From c0ef330f8b24f898674ec3cbdc7abf26496d3e03 Mon Sep 17 00:00:00 2001 From: Bria Morgan Date: Thu, 31 Oct 2024 16:07:52 -0400 Subject: [PATCH 01/31] add getownerworkspaces --- .../org/broadinstitute/dsde/rawls/Boot.scala | 3 +- .../SpendReportingService.scala | 128 +++++++++++++++++- .../SpendReportingServiceSpec.scala | 123 ++++++++++++++--- .../rawls/webservice/ApiServiceSpec.scala | 25 ++-- 4 files changed, 245 insertions(+), 34 deletions(-) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/Boot.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/Boot.scala index 6615b7d605..0654ef5b13 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/Boot.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/Boot.scala @@ -515,7 +515,8 @@ object Boot extends IOApp with LazyLogging { billingRepository, billingProfileManagerDAO, samDAO, - spendReportingServiceConfig + spendReportingServiceConfig, + workspaceServiceConstructor ) val billingAdminServiceConstructor: RawlsRequestContext => BillingAdminService = diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala index c9903b3e0b..d93d0ed629 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala @@ -17,15 +17,22 @@ import org.broadinstitute.dsde.rawls.dataaccess.{SamDAO, SlickDataSource} import org.broadinstitute.dsde.rawls.metrics.{GoogleInstrumented, HitRatioGauge, RawlsInstrumented} import org.broadinstitute.dsde.rawls.model.{SpendReportingAggregationKeyWithSub, _} import org.broadinstitute.dsde.rawls.spendreporting.SpendReportingService._ +import org.broadinstitute.dsde.rawls.workspace.WorkspaceService import org.broadinstitute.dsde.workbench.google2.GoogleBigQueryService import org.broadinstitute.dsde.rawls.{RawlsException, RawlsExceptionWithErrorReport} import org.broadinstitute.dsde.workbench.model.google.GoogleProject import org.joda.time.format.ISODateTimeFormat import org.joda.time.{DateTime, Days} +import spray.json.{JsArray, JsValue} +import spray.json._ +import DefaultJsonProtocol._ +import org.broadinstitute.dsde.rawls.model.WorkspaceJsonSupport.WorkspaceListResponseFormat -import scala.concurrent.{ExecutionContext, Future} +import scala.concurrent.duration.{Duration, DurationInt} +import scala.concurrent.{Await, ExecutionContext, Future} import scala.jdk.CollectionConverters._ import scala.math.BigDecimal.RoundingMode +import org.broadinstitute.dsde.rawls.model.WorkspaceAccessLevels.{Owner, WorkspaceAccessLevel} object SpendReportingService { def constructor( @@ -34,7 +41,8 @@ object SpendReportingService { billingRepository: BillingRepository, bpmDao: BillingProfileManagerDAO, samDAO: SamDAO, - spendReportingServiceConfig: SpendReportingServiceConfig + spendReportingServiceConfig: SpendReportingServiceConfig, + workspaceServiceConstructor: RawlsRequestContext => WorkspaceService )(ctx: RawlsRequestContext)(implicit executionContext: ExecutionContext): SpendReportingService = new SpendReportingService(ctx, dataSource, @@ -42,7 +50,8 @@ object SpendReportingService { billingRepository: BillingRepository, bpmDao, samDAO, - spendReportingServiceConfig + spendReportingServiceConfig, + workspaceServiceConstructor ) val SpendReportingMetrics = "spendReporting" @@ -142,7 +151,8 @@ class SpendReportingService( billingRepository: BillingRepository, bpmDao: BillingProfileManagerDAO, samDAO: SamDAO, - spendReportingServiceConfig: SpendReportingServiceConfig + spendReportingServiceConfig: SpendReportingServiceConfig, + workspaceServiceConstructor: RawlsRequestContext => WorkspaceService )(implicit val executionContext: ExecutionContext) extends LazyLogging with RawlsInstrumented { @@ -241,6 +251,102 @@ class SpendReportingService( .replace("REPLACE_TIME_PARTITION_COLUMN", timePartitionColumn) } +// def getAllUserWorkspaceQuery(aggregations: Set[SpendReportingAggregationKeyWithSub], +// config: BillingProjectSpendExport +// ): String = { +// // Unbox potentially many SpendReportingAggregationKeyWithSubs for query, +// // all of which have optional subAggregationKeys and convert to Set[SpendReportingAggregationKey] +// val queryKeys = aggregations.flatMap(a => Set(Option(a.key), a.subAggregationKey).flatten) +// val tableName = config.spendExportTable.getOrElse(spendReportingServiceConfig.defaultTableName) +// val timePartitionColumn: String = { +// val isBroadTable = tableName == spendReportingServiceConfig.defaultTableName +// // The Broad table uses a view with a different column name. +// if (isBroadTable) spendReportingServiceConfig.defaultTimePartitionColumn else "_PARTITIONTIME" +// } +//// s""" +//// | SELECT +//// | SUM(cost) as cost, +//// | SUM(IFNULL((SELECT SUM(c.amount) FROM UNNEST(credits) c), 0)) as credits, +//// | currency ${queryKeys.map(_.bigQueryAliasClause()).mkString} +//// | FROM `$tableName` +//// | WHERE billing_account_id = @billingAccountId +//// | AND $timePartitionColumn BETWEEN @startDate AND @endDate +//// | AND project.id in UNNEST(@projects) +//// | GROUP BY currency ${queryKeys.map(_.bigQueryGroupByClause()).mkString} +//// |""".stripMargin +//// .replace("REPLACE_TIME_PARTITION_COLUMN", timePartitionColumn) +// +// s""" +// |WITH spend_categories AS ( +// | SELECT +// | project.id AS project_id, +// | project.name AS project_name, +// | CASE +// | WHEN service.description IN ('Cloud Storage') THEN 'Storage' +// | WHEN service.description IN ('Compute Engine', 'Google Kubernetes Engine') THEN 'Compute' +// | ELSE 'Other' +// | END AS spend_category, +// | SUM(CAST(cost AS FLOAT64)) AS category_cost +// | FROM +// | `$first_billing_account_user_has_access_to` +// | where +// | project_id in @listOfWorkspaceProjects AND +// | _PARTITIONTIME BETWEEN @startDate AND @endDate +// | GROUP BY +// | project_id, +// | project_name, +// | spend_category +// | UNION ALL +// | SELECT +// | project.id AS project_id, +// | project.name AS project_name, +// | CASE +// | WHEN service.description IN ('Cloud Storage') THEN 'Storage' +// | WHEN service.description IN ('Compute Engine', 'Google Kubernetes Engine') THEN 'Compute' +// | ELSE 'Other' +// | END AS spend_category, +// | SUM(CAST(cost AS FLOAT64)) AS category_cost +// | FROM +// | $`second_billing_account_user_has_access_to` +// | where +// | project_id in @moreProjectWorkspaces AND +// | _PARTITIONTIME BETWEEN @startDate AND @endDate +// | GROUP BY +// | project_id, +// | project_name, +// | spend_category +// | UNION ALL +// | select +// | project_id, +// | project_name, +// | spend_category, +// | category_cost +// | from +// | `broad_materialized_view` +// | where +// | project_id in ('broad', 'list') AND +// | _PARTITIONTIME BETWEEN @startDate AND @endDate +// |) +// | +// |SELECT +// | project_id, +// | project_name, +// | SUM(category_cost) AS total_cost, +// | SUM(CASE WHEN spend_category = 'Storage' THEN category_cost ELSE 0 END) AS storage_cost, +// | SUM(CASE WHEN spend_category = 'Compute' THEN category_cost ELSE 0 END) AS compute_cost, +// | SUM(CASE WHEN spend_category = 'Other' THEN category_cost ELSE 0 END) AS other_cost +// |FROM +// | spend_categories +// |GROUP BY +// | project_id, +// | project_name +// |ORDER BY +// | total_cost DESC +// |limit 5 +// |""".stripMargin +// .replace("REPLACE_TIME_PARTITION_COLUMN", timePartitionColumn) +// } + def setUpQuery( query: String, exportConf: BillingProjectSpendExport, @@ -359,4 +465,18 @@ class SpendReportingService( case ex: Exception => Future.failed(RawlsExceptionWithErrorReport(ErrorReport(StatusCodes.InternalServerError, ex))) } + + def getOwnerWorkspaces(): Seq[WorkspaceListResponse] = { + val ws = Await.result(workspaceServiceConstructor(ctx).listWorkspaces(WorkspaceFieldSpecs(), -1), Duration.Inf) + val result = ws match { + case JsArray(jsArray) => + val workspaces = jsArray.map(_.convertTo[WorkspaceListResponse]) + println(s"workspaces: $workspaces") + val filtered = workspaces.filter(_.accessLevel == Owner) + println(s"filtered: $filtered") + filtered + case _ => throw new IllegalArgumentException("Expected a JsArray") + } + result + } } diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala index b5b544b1ae..fd5de31182 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala @@ -1,5 +1,6 @@ package org.broadinstitute.dsde.rawls.spendreporting +import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport import akka.http.scaladsl.model.StatusCodes import akka.http.scaladsl.model.headers.OAuth2BearerToken import bio.terra.profile.model.SpendReportingAggregation.AggregationKeyEnum @@ -20,7 +21,7 @@ import org.broadinstitute.dsde.rawls.billing.{ BpmAzureSpendReportApiException } import org.broadinstitute.dsde.rawls.config.SpendReportingServiceConfig -import org.broadinstitute.dsde.rawls.dataaccess.{SamDAO, SlickDataSource} +import org.broadinstitute.dsde.rawls.dataaccess.{GoogleServicesDAO, SamDAO, SlickDataSource} import org.broadinstitute.dsde.rawls.model.Attributable.AttributeMap import org.broadinstitute.dsde.rawls.model._ import org.broadinstitute.dsde.rawls.util.MockitoTestUtils @@ -40,8 +41,13 @@ import scala.concurrent.duration.Duration import scala.concurrent.{Await, Future} import scala.jdk.CollectionConverters._ import scala.math.BigDecimal.RoundingMode +import org.broadinstitute.dsde.rawls.workspace.WorkspaceService +import org.scalatestplus.mockito.MockitoSugar.mock +import spray.json.DefaultJsonProtocol._ +import spray.json._ +import org.broadinstitute.dsde.rawls.model.WorkspaceJsonSupport.WorkspaceListResponseFormat -class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with MockitoTestUtils { +class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with MockitoTestUtils with SprayJsonSupport { implicit val executionContext: TestExecutionContext = TestExecutionContext.testExecutionContext @@ -60,6 +66,11 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki None ) + val mockWorkspaceServiceConstructor: RawlsRequestContext => WorkspaceService = { + lazy val mockWorkspaceService: WorkspaceService = mock[WorkspaceService] + _ => mockWorkspaceService + } + val testContext: RawlsRequestContext = RawlsRequestContext(userInfo) object TestData { val workspace1: Workspace = workspace("workspace1", GoogleProjectId("project1")) @@ -558,7 +569,8 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki billingRepository, bpmDAO, samDAO, - spendReportingServiceConfig + spendReportingServiceConfig, + mockWorkspaceServiceConstructor ) ) val billingProjectSpendExport = @@ -600,7 +612,8 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki billingRepository, bpmDAO, samDAO, - spendReportingServiceConfig + spendReportingServiceConfig, + mockWorkspaceServiceConstructor ) val e = intercept[RawlsExceptionWithErrorReport] { @@ -631,7 +644,8 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki billingRepository, bpmDAO, samDAO, - spendReportingServiceConfig + spendReportingServiceConfig, + mockWorkspaceServiceConstructor ) val e = intercept[RawlsExceptionWithErrorReport] { @@ -646,6 +660,7 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki val samDAO = mock[SamDAO] val billingRepository = mock[BillingRepository] val bpmDAO = mock[BillingProfileManagerDAO] + when(samDAO.userHasAction(any(), any(), any(), any())).thenReturn(Future.successful(true)) val dataSource = mock[SlickDataSource] when(dataSource.inTransaction[Option[BillingProjectSpendExport]](any(), any())) @@ -657,7 +672,8 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki billingRepository, bpmDAO, samDAO, - spendReportingServiceConfig + spendReportingServiceConfig, + mockWorkspaceServiceConstructor ) val projectName = RawlsBillingProjectName("fakeProject") @@ -673,6 +689,7 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki val samDAO = mock[SamDAO] val billingRepository = mock[BillingRepository] val bpmDAO = mock[BillingProfileManagerDAO] + when(samDAO.userHasAction(any(), any(), any(), any())).thenReturn(Future.successful(true)) when(billingRepository.getBillingProject(any())).thenReturn(Future.successful(Option.apply(billingProject))) val badRow = Map( @@ -691,7 +708,8 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki billingRepository, bpmDAO, samDAO, - spendReportingServiceConfig + spendReportingServiceConfig, + mockWorkspaceServiceConstructor ) ) val billingProjectSpendExport = @@ -754,7 +772,8 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki billingRepository, bpmDAO, samDAO, - spendReportingServiceConfig + spendReportingServiceConfig, + mockWorkspaceServiceConstructor ) val result = Await.result( @@ -796,6 +815,7 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki val samDAO = mock[SamDAO](RETURNS_SMART_NULLS) when(samDAO.userHasAction(any(), any(), any(), any())).thenReturn(Future.successful(true)) + val bigQueryService = mockBigQuery(TestData.Workspace.table) val service = spy( new SpendReportingService( @@ -805,7 +825,8 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki billingRepository, bpmDAO, samDAO, - spendReportingServiceConfig + spendReportingServiceConfig, + mockWorkspaceServiceConstructor ) ) val billingProjectSpendExport = @@ -843,6 +864,7 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki val samDAO = mock[SamDAO](RETURNS_SMART_NULLS) when(samDAO.userHasAction(any(), any(), any(), any())).thenReturn(Future.successful(true)) + val bigQueryService = mockBigQuery(TestData.Workspace.table) val service = spy( new SpendReportingService( @@ -852,7 +874,8 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki billingRepository, bpmDAO, samDAO, - spendReportingServiceConfig + spendReportingServiceConfig, + mockWorkspaceServiceConstructor ) ) val billingProjectSpendExport = @@ -914,7 +937,8 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki billingRepository, bpmDAO, samDAO, - spendReportingServiceConfig + spendReportingServiceConfig, + mockWorkspaceServiceConstructor ) val e = intercept[RawlsExceptionWithErrorReport] { @@ -936,7 +960,8 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki mock[BillingRepository], mock[BillingProfileManagerDAO], mock[SamDAO], - spendReportingServiceConfig + spendReportingServiceConfig, + mockWorkspaceServiceConstructor ) val startDate = DateTime.now().minusDays(spendReportingServiceConfig.maxDateRange) val endDate = DateTime.now() @@ -951,7 +976,8 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki mock[BillingRepository], mock[BillingProfileManagerDAO], mock[SamDAO], - spendReportingServiceConfig + spendReportingServiceConfig, + mockWorkspaceServiceConstructor ) val startDate = DateTime.now() val endDate = DateTime.now().minusDays(1) @@ -967,7 +993,8 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki mock[BillingRepository], mock[BillingProfileManagerDAO], mock[SamDAO], - spendReportingServiceConfig + spendReportingServiceConfig, + mockWorkspaceServiceConstructor ) val startDate = DateTime.now().minusDays(spendReportingServiceConfig.maxDateRange + 1) val endDate = DateTime.now() @@ -996,7 +1023,8 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki mock[BillingRepository], mock[BillingProfileManagerDAO], mock[SamDAO], - spendReportingServiceConfig + spendReportingServiceConfig, + mockWorkspaceServiceConstructor ) val result = service.getQuery( Set( @@ -1029,7 +1057,8 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki mock[BillingRepository], mock[BillingProfileManagerDAO], mock[SamDAO], - spendReportingServiceConfig + spendReportingServiceConfig, + mockWorkspaceServiceConstructor ) val result = service.getQuery( Set( @@ -1058,7 +1087,8 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki mock[BillingRepository], mock[BillingProfileManagerDAO], mock[SamDAO], - spendReportingServiceConfig + spendReportingServiceConfig, + mockWorkspaceServiceConstructor ) val result = Await.result(service.getWorkspaceGoogleProjects(RawlsBillingProjectName("")), Duration.Inf) @@ -1066,4 +1096,63 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki result shouldBe Map(GoogleProjectId("v2ProjectId") -> v2Workspace.toWorkspaceName) } + "getOwnerWorkspaces" should "return any and all workspaces user has owner access to" in { + val v1Workspace = TestData.workspace("v1name", GoogleProjectId("v1ProjectId"), WorkspaceVersions.V1) + val v2Workspace = TestData.workspace("v2name", GoogleProjectId("v2ProjectId"), WorkspaceVersions.V2) + val workspace3 = TestData.workspace("name3", GoogleProjectId("3ProjectId"), WorkspaceVersions.V2) + + val dataSource = mock[SlickDataSource] + val mockWorkspaceService = mock[WorkspaceService](RETURNS_SMART_NULLS) + + val workspace1Response = WorkspaceListResponse( + WorkspaceAccessLevels.Owner, + Some(true), + Some(true), + WorkspaceDetails.fromWorkspaceAndOptions(v1Workspace, Option(Set.empty), true, Some(WorkspaceCloudPlatform.Gcp)), + Option.empty, + false, + Some(List.empty) + ) + + val workspace2Response = WorkspaceListResponse( + WorkspaceAccessLevels.Owner, + Some(true), + Some(true), + WorkspaceDetails.fromWorkspaceAndOptions(v2Workspace, Option(Set.empty), true, Some(WorkspaceCloudPlatform.Gcp)), + Option.empty, + false, + Some(List.empty) + ) + + val workspace3Response = WorkspaceListResponse( + WorkspaceAccessLevels.Read, + Some(true), + Some(true), + WorkspaceDetails.fromWorkspaceAndOptions(workspace3, Option(Set.empty), true, Some(WorkspaceCloudPlatform.Gcp)), + Option.empty, + false, + Some(List.empty) + ) + + when(mockWorkspaceService.listWorkspaces(any(), any())) + .thenReturn(Future.successful(Seq(workspace1Response, workspace2Response, workspace3Response).toJson)) + val mockWorkspaceServiceConstructor: RawlsRequestContext => WorkspaceService = { _ => + mockWorkspaceService + } + val service = new SpendReportingService( + testContext, + dataSource, + Resource.pure[IO, GoogleBigQueryService[IO]](mock[GoogleBigQueryService[IO]]), + mock[BillingRepository], + mock[BillingProfileManagerDAO], + mock[SamDAO], + spendReportingServiceConfig, + mockWorkspaceServiceConstructor + ) + + val result = service.getOwnerWorkspaces() + + result shouldBe Seq(workspace1Response, workspace2Response) + } + } diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/ApiServiceSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/ApiServiceSpec.scala index 21361f5924..f0d47d63a7 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/ApiServiceSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/ApiServiceSpec.scala @@ -274,18 +274,6 @@ trait ApiServiceSpec gcsDAO ) _ - val spendReportingBigQueryService = bigQueryServiceFactory.getServiceFromJson("json", GoogleProject("test-project")) - val spendReportingServiceConfig = - SpendReportingServiceConfig("fakeTableName", "fakeTimePartitionColumn", 90, "test.metrics") - override val spendReportingConstructor = SpendReportingService.constructor( - slickDataSource, - spendReportingBigQueryService, - mock[BillingRepository], - mock[BillingProfileManagerDAO], - samDAO, - spendReportingServiceConfig - ) - override val billingAdminServiceConstructor: RawlsRequestContext => BillingAdminService = new BillingAdminService(samDAO, billingRepository, new WorkspaceRepository(slickDataSource), _)( testExecutionContext @@ -417,6 +405,19 @@ trait ApiServiceSpec samDAO ) + val spendReportingBigQueryService = bigQueryServiceFactory.getServiceFromJson("json", GoogleProject("test-project")) + val spendReportingServiceConfig = + SpendReportingServiceConfig("fakeTableName", "fakeTimePartitionColumn", 90, "test.metrics") + override val spendReportingConstructor = SpendReportingService.constructor( + slickDataSource, + spendReportingBigQueryService, + mock[BillingRepository], + mock[BillingProfileManagerDAO], + samDAO, + spendReportingServiceConfig, + workspaceServiceConstructor + ) + override val methodConfigurationServiceConstructor: RawlsRequestContext => MethodConfigurationService = MethodConfigurationService.constructor( slickDataSource, From 1a396b95b992028c9bf81d944c8a4c2d5ae332b6 Mon Sep 17 00:00:00 2001 From: Bria Morgan Date: Fri, 1 Nov 2024 11:06:59 -0400 Subject: [PATCH 02/31] update getOwnerWorkspaces, start to add getBilling --- .../org/broadinstitute/dsde/rawls/Boot.scala | 3 +- .../SpendReportingService.scala | 45 ++++++----- .../SpendReportingServiceSpec.scala | 79 +++++++++++++------ .../rawls/webservice/ApiServiceSpec.scala | 3 +- 4 files changed, 86 insertions(+), 44 deletions(-) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/Boot.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/Boot.scala index 0654ef5b13..ef16f4d122 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/Boot.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/Boot.scala @@ -516,7 +516,8 @@ object Boot extends IOApp with LazyLogging { billingProfileManagerDAO, samDAO, spendReportingServiceConfig, - workspaceServiceConstructor + workspaceServiceConstructor, + userServiceConstructor ) val billingAdminServiceConstructor: RawlsRequestContext => BillingAdminService = diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala index d93d0ed629..d8966d7eb5 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala @@ -33,6 +33,7 @@ import scala.concurrent.{Await, ExecutionContext, Future} import scala.jdk.CollectionConverters._ import scala.math.BigDecimal.RoundingMode import org.broadinstitute.dsde.rawls.model.WorkspaceAccessLevels.{Owner, WorkspaceAccessLevel} +import org.broadinstitute.dsde.rawls.user.UserService object SpendReportingService { def constructor( @@ -42,16 +43,19 @@ object SpendReportingService { bpmDao: BillingProfileManagerDAO, samDAO: SamDAO, spendReportingServiceConfig: SpendReportingServiceConfig, - workspaceServiceConstructor: RawlsRequestContext => WorkspaceService + workspaceServiceConstructor: RawlsRequestContext => WorkspaceService, + userServiceConstructor: RawlsRequestContext => UserService )(ctx: RawlsRequestContext)(implicit executionContext: ExecutionContext): SpendReportingService = - new SpendReportingService(ctx, - dataSource, - bigQueryService, - billingRepository: BillingRepository, - bpmDao, - samDAO, - spendReportingServiceConfig, - workspaceServiceConstructor + new SpendReportingService( + ctx, + dataSource, + bigQueryService, + billingRepository: BillingRepository, + bpmDao, + samDAO, + spendReportingServiceConfig, + workspaceServiceConstructor, + userServiceConstructor ) val SpendReportingMetrics = "spendReporting" @@ -152,7 +156,8 @@ class SpendReportingService( bpmDao: BillingProfileManagerDAO, samDAO: SamDAO, spendReportingServiceConfig: SpendReportingServiceConfig, - workspaceServiceConstructor: RawlsRequestContext => WorkspaceService + workspaceServiceConstructor: RawlsRequestContext => WorkspaceService, + userServiceConstructor: RawlsRequestContext => UserService )(implicit val executionContext: ExecutionContext) extends LazyLogging with RawlsInstrumented { @@ -466,17 +471,19 @@ class SpendReportingService( Future.failed(RawlsExceptionWithErrorReport(ErrorReport(StatusCodes.InternalServerError, ex))) } - def getOwnerWorkspaces(): Seq[WorkspaceListResponse] = { - val ws = Await.result(workspaceServiceConstructor(ctx).listWorkspaces(WorkspaceFieldSpecs(), -1), Duration.Inf) - val result = ws match { + def getOwnerWorkspaces(): Future[Seq[WorkspaceListResponse]] = + workspaceServiceConstructor(ctx).listWorkspaces(WorkspaceFieldSpecs(), -1) map { case JsArray(jsArray) => val workspaces = jsArray.map(_.convertTo[WorkspaceListResponse]) - println(s"workspaces: $workspaces") - val filtered = workspaces.filter(_.accessLevel == Owner) - println(s"filtered: $filtered") - filtered + workspaces.filter(_.accessLevel == Owner) case _ => throw new IllegalArgumentException("Expected a JsArray") } - result - } + +// def getBillingForWorkspaces(workspaces: Future[Seq[WorkspaceListResponse]]): Unit = { +// val billingAccounts = workspaces.map(wsList => wsList.map(ws => ws.workspace.namespace +// val result = billingAccounts.foreach(ba => +// userServiceConstructor(ctx).getBillingProjectSpendConfiguration(RawlsBillingProjectName(ba)) +// )) +// ) +// } } diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala index fd5de31182..5097ef52ed 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala @@ -46,6 +46,7 @@ import org.scalatestplus.mockito.MockitoSugar.mock import spray.json.DefaultJsonProtocol._ import spray.json._ import org.broadinstitute.dsde.rawls.model.WorkspaceJsonSupport.WorkspaceListResponseFormat +import org.broadinstitute.dsde.rawls.user.UserService class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with MockitoTestUtils with SprayJsonSupport { @@ -71,6 +72,10 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki _ => mockWorkspaceService } + val mockUserServiceConstructor: RawlsRequestContext => UserService = { + lazy val mockUserService: UserService = mock[UserService] + _ => mockUserService + } val testContext: RawlsRequestContext = RawlsRequestContext(userInfo) object TestData { val workspace1: Workspace = workspace("workspace1", GoogleProjectId("project1")) @@ -570,7 +575,8 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki bpmDAO, samDAO, spendReportingServiceConfig, - mockWorkspaceServiceConstructor + mockWorkspaceServiceConstructor, + mockUserServiceConstructor ) ) val billingProjectSpendExport = @@ -613,7 +619,8 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki bpmDAO, samDAO, spendReportingServiceConfig, - mockWorkspaceServiceConstructor + mockWorkspaceServiceConstructor, + mockUserServiceConstructor ) val e = intercept[RawlsExceptionWithErrorReport] { @@ -645,7 +652,8 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki bpmDAO, samDAO, spendReportingServiceConfig, - mockWorkspaceServiceConstructor + mockWorkspaceServiceConstructor, + mockUserServiceConstructor ) val e = intercept[RawlsExceptionWithErrorReport] { @@ -673,7 +681,8 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki bpmDAO, samDAO, spendReportingServiceConfig, - mockWorkspaceServiceConstructor + mockWorkspaceServiceConstructor, + mockUserServiceConstructor ) val projectName = RawlsBillingProjectName("fakeProject") @@ -709,7 +718,8 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki bpmDAO, samDAO, spendReportingServiceConfig, - mockWorkspaceServiceConstructor + mockWorkspaceServiceConstructor, + mockUserServiceConstructor ) ) val billingProjectSpendExport = @@ -773,7 +783,8 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki bpmDAO, samDAO, spendReportingServiceConfig, - mockWorkspaceServiceConstructor + mockWorkspaceServiceConstructor, + mockUserServiceConstructor ) val result = Await.result( @@ -826,7 +837,8 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki bpmDAO, samDAO, spendReportingServiceConfig, - mockWorkspaceServiceConstructor + mockWorkspaceServiceConstructor, + mockUserServiceConstructor ) ) val billingProjectSpendExport = @@ -875,7 +887,8 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki bpmDAO, samDAO, spendReportingServiceConfig, - mockWorkspaceServiceConstructor + mockWorkspaceServiceConstructor, + mockUserServiceConstructor ) ) val billingProjectSpendExport = @@ -938,7 +951,8 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki bpmDAO, samDAO, spendReportingServiceConfig, - mockWorkspaceServiceConstructor + mockWorkspaceServiceConstructor, + mockUserServiceConstructor ) val e = intercept[RawlsExceptionWithErrorReport] { @@ -961,7 +975,8 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki mock[BillingProfileManagerDAO], mock[SamDAO], spendReportingServiceConfig, - mockWorkspaceServiceConstructor + mockWorkspaceServiceConstructor, + mockUserServiceConstructor ) val startDate = DateTime.now().minusDays(spendReportingServiceConfig.maxDateRange) val endDate = DateTime.now() @@ -977,7 +992,8 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki mock[BillingProfileManagerDAO], mock[SamDAO], spendReportingServiceConfig, - mockWorkspaceServiceConstructor + mockWorkspaceServiceConstructor, + mockUserServiceConstructor ) val startDate = DateTime.now() val endDate = DateTime.now().minusDays(1) @@ -994,7 +1010,8 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki mock[BillingProfileManagerDAO], mock[SamDAO], spendReportingServiceConfig, - mockWorkspaceServiceConstructor + mockWorkspaceServiceConstructor, + mockUserServiceConstructor ) val startDate = DateTime.now().minusDays(spendReportingServiceConfig.maxDateRange + 1) val endDate = DateTime.now() @@ -1024,7 +1041,8 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki mock[BillingProfileManagerDAO], mock[SamDAO], spendReportingServiceConfig, - mockWorkspaceServiceConstructor + mockWorkspaceServiceConstructor, + mockUserServiceConstructor ) val result = service.getQuery( Set( @@ -1058,7 +1076,8 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki mock[BillingProfileManagerDAO], mock[SamDAO], spendReportingServiceConfig, - mockWorkspaceServiceConstructor + mockWorkspaceServiceConstructor, + mockUserServiceConstructor ) val result = service.getQuery( Set( @@ -1088,7 +1107,8 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki mock[BillingProfileManagerDAO], mock[SamDAO], spendReportingServiceConfig, - mockWorkspaceServiceConstructor + mockWorkspaceServiceConstructor, + mockUserServiceConstructor ) val result = Await.result(service.getWorkspaceGoogleProjects(RawlsBillingProjectName("")), Duration.Inf) @@ -1097,9 +1117,9 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki } "getOwnerWorkspaces" should "return any and all workspaces user has owner access to" in { - val v1Workspace = TestData.workspace("v1name", GoogleProjectId("v1ProjectId"), WorkspaceVersions.V1) - val v2Workspace = TestData.workspace("v2name", GoogleProjectId("v2ProjectId"), WorkspaceVersions.V2) - val workspace3 = TestData.workspace("name3", GoogleProjectId("3ProjectId"), WorkspaceVersions.V2) + val ownerWorkspace1 = TestData.workspace("owner1", GoogleProjectId("owner1ProjectId"), WorkspaceVersions.V1) + val ownerWorkspace2 = TestData.workspace("owner2", GoogleProjectId("owner2ProjectId"), WorkspaceVersions.V2) + val readerWorkspace = TestData.workspace("reader1", GoogleProjectId("reader1ProjectId"), WorkspaceVersions.V2) val dataSource = mock[SlickDataSource] val mockWorkspaceService = mock[WorkspaceService](RETURNS_SMART_NULLS) @@ -1108,7 +1128,11 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki WorkspaceAccessLevels.Owner, Some(true), Some(true), - WorkspaceDetails.fromWorkspaceAndOptions(v1Workspace, Option(Set.empty), true, Some(WorkspaceCloudPlatform.Gcp)), + WorkspaceDetails.fromWorkspaceAndOptions(ownerWorkspace1, + Option(Set.empty), + true, + Some(WorkspaceCloudPlatform.Gcp) + ), Option.empty, false, Some(List.empty) @@ -1118,7 +1142,11 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki WorkspaceAccessLevels.Owner, Some(true), Some(true), - WorkspaceDetails.fromWorkspaceAndOptions(v2Workspace, Option(Set.empty), true, Some(WorkspaceCloudPlatform.Gcp)), + WorkspaceDetails.fromWorkspaceAndOptions(ownerWorkspace2, + Option(Set.empty), + true, + Some(WorkspaceCloudPlatform.Gcp) + ), Option.empty, false, Some(List.empty) @@ -1128,7 +1156,11 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki WorkspaceAccessLevels.Read, Some(true), Some(true), - WorkspaceDetails.fromWorkspaceAndOptions(workspace3, Option(Set.empty), true, Some(WorkspaceCloudPlatform.Gcp)), + WorkspaceDetails.fromWorkspaceAndOptions(readerWorkspace, + Option(Set.empty), + true, + Some(WorkspaceCloudPlatform.Gcp) + ), Option.empty, false, Some(List.empty) @@ -1147,10 +1179,11 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki mock[BillingProfileManagerDAO], mock[SamDAO], spendReportingServiceConfig, - mockWorkspaceServiceConstructor + mockWorkspaceServiceConstructor, + mockUserServiceConstructor ) - val result = service.getOwnerWorkspaces() + val result = Await.result(service.getOwnerWorkspaces(), Duration.Inf) result shouldBe Seq(workspace1Response, workspace2Response) } diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/ApiServiceSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/ApiServiceSpec.scala index f0d47d63a7..6ecdc40cf9 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/ApiServiceSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/ApiServiceSpec.scala @@ -415,7 +415,8 @@ trait ApiServiceSpec mock[BillingProfileManagerDAO], samDAO, spendReportingServiceConfig, - workspaceServiceConstructor + workspaceServiceConstructor, + userServiceConstructor ) override val methodConfigurationServiceConstructor: RawlsRequestContext => MethodConfigurationService = From 3a4dc82e26791ff88bea72458fbf7235fd1fbe72 Mon Sep 17 00:00:00 2001 From: Bria Morgan Date: Fri, 1 Nov 2024 13:34:46 -0400 Subject: [PATCH 03/31] interim getBillingForWorkspaces --- .../SpendReportingService.scala | 16 +- .../SpendReportingServiceSpec.scala | 155 +++++++++++++++++- 2 files changed, 155 insertions(+), 16 deletions(-) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala index d8966d7eb5..6687a57aa3 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala @@ -479,11 +479,13 @@ class SpendReportingService( case _ => throw new IllegalArgumentException("Expected a JsArray") } -// def getBillingForWorkspaces(workspaces: Future[Seq[WorkspaceListResponse]]): Unit = { -// val billingAccounts = workspaces.map(wsList => wsList.map(ws => ws.workspace.namespace -// val result = billingAccounts.foreach(ba => -// userServiceConstructor(ctx).getBillingProjectSpendConfiguration(RawlsBillingProjectName(ba)) -// )) -// ) -// } + def getBillingForWorkspaces( + workspaces: Future[Seq[WorkspaceListResponse]] + ): Future[Seq[BillingProjectSpendConfiguration]] = + workspaces.flatMap { wsList => + val billingFutures = wsList.map(ws => + userServiceConstructor(ctx).getBillingProjectSpendConfiguration(RawlsBillingProjectName(ws.workspace.namespace)) + ) + Future.sequence(billingFutures).map(_.flatten.distinct) + } } diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala index 5097ef52ed..7b5c6d05bb 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala @@ -21,13 +21,14 @@ import org.broadinstitute.dsde.rawls.billing.{ BpmAzureSpendReportApiException } import org.broadinstitute.dsde.rawls.config.SpendReportingServiceConfig +import org.broadinstitute.dsde.rawls.dataaccess.slick.TestDriverComponent import org.broadinstitute.dsde.rawls.dataaccess.{GoogleServicesDAO, SamDAO, SlickDataSource} import org.broadinstitute.dsde.rawls.model.Attributable.AttributeMap import org.broadinstitute.dsde.rawls.model._ import org.broadinstitute.dsde.rawls.util.MockitoTestUtils import org.broadinstitute.dsde.rawls.{model, RawlsException, RawlsExceptionWithErrorReport, TestExecutionContext} import org.broadinstitute.dsde.workbench.google2.GoogleBigQueryService -import org.broadinstitute.dsde.workbench.model.google.GoogleProject +import org.broadinstitute.dsde.workbench.model.google.{BigQueryDatasetName, GoogleProject} import org.joda.time.DateTime import org.joda.time.format.ISODateTimeFormat import org.mockito.ArgumentMatchers.{any, eq => mockitoEq} @@ -48,15 +49,20 @@ import spray.json._ import org.broadinstitute.dsde.rawls.model.WorkspaceJsonSupport.WorkspaceListResponseFormat import org.broadinstitute.dsde.rawls.user.UserService -class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with MockitoTestUtils with SprayJsonSupport { +class SpendReportingServiceSpec + extends AnyFlatSpecLike + with Matchers + with MockitoTestUtils + with SprayJsonSupport + with TestDriverComponent { - implicit val executionContext: TestExecutionContext = TestExecutionContext.testExecutionContext +// implicit val executionContext: TestExecutionContext = TestExecutionContext.testExecutionContext - val userInfo: UserInfo = UserInfo(RawlsUserEmail("owner-access"), - OAuth2BearerToken("token"), - 123, - RawlsUserSubjectId("123456789876543212345") - ) +// val userInfo: UserInfo = UserInfo(RawlsUserEmail("owner-access"), +// OAuth2BearerToken("token"), +// 123, +// RawlsUserSubjectId("123456789876543212345") +// ) val wsName: WorkspaceName = WorkspaceName("myNamespace", "myWorkspace") val billingAccountName: RawlsBillingAccountName = RawlsBillingAccountName("fakeBillingAcct") @@ -76,7 +82,7 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki lazy val mockUserService: UserService = mock[UserService] _ => mockUserService } - val testContext: RawlsRequestContext = RawlsRequestContext(userInfo) + override val testContext: RawlsRequestContext = RawlsRequestContext(userInfo) object TestData { val workspace1: Workspace = workspace("workspace1", GoogleProjectId("project1")) val workspace2: Workspace = workspace("workspace2", GoogleProjectId("project2")) @@ -1188,4 +1194,135 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki result shouldBe Seq(workspace1Response, workspace2Response) } + "getBillingForWorkspaces" should "return spendConfigurations for workspaces" in { + + val dataSource = mock[SlickDataSource] + + val billingProject1 = billingProjectFromName("billingProject1") + val spendReportDatasetName1 = BigQueryDatasetName("test_dataset") + val spendReportGoogleProject1 = GoogleProject("some_other_google_project") + val spendReportConfiguration1 = + BillingProjectSpendConfiguration(spendReportGoogleProject1, spendReportDatasetName1) + + val billingProject2 = billingProjectFromName("billingProject2") + val spendReportDatasetName2 = BigQueryDatasetName("test_dataset2") + val spendReportGoogleProject2 = GoogleProject("some_other_google_project2") + val spendReportConfiguration2 = + BillingProjectSpendConfiguration(spendReportGoogleProject2, spendReportDatasetName2) + + val billingProject3 = billingProjectFromName("billingProject3") + + val mockUserService = mock[UserService](RETURNS_SMART_NULLS) + when(mockUserService.getBillingProjectSpendConfiguration(billingProject1.projectName)) + .thenReturn(Future.successful(Some(spendReportConfiguration1))) + + when(mockUserService.getBillingProjectSpendConfiguration(billingProject2.projectName)) + .thenReturn(Future.successful(Some(spendReportConfiguration2))) + + when(mockUserService.getBillingProjectSpendConfiguration(billingProject3.projectName)) + .thenReturn(Future.successful(None)) + + val mockUserServiceConstructor: RawlsRequestContext => UserService = { _ => + mockUserService + } + + val service = new SpendReportingService( + testContext, + dataSource, + Resource.pure[IO, GoogleBigQueryService[IO]](mock[GoogleBigQueryService[IO]]), + mock[BillingRepository], + mock[BillingProfileManagerDAO], + mock[SamDAO], + spendReportingServiceConfig, + mockWorkspaceServiceConstructor, + mockUserServiceConstructor + ) + + val workspace1Billing1 = + TestData.workspace("workspace1Billing1", + GoogleProjectId("owner1ProjectId"), + WorkspaceVersions.V1, + billingProject1.projectName.value + ) + val workspace2Billing1 = + TestData.workspace("workspace2Billing1", + GoogleProjectId("owner1ProjectId"), + WorkspaceVersions.V2, + billingProject1.projectName.value + ) + val workspace1Billing2 = + TestData.workspace("workspace1Billing2", + GoogleProjectId("owner1ProjectId"), + WorkspaceVersions.V2, + billingProject2.projectName.value + ) + val workspace1Billing3 = + TestData.workspace("workspace1Billing3", + GoogleProjectId("owner1ProjectId"), + WorkspaceVersions.V2, + billingProject3.projectName.value + ) + + val workspace1Response = WorkspaceListResponse( + WorkspaceAccessLevels.Read, + Some(true), + Some(true), + WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing1, + Option(Set.empty), + true, + Some(WorkspaceCloudPlatform.Gcp) + ), + Option.empty, + false, + Some(List.empty) + ) + val workspace2Response = WorkspaceListResponse( + WorkspaceAccessLevels.Read, + Some(true), + Some(true), + WorkspaceDetails.fromWorkspaceAndOptions(workspace2Billing1, + Option(Set.empty), + true, + Some(WorkspaceCloudPlatform.Gcp) + ), + Option.empty, + false, + Some(List.empty) + ) + val workspace3Response = WorkspaceListResponse( + WorkspaceAccessLevels.Read, + Some(true), + Some(true), + WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing2, + Option(Set.empty), + true, + Some(WorkspaceCloudPlatform.Gcp) + ), + Option.empty, + false, + Some(List.empty) + ) + val workspace4Response = WorkspaceListResponse( + WorkspaceAccessLevels.Read, + Some(true), + Some(true), + WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing3, + Option(Set.empty), + true, + Some(WorkspaceCloudPlatform.Gcp) + ), + Option.empty, + false, + Some(List.empty) + ) + + val result = Await.result( + service.getBillingForWorkspaces( + Future.successful(Seq(workspace2Response, workspace3Response, workspace1Response, workspace4Response)) + ), + Duration.Inf + ) + result shouldBe Seq(spendReportConfiguration1, spendReportConfiguration2) + } + } From 56cf2bce086fa3fb616a17f328de7b3c5bd806f7 Mon Sep 17 00:00:00 2001 From: Bria Morgan Date: Mon, 4 Nov 2024 15:24:31 -0500 Subject: [PATCH 04/31] update getbillingforworkspaces to get what we need --- .../org/broadinstitute/dsde/rawls/Boot.scala | 3 +- .../SpendReportingService.scala | 118 +++++++++++---- .../SpendReportingServiceSpec.scala | 137 ++++++++---------- .../rawls/webservice/ApiServiceSpec.scala | 3 +- 4 files changed, 151 insertions(+), 110 deletions(-) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/Boot.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/Boot.scala index ef16f4d122..0654ef5b13 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/Boot.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/Boot.scala @@ -516,8 +516,7 @@ object Boot extends IOApp with LazyLogging { billingProfileManagerDAO, samDAO, spendReportingServiceConfig, - workspaceServiceConstructor, - userServiceConstructor + workspaceServiceConstructor ) val billingAdminServiceConstructor: RawlsRequestContext => BillingAdminService = diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala index 6687a57aa3..0cd94a430c 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala @@ -23,17 +23,13 @@ import org.broadinstitute.dsde.rawls.{RawlsException, RawlsExceptionWithErrorRep import org.broadinstitute.dsde.workbench.model.google.GoogleProject import org.joda.time.format.ISODateTimeFormat import org.joda.time.{DateTime, Days} -import spray.json.{JsArray, JsValue} -import spray.json._ -import DefaultJsonProtocol._ +import spray.json.JsArray import org.broadinstitute.dsde.rawls.model.WorkspaceJsonSupport.WorkspaceListResponseFormat -import scala.concurrent.duration.{Duration, DurationInt} -import scala.concurrent.{Await, ExecutionContext, Future} +import scala.concurrent.{ExecutionContext, Future} import scala.jdk.CollectionConverters._ import scala.math.BigDecimal.RoundingMode -import org.broadinstitute.dsde.rawls.model.WorkspaceAccessLevels.{Owner, WorkspaceAccessLevel} -import org.broadinstitute.dsde.rawls.user.UserService +import org.broadinstitute.dsde.rawls.model.WorkspaceAccessLevels.Owner object SpendReportingService { def constructor( @@ -43,8 +39,7 @@ object SpendReportingService { bpmDao: BillingProfileManagerDAO, samDAO: SamDAO, spendReportingServiceConfig: SpendReportingServiceConfig, - workspaceServiceConstructor: RawlsRequestContext => WorkspaceService, - userServiceConstructor: RawlsRequestContext => UserService + workspaceServiceConstructor: RawlsRequestContext => WorkspaceService )(ctx: RawlsRequestContext)(implicit executionContext: ExecutionContext): SpendReportingService = new SpendReportingService( ctx, @@ -54,8 +49,7 @@ object SpendReportingService { bpmDao, samDAO, spendReportingServiceConfig, - workspaceServiceConstructor, - userServiceConstructor + workspaceServiceConstructor ) val SpendReportingMetrics = "spendReporting" @@ -156,8 +150,7 @@ class SpendReportingService( bpmDao: BillingProfileManagerDAO, samDAO: SamDAO, spendReportingServiceConfig: SpendReportingServiceConfig, - workspaceServiceConstructor: RawlsRequestContext => WorkspaceService, - userServiceConstructor: RawlsRequestContext => UserService + workspaceServiceConstructor: RawlsRequestContext => WorkspaceService )(implicit val executionContext: ExecutionContext) extends LazyLogging with RawlsInstrumented { @@ -256,18 +249,14 @@ class SpendReportingService( .replace("REPLACE_TIME_PARTITION_COLUMN", timePartitionColumn) } -// def getAllUserWorkspaceQuery(aggregations: Set[SpendReportingAggregationKeyWithSub], -// config: BillingProjectSpendExport +// def getAllUserWorkspaceQuery(billingProjects: Seq[BillingProjectSpendConfiguration] // ): String = { -// // Unbox potentially many SpendReportingAggregationKeyWithSubs for query, -// // all of which have optional subAggregationKeys and convert to Set[SpendReportingAggregationKey] -// val queryKeys = aggregations.flatMap(a => Set(Option(a.key), a.subAggregationKey).flatten) -// val tableName = config.spendExportTable.getOrElse(spendReportingServiceConfig.defaultTableName) -// val timePartitionColumn: String = { -// val isBroadTable = tableName == spendReportingServiceConfig.defaultTableName -// // The Broad table uses a view with a different column name. -// if (isBroadTable) spendReportingServiceConfig.defaultTimePartitionColumn else "_PARTITIONTIME" -// } +//// val tableName = config.spendExportTable.getOrElse(spendReportingServiceConfig.defaultTableName) +//// val timePartitionColumn: String = { +//// val isBroadTable = tableName == spendReportingServiceConfig.defaultTableName +//// // The Broad table uses a view with a different column name. +//// if (isBroadTable) spendReportingServiceConfig.defaultTimePartitionColumn else "_PARTITIONTIME" +//// } //// s""" //// | SELECT //// | SUM(cost) as cost, @@ -280,6 +269,7 @@ class SpendReportingService( //// | GROUP BY currency ${queryKeys.map(_.bigQueryGroupByClause()).mkString} //// |""".stripMargin //// .replace("REPLACE_TIME_PARTITION_COLUMN", timePartitionColumn) +// val first_billing_account_user_has_access_to = billingProjects.head.datasetGoogleProject // // s""" // |WITH spend_categories AS ( @@ -471,6 +461,46 @@ class SpendReportingService( Future.failed(RawlsExceptionWithErrorReport(ErrorReport(StatusCodes.InternalServerError, ex))) } +// def getSpendForUserWorkspaces( +// start: DateTime, +// end: DateTime +// ): Future[SpendReportingResults] = +// for { +// workspaces <- getOwnerWorkspaces() +// billing <- getBillingForWorkspaces(workspaces) +// query = getAllUserWorkspaceQuery(billing) +// +// } yield query + + def getSpendForAllWorkspaces( + project: RawlsBillingProjectName, + start: DateTime, + end: DateTime, + aggregations: Set[SpendReportingAggregationKeyWithSub] + ): Future[SpendReportingResults] = { + validateReportParameters(start, end) + requireProjectAction(project, SamBillingProjectActions.readSpendReport) { + for { + spendExportConf <- getSpendExportConfiguration(project) + projectNames <- getWorkspaceGoogleProjects(project) + + query = getQuery(aggregations, spendExportConf) + queryJob = setUpQuery(query, spendExportConf, start, end, projectNames) + + job: Job <- bigQueryService.use(_.runJob(queryJob)).unsafeToFuture().map(_.waitFor()) + _ = logSpendQueryStats(job.getStatistics[JobStatistics.QueryStatistics]) + result = job.getQueryResults() + } yield result.getValues.asScala.toList match { + case Nil => + throw RawlsExceptionWithErrorReport( + StatusCodes.NotFound, + s"no spend data found for billing project ${project.value} between dates ${toISODateString(start)} and ${toISODateString(end)}" + ) + case rows => extractSpendReportingResults(rows, start, end, projectNames, aggregations) + } + } + } + def getOwnerWorkspaces(): Future[Seq[WorkspaceListResponse]] = workspaceServiceConstructor(ctx).listWorkspaces(WorkspaceFieldSpecs(), -1) map { case JsArray(jsArray) => @@ -480,12 +510,38 @@ class SpendReportingService( } def getBillingForWorkspaces( - workspaces: Future[Seq[WorkspaceListResponse]] - ): Future[Seq[BillingProjectSpendConfiguration]] = - workspaces.flatMap { wsList => - val billingFutures = wsList.map(ws => - userServiceConstructor(ctx).getBillingProjectSpendConfiguration(RawlsBillingProjectName(ws.workspace.namespace)) - ) - Future.sequence(billingFutures).map(_.flatten.distinct) + workspaces: Seq[WorkspaceListResponse] + ): Future[Map[String, Seq[GoogleProjectId]]] = { + val groupedWorkspaces = workspaces.groupBy(_.workspace.namespace) + val billingFutures: Iterable[Future[Option[(String, Seq[GoogleProjectId])]]] = groupedWorkspaces.map { + case (namespace, wsList) => + billingRepository.getBillingProject(RawlsBillingProjectName(namespace)).map { + case Some( + RawlsBillingProject(_, + _, + _, + _, + _, + _, + _, + _, + Some(spendReportDataset), + Some(spendReportTable), + Some(spendReportDatasetGoogleProject), + _, + _, + _ + ) + ) => + Some( + spendReportDatasetGoogleProject + "." + spendReportDataset + "." + spendReportTable -> wsList + .map(_.workspace.googleProject) + ) + case _ => + None + } } + Future.sequence(billingFutures).map(_.flatten.toMap) + } + } diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala index 7b5c6d05bb..bd22d7481a 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala @@ -2,7 +2,6 @@ package org.broadinstitute.dsde.rawls.spendreporting import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport import akka.http.scaladsl.model.StatusCodes -import akka.http.scaladsl.model.headers.OAuth2BearerToken import bio.terra.profile.model.SpendReportingAggregation.AggregationKeyEnum import bio.terra.profile.model.SpendReportingForDateRange.CategoryEnum import bio.terra.profile.model.{ @@ -22,13 +21,13 @@ import org.broadinstitute.dsde.rawls.billing.{ } import org.broadinstitute.dsde.rawls.config.SpendReportingServiceConfig import org.broadinstitute.dsde.rawls.dataaccess.slick.TestDriverComponent -import org.broadinstitute.dsde.rawls.dataaccess.{GoogleServicesDAO, SamDAO, SlickDataSource} +import org.broadinstitute.dsde.rawls.dataaccess.{SamDAO, SlickDataSource} import org.broadinstitute.dsde.rawls.model.Attributable.AttributeMap import org.broadinstitute.dsde.rawls.model._ import org.broadinstitute.dsde.rawls.util.MockitoTestUtils -import org.broadinstitute.dsde.rawls.{model, RawlsException, RawlsExceptionWithErrorReport, TestExecutionContext} +import org.broadinstitute.dsde.rawls.{model, RawlsException, RawlsExceptionWithErrorReport} import org.broadinstitute.dsde.workbench.google2.GoogleBigQueryService -import org.broadinstitute.dsde.workbench.model.google.{BigQueryDatasetName, GoogleProject} +import org.broadinstitute.dsde.workbench.model.google.{BigQueryDatasetName, BigQueryTableName, GoogleProject} import org.joda.time.DateTime import org.joda.time.format.ISODateTimeFormat import org.mockito.ArgumentMatchers.{any, eq => mockitoEq} @@ -43,11 +42,9 @@ import scala.concurrent.{Await, Future} import scala.jdk.CollectionConverters._ import scala.math.BigDecimal.RoundingMode import org.broadinstitute.dsde.rawls.workspace.WorkspaceService -import org.scalatestplus.mockito.MockitoSugar.mock import spray.json.DefaultJsonProtocol._ import spray.json._ import org.broadinstitute.dsde.rawls.model.WorkspaceJsonSupport.WorkspaceListResponseFormat -import org.broadinstitute.dsde.rawls.user.UserService class SpendReportingServiceSpec extends AnyFlatSpecLike @@ -78,10 +75,6 @@ class SpendReportingServiceSpec _ => mockWorkspaceService } - val mockUserServiceConstructor: RawlsRequestContext => UserService = { - lazy val mockUserService: UserService = mock[UserService] - _ => mockUserService - } override val testContext: RawlsRequestContext = RawlsRequestContext(userInfo) object TestData { val workspace1: Workspace = workspace("workspace1", GoogleProjectId("project1")) @@ -581,8 +574,7 @@ class SpendReportingServiceSpec bpmDAO, samDAO, spendReportingServiceConfig, - mockWorkspaceServiceConstructor, - mockUserServiceConstructor + mockWorkspaceServiceConstructor ) ) val billingProjectSpendExport = @@ -625,8 +617,7 @@ class SpendReportingServiceSpec bpmDAO, samDAO, spendReportingServiceConfig, - mockWorkspaceServiceConstructor, - mockUserServiceConstructor + mockWorkspaceServiceConstructor ) val e = intercept[RawlsExceptionWithErrorReport] { @@ -658,8 +649,7 @@ class SpendReportingServiceSpec bpmDAO, samDAO, spendReportingServiceConfig, - mockWorkspaceServiceConstructor, - mockUserServiceConstructor + mockWorkspaceServiceConstructor ) val e = intercept[RawlsExceptionWithErrorReport] { @@ -687,8 +677,7 @@ class SpendReportingServiceSpec bpmDAO, samDAO, spendReportingServiceConfig, - mockWorkspaceServiceConstructor, - mockUserServiceConstructor + mockWorkspaceServiceConstructor ) val projectName = RawlsBillingProjectName("fakeProject") @@ -724,8 +713,7 @@ class SpendReportingServiceSpec bpmDAO, samDAO, spendReportingServiceConfig, - mockWorkspaceServiceConstructor, - mockUserServiceConstructor + mockWorkspaceServiceConstructor ) ) val billingProjectSpendExport = @@ -789,8 +777,7 @@ class SpendReportingServiceSpec bpmDAO, samDAO, spendReportingServiceConfig, - mockWorkspaceServiceConstructor, - mockUserServiceConstructor + mockWorkspaceServiceConstructor ) val result = Await.result( @@ -843,8 +830,7 @@ class SpendReportingServiceSpec bpmDAO, samDAO, spendReportingServiceConfig, - mockWorkspaceServiceConstructor, - mockUserServiceConstructor + mockWorkspaceServiceConstructor ) ) val billingProjectSpendExport = @@ -893,8 +879,7 @@ class SpendReportingServiceSpec bpmDAO, samDAO, spendReportingServiceConfig, - mockWorkspaceServiceConstructor, - mockUserServiceConstructor + mockWorkspaceServiceConstructor ) ) val billingProjectSpendExport = @@ -957,8 +942,7 @@ class SpendReportingServiceSpec bpmDAO, samDAO, spendReportingServiceConfig, - mockWorkspaceServiceConstructor, - mockUserServiceConstructor + mockWorkspaceServiceConstructor ) val e = intercept[RawlsExceptionWithErrorReport] { @@ -981,8 +965,7 @@ class SpendReportingServiceSpec mock[BillingProfileManagerDAO], mock[SamDAO], spendReportingServiceConfig, - mockWorkspaceServiceConstructor, - mockUserServiceConstructor + mockWorkspaceServiceConstructor ) val startDate = DateTime.now().minusDays(spendReportingServiceConfig.maxDateRange) val endDate = DateTime.now() @@ -998,8 +981,7 @@ class SpendReportingServiceSpec mock[BillingProfileManagerDAO], mock[SamDAO], spendReportingServiceConfig, - mockWorkspaceServiceConstructor, - mockUserServiceConstructor + mockWorkspaceServiceConstructor ) val startDate = DateTime.now() val endDate = DateTime.now().minusDays(1) @@ -1016,8 +998,7 @@ class SpendReportingServiceSpec mock[BillingProfileManagerDAO], mock[SamDAO], spendReportingServiceConfig, - mockWorkspaceServiceConstructor, - mockUserServiceConstructor + mockWorkspaceServiceConstructor ) val startDate = DateTime.now().minusDays(spendReportingServiceConfig.maxDateRange + 1) val endDate = DateTime.now() @@ -1047,8 +1028,7 @@ class SpendReportingServiceSpec mock[BillingProfileManagerDAO], mock[SamDAO], spendReportingServiceConfig, - mockWorkspaceServiceConstructor, - mockUserServiceConstructor + mockWorkspaceServiceConstructor ) val result = service.getQuery( Set( @@ -1082,8 +1062,7 @@ class SpendReportingServiceSpec mock[BillingProfileManagerDAO], mock[SamDAO], spendReportingServiceConfig, - mockWorkspaceServiceConstructor, - mockUserServiceConstructor + mockWorkspaceServiceConstructor ) val result = service.getQuery( Set( @@ -1113,8 +1092,7 @@ class SpendReportingServiceSpec mock[BillingProfileManagerDAO], mock[SamDAO], spendReportingServiceConfig, - mockWorkspaceServiceConstructor, - mockUserServiceConstructor + mockWorkspaceServiceConstructor ) val result = Await.result(service.getWorkspaceGoogleProjects(RawlsBillingProjectName("")), Duration.Inf) @@ -1185,8 +1163,7 @@ class SpendReportingServiceSpec mock[BillingProfileManagerDAO], mock[SamDAO], spendReportingServiceConfig, - mockWorkspaceServiceConstructor, - mockUserServiceConstructor + mockWorkspaceServiceConstructor ) val result = Await.result(service.getOwnerWorkspaces(), Duration.Inf) @@ -1198,67 +1175,71 @@ class SpendReportingServiceSpec val dataSource = mock[SlickDataSource] - val billingProject1 = billingProjectFromName("billingProject1") - val spendReportDatasetName1 = BigQueryDatasetName("test_dataset") - val spendReportGoogleProject1 = GoogleProject("some_other_google_project") - val spendReportConfiguration1 = - BillingProjectSpendConfiguration(spendReportGoogleProject1, spendReportDatasetName1) - - val billingProject2 = billingProjectFromName("billingProject2") - val spendReportDatasetName2 = BigQueryDatasetName("test_dataset2") - val spendReportGoogleProject2 = GoogleProject("some_other_google_project2") - val spendReportConfiguration2 = - BillingProjectSpendConfiguration(spendReportGoogleProject2, spendReportDatasetName2) - - val billingProject3 = billingProjectFromName("billingProject3") - - val mockUserService = mock[UserService](RETURNS_SMART_NULLS) - when(mockUserService.getBillingProjectSpendConfiguration(billingProject1.projectName)) - .thenReturn(Future.successful(Some(spendReportConfiguration1))) + val billingProject1 = + RawlsBillingProject( + RawlsBillingProjectName("billingProject1"), + CreationStatuses.Ready, + None, + None, + spendReportDataset = Some(BigQueryDatasetName("billing1_dataset")), + spendReportTable = Some(BigQueryTableName("billing1_table")), + spendReportDatasetGoogleProject = Some(GoogleProject("billing1_bq_project")) + ) - when(mockUserService.getBillingProjectSpendConfiguration(billingProject2.projectName)) - .thenReturn(Future.successful(Some(spendReportConfiguration2))) + val billingProject2 = + RawlsBillingProject( + RawlsBillingProjectName("billingProject2"), + CreationStatuses.Ready, + None, + None, + spendReportDataset = Some(BigQueryDatasetName("billing2_dataset")), + spendReportTable = Some(BigQueryTableName("billing2_table")), + spendReportDatasetGoogleProject = Some(GoogleProject("billing2_bq_project")) + ) - when(mockUserService.getBillingProjectSpendConfiguration(billingProject3.projectName)) - .thenReturn(Future.successful(None)) + val billingProject3 = + RawlsBillingProject(RawlsBillingProjectName("billingProject3"), CreationStatuses.Ready, None, None) - val mockUserServiceConstructor: RawlsRequestContext => UserService = { _ => - mockUserService - } + val billingRepository = mock[BillingRepository] + when(billingRepository.getBillingProject(RawlsBillingProjectName("billingProject1"))) + .thenReturn(Future.successful(Option.apply(billingProject1))) + when(billingRepository.getBillingProject(RawlsBillingProjectName("billingProject2"))) + .thenReturn(Future.successful(Option.apply(billingProject2))) + when(billingRepository.getBillingProject(RawlsBillingProjectName("billingProject3"))) + .thenReturn(Future.successful(Option.apply(billingProject3))) val service = new SpendReportingService( testContext, dataSource, Resource.pure[IO, GoogleBigQueryService[IO]](mock[GoogleBigQueryService[IO]]), - mock[BillingRepository], + billingRepository, mock[BillingProfileManagerDAO], mock[SamDAO], spendReportingServiceConfig, - mockWorkspaceServiceConstructor, - mockUserServiceConstructor + mockWorkspaceServiceConstructor ) val workspace1Billing1 = TestData.workspace("workspace1Billing1", - GoogleProjectId("owner1ProjectId"), + GoogleProjectId("workspace1ProjectId"), WorkspaceVersions.V1, billingProject1.projectName.value ) val workspace2Billing1 = TestData.workspace("workspace2Billing1", - GoogleProjectId("owner1ProjectId"), + GoogleProjectId("workspace2ProjectId"), WorkspaceVersions.V2, billingProject1.projectName.value ) val workspace1Billing2 = TestData.workspace("workspace1Billing2", - GoogleProjectId("owner1ProjectId"), + GoogleProjectId("workspace3ProjectId"), WorkspaceVersions.V2, billingProject2.projectName.value ) val workspace1Billing3 = TestData.workspace("workspace1Billing3", - GoogleProjectId("owner1ProjectId"), + GoogleProjectId("workspace4ProjectId"), WorkspaceVersions.V2, billingProject3.projectName.value ) @@ -1318,11 +1299,17 @@ class SpendReportingServiceSpec val result = Await.result( service.getBillingForWorkspaces( - Future.successful(Seq(workspace2Response, workspace3Response, workspace1Response, workspace4Response)) + Seq(workspace2Response, workspace3Response, workspace1Response, workspace4Response) ), Duration.Inf ) - result shouldBe Seq(spendReportConfiguration1, spendReportConfiguration2) + + result shouldBe Map( + "billing1_bq_project.billing1_dataset.billing1_table" -> Seq(GoogleProjectId("workspace2ProjectId"), + GoogleProjectId("workspace1ProjectId") + ), + "billing2_bq_project.billing2_dataset.billing2_table" -> Seq(GoogleProjectId("workspace3ProjectId")) + ) } } diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/ApiServiceSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/ApiServiceSpec.scala index 6ecdc40cf9..f0d47d63a7 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/ApiServiceSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/ApiServiceSpec.scala @@ -415,8 +415,7 @@ trait ApiServiceSpec mock[BillingProfileManagerDAO], samDAO, spendReportingServiceConfig, - workspaceServiceConstructor, - userServiceConstructor + workspaceServiceConstructor ) override val methodConfigurationServiceConstructor: RawlsRequestContext => MethodConfigurationService = From cc2c3bd69d227c75a315015444b66f91c986b63a Mon Sep 17 00:00:00 2001 From: Bria Morgan Date: Wed, 6 Nov 2024 11:32:28 -0500 Subject: [PATCH 05/31] update getbilling and query --- .../SpendReportingService.scala | 210 +++++++---------- .../SpendReportingServiceSpec.scala | 219 ++++++++++++++---- 2 files changed, 257 insertions(+), 172 deletions(-) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala index 0cd94a430c..d5a0e6e30c 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala @@ -230,11 +230,8 @@ class SpendReportingService( // all of which have optional subAggregationKeys and convert to Set[SpendReportingAggregationKey] val queryKeys = aggregations.flatMap(a => Set(Option(a.key), a.subAggregationKey).flatten) val tableName = config.spendExportTable.getOrElse(spendReportingServiceConfig.defaultTableName) - val timePartitionColumn: String = { - val isBroadTable = tableName == spendReportingServiceConfig.defaultTableName - // The Broad table uses a view with a different column name. - if (isBroadTable) spendReportingServiceConfig.defaultTimePartitionColumn else "_PARTITIONTIME" - } + val timePartitionColumn: String = getTimePartitionColumn(tableName) + s""" | SELECT | SUM(cost) as cost, @@ -245,102 +242,80 @@ class SpendReportingService( | AND $timePartitionColumn BETWEEN @startDate AND @endDate | AND project.id in UNNEST(@projects) | GROUP BY currency ${queryKeys.map(_.bigQueryGroupByClause()).mkString} - |""".stripMargin - .replace("REPLACE_TIME_PARTITION_COLUMN", timePartitionColumn) + |""".stripMargin.replace("REPLACE_TIME_PARTITION_COLUMN", timePartitionColumn) } -// def getAllUserWorkspaceQuery(billingProjects: Seq[BillingProjectSpendConfiguration] -// ): String = { -//// val tableName = config.spendExportTable.getOrElse(spendReportingServiceConfig.defaultTableName) -//// val timePartitionColumn: String = { -//// val isBroadTable = tableName == spendReportingServiceConfig.defaultTableName -//// // The Broad table uses a view with a different column name. -//// if (isBroadTable) spendReportingServiceConfig.defaultTimePartitionColumn else "_PARTITIONTIME" -//// } -//// s""" -//// | SELECT -//// | SUM(cost) as cost, -//// | SUM(IFNULL((SELECT SUM(c.amount) FROM UNNEST(credits) c), 0)) as credits, -//// | currency ${queryKeys.map(_.bigQueryAliasClause()).mkString} -//// | FROM `$tableName` -//// | WHERE billing_account_id = @billingAccountId -//// | AND $timePartitionColumn BETWEEN @startDate AND @endDate -//// | AND project.id in UNNEST(@projects) -//// | GROUP BY currency ${queryKeys.map(_.bigQueryGroupByClause()).mkString} -//// |""".stripMargin -//// .replace("REPLACE_TIME_PARTITION_COLUMN", timePartitionColumn) -// val first_billing_account_user_has_access_to = billingProjects.head.datasetGoogleProject -// -// s""" -// |WITH spend_categories AS ( -// | SELECT -// | project.id AS project_id, -// | project.name AS project_name, -// | CASE -// | WHEN service.description IN ('Cloud Storage') THEN 'Storage' -// | WHEN service.description IN ('Compute Engine', 'Google Kubernetes Engine') THEN 'Compute' -// | ELSE 'Other' -// | END AS spend_category, -// | SUM(CAST(cost AS FLOAT64)) AS category_cost -// | FROM -// | `$first_billing_account_user_has_access_to` -// | where -// | project_id in @listOfWorkspaceProjects AND -// | _PARTITIONTIME BETWEEN @startDate AND @endDate -// | GROUP BY -// | project_id, -// | project_name, -// | spend_category -// | UNION ALL -// | SELECT -// | project.id AS project_id, -// | project.name AS project_name, -// | CASE -// | WHEN service.description IN ('Cloud Storage') THEN 'Storage' -// | WHEN service.description IN ('Compute Engine', 'Google Kubernetes Engine') THEN 'Compute' -// | ELSE 'Other' -// | END AS spend_category, -// | SUM(CAST(cost AS FLOAT64)) AS category_cost -// | FROM -// | $`second_billing_account_user_has_access_to` -// | where -// | project_id in @moreProjectWorkspaces AND -// | _PARTITIONTIME BETWEEN @startDate AND @endDate -// | GROUP BY -// | project_id, -// | project_name, -// | spend_category -// | UNION ALL -// | select -// | project_id, -// | project_name, -// | spend_category, -// | category_cost -// | from -// | `broad_materialized_view` -// | where -// | project_id in ('broad', 'list') AND -// | _PARTITIONTIME BETWEEN @startDate AND @endDate -// |) -// | -// |SELECT -// | project_id, -// | project_name, -// | SUM(category_cost) AS total_cost, -// | SUM(CASE WHEN spend_category = 'Storage' THEN category_cost ELSE 0 END) AS storage_cost, -// | SUM(CASE WHEN spend_category = 'Compute' THEN category_cost ELSE 0 END) AS compute_cost, -// | SUM(CASE WHEN spend_category = 'Other' THEN category_cost ELSE 0 END) AS other_cost -// |FROM -// | spend_categories -// |GROUP BY -// | project_id, -// | project_name -// |ORDER BY -// | total_cost DESC -// |limit 5 -// |""".stripMargin -// .replace("REPLACE_TIME_PARTITION_COLUMN", timePartitionColumn) -// } + private def getTimePartitionColumn(tableName: String): String = { + val isBroadTable = tableName == spendReportingServiceConfig.defaultTableName + // The Broad table uses a view with a different column name. + if (isBroadTable) spendReportingServiceConfig.defaultTimePartitionColumn else "_PARTITIONTIME" + } + + def getAllUserWorkspaceQuery(billingProjects: Map[BillingProjectSpendExport, Seq[GoogleProjectId]]): String = { + val baseQuery = s""" + | SELECT + | project.id AS project_id, + | project.name AS project_name, + | CASE + | WHEN service.description IN ('Cloud Storage') THEN 'Storage' + | WHEN service.description IN ('Compute Engine', 'Google Kubernetes Engine') THEN 'Compute' + | ELSE 'Other' + | END AS spend_category, + | SUM(CAST(cost AS FLOAT64)) AS category_cost + | FROM + | _BILLING_ACCOUNT_TABLE + | where + | project_id in _PROJECT_ID_LIST AND + | _PARTITIONTIME BETWEEN @startDate AND @endDate + | GROUP BY + | project_id, + | project_name, + | spend_category""".stripMargin.trim + + val bpSubQuery = billingProjects + .map { bp => + val tableName = bp._1.spendExportTable.getOrElse(spendReportingServiceConfig.defaultTableName) + val timePartitionColumn: String = getTimePartitionColumn(tableName) + baseQuery + .replace("_PARTITIONTIME", timePartitionColumn) + .replace("_BILLING_ACCOUNT_TABLE", tableName) + .replace("_PROJECT_ID_LIST", "(" + bp._2.mkString(", ") + ")") + "\nUNION ALL" + } + .mkString("\n") + + val allBPQuery = s"""WITH spend_categories AS ( + |$bpSubQuery + | select + | project_id, + | project_name, + | spend_category, + | category_cost + | from + | `broad_materialized_view` + | where + | project_id in ('broad', 'list') AND + | _PARTITIONTIME BETWEEN @startDate AND @endDate + |)""" + + s""" + |$allBPQuery + |SELECT + | project_id, + | project_name, + | SUM(category_cost) AS total_cost, + | SUM(CASE WHEN spend_category = 'Storage' THEN category_cost ELSE 0 END) AS storage_cost, + | SUM(CASE WHEN spend_category = 'Compute' THEN category_cost ELSE 0 END) AS compute_cost, + | SUM(CASE WHEN spend_category = 'Other' THEN category_cost ELSE 0 END) AS other_cost + |FROM + | spend_categories + |GROUP BY + | project_id, + | project_name + |ORDER BY + | total_cost DESC + |limit 5 + |""".stripMargin.trim + } def setUpQuery( query: String, @@ -511,36 +486,17 @@ class SpendReportingService( def getBillingForWorkspaces( workspaces: Seq[WorkspaceListResponse] - ): Future[Map[String, Seq[GoogleProjectId]]] = { + ): Future[Map[BillingProjectSpendExport, Seq[GoogleProjectId]]] = { val groupedWorkspaces = workspaces.groupBy(_.workspace.namespace) - val billingFutures: Iterable[Future[Option[(String, Seq[GoogleProjectId])]]] = groupedWorkspaces.map { - case (namespace, wsList) => - billingRepository.getBillingProject(RawlsBillingProjectName(namespace)).map { - case Some( - RawlsBillingProject(_, - _, - _, - _, - _, - _, - _, - _, - Some(spendReportDataset), - Some(spendReportTable), - Some(spendReportDatasetGoogleProject), - _, - _, - _ - ) - ) => - Some( - spendReportDatasetGoogleProject + "." + spendReportDataset + "." + spendReportTable -> wsList - .map(_.workspace.googleProject) - ) - case _ => - None + val billingFutures: Iterable[Future[Option[(BillingProjectSpendExport, Seq[GoogleProjectId])]]] = + groupedWorkspaces.map { case (namespace, wsList) => + getSpendExportConfiguration(RawlsBillingProjectName(namespace)).map { exportConfig => + Some( + exportConfig -> wsList + .map(_.workspace.googleProject) + ) } - } + } Future.sequence(billingFutures).map(_.flatten.toMap) } diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala index bd22d7481a..9a9c172ba6 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala @@ -1175,73 +1175,70 @@ class SpendReportingServiceSpec val dataSource = mock[SlickDataSource] - val billingProject1 = - RawlsBillingProject( - RawlsBillingProjectName("billingProject1"), - CreationStatuses.Ready, - None, - None, - spendReportDataset = Some(BigQueryDatasetName("billing1_dataset")), - spendReportTable = Some(BigQueryTableName("billing1_table")), - spendReportDatasetGoogleProject = Some(GoogleProject("billing1_bq_project")) + val billingProject1SpendExport = + BillingProjectSpendExport(RawlsBillingProjectName("billingProject1"), + RawlsBillingAccountName("billingAccount1"), + Some("billing1_bq_project.billing1_dataset.billing1_table") ) - val billingProject2 = - RawlsBillingProject( - RawlsBillingProjectName("billingProject2"), - CreationStatuses.Ready, - None, - None, - spendReportDataset = Some(BigQueryDatasetName("billing2_dataset")), - spendReportTable = Some(BigQueryTableName("billing2_table")), - spendReportDatasetGoogleProject = Some(GoogleProject("billing2_bq_project")) + val billingProject2SpendExport = + BillingProjectSpendExport(RawlsBillingProjectName("billingProject2"), + RawlsBillingAccountName("billingAccount2"), + Some("billing2_bq_project.billing2_dataset.billing2_table") ) - val billingProject3 = - RawlsBillingProject(RawlsBillingProjectName("billingProject3"), CreationStatuses.Ready, None, None) - - val billingRepository = mock[BillingRepository] - when(billingRepository.getBillingProject(RawlsBillingProjectName("billingProject1"))) - .thenReturn(Future.successful(Option.apply(billingProject1))) - when(billingRepository.getBillingProject(RawlsBillingProjectName("billingProject2"))) - .thenReturn(Future.successful(Option.apply(billingProject2))) - when(billingRepository.getBillingProject(RawlsBillingProjectName("billingProject3"))) - .thenReturn(Future.successful(Option.apply(billingProject3))) + val billingProject3SpendExport = + BillingProjectSpendExport(RawlsBillingProjectName("billingProject3"), + RawlsBillingAccountName("billingAccount3"), + None + ) - val service = new SpendReportingService( - testContext, - dataSource, - Resource.pure[IO, GoogleBigQueryService[IO]](mock[GoogleBigQueryService[IO]]), - billingRepository, - mock[BillingProfileManagerDAO], - mock[SamDAO], - spendReportingServiceConfig, - mockWorkspaceServiceConstructor + val service = spy( + new SpendReportingService( + testContext, + dataSource, + Resource.pure[IO, GoogleBigQueryService[IO]](mock[GoogleBigQueryService[IO]]), + mock[BillingRepository], + mock[BillingProfileManagerDAO], + mock[SamDAO], + spendReportingServiceConfig, + mockWorkspaceServiceConstructor + ) ) + doReturn(Future.successful(billingProject1SpendExport)) + .when(service) + .getSpendExportConfiguration(RawlsBillingProjectName("billingProject1")) + doReturn(Future.successful(billingProject2SpendExport)) + .when(service) + .getSpendExportConfiguration(RawlsBillingProjectName("billingProject2")) + doReturn(Future.successful(billingProject3SpendExport)) + .when(service) + .getSpendExportConfiguration(RawlsBillingProjectName("billingProject3")) + val workspace1Billing1 = TestData.workspace("workspace1Billing1", GoogleProjectId("workspace1ProjectId"), WorkspaceVersions.V1, - billingProject1.projectName.value + "billingProject1" ) val workspace2Billing1 = TestData.workspace("workspace2Billing1", GoogleProjectId("workspace2ProjectId"), WorkspaceVersions.V2, - billingProject1.projectName.value + "billingProject1" ) val workspace1Billing2 = TestData.workspace("workspace1Billing2", GoogleProjectId("workspace3ProjectId"), WorkspaceVersions.V2, - billingProject2.projectName.value + "billingProject2" ) val workspace1Billing3 = TestData.workspace("workspace1Billing3", GoogleProjectId("workspace4ProjectId"), WorkspaceVersions.V2, - billingProject3.projectName.value + "billingProject3" ) val workspace1Response = WorkspaceListResponse( @@ -1305,11 +1302,143 @@ class SpendReportingServiceSpec ) result shouldBe Map( - "billing1_bq_project.billing1_dataset.billing1_table" -> Seq(GoogleProjectId("workspace2ProjectId"), - GoogleProjectId("workspace1ProjectId") - ), - "billing2_bq_project.billing2_dataset.billing2_table" -> Seq(GoogleProjectId("workspace3ProjectId")) + billingProject1SpendExport -> Seq(GoogleProjectId("workspace2ProjectId"), GoogleProjectId("workspace1ProjectId")), + billingProject2SpendExport -> Seq(GoogleProjectId("workspace3ProjectId")), + billingProject3SpendExport -> Seq(GoogleProjectId("workspace4ProjectId")) + ) + } + + "getAllUserWorkspaceQuery" should "union all billingProjects with their workspace projects" in { + + val billingProject1SpendExport = + BillingProjectSpendExport(RawlsBillingProjectName("billingProject1"), + RawlsBillingAccountName("billingAccount1"), + Some("billing1_bq_project.billing1_dataset.billing1_table") + ) + + val billingProject2SpendExport = + BillingProjectSpendExport(RawlsBillingProjectName("billingProject2"), + RawlsBillingAccountName("billingAccount2"), + Some("billing2_bq_project.billing2_dataset.billing2_table") + ) + + val billingProject3SpendExport = + BillingProjectSpendExport(RawlsBillingProjectName("billingProject3"), + RawlsBillingAccountName("billingAccount3"), + None + ) + + val inputMap = Map( + billingProject1SpendExport -> Seq(GoogleProjectId("workspace2ProjectId"), GoogleProjectId("workspace1ProjectId")), + billingProject2SpendExport -> Seq(GoogleProjectId("workspace3ProjectId")), + billingProject3SpendExport -> Seq(GoogleProjectId("workspace4ProjectId")) ) + + val expectedQuery = + s"""|WITH spend_categories AS ( + | SELECT + | project.id AS project_id, + | project.name AS project_name, + | CASE + | WHEN service.description IN ('Cloud Storage') THEN 'Storage' + | WHEN service.description IN ('Compute Engine', 'Google Kubernetes Engine') THEN 'Compute' + | ELSE 'Other' + | END AS spend_category, + | SUM(CAST(cost AS FLOAT64)) AS category_cost + | FROM + | billing1_bq_project.billing1_dataset.billing1_table + | where + | project_id in (workspace2ProjectId, workspace1ProjectId) AND + | _PARTITIONTIME BETWEEN @startDate AND @endDate + | GROUP BY + | project_id, + | project_name, + | spend_category + | UNION ALL + | SELECT + | project.id AS project_id, + | project.name AS project_name, + | CASE + | WHEN service.description IN ('Cloud Storage') THEN 'Storage' + | WHEN service.description IN ('Compute Engine', 'Google Kubernetes Engine') THEN 'Compute' + | ELSE 'Other' + | END AS spend_category, + | SUM(CAST(cost AS FLOAT64)) AS category_cost + | FROM + | billing2_bq_project.billing2_dataset.billing2_table + | where + | project_id in (workspace3ProjectId) AND + | _PARTITIONTIME BETWEEN @startDate AND @endDate + | GROUP BY + | project_id, + | project_name, + | spend_category + | UNION ALL + | SELECT + | project.id AS project_id, + | project.name AS project_name, + | CASE + | WHEN service.description IN ('Cloud Storage') THEN 'Storage' + | WHEN service.description IN ('Compute Engine', 'Google Kubernetes Engine') THEN 'Compute' + | ELSE 'Other' + | END AS spend_category, + | SUM(CAST(cost AS FLOAT64)) AS category_cost + | FROM + | fakeTable + | where + | project_id in (workspace4ProjectId) AND + | fakeTimePartitionColumn BETWEEN @startDate AND @endDate + | GROUP BY + | project_id, + | project_name, + | spend_category + | UNION ALL + | select + | project_id, + | project_name, + | spend_category, + | category_cost + | from + | `broad_materialized_view` + | where + | project_id in ('broad', 'list') AND + | _PARTITIONTIME BETWEEN @startDate AND @endDate + |) + |SELECT + | project_id, + | project_name, + | SUM(category_cost) AS total_cost, + | SUM(CASE WHEN spend_category = 'Storage' THEN category_cost ELSE 0 END) AS storage_cost, + | SUM(CASE WHEN spend_category = 'Compute' THEN category_cost ELSE 0 END) AS compute_cost, + | SUM(CASE WHEN spend_category = 'Other' THEN category_cost ELSE 0 END) AS other_cost + |FROM + | spend_categories + |GROUP BY + | project_id, + | project_name + |ORDER BY + | total_cost DESC + |limit 5 + |""".stripMargin + + val service = new SpendReportingService( + testContext, + mock[SlickDataSource], + Resource.pure[IO, GoogleBigQueryService[IO]](mock[GoogleBigQueryService[IO]]), + mock[BillingRepository], + mock[BillingProfileManagerDAO], + mock[SamDAO], + spendReportingServiceConfig, + mockWorkspaceServiceConstructor + ) + val result = service.getAllUserWorkspaceQuery( + inputMap + ) + + // It's easier and more reliable to do this than tweak line changes in the query or expected query + def normalizeWhitespace(str: String): String = str.replaceAll("\\s+", " ").trim + normalizeWhitespace(result) shouldEqual normalizeWhitespace(expectedQuery) + } } From 2adf61b1106e6b4d5b6115126930cb9f14b18284 Mon Sep 17 00:00:00 2001 From: Bria Morgan Date: Wed, 6 Nov 2024 11:35:56 -0500 Subject: [PATCH 06/31] remove unnecessary changes --- .../SpendReportingServiceSpec.scala | 27 ++++++++----------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala index 9a9c172ba6..864c496b91 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala @@ -20,14 +20,13 @@ import org.broadinstitute.dsde.rawls.billing.{ BpmAzureSpendReportApiException } import org.broadinstitute.dsde.rawls.config.SpendReportingServiceConfig -import org.broadinstitute.dsde.rawls.dataaccess.slick.TestDriverComponent import org.broadinstitute.dsde.rawls.dataaccess.{SamDAO, SlickDataSource} import org.broadinstitute.dsde.rawls.model.Attributable.AttributeMap import org.broadinstitute.dsde.rawls.model._ import org.broadinstitute.dsde.rawls.util.MockitoTestUtils -import org.broadinstitute.dsde.rawls.{model, RawlsException, RawlsExceptionWithErrorReport} +import org.broadinstitute.dsde.rawls.{model, RawlsException, RawlsExceptionWithErrorReport, TestExecutionContext} import org.broadinstitute.dsde.workbench.google2.GoogleBigQueryService -import org.broadinstitute.dsde.workbench.model.google.{BigQueryDatasetName, BigQueryTableName, GoogleProject} +import org.broadinstitute.dsde.workbench.model.google.GoogleProject import org.joda.time.DateTime import org.joda.time.format.ISODateTimeFormat import org.mockito.ArgumentMatchers.{any, eq => mockitoEq} @@ -35,6 +34,7 @@ import org.mockito.Mockito._ import org.mockito.{ArgumentCaptor, Mockito} import org.scalatest.flatspec.AnyFlatSpecLike import org.scalatest.matchers.should.Matchers +import akka.http.scaladsl.model.headers.OAuth2BearerToken import java.util.{Date, UUID} import scala.concurrent.duration.Duration @@ -46,20 +46,15 @@ import spray.json.DefaultJsonProtocol._ import spray.json._ import org.broadinstitute.dsde.rawls.model.WorkspaceJsonSupport.WorkspaceListResponseFormat -class SpendReportingServiceSpec - extends AnyFlatSpecLike - with Matchers - with MockitoTestUtils - with SprayJsonSupport - with TestDriverComponent { +class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with MockitoTestUtils with SprayJsonSupport { -// implicit val executionContext: TestExecutionContext = TestExecutionContext.testExecutionContext + implicit val executionContext: TestExecutionContext = TestExecutionContext.testExecutionContext -// val userInfo: UserInfo = UserInfo(RawlsUserEmail("owner-access"), -// OAuth2BearerToken("token"), -// 123, -// RawlsUserSubjectId("123456789876543212345") -// ) + val userInfo: UserInfo = UserInfo(RawlsUserEmail("owner-access"), + OAuth2BearerToken("token"), + 123, + RawlsUserSubjectId("123456789876543212345") + ) val wsName: WorkspaceName = WorkspaceName("myNamespace", "myWorkspace") val billingAccountName: RawlsBillingAccountName = RawlsBillingAccountName("fakeBillingAcct") @@ -75,7 +70,7 @@ class SpendReportingServiceSpec _ => mockWorkspaceService } - override val testContext: RawlsRequestContext = RawlsRequestContext(userInfo) + val testContext: RawlsRequestContext = RawlsRequestContext(userInfo) object TestData { val workspace1: Workspace = workspace("workspace1", GoogleProjectId("project1")) val workspace2: Workspace = workspace("workspace2", GoogleProjectId("project2")) From 775616e24e50189bbe3378ef0434032f6873fbdd Mon Sep 17 00:00:00 2001 From: Bria Morgan Date: Thu, 7 Nov 2024 11:09:05 -0500 Subject: [PATCH 07/31] update getbillingworkspaces to be one sql query --- .../slick/RawlsBillingProjectComponent.scala | 10 ++ .../SpendReportingService.scala | 109 ++++++++++-------- .../RawlsBillingProjectComponentSpec.scala | 12 ++ .../SpendReportingServiceSpec.scala | 12 +- 4 files changed, 88 insertions(+), 55 deletions(-) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/RawlsBillingProjectComponent.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/RawlsBillingProjectComponent.scala index 418af55f1a..82059ebd21 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/RawlsBillingProjectComponent.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/RawlsBillingProjectComponent.scala @@ -350,6 +350,16 @@ trait RawlsBillingProjectComponent { .result .map(_.headOption.map(RawlsBillingProjectRecord.toBillingProjectSpendExport)) + def getBillingProjectsSpendConfiguration( + billingProjectNames: Seq[RawlsBillingProjectName] + ): ReadAction[Seq[Option[BillingProjectSpendExport]]] = + rawlsBillingProjectQuery + .withProjectNames(billingProjectNames) + .result + .map(projectRecords => + projectRecords.map(record => Some(RawlsBillingProjectRecord.toBillingProjectSpendExport(record))) + ) + def insertOperations(operations: Seq[RawlsBillingProjectOperationRecord]): WriteAction[Unit] = (rawlsBillingProjectOperationQuery ++= operations).map(_ => ()) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala index d5a0e6e30c..32fffdbe8f 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala @@ -206,6 +206,20 @@ class SpendReportingService( ) } + def getSpendExportConfigurations(projects: Seq[RawlsBillingProjectName]): Future[Seq[BillingProjectSpendExport]] = + dataSource + .inTransaction(_.rawlsBillingProjectQuery.getBillingProjectsSpendConfiguration(projects)) + .recover { case _: RawlsException => + throw RawlsExceptionWithErrorReport( + StatusCodes.BadRequest, + s"billing account not found on billing project" // TODO: identify problem + ) + } + .map { exportOptions => + exportOptions.collect { case Some(export) => export } + + } + def getWorkspaceGoogleProjects(projectName: RawlsBillingProjectName): Future[Map[GoogleProjectId, WorkspaceName]] = dataSource.inTransaction(_.workspaceQuery.listWithBillingProject(projectName)).map { _.collect { @@ -346,6 +360,23 @@ class SpendReportingService( JobInfo.newBuilder(queryConfig).build() } + def setUpAllUserWorkspaceQuery( + query: String, + start: DateTime, + end: DateTime + ): JobInfo = { + def queryParam(value: String): QueryParameterValue = + QueryParameterValue.newBuilder().setType(StandardSQLTypeName.STRING).setValue(value).build() + + val queryConfig = QueryJobConfiguration + .newBuilder(query) + .addNamedParameter("startDate", queryParam(toISODateString(start))) + .addNamedParameter("endDate", queryParam(toISODateString(end))) + .build() + + JobInfo.newBuilder(queryConfig).build() + } + def logSpendQueryStats(stats: JobStatistics.QueryStatistics): Unit = { if (stats.getCacheHit) cacheHitRate().hit() else cacheHitRate().miss() bytesProcessedCounter += stats.getEstimatedBytesProcessed @@ -436,45 +467,33 @@ class SpendReportingService( Future.failed(RawlsExceptionWithErrorReport(ErrorReport(StatusCodes.InternalServerError, ex))) } -// def getSpendForUserWorkspaces( +// def getSpendForAllWorkspaces( +// project: RawlsBillingProjectName, // start: DateTime, -// end: DateTime -// ): Future[SpendReportingResults] = -// for { -// workspaces <- getOwnerWorkspaces() -// billing <- getBillingForWorkspaces(workspaces) -// query = getAllUserWorkspaceQuery(billing) +// end: DateTime, +// aggregations: Set[SpendReportingAggregationKeyWithSub] +// ): Future[SpendReportingResults] = { +// validateReportParameters(start, end) +// requireProjectAction(project, SamBillingProjectActions.readSpendReport) { +// for { +// workspaces <- getOwnerWorkspaces() +// billing <- getBillingForWorkspaces(workspaces) +// query = getAllUserWorkspaceQuery(billing) +// queryJob = setUpAllUserWorkspaceQuery(query, start, end) // -// } yield query - - def getSpendForAllWorkspaces( - project: RawlsBillingProjectName, - start: DateTime, - end: DateTime, - aggregations: Set[SpendReportingAggregationKeyWithSub] - ): Future[SpendReportingResults] = { - validateReportParameters(start, end) - requireProjectAction(project, SamBillingProjectActions.readSpendReport) { - for { - spendExportConf <- getSpendExportConfiguration(project) - projectNames <- getWorkspaceGoogleProjects(project) - - query = getQuery(aggregations, spendExportConf) - queryJob = setUpQuery(query, spendExportConf, start, end, projectNames) - - job: Job <- bigQueryService.use(_.runJob(queryJob)).unsafeToFuture().map(_.waitFor()) - _ = logSpendQueryStats(job.getStatistics[JobStatistics.QueryStatistics]) - result = job.getQueryResults() - } yield result.getValues.asScala.toList match { - case Nil => - throw RawlsExceptionWithErrorReport( - StatusCodes.NotFound, - s"no spend data found for billing project ${project.value} between dates ${toISODateString(start)} and ${toISODateString(end)}" - ) - case rows => extractSpendReportingResults(rows, start, end, projectNames, aggregations) - } - } - } +// job: Job <- bigQueryService.use(_.runJob(queryJob)).unsafeToFuture().map(_.waitFor()) +// _ = logSpendQueryStats(job.getStatistics[JobStatistics.QueryStatistics]) +// result = job.getQueryResults() +// } yield result.getValues.asScala.toList match { +// case Nil => +// throw RawlsExceptionWithErrorReport( +// StatusCodes.NotFound, +// s"no spend data found for billing project ${project.value} between dates ${toISODateString(start)} and ${toISODateString(end)}" +// ) +// case rows => extractSpendReportingResults(rows, start, end, projectNames, aggregations) +// } +// } +// } def getOwnerWorkspaces(): Future[Seq[WorkspaceListResponse]] = workspaceServiceConstructor(ctx).listWorkspaces(WorkspaceFieldSpecs(), -1) map { @@ -488,16 +507,12 @@ class SpendReportingService( workspaces: Seq[WorkspaceListResponse] ): Future[Map[BillingProjectSpendExport, Seq[GoogleProjectId]]] = { val groupedWorkspaces = workspaces.groupBy(_.workspace.namespace) - val billingFutures: Iterable[Future[Option[(BillingProjectSpendExport, Seq[GoogleProjectId])]]] = - groupedWorkspaces.map { case (namespace, wsList) => - getSpendExportConfiguration(RawlsBillingProjectName(namespace)).map { exportConfig => - Some( - exportConfig -> wsList - .map(_.workspace.googleProject) - ) - } - } - Future.sequence(billingFutures).map(_.flatten.toMap) + val billingProjects = groupedWorkspaces.keys.map(RawlsBillingProjectName).toList + getSpendExportConfigurations(billingProjects).map { exportConfigs => + exportConfigs.map { config => + config -> groupedWorkspaces(config.billingProjectName.value).map(ws => ws.workspace.googleProject) + }.toMap + } } } diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/RawlsBillingProjectComponentSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/RawlsBillingProjectComponentSpec.scala index 75e55acf48..9c3c2917ff 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/RawlsBillingProjectComponentSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/RawlsBillingProjectComponentSpec.scala @@ -176,6 +176,18 @@ class RawlsBillingProjectComponentSpec } } + it should "return the appropriate list of Option[BillingProjectSpendExport]s for given RawlsBillingProjectNames" in withDefaultTestDatabase { + val billingProjectNames = Seq(testData.testProject1.projectName, testData.testProject2.projectName) + + val expectedSpendExports = billingProjectNames.map { projectName => + runAndWait(rawlsBillingProjectQuery.getBillingProjectSpendConfiguration(projectName)) + } + + val actualSpendExports = runAndWait(rawlsBillingProjectQuery.getBillingProjectsSpendConfiguration(billingProjectNames)) + + actualSpendExports shouldBe expectedSpendExports + } + "BillingAccountChange" should "be able to load records that need to be sync'd" in withDefaultTestDatabase { runAndWait { import driver.api._ diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala index 864c496b91..97877de5f0 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala @@ -1201,15 +1201,11 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki ) ) - doReturn(Future.successful(billingProject1SpendExport)) + doReturn(Future.successful(Seq(billingProject1SpendExport, billingProject2SpendExport, billingProject3SpendExport))) .when(service) - .getSpendExportConfiguration(RawlsBillingProjectName("billingProject1")) - doReturn(Future.successful(billingProject2SpendExport)) - .when(service) - .getSpendExportConfiguration(RawlsBillingProjectName("billingProject2")) - doReturn(Future.successful(billingProject3SpendExport)) - .when(service) - .getSpendExportConfiguration(RawlsBillingProjectName("billingProject3")) + .getSpendExportConfigurations( + any() + ) val workspace1Billing1 = TestData.workspace("workspace1Billing1", From 414207cc26bb3a91e2e3a42e943f8514400539b3 Mon Sep 17 00:00:00 2001 From: Bria Morgan Date: Thu, 7 Nov 2024 15:14:35 -0500 Subject: [PATCH 08/31] start putting it all together --- .../SpendReportingService.scala | 61 +++--- .../SpendReportingServiceSpec.scala | 203 +++++++++++++++++- 2 files changed, 235 insertions(+), 29 deletions(-) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala index 32fffdbe8f..6106adac78 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala @@ -467,33 +467,34 @@ class SpendReportingService( Future.failed(RawlsExceptionWithErrorReport(ErrorReport(StatusCodes.InternalServerError, ex))) } -// def getSpendForAllWorkspaces( -// project: RawlsBillingProjectName, -// start: DateTime, -// end: DateTime, -// aggregations: Set[SpendReportingAggregationKeyWithSub] -// ): Future[SpendReportingResults] = { -// validateReportParameters(start, end) -// requireProjectAction(project, SamBillingProjectActions.readSpendReport) { -// for { -// workspaces <- getOwnerWorkspaces() -// billing <- getBillingForWorkspaces(workspaces) -// query = getAllUserWorkspaceQuery(billing) -// queryJob = setUpAllUserWorkspaceQuery(query, start, end) -// -// job: Job <- bigQueryService.use(_.runJob(queryJob)).unsafeToFuture().map(_.waitFor()) -// _ = logSpendQueryStats(job.getStatistics[JobStatistics.QueryStatistics]) -// result = job.getQueryResults() -// } yield result.getValues.asScala.toList match { -// case Nil => -// throw RawlsExceptionWithErrorReport( -// StatusCodes.NotFound, -// s"no spend data found for billing project ${project.value} between dates ${toISODateString(start)} and ${toISODateString(end)}" -// ) -// case rows => extractSpendReportingResults(rows, start, end, projectNames, aggregations) -// } -// } -// } + def getSpendForAllWorkspaces( + start: DateTime, + end: DateTime + ): Future[SpendReportingResults] = { + validateReportParameters(start, end) + for { + workspaces <- getOwnerWorkspaces() + projectNames = workspaces + .map(wsResp => + wsResp.workspace.googleProject -> WorkspaceName(wsResp.workspace.namespace, wsResp.workspace.name) + ) + .toMap // Map[GoogleProjectId, WorkspaceName] + billing <- getBillingSpendExportsForWorkspaces(workspaces) + query = getAllUserWorkspaceQuery(billing) + queryJob = setUpAllUserWorkspaceQuery(query, start, end) + + job: Job <- bigQueryService.use(_.runJob(queryJob)).unsafeToFuture().map(_.waitFor()) + _ = logSpendQueryStats(job.getStatistics[JobStatistics.QueryStatistics]) + result = job.getQueryResults() + } yield result.getValues.asScala.toList match { + case Nil => + throw RawlsExceptionWithErrorReport( + StatusCodes.NotFound, + s"no spend data found between dates ${toISODateString(start)} and ${toISODateString(end)}" + ) // TODO update this + case rows => extractSpendReportingResults(rows, start, end, projectNames, Set.empty) + } + } def getOwnerWorkspaces(): Future[Seq[WorkspaceListResponse]] = workspaceServiceConstructor(ctx).listWorkspaces(WorkspaceFieldSpecs(), -1) map { @@ -503,11 +504,15 @@ class SpendReportingService( case _ => throw new IllegalArgumentException("Expected a JsArray") } - def getBillingForWorkspaces( + def getBillingSpendExportsForWorkspaces( workspaces: Seq[WorkspaceListResponse] ): Future[Map[BillingProjectSpendExport, Seq[GoogleProjectId]]] = { val groupedWorkspaces = workspaces.groupBy(_.workspace.namespace) val billingProjects = groupedWorkspaces.keys.map(RawlsBillingProjectName).toList +// billingProjects.map(project => +// requireProjectAction(project, SamBillingProjectActions.readSpendReport) +// ) // TODO a better way to handle this? + getSpendExportConfigurations(billingProjects).map { exportConfigs => exportConfigs.map { config => config -> groupedWorkspaces(config.billingProjectName.value).map(ws => ws.workspace.googleProject) diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala index 97877de5f0..eda4a4bcd3 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala @@ -1286,7 +1286,7 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki ) val result = Await.result( - service.getBillingForWorkspaces( + service.getBillingSpendExportsForWorkspaces( Seq(workspace2Response, workspace3Response, workspace1Response, workspace4Response) ), Duration.Inf @@ -1432,4 +1432,205 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki } + "getSpendForAllWorkspaces" should "get the spend report from multiple billing projects" in { + val from = DateTime.now().minusMonths(2) + val to = from.plusMonths(1) + + val price1 = BigDecimal("10.22") + val price2 = BigDecimal("50.74") + val currency = "USD" + + val samDAO = mock[SamDAO](RETURNS_SMART_NULLS) + val billingRepository = mock[BillingRepository](RETURNS_SMART_NULLS) + val bpmDAO = mock[BillingProfileManagerDAO](RETURNS_SMART_NULLS) + + // Billing projects + val billingProfileId1 = UUID.randomUUID() + val projectName1 = RawlsBillingProjectName("billingProject1") + val billingAccount1 = RawlsBillingAccountName("billingAcct1") + val billingProject1 = RawlsBillingProject( + projectName1, + CreationStatuses.Ready, + Option(billingAccount1), + None, + billingProfileId = Option.apply(billingProfileId1.toString) + ) + val billingProfileId2 = UUID.randomUUID() + val projectName2 = RawlsBillingProjectName("billingProject2") + val billingAccount2 = RawlsBillingAccountName("billingAcct2") + val billingProject2 = RawlsBillingProject( + projectName2, + CreationStatuses.Ready, + Option(billingAccount2), + None, + billingProfileId = Option.apply(billingProfileId2.toString) + ) + val billingProfileId3 = UUID.randomUUID() + val projectName3 = RawlsBillingProjectName("billingProject3") + val billingAccount3 = RawlsBillingAccountName("billingAcct3") + val billingProject3 = RawlsBillingProject( + projectName3, + CreationStatuses.Ready, + Option(billingAccount3), + None, + billingProfileId = Option.apply(billingProfileId3.toString) + ) + + // Billing project spend exports + val billingProject1SpendExport = + BillingProjectSpendExport(RawlsBillingProjectName("billingProject1"), + RawlsBillingAccountName("billingAccount1"), + Some("billing1_bq_project.billing1_dataset.billing1_table") + ) + + val billingProject3SpendExport = + BillingProjectSpendExport(RawlsBillingProjectName("billingProject3"), + RawlsBillingAccountName("billingAccount3"), + None + ) + + // Workspaces + val workspace1Billing1 = + TestData.workspace("workspace1Billing1", + GoogleProjectId("workspace1ProjectId"), + WorkspaceVersions.V1, + "billingProject1" + ) + val workspace2Billing1 = + TestData.workspace("workspace2Billing1", + GoogleProjectId("workspace2ProjectId"), + WorkspaceVersions.V2, + "billingProject1" + ) + val workspace1Billing2 = + TestData.workspace("workspace1Billing2", + GoogleProjectId("workspace3ProjectId"), + WorkspaceVersions.V2, + "billingProject2" + ) + val workspace1Billing3 = + TestData.workspace("workspace1Billing3", + GoogleProjectId("workspace4ProjectId"), + WorkspaceVersions.V2, + "billingProject3" + ) + + // Only workspaces 1 and 4 are owned + val workspace1Response = WorkspaceListResponse( + WorkspaceAccessLevels.Owner, + Some(true), + Some(true), + WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing1, + Option(Set.empty), + true, + Some(WorkspaceCloudPlatform.Gcp) + ), + Option.empty, + false, + Some(List.empty) + ) + val workspace2Response = WorkspaceListResponse( + WorkspaceAccessLevels.Read, + Some(true), + Some(true), + WorkspaceDetails.fromWorkspaceAndOptions(workspace2Billing1, + Option(Set.empty), + true, + Some(WorkspaceCloudPlatform.Gcp) + ), + Option.empty, + false, + Some(List.empty) + ) + val workspace3Response = WorkspaceListResponse( + WorkspaceAccessLevels.Write, + Some(true), + Some(true), + WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing2, + Option(Set.empty), + true, + Some(WorkspaceCloudPlatform.Gcp) + ), + Option.empty, + false, + Some(List.empty) + ) + val workspace4Response = WorkspaceListResponse( + WorkspaceAccessLevels.Owner, + Some(true), + Some(true), + WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing3, + Option(Set.empty), + true, + Some(WorkspaceCloudPlatform.Gcp) + ), + Option.empty, + false, + Some(List.empty) + ) + + val dataSource = mock[SlickDataSource] + val mockWorkspaceService = mock[WorkspaceService](RETURNS_SMART_NULLS) + + when(mockWorkspaceService.listWorkspaces(any(), any())) + .thenReturn( + Future.successful(Seq(workspace1Response, workspace2Response, workspace3Response, workspace4Response).toJson) + ) + val mockWorkspaceServiceConstructor: RawlsRequestContext => WorkspaceService = { _ => + mockWorkspaceService + } + + when(billingRepository.getBillingProject(mockitoEq(projectName1))) + .thenReturn(Future.successful(Option.apply(billingProject1))) + when(billingRepository.getBillingProject(mockitoEq(projectName2))) + .thenReturn(Future.successful(Option.apply(billingProject2))) + when(billingRepository.getBillingProject(mockitoEq(projectName3))) + .thenReturn(Future.successful(Option.apply(billingProject3))) + + when(samDAO.userHasAction(any(), any(), any(), any())).thenReturn(Future.successful(true)) + + val bigQueryService = mockBigQuery(List[Map[String, String]]()) + + val service = spy( + new SpendReportingService( + testContext, + mock[SlickDataSource], + bigQueryService, + billingRepository, + bpmDAO, + samDAO, + spendReportingServiceConfig, + mockWorkspaceServiceConstructor + ) + ) + doReturn(Future.successful(Seq(billingProject1SpendExport, billingProject3SpendExport))) + .when(service) + .getSpendExportConfigurations( + any() + ) + + val spendReport = + TestData.BpmSpendReport.spendData(from, to, currency, Map("Compute" -> price1, "Storage" -> price2)) + + val billingProfileIdCapture: ArgumentCaptor[UUID] = ArgumentCaptor.forClass(classOf[UUID]) + val startDateCapture: ArgumentCaptor[Date] = ArgumentCaptor.forClass(classOf[Date]) + val endDateCapture: ArgumentCaptor[Date] = ArgumentCaptor.forClass(classOf[Date]) + + val result = Await.result( + service.getSpendForAllWorkspaces(from, to), + Duration.Inf + ) + + result.spendSummary.credits shouldBe "0" + result.spendSummary.cost shouldBe Seq(price1, price2).sum.toString() + result.spendSummary.currency shouldBe "USD" + result.spendSummary.startTime.get.toString(ISODateTimeFormat.date()) shouldBe from.toString( + ISODateTimeFormat.date() + ) + result.spendSummary.endTime.get.toString(ISODateTimeFormat.date()) shouldBe to.toString(ISODateTimeFormat.date()) + + startDateCapture.getValue shouldBe from.toDate + endDateCapture.getValue shouldBe to.toDate + } + } From c5c736a431a20b6282bb56bb85c68bccffadeef2 Mon Sep 17 00:00:00 2001 From: Bria Morgan Date: Thu, 7 Nov 2024 15:59:01 -0500 Subject: [PATCH 09/31] add new spendreport api --- core/src/main/resources/swagger/api-docs.yaml | 49 +++ .../webservice/BillingApiServiceV2.scala | 286 ++++++++++-------- 2 files changed, 201 insertions(+), 134 deletions(-) diff --git a/core/src/main/resources/swagger/api-docs.yaml b/core/src/main/resources/swagger/api-docs.yaml index 44e808a2bc..66f313908d 100644 --- a/core/src/main/resources/swagger/api-docs.yaml +++ b/core/src/main/resources/swagger/api-docs.yaml @@ -450,6 +450,55 @@ paths: $ref: '#/components/schemas/ErrorReport' 500: $ref: '#/components/responses/RawlsInternalError' + /api/billing/v2/spendReport: + get: + tags: + - billing_v2 + summary: get spend report for all workspaces user has owner access to + description: get spend report for all workspaces user has owner access to + operationId: getSpendReportAllWorkspaces + parameters: + - name: startDate + in: query + description: start date of report (YYYY-MM-DD). Data included in report will start at 12 AM UTC on this date. + required: true + schema: + type: string + format: date + - name: endDate + in: query + description: end date of report (YYYY-MM-DD). Data included in report will end at 11:59 PM UTC on this date. + required: true + schema: + type: string + format: date + responses: + 200: + description: Success + content: + 'application/json': + schema: + $ref: '#/components/schemas/SpendReport' + 400: + description: invalid spend report parameters + content: + 'application/json': + schema: + $ref: '#/components/schemas/ErrorReport' + 403: + description: You must be a project owner to view the spend report of a project + content: + 'application/json': + schema: + $ref: '#/components/schemas/ErrorReport' + 404: + description: The specified billing project could not be found + content: + 'application/json': + schema: + $ref: '#/components/schemas/ErrorReport' + 500: + $ref: '#/components/responses/RawlsInternalError' /api/billing/v2/{projectId}/spendReportConfiguration: get: tags: diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/webservice/BillingApiServiceV2.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/webservice/BillingApiServiceV2.scala index b1204568b4..ae44d9f317 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/webservice/BillingApiServiceV2.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/webservice/BillingApiServiceV2.scala @@ -59,177 +59,195 @@ trait BillingApiServiceV2 extends UserInfoDirectives { requireUserInfo(Option(otelContext)) { userInfo => val ctx = RawlsRequestContext(userInfo, Option(otelContext)) pathPrefix("billing" / "v2") { - - pathPrefix(Segment) { projectId => - pathEnd { + pathPrefix("spendReport") { + pathEndOrSingleSlash { get { - complete { - import spray.json._ - userServiceConstructor(ctx).getBillingProject(RawlsBillingProjectName(projectId)).map { - case Some(projectResponse) => StatusCodes.OK -> Option(projectResponse).toJson - case None => StatusCodes.NotFound -> Option(StatusCodes.NotFound.defaultMessage).toJson + parameters( + "startDate".as[DateTime], + "endDate".as[DateTime] + ) { (startDate, endDate) => + complete { + spendReportingConstructor(ctx).getSpendForAllWorkspaces( + startDate, + endDate.plusDays(1).minusMillis(1) + ) } } - } ~ - delete { + } + } + } ~ + pathPrefix(Segment) { projectId => + pathEnd { + get { complete { - billingProjectOrchestratorConstructor(ctx) - .deleteBillingProjectV2(RawlsBillingProjectName(projectId)) - .map(_ => StatusCodes.NoContent) + import spray.json._ + userServiceConstructor(ctx).getBillingProject(RawlsBillingProjectName(projectId)).map { + case Some(projectResponse) => StatusCodes.OK -> Option(projectResponse).toJson + case None => StatusCodes.NotFound -> Option(StatusCodes.NotFound.defaultMessage).toJson + } } - } - } ~ - pathPrefix("spendReport") { - pathEndOrSingleSlash { - get { - parameters( - "startDate".as[DateTime], - "endDate".as[DateTime], - "aggregationKey" - .as[SpendReportingAggregationKeyWithSub](aggregationKeyParameterUnmarshaller) - .repeated - ) { (startDate, endDate, aggregationKeyParameters) => - complete { - spendReportingConstructor(ctx).getSpendForBillingProject( - RawlsBillingProjectName(projectId), - startDate, - endDate.plusDays(1).minusMillis(1), - aggregationKeyParameters.toSet - ) - } + } ~ + delete { + complete { + billingProjectOrchestratorConstructor(ctx) + .deleteBillingProjectV2(RawlsBillingProjectName(projectId)) + .map(_ => StatusCodes.NoContent) } } - } } ~ - pathPrefix("spendReportConfiguration") { - pathEnd { - put { - entity(as[BillingProjectSpendConfiguration]) { spendConfiguration => - complete { - userServiceConstructor(ctx) - .setBillingProjectSpendConfiguration(RawlsBillingProjectName(projectId), spendConfiguration) - .map(_ => StatusCodes.NoContent) + pathPrefix("spendReport") { + pathEndOrSingleSlash { + get { + parameters( + "startDate".as[DateTime], + "endDate".as[DateTime], + "aggregationKey" + .as[SpendReportingAggregationKeyWithSub](aggregationKeyParameterUnmarshaller) + .repeated + ) { (startDate, endDate, aggregationKeyParameters) => + complete { + spendReportingConstructor(ctx).getSpendForBillingProject( + RawlsBillingProjectName(projectId), + startDate, + endDate.plusDays(1).minusMillis(1), + aggregationKeyParameters.toSet + ) + } } } - } ~ - delete { - complete { - userServiceConstructor(ctx) - .clearBillingProjectSpendConfiguration(RawlsBillingProjectName(projectId)) - .map(_ => StatusCodes.NoContent) + } + } ~ + pathPrefix("spendReportConfiguration") { + pathEnd { + put { + entity(as[BillingProjectSpendConfiguration]) { spendConfiguration => + complete { + userServiceConstructor(ctx) + .setBillingProjectSpendConfiguration(RawlsBillingProjectName(projectId), spendConfiguration) + .map(_ => StatusCodes.NoContent) + } } } ~ - get { - complete { - userServiceConstructor(ctx) - .getBillingProjectSpendConfiguration(RawlsBillingProjectName(projectId)) - .map { - case Some(config) => StatusCodes.OK -> Option(config) - case None => StatusCodes.NoContent -> None - } + delete { + complete { + userServiceConstructor(ctx) + .clearBillingProjectSpendConfiguration(RawlsBillingProjectName(projectId)) + .map(_ => StatusCodes.NoContent) + } + } ~ + get { + complete { + userServiceConstructor(ctx) + .getBillingProjectSpendConfiguration(RawlsBillingProjectName(projectId)) + .map { + case Some(config) => StatusCodes.OK -> Option(config) + case None => StatusCodes.NoContent -> None + } + } } - } - } - } ~ - pathPrefix("billingAccount") { - pathEnd { - put { - entity(as[UpdateRawlsBillingAccountRequest]) { updateProjectRequest => - complete { - userServiceConstructor(ctx) - .updateBillingProjectBillingAccount(RawlsBillingProjectName(projectId), updateProjectRequest) - .map { + } + } ~ + pathPrefix("billingAccount") { + pathEnd { + put { + entity(as[UpdateRawlsBillingAccountRequest]) { updateProjectRequest => + complete { + userServiceConstructor(ctx) + .updateBillingProjectBillingAccount(RawlsBillingProjectName(projectId), updateProjectRequest) + .map { + case Some(billingProject) => StatusCodes.OK -> Option(billingProject) + case None => StatusCodes.NoContent -> None + } + } + } + } ~ + delete { + complete { + userServiceConstructor(ctx).deleteBillingAccount(RawlsBillingProjectName(projectId)).map { case Some(billingProject) => StatusCodes.OK -> Option(billingProject) case None => StatusCodes.NoContent -> None } + } } - } - } ~ - delete { + } + } ~ + pathPrefix("members") { + pathEnd { + get { complete { - userServiceConstructor(ctx).deleteBillingAccount(RawlsBillingProjectName(projectId)).map { - case Some(billingProject) => StatusCodes.OK -> Option(billingProject) - case None => StatusCodes.NoContent -> None + userServiceConstructor(ctx).getBillingProjectMembers(RawlsBillingProjectName(projectId)) + } + } ~ + patch { + parameter(Symbol("inviteUsersNotFound").?) { inviteUsersNotFound => + entity(as[BatchProjectAccessUpdate]) { batchProjectAccessUpdate => + complete { + userServiceConstructor(ctx) + .batchUpdateBillingProjectMembers(RawlsBillingProjectName(projectId), + batchProjectAccessUpdate, + inviteUsersNotFound.getOrElse("false").toBoolean + ) + .map(_ => StatusCodes.NoContent -> None) + } + } } } - } - } - } ~ - pathPrefix("members") { - pathEnd { - get { - complete { - userServiceConstructor(ctx).getBillingProjectMembers(RawlsBillingProjectName(projectId)) - } } ~ - patch { - parameter(Symbol("inviteUsersNotFound").?) { inviteUsersNotFound => - entity(as[BatchProjectAccessUpdate]) { batchProjectAccessUpdate => + // these routes are for adding/removing users from projects + path(Segment / Segment) { (workbenchRole, userEmail) => + put { + complete { + userServiceConstructor(ctx) + .addUserToBillingProjectV2(RawlsBillingProjectName(projectId), + ProjectAccessUpdate(userEmail, + ProjectRoles.withName(workbenchRole) + ) + ) + .map(_ => StatusCodes.OK) + } + } ~ + delete { complete { userServiceConstructor(ctx) - .batchUpdateBillingProjectMembers(RawlsBillingProjectName(projectId), - batchProjectAccessUpdate, - inviteUsersNotFound.getOrElse("false").toBoolean + .removeUserFromBillingProjectV2(RawlsBillingProjectName(projectId), + ProjectAccessUpdate(userEmail, + ProjectRoles.withName(workbenchRole) + ) ) - .map(_ => StatusCodes.NoContent -> None) + .map(_ => StatusCodes.OK) } } - } } } ~ - // these routes are for adding/removing users from projects - path(Segment / Segment) { (workbenchRole, userEmail) => - put { + pathPrefix("bucketMigration") { + val billingProjectName = RawlsBillingProjectName(projectId) + pathEndOrSingleSlash { + post { complete { - userServiceConstructor(ctx) - .addUserToBillingProjectV2(RawlsBillingProjectName(projectId), - ProjectAccessUpdate(userEmail, ProjectRoles.withName(workbenchRole)) - ) - .map(_ => StatusCodes.OK) + bucketMigrationServiceConstructor(ctx) + .migrateWorkspaceBucketsInBillingProject(billingProjectName) + .map(StatusCodes.Created -> _) } } ~ - delete { + get { complete { - userServiceConstructor(ctx) - .removeUserFromBillingProjectV2(RawlsBillingProjectName(projectId), - ProjectAccessUpdate(userEmail, - ProjectRoles.withName(workbenchRole) - ) - ) - .map(_ => StatusCodes.OK) + bucketMigrationServiceConstructor(ctx) + .getBucketMigrationAttemptsForBillingProject(billingProjectName) + .map(ms => StatusCodes.OK -> ms) } } - } - } ~ - pathPrefix("bucketMigration") { - val billingProjectName = RawlsBillingProjectName(projectId) - pathEndOrSingleSlash { - post { - complete { - bucketMigrationServiceConstructor(ctx) - .migrateWorkspaceBucketsInBillingProject(billingProjectName) - .map(StatusCodes.Created -> _) - } } ~ - get { - complete { - bucketMigrationServiceConstructor(ctx) - .getBucketMigrationAttemptsForBillingProject(billingProjectName) - .map(ms => StatusCodes.OK -> ms) - } - } - } ~ - path("progress") { - get { - complete { - bucketMigrationServiceConstructor(ctx) - .getBucketMigrationProgressForBillingProject(billingProjectName) - .map(StatusCodes.OK -> _) + path("progress") { + get { + complete { + bucketMigrationServiceConstructor(ctx) + .getBucketMigrationProgressForBillingProject(billingProjectName) + .map(StatusCodes.OK -> _) + } } } - } - } - } ~ + } + } ~ pathEnd { get { complete { From 81f843b00372867532062f683368394ea6aad7fe Mon Sep 17 00:00:00 2001 From: Bria Morgan Date: Fri, 8 Nov 2024 16:30:02 -0500 Subject: [PATCH 10/31] query roughly working --- .../SpendReportingService.scala | 96 ++++++++++++++----- .../SpendReportingServiceSpec.scala | 64 ++++++++++--- 2 files changed, 126 insertions(+), 34 deletions(-) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala index 6106adac78..3daec4f9f3 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala @@ -140,6 +140,65 @@ object SpendReportingService { SpendReportingResults(aggregations.map(aggregateRows(allRows, _)).toList, summary) } + def extractCrossBillingProjectSpendReportingResults( + allRows: List[FieldValueList], + start: DateTime, + end: DateTime, + names: Map[GoogleProjectId, WorkspaceName] + ): List[SpendReportingResults] = { + + val spendDetails = allRows.flatMap { row => + val projectId = GoogleProjectId(row.get("project_id").getStringValue) + val workspaceName = names.getOrElse( + projectId, + throw RawlsExceptionWithErrorReport( + StatusCodes.InternalServerError, + s"unexpected project $projectId returned by BigQuery" + ) + ) + val totalCost = row.get("total_cost").getDoubleValue.toString + val computeCost = row.get("compute_cost").getDoubleValue.toString + val storageCost = row.get("storage_cost").getDoubleValue.toString + val currency = row.get("currency").getStringValue + + // Each row gets a summary of compute, storage, and total + List( + SpendReportingForDateRange( + totalCost, + "0", // Ignoring credits for now; do we want to include them? + currency, + Option(start), + Option(end), + workspace = Some(workspaceName), + googleProjectId = Some(GoogleProject(projectId.value)) + ), + SpendReportingForDateRange( + computeCost, + "0", + currency, + Option(start), + Option(end), + workspace = Some(workspaceName), + googleProjectId = Some(GoogleProject(projectId.value)), + category = Some(TerraSpendCategories.Compute) + ), + SpendReportingForDateRange( + storageCost, + "0", + currency, + Option(start), + Option(end), + workspace = Some(workspaceName), + googleProjectId = Some(GoogleProject(projectId.value)), + category = Some(TerraSpendCategories.Storage) + ) + ) + } + + // TODO what format should the result be? Do we need a new model? + spendDetails.map(details => SpendReportingResults(List.empty, details)) + } + } class SpendReportingService( @@ -270,6 +329,7 @@ class SpendReportingService( | SELECT | project.id AS project_id, | project.name AS project_name, + | currency, | CASE | WHEN service.description IN ('Cloud Storage') THEN 'Storage' | WHEN service.description IN ('Compute Engine', 'Google Kubernetes Engine') THEN 'Compute' @@ -279,11 +339,12 @@ class SpendReportingService( | FROM | _BILLING_ACCOUNT_TABLE | where - | project_id in _PROJECT_ID_LIST AND + | project.id in _PROJECT_ID_LIST AND | _PARTITIONTIME BETWEEN @startDate AND @endDate | GROUP BY | project_id, | project_name, + | currency, | spend_category""".stripMargin.trim val bpSubQuery = billingProjects @@ -293,38 +354,27 @@ class SpendReportingService( baseQuery .replace("_PARTITIONTIME", timePartitionColumn) .replace("_BILLING_ACCOUNT_TABLE", tableName) - .replace("_PROJECT_ID_LIST", "(" + bp._2.mkString(", ") + ")") + "\nUNION ALL" + .replace("_PROJECT_ID_LIST", "(" + bp._2.map(id => s""""${id.value}"""").mkString(", ") + ")") } - .mkString("\n") - - val allBPQuery = s"""WITH spend_categories AS ( - |$bpSubQuery - | select - | project_id, - | project_name, - | spend_category, - | category_cost - | from - | `broad_materialized_view` - | where - | project_id in ('broad', 'list') AND - | _PARTITIONTIME BETWEEN @startDate AND @endDate - |)""" + .mkString("\nUNION ALL\n") - s""" - |$allBPQuery + s"""WITH spend_categories AS ( + |$bpSubQuery + |) |SELECT | project_id, | project_name, | SUM(category_cost) AS total_cost, | SUM(CASE WHEN spend_category = 'Storage' THEN category_cost ELSE 0 END) AS storage_cost, | SUM(CASE WHEN spend_category = 'Compute' THEN category_cost ELSE 0 END) AS compute_cost, - | SUM(CASE WHEN spend_category = 'Other' THEN category_cost ELSE 0 END) AS other_cost + | SUM(CASE WHEN spend_category = 'Other' THEN category_cost ELSE 0 END) AS other_cost, + | currency |FROM | spend_categories |GROUP BY | project_id, - | project_name + | project_name, + | currency |ORDER BY | total_cost DESC |limit 5 @@ -470,7 +520,7 @@ class SpendReportingService( def getSpendForAllWorkspaces( start: DateTime, end: DateTime - ): Future[SpendReportingResults] = { + ): Future[List[SpendReportingResults]] = { validateReportParameters(start, end) for { workspaces <- getOwnerWorkspaces() @@ -492,7 +542,7 @@ class SpendReportingService( StatusCodes.NotFound, s"no spend data found between dates ${toISODateString(start)} and ${toISODateString(end)}" ) // TODO update this - case rows => extractSpendReportingResults(rows, start, end, projectNames, Set.empty) + case rows => extractCrossBillingProjectSpendReportingResults(rows, start, end, projectNames) } } diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala index eda4a4bcd3..6d1168a6dc 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala @@ -1383,17 +1383,6 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki | project_id, | project_name, | spend_category - | UNION ALL - | select - | project_id, - | project_name, - | spend_category, - | category_cost - | from - | `broad_materialized_view` - | where - | project_id in ('broad', 'list') AND - | _PARTITIONTIME BETWEEN @startDate AND @endDate |) |SELECT | project_id, @@ -1432,6 +1421,59 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki } + "extractSpendReportingResultsAcrossBillingProjects" should "should return correct summary data" in { + val storageCostWs1 = 100.582 + val otherCostWs1 = 0.10111 + val storageCostRoundedWs1: BigDecimal = BigDecimal(storageCostWs1).setScale(2, RoundingMode.HALF_EVEN) + val otherCostRoundedWs1: BigDecimal = BigDecimal(otherCostWs1).setScale(2, RoundingMode.HALF_EVEN) + val totalCostRoundedWs1: BigDecimal = BigDecimal(storageCostWs1 + otherCostWs1).setScale(2, RoundingMode.HALF_EVEN) + + val storageCostWs2 = 20.145 + val computeCostWs2 = 150.4033 + val storageCostRoundedWs2: BigDecimal = BigDecimal(storageCostWs2).setScale(2, RoundingMode.HALF_EVEN) + val otherCostRoundedWs2: BigDecimal = BigDecimal(computeCostWs2).setScale(2, RoundingMode.HALF_EVEN) + val totalCostRoundedWs2: BigDecimal = + BigDecimal(storageCostWs2 + computeCostWs2).setScale(2, RoundingMode.HALF_EVEN) + + val computeCostWs3 = 1111.222 + val otherCostWs3 = 0.02 + val storageCostRoundedWs3: BigDecimal = BigDecimal(computeCostWs3).setScale(2, RoundingMode.HALF_EVEN) + val otherCostRoundedWs3: BigDecimal = BigDecimal(otherCostWs3).setScale(2, RoundingMode.HALF_EVEN) + val totalCostRoundedWs3: BigDecimal = BigDecimal(computeCostWs3 + otherCostWs3).setScale(2, RoundingMode.HALF_EVEN) + + val table: List[Map[String, String]] = List( + Map( + "storage_cost" -> s"$storageCostWs1", + "compute_cost" -> "0.0", + "other_cost" -> s"$otherCostWs1", + "googleProjectId" -> "workspace1ProjectId" + ), + Map( + "storage_cost" -> s"$storageCostWs2", + "compute_cost" -> s"$computeCostWs2", + "other_cost" -> "0.0", + "googleProjectId" -> "workspace2ProjectId" + ), + Map( + "storage_cost" -> "0.0", + "compute_cost" -> s"$computeCostWs3", + "other_cost" -> s"$otherCostWs3", + "googleProjectId" -> "workspace3ProjectId" + ) + ) + + val tableResult: TableResult = createTableResult(table) + + val reportingResults = SpendReportingService.extractCrossBillingProjectSpendReportingResults( + tableResult.getValues.asScala.toList, + DateTime.now().minusDays(1), + DateTime.now(), + Map() + ) + reportingResults.spendSummary.cost shouldBe TestData.Workspace.totalCostRounded.toString + reportingResults.spendDetails shouldBe empty + } + "getSpendForAllWorkspaces" should "get the spend report from multiple billing projects" in { val from = DateTime.now().minusMonths(2) val to = from.plusMonths(1) From a023bbb31f44cc605df86a0a06f6e41334ac43a8 Mon Sep 17 00:00:00 2001 From: Bria Morgan Date: Mon, 18 Nov 2024 15:49:26 -0500 Subject: [PATCH 11/31] update return value of cross billing spend report --- .../rawls/model/SpendReportingModel.scala | 2 + .../SpendReportingService.scala | 185 +++-- .../SpendReportingServiceSpec.scala | 656 +++++++++++------- 3 files changed, 534 insertions(+), 309 deletions(-) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/model/SpendReportingModel.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/model/SpendReportingModel.scala index ec64c5b897..f88df652ea 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/model/SpendReportingModel.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/model/SpendReportingModel.scala @@ -164,6 +164,8 @@ object TerraSpendCategories { case "cloudstorage" => Storage case "computeengine" => Compute case "kubernetesengine" => Compute + case "storage" => Storage + case "compute" => Compute case _ => Other } diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala index 3daec4f9f3..91f4901a8c 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala @@ -5,6 +5,8 @@ import akka.http.scaladsl.model.StatusCodes import cats.effect.IO import cats.effect.unsafe.implicits.global import com.google.cloud.bigquery.{JobStatistics, Option => _, _} +import com.google.cloud.bigquery.{Field, FieldList, FieldValue, FieldValueList} + import com.typesafe.scalalogging.LazyLogging import nl.grons.metrics4.scala.{Counter, Histogram} import org.broadinstitute.dsde.rawls.billing.{ @@ -17,19 +19,21 @@ import org.broadinstitute.dsde.rawls.dataaccess.{SamDAO, SlickDataSource} import org.broadinstitute.dsde.rawls.metrics.{GoogleInstrumented, HitRatioGauge, RawlsInstrumented} import org.broadinstitute.dsde.rawls.model.{SpendReportingAggregationKeyWithSub, _} import org.broadinstitute.dsde.rawls.spendreporting.SpendReportingService._ -import org.broadinstitute.dsde.rawls.workspace.WorkspaceService +import org.broadinstitute.dsde.rawls.workspace.{AggregatedWorkspace, AggregatedWorkspaceService, WorkspaceService} import org.broadinstitute.dsde.workbench.google2.GoogleBigQueryService import org.broadinstitute.dsde.rawls.{RawlsException, RawlsExceptionWithErrorReport} import org.broadinstitute.dsde.workbench.model.google.GoogleProject import org.joda.time.format.ISODateTimeFormat import org.joda.time.{DateTime, Days} -import spray.json.JsArray +import spray.json.{JsArray, JsValue} import org.broadinstitute.dsde.rawls.model.WorkspaceJsonSupport.WorkspaceListResponseFormat import scala.concurrent.{ExecutionContext, Future} import scala.jdk.CollectionConverters._ import scala.math.BigDecimal.RoundingMode -import org.broadinstitute.dsde.rawls.model.WorkspaceAccessLevels.Owner +import org.broadinstitute.dsde.rawls.model.WorkspaceAccessLevels.{Owner, WorkspaceAccessLevel} + +import scala.util.Try object SpendReportingService { def constructor( @@ -140,63 +144,126 @@ object SpendReportingService { SpendReportingResults(aggregations.map(aggregateRows(allRows, _)).toList, summary) } +// def extractCrossBillingProjectSpendReportingResults( +// allRows: List[FieldValueList], +// start: DateTime, +// end: DateTime, +// names: Map[GoogleProjectId, WorkspaceName] +// ): List[SpendReportingResults] = +// allRows +// .map(row => +// extractSpendReportingResults(row, start, end, names, Set.of(SpendReportingAggregationKeyWithSub(Category))) +// ) +// .collect() + def extractCrossBillingProjectSpendReportingResults( allRows: List[FieldValueList], start: DateTime, end: DateTime, names: Map[GoogleProjectId, WorkspaceName] - ): List[SpendReportingResults] = { + ): Map[String, SpendReportingResults] = { - val spendDetails = allRows.flatMap { row => - val projectId = GoogleProjectId(row.get("project_id").getStringValue) - val workspaceName = names.getOrElse( - projectId, - throw RawlsExceptionWithErrorReport( - StatusCodes.InternalServerError, - s"unexpected project $projectId returned by BigQuery" + val schema = FieldList.of( + Field.of("cost", StandardSQLTypeName.FLOAT64), + Field.of("credits", StandardSQLTypeName.STRING), + Field.of("currency", StandardSQLTypeName.STRING), + Field.of("service", StandardSQLTypeName.STRING) + ) + val groupedRows = allRows.groupBy(row => row.get("project_id").getStringValue) + + groupedRows.map { case (projectId, rows) => + val transformedRows = rows.flatMap { row => + val currency = row.get("currency").getStringValue + + List( + FieldValueList.of( + java.util.List.of( + FieldValue.of(FieldValue.Attribute.PRIMITIVE, row.get("compute_cost").getDoubleValue.toString), + FieldValue.of(FieldValue.Attribute.PRIMITIVE, "0"), + FieldValue.of(FieldValue.Attribute.PRIMITIVE, currency), + FieldValue.of(FieldValue.Attribute.PRIMITIVE, "compute") + ), + schema + ), + FieldValueList.of( + java.util.List.of( + FieldValue.of(FieldValue.Attribute.PRIMITIVE, row.get("storage_cost").getDoubleValue.toString), + FieldValue.of(FieldValue.Attribute.PRIMITIVE, "0"), + FieldValue.of(FieldValue.Attribute.PRIMITIVE, currency), + FieldValue.of(FieldValue.Attribute.PRIMITIVE, "storage") + ), + schema + ), + FieldValueList.of( + java.util.List.of( + FieldValue.of(FieldValue.Attribute.PRIMITIVE, row.get("other_cost").getDoubleValue.toString), + FieldValue.of(FieldValue.Attribute.PRIMITIVE, "0"), + FieldValue.of(FieldValue.Attribute.PRIMITIVE, currency), + FieldValue.of(FieldValue.Attribute.PRIMITIVE, "other") + ), + schema + ) ) - ) - val totalCost = row.get("total_cost").getDoubleValue.toString - val computeCost = row.get("compute_cost").getDoubleValue.toString - val storageCost = row.get("storage_cost").getDoubleValue.toString - val currency = row.get("currency").getStringValue + } - // Each row gets a summary of compute, storage, and total - List( - SpendReportingForDateRange( - totalCost, - "0", // Ignoring credits for now; do we want to include them? - currency, - Option(start), - Option(end), - workspace = Some(workspaceName), - googleProjectId = Some(GoogleProject(projectId.value)) - ), - SpendReportingForDateRange( - computeCost, - "0", - currency, - Option(start), - Option(end), - workspace = Some(workspaceName), - googleProjectId = Some(GoogleProject(projectId.value)), - category = Some(TerraSpendCategories.Compute) - ), - SpendReportingForDateRange( - storageCost, - "0", - currency, - Option(start), - Option(end), - workspace = Some(workspaceName), - googleProjectId = Some(GoogleProject(projectId.value)), - category = Some(TerraSpendCategories.Storage) - ) + projectId -> extractSpendReportingResults( + transformedRows, + start, + end, + names, + Set(SpendReportingAggregationKeyWithSub(SpendReportingAggregationKeys.Category)) ) } - - // TODO what format should the result be? Do we need a new model? - spendDetails.map(details => SpendReportingResults(List.empty, details)) +// val spendDetails = allRows.flatMap { row => +// val projectId = GoogleProjectId(row.get("project_id").getStringValue) +// val workspaceName = names.getOrElse( +// projectId, +// throw RawlsExceptionWithErrorReport( +// StatusCodes.InternalServerError, +// s"unexpected project $projectId returned by BigQuery" +// ) +// ) +//// val totalCost = row.get("total_cost").getDoubleValue.toString +//// val computeCost = row.get("compute_cost").getDoubleValue.toString +//// val storageCost = row.get("storage_cost").getDoubleValue.toString +// val currency = row.get("currency").getStringValue +// +// // Each row gets a summary of compute, storage, and total +// List( +// SpendReportingForDateRange( +// totalCost, +// "0", // Ignoring credits for now; do we want to include them? +// currency, +// Option(start), +// Option(end), +// workspace = Some(workspaceName), +// googleProjectId = Some(GoogleProject(projectId.value)) +// ), +// SpendReportingForDateRange( +// computeCost, +// "0", +// currency, +// Option(start), +// Option(end), +// workspace = Some(workspaceName), +// googleProjectId = Some(GoogleProject(projectId.value)), +// category = Some(TerraSpendCategories.Compute) +// ), +// SpendReportingForDateRange( +// storageCost, +// "0", +// currency, +// Option(start), +// Option(end), +// workspace = Some(workspaceName), +// googleProjectId = Some(GoogleProject(projectId.value)), +// category = Some(TerraSpendCategories.Storage) +// ) +// ) +// } +// +// // TODO what format should the result be? Do we need a new model? +// spendDetails.map(details => SpendReportingResults(List.empty, details)) } } @@ -520,9 +587,10 @@ class SpendReportingService( def getSpendForAllWorkspaces( start: DateTime, end: DateTime - ): Future[List[SpendReportingResults]] = { + ): Future[Map[String, SpendReportingResults]] = { validateReportParameters(start, end) for { + // TODO get projectNames from billingMap workspaces <- getOwnerWorkspaces() projectNames = workspaces .map(wsResp => @@ -530,6 +598,8 @@ class SpendReportingService( ) .toMap // Map[GoogleProjectId, WorkspaceName] billing <- getBillingSpendExportsForWorkspaces(workspaces) +// billingMap <- getBillingWithSpendPermission() + query = getAllUserWorkspaceQuery(billing) queryJob = setUpAllUserWorkspaceQuery(query, start, end) @@ -570,4 +640,17 @@ class SpendReportingService( } } +// def getBillingWithSpendPermission( +// ): Future[Map[BillingProjectSpendExport, Seq[GoogleProjectId]]] = +// for { +// billingProjectResources <- samDAO.listResourcesWithActions(SamResourceTypeNames.billingProject, +// SamBillingProjectActions.readSpendReport, +// ctx +// ) +// billingProjectIds = billingProjectResources.map(resource => RawlsBillingProjectName(resource.resourceId)).toList +// groupedWorkspaces <- workspaceServiceConstructor(ctx).getWorkspacesByBillingProjects(billingProjectIds) +// spendConfigs <- getSpendExportConfigurations(billingProjectIds) +// } yield spendConfigs.map { config => +// config -> groupedWorkspaces(config.billingProjectName).map(ws => ws.googleProjectId) +// }.toMap } diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala index 6d1168a6dc..903929f8cc 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala @@ -1330,6 +1330,7 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki | SELECT | project.id AS project_id, | project.name AS project_name, + | currency, | CASE | WHEN service.description IN ('Cloud Storage') THEN 'Storage' | WHEN service.description IN ('Compute Engine', 'Google Kubernetes Engine') THEN 'Compute' @@ -1339,16 +1340,18 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki | FROM | billing1_bq_project.billing1_dataset.billing1_table | where - | project_id in (workspace2ProjectId, workspace1ProjectId) AND + | project.id in ("workspace2ProjectId", "workspace1ProjectId") AND | _PARTITIONTIME BETWEEN @startDate AND @endDate | GROUP BY | project_id, | project_name, + | currency, | spend_category | UNION ALL | SELECT | project.id AS project_id, | project.name AS project_name, + | currency, | CASE | WHEN service.description IN ('Cloud Storage') THEN 'Storage' | WHEN service.description IN ('Compute Engine', 'Google Kubernetes Engine') THEN 'Compute' @@ -1358,16 +1361,18 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki | FROM | billing2_bq_project.billing2_dataset.billing2_table | where - | project_id in (workspace3ProjectId) AND + | project.id in ("workspace3ProjectId") AND | _PARTITIONTIME BETWEEN @startDate AND @endDate | GROUP BY | project_id, | project_name, + | currency, | spend_category | UNION ALL | SELECT | project.id AS project_id, | project.name AS project_name, + | currency, | CASE | WHEN service.description IN ('Cloud Storage') THEN 'Storage' | WHEN service.description IN ('Compute Engine', 'Google Kubernetes Engine') THEN 'Compute' @@ -1377,11 +1382,12 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki | FROM | fakeTable | where - | project_id in (workspace4ProjectId) AND + | project.id in ("workspace4ProjectId") AND | fakeTimePartitionColumn BETWEEN @startDate AND @endDate | GROUP BY | project_id, | project_name, + | currency, | spend_category |) |SELECT @@ -1390,12 +1396,14 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki | SUM(category_cost) AS total_cost, | SUM(CASE WHEN spend_category = 'Storage' THEN category_cost ELSE 0 END) AS storage_cost, | SUM(CASE WHEN spend_category = 'Compute' THEN category_cost ELSE 0 END) AS compute_cost, - | SUM(CASE WHEN spend_category = 'Other' THEN category_cost ELSE 0 END) AS other_cost + | SUM(CASE WHEN spend_category = 'Other' THEN category_cost ELSE 0 END) AS other_cost, + | currency |FROM | spend_categories |GROUP BY | project_id, - | project_name + | project_name, + | currency |ORDER BY | total_cost DESC |limit 5 @@ -1421,258 +1429,390 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki } - "extractSpendReportingResultsAcrossBillingProjects" should "should return correct summary data" in { - val storageCostWs1 = 100.582 - val otherCostWs1 = 0.10111 - val storageCostRoundedWs1: BigDecimal = BigDecimal(storageCostWs1).setScale(2, RoundingMode.HALF_EVEN) - val otherCostRoundedWs1: BigDecimal = BigDecimal(otherCostWs1).setScale(2, RoundingMode.HALF_EVEN) - val totalCostRoundedWs1: BigDecimal = BigDecimal(storageCostWs1 + otherCostWs1).setScale(2, RoundingMode.HALF_EVEN) - - val storageCostWs2 = 20.145 - val computeCostWs2 = 150.4033 - val storageCostRoundedWs2: BigDecimal = BigDecimal(storageCostWs2).setScale(2, RoundingMode.HALF_EVEN) - val otherCostRoundedWs2: BigDecimal = BigDecimal(computeCostWs2).setScale(2, RoundingMode.HALF_EVEN) - val totalCostRoundedWs2: BigDecimal = - BigDecimal(storageCostWs2 + computeCostWs2).setScale(2, RoundingMode.HALF_EVEN) - - val computeCostWs3 = 1111.222 - val otherCostWs3 = 0.02 - val storageCostRoundedWs3: BigDecimal = BigDecimal(computeCostWs3).setScale(2, RoundingMode.HALF_EVEN) - val otherCostRoundedWs3: BigDecimal = BigDecimal(otherCostWs3).setScale(2, RoundingMode.HALF_EVEN) - val totalCostRoundedWs3: BigDecimal = BigDecimal(computeCostWs3 + otherCostWs3).setScale(2, RoundingMode.HALF_EVEN) - - val table: List[Map[String, String]] = List( - Map( - "storage_cost" -> s"$storageCostWs1", - "compute_cost" -> "0.0", - "other_cost" -> s"$otherCostWs1", - "googleProjectId" -> "workspace1ProjectId" - ), - Map( - "storage_cost" -> s"$storageCostWs2", - "compute_cost" -> s"$computeCostWs2", - "other_cost" -> "0.0", - "googleProjectId" -> "workspace2ProjectId" - ), - Map( - "storage_cost" -> "0.0", - "compute_cost" -> s"$computeCostWs3", - "other_cost" -> s"$otherCostWs3", - "googleProjectId" -> "workspace3ProjectId" - ) - ) - - val tableResult: TableResult = createTableResult(table) - - val reportingResults = SpendReportingService.extractCrossBillingProjectSpendReportingResults( - tableResult.getValues.asScala.toList, - DateTime.now().minusDays(1), - DateTime.now(), - Map() - ) - reportingResults.spendSummary.cost shouldBe TestData.Workspace.totalCostRounded.toString - reportingResults.spendDetails shouldBe empty - } - - "getSpendForAllWorkspaces" should "get the spend report from multiple billing projects" in { - val from = DateTime.now().minusMonths(2) - val to = from.plusMonths(1) - - val price1 = BigDecimal("10.22") - val price2 = BigDecimal("50.74") - val currency = "USD" - - val samDAO = mock[SamDAO](RETURNS_SMART_NULLS) - val billingRepository = mock[BillingRepository](RETURNS_SMART_NULLS) - val bpmDAO = mock[BillingProfileManagerDAO](RETURNS_SMART_NULLS) - - // Billing projects - val billingProfileId1 = UUID.randomUUID() - val projectName1 = RawlsBillingProjectName("billingProject1") - val billingAccount1 = RawlsBillingAccountName("billingAcct1") - val billingProject1 = RawlsBillingProject( - projectName1, - CreationStatuses.Ready, - Option(billingAccount1), - None, - billingProfileId = Option.apply(billingProfileId1.toString) - ) - val billingProfileId2 = UUID.randomUUID() - val projectName2 = RawlsBillingProjectName("billingProject2") - val billingAccount2 = RawlsBillingAccountName("billingAcct2") - val billingProject2 = RawlsBillingProject( - projectName2, - CreationStatuses.Ready, - Option(billingAccount2), - None, - billingProfileId = Option.apply(billingProfileId2.toString) - ) - val billingProfileId3 = UUID.randomUUID() - val projectName3 = RawlsBillingProjectName("billingProject3") - val billingAccount3 = RawlsBillingAccountName("billingAcct3") - val billingProject3 = RawlsBillingProject( - projectName3, - CreationStatuses.Ready, - Option(billingAccount3), - None, - billingProfileId = Option.apply(billingProfileId3.toString) - ) - - // Billing project spend exports - val billingProject1SpendExport = - BillingProjectSpendExport(RawlsBillingProjectName("billingProject1"), - RawlsBillingAccountName("billingAccount1"), - Some("billing1_bq_project.billing1_dataset.billing1_table") - ) - - val billingProject3SpendExport = - BillingProjectSpendExport(RawlsBillingProjectName("billingProject3"), - RawlsBillingAccountName("billingAccount3"), - None - ) - - // Workspaces - val workspace1Billing1 = - TestData.workspace("workspace1Billing1", - GoogleProjectId("workspace1ProjectId"), - WorkspaceVersions.V1, - "billingProject1" - ) - val workspace2Billing1 = - TestData.workspace("workspace2Billing1", - GoogleProjectId("workspace2ProjectId"), - WorkspaceVersions.V2, - "billingProject1" - ) - val workspace1Billing2 = - TestData.workspace("workspace1Billing2", - GoogleProjectId("workspace3ProjectId"), - WorkspaceVersions.V2, - "billingProject2" - ) - val workspace1Billing3 = - TestData.workspace("workspace1Billing3", - GoogleProjectId("workspace4ProjectId"), - WorkspaceVersions.V2, - "billingProject3" - ) - - // Only workspaces 1 and 4 are owned - val workspace1Response = WorkspaceListResponse( - WorkspaceAccessLevels.Owner, - Some(true), - Some(true), - WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing1, - Option(Set.empty), - true, - Some(WorkspaceCloudPlatform.Gcp) - ), - Option.empty, - false, - Some(List.empty) - ) - val workspace2Response = WorkspaceListResponse( - WorkspaceAccessLevels.Read, - Some(true), - Some(true), - WorkspaceDetails.fromWorkspaceAndOptions(workspace2Billing1, - Option(Set.empty), - true, - Some(WorkspaceCloudPlatform.Gcp) - ), - Option.empty, - false, - Some(List.empty) - ) - val workspace3Response = WorkspaceListResponse( - WorkspaceAccessLevels.Write, - Some(true), - Some(true), - WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing2, - Option(Set.empty), - true, - Some(WorkspaceCloudPlatform.Gcp) - ), - Option.empty, - false, - Some(List.empty) - ) - val workspace4Response = WorkspaceListResponse( - WorkspaceAccessLevels.Owner, - Some(true), - Some(true), - WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing3, - Option(Set.empty), - true, - Some(WorkspaceCloudPlatform.Gcp) - ), - Option.empty, - false, - Some(List.empty) - ) - - val dataSource = mock[SlickDataSource] - val mockWorkspaceService = mock[WorkspaceService](RETURNS_SMART_NULLS) - - when(mockWorkspaceService.listWorkspaces(any(), any())) - .thenReturn( - Future.successful(Seq(workspace1Response, workspace2Response, workspace3Response, workspace4Response).toJson) - ) - val mockWorkspaceServiceConstructor: RawlsRequestContext => WorkspaceService = { _ => - mockWorkspaceService - } - - when(billingRepository.getBillingProject(mockitoEq(projectName1))) - .thenReturn(Future.successful(Option.apply(billingProject1))) - when(billingRepository.getBillingProject(mockitoEq(projectName2))) - .thenReturn(Future.successful(Option.apply(billingProject2))) - when(billingRepository.getBillingProject(mockitoEq(projectName3))) - .thenReturn(Future.successful(Option.apply(billingProject3))) - - when(samDAO.userHasAction(any(), any(), any(), any())).thenReturn(Future.successful(true)) - - val bigQueryService = mockBigQuery(List[Map[String, String]]()) - - val service = spy( - new SpendReportingService( - testContext, - mock[SlickDataSource], - bigQueryService, - billingRepository, - bpmDAO, - samDAO, - spendReportingServiceConfig, - mockWorkspaceServiceConstructor - ) - ) - doReturn(Future.successful(Seq(billingProject1SpendExport, billingProject3SpendExport))) - .when(service) - .getSpendExportConfigurations( - any() - ) - - val spendReport = - TestData.BpmSpendReport.spendData(from, to, currency, Map("Compute" -> price1, "Storage" -> price2)) - - val billingProfileIdCapture: ArgumentCaptor[UUID] = ArgumentCaptor.forClass(classOf[UUID]) - val startDateCapture: ArgumentCaptor[Date] = ArgumentCaptor.forClass(classOf[Date]) - val endDateCapture: ArgumentCaptor[Date] = ArgumentCaptor.forClass(classOf[Date]) - - val result = Await.result( - service.getSpendForAllWorkspaces(from, to), - Duration.Inf - ) - - result.spendSummary.credits shouldBe "0" - result.spendSummary.cost shouldBe Seq(price1, price2).sum.toString() - result.spendSummary.currency shouldBe "USD" - result.spendSummary.startTime.get.toString(ISODateTimeFormat.date()) shouldBe from.toString( - ISODateTimeFormat.date() - ) - result.spendSummary.endTime.get.toString(ISODateTimeFormat.date()) shouldBe to.toString(ISODateTimeFormat.date()) - - startDateCapture.getValue shouldBe from.toDate - endDateCapture.getValue shouldBe to.toDate - } +// "getBillingWithSpendPermission" should "return spendConfigurations for workspaces" in { +// val dataSource = mock[SlickDataSource] +// +// val billingProject1SpendExport = +// BillingProjectSpendExport(RawlsBillingProjectName("billingProject1"), +// RawlsBillingAccountName("billingAccount1"), +// Some("billing1_bq_project.billing1_dataset.billing1_table") +// ) +// +// val billingProject2SpendExport = +// BillingProjectSpendExport(RawlsBillingProjectName("billingProject2"), +// RawlsBillingAccountName("billingAccount2"), +// Some("billing2_bq_project.billing2_dataset.billing2_table") +// ) +// +// val billingProject3SpendExport = +// BillingProjectSpendExport(RawlsBillingProjectName("billingProject3"), +// RawlsBillingAccountName("billingAccount3"), +// None +// ) +// +// val service = spy( +// new SpendReportingService( +// testContext, +// dataSource, +// Resource.pure[IO, GoogleBigQueryService[IO]](mock[GoogleBigQueryService[IO]]), +// mock[BillingRepository], +// mock[BillingProfileManagerDAO], +// mock[SamDAO], +// spendReportingServiceConfig, +// mockWorkspaceServiceConstructor +// ) +// ) +// +// doReturn(Future.successful(Seq(billingProject1SpendExport, billingProject2SpendExport, billingProject3SpendExport))) +// .when(service) +// .getSpendExportConfigurations( +// any() +// ) +// +// val workspace1Billing1 = +// TestData.workspace("workspace1Billing1", +// GoogleProjectId("workspace1ProjectId"), +// WorkspaceVersions.V1, +// "billingProject1" +// ) +// val workspace2Billing1 = +// TestData.workspace("workspace2Billing1", +// GoogleProjectId("workspace2ProjectId"), +// WorkspaceVersions.V2, +// "billingProject1" +// ) +// val workspace1Billing2 = +// TestData.workspace("workspace1Billing2", +// GoogleProjectId("workspace3ProjectId"), +// WorkspaceVersions.V2, +// "billingProject2" +// ) +// val workspace1Billing3 = +// TestData.workspace("workspace1Billing3", +// GoogleProjectId("workspace4ProjectId"), +// WorkspaceVersions.V2, +// "billingProject3" +// ) +// +// val workspace1Response = WorkspaceListResponse( +// WorkspaceAccessLevels.Read, +// Some(true), +// Some(true), +// WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing1, +// Option(Set.empty), +// true, +// Some(WorkspaceCloudPlatform.Gcp) +// ), +// Option.empty, +// false, +// Some(List.empty) +// ) +// val workspace2Response = WorkspaceListResponse( +// WorkspaceAccessLevels.Read, +// Some(true), +// Some(true), +// WorkspaceDetails.fromWorkspaceAndOptions(workspace2Billing1, +// Option(Set.empty), +// true, +// Some(WorkspaceCloudPlatform.Gcp) +// ), +// Option.empty, +// false, +// Some(List.empty) +// ) +// val workspace3Response = WorkspaceListResponse( +// WorkspaceAccessLevels.Read, +// Some(true), +// Some(true), +// WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing2, +// Option(Set.empty), +// true, +// Some(WorkspaceCloudPlatform.Gcp) +// ), +// Option.empty, +// false, +// Some(List.empty) +// ) +// val workspace4Response = WorkspaceListResponse( +// WorkspaceAccessLevels.Read, +// Some(true), +// Some(true), +// WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing3, +// Option(Set.empty), +// true, +// Some(WorkspaceCloudPlatform.Gcp) +// ), +// Option.empty, +// false, +// Some(List.empty) +// ) +// +// val result = Await.result( +// service.getBillingWithSpendPermission( +// ), +// Duration.Inf +// ) +// +// result shouldBe Map( +// billingProject1SpendExport -> Seq(GoogleProjectId("workspace2ProjectId"), GoogleProjectId("workspace1ProjectId")), +// billingProject2SpendExport -> Seq(GoogleProjectId("workspace3ProjectId")), +// billingProject3SpendExport -> Seq(GoogleProjectId("workspace4ProjectId")) +// ) +// +// } + +// "extractSpendReportingResultsAcrossBillingProjects" should "should return correct summary data" in { +// val storageCostWs1 = 100.582 +// val otherCostWs1 = 0.10111 +// val storageCostRoundedWs1: BigDecimal = BigDecimal(storageCostWs1).setScale(2, RoundingMode.HALF_EVEN) +// val otherCostRoundedWs1: BigDecimal = BigDecimal(otherCostWs1).setScale(2, RoundingMode.HALF_EVEN) +// val totalCostRoundedWs1: BigDecimal = BigDecimal(storageCostWs1 + otherCostWs1).setScale(2, RoundingMode.HALF_EVEN) +// +// val storageCostWs2 = 20.145 +// val computeCostWs2 = 150.4033 +// val storageCostRoundedWs2: BigDecimal = BigDecimal(storageCostWs2).setScale(2, RoundingMode.HALF_EVEN) +// val otherCostRoundedWs2: BigDecimal = BigDecimal(computeCostWs2).setScale(2, RoundingMode.HALF_EVEN) +// val totalCostRoundedWs2: BigDecimal = +// BigDecimal(storageCostWs2 + computeCostWs2).setScale(2, RoundingMode.HALF_EVEN) +// +// val computeCostWs3 = 1111.222 +// val otherCostWs3 = 0.02 +// val storageCostRoundedWs3: BigDecimal = BigDecimal(computeCostWs3).setScale(2, RoundingMode.HALF_EVEN) +// val otherCostRoundedWs3: BigDecimal = BigDecimal(otherCostWs3).setScale(2, RoundingMode.HALF_EVEN) +// val totalCostRoundedWs3: BigDecimal = BigDecimal(computeCostWs3 + otherCostWs3).setScale(2, RoundingMode.HALF_EVEN) +// +// val table: List[Map[String, String]] = List( +// Map( +// "storage_cost" -> s"$storageCostWs1", +// "compute_cost" -> "0.0", +// "other_cost" -> s"$otherCostWs1", +// "googleProjectId" -> "workspace1ProjectId" +// ), +// Map( +// "storage_cost" -> s"$storageCostWs2", +// "compute_cost" -> s"$computeCostWs2", +// "other_cost" -> "0.0", +// "googleProjectId" -> "workspace2ProjectId" +// ), +// Map( +// "storage_cost" -> "0.0", +// "compute_cost" -> s"$computeCostWs3", +// "other_cost" -> s"$otherCostWs3", +// "googleProjectId" -> "workspace3ProjectId" +// ) +// ) +// +// val tableResult: TableResult = createTableResult(table) +// +// val reportingResults = SpendReportingService.extractCrossBillingProjectSpendReportingResults( +// tableResult.getValues.asScala.toList, +// DateTime.now().minusDays(1), +// DateTime.now(), +// Map() +// ) +// reportingResults.spendSummary.cost shouldBe TestData.Workspace.totalCostRounded.toString +// reportingResults.spendDetails shouldBe empty +// } + +// "getSpendForAllWorkspaces" should "get the spend report from multiple billing projects" in { +// val from = DateTime.now().minusMonths(2) +// val to = from.plusMonths(1) +// +// val price1 = BigDecimal("10.22") +// val price2 = BigDecimal("50.74") +// val currency = "USD" +// +// val samDAO = mock[SamDAO](RETURNS_SMART_NULLS) +// val billingRepository = mock[BillingRepository](RETURNS_SMART_NULLS) +// val bpmDAO = mock[BillingProfileManagerDAO](RETURNS_SMART_NULLS) +// +// // Billing projects +// val billingProfileId1 = UUID.randomUUID() +// val projectName1 = RawlsBillingProjectName("billingProject1") +// val billingAccount1 = RawlsBillingAccountName("billingAcct1") +// val billingProject1 = RawlsBillingProject( +// projectName1, +// CreationStatuses.Ready, +// Option(billingAccount1), +// None, +// billingProfileId = Option.apply(billingProfileId1.toString) +// ) +// val billingProfileId2 = UUID.randomUUID() +// val projectName2 = RawlsBillingProjectName("billingProject2") +// val billingAccount2 = RawlsBillingAccountName("billingAcct2") +// val billingProject2 = RawlsBillingProject( +// projectName2, +// CreationStatuses.Ready, +// Option(billingAccount2), +// None, +// billingProfileId = Option.apply(billingProfileId2.toString) +// ) +// val billingProfileId3 = UUID.randomUUID() +// val projectName3 = RawlsBillingProjectName("billingProject3") +// val billingAccount3 = RawlsBillingAccountName("billingAcct3") +// val billingProject3 = RawlsBillingProject( +// projectName3, +// CreationStatuses.Ready, +// Option(billingAccount3), +// None, +// billingProfileId = Option.apply(billingProfileId3.toString) +// ) +// +// // Billing project spend exports +// val billingProject1SpendExport = +// BillingProjectSpendExport(RawlsBillingProjectName("billingProject1"), +// RawlsBillingAccountName("billingAccount1"), +// Some("billing1_bq_project.billing1_dataset.billing1_table") +// ) +// +// val billingProject3SpendExport = +// BillingProjectSpendExport(RawlsBillingProjectName("billingProject3"), +// RawlsBillingAccountName("billingAccount3"), +// None +// ) +// +// // Workspaces +// val workspace1Billing1 = +// TestData.workspace("workspace1Billing1", +// GoogleProjectId("workspace1ProjectId"), +// WorkspaceVersions.V1, +// "billingProject1" +// ) +// val workspace2Billing1 = +// TestData.workspace("workspace2Billing1", +// GoogleProjectId("workspace2ProjectId"), +// WorkspaceVersions.V2, +// "billingProject1" +// ) +// val workspace1Billing2 = +// TestData.workspace("workspace1Billing2", +// GoogleProjectId("workspace3ProjectId"), +// WorkspaceVersions.V2, +// "billingProject2" +// ) +// val workspace1Billing3 = +// TestData.workspace("workspace1Billing3", +// GoogleProjectId("workspace4ProjectId"), +// WorkspaceVersions.V2, +// "billingProject3" +// ) +// +// // Only workspaces 1 and 4 are owned +// val workspace1Response = WorkspaceListResponse( +// WorkspaceAccessLevels.Owner, +// Some(true), +// Some(true), +// WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing1, +// Option(Set.empty), +// true, +// Some(WorkspaceCloudPlatform.Gcp) +// ), +// Option.empty, +// false, +// Some(List.empty) +// ) +// val workspace2Response = WorkspaceListResponse( +// WorkspaceAccessLevels.Read, +// Some(true), +// Some(true), +// WorkspaceDetails.fromWorkspaceAndOptions(workspace2Billing1, +// Option(Set.empty), +// true, +// Some(WorkspaceCloudPlatform.Gcp) +// ), +// Option.empty, +// false, +// Some(List.empty) +// ) +// val workspace3Response = WorkspaceListResponse( +// WorkspaceAccessLevels.Write, +// Some(true), +// Some(true), +// WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing2, +// Option(Set.empty), +// true, +// Some(WorkspaceCloudPlatform.Gcp) +// ), +// Option.empty, +// false, +// Some(List.empty) +// ) +// val workspace4Response = WorkspaceListResponse( +// WorkspaceAccessLevels.Owner, +// Some(true), +// Some(true), +// WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing3, +// Option(Set.empty), +// true, +// Some(WorkspaceCloudPlatform.Gcp) +// ), +// Option.empty, +// false, +// Some(List.empty) +// ) +// +// val dataSource = mock[SlickDataSource] +// val mockWorkspaceService = mock[WorkspaceService](RETURNS_SMART_NULLS) +// +// when(mockWorkspaceService.listWorkspaces(any(), any())) +// .thenReturn( +// Future.successful(Seq(workspace1Response, workspace2Response, workspace3Response, workspace4Response).toJson) +// ) +// val mockWorkspaceServiceConstructor: RawlsRequestContext => WorkspaceService = { _ => +// mockWorkspaceService +// } +// +// when(billingRepository.getBillingProject(mockitoEq(projectName1))) +// .thenReturn(Future.successful(Option.apply(billingProject1))) +// when(billingRepository.getBillingProject(mockitoEq(projectName2))) +// .thenReturn(Future.successful(Option.apply(billingProject2))) +// when(billingRepository.getBillingProject(mockitoEq(projectName3))) +// .thenReturn(Future.successful(Option.apply(billingProject3))) +// +// when(samDAO.userHasAction(any(), any(), any(), any())).thenReturn(Future.successful(true)) +// +// val bigQueryService = mockBigQuery(List[Map[String, String]]()) +// +// val service = spy( +// new SpendReportingService( +// testContext, +// mock[SlickDataSource], +// bigQueryService, +// billingRepository, +// bpmDAO, +// samDAO, +// spendReportingServiceConfig, +// mockWorkspaceServiceConstructor +// ) +// ) +// doReturn(Future.successful(Seq(billingProject1SpendExport, billingProject3SpendExport))) +// .when(service) +// .getSpendExportConfigurations( +// any() +// ) +// +// val spendReport = +// TestData.BpmSpendReport.spendData(from, to, currency, Map("Compute" -> price1, "Storage" -> price2)) +// +// val billingProfileIdCapture: ArgumentCaptor[UUID] = ArgumentCaptor.forClass(classOf[UUID]) +// val startDateCapture: ArgumentCaptor[Date] = ArgumentCaptor.forClass(classOf[Date]) +// val endDateCapture: ArgumentCaptor[Date] = ArgumentCaptor.forClass(classOf[Date]) +// +// val result = Await.result( +// service.getSpendForAllWorkspaces(from, to), +// Duration.Inf +// ) +// +// result.spendSummary.credits shouldBe "0" +// result.spendSummary.cost shouldBe Seq(price1, price2).sum.toString() +// result.spendSummary.currency shouldBe "USD" +// result.spendSummary.startTime.get.toString(ISODateTimeFormat.date()) shouldBe from.toString( +// ISODateTimeFormat.date() +// ) +// result.spendSummary.endTime.get.toString(ISODateTimeFormat.date()) shouldBe to.toString(ISODateTimeFormat.date()) +// +// startDateCapture.getValue shouldBe from.toDate +// endDateCapture.getValue shouldBe to.toDate +// } } From 0c6915b5870ea8ab125e659834ca2565f80eb208 Mon Sep 17 00:00:00 2001 From: Bria Morgan Date: Tue, 19 Nov 2024 13:16:26 -0500 Subject: [PATCH 12/31] update how to extract spend report results --- .../SpendReportingService.scala | 175 +++++++----------- 1 file changed, 64 insertions(+), 111 deletions(-) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala index 91f4901a8c..ea6eb3f6aa 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala @@ -144,126 +144,79 @@ object SpendReportingService { SpendReportingResults(aggregations.map(aggregateRows(allRows, _)).toList, summary) } -// def extractCrossBillingProjectSpendReportingResults( -// allRows: List[FieldValueList], -// start: DateTime, -// end: DateTime, -// names: Map[GoogleProjectId, WorkspaceName] -// ): List[SpendReportingResults] = -// allRows -// .map(row => -// extractSpendReportingResults(row, start, end, names, Set.of(SpendReportingAggregationKeyWithSub(Category))) -// ) -// .collect() - def extractCrossBillingProjectSpendReportingResults( allRows: List[FieldValueList], start: DateTime, end: DateTime, names: Map[GoogleProjectId, WorkspaceName] - ): Map[String, SpendReportingResults] = { + ): SpendReportingResults = { - val schema = FieldList.of( - Field.of("cost", StandardSQLTypeName.FLOAT64), - Field.of("credits", StandardSQLTypeName.STRING), - Field.of("currency", StandardSQLTypeName.STRING), - Field.of("service", StandardSQLTypeName.STRING) - ) - val groupedRows = allRows.groupBy(row => row.get("project_id").getStringValue) - - groupedRows.map { case (projectId, rows) => - val transformedRows = rows.flatMap { row => - val currency = row.get("currency").getStringValue - - List( - FieldValueList.of( - java.util.List.of( - FieldValue.of(FieldValue.Attribute.PRIMITIVE, row.get("compute_cost").getDoubleValue.toString), - FieldValue.of(FieldValue.Attribute.PRIMITIVE, "0"), - FieldValue.of(FieldValue.Attribute.PRIMITIVE, currency), - FieldValue.of(FieldValue.Attribute.PRIMITIVE, "compute") - ), - schema - ), - FieldValueList.of( - java.util.List.of( - FieldValue.of(FieldValue.Attribute.PRIMITIVE, row.get("storage_cost").getDoubleValue.toString), - FieldValue.of(FieldValue.Attribute.PRIMITIVE, "0"), - FieldValue.of(FieldValue.Attribute.PRIMITIVE, currency), - FieldValue.of(FieldValue.Attribute.PRIMITIVE, "storage") - ), - schema - ), - FieldValueList.of( - java.util.List.of( - FieldValue.of(FieldValue.Attribute.PRIMITIVE, row.get("other_cost").getDoubleValue.toString), - FieldValue.of(FieldValue.Attribute.PRIMITIVE, "0"), - FieldValue.of(FieldValue.Attribute.PRIMITIVE, currency), - FieldValue.of(FieldValue.Attribute.PRIMITIVE, "other") - ), - schema - ) + var total = BigDecimal(0.0) + val all = allRows.map { row => + val currencyString = row.get("currency").getStringValue + val currencyCode = Currency.getInstance(currencyString) + val projectId = row.get("project_id").getStringValue + val workspaceName = names.getOrElse( + GoogleProjectId(projectId), + throw RawlsExceptionWithErrorReport( + StatusCodes.InternalServerError, + s"unexpected project $projectId returned by BigQuery" ) - } + ) + + def getRoundedNumericValue(field: String): BigDecimal = + BigDecimal(row.get(field).getDoubleValue) + .setScale(currencyCode.getDefaultFractionDigits, RoundingMode.HALF_EVEN) + + val subAggregation = List( + SpendReportingForDateRange( + getRoundedNumericValue("other_cost").toString, + "0.0", + currencyCode.toString, + Option(start), + Option(end), + category = Option(TerraSpendCategories.Other) + ), + SpendReportingForDateRange( + getRoundedNumericValue("storage_cost").toString, + "0.0", + currencyCode.toString, + category = Option(TerraSpendCategories.Storage) + ), + SpendReportingForDateRange( + getRoundedNumericValue("compute_cost").toString, + "0.0", + currencyCode.toString, + category = Option(TerraSpendCategories.Compute) + ) + ) - projectId -> extractSpendReportingResults( - transformedRows, - start, - end, - names, - Set(SpendReportingAggregationKeyWithSub(SpendReportingAggregationKeys.Category)) + val total_cost = getRoundedNumericValue("total_cost") + total = total + total_cost + + val workspaceTotal = SpendReportingForDateRange( + total_cost.toString, + "0.0", + currencyCode.toString, + Option(start), + Option(end), + workspace = Option(workspaceName), + googleProjectId = Option(GoogleProject(projectId)), + subAggregation = Option(SpendReportingAggregation(SpendReportingAggregationKeys.Category, subAggregation)) ) + + SpendReportingAggregation(SpendReportingAggregationKeys.Workspace, List(workspaceTotal)) + } -// val spendDetails = allRows.flatMap { row => -// val projectId = GoogleProjectId(row.get("project_id").getStringValue) -// val workspaceName = names.getOrElse( -// projectId, -// throw RawlsExceptionWithErrorReport( -// StatusCodes.InternalServerError, -// s"unexpected project $projectId returned by BigQuery" -// ) -// ) -//// val totalCost = row.get("total_cost").getDoubleValue.toString -//// val computeCost = row.get("compute_cost").getDoubleValue.toString -//// val storageCost = row.get("storage_cost").getDoubleValue.toString -// val currency = row.get("currency").getStringValue -// -// // Each row gets a summary of compute, storage, and total -// List( -// SpendReportingForDateRange( -// totalCost, -// "0", // Ignoring credits for now; do we want to include them? -// currency, -// Option(start), -// Option(end), -// workspace = Some(workspaceName), -// googleProjectId = Some(GoogleProject(projectId.value)) -// ), -// SpendReportingForDateRange( -// computeCost, -// "0", -// currency, -// Option(start), -// Option(end), -// workspace = Some(workspaceName), -// googleProjectId = Some(GoogleProject(projectId.value)), -// category = Some(TerraSpendCategories.Compute) -// ), -// SpendReportingForDateRange( -// storageCost, -// "0", -// currency, -// Option(start), -// Option(end), -// workspace = Some(workspaceName), -// googleProjectId = Some(GoogleProject(projectId.value)), -// category = Some(TerraSpendCategories.Storage) -// ) -// ) -// } -// -// // TODO what format should the result be? Do we need a new model? -// spendDetails.map(details => SpendReportingResults(List.empty, details)) + + val summary = SpendReportingForDateRange( + total.toString, + "0.0", // TODO + "USD", // TODO + Option(start), + Option(end) + ) + SpendReportingResults(all, summary) } } @@ -587,7 +540,7 @@ class SpendReportingService( def getSpendForAllWorkspaces( start: DateTime, end: DateTime - ): Future[Map[String, SpendReportingResults]] = { + ): Future[SpendReportingResults] = { validateReportParameters(start, end) for { // TODO get projectNames from billingMap From 9a8694bb6b6482b9ebe5f072642d2954b4742b4b Mon Sep 17 00:00:00 2001 From: Bria Morgan Date: Tue, 19 Nov 2024 14:15:02 -0500 Subject: [PATCH 13/31] add test skeleton --- .../SpendReportingServiceSpec.scala | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala index 903929f8cc..828d6444ba 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala @@ -551,6 +551,61 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki e.errorReport.statusCode shouldBe Option(StatusCodes.InternalServerError) } + "extractCrossBillingProjectSpendReportingResults" should "break down results by workspace and category" in { + val computeCost1 = 2.4 + val computeCost2 = 0.10111 + val storageCost1 = 0.33 + val storageCost2 = 0.3561 + val otherCost1 = 3.0 + val otherCost2 = 0.0001 + val computeCost1Rounded: BigDecimal = BigDecimal(computeCost1).setScale(2, RoundingMode.HALF_EVEN) + val computeCost2Rounded: BigDecimal = BigDecimal(computeCost2).setScale(2, RoundingMode.HALF_EVEN) + val storageCost1Rounded: BigDecimal = BigDecimal(storageCost1).setScale(2, RoundingMode.HALF_EVEN) + val storageCost2Rounded: BigDecimal = BigDecimal(storageCost2).setScale(2, RoundingMode.HALF_EVEN) + val otherCost1Rounded: BigDecimal = BigDecimal(otherCost1).setScale(2, RoundingMode.HALF_EVEN) + val otherCost2Rounded: BigDecimal = BigDecimal(otherCost2).setScale(2, RoundingMode.HALF_EVEN) + val totalCost1 = computeCost1 + storageCost1 + otherCost1 + val totalCost2 = computeCost2 + storageCost2 + otherCost2 + val totalCostRounded: BigDecimal = BigDecimal(totalCost1 + totalCost2).setScale(2, RoundingMode.HALF_EVEN) + + val table: List[Map[String, String]] = List( + Map( + "storage_cost" -> s"$storageCost1", + "compute_cost" -> s"$computeCost1", + "other_cost" -> s"$otherCost1", + "total_cost" -> s"$totalCost1", + "currency" -> "USD", + "project_id" -> "terra-workspace-project1", + "project_name" -> "terra-billing-project1" + ), + Map( + "storage_cost" -> s"$storageCost2", + "compute_cost" -> s"$computeCost2", + "other_cost" -> s"$otherCost2", + "total_cost" -> s"$totalCost2", + "currency" -> "USD", + "project_id" -> "terra-workspace-project2", + "project_name" -> "terra-billing-project2" + ) + ) + + val tableResult: TableResult = createTableResult(table) + + val reportingResults = SpendReportingService.extractCrossBillingProjectSpendReportingResults( + tableResult.getValues.asScala.toList, + DateTime.now().minusDays(1), + DateTime.now(), + Map( + GoogleProjectId("terra-workspace-project1") -> WorkspaceName("terra-billing-project1", "workspace1"), + GoogleProjectId("terra-workspace-project2") -> WorkspaceName("terra-billing-project2", "workspace2") + ) + ) + reportingResults.spendSummary.cost shouldBe totalCostRounded.toString + reportingResults.spendDetails.head.aggregationKey shouldBe SpendReportingAggregationKeys.Workspace + reportingResults.spendDetails.length shouldBe 2 + + } + "getSpendForGCPBillingProject" should "throw an exception when BQ returns zero rows" in { val samDAO = mock[SamDAO] val billingRepository = mock[BillingRepository] From 554256bc0c69adb84de910db6ae631da09e19060 Mon Sep 17 00:00:00 2001 From: Bria Morgan Date: Tue, 19 Nov 2024 15:36:07 -0500 Subject: [PATCH 14/31] get workspaces/bps from sam instead --- .../dsde/rawls/dataaccess/HttpSamDAO.scala | 48 ++++++++++++++++ .../dsde/rawls/dataaccess/SamDAO.scala | 5 ++ .../dataaccess/slick/WorkspaceComponent.scala | 6 ++ .../SpendReportingService.scala | 53 +++++++++--------- .../rawls/workspace/WorkspaceRepository.scala | 7 +++ .../rawls/workspace/WorkspaceService.scala | 8 +++ .../dsde/rawls/mock/MockSamDAO.scala | 55 +++++++++++++++++++ .../SpendReportingServiceSpec.scala | 13 ++++- 8 files changed, 166 insertions(+), 29 deletions(-) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpSamDAO.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpSamDAO.scala index 071fbe1c58..d41087066c 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpSamDAO.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpSamDAO.scala @@ -14,12 +14,17 @@ import org.broadinstitute.dsde.rawls.model._ import org.broadinstitute.dsde.rawls.util.{FutureSupport, Retry} import org.broadinstitute.dsde.workbench.client.sam import org.broadinstitute.dsde.workbench.client.sam.api._ +import org.broadinstitute.dsde.workbench.client.sam.model.{ + FilteredHierarchicalResourcePolicy, + ListResourcesV2200Response +} import org.broadinstitute.dsde.workbench.client.sam.{ApiCallback, ApiClient, ApiException} import org.broadinstitute.dsde.workbench.model.{WorkbenchEmail, WorkbenchGroupName} import java.time.Instant import java.time.temporal.ChronoUnit import java.util +import java.util.List import scala.concurrent.duration.FiniteDuration import scala.concurrent.{ExecutionContext, Future, Promise} import scala.jdk.CollectionConverters._ @@ -513,12 +518,55 @@ class HttpSamDAO(baseSamServiceURL: String, rawlsCredential: RawlsCredential, ti } } + override def listResourcesWithActions(resourceTypeName: SamResourceTypeName, + action: SamResourceAction, + ctx: RawlsRequestContext + ): Future[Seq[SamUserResource]] = + retry(when401or5xx) { () => + val callback = new SamApiCallback[ListResourcesV2200Response]("listResourcesV2") + + resourcesApi(ctx).listResourcesV2Async( + /* format = */ "hierarchical", // Todo what to pass in here + /* resourceTypes = */ util.List.of(resourceTypeName.value), + /* policies = */ util.List.of(), + /* roles = */ util.List.of, + /* actions = */ util.List.of(action.value), + /* includePublic = */ false, + callback + ) + + callback.future.map { resourcesResponse => + println(resourcesResponse) + resourcesResponse.getFilteredResourcesHierarchicalResponse + .getResources() + .asScala + .map { resource => + SamUserResource( + resource.getResourceId, + toSamRolesAndActions(resource.getPolicies()), // TODO what to use here? + toSamRolesAndActions(resource.getPolicies()), // What are all these three things? + toSamRolesAndActions(resource.getPolicies()), + resource.getAuthDomainGroups.asScala.map(WorkbenchGroupName).toSet, + resource.getMissingAuthDomainGroups.asScala.map(WorkbenchGroupName).toSet + ) + } + .toSeq + } + } + private def toSamRolesAndActions(rolesAndActions: sam.model.RolesAndActions) = SamRolesAndActions( rolesAndActions.getRoles.asScala.map(SamResourceRole).toSet, rolesAndActions.getActions.asScala.map(SamResourceAction).toSet ) + private def toSamRolesAndActions(policies: util.List[FilteredHierarchicalResourcePolicy]) = { + val scalaPolicies = policies.asScala.toList + val roles = scalaPolicies.flatMap(_.getRoles.asScala) + val actions = scalaPolicies.flatMap(_.getActions.asScala) + SamRolesAndActions(roles.map(role => SamResourceRole(role.toString)).toSet, actions.map(SamResourceAction).toSet) + } + override def getPetServiceAccountKeyForUser(googleProject: GoogleProjectId, userEmail: RawlsUserEmail ): Future[String] = diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/SamDAO.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/SamDAO.scala index cc95dd32dc..dd19bb20c7 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/SamDAO.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/SamDAO.scala @@ -100,6 +100,11 @@ trait SamDAO { def listUserResources(resourceTypeName: SamResourceTypeName, ctx: RawlsRequestContext): Future[Seq[SamUserResource]] + def listResourcesWithActions(resourceTypeName: SamResourceTypeName, + action: SamResourceAction, + ctx: RawlsRequestContext + ): Future[Seq[SamUserResource]] + def listPoliciesForResource(resourceTypeName: SamResourceTypeName, resourceId: String, ctx: RawlsRequestContext diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceComponent.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceComponent.scala index 31404e49aa..0f4dccbf05 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceComponent.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceComponent.scala @@ -260,6 +260,9 @@ trait WorkspaceComponent { def listWithBillingProject(billingProject: RawlsBillingProjectName): ReadAction[Seq[Workspace]] = workspaceQuery.withBillingProject(billingProject).read + def listWithBillingProjects(billingProjects: List[RawlsBillingProjectName]): ReadAction[Seq[Workspace]] = + workspaceQuery.withBillingProjects(billingProjects).read + def getTags(queryString: Option[String], limit: Option[Int] = None, ownerIds: Option[Seq[UUID]] = None @@ -644,6 +647,9 @@ trait WorkspaceComponent { def withBillingProject(projectName: RawlsBillingProjectName): WorkspaceQueryType = query.filter(_.namespace === projectName.value) + def withBillingProjects(projectNames: List[RawlsBillingProjectName]): WorkspaceQueryType = + query.filter(_.namespace.inSetBind(projectNames.map(_.value))) + def withGoogleProjectId(googleProjectId: GoogleProjectId): WorkspaceQueryType = query.filter(_.googleProjectId === googleProjectId.value) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala index ea6eb3f6aa..2e4c189a54 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala @@ -6,7 +6,6 @@ import cats.effect.IO import cats.effect.unsafe.implicits.global import com.google.cloud.bigquery.{JobStatistics, Option => _, _} import com.google.cloud.bigquery.{Field, FieldList, FieldValue, FieldValueList} - import com.typesafe.scalalogging.LazyLogging import nl.grons.metrics4.scala.{Counter, Histogram} import org.broadinstitute.dsde.rawls.billing.{ @@ -32,6 +31,7 @@ import scala.concurrent.{ExecutionContext, Future} import scala.jdk.CollectionConverters._ import scala.math.BigDecimal.RoundingMode import org.broadinstitute.dsde.rawls.model.WorkspaceAccessLevels.{Owner, WorkspaceAccessLevel} +import shapeless.syntax.std.tuple.productTupleOps import scala.util.Try @@ -344,7 +344,9 @@ class SpendReportingService( if (isBroadTable) spendReportingServiceConfig.defaultTimePartitionColumn else "_PARTITIONTIME" } - def getAllUserWorkspaceQuery(billingProjects: Map[BillingProjectSpendExport, Seq[GoogleProjectId]]): String = { + def getAllUserWorkspaceQuery( + billingProjects: Map[BillingProjectSpendExport, Seq[(GoogleProjectId, WorkspaceName)]] + ): String = { val baseQuery = s""" | SELECT | project.id AS project_id, @@ -374,7 +376,7 @@ class SpendReportingService( baseQuery .replace("_PARTITIONTIME", timePartitionColumn) .replace("_BILLING_ACCOUNT_TABLE", tableName) - .replace("_PROJECT_ID_LIST", "(" + bp._2.map(id => s""""${id.value}"""").mkString(", ") + ")") + .replace("_PROJECT_ID_LIST", "(" + bp._2.map(tuple => s""""${tuple._1.value}"""").mkString(", ") + ")") } .mkString("\nUNION ALL\n") @@ -543,17 +545,10 @@ class SpendReportingService( ): Future[SpendReportingResults] = { validateReportParameters(start, end) for { - // TODO get projectNames from billingMap - workspaces <- getOwnerWorkspaces() - projectNames = workspaces - .map(wsResp => - wsResp.workspace.googleProject -> WorkspaceName(wsResp.workspace.namespace, wsResp.workspace.name) - ) - .toMap // Map[GoogleProjectId, WorkspaceName] - billing <- getBillingSpendExportsForWorkspaces(workspaces) -// billingMap <- getBillingWithSpendPermission() + billingMap <- getBillingWithSpendPermission() + projectNames: Map[GoogleProjectId, WorkspaceName] = billingMap.values.flatten.toMap - query = getAllUserWorkspaceQuery(billing) + query = getAllUserWorkspaceQuery(billingMap) queryJob = setUpAllUserWorkspaceQuery(query, start, end) job: Job <- bigQueryService.use(_.runJob(queryJob)).unsafeToFuture().map(_.waitFor()) @@ -593,17 +588,23 @@ class SpendReportingService( } } -// def getBillingWithSpendPermission( -// ): Future[Map[BillingProjectSpendExport, Seq[GoogleProjectId]]] = -// for { -// billingProjectResources <- samDAO.listResourcesWithActions(SamResourceTypeNames.billingProject, -// SamBillingProjectActions.readSpendReport, -// ctx -// ) -// billingProjectIds = billingProjectResources.map(resource => RawlsBillingProjectName(resource.resourceId)).toList -// groupedWorkspaces <- workspaceServiceConstructor(ctx).getWorkspacesByBillingProjects(billingProjectIds) -// spendConfigs <- getSpendExportConfigurations(billingProjectIds) -// } yield spendConfigs.map { config => -// config -> groupedWorkspaces(config.billingProjectName).map(ws => ws.googleProjectId) -// }.toMap + def getBillingWithSpendPermission( + ): Future[Map[BillingProjectSpendExport, Seq[(GoogleProjectId, WorkspaceName)]]] = + for { + billingProjectResources <- samDAO.listResourcesWithActions(SamResourceTypeNames.billingProject, + SamBillingProjectActions.readSpendReport, + ctx + ) + billingProjectIds = billingProjectResources.map(resource => RawlsBillingProjectName(resource.resourceId)).toList + groupedWorkspaces <- workspaceServiceConstructor(ctx).getWorkspacesByBillingProjects(billingProjectIds) + gcpOnlyGroupedWorkspaces = groupedWorkspaces + .map { case (key, workspaces) => + key -> workspaces.filter(_.workspaceType == WorkspaceType.RawlsWorkspace) + } + .filter { case (_, workspaces) => workspaces.nonEmpty } + // Only use the BPs we know exist in the DB and are GCP + spendConfigs <- getSpendExportConfigurations(gcpOnlyGroupedWorkspaces.keys.toList) + } yield spendConfigs.map { config => + config -> gcpOnlyGroupedWorkspaces(config.billingProjectName).map(ws => (ws.googleProjectId, ws.toWorkspaceName)) + }.toMap } diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala index f799cb1a76..2fab668572 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala @@ -69,6 +69,13 @@ class WorkspaceRepository(dataSource: SlickDataSource) { _.workspaceQuery.listWithBillingProject(billingProjectName) } + def listWorkspacesByMultipleBillingProjects( + billingProjectNames: List[RawlsBillingProjectName] + ): Future[Seq[Workspace]] = + dataSource.inTransaction { + _.workspaceQuery.listWithBillingProjects(billingProjectNames) + } + def createWorkspace(workspace: Workspace): Future[Workspace] = dataSource.inTransaction { access => access.workspaceQuery.createOrUpdate(workspace) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala index c6329024b4..b152a6f46e 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala @@ -433,6 +433,14 @@ class WorkspaceService( } yield deepFilterJsValue(responseWorkspaces.toJson, options.options) } + // TODO have the DB query do the grouping + def getWorkspacesByBillingProjects( + billingProjects: List[RawlsBillingProjectName] + ): Future[Map[RawlsBillingProjectName, Seq[Workspace]]] = + for { + workspaces <- workspaceRepository.listWorkspacesByMultipleBillingProjects(billingProjects) + } yield workspaces.groupBy(ws => RawlsBillingProjectName(ws.namespace)) + /** Returns the Set of legal field names supplied by the user, trimmed of whitespace. * Throws an error if the user supplied an unrecognized field name. * Legal field names are any member of `WorkspaceResponse`, `WorkspaceDetails`, diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/mock/MockSamDAO.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/mock/MockSamDAO.scala index 2b417d9acd..31e6110cbb 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/mock/MockSamDAO.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/mock/MockSamDAO.scala @@ -239,6 +239,46 @@ class MockSamDAO(dataSource: SlickDataSource)(implicit executionContext: Executi case _ => Future.successful(Seq.empty) } + override def listResourcesWithActions(resourceTypeName: SamResourceTypeName, + action: SamResourceAction, + ctx: RawlsRequestContext + ): Future[Seq[SamUserResource]] = + resourceTypeName match { + case SamResourceTypeNames.workspace => + dataSource + .inTransaction(_ => workspaceQuery.listAll()) + .map( + _.map(workspace => + SamUserResource( + workspace.workspaceId, + SamRolesAndActions(Set(SamWorkspaceRoles.owner), Set(action)), + SamRolesAndActions(Set.empty, Set.empty), + SamRolesAndActions(Set.empty, Set.empty), + Set.empty, + Set.empty + ) + ) + ) + + case SamResourceTypeNames.billingProject => + dataSource + .inTransaction(_ => rawlsBillingProjectQuery.read) + .map( + _.map(project => + SamUserResource( + project.projectName.value, + SamRolesAndActions(Set(SamBillingProjectRoles.owner), Set(action)), + SamRolesAndActions(Set.empty, Set.empty), + SamRolesAndActions(Set.empty, Set.empty), + Set.empty, + Set.empty + ) + ) + ) + + case _ => Future.successful(Seq.empty) + } + override def admin: SamAdminDAO = new MockSamAdminDAO() class MockSamAdminDAO extends SamAdminDAO { @@ -386,6 +426,21 @@ class CustomizableMockSamDAO(dataSource: SlickDataSource)(implicit executionCont } } + override def listResourcesWithActions(resourceTypeName: SamResourceTypeName, + action: SamResourceAction, + ctx: RawlsRequestContext + ): Future[Seq[SamUserResource]] = { + val userResources = for { + ((typeName, resourceId), resourcePolicies) <- policies if typeName == resourceTypeName + userResource <- constructResourceFromPolicies(ctx, resourceId, resourcePolicies.values) + } yield userResource + if (userResources.isEmpty) { + super.listResourcesWithActions(resourceTypeName, action, ctx) + } else { + Future.successful(userResources.toSeq) + } + } + override def listUserRolesForResource(resourceTypeName: SamResourceTypeName, resourceId: String, ctx: RawlsRequestContext diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala index 828d6444ba..e478484a84 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala @@ -1375,9 +1375,16 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki ) val inputMap = Map( - billingProject1SpendExport -> Seq(GoogleProjectId("workspace2ProjectId"), GoogleProjectId("workspace1ProjectId")), - billingProject2SpendExport -> Seq(GoogleProjectId("workspace3ProjectId")), - billingProject3SpendExport -> Seq(GoogleProjectId("workspace4ProjectId")) + billingProject1SpendExport -> Seq( + (GoogleProjectId("workspace2ProjectId"), WorkspaceName("billingProject1", "workspace2")), + (GoogleProjectId("workspace1ProjectId"), WorkspaceName("billingProject1", "workspace1")) + ), + billingProject2SpendExport -> Seq( + (GoogleProjectId("workspace3ProjectId"), WorkspaceName("billingProject2", "workspace3")) + ), + billingProject3SpendExport -> Seq( + (GoogleProjectId("workspace4ProjectId"), WorkspaceName("billingProject3", "workspace4")) + ) ) val expectedQuery = From 8f52a841e14a5a47df1b88373f1c6f6c755f9f9f Mon Sep 17 00:00:00 2001 From: Bria Morgan Date: Tue, 19 Nov 2024 16:05:23 -0500 Subject: [PATCH 15/31] some cleanup --- .../dsde/rawls/dataaccess/HttpSamDAO.scala | 3 +- .../rawls/model/SpendReportingModel.scala | 2 - .../SpendReportingService.scala | 29 +- .../SpendReportingServiceSpec.scala | 1179 +++++++++-------- 4 files changed, 596 insertions(+), 617 deletions(-) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpSamDAO.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpSamDAO.scala index d41087066c..3fca278c2e 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpSamDAO.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpSamDAO.scala @@ -526,7 +526,7 @@ class HttpSamDAO(baseSamServiceURL: String, rawlsCredential: RawlsCredential, ti val callback = new SamApiCallback[ListResourcesV2200Response]("listResourcesV2") resourcesApi(ctx).listResourcesV2Async( - /* format = */ "hierarchical", // Todo what to pass in here + /* format = */ "hierarchical", /* resourceTypes = */ util.List.of(resourceTypeName.value), /* policies = */ util.List.of(), /* roles = */ util.List.of, @@ -536,7 +536,6 @@ class HttpSamDAO(baseSamServiceURL: String, rawlsCredential: RawlsCredential, ti ) callback.future.map { resourcesResponse => - println(resourcesResponse) resourcesResponse.getFilteredResourcesHierarchicalResponse .getResources() .asScala diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/model/SpendReportingModel.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/model/SpendReportingModel.scala index f88df652ea..ec64c5b897 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/model/SpendReportingModel.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/model/SpendReportingModel.scala @@ -164,8 +164,6 @@ object TerraSpendCategories { case "cloudstorage" => Storage case "computeengine" => Compute case "kubernetesengine" => Compute - case "storage" => Storage - case "compute" => Compute case _ => Other } diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala index 2e4c189a54..29874a7d59 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala @@ -285,13 +285,14 @@ class SpendReportingService( ) } + // TODO if there is a problem with just one BP, the whole thing fails. def getSpendExportConfigurations(projects: Seq[RawlsBillingProjectName]): Future[Seq[BillingProjectSpendExport]] = dataSource .inTransaction(_.rawlsBillingProjectQuery.getBillingProjectsSpendConfiguration(projects)) - .recover { case _: RawlsException => + .recover { case ex: RawlsException => throw RawlsExceptionWithErrorReport( StatusCodes.BadRequest, - s"billing account not found on billing project" // TODO: identify problem + ex.getMessage ) } .map { exportOptions => @@ -564,30 +565,6 @@ class SpendReportingService( } } - def getOwnerWorkspaces(): Future[Seq[WorkspaceListResponse]] = - workspaceServiceConstructor(ctx).listWorkspaces(WorkspaceFieldSpecs(), -1) map { - case JsArray(jsArray) => - val workspaces = jsArray.map(_.convertTo[WorkspaceListResponse]) - workspaces.filter(_.accessLevel == Owner) - case _ => throw new IllegalArgumentException("Expected a JsArray") - } - - def getBillingSpendExportsForWorkspaces( - workspaces: Seq[WorkspaceListResponse] - ): Future[Map[BillingProjectSpendExport, Seq[GoogleProjectId]]] = { - val groupedWorkspaces = workspaces.groupBy(_.workspace.namespace) - val billingProjects = groupedWorkspaces.keys.map(RawlsBillingProjectName).toList -// billingProjects.map(project => -// requireProjectAction(project, SamBillingProjectActions.readSpendReport) -// ) // TODO a better way to handle this? - - getSpendExportConfigurations(billingProjects).map { exportConfigs => - exportConfigs.map { config => - config -> groupedWorkspaces(config.billingProjectName.value).map(ws => ws.workspace.googleProject) - }.toMap - } - } - def getBillingWithSpendPermission( ): Future[Map[BillingProjectSpendExport, Seq[(GoogleProjectId, WorkspaceName)]]] = for { diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala index e478484a84..a02f1ec216 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala @@ -35,6 +35,7 @@ import org.mockito.{ArgumentCaptor, Mockito} import org.scalatest.flatspec.AnyFlatSpecLike import org.scalatest.matchers.should.Matchers import akka.http.scaladsl.model.headers.OAuth2BearerToken +import org.broadinstitute.dsde.rawls.mock.MockSamDAO import java.util.{Date, UUID} import scala.concurrent.duration.Duration @@ -1150,211 +1151,211 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki result shouldBe Map(GoogleProjectId("v2ProjectId") -> v2Workspace.toWorkspaceName) } - "getOwnerWorkspaces" should "return any and all workspaces user has owner access to" in { - val ownerWorkspace1 = TestData.workspace("owner1", GoogleProjectId("owner1ProjectId"), WorkspaceVersions.V1) - val ownerWorkspace2 = TestData.workspace("owner2", GoogleProjectId("owner2ProjectId"), WorkspaceVersions.V2) - val readerWorkspace = TestData.workspace("reader1", GoogleProjectId("reader1ProjectId"), WorkspaceVersions.V2) - - val dataSource = mock[SlickDataSource] - val mockWorkspaceService = mock[WorkspaceService](RETURNS_SMART_NULLS) - - val workspace1Response = WorkspaceListResponse( - WorkspaceAccessLevels.Owner, - Some(true), - Some(true), - WorkspaceDetails.fromWorkspaceAndOptions(ownerWorkspace1, - Option(Set.empty), - true, - Some(WorkspaceCloudPlatform.Gcp) - ), - Option.empty, - false, - Some(List.empty) - ) - - val workspace2Response = WorkspaceListResponse( - WorkspaceAccessLevels.Owner, - Some(true), - Some(true), - WorkspaceDetails.fromWorkspaceAndOptions(ownerWorkspace2, - Option(Set.empty), - true, - Some(WorkspaceCloudPlatform.Gcp) - ), - Option.empty, - false, - Some(List.empty) - ) - - val workspace3Response = WorkspaceListResponse( - WorkspaceAccessLevels.Read, - Some(true), - Some(true), - WorkspaceDetails.fromWorkspaceAndOptions(readerWorkspace, - Option(Set.empty), - true, - Some(WorkspaceCloudPlatform.Gcp) - ), - Option.empty, - false, - Some(List.empty) - ) - - when(mockWorkspaceService.listWorkspaces(any(), any())) - .thenReturn(Future.successful(Seq(workspace1Response, workspace2Response, workspace3Response).toJson)) - val mockWorkspaceServiceConstructor: RawlsRequestContext => WorkspaceService = { _ => - mockWorkspaceService - } - val service = new SpendReportingService( - testContext, - dataSource, - Resource.pure[IO, GoogleBigQueryService[IO]](mock[GoogleBigQueryService[IO]]), - mock[BillingRepository], - mock[BillingProfileManagerDAO], - mock[SamDAO], - spendReportingServiceConfig, - mockWorkspaceServiceConstructor - ) - - val result = Await.result(service.getOwnerWorkspaces(), Duration.Inf) - - result shouldBe Seq(workspace1Response, workspace2Response) - } - - "getBillingForWorkspaces" should "return spendConfigurations for workspaces" in { +// "getOwnerWorkspaces" should "return any and all workspaces user has owner access to" in { +// val ownerWorkspace1 = TestData.workspace("owner1", GoogleProjectId("owner1ProjectId"), WorkspaceVersions.V1) +// val ownerWorkspace2 = TestData.workspace("owner2", GoogleProjectId("owner2ProjectId"), WorkspaceVersions.V2) +// val readerWorkspace = TestData.workspace("reader1", GoogleProjectId("reader1ProjectId"), WorkspaceVersions.V2) +// +// val dataSource = mock[SlickDataSource] +// val mockWorkspaceService = mock[WorkspaceService](RETURNS_SMART_NULLS) +// +// val workspace1Response = WorkspaceListResponse( +// WorkspaceAccessLevels.Owner, +// Some(true), +// Some(true), +// WorkspaceDetails.fromWorkspaceAndOptions(ownerWorkspace1, +// Option(Set.empty), +// true, +// Some(WorkspaceCloudPlatform.Gcp) +// ), +// Option.empty, +// false, +// Some(List.empty) +// ) +// +// val workspace2Response = WorkspaceListResponse( +// WorkspaceAccessLevels.Owner, +// Some(true), +// Some(true), +// WorkspaceDetails.fromWorkspaceAndOptions(ownerWorkspace2, +// Option(Set.empty), +// true, +// Some(WorkspaceCloudPlatform.Gcp) +// ), +// Option.empty, +// false, +// Some(List.empty) +// ) +// +// val workspace3Response = WorkspaceListResponse( +// WorkspaceAccessLevels.Read, +// Some(true), +// Some(true), +// WorkspaceDetails.fromWorkspaceAndOptions(readerWorkspace, +// Option(Set.empty), +// true, +// Some(WorkspaceCloudPlatform.Gcp) +// ), +// Option.empty, +// false, +// Some(List.empty) +// ) +// +// when(mockWorkspaceService.listWorkspaces(any(), any())) +// .thenReturn(Future.successful(Seq(workspace1Response, workspace2Response, workspace3Response).toJson)) +// val mockWorkspaceServiceConstructor: RawlsRequestContext => WorkspaceService = { _ => +// mockWorkspaceService +// } +// val service = new SpendReportingService( +// testContext, +// dataSource, +// Resource.pure[IO, GoogleBigQueryService[IO]](mock[GoogleBigQueryService[IO]]), +// mock[BillingRepository], +// mock[BillingProfileManagerDAO], +// mock[SamDAO], +// spendReportingServiceConfig, +// mockWorkspaceServiceConstructor +// ) +// +// val result = Await.result(service.getOwnerWorkspaces(), Duration.Inf) +// +// result shouldBe Seq(workspace1Response, workspace2Response) +// } +// +// "getBillingForWorkspaces" should "return spendConfigurations for workspaces" in { +// +// val dataSource = mock[SlickDataSource] +// +// val billingProject1SpendExport = +// BillingProjectSpendExport(RawlsBillingProjectName("billingProject1"), +// RawlsBillingAccountName("billingAccount1"), +// Some("billing1_bq_project.billing1_dataset.billing1_table") +// ) +// +// val billingProject2SpendExport = +// BillingProjectSpendExport(RawlsBillingProjectName("billingProject2"), +// RawlsBillingAccountName("billingAccount2"), +// Some("billing2_bq_project.billing2_dataset.billing2_table") +// ) +// +// val billingProject3SpendExport = +// BillingProjectSpendExport(RawlsBillingProjectName("billingProject3"), +// RawlsBillingAccountName("billingAccount3"), +// None +// ) +// +// val service = spy( +// new SpendReportingService( +// testContext, +// dataSource, +// Resource.pure[IO, GoogleBigQueryService[IO]](mock[GoogleBigQueryService[IO]]), +// mock[BillingRepository], +// mock[BillingProfileManagerDAO], +// mock[SamDAO], +// spendReportingServiceConfig, +// mockWorkspaceServiceConstructor +// ) +// ) +// +// doReturn(Future.successful(Seq(billingProject1SpendExport, billingProject2SpendExport, billingProject3SpendExport))) +// .when(service) +// .getSpendExportConfigurations( +// any() +// ) +// +// val workspace1Billing1 = +// TestData.workspace("workspace1Billing1", +// GoogleProjectId("workspace1ProjectId"), +// WorkspaceVersions.V1, +// "billingProject1" +// ) +// val workspace2Billing1 = +// TestData.workspace("workspace2Billing1", +// GoogleProjectId("workspace2ProjectId"), +// WorkspaceVersions.V2, +// "billingProject1" +// ) +// val workspace1Billing2 = +// TestData.workspace("workspace1Billing2", +// GoogleProjectId("workspace3ProjectId"), +// WorkspaceVersions.V2, +// "billingProject2" +// ) +// val workspace1Billing3 = +// TestData.workspace("workspace1Billing3", +// GoogleProjectId("workspace4ProjectId"), +// WorkspaceVersions.V2, +// "billingProject3" +// ) +// +// val workspace1Response = WorkspaceListResponse( +// WorkspaceAccessLevels.Read, +// Some(true), +// Some(true), +// WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing1, +// Option(Set.empty), +// true, +// Some(WorkspaceCloudPlatform.Gcp) +// ), +// Option.empty, +// false, +// Some(List.empty) +// ) +// val workspace2Response = WorkspaceListResponse( +// WorkspaceAccessLevels.Read, +// Some(true), +// Some(true), +// WorkspaceDetails.fromWorkspaceAndOptions(workspace2Billing1, +// Option(Set.empty), +// true, +// Some(WorkspaceCloudPlatform.Gcp) +// ), +// Option.empty, +// false, +// Some(List.empty) +// ) +// val workspace3Response = WorkspaceListResponse( +// WorkspaceAccessLevels.Read, +// Some(true), +// Some(true), +// WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing2, +// Option(Set.empty), +// true, +// Some(WorkspaceCloudPlatform.Gcp) +// ), +// Option.empty, +// false, +// Some(List.empty) +// ) +// val workspace4Response = WorkspaceListResponse( +// WorkspaceAccessLevels.Read, +// Some(true), +// Some(true), +// WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing3, +// Option(Set.empty), +// true, +// Some(WorkspaceCloudPlatform.Gcp) +// ), +// Option.empty, +// false, +// Some(List.empty) +// ) +// +// val result = Await.result( +// service.getBillingSpendExportsForWorkspaces( +// Seq(workspace2Response, workspace3Response, workspace1Response, workspace4Response) +// ), +// Duration.Inf +// ) +// +// result shouldBe Map( +// billingProject1SpendExport -> Seq(GoogleProjectId("workspace2ProjectId"), GoogleProjectId("workspace1ProjectId")), +// billingProject2SpendExport -> Seq(GoogleProjectId("workspace3ProjectId")), +// billingProject3SpendExport -> Seq(GoogleProjectId("workspace4ProjectId")) +// ) +// } - val dataSource = mock[SlickDataSource] - - val billingProject1SpendExport = - BillingProjectSpendExport(RawlsBillingProjectName("billingProject1"), - RawlsBillingAccountName("billingAccount1"), - Some("billing1_bq_project.billing1_dataset.billing1_table") - ) - - val billingProject2SpendExport = - BillingProjectSpendExport(RawlsBillingProjectName("billingProject2"), - RawlsBillingAccountName("billingAccount2"), - Some("billing2_bq_project.billing2_dataset.billing2_table") - ) - - val billingProject3SpendExport = - BillingProjectSpendExport(RawlsBillingProjectName("billingProject3"), - RawlsBillingAccountName("billingAccount3"), - None - ) - - val service = spy( - new SpendReportingService( - testContext, - dataSource, - Resource.pure[IO, GoogleBigQueryService[IO]](mock[GoogleBigQueryService[IO]]), - mock[BillingRepository], - mock[BillingProfileManagerDAO], - mock[SamDAO], - spendReportingServiceConfig, - mockWorkspaceServiceConstructor - ) - ) - - doReturn(Future.successful(Seq(billingProject1SpendExport, billingProject2SpendExport, billingProject3SpendExport))) - .when(service) - .getSpendExportConfigurations( - any() - ) - - val workspace1Billing1 = - TestData.workspace("workspace1Billing1", - GoogleProjectId("workspace1ProjectId"), - WorkspaceVersions.V1, - "billingProject1" - ) - val workspace2Billing1 = - TestData.workspace("workspace2Billing1", - GoogleProjectId("workspace2ProjectId"), - WorkspaceVersions.V2, - "billingProject1" - ) - val workspace1Billing2 = - TestData.workspace("workspace1Billing2", - GoogleProjectId("workspace3ProjectId"), - WorkspaceVersions.V2, - "billingProject2" - ) - val workspace1Billing3 = - TestData.workspace("workspace1Billing3", - GoogleProjectId("workspace4ProjectId"), - WorkspaceVersions.V2, - "billingProject3" - ) - - val workspace1Response = WorkspaceListResponse( - WorkspaceAccessLevels.Read, - Some(true), - Some(true), - WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing1, - Option(Set.empty), - true, - Some(WorkspaceCloudPlatform.Gcp) - ), - Option.empty, - false, - Some(List.empty) - ) - val workspace2Response = WorkspaceListResponse( - WorkspaceAccessLevels.Read, - Some(true), - Some(true), - WorkspaceDetails.fromWorkspaceAndOptions(workspace2Billing1, - Option(Set.empty), - true, - Some(WorkspaceCloudPlatform.Gcp) - ), - Option.empty, - false, - Some(List.empty) - ) - val workspace3Response = WorkspaceListResponse( - WorkspaceAccessLevels.Read, - Some(true), - Some(true), - WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing2, - Option(Set.empty), - true, - Some(WorkspaceCloudPlatform.Gcp) - ), - Option.empty, - false, - Some(List.empty) - ) - val workspace4Response = WorkspaceListResponse( - WorkspaceAccessLevels.Read, - Some(true), - Some(true), - WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing3, - Option(Set.empty), - true, - Some(WorkspaceCloudPlatform.Gcp) - ), - Option.empty, - false, - Some(List.empty) - ) - - val result = Await.result( - service.getBillingSpendExportsForWorkspaces( - Seq(workspace2Response, workspace3Response, workspace1Response, workspace4Response) - ), - Duration.Inf - ) - - result shouldBe Map( - billingProject1SpendExport -> Seq(GoogleProjectId("workspace2ProjectId"), GoogleProjectId("workspace1ProjectId")), - billingProject2SpendExport -> Seq(GoogleProjectId("workspace3ProjectId")), - billingProject3SpendExport -> Seq(GoogleProjectId("workspace4ProjectId")) - ) - } - - "getAllUserWorkspaceQuery" should "union all billingProjects with their workspace projects" in { + "getAllUserWorkspaceQuery" should "union all billingProjects with their workspace projects" in { val billingProject1SpendExport = BillingProjectSpendExport(RawlsBillingProjectName("billingProject1"), @@ -1491,390 +1492,394 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki } -// "getBillingWithSpendPermission" should "return spendConfigurations for workspaces" in { -// val dataSource = mock[SlickDataSource] -// -// val billingProject1SpendExport = -// BillingProjectSpendExport(RawlsBillingProjectName("billingProject1"), -// RawlsBillingAccountName("billingAccount1"), -// Some("billing1_bq_project.billing1_dataset.billing1_table") -// ) -// -// val billingProject2SpendExport = -// BillingProjectSpendExport(RawlsBillingProjectName("billingProject2"), -// RawlsBillingAccountName("billingAccount2"), -// Some("billing2_bq_project.billing2_dataset.billing2_table") -// ) -// -// val billingProject3SpendExport = -// BillingProjectSpendExport(RawlsBillingProjectName("billingProject3"), -// RawlsBillingAccountName("billingAccount3"), -// None -// ) -// -// val service = spy( -// new SpendReportingService( -// testContext, -// dataSource, -// Resource.pure[IO, GoogleBigQueryService[IO]](mock[GoogleBigQueryService[IO]]), -// mock[BillingRepository], -// mock[BillingProfileManagerDAO], -// mock[SamDAO], -// spendReportingServiceConfig, -// mockWorkspaceServiceConstructor -// ) -// ) -// -// doReturn(Future.successful(Seq(billingProject1SpendExport, billingProject2SpendExport, billingProject3SpendExport))) -// .when(service) -// .getSpendExportConfigurations( -// any() -// ) -// -// val workspace1Billing1 = -// TestData.workspace("workspace1Billing1", -// GoogleProjectId("workspace1ProjectId"), -// WorkspaceVersions.V1, -// "billingProject1" -// ) -// val workspace2Billing1 = -// TestData.workspace("workspace2Billing1", -// GoogleProjectId("workspace2ProjectId"), -// WorkspaceVersions.V2, -// "billingProject1" -// ) -// val workspace1Billing2 = -// TestData.workspace("workspace1Billing2", -// GoogleProjectId("workspace3ProjectId"), -// WorkspaceVersions.V2, -// "billingProject2" -// ) -// val workspace1Billing3 = -// TestData.workspace("workspace1Billing3", -// GoogleProjectId("workspace4ProjectId"), -// WorkspaceVersions.V2, -// "billingProject3" -// ) -// -// val workspace1Response = WorkspaceListResponse( -// WorkspaceAccessLevels.Read, -// Some(true), -// Some(true), -// WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing1, -// Option(Set.empty), -// true, -// Some(WorkspaceCloudPlatform.Gcp) -// ), -// Option.empty, -// false, -// Some(List.empty) -// ) -// val workspace2Response = WorkspaceListResponse( -// WorkspaceAccessLevels.Read, -// Some(true), -// Some(true), -// WorkspaceDetails.fromWorkspaceAndOptions(workspace2Billing1, -// Option(Set.empty), -// true, -// Some(WorkspaceCloudPlatform.Gcp) -// ), -// Option.empty, -// false, -// Some(List.empty) -// ) -// val workspace3Response = WorkspaceListResponse( -// WorkspaceAccessLevels.Read, -// Some(true), -// Some(true), -// WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing2, -// Option(Set.empty), -// true, -// Some(WorkspaceCloudPlatform.Gcp) -// ), -// Option.empty, -// false, -// Some(List.empty) -// ) -// val workspace4Response = WorkspaceListResponse( -// WorkspaceAccessLevels.Read, -// Some(true), -// Some(true), -// WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing3, -// Option(Set.empty), -// true, -// Some(WorkspaceCloudPlatform.Gcp) -// ), -// Option.empty, -// false, -// Some(List.empty) -// ) -// -// val result = Await.result( -// service.getBillingWithSpendPermission( -// ), -// Duration.Inf -// ) -// -// result shouldBe Map( -// billingProject1SpendExport -> Seq(GoogleProjectId("workspace2ProjectId"), GoogleProjectId("workspace1ProjectId")), -// billingProject2SpendExport -> Seq(GoogleProjectId("workspace3ProjectId")), -// billingProject3SpendExport -> Seq(GoogleProjectId("workspace4ProjectId")) -// ) -// -// } + "getBillingWithSpendPermission" should "return spendConfigurations for workspaces" ignore { + val dataSource = mock[SlickDataSource] -// "extractSpendReportingResultsAcrossBillingProjects" should "should return correct summary data" in { -// val storageCostWs1 = 100.582 -// val otherCostWs1 = 0.10111 -// val storageCostRoundedWs1: BigDecimal = BigDecimal(storageCostWs1).setScale(2, RoundingMode.HALF_EVEN) -// val otherCostRoundedWs1: BigDecimal = BigDecimal(otherCostWs1).setScale(2, RoundingMode.HALF_EVEN) -// val totalCostRoundedWs1: BigDecimal = BigDecimal(storageCostWs1 + otherCostWs1).setScale(2, RoundingMode.HALF_EVEN) -// -// val storageCostWs2 = 20.145 -// val computeCostWs2 = 150.4033 -// val storageCostRoundedWs2: BigDecimal = BigDecimal(storageCostWs2).setScale(2, RoundingMode.HALF_EVEN) -// val otherCostRoundedWs2: BigDecimal = BigDecimal(computeCostWs2).setScale(2, RoundingMode.HALF_EVEN) -// val totalCostRoundedWs2: BigDecimal = -// BigDecimal(storageCostWs2 + computeCostWs2).setScale(2, RoundingMode.HALF_EVEN) -// -// val computeCostWs3 = 1111.222 -// val otherCostWs3 = 0.02 -// val storageCostRoundedWs3: BigDecimal = BigDecimal(computeCostWs3).setScale(2, RoundingMode.HALF_EVEN) -// val otherCostRoundedWs3: BigDecimal = BigDecimal(otherCostWs3).setScale(2, RoundingMode.HALF_EVEN) -// val totalCostRoundedWs3: BigDecimal = BigDecimal(computeCostWs3 + otherCostWs3).setScale(2, RoundingMode.HALF_EVEN) -// -// val table: List[Map[String, String]] = List( -// Map( -// "storage_cost" -> s"$storageCostWs1", -// "compute_cost" -> "0.0", -// "other_cost" -> s"$otherCostWs1", -// "googleProjectId" -> "workspace1ProjectId" -// ), -// Map( -// "storage_cost" -> s"$storageCostWs2", -// "compute_cost" -> s"$computeCostWs2", -// "other_cost" -> "0.0", -// "googleProjectId" -> "workspace2ProjectId" -// ), -// Map( -// "storage_cost" -> "0.0", -// "compute_cost" -> s"$computeCostWs3", -// "other_cost" -> s"$otherCostWs3", -// "googleProjectId" -> "workspace3ProjectId" -// ) -// ) -// -// val tableResult: TableResult = createTableResult(table) -// -// val reportingResults = SpendReportingService.extractCrossBillingProjectSpendReportingResults( -// tableResult.getValues.asScala.toList, -// DateTime.now().minusDays(1), -// DateTime.now(), -// Map() -// ) -// reportingResults.spendSummary.cost shouldBe TestData.Workspace.totalCostRounded.toString -// reportingResults.spendDetails shouldBe empty -// } + val billingProject1SpendExport = + BillingProjectSpendExport(RawlsBillingProjectName("billingProject1"), + RawlsBillingAccountName("billingAccount1"), + Some("billing1_bq_project.billing1_dataset.billing1_table") + ) -// "getSpendForAllWorkspaces" should "get the spend report from multiple billing projects" in { -// val from = DateTime.now().minusMonths(2) -// val to = from.plusMonths(1) -// -// val price1 = BigDecimal("10.22") -// val price2 = BigDecimal("50.74") -// val currency = "USD" -// -// val samDAO = mock[SamDAO](RETURNS_SMART_NULLS) -// val billingRepository = mock[BillingRepository](RETURNS_SMART_NULLS) -// val bpmDAO = mock[BillingProfileManagerDAO](RETURNS_SMART_NULLS) -// -// // Billing projects -// val billingProfileId1 = UUID.randomUUID() -// val projectName1 = RawlsBillingProjectName("billingProject1") -// val billingAccount1 = RawlsBillingAccountName("billingAcct1") -// val billingProject1 = RawlsBillingProject( -// projectName1, -// CreationStatuses.Ready, -// Option(billingAccount1), -// None, -// billingProfileId = Option.apply(billingProfileId1.toString) -// ) -// val billingProfileId2 = UUID.randomUUID() -// val projectName2 = RawlsBillingProjectName("billingProject2") -// val billingAccount2 = RawlsBillingAccountName("billingAcct2") -// val billingProject2 = RawlsBillingProject( -// projectName2, -// CreationStatuses.Ready, -// Option(billingAccount2), -// None, -// billingProfileId = Option.apply(billingProfileId2.toString) -// ) -// val billingProfileId3 = UUID.randomUUID() -// val projectName3 = RawlsBillingProjectName("billingProject3") -// val billingAccount3 = RawlsBillingAccountName("billingAcct3") -// val billingProject3 = RawlsBillingProject( -// projectName3, -// CreationStatuses.Ready, -// Option(billingAccount3), -// None, -// billingProfileId = Option.apply(billingProfileId3.toString) -// ) -// -// // Billing project spend exports -// val billingProject1SpendExport = -// BillingProjectSpendExport(RawlsBillingProjectName("billingProject1"), -// RawlsBillingAccountName("billingAccount1"), -// Some("billing1_bq_project.billing1_dataset.billing1_table") -// ) -// -// val billingProject3SpendExport = -// BillingProjectSpendExport(RawlsBillingProjectName("billingProject3"), -// RawlsBillingAccountName("billingAccount3"), -// None -// ) -// -// // Workspaces -// val workspace1Billing1 = -// TestData.workspace("workspace1Billing1", -// GoogleProjectId("workspace1ProjectId"), -// WorkspaceVersions.V1, -// "billingProject1" -// ) -// val workspace2Billing1 = -// TestData.workspace("workspace2Billing1", -// GoogleProjectId("workspace2ProjectId"), -// WorkspaceVersions.V2, -// "billingProject1" -// ) -// val workspace1Billing2 = -// TestData.workspace("workspace1Billing2", -// GoogleProjectId("workspace3ProjectId"), -// WorkspaceVersions.V2, -// "billingProject2" -// ) -// val workspace1Billing3 = -// TestData.workspace("workspace1Billing3", -// GoogleProjectId("workspace4ProjectId"), -// WorkspaceVersions.V2, -// "billingProject3" -// ) -// -// // Only workspaces 1 and 4 are owned -// val workspace1Response = WorkspaceListResponse( -// WorkspaceAccessLevels.Owner, -// Some(true), -// Some(true), -// WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing1, -// Option(Set.empty), -// true, -// Some(WorkspaceCloudPlatform.Gcp) -// ), -// Option.empty, -// false, -// Some(List.empty) -// ) -// val workspace2Response = WorkspaceListResponse( -// WorkspaceAccessLevels.Read, -// Some(true), -// Some(true), -// WorkspaceDetails.fromWorkspaceAndOptions(workspace2Billing1, -// Option(Set.empty), -// true, -// Some(WorkspaceCloudPlatform.Gcp) -// ), -// Option.empty, -// false, -// Some(List.empty) -// ) -// val workspace3Response = WorkspaceListResponse( -// WorkspaceAccessLevels.Write, -// Some(true), -// Some(true), -// WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing2, -// Option(Set.empty), -// true, -// Some(WorkspaceCloudPlatform.Gcp) -// ), -// Option.empty, -// false, -// Some(List.empty) -// ) -// val workspace4Response = WorkspaceListResponse( -// WorkspaceAccessLevels.Owner, -// Some(true), -// Some(true), -// WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing3, -// Option(Set.empty), -// true, -// Some(WorkspaceCloudPlatform.Gcp) -// ), -// Option.empty, -// false, -// Some(List.empty) -// ) -// -// val dataSource = mock[SlickDataSource] -// val mockWorkspaceService = mock[WorkspaceService](RETURNS_SMART_NULLS) -// -// when(mockWorkspaceService.listWorkspaces(any(), any())) -// .thenReturn( -// Future.successful(Seq(workspace1Response, workspace2Response, workspace3Response, workspace4Response).toJson) -// ) -// val mockWorkspaceServiceConstructor: RawlsRequestContext => WorkspaceService = { _ => -// mockWorkspaceService -// } -// -// when(billingRepository.getBillingProject(mockitoEq(projectName1))) -// .thenReturn(Future.successful(Option.apply(billingProject1))) -// when(billingRepository.getBillingProject(mockitoEq(projectName2))) -// .thenReturn(Future.successful(Option.apply(billingProject2))) -// when(billingRepository.getBillingProject(mockitoEq(projectName3))) -// .thenReturn(Future.successful(Option.apply(billingProject3))) -// -// when(samDAO.userHasAction(any(), any(), any(), any())).thenReturn(Future.successful(true)) -// -// val bigQueryService = mockBigQuery(List[Map[String, String]]()) -// -// val service = spy( -// new SpendReportingService( -// testContext, -// mock[SlickDataSource], -// bigQueryService, -// billingRepository, -// bpmDAO, -// samDAO, -// spendReportingServiceConfig, -// mockWorkspaceServiceConstructor -// ) -// ) -// doReturn(Future.successful(Seq(billingProject1SpendExport, billingProject3SpendExport))) -// .when(service) -// .getSpendExportConfigurations( -// any() -// ) -// -// val spendReport = -// TestData.BpmSpendReport.spendData(from, to, currency, Map("Compute" -> price1, "Storage" -> price2)) -// -// val billingProfileIdCapture: ArgumentCaptor[UUID] = ArgumentCaptor.forClass(classOf[UUID]) -// val startDateCapture: ArgumentCaptor[Date] = ArgumentCaptor.forClass(classOf[Date]) -// val endDateCapture: ArgumentCaptor[Date] = ArgumentCaptor.forClass(classOf[Date]) -// -// val result = Await.result( -// service.getSpendForAllWorkspaces(from, to), -// Duration.Inf -// ) -// -// result.spendSummary.credits shouldBe "0" -// result.spendSummary.cost shouldBe Seq(price1, price2).sum.toString() -// result.spendSummary.currency shouldBe "USD" -// result.spendSummary.startTime.get.toString(ISODateTimeFormat.date()) shouldBe from.toString( -// ISODateTimeFormat.date() -// ) -// result.spendSummary.endTime.get.toString(ISODateTimeFormat.date()) shouldBe to.toString(ISODateTimeFormat.date()) -// -// startDateCapture.getValue shouldBe from.toDate -// endDateCapture.getValue shouldBe to.toDate -// } + val billingProject2SpendExport = + BillingProjectSpendExport(RawlsBillingProjectName("billingProject2"), + RawlsBillingAccountName("billingAccount2"), + Some("billing2_bq_project.billing2_dataset.billing2_table") + ) + + val billingProject3SpendExport = + BillingProjectSpendExport(RawlsBillingProjectName("billingProject3"), + RawlsBillingAccountName("billingAccount3"), + None + ) + val samDAO: MockSamDAO = new MockSamDAO(dataSource) + + val service = spy( + new SpendReportingService( + testContext, + dataSource, + Resource.pure[IO, GoogleBigQueryService[IO]](mock[GoogleBigQueryService[IO]]), + mock[BillingRepository], + mock[BillingProfileManagerDAO], + samDAO, + spendReportingServiceConfig, + mockWorkspaceServiceConstructor + ) + ) + + doReturn(Future.successful(Seq(billingProject1SpendExport, billingProject2SpendExport, billingProject3SpendExport))) + .when(service) + .getSpendExportConfigurations( + any() + ) + + val workspace1Billing1 = + TestData.workspace("workspace1Billing1", + GoogleProjectId("workspace1ProjectId"), + WorkspaceVersions.V1, + "billingProject1" + ) + val workspace2Billing1 = + TestData.workspace("workspace2Billing1", + GoogleProjectId("workspace2ProjectId"), + WorkspaceVersions.V2, + "billingProject1" + ) + val workspace1Billing2 = + TestData.workspace("workspace1Billing2", + GoogleProjectId("workspace3ProjectId"), + WorkspaceVersions.V2, + "billingProject2" + ) + val workspace1Billing3 = + TestData.workspace("workspace1Billing3", + GoogleProjectId("workspace4ProjectId"), + WorkspaceVersions.V2, + "billingProject3" + ) + + val workspace1Response = WorkspaceListResponse( + WorkspaceAccessLevels.Read, + Some(true), + Some(true), + WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing1, + Option(Set.empty), + true, + Some(WorkspaceCloudPlatform.Gcp) + ), + Option.empty, + false, + Some(List.empty) + ) + val workspace2Response = WorkspaceListResponse( + WorkspaceAccessLevels.Read, + Some(true), + Some(true), + WorkspaceDetails.fromWorkspaceAndOptions(workspace2Billing1, + Option(Set.empty), + true, + Some(WorkspaceCloudPlatform.Gcp) + ), + Option.empty, + false, + Some(List.empty) + ) + val workspace3Response = WorkspaceListResponse( + WorkspaceAccessLevels.Read, + Some(true), + Some(true), + WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing2, + Option(Set.empty), + true, + Some(WorkspaceCloudPlatform.Gcp) + ), + Option.empty, + false, + Some(List.empty) + ) + val workspace4Response = WorkspaceListResponse( + WorkspaceAccessLevels.Read, + Some(true), + Some(true), + WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing3, + Option(Set.empty), + true, + Some(WorkspaceCloudPlatform.Gcp) + ), + Option.empty, + false, + Some(List.empty) + ) + + val result = Await.result( + service.getBillingWithSpendPermission( + ), + Duration.Inf + ) + + result shouldBe Map( + billingProject1SpendExport -> Seq(GoogleProjectId("workspace2ProjectId"), GoogleProjectId("workspace1ProjectId")), + billingProject2SpendExport -> Seq(GoogleProjectId("workspace3ProjectId")), + billingProject3SpendExport -> Seq(GoogleProjectId("workspace4ProjectId")) + ) + + } + + "extractSpendReportingResultsAcrossBillingProjects" should "should return correct summary data" ignore { + val storageCostWs1 = 100.582 + val otherCostWs1 = 0.10111 + val storageCostRoundedWs1: BigDecimal = BigDecimal(storageCostWs1).setScale(2, RoundingMode.HALF_EVEN) + val otherCostRoundedWs1: BigDecimal = BigDecimal(otherCostWs1).setScale(2, RoundingMode.HALF_EVEN) + val totalCostRoundedWs1: BigDecimal = BigDecimal(storageCostWs1 + otherCostWs1).setScale(2, RoundingMode.HALF_EVEN) + + val storageCostWs2 = 20.145 + val computeCostWs2 = 150.4033 + val storageCostRoundedWs2: BigDecimal = BigDecimal(storageCostWs2).setScale(2, RoundingMode.HALF_EVEN) + val otherCostRoundedWs2: BigDecimal = BigDecimal(computeCostWs2).setScale(2, RoundingMode.HALF_EVEN) + val totalCostRoundedWs2: BigDecimal = + BigDecimal(storageCostWs2 + computeCostWs2).setScale(2, RoundingMode.HALF_EVEN) + + val computeCostWs3 = 1111.222 + val otherCostWs3 = 0.02 + val storageCostRoundedWs3: BigDecimal = BigDecimal(computeCostWs3).setScale(2, RoundingMode.HALF_EVEN) + val otherCostRoundedWs3: BigDecimal = BigDecimal(otherCostWs3).setScale(2, RoundingMode.HALF_EVEN) + val totalCostRoundedWs3: BigDecimal = BigDecimal(computeCostWs3 + otherCostWs3).setScale(2, RoundingMode.HALF_EVEN) + + val table: List[Map[String, String]] = List( + Map( + "storage_cost" -> s"$storageCostWs1", + "compute_cost" -> "0.0", + "other_cost" -> s"$otherCostWs1", + "project_id" -> "workspace1ProjectId", + "currency" -> "USD" + ), + Map( + "storage_cost" -> s"$storageCostWs2", + "compute_cost" -> s"$computeCostWs2", + "other_cost" -> "0.0", + "project_id" -> "workspace2ProjectId", + "currency" -> "USD" + ), + Map( + "storage_cost" -> "0.0", + "compute_cost" -> s"$computeCostWs3", + "other_cost" -> s"$otherCostWs3", + "project_id" -> "workspace3ProjectId", + "currency" -> "USD" + ) + ) + + val tableResult: TableResult = createTableResult(table) + + val reportingResults = SpendReportingService.extractCrossBillingProjectSpendReportingResults( + tableResult.getValues.asScala.toList, + DateTime.now().minusDays(1), + DateTime.now(), + Map() + ) + reportingResults.spendSummary.cost shouldBe TestData.Workspace.totalCostRounded.toString + reportingResults.spendDetails shouldBe empty + } + + "getSpendForAllWorkspaces" should "get the spend report from multiple billing projects" ignore { + val from = DateTime.now().minusMonths(2) + val to = from.plusMonths(1) + + val price1 = BigDecimal("10.22") + val price2 = BigDecimal("50.74") + val currency = "USD" + + val samDAO = mock[SamDAO](RETURNS_SMART_NULLS) + val billingRepository = mock[BillingRepository](RETURNS_SMART_NULLS) + val bpmDAO = mock[BillingProfileManagerDAO](RETURNS_SMART_NULLS) + + // Billing projects + val billingProfileId1 = UUID.randomUUID() + val projectName1 = RawlsBillingProjectName("billingProject1") + val billingAccount1 = RawlsBillingAccountName("billingAcct1") + val billingProject1 = RawlsBillingProject( + projectName1, + CreationStatuses.Ready, + Option(billingAccount1), + None, + billingProfileId = Option.apply(billingProfileId1.toString) + ) + val billingProfileId2 = UUID.randomUUID() + val projectName2 = RawlsBillingProjectName("billingProject2") + val billingAccount2 = RawlsBillingAccountName("billingAcct2") + val billingProject2 = RawlsBillingProject( + projectName2, + CreationStatuses.Ready, + Option(billingAccount2), + None, + billingProfileId = Option.apply(billingProfileId2.toString) + ) + val billingProfileId3 = UUID.randomUUID() + val projectName3 = RawlsBillingProjectName("billingProject3") + val billingAccount3 = RawlsBillingAccountName("billingAcct3") + val billingProject3 = RawlsBillingProject( + projectName3, + CreationStatuses.Ready, + Option(billingAccount3), + None, + billingProfileId = Option.apply(billingProfileId3.toString) + ) + + // Billing project spend exports + val billingProject1SpendExport = + BillingProjectSpendExport(RawlsBillingProjectName("billingProject1"), + RawlsBillingAccountName("billingAccount1"), + Some("billing1_bq_project.billing1_dataset.billing1_table") + ) + + val billingProject3SpendExport = + BillingProjectSpendExport(RawlsBillingProjectName("billingProject3"), + RawlsBillingAccountName("billingAccount3"), + None + ) + + // Workspaces + val workspace1Billing1 = + TestData.workspace("workspace1Billing1", + GoogleProjectId("workspace1ProjectId"), + WorkspaceVersions.V1, + "billingProject1" + ) + val workspace2Billing1 = + TestData.workspace("workspace2Billing1", + GoogleProjectId("workspace2ProjectId"), + WorkspaceVersions.V2, + "billingProject1" + ) + val workspace1Billing2 = + TestData.workspace("workspace1Billing2", + GoogleProjectId("workspace3ProjectId"), + WorkspaceVersions.V2, + "billingProject2" + ) + val workspace1Billing3 = + TestData.workspace("workspace1Billing3", + GoogleProjectId("workspace4ProjectId"), + WorkspaceVersions.V2, + "billingProject3" + ) + + // Only workspaces 1 and 4 are owned + val workspace1Response = WorkspaceListResponse( + WorkspaceAccessLevels.Owner, + Some(true), + Some(true), + WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing1, + Option(Set.empty), + true, + Some(WorkspaceCloudPlatform.Gcp) + ), + Option.empty, + false, + Some(List.empty) + ) + val workspace2Response = WorkspaceListResponse( + WorkspaceAccessLevels.Read, + Some(true), + Some(true), + WorkspaceDetails.fromWorkspaceAndOptions(workspace2Billing1, + Option(Set.empty), + true, + Some(WorkspaceCloudPlatform.Gcp) + ), + Option.empty, + false, + Some(List.empty) + ) + val workspace3Response = WorkspaceListResponse( + WorkspaceAccessLevels.Write, + Some(true), + Some(true), + WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing2, + Option(Set.empty), + true, + Some(WorkspaceCloudPlatform.Gcp) + ), + Option.empty, + false, + Some(List.empty) + ) + val workspace4Response = WorkspaceListResponse( + WorkspaceAccessLevels.Owner, + Some(true), + Some(true), + WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing3, + Option(Set.empty), + true, + Some(WorkspaceCloudPlatform.Gcp) + ), + Option.empty, + false, + Some(List.empty) + ) + + val dataSource = mock[SlickDataSource] + val mockWorkspaceService = mock[WorkspaceService](RETURNS_SMART_NULLS) + + when(mockWorkspaceService.listWorkspaces(any(), any())) + .thenReturn( + Future.successful(Seq(workspace1Response, workspace2Response, workspace3Response, workspace4Response).toJson) + ) + val mockWorkspaceServiceConstructor: RawlsRequestContext => WorkspaceService = { _ => + mockWorkspaceService + } + + when(billingRepository.getBillingProject(mockitoEq(projectName1))) + .thenReturn(Future.successful(Option.apply(billingProject1))) + when(billingRepository.getBillingProject(mockitoEq(projectName2))) + .thenReturn(Future.successful(Option.apply(billingProject2))) + when(billingRepository.getBillingProject(mockitoEq(projectName3))) + .thenReturn(Future.successful(Option.apply(billingProject3))) + + when(samDAO.userHasAction(any(), any(), any(), any())).thenReturn(Future.successful(true)) + + val bigQueryService = mockBigQuery(List[Map[String, String]]()) + + val service = spy( + new SpendReportingService( + testContext, + mock[SlickDataSource], + bigQueryService, + billingRepository, + bpmDAO, + samDAO, + spendReportingServiceConfig, + mockWorkspaceServiceConstructor + ) + ) + doReturn(Future.successful(Seq(billingProject1SpendExport, billingProject3SpendExport))) + .when(service) + .getSpendExportConfigurations( + any() + ) + + val spendReport = + TestData.BpmSpendReport.spendData(from, to, currency, Map("Compute" -> price1, "Storage" -> price2)) + + val billingProfileIdCapture: ArgumentCaptor[UUID] = ArgumentCaptor.forClass(classOf[UUID]) + val startDateCapture: ArgumentCaptor[Date] = ArgumentCaptor.forClass(classOf[Date]) + val endDateCapture: ArgumentCaptor[Date] = ArgumentCaptor.forClass(classOf[Date]) + + val result = Await.result( + service.getSpendForAllWorkspaces(from, to), + Duration.Inf + ) + + result.spendSummary.credits shouldBe "0" + result.spendSummary.cost shouldBe Seq(price1, price2).sum.toString() + result.spendSummary.currency shouldBe "USD" + result.spendSummary.startTime.get.toString(ISODateTimeFormat.date()) shouldBe from.toString( + ISODateTimeFormat.date() + ) + result.spendSummary.endTime.get.toString(ISODateTimeFormat.date()) shouldBe to.toString(ISODateTimeFormat.date()) + + startDateCapture.getValue shouldBe from.toDate + endDateCapture.getValue shouldBe to.toDate + } } From 9fd6bc0c4fd84e68bc6f646787e86a3ee2dc6cf5 Mon Sep 17 00:00:00 2001 From: Bria Morgan Date: Thu, 21 Nov 2024 14:16:48 -0500 Subject: [PATCH 16/31] fix unit tests --- .../dsde/rawls/dataaccess/HttpSamDAO.scala | 18 +- .../SpendReportingServiceSpec.scala | 376 +++++------------- 2 files changed, 105 insertions(+), 289 deletions(-) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpSamDAO.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpSamDAO.scala index 3fca278c2e..c5f95ed0ab 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpSamDAO.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpSamDAO.scala @@ -15,6 +15,7 @@ import org.broadinstitute.dsde.rawls.util.{FutureSupport, Retry} import org.broadinstitute.dsde.workbench.client.sam import org.broadinstitute.dsde.workbench.client.sam.api._ import org.broadinstitute.dsde.workbench.client.sam.model.{ + FilteredFlatResourcePolicy, FilteredHierarchicalResourcePolicy, ListResourcesV2200Response } @@ -540,11 +541,15 @@ class HttpSamDAO(baseSamServiceURL: String, rawlsCredential: RawlsCredential, ti .getResources() .asScala .map { resource => + val resourcePolicies = resource.getPolicies.asScala.toList + val publicPolicies = resourcePolicies.filter(_.getIsPublic) + // might need to filter public policies out of this next statement: + val (inheritedPolicies, directPolicies) = resourcePolicies.partition(_.getInherited) SamUserResource( resource.getResourceId, - toSamRolesAndActions(resource.getPolicies()), // TODO what to use here? - toSamRolesAndActions(resource.getPolicies()), // What are all these three things? - toSamRolesAndActions(resource.getPolicies()), + toSamRolesAndActions(directPolicies), + toSamRolesAndActions(inheritedPolicies), + toSamRolesAndActions(publicPolicies), resource.getAuthDomainGroups.asScala.map(WorkbenchGroupName).toSet, resource.getMissingAuthDomainGroups.asScala.map(WorkbenchGroupName).toSet ) @@ -559,10 +564,9 @@ class HttpSamDAO(baseSamServiceURL: String, rawlsCredential: RawlsCredential, ti rolesAndActions.getActions.asScala.map(SamResourceAction).toSet ) - private def toSamRolesAndActions(policies: util.List[FilteredHierarchicalResourcePolicy]) = { - val scalaPolicies = policies.asScala.toList - val roles = scalaPolicies.flatMap(_.getRoles.asScala) - val actions = scalaPolicies.flatMap(_.getActions.asScala) + private def toSamRolesAndActions(policies: scala.collection.immutable.List[FilteredHierarchicalResourcePolicy]) = { + val roles = policies.flatMap(_.getRoles.asScala) + val actions = policies.flatMap(_.getActions.asScala) SamRolesAndActions(roles.map(role => SamResourceRole(role.toString)).toSet, actions.map(SamResourceAction).toSet) } diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala index a02f1ec216..49234eead8 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala @@ -1151,210 +1151,6 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki result shouldBe Map(GoogleProjectId("v2ProjectId") -> v2Workspace.toWorkspaceName) } -// "getOwnerWorkspaces" should "return any and all workspaces user has owner access to" in { -// val ownerWorkspace1 = TestData.workspace("owner1", GoogleProjectId("owner1ProjectId"), WorkspaceVersions.V1) -// val ownerWorkspace2 = TestData.workspace("owner2", GoogleProjectId("owner2ProjectId"), WorkspaceVersions.V2) -// val readerWorkspace = TestData.workspace("reader1", GoogleProjectId("reader1ProjectId"), WorkspaceVersions.V2) -// -// val dataSource = mock[SlickDataSource] -// val mockWorkspaceService = mock[WorkspaceService](RETURNS_SMART_NULLS) -// -// val workspace1Response = WorkspaceListResponse( -// WorkspaceAccessLevels.Owner, -// Some(true), -// Some(true), -// WorkspaceDetails.fromWorkspaceAndOptions(ownerWorkspace1, -// Option(Set.empty), -// true, -// Some(WorkspaceCloudPlatform.Gcp) -// ), -// Option.empty, -// false, -// Some(List.empty) -// ) -// -// val workspace2Response = WorkspaceListResponse( -// WorkspaceAccessLevels.Owner, -// Some(true), -// Some(true), -// WorkspaceDetails.fromWorkspaceAndOptions(ownerWorkspace2, -// Option(Set.empty), -// true, -// Some(WorkspaceCloudPlatform.Gcp) -// ), -// Option.empty, -// false, -// Some(List.empty) -// ) -// -// val workspace3Response = WorkspaceListResponse( -// WorkspaceAccessLevels.Read, -// Some(true), -// Some(true), -// WorkspaceDetails.fromWorkspaceAndOptions(readerWorkspace, -// Option(Set.empty), -// true, -// Some(WorkspaceCloudPlatform.Gcp) -// ), -// Option.empty, -// false, -// Some(List.empty) -// ) -// -// when(mockWorkspaceService.listWorkspaces(any(), any())) -// .thenReturn(Future.successful(Seq(workspace1Response, workspace2Response, workspace3Response).toJson)) -// val mockWorkspaceServiceConstructor: RawlsRequestContext => WorkspaceService = { _ => -// mockWorkspaceService -// } -// val service = new SpendReportingService( -// testContext, -// dataSource, -// Resource.pure[IO, GoogleBigQueryService[IO]](mock[GoogleBigQueryService[IO]]), -// mock[BillingRepository], -// mock[BillingProfileManagerDAO], -// mock[SamDAO], -// spendReportingServiceConfig, -// mockWorkspaceServiceConstructor -// ) -// -// val result = Await.result(service.getOwnerWorkspaces(), Duration.Inf) -// -// result shouldBe Seq(workspace1Response, workspace2Response) -// } -// -// "getBillingForWorkspaces" should "return spendConfigurations for workspaces" in { -// -// val dataSource = mock[SlickDataSource] -// -// val billingProject1SpendExport = -// BillingProjectSpendExport(RawlsBillingProjectName("billingProject1"), -// RawlsBillingAccountName("billingAccount1"), -// Some("billing1_bq_project.billing1_dataset.billing1_table") -// ) -// -// val billingProject2SpendExport = -// BillingProjectSpendExport(RawlsBillingProjectName("billingProject2"), -// RawlsBillingAccountName("billingAccount2"), -// Some("billing2_bq_project.billing2_dataset.billing2_table") -// ) -// -// val billingProject3SpendExport = -// BillingProjectSpendExport(RawlsBillingProjectName("billingProject3"), -// RawlsBillingAccountName("billingAccount3"), -// None -// ) -// -// val service = spy( -// new SpendReportingService( -// testContext, -// dataSource, -// Resource.pure[IO, GoogleBigQueryService[IO]](mock[GoogleBigQueryService[IO]]), -// mock[BillingRepository], -// mock[BillingProfileManagerDAO], -// mock[SamDAO], -// spendReportingServiceConfig, -// mockWorkspaceServiceConstructor -// ) -// ) -// -// doReturn(Future.successful(Seq(billingProject1SpendExport, billingProject2SpendExport, billingProject3SpendExport))) -// .when(service) -// .getSpendExportConfigurations( -// any() -// ) -// -// val workspace1Billing1 = -// TestData.workspace("workspace1Billing1", -// GoogleProjectId("workspace1ProjectId"), -// WorkspaceVersions.V1, -// "billingProject1" -// ) -// val workspace2Billing1 = -// TestData.workspace("workspace2Billing1", -// GoogleProjectId("workspace2ProjectId"), -// WorkspaceVersions.V2, -// "billingProject1" -// ) -// val workspace1Billing2 = -// TestData.workspace("workspace1Billing2", -// GoogleProjectId("workspace3ProjectId"), -// WorkspaceVersions.V2, -// "billingProject2" -// ) -// val workspace1Billing3 = -// TestData.workspace("workspace1Billing3", -// GoogleProjectId("workspace4ProjectId"), -// WorkspaceVersions.V2, -// "billingProject3" -// ) -// -// val workspace1Response = WorkspaceListResponse( -// WorkspaceAccessLevels.Read, -// Some(true), -// Some(true), -// WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing1, -// Option(Set.empty), -// true, -// Some(WorkspaceCloudPlatform.Gcp) -// ), -// Option.empty, -// false, -// Some(List.empty) -// ) -// val workspace2Response = WorkspaceListResponse( -// WorkspaceAccessLevels.Read, -// Some(true), -// Some(true), -// WorkspaceDetails.fromWorkspaceAndOptions(workspace2Billing1, -// Option(Set.empty), -// true, -// Some(WorkspaceCloudPlatform.Gcp) -// ), -// Option.empty, -// false, -// Some(List.empty) -// ) -// val workspace3Response = WorkspaceListResponse( -// WorkspaceAccessLevels.Read, -// Some(true), -// Some(true), -// WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing2, -// Option(Set.empty), -// true, -// Some(WorkspaceCloudPlatform.Gcp) -// ), -// Option.empty, -// false, -// Some(List.empty) -// ) -// val workspace4Response = WorkspaceListResponse( -// WorkspaceAccessLevels.Read, -// Some(true), -// Some(true), -// WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing3, -// Option(Set.empty), -// true, -// Some(WorkspaceCloudPlatform.Gcp) -// ), -// Option.empty, -// false, -// Some(List.empty) -// ) -// -// val result = Await.result( -// service.getBillingSpendExportsForWorkspaces( -// Seq(workspace2Response, workspace3Response, workspace1Response, workspace4Response) -// ), -// Duration.Inf -// ) -// -// result shouldBe Map( -// billingProject1SpendExport -> Seq(GoogleProjectId("workspace2ProjectId"), GoogleProjectId("workspace1ProjectId")), -// billingProject2SpendExport -> Seq(GoogleProjectId("workspace3ProjectId")), -// billingProject3SpendExport -> Seq(GoogleProjectId("workspace4ProjectId")) -// ) -// } - "getAllUserWorkspaceQuery" should "union all billingProjects with their workspace projects" in { val billingProject1SpendExport = @@ -1492,7 +1288,9 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki } - "getBillingWithSpendPermission" should "return spendConfigurations for workspaces" ignore { + "getBillingWithSpendPermission" should "return spendConfigurations for workspaces" in { + // todo: include non-rawls bps + val dataSource = mock[SlickDataSource] val billingProject1SpendExport = @@ -1512,26 +1310,40 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki RawlsBillingAccountName("billingAccount3"), None ) - val samDAO: MockSamDAO = new MockSamDAO(dataSource) + val samDAO = mock[SamDAO] - val service = spy( - new SpendReportingService( - testContext, - dataSource, - Resource.pure[IO, GoogleBigQueryService[IO]](mock[GoogleBigQueryService[IO]]), - mock[BillingRepository], - mock[BillingProfileManagerDAO], - samDAO, - spendReportingServiceConfig, - mockWorkspaceServiceConstructor + doReturn( + Future.successful( + Seq( + SamUserResource( + "billingProject1", + SamRolesAndActions(Set.empty, Set.empty), + SamRolesAndActions(Set.empty, Set.empty), + SamRolesAndActions(Set.empty, Set.empty), + Set.empty, + Set.empty + ), + SamUserResource( + "billingProject2", + SamRolesAndActions(Set.empty, Set.empty), + SamRolesAndActions(Set.empty, Set.empty), + SamRolesAndActions(Set.empty, Set.empty), + Set.empty, + Set.empty + ), + SamUserResource( + "billingProject3", + SamRolesAndActions(Set.empty, Set.empty), + SamRolesAndActions(Set.empty, Set.empty), + SamRolesAndActions(Set.empty, Set.empty), + Set.empty, + Set.empty + ) + ) ) ) - - doReturn(Future.successful(Seq(billingProject1SpendExport, billingProject2SpendExport, billingProject3SpendExport))) - .when(service) - .getSpendExportConfigurations( - any() - ) + .when(samDAO) + .listResourcesWithActions(any(), any(), any()) val workspace1Billing1 = TestData.workspace("workspace1Billing1", @@ -1558,59 +1370,38 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki "billingProject3" ) - val workspace1Response = WorkspaceListResponse( - WorkspaceAccessLevels.Read, - Some(true), - Some(true), - WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing1, - Option(Set.empty), - true, - Some(WorkspaceCloudPlatform.Gcp) - ), - Option.empty, - false, - Some(List.empty) - ) - val workspace2Response = WorkspaceListResponse( - WorkspaceAccessLevels.Read, - Some(true), - Some(true), - WorkspaceDetails.fromWorkspaceAndOptions(workspace2Billing1, - Option(Set.empty), - true, - Some(WorkspaceCloudPlatform.Gcp) - ), - Option.empty, - false, - Some(List.empty) + val workspaces = Map( + RawlsBillingProjectName("billingProject1") -> Seq(workspace1Billing1, workspace2Billing1), + RawlsBillingProjectName("billingProject2") -> Seq(workspace1Billing2), + RawlsBillingProjectName("billingProject3") -> Seq(workspace1Billing3) ) - val workspace3Response = WorkspaceListResponse( - WorkspaceAccessLevels.Read, - Some(true), - Some(true), - WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing2, - Option(Set.empty), - true, - Some(WorkspaceCloudPlatform.Gcp) - ), - Option.empty, - false, - Some(List.empty) - ) - val workspace4Response = WorkspaceListResponse( - WorkspaceAccessLevels.Read, - Some(true), - Some(true), - WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing3, - Option(Set.empty), - true, - Some(WorkspaceCloudPlatform.Gcp) - ), - Option.empty, - false, - Some(List.empty) + val mockWorkspaceService = mock[WorkspaceService](RETURNS_SMART_NULLS) + + when(mockWorkspaceService.getWorkspacesByBillingProjects(any())) + .thenReturn(Future.successful(workspaces)) + val mockWorkspaceServiceConstructor: RawlsRequestContext => WorkspaceService = { _ => + mockWorkspaceService + } + + val service = spy( + new SpendReportingService( + testContext, + dataSource, + Resource.pure[IO, GoogleBigQueryService[IO]](mock[GoogleBigQueryService[IO]]), + mock[BillingRepository], + mock[BillingProfileManagerDAO], + samDAO, + spendReportingServiceConfig, + mockWorkspaceServiceConstructor + ) ) + doReturn(Future.successful(Seq(billingProject1SpendExport, billingProject2SpendExport, billingProject3SpendExport))) + .when(service) + .getSpendExportConfigurations( + any() + ) + val result = Await.result( service.getBillingWithSpendPermission( ), @@ -1618,38 +1409,45 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki ) result shouldBe Map( - billingProject1SpendExport -> Seq(GoogleProjectId("workspace2ProjectId"), GoogleProjectId("workspace1ProjectId")), - billingProject2SpendExport -> Seq(GoogleProjectId("workspace3ProjectId")), - billingProject3SpendExport -> Seq(GoogleProjectId("workspace4ProjectId")) + billingProject1SpendExport -> Seq( + (workspace1Billing1.googleProjectId, workspace1Billing1.toWorkspaceName), + (workspace2Billing1.googleProjectId, workspace2Billing1.toWorkspaceName) + ), + billingProject2SpendExport -> Seq((workspace1Billing2.googleProjectId, workspace1Billing2.toWorkspaceName)), + billingProject3SpendExport -> Seq((workspace1Billing3.googleProjectId, workspace1Billing3.toWorkspaceName)) ) } - "extractSpendReportingResultsAcrossBillingProjects" should "should return correct summary data" ignore { + "extractSpendReportingResultsAcrossBillingProjects" should "should return correct summary data" in { val storageCostWs1 = 100.582 val otherCostWs1 = 0.10111 + val totalCostWs1 = storageCostWs1 + otherCostWs1 val storageCostRoundedWs1: BigDecimal = BigDecimal(storageCostWs1).setScale(2, RoundingMode.HALF_EVEN) val otherCostRoundedWs1: BigDecimal = BigDecimal(otherCostWs1).setScale(2, RoundingMode.HALF_EVEN) - val totalCostRoundedWs1: BigDecimal = BigDecimal(storageCostWs1 + otherCostWs1).setScale(2, RoundingMode.HALF_EVEN) + val totalCostRoundedWs1: BigDecimal = BigDecimal(totalCostWs1).setScale(2, RoundingMode.HALF_EVEN) val storageCostWs2 = 20.145 val computeCostWs2 = 150.4033 + val totalCostWs2 = storageCostWs2 + computeCostWs2 val storageCostRoundedWs2: BigDecimal = BigDecimal(storageCostWs2).setScale(2, RoundingMode.HALF_EVEN) val otherCostRoundedWs2: BigDecimal = BigDecimal(computeCostWs2).setScale(2, RoundingMode.HALF_EVEN) val totalCostRoundedWs2: BigDecimal = - BigDecimal(storageCostWs2 + computeCostWs2).setScale(2, RoundingMode.HALF_EVEN) + BigDecimal(totalCostWs2).setScale(2, RoundingMode.HALF_EVEN) val computeCostWs3 = 1111.222 val otherCostWs3 = 0.02 + val totalCostWs3 = otherCostWs3 + computeCostWs3 val storageCostRoundedWs3: BigDecimal = BigDecimal(computeCostWs3).setScale(2, RoundingMode.HALF_EVEN) val otherCostRoundedWs3: BigDecimal = BigDecimal(otherCostWs3).setScale(2, RoundingMode.HALF_EVEN) - val totalCostRoundedWs3: BigDecimal = BigDecimal(computeCostWs3 + otherCostWs3).setScale(2, RoundingMode.HALF_EVEN) + val totalCostRoundedWs3: BigDecimal = BigDecimal(totalCostWs3).setScale(2, RoundingMode.HALF_EVEN) val table: List[Map[String, String]] = List( Map( "storage_cost" -> s"$storageCostWs1", "compute_cost" -> "0.0", "other_cost" -> s"$otherCostWs1", + "total_cost" -> s"$totalCostWs1", "project_id" -> "workspace1ProjectId", "currency" -> "USD" ), @@ -1657,6 +1455,7 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki "storage_cost" -> s"$storageCostWs2", "compute_cost" -> s"$computeCostWs2", "other_cost" -> "0.0", + "total_cost" -> s"$totalCostWs2", "project_id" -> "workspace2ProjectId", "currency" -> "USD" ), @@ -1665,6 +1464,7 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki "compute_cost" -> s"$computeCostWs3", "other_cost" -> s"$otherCostWs3", "project_id" -> "workspace3ProjectId", + "total_cost" -> s"$totalCostWs3", "currency" -> "USD" ) ) @@ -1675,10 +1475,22 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki tableResult.getValues.asScala.toList, DateTime.now().minusDays(1), DateTime.now(), - Map() + Map( + GoogleProjectId("workspace1ProjectId") -> WorkspaceName("workspace1", "namespace1"), + GoogleProjectId("workspace2ProjectId") -> WorkspaceName("workspace2", "namespace1"), + GoogleProjectId("workspace3ProjectId") -> WorkspaceName("workspace3", "namespace2") + ) ) - reportingResults.spendSummary.cost shouldBe TestData.Workspace.totalCostRounded.toString - reportingResults.spendDetails shouldBe empty + + reportingResults.spendDetails.length shouldBe 3 + reportingResults.spendDetails.head.spendData.length shouldBe 1 + reportingResults.spendDetails.head.spendData.head.cost shouldBe totalCostRoundedWs1.toString + + reportingResults.spendDetails(1).spendData.length shouldBe 1 +// reportingResults.spendDetails(1).spendData.head. shouldBe totalCostRoundedWs1.toString + + reportingResults.spendDetails.head.spendData.length shouldBe 1 + reportingResults.spendDetails.head.spendData.head.cost shouldBe totalCostRoundedWs1.toString } "getSpendForAllWorkspaces" should "get the spend report from multiple billing projects" ignore { From 9ad4c7e796dbfa1855ecbd81eba919c58ed0bb15 Mon Sep 17 00:00:00 2001 From: Bria Morgan Date: Thu, 21 Nov 2024 15:38:42 -0500 Subject: [PATCH 17/31] move logic around and expand test --- .../dataaccess/slick/WorkspaceComponent.scala | 17 +++- .../SpendReportingService.scala | 11 +-- .../rawls/workspace/WorkspaceRepository.scala | 22 ++--- .../rawls/workspace/WorkspaceService.scala | 7 +- .../SpendReportingServiceSpec.scala | 84 ++++++++++++++++--- 5 files changed, 99 insertions(+), 42 deletions(-) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceComponent.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceComponent.scala index 0f4dccbf05..53ec801622 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceComponent.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceComponent.scala @@ -7,6 +7,7 @@ import cats.{Monoid, MonoidK} import org.broadinstitute.dsde.rawls.RawlsException import org.broadinstitute.dsde.rawls.model.Attributable.AttributeMap import org.broadinstitute.dsde.rawls.model.WorkspaceState.WorkspaceState +import org.broadinstitute.dsde.rawls.model.WorkspaceType.WorkspaceType import org.broadinstitute.dsde.rawls.model.WorkspaceVersions.WorkspaceVersion import org.broadinstitute.dsde.rawls.model._ import org.broadinstitute.dsde.rawls.util.CollectionUtils @@ -260,8 +261,20 @@ trait WorkspaceComponent { def listWithBillingProject(billingProject: RawlsBillingProjectName): ReadAction[Seq[Workspace]] = workspaceQuery.withBillingProject(billingProject).read - def listWithBillingProjects(billingProjects: List[RawlsBillingProjectName]): ReadAction[Seq[Workspace]] = - workspaceQuery.withBillingProjects(billingProjects).read + def listWithBillingProjectsOfType(billingProjects: List[RawlsBillingProjectName], + workspaceType: WorkspaceType + ): ReadWriteAction[Map[RawlsBillingProjectName, Seq[Workspace]]] = { + val query = for { + workspace <- workspaceQuery if workspace.namespace inSet billingProjects.map(_.value) + if workspace.workspaceType === workspaceType.toString + } yield (workspace.namespace, workspace) + + query.result.map { rows => + rows.groupBy(_._1).map { case (billingProjectName, workspaces) => + RawlsBillingProjectName(billingProjectName) -> workspaces.map(_._2).map(WorkspaceRecord.toWorkspace) + } + } + } def getTags(queryString: Option[String], limit: Option[Int] = None, diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala index 29874a7d59..dc001a81ab 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala @@ -573,15 +573,10 @@ class SpendReportingService( ctx ) billingProjectIds = billingProjectResources.map(resource => RawlsBillingProjectName(resource.resourceId)).toList - groupedWorkspaces <- workspaceServiceConstructor(ctx).getWorkspacesByBillingProjects(billingProjectIds) - gcpOnlyGroupedWorkspaces = groupedWorkspaces - .map { case (key, workspaces) => - key -> workspaces.filter(_.workspaceType == WorkspaceType.RawlsWorkspace) - } - .filter { case (_, workspaces) => workspaces.nonEmpty } + groupedWorkspaces <- workspaceServiceConstructor(ctx).getGCPWorkspacesByBillingProjects(billingProjectIds) // Only use the BPs we know exist in the DB and are GCP - spendConfigs <- getSpendExportConfigurations(gcpOnlyGroupedWorkspaces.keys.toList) + spendConfigs <- getSpendExportConfigurations(groupedWorkspaces.keys.toList) } yield spendConfigs.map { config => - config -> gcpOnlyGroupedWorkspaces(config.billingProjectName).map(ws => (ws.googleProjectId, ws.toWorkspaceName)) + config -> groupedWorkspaces(config.billingProjectName).map(ws => (ws.googleProjectId, ws.toWorkspaceName)) }.toMap } diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala index 2fab668572..221711575b 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala @@ -5,20 +5,9 @@ import org.broadinstitute.dsde.rawls.RawlsExceptionWithErrorReport import org.broadinstitute.dsde.rawls.dataaccess.SlickDataSource import org.broadinstitute.dsde.rawls.dataaccess.slick.PendingBucketDeletionRecord import org.broadinstitute.dsde.rawls.model.Attributable.AttributeMap -import org.broadinstitute.dsde.rawls.model.{ - ErrorReport, - GoogleProjectId, - PendingCloneWorkspaceFileTransfer, - RawlsBillingProjectName, - RawlsRequestContext, - Workspace, - WorkspaceAttributeSpecs, - WorkspaceName, - WorkspaceState, - WorkspaceSubmissionStats, - WorkspaceTag -} +import org.broadinstitute.dsde.rawls.model.{ErrorReport, GoogleProjectId, PendingCloneWorkspaceFileTransfer, RawlsBillingProjectName, RawlsRequestContext, Workspace, WorkspaceAttributeSpecs, WorkspaceName, WorkspaceState, WorkspaceSubmissionStats, WorkspaceTag, WorkspaceType} import org.broadinstitute.dsde.rawls.model.WorkspaceState.WorkspaceState +import org.broadinstitute.dsde.rawls.model.WorkspaceType.WorkspaceType import org.broadinstitute.dsde.rawls.util.TracingUtils.traceDBIOWithParent import org.joda.time.DateTime @@ -70,10 +59,11 @@ class WorkspaceRepository(dataSource: SlickDataSource) { } def listWorkspacesByMultipleBillingProjects( - billingProjectNames: List[RawlsBillingProjectName] - ): Future[Seq[Workspace]] = + billingProjectNames: List[RawlsBillingProjectName], + workspaceType: WorkspaceType + ): Future[Map[RawlsBillingProjectName, Seq[Workspace]]] = dataSource.inTransaction { - _.workspaceQuery.listWithBillingProjects(billingProjectNames) + _.workspaceQuery.listWithBillingProjectsOfType(billingProjectNames, workspaceType) } def createWorkspace(workspace: Workspace): Future[Workspace] = diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala index b152a6f46e..040b72c099 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala @@ -433,13 +433,10 @@ class WorkspaceService( } yield deepFilterJsValue(responseWorkspaces.toJson, options.options) } - // TODO have the DB query do the grouping - def getWorkspacesByBillingProjects( + def getGCPWorkspacesByBillingProjects( billingProjects: List[RawlsBillingProjectName] ): Future[Map[RawlsBillingProjectName, Seq[Workspace]]] = - for { - workspaces <- workspaceRepository.listWorkspacesByMultipleBillingProjects(billingProjects) - } yield workspaces.groupBy(ws => RawlsBillingProjectName(ws.namespace)) + workspaceRepository.listWorkspacesByMultipleBillingProjects(billingProjects, WorkspaceType.RawlsWorkspace) /** Returns the Set of legal field names supplied by the user, trimmed of whitespace. * Throws an error if the user supplied an unrecognized field name. diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala index 49234eead8..72b8acd7db 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala @@ -22,7 +22,7 @@ import org.broadinstitute.dsde.rawls.billing.{ import org.broadinstitute.dsde.rawls.config.SpendReportingServiceConfig import org.broadinstitute.dsde.rawls.dataaccess.{SamDAO, SlickDataSource} import org.broadinstitute.dsde.rawls.model.Attributable.AttributeMap -import org.broadinstitute.dsde.rawls.model._ +import org.broadinstitute.dsde.rawls.model.{SpendReportingAggregationKeys, _} import org.broadinstitute.dsde.rawls.util.MockitoTestUtils import org.broadinstitute.dsde.rawls.{model, RawlsException, RawlsExceptionWithErrorReport, TestExecutionContext} import org.broadinstitute.dsde.workbench.google2.GoogleBigQueryService @@ -36,6 +36,7 @@ import org.scalatest.flatspec.AnyFlatSpecLike import org.scalatest.matchers.should.Matchers import akka.http.scaladsl.model.headers.OAuth2BearerToken import org.broadinstitute.dsde.rawls.mock.MockSamDAO +import org.broadinstitute.dsde.rawls.model.TerraSpendCategories.TerraSpendCategory import java.util.{Date, UUID} import scala.concurrent.duration.Duration @@ -1377,7 +1378,7 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki ) val mockWorkspaceService = mock[WorkspaceService](RETURNS_SMART_NULLS) - when(mockWorkspaceService.getWorkspacesByBillingProjects(any())) + when(mockWorkspaceService.getGCPWorkspacesByBillingProjects(any())) .thenReturn(Future.successful(workspaces)) val mockWorkspaceServiceConstructor: RawlsRequestContext => WorkspaceService = { _ => mockWorkspaceService @@ -1431,14 +1432,14 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki val computeCostWs2 = 150.4033 val totalCostWs2 = storageCostWs2 + computeCostWs2 val storageCostRoundedWs2: BigDecimal = BigDecimal(storageCostWs2).setScale(2, RoundingMode.HALF_EVEN) - val otherCostRoundedWs2: BigDecimal = BigDecimal(computeCostWs2).setScale(2, RoundingMode.HALF_EVEN) + val computeCostRoundedWs2: BigDecimal = BigDecimal(computeCostWs2).setScale(2, RoundingMode.HALF_EVEN) val totalCostRoundedWs2: BigDecimal = BigDecimal(totalCostWs2).setScale(2, RoundingMode.HALF_EVEN) val computeCostWs3 = 1111.222 val otherCostWs3 = 0.02 val totalCostWs3 = otherCostWs3 + computeCostWs3 - val storageCostRoundedWs3: BigDecimal = BigDecimal(computeCostWs3).setScale(2, RoundingMode.HALF_EVEN) + val computeCostRoundedWs3: BigDecimal = BigDecimal(computeCostWs3).setScale(2, RoundingMode.HALF_EVEN) val otherCostRoundedWs3: BigDecimal = BigDecimal(otherCostWs3).setScale(2, RoundingMode.HALF_EVEN) val totalCostRoundedWs3: BigDecimal = BigDecimal(totalCostWs3).setScale(2, RoundingMode.HALF_EVEN) @@ -1482,15 +1483,76 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki ) ) - reportingResults.spendDetails.length shouldBe 3 - reportingResults.spendDetails.head.spendData.length shouldBe 1 - reportingResults.spendDetails.head.spendData.head.cost shouldBe totalCostRoundedWs1.toString + val spendDetails = reportingResults.spendDetails - reportingResults.spendDetails(1).spendData.length shouldBe 1 -// reportingResults.spendDetails(1).spendData.head. shouldBe totalCostRoundedWs1.toString + // We have 3 workspaces + spendDetails.length shouldBe 3 + + // Workspace 1 + spendDetails.head.aggregationKey shouldBe SpendReportingAggregationKeys.Workspace + val ws1SpendData = spendDetails.head.spendData + ws1SpendData.length shouldBe 1 + verifyWorkspaceSpendData(ws1SpendData.head, + totalCostRoundedWs1, + BigDecimal(0.00).setScale(2, RoundingMode.HALF_EVEN), + storageCostRoundedWs1, + otherCostRoundedWs1 + ) + + // Workspace 2 + spendDetails(1).aggregationKey shouldBe SpendReportingAggregationKeys.Workspace + val ws2SpendData = spendDetails(1).spendData + ws2SpendData.length shouldBe 1 + verifyWorkspaceSpendData(ws2SpendData.head, + totalCostRoundedWs2, + computeCostRoundedWs2, + storageCostRoundedWs2, + BigDecimal(0.00).setScale(2, RoundingMode.HALF_EVEN) + ) + + // Workspace 3 + spendDetails(2).aggregationKey shouldBe SpendReportingAggregationKeys.Workspace + val ws3SpendData = spendDetails(2).spendData + ws3SpendData.length shouldBe 1 + verifyWorkspaceSpendData(ws3SpendData.head, + totalCostRoundedWs3, + computeCostRoundedWs3, + BigDecimal(0.00).setScale(2, RoundingMode.HALF_EVEN), + otherCostRoundedWs3 + ) + + } + + def verifyWorkspaceSpendData(actualSpendData: SpendReportingForDateRange, + expectedTotal: BigDecimal, + expectedCompute: BigDecimal, + expectedStorage: BigDecimal, + expectedOther: BigDecimal + ) = { + actualSpendData.cost shouldBe expectedTotal.toString + val aggSub = actualSpendData.subAggregation.get + aggSub.aggregationKey shouldBe SpendReportingAggregationKeys.Category + verifyCategoricalSpendData(aggSub.spendData, expectedCompute, expectedStorage, expectedOther) + } + + def verifyCategoricalSpendData(actualSpendData: Seq[SpendReportingForDateRange], + expectedCompute: BigDecimal, + expectedStorage: BigDecimal, + expectedOther: BigDecimal + ) = { + actualSpendData.length shouldBe 3 + actualSpendData.foreach { spendData => + spendData.category match { + case Some(TerraSpendCategories.Other) => + spendData.cost shouldBe expectedOther.toString + case Some(TerraSpendCategories.Compute) => + spendData.cost shouldBe expectedCompute.toString + case Some(TerraSpendCategories.Storage) => + spendData.cost shouldBe expectedStorage.toString + case _ => fail("Unexpected category") + } + } - reportingResults.spendDetails.head.spendData.length shouldBe 1 - reportingResults.spendDetails.head.spendData.head.cost shouldBe totalCostRoundedWs1.toString } "getSpendForAllWorkspaces" should "get the spend report from multiple billing projects" ignore { From b07bbb1def7fa25ef038cb8dd4a4cd8f5ae95218 Mon Sep 17 00:00:00 2001 From: Bria Morgan Date: Thu, 21 Nov 2024 16:08:02 -0500 Subject: [PATCH 18/31] compare spend report permission to workspace ownership --- .../dsde/rawls/dataaccess/HttpSamDAO.scala | 11 ++--- .../dsde/rawls/dataaccess/SamDAO.scala | 7 +-- .../SpendReportingService.scala | 21 ++++++--- .../dsde/rawls/mock/MockSamDAO.scala | 20 +++++---- .../SpendReportingServiceSpec.scala | 43 ++++++++++++++++++- 5 files changed, 79 insertions(+), 23 deletions(-) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpSamDAO.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpSamDAO.scala index c5f95ed0ab..dbfa0557da 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpSamDAO.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpSamDAO.scala @@ -519,9 +519,10 @@ class HttpSamDAO(baseSamServiceURL: String, rawlsCredential: RawlsCredential, ti } } - override def listResourcesWithActions(resourceTypeName: SamResourceTypeName, - action: SamResourceAction, - ctx: RawlsRequestContext + override def listResourcesWithRolesOrActions(resourceTypeName: SamResourceTypeName, + actions: Seq[SamResourceAction], + roles: Seq[SamResourceRole], + ctx: RawlsRequestContext ): Future[Seq[SamUserResource]] = retry(when401or5xx) { () => val callback = new SamApiCallback[ListResourcesV2200Response]("listResourcesV2") @@ -530,8 +531,8 @@ class HttpSamDAO(baseSamServiceURL: String, rawlsCredential: RawlsCredential, ti /* format = */ "hierarchical", /* resourceTypes = */ util.List.of(resourceTypeName.value), /* policies = */ util.List.of(), - /* roles = */ util.List.of, - /* actions = */ util.List.of(action.value), + /* roles = */ roles.map(_.toString).asJava, + /* actions = */ actions.map(_.toString).asJava, /* includePublic = */ false, callback ) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/SamDAO.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/SamDAO.scala index dd19bb20c7..185e190ccc 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/SamDAO.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/SamDAO.scala @@ -100,9 +100,10 @@ trait SamDAO { def listUserResources(resourceTypeName: SamResourceTypeName, ctx: RawlsRequestContext): Future[Seq[SamUserResource]] - def listResourcesWithActions(resourceTypeName: SamResourceTypeName, - action: SamResourceAction, - ctx: RawlsRequestContext + def listResourcesWithRolesOrActions(resourceTypeName: SamResourceTypeName, + actions: Seq[SamResourceAction], + roles: Seq[SamResourceRole], + ctx: RawlsRequestContext ): Future[Seq[SamUserResource]] def listPoliciesForResource(resourceTypeName: SamResourceTypeName, diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala index dc001a81ab..9353803f19 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala @@ -568,15 +568,26 @@ class SpendReportingService( def getBillingWithSpendPermission( ): Future[Map[BillingProjectSpendExport, Seq[(GoogleProjectId, WorkspaceName)]]] = for { - billingProjectResources <- samDAO.listResourcesWithActions(SamResourceTypeNames.billingProject, - SamBillingProjectActions.readSpendReport, - ctx + billingProjectResources <- samDAO.listResourcesWithRolesOrActions(SamResourceTypeNames.billingProject, + List(SamBillingProjectActions.readSpendReport), + List(), + ctx + ) + ownerWorkspaces <- samDAO.listResourcesWithRolesOrActions( + SamResourceTypeNames.workspace, + List(), + List(SamWorkspaceRoles.owner), // TODO: owner only or owner + project-owner? + ctx ) billingProjectIds = billingProjectResources.map(resource => RawlsBillingProjectName(resource.resourceId)).toList groupedWorkspaces <- workspaceServiceConstructor(ctx).getGCPWorkspacesByBillingProjects(billingProjectIds) + ownerWorkspaceSet = ownerWorkspaces.map(_.resourceId).toSet + filteredGroupedWorkspaces = groupedWorkspaces.map { case (key, workspaces) => + key -> workspaces.filter(ws => ownerWorkspaceSet.contains(ws.name)) + } // Only use the BPs we know exist in the DB and are GCP - spendConfigs <- getSpendExportConfigurations(groupedWorkspaces.keys.toList) + spendConfigs <- getSpendExportConfigurations(filteredGroupedWorkspaces.keys.toList) } yield spendConfigs.map { config => - config -> groupedWorkspaces(config.billingProjectName).map(ws => (ws.googleProjectId, ws.toWorkspaceName)) + config -> filteredGroupedWorkspaces(config.billingProjectName).map(ws => (ws.googleProjectId, ws.toWorkspaceName)) }.toMap } diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/mock/MockSamDAO.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/mock/MockSamDAO.scala index 31e6110cbb..7ee3dbf1c6 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/mock/MockSamDAO.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/mock/MockSamDAO.scala @@ -239,9 +239,10 @@ class MockSamDAO(dataSource: SlickDataSource)(implicit executionContext: Executi case _ => Future.successful(Seq.empty) } - override def listResourcesWithActions(resourceTypeName: SamResourceTypeName, - action: SamResourceAction, - ctx: RawlsRequestContext + override def listResourcesWithRolesOrActions(resourceTypeName: SamResourceTypeName, + actions: Seq[SamResourceAction], + roles: Seq[SamResourceRole], + ctx: RawlsRequestContext ): Future[Seq[SamUserResource]] = resourceTypeName match { case SamResourceTypeNames.workspace => @@ -251,7 +252,7 @@ class MockSamDAO(dataSource: SlickDataSource)(implicit executionContext: Executi _.map(workspace => SamUserResource( workspace.workspaceId, - SamRolesAndActions(Set(SamWorkspaceRoles.owner), Set(action)), + SamRolesAndActions(roles.toSet, actions.toSet), SamRolesAndActions(Set.empty, Set.empty), SamRolesAndActions(Set.empty, Set.empty), Set.empty, @@ -267,7 +268,7 @@ class MockSamDAO(dataSource: SlickDataSource)(implicit executionContext: Executi _.map(project => SamUserResource( project.projectName.value, - SamRolesAndActions(Set(SamBillingProjectRoles.owner), Set(action)), + SamRolesAndActions(roles.toSet, actions.toSet), SamRolesAndActions(Set.empty, Set.empty), SamRolesAndActions(Set.empty, Set.empty), Set.empty, @@ -426,16 +427,17 @@ class CustomizableMockSamDAO(dataSource: SlickDataSource)(implicit executionCont } } - override def listResourcesWithActions(resourceTypeName: SamResourceTypeName, - action: SamResourceAction, - ctx: RawlsRequestContext + override def listResourcesWithRolesOrActions(resourceTypeName: SamResourceTypeName, + actions: Seq[SamResourceAction], + roles: Seq[SamResourceRole], + ctx: RawlsRequestContext ): Future[Seq[SamUserResource]] = { val userResources = for { ((typeName, resourceId), resourcePolicies) <- policies if typeName == resourceTypeName userResource <- constructResourceFromPolicies(ctx, resourceId, resourcePolicies.values) } yield userResource if (userResources.isEmpty) { - super.listResourcesWithActions(resourceTypeName, action, ctx) + super.listResourcesWithRolesOrActions(resourceTypeName, actions, roles, ctx) } else { Future.successful(userResources.toSeq) } diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala index 72b8acd7db..a5f1477c7f 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala @@ -1344,7 +1344,48 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki ) ) .when(samDAO) - .listResourcesWithActions(any(), any(), any()) + .listResourcesWithRolesOrActions(mockitoEq(SamResourceTypeNames.billingProject), any(), any(), any()) + + doReturn( + Future.successful( + Seq( + SamUserResource( + "workspace1Billing1", + SamRolesAndActions(Set.empty, Set.empty), + SamRolesAndActions(Set.empty, Set.empty), + SamRolesAndActions(Set.empty, Set.empty), + Set.empty, + Set.empty + ), + SamUserResource( + "workspace2Billing1", + SamRolesAndActions(Set.empty, Set.empty), + SamRolesAndActions(Set.empty, Set.empty), + SamRolesAndActions(Set.empty, Set.empty), + Set.empty, + Set.empty + ), + SamUserResource( + "workspace1Billing2", + SamRolesAndActions(Set.empty, Set.empty), + SamRolesAndActions(Set.empty, Set.empty), + SamRolesAndActions(Set.empty, Set.empty), + Set.empty, + Set.empty + ), + SamUserResource( + "workspace1Billing3", + SamRolesAndActions(Set.empty, Set.empty), + SamRolesAndActions(Set.empty, Set.empty), + SamRolesAndActions(Set.empty, Set.empty), + Set.empty, + Set.empty + ) + ) + ) + ) + .when(samDAO) + .listResourcesWithRolesOrActions(mockitoEq(SamResourceTypeNames.workspace), any(), any(), any()) val workspace1Billing1 = TestData.workspace("workspace1Billing1", From a0e9897137fa17702462b777a12ee72d62fa78ee Mon Sep 17 00:00:00 2001 From: Bria Morgan Date: Fri, 22 Nov 2024 09:27:12 -0500 Subject: [PATCH 19/31] check own action on workspaces # Conflicts: # core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala --- .../dsde/rawls/spendreporting/SpendReportingService.scala | 6 ++++++ .../rawls/spendreporting/SpendReportingServiceSpec.scala | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala index 9353803f19..8125331a0d 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala @@ -579,6 +579,12 @@ class SpendReportingService( List(SamWorkspaceRoles.owner), // TODO: owner only or owner + project-owner? ctx ) + ownerWorkspaces <- samDAO.listResourcesWithActions( + SamResourceTypeNames.workspace, + SamWorkspaceActions.own, + ctx + ) + billingProjectIds = billingProjectResources.map(resource => RawlsBillingProjectName(resource.resourceId)).toList groupedWorkspaces <- workspaceServiceConstructor(ctx).getGCPWorkspacesByBillingProjects(billingProjectIds) ownerWorkspaceSet = ownerWorkspaces.map(_.resourceId).toSet diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala index a5f1477c7f..d8e4d6ea8e 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala @@ -1344,7 +1344,7 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki ) ) .when(samDAO) - .listResourcesWithRolesOrActions(mockitoEq(SamResourceTypeNames.billingProject), any(), any(), any()) + .listResourcesWithActions(mockitoEq(SamResourceTypeNames.billingProject), any(), any()) doReturn( Future.successful( @@ -1385,7 +1385,7 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki ) ) .when(samDAO) - .listResourcesWithRolesOrActions(mockitoEq(SamResourceTypeNames.workspace), any(), any(), any()) + .listResourcesWithActions(mockitoEq(SamResourceTypeNames.workspace), any(), any()) val workspace1Billing1 = TestData.workspace("workspace1Billing1", From 9da40c0d54531253a5f9b53eabc895548ace0f19 Mon Sep 17 00:00:00 2001 From: Bria Morgan Date: Fri, 22 Nov 2024 09:49:51 -0500 Subject: [PATCH 20/31] fix rebase --- .../dsde/rawls/dataaccess/HttpSamDAO.scala | 11 +++++----- .../dsde/rawls/dataaccess/SamDAO.scala | 7 +++---- .../SpendReportingService.scala | 13 +++--------- .../dsde/rawls/mock/MockSamDAO.scala | 20 +++++++++---------- 4 files changed, 20 insertions(+), 31 deletions(-) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpSamDAO.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpSamDAO.scala index dbfa0557da..838060d7a4 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpSamDAO.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpSamDAO.scala @@ -519,10 +519,9 @@ class HttpSamDAO(baseSamServiceURL: String, rawlsCredential: RawlsCredential, ti } } - override def listResourcesWithRolesOrActions(resourceTypeName: SamResourceTypeName, - actions: Seq[SamResourceAction], - roles: Seq[SamResourceRole], - ctx: RawlsRequestContext + override def listResourcesWithActions(resourceTypeName: SamResourceTypeName, + action: SamResourceAction, + ctx: RawlsRequestContext ): Future[Seq[SamUserResource]] = retry(when401or5xx) { () => val callback = new SamApiCallback[ListResourcesV2200Response]("listResourcesV2") @@ -531,8 +530,8 @@ class HttpSamDAO(baseSamServiceURL: String, rawlsCredential: RawlsCredential, ti /* format = */ "hierarchical", /* resourceTypes = */ util.List.of(resourceTypeName.value), /* policies = */ util.List.of(), - /* roles = */ roles.map(_.toString).asJava, - /* actions = */ actions.map(_.toString).asJava, + /* roles = */ util.List.of(), + /* actions = */ util.List.of(action.value), /* includePublic = */ false, callback ) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/SamDAO.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/SamDAO.scala index 185e190ccc..dd19bb20c7 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/SamDAO.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/SamDAO.scala @@ -100,10 +100,9 @@ trait SamDAO { def listUserResources(resourceTypeName: SamResourceTypeName, ctx: RawlsRequestContext): Future[Seq[SamUserResource]] - def listResourcesWithRolesOrActions(resourceTypeName: SamResourceTypeName, - actions: Seq[SamResourceAction], - roles: Seq[SamResourceRole], - ctx: RawlsRequestContext + def listResourcesWithActions(resourceTypeName: SamResourceTypeName, + action: SamResourceAction, + ctx: RawlsRequestContext ): Future[Seq[SamUserResource]] def listPoliciesForResource(resourceTypeName: SamResourceTypeName, diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala index 8125331a0d..5619e2a9be 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala @@ -568,16 +568,9 @@ class SpendReportingService( def getBillingWithSpendPermission( ): Future[Map[BillingProjectSpendExport, Seq[(GoogleProjectId, WorkspaceName)]]] = for { - billingProjectResources <- samDAO.listResourcesWithRolesOrActions(SamResourceTypeNames.billingProject, - List(SamBillingProjectActions.readSpendReport), - List(), - ctx - ) - ownerWorkspaces <- samDAO.listResourcesWithRolesOrActions( - SamResourceTypeNames.workspace, - List(), - List(SamWorkspaceRoles.owner), // TODO: owner only or owner + project-owner? - ctx + billingProjectResources <- samDAO.listResourcesWithActions(SamResourceTypeNames.billingProject, + SamBillingProjectActions.readSpendReport, + ctx ) ownerWorkspaces <- samDAO.listResourcesWithActions( SamResourceTypeNames.workspace, diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/mock/MockSamDAO.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/mock/MockSamDAO.scala index 7ee3dbf1c6..c6bdb4756b 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/mock/MockSamDAO.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/mock/MockSamDAO.scala @@ -239,10 +239,9 @@ class MockSamDAO(dataSource: SlickDataSource)(implicit executionContext: Executi case _ => Future.successful(Seq.empty) } - override def listResourcesWithRolesOrActions(resourceTypeName: SamResourceTypeName, - actions: Seq[SamResourceAction], - roles: Seq[SamResourceRole], - ctx: RawlsRequestContext + override def listResourcesWithActions(resourceTypeName: SamResourceTypeName, + action: SamResourceAction, + ctx: RawlsRequestContext ): Future[Seq[SamUserResource]] = resourceTypeName match { case SamResourceTypeNames.workspace => @@ -252,7 +251,7 @@ class MockSamDAO(dataSource: SlickDataSource)(implicit executionContext: Executi _.map(workspace => SamUserResource( workspace.workspaceId, - SamRolesAndActions(roles.toSet, actions.toSet), + SamRolesAndActions(Set.empty, Set(action)), SamRolesAndActions(Set.empty, Set.empty), SamRolesAndActions(Set.empty, Set.empty), Set.empty, @@ -268,7 +267,7 @@ class MockSamDAO(dataSource: SlickDataSource)(implicit executionContext: Executi _.map(project => SamUserResource( project.projectName.value, - SamRolesAndActions(roles.toSet, actions.toSet), + SamRolesAndActions(Set.empty, Set(action)), SamRolesAndActions(Set.empty, Set.empty), SamRolesAndActions(Set.empty, Set.empty), Set.empty, @@ -427,17 +426,16 @@ class CustomizableMockSamDAO(dataSource: SlickDataSource)(implicit executionCont } } - override def listResourcesWithRolesOrActions(resourceTypeName: SamResourceTypeName, - actions: Seq[SamResourceAction], - roles: Seq[SamResourceRole], - ctx: RawlsRequestContext + override def listResourcesWithActions(resourceTypeName: SamResourceTypeName, + action: SamResourceAction, + ctx: RawlsRequestContext ): Future[Seq[SamUserResource]] = { val userResources = for { ((typeName, resourceId), resourcePolicies) <- policies if typeName == resourceTypeName userResource <- constructResourceFromPolicies(ctx, resourceId, resourcePolicies.values) } yield userResource if (userResources.isEmpty) { - super.listResourcesWithRolesOrActions(resourceTypeName, actions, roles, ctx) + super.listResourcesWithActions(resourceTypeName, action, ctx) } else { Future.successful(userResources.toSeq) } From 6a0327465a0294b6322bb995386174e514041bb2 Mon Sep 17 00:00:00 2001 From: Bria Morgan Date: Fri, 22 Nov 2024 11:40:11 -0500 Subject: [PATCH 21/31] get workspaces by spend report action --- .../dataaccess/slick/WorkspaceComponent.scala | 6 ++-- .../dsde/rawls/model/SamModel.scala | 1 + .../SpendReportingService.scala | 22 ++++--------- .../rawls/workspace/WorkspaceRepository.scala | 21 +++++++++--- .../rawls/workspace/WorkspaceService.scala | 6 ++-- .../SpendReportingServiceSpec.scala | 33 ------------------- 6 files changed, 32 insertions(+), 57 deletions(-) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceComponent.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceComponent.scala index 53ec801622..bdfe6bb597 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceComponent.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceComponent.scala @@ -261,11 +261,11 @@ trait WorkspaceComponent { def listWithBillingProject(billingProject: RawlsBillingProjectName): ReadAction[Seq[Workspace]] = workspaceQuery.withBillingProject(billingProject).read - def listWithBillingProjectsOfType(billingProjects: List[RawlsBillingProjectName], - workspaceType: WorkspaceType + def groupByBillingProjectOfType(workspaceIds: List[UUID], + workspaceType: WorkspaceType ): ReadWriteAction[Map[RawlsBillingProjectName, Seq[Workspace]]] = { val query = for { - workspace <- workspaceQuery if workspace.namespace inSet billingProjects.map(_.value) + workspace <- workspaceQuery if workspace.id inSet workspaceIds.toSet if workspace.workspaceType === workspaceType.toString } yield (workspace.namespace, workspace) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/model/SamModel.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/model/SamModel.scala index 15d2a7205b..a723881e01 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/model/SamModel.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/model/SamModel.scala @@ -70,6 +70,7 @@ object SamWorkspaceActions { val delete = SamResourceAction("delete") val migrate = SamResourceAction("migrate") val viewMigrationStatus = SamResourceAction("view_migration_status") + val readSpendReport = SamResourceAction("read_spend_report") def sharePolicy(policy: String) = SamResourceAction(s"share_policy::$policy") } diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala index 5619e2a9be..b385151b69 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala @@ -560,7 +560,7 @@ class SpendReportingService( throw RawlsExceptionWithErrorReport( StatusCodes.NotFound, s"no spend data found between dates ${toISODateString(start)} and ${toISODateString(end)}" - ) // TODO update this + ) case rows => extractCrossBillingProjectSpendReportingResults(rows, start, end, projectNames) } } @@ -568,25 +568,17 @@ class SpendReportingService( def getBillingWithSpendPermission( ): Future[Map[BillingProjectSpendExport, Seq[(GoogleProjectId, WorkspaceName)]]] = for { - billingProjectResources <- samDAO.listResourcesWithActions(SamResourceTypeNames.billingProject, - SamBillingProjectActions.readSpendReport, - ctx - ) ownerWorkspaces <- samDAO.listResourcesWithActions( SamResourceTypeNames.workspace, - SamWorkspaceActions.own, + SamWorkspaceActions.readSpendReport, ctx ) - - billingProjectIds = billingProjectResources.map(resource => RawlsBillingProjectName(resource.resourceId)).toList - groupedWorkspaces <- workspaceServiceConstructor(ctx).getGCPWorkspacesByBillingProjects(billingProjectIds) - ownerWorkspaceSet = ownerWorkspaces.map(_.resourceId).toSet - filteredGroupedWorkspaces = groupedWorkspaces.map { case (key, workspaces) => - key -> workspaces.filter(ws => ownerWorkspaceSet.contains(ws.name)) - } + groupedWorkspaces <- workspaceServiceConstructor(ctx).getGCPWorkspacesByBillingProjects( + ownerWorkspaces.map(_.resourceId).toList + ) // Only use the BPs we know exist in the DB and are GCP - spendConfigs <- getSpendExportConfigurations(filteredGroupedWorkspaces.keys.toList) + spendConfigs <- getSpendExportConfigurations(groupedWorkspaces.keys.toList) } yield spendConfigs.map { config => - config -> filteredGroupedWorkspaces(config.billingProjectName).map(ws => (ws.googleProjectId, ws.toWorkspaceName)) + config -> groupedWorkspaces(config.billingProjectName).map(ws => (ws.googleProjectId, ws.toWorkspaceName)) }.toMap } diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala index 221711575b..fa0ca5cc54 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceRepository.scala @@ -5,7 +5,20 @@ import org.broadinstitute.dsde.rawls.RawlsExceptionWithErrorReport import org.broadinstitute.dsde.rawls.dataaccess.SlickDataSource import org.broadinstitute.dsde.rawls.dataaccess.slick.PendingBucketDeletionRecord import org.broadinstitute.dsde.rawls.model.Attributable.AttributeMap -import org.broadinstitute.dsde.rawls.model.{ErrorReport, GoogleProjectId, PendingCloneWorkspaceFileTransfer, RawlsBillingProjectName, RawlsRequestContext, Workspace, WorkspaceAttributeSpecs, WorkspaceName, WorkspaceState, WorkspaceSubmissionStats, WorkspaceTag, WorkspaceType} +import org.broadinstitute.dsde.rawls.model.{ + ErrorReport, + GoogleProjectId, + PendingCloneWorkspaceFileTransfer, + RawlsBillingProjectName, + RawlsRequestContext, + Workspace, + WorkspaceAttributeSpecs, + WorkspaceName, + WorkspaceState, + WorkspaceSubmissionStats, + WorkspaceTag, + WorkspaceType +} import org.broadinstitute.dsde.rawls.model.WorkspaceState.WorkspaceState import org.broadinstitute.dsde.rawls.model.WorkspaceType.WorkspaceType import org.broadinstitute.dsde.rawls.util.TracingUtils.traceDBIOWithParent @@ -58,12 +71,12 @@ class WorkspaceRepository(dataSource: SlickDataSource) { _.workspaceQuery.listWithBillingProject(billingProjectName) } - def listWorkspacesByMultipleBillingProjects( - billingProjectNames: List[RawlsBillingProjectName], + def groupByBillingProjectOfType( + workspaceIds: List[UUID], workspaceType: WorkspaceType ): Future[Map[RawlsBillingProjectName, Seq[Workspace]]] = dataSource.inTransaction { - _.workspaceQuery.listWithBillingProjectsOfType(billingProjectNames, workspaceType) + _.workspaceQuery.groupByBillingProjectOfType(workspaceIds, workspaceType) } def createWorkspace(workspace: Workspace): Future[Workspace] = diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala index 040b72c099..38a304af73 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala @@ -434,9 +434,11 @@ class WorkspaceService( } def getGCPWorkspacesByBillingProjects( - billingProjects: List[RawlsBillingProjectName] + workspaceIds: List[String] ): Future[Map[RawlsBillingProjectName, Seq[Workspace]]] = - workspaceRepository.listWorkspacesByMultipleBillingProjects(billingProjects, WorkspaceType.RawlsWorkspace) + workspaceRepository.groupByBillingProjectOfType(workspaceIds.map(id => UUID.fromString(id)), + WorkspaceType.RawlsWorkspace + ) /** Returns the Set of legal field names supplied by the user, trimmed of whitespace. * Throws an error if the user supplied an unrecognized field name. diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala index d8e4d6ea8e..0051181ceb 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala @@ -1313,39 +1313,6 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki ) val samDAO = mock[SamDAO] - doReturn( - Future.successful( - Seq( - SamUserResource( - "billingProject1", - SamRolesAndActions(Set.empty, Set.empty), - SamRolesAndActions(Set.empty, Set.empty), - SamRolesAndActions(Set.empty, Set.empty), - Set.empty, - Set.empty - ), - SamUserResource( - "billingProject2", - SamRolesAndActions(Set.empty, Set.empty), - SamRolesAndActions(Set.empty, Set.empty), - SamRolesAndActions(Set.empty, Set.empty), - Set.empty, - Set.empty - ), - SamUserResource( - "billingProject3", - SamRolesAndActions(Set.empty, Set.empty), - SamRolesAndActions(Set.empty, Set.empty), - SamRolesAndActions(Set.empty, Set.empty), - Set.empty, - Set.empty - ) - ) - ) - ) - .when(samDAO) - .listResourcesWithActions(mockitoEq(SamResourceTypeNames.billingProject), any(), any()) - doReturn( Future.successful( Seq( From e1e31a1dc5189edcb1b64c4931ffe4852627d1fa Mon Sep 17 00:00:00 2001 From: Bria Morgan Date: Fri, 22 Nov 2024 13:56:57 -0500 Subject: [PATCH 22/31] add pagination --- core/src/main/resources/swagger/api-docs.yaml | 10 ++++++++++ .../spendreporting/SpendReportingService.scala | 14 +++++++++----- .../rawls/webservice/BillingApiServiceV2.scala | 10 +++++++--- .../spendreporting/SpendReportingServiceSpec.scala | 8 +++++--- 4 files changed, 31 insertions(+), 11 deletions(-) diff --git a/core/src/main/resources/swagger/api-docs.yaml b/core/src/main/resources/swagger/api-docs.yaml index 315bcce845..ac05060aa0 100644 --- a/core/src/main/resources/swagger/api-docs.yaml +++ b/core/src/main/resources/swagger/api-docs.yaml @@ -472,6 +472,16 @@ paths: schema: type: string format: date + - name: pageSize + in: query + description: how many workspaces to return at a time + required: false + default: 100 + - name: offset + in: query + description: The number of items to skip before starting to collect the result + required: false + default: 0 responses: 200: description: Success diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala index b385151b69..805347d553 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala @@ -346,7 +346,9 @@ class SpendReportingService( } def getAllUserWorkspaceQuery( - billingProjects: Map[BillingProjectSpendExport, Seq[(GoogleProjectId, WorkspaceName)]] + billingProjects: Map[BillingProjectSpendExport, Seq[(GoogleProjectId, WorkspaceName)]], + pageSize: Int, + offset: Int ): String = { val baseQuery = s""" | SELECT @@ -400,7 +402,7 @@ class SpendReportingService( | currency |ORDER BY | total_cost DESC - |limit 5 + |limit $pageSize offset $offset |""".stripMargin.trim } @@ -542,14 +544,16 @@ class SpendReportingService( def getSpendForAllWorkspaces( start: DateTime, - end: DateTime + end: DateTime, + pageSize: Int, + offset: Int ): Future[SpendReportingResults] = { validateReportParameters(start, end) for { billingMap <- getBillingWithSpendPermission() projectNames: Map[GoogleProjectId, WorkspaceName] = billingMap.values.flatten.toMap - - query = getAllUserWorkspaceQuery(billingMap) + // TODO if there's no workspaces returned, don't run the query + query = getAllUserWorkspaceQuery(billingMap, pageSize, offset) queryJob = setUpAllUserWorkspaceQuery(query, start, end) job: Job <- bigQueryService.use(_.runJob(queryJob)).unsafeToFuture().map(_.waitFor()) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/webservice/BillingApiServiceV2.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/webservice/BillingApiServiceV2.scala index ae44d9f317..1487a82fc7 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/webservice/BillingApiServiceV2.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/webservice/BillingApiServiceV2.scala @@ -64,12 +64,16 @@ trait BillingApiServiceV2 extends UserInfoDirectives { get { parameters( "startDate".as[DateTime], - "endDate".as[DateTime] - ) { (startDate, endDate) => + "endDate".as[DateTime], + "pageSize".as[Int], + "offset".as[Int] + ) { (startDate, endDate, pageSize, offset) => complete { spendReportingConstructor(ctx).getSpendForAllWorkspaces( startDate, - endDate.plusDays(1).minusMillis(1) + endDate.plusDays(1).minusMillis(1), + pageSize, + offset ) } } diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala index 0051181ceb..7618555879 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala @@ -1266,7 +1266,7 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki | currency |ORDER BY | total_cost DESC - |limit 5 + |limit 5 offset 5 |""".stripMargin val service = new SpendReportingService( @@ -1280,7 +1280,9 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki mockWorkspaceServiceConstructor ) val result = service.getAllUserWorkspaceQuery( - inputMap + inputMap, + 5, + 5 ) // It's easier and more reliable to do this than tweak line changes in the query or expected query @@ -1748,7 +1750,7 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki val endDateCapture: ArgumentCaptor[Date] = ArgumentCaptor.forClass(classOf[Date]) val result = Await.result( - service.getSpendForAllWorkspaces(from, to), + service.getSpendForAllWorkspaces(from, to, 100, 0), Duration.Inf ) From e4a13db1231cc3e58eb88f4f02567d59702bda88 Mon Sep 17 00:00:00 2001 From: Bria Morgan Date: Mon, 25 Nov 2024 11:21:08 -0500 Subject: [PATCH 23/31] include credits and deal with edge cases --- .../slick/RawlsBillingProjectComponent.scala | 5 +- .../SpendReportingService.scala | 64 ++-- .../SpendReportingServiceSpec.scala | 336 +++++++++--------- 3 files changed, 213 insertions(+), 192 deletions(-) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/RawlsBillingProjectComponent.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/RawlsBillingProjectComponent.scala index 82059ebd21..d4b580a880 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/RawlsBillingProjectComponent.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/RawlsBillingProjectComponent.scala @@ -17,6 +17,7 @@ import slick.jdbc.JdbcType import java.sql.Timestamp import java.time.Instant import java.util.UUID +import scala.util.Try final case class RawlsBillingProjectRecord(projectName: String, creationStatus: String, @@ -342,6 +343,7 @@ trait RawlsBillingProjectComponent { def clearBillingProjectSpendConfiguration(billingProjectName: RawlsBillingProjectName): WriteAction[Int] = setBillingProjectSpendConfiguration(billingProjectName, None, None, None) + // Throws an error if the Billing Project does not have a Billing Account def getBillingProjectSpendConfiguration( billingProjectName: RawlsBillingProjectName ): ReadAction[Option[BillingProjectSpendExport]] = @@ -350,6 +352,7 @@ trait RawlsBillingProjectComponent { .result .map(_.headOption.map(RawlsBillingProjectRecord.toBillingProjectSpendExport)) + // Ignores any Billing Projects that don't have Billing Accounts def getBillingProjectsSpendConfiguration( billingProjectNames: Seq[RawlsBillingProjectName] ): ReadAction[Seq[Option[BillingProjectSpendExport]]] = @@ -357,7 +360,7 @@ trait RawlsBillingProjectComponent { .withProjectNames(billingProjectNames) .result .map(projectRecords => - projectRecords.map(record => Some(RawlsBillingProjectRecord.toBillingProjectSpendExport(record))) + projectRecords.map(record => Try(RawlsBillingProjectRecord.toBillingProjectSpendExport(record)).toOption) ) def insertOperations(operations: Seq[RawlsBillingProjectOperationRecord]): WriteAction[Unit] = diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala index 805347d553..f53816fbe6 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala @@ -152,6 +152,13 @@ object SpendReportingService { ): SpendReportingResults = { var total = BigDecimal(0.0) + var total_credits = BigDecimal(0.0) + + val currency = allRows.map(_.get("currency").getStringValue).distinct match { + case head :: _ => Currency.getInstance(head) + case _ => throw RawlsExceptionWithErrorReport(StatusCodes.NotFound, "No currencies found for spend data") + } + val all = allRows.map { row => val currencyString = row.get("currency").getStringValue val currencyCode = Currency.getInstance(currencyString) @@ -171,7 +178,7 @@ object SpendReportingService { val subAggregation = List( SpendReportingForDateRange( getRoundedNumericValue("other_cost").toString, - "0.0", + getRoundedNumericValue("credits").toString, currencyCode.toString, Option(start), Option(end), @@ -179,24 +186,26 @@ object SpendReportingService { ), SpendReportingForDateRange( getRoundedNumericValue("storage_cost").toString, - "0.0", + getRoundedNumericValue("credits").toString, currencyCode.toString, category = Option(TerraSpendCategories.Storage) ), SpendReportingForDateRange( getRoundedNumericValue("compute_cost").toString, - "0.0", + getRoundedNumericValue("credits").toString, currencyCode.toString, category = Option(TerraSpendCategories.Compute) ) ) val total_cost = getRoundedNumericValue("total_cost") + val credits = getRoundedNumericValue("credits") total = total + total_cost + total_credits = total_credits + credits val workspaceTotal = SpendReportingForDateRange( total_cost.toString, - "0.0", + credits.toString, currencyCode.toString, Option(start), Option(end), @@ -211,8 +220,8 @@ object SpendReportingService { val summary = SpendReportingForDateRange( total.toString, - "0.0", // TODO - "USD", // TODO + total_credits.toString, + currency.toString, // TODO: what to do about combined summary for currencies? Option(start), Option(end) ) @@ -285,7 +294,6 @@ class SpendReportingService( ) } - // TODO if there is a problem with just one BP, the whole thing fails. def getSpendExportConfigurations(projects: Seq[RawlsBillingProjectName]): Future[Seq[BillingProjectSpendExport]] = dataSource .inTransaction(_.rawlsBillingProjectQuery.getBillingProjectsSpendConfiguration(projects)) @@ -355,6 +363,7 @@ class SpendReportingService( | project.id AS project_id, | project.name AS project_name, | currency, + | SUM(IFNULL((SELECT SUM(c.amount) FROM UNNEST(credits) c), 0)) as credits, | CASE | WHEN service.description IN ('Cloud Storage') THEN 'Storage' | WHEN service.description IN ('Compute Engine', 'Google Kubernetes Engine') THEN 'Compute' @@ -369,8 +378,8 @@ class SpendReportingService( | GROUP BY | project_id, | project_name, - | currency, - | spend_category""".stripMargin.trim + | spend_category, + | currency""".stripMargin.trim val bpSubQuery = billingProjects .map { bp => @@ -393,7 +402,8 @@ class SpendReportingService( | SUM(CASE WHEN spend_category = 'Storage' THEN category_cost ELSE 0 END) AS storage_cost, | SUM(CASE WHEN spend_category = 'Compute' THEN category_cost ELSE 0 END) AS compute_cost, | SUM(CASE WHEN spend_category = 'Other' THEN category_cost ELSE 0 END) AS other_cost, - | currency + | currency, + | SUM(credits) as credits |FROM | spend_categories |GROUP BY @@ -547,12 +557,14 @@ class SpendReportingService( end: DateTime, pageSize: Int, offset: Int - ): Future[SpendReportingResults] = { + ): Future[Option[SpendReportingResults]] = { validateReportParameters(start, end) for { billingMap <- getBillingWithSpendPermission() + _ = if (billingMap.isEmpty) { + return Future.successful(None) + } projectNames: Map[GoogleProjectId, WorkspaceName] = billingMap.values.flatten.toMap - // TODO if there's no workspaces returned, don't run the query query = getAllUserWorkspaceQuery(billingMap, pageSize, offset) queryJob = setUpAllUserWorkspaceQuery(query, start, end) @@ -561,11 +573,8 @@ class SpendReportingService( result = job.getQueryResults() } yield result.getValues.asScala.toList match { case Nil => - throw RawlsExceptionWithErrorReport( - StatusCodes.NotFound, - s"no spend data found between dates ${toISODateString(start)} and ${toISODateString(end)}" - ) - case rows => extractCrossBillingProjectSpendReportingResults(rows, start, end, projectNames) + None + case rows => Some(extractCrossBillingProjectSpendReportingResults(rows, start, end, projectNames)) } } @@ -577,12 +586,23 @@ class SpendReportingService( SamWorkspaceActions.readSpendReport, ctx ) - groupedWorkspaces <- workspaceServiceConstructor(ctx).getGCPWorkspacesByBillingProjects( - ownerWorkspaces.map(_.resourceId).toList - ) + groupedWorkspaces <- + if (ownerWorkspaces.isEmpty) { + Future.successful(Map.empty[RawlsBillingProjectName, Seq[Workspace]]) + } else { + // Ignore non-UUID workspaceIds; these shouldn't happen but if they do, we don't want them + workspaceServiceConstructor(ctx).getGCPWorkspacesByBillingProjects( + ownerWorkspaces.map(_.resourceId).filter(resourceId => Try(UUID.fromString(resourceId)).isSuccess).toList + ) + } // Only use the BPs we know exist in the DB and are GCP - spendConfigs <- getSpendExportConfigurations(groupedWorkspaces.keys.toList) + spendConfigs <- + if (groupedWorkspaces.isEmpty) { + Future.successful(Seq.empty[BillingProjectSpendExport]) + } else { getSpendExportConfigurations(groupedWorkspaces.keys.toList) } } yield spendConfigs.map { config => - config -> groupedWorkspaces(config.billingProjectName).map(ws => (ws.googleProjectId, ws.toWorkspaceName)) + config -> groupedWorkspaces + .getOrElse(RawlsBillingProjectName(config.billingProjectName.value), Seq.empty) + .map(ws => (ws.googleProjectId, ws.toWorkspaceName)) }.toMap } diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala index 7618555879..586cd18440 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala @@ -47,6 +47,7 @@ import org.broadinstitute.dsde.rawls.workspace.WorkspaceService import spray.json.DefaultJsonProtocol._ import spray.json._ import org.broadinstitute.dsde.rawls.model.WorkspaceJsonSupport.WorkspaceListResponseFormat +import org.scalatest.RecoverMethods.recoverToExceptionIf class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with MockitoTestUtils with SprayJsonSupport { @@ -554,40 +555,60 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki } "extractCrossBillingProjectSpendReportingResults" should "break down results by workspace and category" in { - val computeCost1 = 2.4 - val computeCost2 = 0.10111 - val storageCost1 = 0.33 - val storageCost2 = 0.3561 - val otherCost1 = 3.0 - val otherCost2 = 0.0001 - val computeCost1Rounded: BigDecimal = BigDecimal(computeCost1).setScale(2, RoundingMode.HALF_EVEN) - val computeCost2Rounded: BigDecimal = BigDecimal(computeCost2).setScale(2, RoundingMode.HALF_EVEN) - val storageCost1Rounded: BigDecimal = BigDecimal(storageCost1).setScale(2, RoundingMode.HALF_EVEN) - val storageCost2Rounded: BigDecimal = BigDecimal(storageCost2).setScale(2, RoundingMode.HALF_EVEN) - val otherCost1Rounded: BigDecimal = BigDecimal(otherCost1).setScale(2, RoundingMode.HALF_EVEN) - val otherCost2Rounded: BigDecimal = BigDecimal(otherCost2).setScale(2, RoundingMode.HALF_EVEN) - val totalCost1 = computeCost1 + storageCost1 + otherCost1 - val totalCost2 = computeCost2 + storageCost2 + otherCost2 - val totalCostRounded: BigDecimal = BigDecimal(totalCost1 + totalCost2).setScale(2, RoundingMode.HALF_EVEN) + val credits1 = 0.0 + val credits2 = 3.012 + val credits3 = 1.12345 + val storageCostWs1 = 100.582 + val otherCostWs1 = 0.10111 + val totalCostWs1 = storageCostWs1 + otherCostWs1 + val storageCostRoundedWs1: BigDecimal = BigDecimal(storageCostWs1).setScale(2, RoundingMode.HALF_EVEN) + val otherCostRoundedWs1: BigDecimal = BigDecimal(otherCostWs1).setScale(2, RoundingMode.HALF_EVEN) + val totalCostRoundedWs1: BigDecimal = BigDecimal(totalCostWs1).setScale(2, RoundingMode.HALF_EVEN) + val storageCostWs2 = 20.145 + val computeCostWs2 = 150.4033 + val totalCostWs2 = storageCostWs2 + computeCostWs2 + val storageCostRoundedWs2: BigDecimal = BigDecimal(storageCostWs2).setScale(2, RoundingMode.HALF_EVEN) + val computeCostRoundedWs2: BigDecimal = BigDecimal(computeCostWs2).setScale(2, RoundingMode.HALF_EVEN) + val totalCostRoundedWs2: BigDecimal = + BigDecimal(totalCostWs2).setScale(2, RoundingMode.HALF_EVEN) + + val computeCostWs3 = 1111.222 + val otherCostWs3 = 0.02 + val totalCostWs3 = otherCostWs3 + computeCostWs3 + val computeCostRoundedWs3: BigDecimal = BigDecimal(computeCostWs3).setScale(2, RoundingMode.HALF_EVEN) + val otherCostRoundedWs3: BigDecimal = BigDecimal(otherCostWs3).setScale(2, RoundingMode.HALF_EVEN) + val totalCostRoundedWs3: BigDecimal = BigDecimal(totalCostWs3).setScale(2, RoundingMode.HALF_EVEN) val table: List[Map[String, String]] = List( Map( - "storage_cost" -> s"$storageCost1", - "compute_cost" -> s"$computeCost1", - "other_cost" -> s"$otherCost1", - "total_cost" -> s"$totalCost1", + "storage_cost" -> s"$storageCostWs1", + "compute_cost" -> "0.0", + "other_cost" -> s"$otherCostWs1", + "total_cost" -> s"$totalCostWs1", "currency" -> "USD", - "project_id" -> "terra-workspace-project1", - "project_name" -> "terra-billing-project1" + "project_id" -> "workspace1ProjectId", + "project_name" -> "terra-billing-project1", + "credits" -> s"$credits1" ), Map( - "storage_cost" -> s"$storageCost2", - "compute_cost" -> s"$computeCost2", - "other_cost" -> s"$otherCost2", - "total_cost" -> s"$totalCost2", + "storage_cost" -> s"$storageCostWs2", + "compute_cost" -> s"$computeCostWs2", + "other_cost" -> "0.0", + "total_cost" -> s"$totalCostWs2", + "project_id" -> "workspace2ProjectId", + "project_name" -> "terra-billing-project1", "currency" -> "USD", - "project_id" -> "terra-workspace-project2", - "project_name" -> "terra-billing-project2" + "credits" -> s"$credits2" + ), + Map( + "storage_cost" -> "0.0", + "compute_cost" -> s"$computeCostWs3", + "other_cost" -> s"$otherCostWs3", + "project_id" -> "workspace3ProjectId", + "project_name" -> "terra-billing-project2", + "total_cost" -> s"$totalCostWs3", + "currency" -> "USD", + "credits" -> s"$credits3" ) ) @@ -598,15 +619,79 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki DateTime.now().minusDays(1), DateTime.now(), Map( - GoogleProjectId("terra-workspace-project1") -> WorkspaceName("terra-billing-project1", "workspace1"), - GoogleProjectId("terra-workspace-project2") -> WorkspaceName("terra-billing-project2", "workspace2") + GoogleProjectId("workspace1ProjectId") -> WorkspaceName("workspace1", "namespace1"), + GoogleProjectId("workspace2ProjectId") -> WorkspaceName("workspace2", "namespace1"), + GoogleProjectId("workspace3ProjectId") -> WorkspaceName("workspace3", "namespace2") ) ) - reportingResults.spendSummary.cost shouldBe totalCostRounded.toString - reportingResults.spendDetails.head.aggregationKey shouldBe SpendReportingAggregationKeys.Workspace - reportingResults.spendDetails.length shouldBe 2 + val spendDetails = reportingResults.spendDetails + // We have 3 workspaces + spendDetails.length shouldBe 3 + + // Workspace 1 + spendDetails.head.aggregationKey shouldBe SpendReportingAggregationKeys.Workspace + val ws1SpendData = spendDetails.head.spendData + ws1SpendData.length shouldBe 1 + verifyWorkspaceSpendData(ws1SpendData.head, + totalCostRoundedWs1, + BigDecimal(0.00).setScale(2, RoundingMode.HALF_EVEN), + storageCostRoundedWs1, + otherCostRoundedWs1 + ) + + // Workspace 2 + spendDetails(1).aggregationKey shouldBe SpendReportingAggregationKeys.Workspace + val ws2SpendData = spendDetails(1).spendData + ws2SpendData.length shouldBe 1 + verifyWorkspaceSpendData(ws2SpendData.head, + totalCostRoundedWs2, + computeCostRoundedWs2, + storageCostRoundedWs2, + BigDecimal(0.00).setScale(2, RoundingMode.HALF_EVEN) + ) + + // Workspace 3 + spendDetails(2).aggregationKey shouldBe SpendReportingAggregationKeys.Workspace + val ws3SpendData = spendDetails(2).spendData + ws3SpendData.length shouldBe 1 + verifyWorkspaceSpendData(ws3SpendData.head, + totalCostRoundedWs3, + computeCostRoundedWs3, + BigDecimal(0.00).setScale(2, RoundingMode.HALF_EVEN), + otherCostRoundedWs3 + ) } + def verifyWorkspaceSpendData(actualSpendData: SpendReportingForDateRange, + expectedTotal: BigDecimal, + expectedCompute: BigDecimal, + expectedStorage: BigDecimal, + expectedOther: BigDecimal + ) = { + actualSpendData.cost shouldBe expectedTotal.toString + val aggSub = actualSpendData.subAggregation.get + aggSub.aggregationKey shouldBe SpendReportingAggregationKeys.Category + verifyCategoricalSpendData(aggSub.spendData, expectedCompute, expectedStorage, expectedOther) + } + + def verifyCategoricalSpendData(actualSpendData: Seq[SpendReportingForDateRange], + expectedCompute: BigDecimal, + expectedStorage: BigDecimal, + expectedOther: BigDecimal + ) = { + actualSpendData.length shouldBe 3 + actualSpendData.foreach { spendData => + spendData.category match { + case Some(TerraSpendCategories.Other) => + spendData.cost shouldBe expectedOther.toString + case Some(TerraSpendCategories.Compute) => + spendData.cost shouldBe expectedCompute.toString + case Some(TerraSpendCategories.Storage) => + spendData.cost shouldBe expectedStorage.toString + case _ => fail("Unexpected category") + } + } + } "getSpendForGCPBillingProject" should "throw an exception when BQ returns zero rows" in { val samDAO = mock[SamDAO] @@ -1191,6 +1276,7 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki | project.id AS project_id, | project.name AS project_name, | currency, + | SUM(IFNULL((SELECT SUM(c.amount) FROM UNNEST(credits) c), 0)) as credits, | CASE | WHEN service.description IN ('Cloud Storage') THEN 'Storage' | WHEN service.description IN ('Compute Engine', 'Google Kubernetes Engine') THEN 'Compute' @@ -1205,13 +1291,14 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki | GROUP BY | project_id, | project_name, - | currency, - | spend_category + | spend_category, + | currency | UNION ALL | SELECT | project.id AS project_id, | project.name AS project_name, | currency, + | SUM(IFNULL((SELECT SUM(c.amount) FROM UNNEST(credits) c), 0)) as credits, | CASE | WHEN service.description IN ('Cloud Storage') THEN 'Storage' | WHEN service.description IN ('Compute Engine', 'Google Kubernetes Engine') THEN 'Compute' @@ -1226,13 +1313,14 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki | GROUP BY | project_id, | project_name, - | currency, - | spend_category + | spend_category, + | currency | UNION ALL | SELECT | project.id AS project_id, | project.name AS project_name, | currency, + | SUM(IFNULL((SELECT SUM(c.amount) FROM UNNEST(credits) c), 0)) as credits, | CASE | WHEN service.description IN ('Cloud Storage') THEN 'Storage' | WHEN service.description IN ('Compute Engine', 'Google Kubernetes Engine') THEN 'Compute' @@ -1247,8 +1335,8 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki | GROUP BY | project_id, | project_name, - | currency, - | spend_category + | spend_category, + | currency |) |SELECT | project_id, @@ -1257,7 +1345,8 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki | SUM(CASE WHEN spend_category = 'Storage' THEN category_cost ELSE 0 END) AS storage_cost, | SUM(CASE WHEN spend_category = 'Compute' THEN category_cost ELSE 0 END) AS compute_cost, | SUM(CASE WHEN spend_category = 'Other' THEN category_cost ELSE 0 END) AS other_cost, - | currency + | currency, + | SUM(credits) as credits |FROM | spend_categories |GROUP BY @@ -1292,7 +1381,7 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki } "getBillingWithSpendPermission" should "return spendConfigurations for workspaces" in { - // todo: include non-rawls bps + // todo: include non-rawls bps and bps without accounts val dataSource = mock[SlickDataSource] @@ -1430,139 +1519,46 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki } - "extractSpendReportingResultsAcrossBillingProjects" should "should return correct summary data" in { - val storageCostWs1 = 100.582 - val otherCostWs1 = 0.10111 - val totalCostWs1 = storageCostWs1 + otherCostWs1 - val storageCostRoundedWs1: BigDecimal = BigDecimal(storageCostWs1).setScale(2, RoundingMode.HALF_EVEN) - val otherCostRoundedWs1: BigDecimal = BigDecimal(otherCostWs1).setScale(2, RoundingMode.HALF_EVEN) - val totalCostRoundedWs1: BigDecimal = BigDecimal(totalCostWs1).setScale(2, RoundingMode.HALF_EVEN) - - val storageCostWs2 = 20.145 - val computeCostWs2 = 150.4033 - val totalCostWs2 = storageCostWs2 + computeCostWs2 - val storageCostRoundedWs2: BigDecimal = BigDecimal(storageCostWs2).setScale(2, RoundingMode.HALF_EVEN) - val computeCostRoundedWs2: BigDecimal = BigDecimal(computeCostWs2).setScale(2, RoundingMode.HALF_EVEN) - val totalCostRoundedWs2: BigDecimal = - BigDecimal(totalCostWs2).setScale(2, RoundingMode.HALF_EVEN) - - val computeCostWs3 = 1111.222 - val otherCostWs3 = 0.02 - val totalCostWs3 = otherCostWs3 + computeCostWs3 - val computeCostRoundedWs3: BigDecimal = BigDecimal(computeCostWs3).setScale(2, RoundingMode.HALF_EVEN) - val otherCostRoundedWs3: BigDecimal = BigDecimal(otherCostWs3).setScale(2, RoundingMode.HALF_EVEN) - val totalCostRoundedWs3: BigDecimal = BigDecimal(totalCostWs3).setScale(2, RoundingMode.HALF_EVEN) - - val table: List[Map[String, String]] = List( - Map( - "storage_cost" -> s"$storageCostWs1", - "compute_cost" -> "0.0", - "other_cost" -> s"$otherCostWs1", - "total_cost" -> s"$totalCostWs1", - "project_id" -> "workspace1ProjectId", - "currency" -> "USD" - ), - Map( - "storage_cost" -> s"$storageCostWs2", - "compute_cost" -> s"$computeCostWs2", - "other_cost" -> "0.0", - "total_cost" -> s"$totalCostWs2", - "project_id" -> "workspace2ProjectId", - "currency" -> "USD" - ), - Map( - "storage_cost" -> "0.0", - "compute_cost" -> s"$computeCostWs3", - "other_cost" -> s"$otherCostWs3", - "project_id" -> "workspace3ProjectId", - "total_cost" -> s"$totalCostWs3", - "currency" -> "USD" - ) - ) - - val tableResult: TableResult = createTableResult(table) + "getSpendForAllWorkspaces" should "handle no owned workspaces" in { + val from = DateTime.now().minusMonths(2) + val to = from.plusMonths(1) - val reportingResults = SpendReportingService.extractCrossBillingProjectSpendReportingResults( - tableResult.getValues.asScala.toList, - DateTime.now().minusDays(1), - DateTime.now(), - Map( - GoogleProjectId("workspace1ProjectId") -> WorkspaceName("workspace1", "namespace1"), - GoogleProjectId("workspace2ProjectId") -> WorkspaceName("workspace2", "namespace1"), - GoogleProjectId("workspace3ProjectId") -> WorkspaceName("workspace3", "namespace2") - ) - ) + val samDAO = mock[SamDAO](RETURNS_SMART_NULLS) + val billingRepository = mock[BillingRepository](RETURNS_SMART_NULLS) + val bpmDAO = mock[BillingProfileManagerDAO](RETURNS_SMART_NULLS) - val spendDetails = reportingResults.spendDetails + when(samDAO.listResourcesWithActions(any(), any(), any())).thenReturn(Future.successful(List.empty)) - // We have 3 workspaces - spendDetails.length shouldBe 3 + val mockWorkspaceService = mock[WorkspaceService](RETURNS_SMART_NULLS) - // Workspace 1 - spendDetails.head.aggregationKey shouldBe SpendReportingAggregationKeys.Workspace - val ws1SpendData = spendDetails.head.spendData - ws1SpendData.length shouldBe 1 - verifyWorkspaceSpendData(ws1SpendData.head, - totalCostRoundedWs1, - BigDecimal(0.00).setScale(2, RoundingMode.HALF_EVEN), - storageCostRoundedWs1, - otherCostRoundedWs1 - ) + when(mockWorkspaceService.getGCPWorkspacesByBillingProjects(any())) + .thenReturn(Future.successful(Map.empty)) + val mockWorkspaceServiceConstructor: RawlsRequestContext => WorkspaceService = { _ => + mockWorkspaceService + } - // Workspace 2 - spendDetails(1).aggregationKey shouldBe SpendReportingAggregationKeys.Workspace - val ws2SpendData = spendDetails(1).spendData - ws2SpendData.length shouldBe 1 - verifyWorkspaceSpendData(ws2SpendData.head, - totalCostRoundedWs2, - computeCostRoundedWs2, - storageCostRoundedWs2, - BigDecimal(0.00).setScale(2, RoundingMode.HALF_EVEN) - ) + val bigQueryService = mockBigQuery(List[Map[String, String]]()) - // Workspace 3 - spendDetails(2).aggregationKey shouldBe SpendReportingAggregationKeys.Workspace - val ws3SpendData = spendDetails(2).spendData - ws3SpendData.length shouldBe 1 - verifyWorkspaceSpendData(ws3SpendData.head, - totalCostRoundedWs3, - computeCostRoundedWs3, - BigDecimal(0.00).setScale(2, RoundingMode.HALF_EVEN), - otherCostRoundedWs3 + val service = spy( + new SpendReportingService( + testContext, + mock[SlickDataSource], + bigQueryService, + billingRepository, + bpmDAO, + samDAO, + spendReportingServiceConfig, + mockWorkspaceServiceConstructor + ) ) - } - - def verifyWorkspaceSpendData(actualSpendData: SpendReportingForDateRange, - expectedTotal: BigDecimal, - expectedCompute: BigDecimal, - expectedStorage: BigDecimal, - expectedOther: BigDecimal - ) = { - actualSpendData.cost shouldBe expectedTotal.toString - val aggSub = actualSpendData.subAggregation.get - aggSub.aggregationKey shouldBe SpendReportingAggregationKeys.Category - verifyCategoricalSpendData(aggSub.spendData, expectedCompute, expectedStorage, expectedOther) - } - - def verifyCategoricalSpendData(actualSpendData: Seq[SpendReportingForDateRange], - expectedCompute: BigDecimal, - expectedStorage: BigDecimal, - expectedOther: BigDecimal - ) = { - actualSpendData.length shouldBe 3 - actualSpendData.foreach { spendData => - spendData.category match { - case Some(TerraSpendCategories.Other) => - spendData.cost shouldBe expectedOther.toString - case Some(TerraSpendCategories.Compute) => - spendData.cost shouldBe expectedCompute.toString - case Some(TerraSpendCategories.Storage) => - spendData.cost shouldBe expectedStorage.toString - case _ => fail("Unexpected category") - } + val exceptionFuture = recoverToExceptionIf[RawlsExceptionWithErrorReport] { + service.getSpendForAllWorkspaces(from, to, 100, 0) + } + exceptionFuture.map { e => + e.errorReport.statusCode shouldBe Option(StatusCodes.InternalServerError) + e.errorReport.message.contains("no workspaces") shouldBe true } - } "getSpendForAllWorkspaces" should "get the spend report from multiple billing projects" ignore { @@ -1754,13 +1750,15 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki Duration.Inf ) - result.spendSummary.credits shouldBe "0" - result.spendSummary.cost shouldBe Seq(price1, price2).sum.toString() - result.spendSummary.currency shouldBe "USD" - result.spendSummary.startTime.get.toString(ISODateTimeFormat.date()) shouldBe from.toString( + val spendSummary = result.get.spendSummary + + spendSummary.credits shouldBe "0" + spendSummary.cost shouldBe Seq(price1, price2).sum.toString() + spendSummary.currency shouldBe "USD" + spendSummary.startTime.get.toString(ISODateTimeFormat.date()) shouldBe from.toString( ISODateTimeFormat.date() ) - result.spendSummary.endTime.get.toString(ISODateTimeFormat.date()) shouldBe to.toString(ISODateTimeFormat.date()) + spendSummary.endTime.get.toString(ISODateTimeFormat.date()) shouldBe to.toString(ISODateTimeFormat.date()) startDateCapture.getValue shouldBe from.toDate endDateCapture.getValue shouldBe to.toDate From 85ad0f28564cf6bdf1f79c95ddd50e3bfb4ca06a Mon Sep 17 00:00:00 2001 From: Bria Morgan Date: Mon, 25 Nov 2024 15:07:33 -0500 Subject: [PATCH 24/31] start to add trace --- .../SpendReportingService.scala | 93 +++++---- .../SpendReportingServiceSpec.scala | 186 +++++++----------- 2 files changed, 123 insertions(+), 156 deletions(-) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala index f53816fbe6..6db472c513 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala @@ -31,6 +31,7 @@ import scala.concurrent.{ExecutionContext, Future} import scala.jdk.CollectionConverters._ import scala.math.BigDecimal.RoundingMode import org.broadinstitute.dsde.rawls.model.WorkspaceAccessLevels.{Owner, WorkspaceAccessLevel} +import org.broadinstitute.dsde.rawls.util.TracingUtils.traceFutureWithParent import shapeless.syntax.std.tuple.productTupleOps import scala.util.Try @@ -557,52 +558,58 @@ class SpendReportingService( end: DateTime, pageSize: Int, offset: Int - ): Future[Option[SpendReportingResults]] = { - validateReportParameters(start, end) - for { - billingMap <- getBillingWithSpendPermission() - _ = if (billingMap.isEmpty) { - return Future.successful(None) + ): Future[Option[SpendReportingResults]] = + traceFutureWithParent("getSpendForAllWorkspaces", ctx) { childContext => + validateReportParameters(start, end) + for { + billingMap <- getBillingWithSpendPermission(childContext) + _ = if (billingMap.isEmpty) { + return Future.successful(None) + } + projectNames: Map[GoogleProjectId, WorkspaceName] = billingMap.values.flatten.toMap + query = getAllUserWorkspaceQuery(billingMap, pageSize, offset) + queryJob = setUpAllUserWorkspaceQuery(query, start, end) + + job: Job <- bigQueryService.use(_.runJob(queryJob)).unsafeToFuture().map(_.waitFor()) + _ = logSpendQueryStats(job.getStatistics[JobStatistics.QueryStatistics]) + result = job.getQueryResults() + } yield result.getValues.asScala.toList match { + case Nil => + None + case rows => Some(extractCrossBillingProjectSpendReportingResults(rows, start, end, projectNames)) } - projectNames: Map[GoogleProjectId, WorkspaceName] = billingMap.values.flatten.toMap - query = getAllUserWorkspaceQuery(billingMap, pageSize, offset) - queryJob = setUpAllUserWorkspaceQuery(query, start, end) - - job: Job <- bigQueryService.use(_.runJob(queryJob)).unsafeToFuture().map(_.waitFor()) - _ = logSpendQueryStats(job.getStatistics[JobStatistics.QueryStatistics]) - result = job.getQueryResults() - } yield result.getValues.asScala.toList match { - case Nil => - None - case rows => Some(extractCrossBillingProjectSpendReportingResults(rows, start, end, projectNames)) } - } def getBillingWithSpendPermission( + parentContext: RawlsRequestContext ): Future[Map[BillingProjectSpendExport, Seq[(GoogleProjectId, WorkspaceName)]]] = - for { - ownerWorkspaces <- samDAO.listResourcesWithActions( - SamResourceTypeNames.workspace, - SamWorkspaceActions.readSpendReport, - ctx - ) - groupedWorkspaces <- - if (ownerWorkspaces.isEmpty) { - Future.successful(Map.empty[RawlsBillingProjectName, Seq[Workspace]]) - } else { - // Ignore non-UUID workspaceIds; these shouldn't happen but if they do, we don't want them - workspaceServiceConstructor(ctx).getGCPWorkspacesByBillingProjects( - ownerWorkspaces.map(_.resourceId).filter(resourceId => Try(UUID.fromString(resourceId)).isSuccess).toList - ) - } - // Only use the BPs we know exist in the DB and are GCP - spendConfigs <- - if (groupedWorkspaces.isEmpty) { - Future.successful(Seq.empty[BillingProjectSpendExport]) - } else { getSpendExportConfigurations(groupedWorkspaces.keys.toList) } - } yield spendConfigs.map { config => - config -> groupedWorkspaces - .getOrElse(RawlsBillingProjectName(config.billingProjectName.value), Seq.empty) - .map(ws => (ws.googleProjectId, ws.toWorkspaceName)) - }.toMap + traceFutureWithParent("getBillingWithSpendPermission", parentContext) { childContext => + for { + ownerWorkspaces <- samDAO.listResourcesWithActions( + SamResourceTypeNames.workspace, + SamWorkspaceActions.readSpendReport, + childContext + ) + groupedWorkspaces <- + if (ownerWorkspaces.isEmpty) { + Future.successful(Map.empty[RawlsBillingProjectName, Seq[Workspace]]) + } else { + // Ignore non-UUID workspaceIds; these shouldn't happen but if they do, we don't want them + workspaceServiceConstructor(childContext).getGCPWorkspacesByBillingProjects( + ownerWorkspaces.map(_.resourceId).filter(resourceId => Try(UUID.fromString(resourceId)).isSuccess).toList + ) + } + // Only use the BPs we know exist in the DB and are GCP + spendConfigs <- + if (groupedWorkspaces.isEmpty) { + Future.successful(Seq.empty[BillingProjectSpendExport]) + } else { + getSpendExportConfigurations(groupedWorkspaces.keys.toList) + } + } yield spendConfigs.map { config => + config -> groupedWorkspaces + .getOrElse(RawlsBillingProjectName(config.billingProjectName.value), Seq.empty) + .map(ws => (ws.googleProjectId, ws.toWorkspaceName)) + }.toMap + } } diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala index 586cd18440..8561973236 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala @@ -35,8 +35,6 @@ import org.mockito.{ArgumentCaptor, Mockito} import org.scalatest.flatspec.AnyFlatSpecLike import org.scalatest.matchers.should.Matchers import akka.http.scaladsl.model.headers.OAuth2BearerToken -import org.broadinstitute.dsde.rawls.mock.MockSamDAO -import org.broadinstitute.dsde.rawls.model.TerraSpendCategories.TerraSpendCategory import java.util.{Date, UUID} import scala.concurrent.duration.Duration @@ -1503,8 +1501,7 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki ) val result = Await.result( - service.getBillingWithSpendPermission( - ), + service.getBillingWithSpendPermission(testContext), Duration.Inf ) @@ -1561,14 +1558,7 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki } } - "getSpendForAllWorkspaces" should "get the spend report from multiple billing projects" ignore { - val from = DateTime.now().minusMonths(2) - val to = from.plusMonths(1) - - val price1 = BigDecimal("10.22") - val price2 = BigDecimal("50.74") - val currency = "USD" - + "getSpendForAllWorkspaces" should "get the spend report from multiple billing projects" in { val samDAO = mock[SamDAO](RETURNS_SMART_NULLS) val billingRepository = mock[BillingRepository](RETURNS_SMART_NULLS) val bpmDAO = mock[BillingProfileManagerDAO](RETURNS_SMART_NULLS) @@ -1594,16 +1584,6 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki None, billingProfileId = Option.apply(billingProfileId2.toString) ) - val billingProfileId3 = UUID.randomUUID() - val projectName3 = RawlsBillingProjectName("billingProject3") - val billingAccount3 = RawlsBillingAccountName("billingAcct3") - val billingProject3 = RawlsBillingProject( - projectName3, - CreationStatuses.Ready, - Option(billingAccount3), - None, - billingProfileId = Option.apply(billingProfileId3.toString) - ) // Billing project spend exports val billingProject1SpendExport = @@ -1612,9 +1592,9 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki Some("billing1_bq_project.billing1_dataset.billing1_table") ) - val billingProject3SpendExport = - BillingProjectSpendExport(RawlsBillingProjectName("billingProject3"), - RawlsBillingAccountName("billingAccount3"), + val billingProject2SpendExport = + BillingProjectSpendExport(RawlsBillingProjectName("billingProject2"), + RawlsBillingAccountName("billingAccount2"), None ) @@ -1625,86 +1605,23 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki WorkspaceVersions.V1, "billingProject1" ) - val workspace2Billing1 = - TestData.workspace("workspace2Billing1", + val workspace2Billing2 = + TestData.workspace("workspace2Billing2", GoogleProjectId("workspace2ProjectId"), WorkspaceVersions.V2, - "billingProject1" - ) - val workspace1Billing2 = - TestData.workspace("workspace1Billing2", - GoogleProjectId("workspace3ProjectId"), - WorkspaceVersions.V2, "billingProject2" ) - val workspace1Billing3 = - TestData.workspace("workspace1Billing3", - GoogleProjectId("workspace4ProjectId"), - WorkspaceVersions.V2, - "billingProject3" - ) - - // Only workspaces 1 and 4 are owned - val workspace1Response = WorkspaceListResponse( - WorkspaceAccessLevels.Owner, - Some(true), - Some(true), - WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing1, - Option(Set.empty), - true, - Some(WorkspaceCloudPlatform.Gcp) - ), - Option.empty, - false, - Some(List.empty) - ) - val workspace2Response = WorkspaceListResponse( - WorkspaceAccessLevels.Read, - Some(true), - Some(true), - WorkspaceDetails.fromWorkspaceAndOptions(workspace2Billing1, - Option(Set.empty), - true, - Some(WorkspaceCloudPlatform.Gcp) - ), - Option.empty, - false, - Some(List.empty) - ) - val workspace3Response = WorkspaceListResponse( - WorkspaceAccessLevels.Write, - Some(true), - Some(true), - WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing2, - Option(Set.empty), - true, - Some(WorkspaceCloudPlatform.Gcp) - ), - Option.empty, - false, - Some(List.empty) - ) - val workspace4Response = WorkspaceListResponse( - WorkspaceAccessLevels.Owner, - Some(true), - Some(true), - WorkspaceDetails.fromWorkspaceAndOptions(workspace1Billing3, - Option(Set.empty), - true, - Some(WorkspaceCloudPlatform.Gcp) - ), - Option.empty, - false, - Some(List.empty) - ) - val dataSource = mock[SlickDataSource] val mockWorkspaceService = mock[WorkspaceService](RETURNS_SMART_NULLS) - - when(mockWorkspaceService.listWorkspaces(any(), any())) + when(mockWorkspaceService.getGCPWorkspacesByBillingProjects(any())) .thenReturn( - Future.successful(Seq(workspace1Response, workspace2Response, workspace3Response, workspace4Response).toJson) + Future.successful( + Map(billingProject1.projectName -> List(workspace1Billing1), + billingProject2.projectName -> List(workspace2Billing2) + ) + ) ) + val mockWorkspaceServiceConstructor: RawlsRequestContext => WorkspaceService = { _ => mockWorkspaceService } @@ -1713,12 +1630,62 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki .thenReturn(Future.successful(Option.apply(billingProject1))) when(billingRepository.getBillingProject(mockitoEq(projectName2))) .thenReturn(Future.successful(Option.apply(billingProject2))) - when(billingRepository.getBillingProject(mockitoEq(projectName3))) - .thenReturn(Future.successful(Option.apply(billingProject3))) - when(samDAO.userHasAction(any(), any(), any(), any())).thenReturn(Future.successful(true)) + when(samDAO.listResourcesWithActions(any(), any(), any())).thenReturn( + Future.successful( + List( + SamUserResource( + UUID.randomUUID().toString, + SamRolesAndActions(Set.empty, Set.empty), + SamRolesAndActions(Set.empty, Set.empty), + SamRolesAndActions(Set.empty, Set.empty), + Set.empty, + Set.empty + ), + SamUserResource( + UUID.randomUUID().toString, + SamRolesAndActions(Set.empty, Set.empty), + SamRolesAndActions(Set.empty, Set.empty), + SamRolesAndActions(Set.empty, Set.empty), + Set.empty, + Set.empty + ) + ) + ) + ) - val bigQueryService = mockBigQuery(List[Map[String, String]]()) + val from = DateTime.now().minusMonths(2) + val to = from.plusMonths(1) + + val price1 = BigDecimal("10.22") + val price2 = BigDecimal("50.74") + val zero = BigDecimal("0.00") + val total = price1 + price2 + + val table: List[Map[String, String]] = List( + Map( + "storage_cost" -> s"$price1", + "compute_cost" -> s"$zero", + "other_cost" -> s"$price2", + "total_cost" -> s"$total", + "currency" -> "USD", + "project_id" -> "workspace1ProjectId", + "project_name" -> "terra-billing-project1", + "credits" -> s"$zero" + ), + Map( + "storage_cost" -> s"$price2", + "compute_cost" -> s"$zero", + "other_cost" -> s"$zero", + "total_cost" -> s"$price2", + "project_id" -> "workspace2ProjectId", + "project_name" -> "terra-billing-project1", + "currency" -> "USD", + "credits" -> s"$zero" + ) + ) + + val bigQueryService = mockBigQuery(table) val service = spy( new SpendReportingService( @@ -1732,36 +1699,29 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki mockWorkspaceServiceConstructor ) ) - doReturn(Future.successful(Seq(billingProject1SpendExport, billingProject3SpendExport))) + doReturn(Future.successful(Seq(billingProject1SpendExport, billingProject2SpendExport))) .when(service) .getSpendExportConfigurations( any() ) - val spendReport = - TestData.BpmSpendReport.spendData(from, to, currency, Map("Compute" -> price1, "Storage" -> price2)) - - val billingProfileIdCapture: ArgumentCaptor[UUID] = ArgumentCaptor.forClass(classOf[UUID]) - val startDateCapture: ArgumentCaptor[Date] = ArgumentCaptor.forClass(classOf[Date]) - val endDateCapture: ArgumentCaptor[Date] = ArgumentCaptor.forClass(classOf[Date]) - val result = Await.result( service.getSpendForAllWorkspaces(from, to, 100, 0), Duration.Inf ) + result.get.spendDetails.length shouldBe 2 + val spendSummary = result.get.spendSummary - spendSummary.credits shouldBe "0" - spendSummary.cost shouldBe Seq(price1, price2).sum.toString() + spendSummary.credits shouldBe zero.toString() + spendSummary.cost shouldBe (total + price2).toString() spendSummary.currency shouldBe "USD" spendSummary.startTime.get.toString(ISODateTimeFormat.date()) shouldBe from.toString( ISODateTimeFormat.date() ) spendSummary.endTime.get.toString(ISODateTimeFormat.date()) shouldBe to.toString(ISODateTimeFormat.date()) - startDateCapture.getValue shouldBe from.toDate - endDateCapture.getValue shouldBe to.toDate } } From 0c4cbe17c510b00af8dd4012246ba407939e0f0e Mon Sep 17 00:00:00 2001 From: Bria Morgan Date: Mon, 25 Nov 2024 15:27:12 -0500 Subject: [PATCH 25/31] scalafmt --- .../RawlsBillingProjectComponentSpec.scala | 5 ++-- .../rawls/jobexec/SubmissionMonitorSpec.scala | 29 ++++++++++++++----- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/RawlsBillingProjectComponentSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/RawlsBillingProjectComponentSpec.scala index bb961d5e63..3c482e0bf6 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/RawlsBillingProjectComponentSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/RawlsBillingProjectComponentSpec.scala @@ -231,11 +231,12 @@ class RawlsBillingProjectComponentSpec runAndWait(rawlsBillingProjectQuery.getBillingProjectSpendConfiguration(projectName)) } - val actualSpendExports = runAndWait(rawlsBillingProjectQuery.getBillingProjectsSpendConfiguration(billingProjectNames)) + val actualSpendExports = + runAndWait(rawlsBillingProjectQuery.getBillingProjectsSpendConfiguration(billingProjectNames)) actualSpendExports shouldBe expectedSpendExports } - + it should "set statuses properly in ignoreAllOutstanding" in withDefaultTestDatabase { import driver.api._ diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/jobexec/SubmissionMonitorSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/jobexec/SubmissionMonitorSpec.scala index 30a6af9349..50159695f2 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/jobexec/SubmissionMonitorSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/jobexec/SubmissionMonitorSpec.scala @@ -11,7 +11,10 @@ import org.broadinstitute.dsde.rawls.coordination.{DataSourceAccess, Uncoordinat import org.broadinstitute.dsde.rawls.dataaccess._ import org.broadinstitute.dsde.rawls.dataaccess.slick.{TestDriverComponent, WorkflowRecord} import org.broadinstitute.dsde.rawls.expressions.{BoundOutputExpression, OutputExpression} -import org.broadinstitute.dsde.rawls.jobexec.SubmissionMonitorActor.{ExecutionServiceStatusResponse, StatusCheckComplete} +import org.broadinstitute.dsde.rawls.jobexec.SubmissionMonitorActor.{ + ExecutionServiceStatusResponse, + StatusCheckComplete +} import org.broadinstitute.dsde.rawls.metrics.RawlsStatsDTestUtils import org.broadinstitute.dsde.rawls.mock.{MockSamDAO, RemoteServicesMockServer} import org.broadinstitute.dsde.rawls.model._ @@ -1348,14 +1351,26 @@ class SubmissionMonitorSpec(_system: ActorSystem) dataSource: SlickDataSource => val cheapWorkflowId = UUID.randomUUID().toString val expensiveWorkflowId = UUID.randomUUID().toString - val cheapWorkflow = Workflow(Some(cheapWorkflowId), WorkflowStatuses.Submitted, new DateTime(), Some(testData.sample1.toReference), Seq.empty) - val expensiveWorkflow = Workflow(Some(expensiveWorkflowId), WorkflowStatuses.Submitted, new DateTime(), Some(testData.sample2.toReference), Seq.empty) - val submission = testData.submission1.copy(submissionId = UUID.randomUUID().toString, workflows = Seq(cheapWorkflow, expensiveWorkflow)) + val cheapWorkflow = Workflow(Some(cheapWorkflowId), + WorkflowStatuses.Submitted, + new DateTime(), + Some(testData.sample1.toReference), + Seq.empty + ) + val expensiveWorkflow = Workflow(Some(expensiveWorkflowId), + WorkflowStatuses.Submitted, + new DateTime(), + Some(testData.sample2.toReference), + Seq.empty + ) + val submission = testData.submission1.copy(submissionId = UUID.randomUUID().toString, + workflows = Seq(cheapWorkflow, expensiveWorkflow) + ) runAndWait(submissionQuery.create(testData.workspace, submission)) runAndWait(updateWorkflowExecutionServiceKey("unittestdefault")) class CostCapTestExecutionServiceDAO(status: String) extends SubmissionTestExecutionServiceDAO(status) { - override def getCost(id: String, userInfo: UserInfo): Future[WorkflowCostBreakdown] = { + override def getCost(id: String, userInfo: UserInfo): Future[WorkflowCostBreakdown] = if (id.equals(cheapWorkflowId)) { Future.successful(WorkflowCostBreakdown(id, BigDecimal(1), "USD", status, Seq.empty)) } else if (id.equals(expensiveWorkflowId)) { @@ -1363,7 +1378,6 @@ class SubmissionMonitorSpec(_system: ActorSystem) } else { Future.failed(new Exception("Unexpected workflow ID")) } - } } val monitor = createSubmissionMonitor( @@ -1377,7 +1391,8 @@ class SubmissionMonitorSpec(_system: ActorSystem) ) val workflowCosts = await(monitor.queryExecutionServiceForStatus()).statusResponse.collect { - case Success(Some(recordWithOutputs)) => recordWithOutputs._1.externalId.get -> (recordWithOutputs._1.status, recordWithOutputs._1.cost) + case Success(Some(recordWithOutputs)) => + recordWithOutputs._1.externalId.get -> (recordWithOutputs._1.status, recordWithOutputs._1.cost) }.toMap workflowCosts(cheapWorkflowId) shouldEqual (WorkflowStatuses.Running.toString, Option(BigDecimal(1))) From 7d9f3790ac053bb135139654e044ced8361abd3d Mon Sep 17 00:00:00 2001 From: Bria Morgan Date: Tue, 26 Nov 2024 14:39:35 -0500 Subject: [PATCH 26/31] pr comments round 1 --- core/src/main/resources/swagger/api-docs.yaml | 8 +-- .../dsde/rawls/dataaccess/HttpSamDAO.scala | 24 +------- .../dsde/rawls/dataaccess/SamDAO.scala | 3 +- .../dataaccess/slick/WorkspaceComponent.scala | 2 +- .../SpendReportingService.scala | 8 ++- .../dsde/rawls/mock/MockSamDAO.scala | 36 +++-------- .../SpendReportingServiceSpec.scala | 59 +++++-------------- 7 files changed, 36 insertions(+), 104 deletions(-) diff --git a/core/src/main/resources/swagger/api-docs.yaml b/core/src/main/resources/swagger/api-docs.yaml index ac05060aa0..a6b6cdd8e0 100644 --- a/core/src/main/resources/swagger/api-docs.yaml +++ b/core/src/main/resources/swagger/api-docs.yaml @@ -496,13 +496,7 @@ paths: schema: $ref: '#/components/schemas/ErrorReport' 403: - description: You must be a project owner to view the spend report of a project - content: - 'application/json': - schema: - $ref: '#/components/schemas/ErrorReport' - 404: - description: The specified billing project could not be found + description: You must be a workspace owner to view the spend report of a workspace content: 'application/json': schema: diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpSamDAO.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpSamDAO.scala index 838060d7a4..392515ca7a 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpSamDAO.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpSamDAO.scala @@ -16,7 +16,9 @@ import org.broadinstitute.dsde.workbench.client.sam import org.broadinstitute.dsde.workbench.client.sam.api._ import org.broadinstitute.dsde.workbench.client.sam.model.{ FilteredFlatResourcePolicy, + FilteredHierarchicalResource, FilteredHierarchicalResourcePolicy, + FilteredResourcesHierarchicalResponse, ListResourcesV2200Response } import org.broadinstitute.dsde.workbench.client.sam.{ApiCallback, ApiClient, ApiException} @@ -522,7 +524,7 @@ class HttpSamDAO(baseSamServiceURL: String, rawlsCredential: RawlsCredential, ti override def listResourcesWithActions(resourceTypeName: SamResourceTypeName, action: SamResourceAction, ctx: RawlsRequestContext - ): Future[Seq[SamUserResource]] = + ): Future[Seq[FilteredHierarchicalResource]] = retry(when401or5xx) { () => val callback = new SamApiCallback[ListResourcesV2200Response]("listResourcesV2") @@ -540,20 +542,6 @@ class HttpSamDAO(baseSamServiceURL: String, rawlsCredential: RawlsCredential, ti resourcesResponse.getFilteredResourcesHierarchicalResponse .getResources() .asScala - .map { resource => - val resourcePolicies = resource.getPolicies.asScala.toList - val publicPolicies = resourcePolicies.filter(_.getIsPublic) - // might need to filter public policies out of this next statement: - val (inheritedPolicies, directPolicies) = resourcePolicies.partition(_.getInherited) - SamUserResource( - resource.getResourceId, - toSamRolesAndActions(directPolicies), - toSamRolesAndActions(inheritedPolicies), - toSamRolesAndActions(publicPolicies), - resource.getAuthDomainGroups.asScala.map(WorkbenchGroupName).toSet, - resource.getMissingAuthDomainGroups.asScala.map(WorkbenchGroupName).toSet - ) - } .toSeq } } @@ -564,12 +552,6 @@ class HttpSamDAO(baseSamServiceURL: String, rawlsCredential: RawlsCredential, ti rolesAndActions.getActions.asScala.map(SamResourceAction).toSet ) - private def toSamRolesAndActions(policies: scala.collection.immutable.List[FilteredHierarchicalResourcePolicy]) = { - val roles = policies.flatMap(_.getRoles.asScala) - val actions = policies.flatMap(_.getActions.asScala) - SamRolesAndActions(roles.map(role => SamResourceRole(role.toString)).toSet, actions.map(SamResourceAction).toSet) - } - override def getPetServiceAccountKeyForUser(googleProject: GoogleProjectId, userEmail: RawlsUserEmail ): Future[String] = diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/SamDAO.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/SamDAO.scala index dd19bb20c7..754c1cce44 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/SamDAO.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/SamDAO.scala @@ -26,6 +26,7 @@ import org.broadinstitute.dsde.rawls.model.{ UserIdInfo, UserInfo } +import org.broadinstitute.dsde.workbench.client.sam.model.FilteredHierarchicalResource import org.broadinstitute.dsde.workbench.model._ import scala.concurrent.Future @@ -103,7 +104,7 @@ trait SamDAO { def listResourcesWithActions(resourceTypeName: SamResourceTypeName, action: SamResourceAction, ctx: RawlsRequestContext - ): Future[Seq[SamUserResource]] + ): Future[Seq[FilteredHierarchicalResource]] def listPoliciesForResource(resourceTypeName: SamResourceTypeName, resourceId: String, diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceComponent.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceComponent.scala index bdfe6bb597..8e402adf0e 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceComponent.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceComponent.scala @@ -265,7 +265,7 @@ trait WorkspaceComponent { workspaceType: WorkspaceType ): ReadWriteAction[Map[RawlsBillingProjectName, Seq[Workspace]]] = { val query = for { - workspace <- workspaceQuery if workspace.id inSet workspaceIds.toSet + workspace <- workspaceQuery if workspace.id inSetBind workspaceIds.toSet if workspace.workspaceType === workspaceType.toString } yield (workspace.namespace, workspace) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala index 6db472c513..c96f943ad1 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala @@ -152,6 +152,7 @@ object SpendReportingService { names: Map[GoogleProjectId, WorkspaceName] ): SpendReportingResults = { + // Using vars because they will get updated as we process the rows var total = BigDecimal(0.0) var total_credits = BigDecimal(0.0) @@ -181,8 +182,6 @@ object SpendReportingService { getRoundedNumericValue("other_cost").toString, getRoundedNumericValue("credits").toString, currencyCode.toString, - Option(start), - Option(end), category = Option(TerraSpendCategories.Other) ), SpendReportingForDateRange( @@ -596,7 +595,10 @@ class SpendReportingService( } else { // Ignore non-UUID workspaceIds; these shouldn't happen but if they do, we don't want them workspaceServiceConstructor(childContext).getGCPWorkspacesByBillingProjects( - ownerWorkspaces.map(_.resourceId).filter(resourceId => Try(UUID.fromString(resourceId)).isSuccess).toList + ownerWorkspaces + .map(_.getResourceId) + .filter(resourceId => Try(UUID.fromString(resourceId)).isSuccess) + .toList ) } // Only use the BPs we know exist in the DB and are GCP diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/mock/MockSamDAO.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/mock/MockSamDAO.scala index c6bdb4756b..6ccdcd66d3 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/mock/MockSamDAO.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/mock/MockSamDAO.scala @@ -2,6 +2,7 @@ package org.broadinstitute.dsde.rawls.mock import org.broadinstitute.dsde.rawls.dataaccess._ import org.broadinstitute.dsde.rawls.model._ +import org.broadinstitute.dsde.workbench.client.sam.model.FilteredHierarchicalResource import org.broadinstitute.dsde.workbench.model.{WorkbenchEmail, WorkbenchGroupName} import java.util.concurrent.ConcurrentLinkedDeque @@ -242,37 +243,16 @@ class MockSamDAO(dataSource: SlickDataSource)(implicit executionContext: Executi override def listResourcesWithActions(resourceTypeName: SamResourceTypeName, action: SamResourceAction, ctx: RawlsRequestContext - ): Future[Seq[SamUserResource]] = + ): Future[Seq[FilteredHierarchicalResource]] = resourceTypeName match { case SamResourceTypeNames.workspace => dataSource .inTransaction(_ => workspaceQuery.listAll()) .map( _.map(workspace => - SamUserResource( - workspace.workspaceId, - SamRolesAndActions(Set.empty, Set(action)), - SamRolesAndActions(Set.empty, Set.empty), - SamRolesAndActions(Set.empty, Set.empty), - Set.empty, - Set.empty - ) - ) - ) - - case SamResourceTypeNames.billingProject => - dataSource - .inTransaction(_ => rawlsBillingProjectQuery.read) - .map( - _.map(project => - SamUserResource( - project.projectName.value, - SamRolesAndActions(Set.empty, Set(action)), - SamRolesAndActions(Set.empty, Set.empty), - SamRolesAndActions(Set.empty, Set.empty), - Set.empty, - Set.empty - ) + new FilteredHierarchicalResource() + .resourceType(SamResourceTypeNames.workspace.value) + .resourceId(workspace.workspaceId) ) ) @@ -429,7 +409,7 @@ class CustomizableMockSamDAO(dataSource: SlickDataSource)(implicit executionCont override def listResourcesWithActions(resourceTypeName: SamResourceTypeName, action: SamResourceAction, ctx: RawlsRequestContext - ): Future[Seq[SamUserResource]] = { + ): Future[Seq[FilteredHierarchicalResource]] = { val userResources = for { ((typeName, resourceId), resourcePolicies) <- policies if typeName == resourceTypeName userResource <- constructResourceFromPolicies(ctx, resourceId, resourcePolicies.values) @@ -437,7 +417,9 @@ class CustomizableMockSamDAO(dataSource: SlickDataSource)(implicit executionCont if (userResources.isEmpty) { super.listResourcesWithActions(resourceTypeName, action, ctx) } else { - Future.successful(userResources.toSeq) + Future.successful(userResources.map { resource => + new FilteredHierarchicalResource().resourceType(resourceTypeName.value).resourceId(resource.resourceId) + }.toSeq) } } diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala index 8561973236..c09555f9b2 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala @@ -45,6 +45,7 @@ import org.broadinstitute.dsde.rawls.workspace.WorkspaceService import spray.json.DefaultJsonProtocol._ import spray.json._ import org.broadinstitute.dsde.rawls.model.WorkspaceJsonSupport.WorkspaceListResponseFormat +import org.broadinstitute.dsde.workbench.client.sam.model.FilteredHierarchicalResource import org.scalatest.RecoverMethods.recoverToExceptionIf class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with MockitoTestUtils with SprayJsonSupport { @@ -1405,37 +1406,21 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki doReturn( Future.successful( Seq( - SamUserResource( - "workspace1Billing1", - SamRolesAndActions(Set.empty, Set.empty), - SamRolesAndActions(Set.empty, Set.empty), - SamRolesAndActions(Set.empty, Set.empty), - Set.empty, - Set.empty + new FilteredHierarchicalResource( + ).resourceId( + "workspace1Billing1" ), - SamUserResource( - "workspace2Billing1", - SamRolesAndActions(Set.empty, Set.empty), - SamRolesAndActions(Set.empty, Set.empty), - SamRolesAndActions(Set.empty, Set.empty), - Set.empty, - Set.empty + new FilteredHierarchicalResource( + ).resourceId( + "workspace2Billing1" ), - SamUserResource( - "workspace1Billing2", - SamRolesAndActions(Set.empty, Set.empty), - SamRolesAndActions(Set.empty, Set.empty), - SamRolesAndActions(Set.empty, Set.empty), - Set.empty, - Set.empty + new FilteredHierarchicalResource( + ).resourceId( + "workspace1Billing2" ), - SamUserResource( - "workspace1Billing3", - SamRolesAndActions(Set.empty, Set.empty), - SamRolesAndActions(Set.empty, Set.empty), - SamRolesAndActions(Set.empty, Set.empty), - Set.empty, - Set.empty + new FilteredHierarchicalResource( + ).resourceId( + "workspace1Billing3" ) ) ) @@ -1634,22 +1619,8 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki when(samDAO.listResourcesWithActions(any(), any(), any())).thenReturn( Future.successful( List( - SamUserResource( - UUID.randomUUID().toString, - SamRolesAndActions(Set.empty, Set.empty), - SamRolesAndActions(Set.empty, Set.empty), - SamRolesAndActions(Set.empty, Set.empty), - Set.empty, - Set.empty - ), - SamUserResource( - UUID.randomUUID().toString, - SamRolesAndActions(Set.empty, Set.empty), - SamRolesAndActions(Set.empty, Set.empty), - SamRolesAndActions(Set.empty, Set.empty), - Set.empty, - Set.empty - ) + new FilteredHierarchicalResource().resourceId(UUID.randomUUID().toString), + new FilteredHierarchicalResource().resourceId(UUID.randomUUID().toString) ) ) ) From 20dfd8cf29a718019e71d7ee0c017bed80682cf2 Mon Sep 17 00:00:00 2001 From: Bria Morgan Date: Tue, 26 Nov 2024 15:22:38 -0500 Subject: [PATCH 27/31] pr comments round 2 --- .../SpendReportingService.scala | 48 ++++++++++++------- .../SpendReportingServiceSpec.scala | 27 +++++++---- 2 files changed, 49 insertions(+), 26 deletions(-) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala index c96f943ad1..178eee8e51 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala @@ -5,7 +5,7 @@ import akka.http.scaladsl.model.StatusCodes import cats.effect.IO import cats.effect.unsafe.implicits.global import com.google.cloud.bigquery.{JobStatistics, Option => _, _} -import com.google.cloud.bigquery.{Field, FieldList, FieldValue, FieldValueList} +import com.google.cloud.bigquery.FieldValueList import com.typesafe.scalalogging.LazyLogging import nl.grons.metrics4.scala.{Counter, Histogram} import org.broadinstitute.dsde.rawls.billing.{ @@ -18,21 +18,17 @@ import org.broadinstitute.dsde.rawls.dataaccess.{SamDAO, SlickDataSource} import org.broadinstitute.dsde.rawls.metrics.{GoogleInstrumented, HitRatioGauge, RawlsInstrumented} import org.broadinstitute.dsde.rawls.model.{SpendReportingAggregationKeyWithSub, _} import org.broadinstitute.dsde.rawls.spendreporting.SpendReportingService._ -import org.broadinstitute.dsde.rawls.workspace.{AggregatedWorkspace, AggregatedWorkspaceService, WorkspaceService} +import org.broadinstitute.dsde.rawls.workspace.WorkspaceService import org.broadinstitute.dsde.workbench.google2.GoogleBigQueryService import org.broadinstitute.dsde.rawls.{RawlsException, RawlsExceptionWithErrorReport} import org.broadinstitute.dsde.workbench.model.google.GoogleProject import org.joda.time.format.ISODateTimeFormat import org.joda.time.{DateTime, Days} -import spray.json.{JsArray, JsValue} -import org.broadinstitute.dsde.rawls.model.WorkspaceJsonSupport.WorkspaceListResponseFormat import scala.concurrent.{ExecutionContext, Future} import scala.jdk.CollectionConverters._ import scala.math.BigDecimal.RoundingMode -import org.broadinstitute.dsde.rawls.model.WorkspaceAccessLevels.{Owner, WorkspaceAccessLevel} import org.broadinstitute.dsde.rawls.util.TracingUtils.traceFutureWithParent -import shapeless.syntax.std.tuple.productTupleOps import scala.util.Try @@ -156,9 +152,15 @@ object SpendReportingService { var total = BigDecimal(0.0) var total_credits = BigDecimal(0.0) + // TODO: We may want to allow multiple currencies someday val currency = allRows.map(_.get("currency").getStringValue).distinct match { - case head :: _ => Currency.getInstance(head) - case _ => throw RawlsExceptionWithErrorReport(StatusCodes.NotFound, "No currencies found for spend data") + case head :: List() => Currency.getInstance(head) + case head :: tail => + throw RawlsExceptionWithErrorReport( + StatusCodes.InternalServerError, + s"Inconsistent currencies found while aggregating spend data: $head and ${tail.head} cannot be combined" + ) + case List() => throw RawlsExceptionWithErrorReport(StatusCodes.NotFound, "No currencies found for spend data") } val all = allRows.map { row => @@ -180,26 +182,29 @@ object SpendReportingService { val subAggregation = List( SpendReportingForDateRange( getRoundedNumericValue("other_cost").toString, - getRoundedNumericValue("credits").toString, + getRoundedNumericValue("other_credits").toString, currencyCode.toString, category = Option(TerraSpendCategories.Other) ), SpendReportingForDateRange( getRoundedNumericValue("storage_cost").toString, - getRoundedNumericValue("credits").toString, + getRoundedNumericValue("storage_credits").toString, currencyCode.toString, category = Option(TerraSpendCategories.Storage) ), SpendReportingForDateRange( getRoundedNumericValue("compute_cost").toString, - getRoundedNumericValue("credits").toString, + getRoundedNumericValue("compute_credits").toString, currencyCode.toString, category = Option(TerraSpendCategories.Compute) ) ) val total_cost = getRoundedNumericValue("total_cost") - val credits = getRoundedNumericValue("credits") + val credits = + getRoundedNumericValue("other_credits") + getRoundedNumericValue("storage_credits") + getRoundedNumericValue( + "compute_credits" + ) total = total + total_cost total_credits = total_credits + credits @@ -221,7 +226,7 @@ object SpendReportingService { val summary = SpendReportingForDateRange( total.toString, total_credits.toString, - currency.toString, // TODO: what to do about combined summary for currencies? + currency.toString, Option(start), Option(end) ) @@ -403,7 +408,9 @@ class SpendReportingService( | SUM(CASE WHEN spend_category = 'Compute' THEN category_cost ELSE 0 END) AS compute_cost, | SUM(CASE WHEN spend_category = 'Other' THEN category_cost ELSE 0 END) AS other_cost, | currency, - | SUM(credits) as credits + | SUM(CASE WHEN spend_category = 'Storage' THEN credits ELSE 0 END) AS storage_credits, + | SUM(CASE WHEN spend_category = 'Compute' THEN credits ELSE 0 END) AS compute_credits, + | SUM(CASE WHEN spend_category = 'Other' THEN credits ELSE 0 END) AS other_credits, |FROM | spend_categories |GROUP BY @@ -569,9 +576,7 @@ class SpendReportingService( query = getAllUserWorkspaceQuery(billingMap, pageSize, offset) queryJob = setUpAllUserWorkspaceQuery(query, start, end) - job: Job <- bigQueryService.use(_.runJob(queryJob)).unsafeToFuture().map(_.waitFor()) - _ = logSpendQueryStats(job.getStatistics[JobStatistics.QueryStatistics]) - result = job.getQueryResults() + result <- runBigQueryJob(queryJob, childContext) } yield result.getValues.asScala.toList match { case Nil => None @@ -579,6 +584,15 @@ class SpendReportingService( } } + def runBigQueryJob(queryJob: JobInfo, ctx: RawlsRequestContext): Future[TableResult] = + traceFutureWithParent("runBigQueryJob", ctx) { childContext => + for { + job: Job <- bigQueryService.use(_.runJob(queryJob)).unsafeToFuture().map(_.waitFor()) + _ = logSpendQueryStats(job.getStatistics[JobStatistics.QueryStatistics]) + result = job.getQueryResults() + } yield result + } + def getBillingWithSpendPermission( parentContext: RawlsRequestContext ): Future[Map[BillingProjectSpendExport, Seq[(GoogleProjectId, WorkspaceName)]]] = diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala index c09555f9b2..8658d425a2 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala @@ -42,9 +42,6 @@ import scala.concurrent.{Await, Future} import scala.jdk.CollectionConverters._ import scala.math.BigDecimal.RoundingMode import org.broadinstitute.dsde.rawls.workspace.WorkspaceService -import spray.json.DefaultJsonProtocol._ -import spray.json._ -import org.broadinstitute.dsde.rawls.model.WorkspaceJsonSupport.WorkspaceListResponseFormat import org.broadinstitute.dsde.workbench.client.sam.model.FilteredHierarchicalResource import org.scalatest.RecoverMethods.recoverToExceptionIf @@ -587,7 +584,9 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki "currency" -> "USD", "project_id" -> "workspace1ProjectId", "project_name" -> "terra-billing-project1", - "credits" -> s"$credits1" + "storage_credits" -> s"$credits1", + "compute_credits" -> "0.0", + "other_credits" -> "0.0" ), Map( "storage_cost" -> s"$storageCostWs2", @@ -597,7 +596,9 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki "project_id" -> "workspace2ProjectId", "project_name" -> "terra-billing-project1", "currency" -> "USD", - "credits" -> s"$credits2" + "compute_credits" -> s"$credits2", + "storage_credits" -> "0.0", + "other_credits" -> "0.0" ), Map( "storage_cost" -> "0.0", @@ -607,7 +608,9 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki "project_name" -> "terra-billing-project2", "total_cost" -> s"$totalCostWs3", "currency" -> "USD", - "credits" -> s"$credits3" + "other_credits" -> s"$credits3", + "compute_credits" -> "0.0", + "storage_credits" -> "0.0" ) ) @@ -1345,7 +1348,9 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki | SUM(CASE WHEN spend_category = 'Compute' THEN category_cost ELSE 0 END) AS compute_cost, | SUM(CASE WHEN spend_category = 'Other' THEN category_cost ELSE 0 END) AS other_cost, | currency, - | SUM(credits) as credits + | SUM(CASE WHEN spend_category = 'Storage' THEN credits ELSE 0 END) AS storage_credits, + | SUM(CASE WHEN spend_category = 'Compute' THEN credits ELSE 0 END) AS compute_credits, + | SUM(CASE WHEN spend_category = 'Other' THEN credits ELSE 0 END) AS other_credits, |FROM | spend_categories |GROUP BY @@ -1642,7 +1647,9 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki "currency" -> "USD", "project_id" -> "workspace1ProjectId", "project_name" -> "terra-billing-project1", - "credits" -> s"$zero" + "storage_credits" -> s"$zero", + "compute_credits" -> s"$zero", + "other_credits" -> s"$zero" ), Map( "storage_cost" -> s"$price2", @@ -1652,7 +1659,9 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki "project_id" -> "workspace2ProjectId", "project_name" -> "terra-billing-project1", "currency" -> "USD", - "credits" -> s"$zero" + "storage_credits" -> s"$zero", + "compute_credits" -> s"$zero", + "other_credits" -> s"$zero" ) ) From a6b900f78962915beb9b43375a300d8f9b6c0714 Mon Sep 17 00:00:00 2001 From: Bria Morgan Date: Wed, 27 Nov 2024 09:47:27 -0500 Subject: [PATCH 28/31] remove extraneous todo --- .../dsde/rawls/spendreporting/SpendReportingServiceSpec.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala index 8658d425a2..ec61e78bb2 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala @@ -1385,7 +1385,6 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki } "getBillingWithSpendPermission" should "return spendConfigurations for workspaces" in { - // todo: include non-rawls bps and bps without accounts val dataSource = mock[SlickDataSource] From e079f56ba68d9b8cade2e97147694a3e9ab01bda Mon Sep 17 00:00:00 2001 From: Bria Morgan Date: Wed, 4 Dec 2024 09:10:42 -0500 Subject: [PATCH 29/31] include public workspaces --- .../org/broadinstitute/dsde/rawls/dataaccess/HttpSamDAO.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpSamDAO.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpSamDAO.scala index 392515ca7a..d13a3ce037 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpSamDAO.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpSamDAO.scala @@ -534,7 +534,7 @@ class HttpSamDAO(baseSamServiceURL: String, rawlsCredential: RawlsCredential, ti /* policies = */ util.List.of(), /* roles = */ util.List.of(), /* actions = */ util.List.of(action.value), - /* includePublic = */ false, + /* includePublic = */ true, callback ) From 780974fc59186ab491639145912b4ef9ec030e43 Mon Sep 17 00:00:00 2001 From: Bria Morgan Date: Wed, 4 Dec 2024 10:32:18 -0500 Subject: [PATCH 30/31] hierarchical to flat --- .../dsde/rawls/dataaccess/HttpSamDAO.scala | 7 ++++--- .../broadinstitute/dsde/rawls/dataaccess/SamDAO.scala | 4 ++-- .../broadinstitute/dsde/rawls/mock/MockSamDAO.scala | 10 +++++----- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpSamDAO.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpSamDAO.scala index d13a3ce037..45da7fe6c9 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpSamDAO.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpSamDAO.scala @@ -15,6 +15,7 @@ import org.broadinstitute.dsde.rawls.util.{FutureSupport, Retry} import org.broadinstitute.dsde.workbench.client.sam import org.broadinstitute.dsde.workbench.client.sam.api._ import org.broadinstitute.dsde.workbench.client.sam.model.{ + FilteredFlatResource, FilteredFlatResourcePolicy, FilteredHierarchicalResource, FilteredHierarchicalResourcePolicy, @@ -524,12 +525,12 @@ class HttpSamDAO(baseSamServiceURL: String, rawlsCredential: RawlsCredential, ti override def listResourcesWithActions(resourceTypeName: SamResourceTypeName, action: SamResourceAction, ctx: RawlsRequestContext - ): Future[Seq[FilteredHierarchicalResource]] = + ): Future[Seq[FilteredFlatResource]] = retry(when401or5xx) { () => val callback = new SamApiCallback[ListResourcesV2200Response]("listResourcesV2") resourcesApi(ctx).listResourcesV2Async( - /* format = */ "hierarchical", + /* format = */ "flat", /* resourceTypes = */ util.List.of(resourceTypeName.value), /* policies = */ util.List.of(), /* roles = */ util.List.of(), @@ -539,7 +540,7 @@ class HttpSamDAO(baseSamServiceURL: String, rawlsCredential: RawlsCredential, ti ) callback.future.map { resourcesResponse => - resourcesResponse.getFilteredResourcesHierarchicalResponse + resourcesResponse.getFilteredResourcesFlatResponse .getResources() .asScala .toSeq diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/SamDAO.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/SamDAO.scala index 754c1cce44..0a4cead561 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/SamDAO.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/SamDAO.scala @@ -26,7 +26,7 @@ import org.broadinstitute.dsde.rawls.model.{ UserIdInfo, UserInfo } -import org.broadinstitute.dsde.workbench.client.sam.model.FilteredHierarchicalResource +import org.broadinstitute.dsde.workbench.client.sam.model.{FilteredFlatResource, FilteredHierarchicalResource} import org.broadinstitute.dsde.workbench.model._ import scala.concurrent.Future @@ -104,7 +104,7 @@ trait SamDAO { def listResourcesWithActions(resourceTypeName: SamResourceTypeName, action: SamResourceAction, ctx: RawlsRequestContext - ): Future[Seq[FilteredHierarchicalResource]] + ): Future[Seq[FilteredFlatResource]] def listPoliciesForResource(resourceTypeName: SamResourceTypeName, resourceId: String, diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/mock/MockSamDAO.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/mock/MockSamDAO.scala index 6ccdcd66d3..fb3bf25a08 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/mock/MockSamDAO.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/mock/MockSamDAO.scala @@ -2,7 +2,7 @@ package org.broadinstitute.dsde.rawls.mock import org.broadinstitute.dsde.rawls.dataaccess._ import org.broadinstitute.dsde.rawls.model._ -import org.broadinstitute.dsde.workbench.client.sam.model.FilteredHierarchicalResource +import org.broadinstitute.dsde.workbench.client.sam.model.{FilteredFlatResource, FilteredHierarchicalResource} import org.broadinstitute.dsde.workbench.model.{WorkbenchEmail, WorkbenchGroupName} import java.util.concurrent.ConcurrentLinkedDeque @@ -243,14 +243,14 @@ class MockSamDAO(dataSource: SlickDataSource)(implicit executionContext: Executi override def listResourcesWithActions(resourceTypeName: SamResourceTypeName, action: SamResourceAction, ctx: RawlsRequestContext - ): Future[Seq[FilteredHierarchicalResource]] = + ): Future[Seq[FilteredFlatResource]] = resourceTypeName match { case SamResourceTypeNames.workspace => dataSource .inTransaction(_ => workspaceQuery.listAll()) .map( _.map(workspace => - new FilteredHierarchicalResource() + new FilteredFlatResource() .resourceType(SamResourceTypeNames.workspace.value) .resourceId(workspace.workspaceId) ) @@ -409,7 +409,7 @@ class CustomizableMockSamDAO(dataSource: SlickDataSource)(implicit executionCont override def listResourcesWithActions(resourceTypeName: SamResourceTypeName, action: SamResourceAction, ctx: RawlsRequestContext - ): Future[Seq[FilteredHierarchicalResource]] = { + ): Future[Seq[FilteredFlatResource]] = { val userResources = for { ((typeName, resourceId), resourcePolicies) <- policies if typeName == resourceTypeName userResource <- constructResourceFromPolicies(ctx, resourceId, resourcePolicies.values) @@ -418,7 +418,7 @@ class CustomizableMockSamDAO(dataSource: SlickDataSource)(implicit executionCont super.listResourcesWithActions(resourceTypeName, action, ctx) } else { Future.successful(userResources.map { resource => - new FilteredHierarchicalResource().resourceType(resourceTypeName.value).resourceId(resource.resourceId) + new FilteredFlatResource().resourceType(resourceTypeName.value).resourceId(resource.resourceId) }.toSeq) } } From 9dcfd0f280cd23dd0fcec97febfa5a3154d5de5a Mon Sep 17 00:00:00 2001 From: Bria Morgan Date: Wed, 4 Dec 2024 10:38:41 -0500 Subject: [PATCH 31/31] whoops --- .../spendreporting/SpendReportingServiceSpec.scala | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala index ec61e78bb2..c6e61fac66 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala @@ -42,7 +42,7 @@ import scala.concurrent.{Await, Future} import scala.jdk.CollectionConverters._ import scala.math.BigDecimal.RoundingMode import org.broadinstitute.dsde.rawls.workspace.WorkspaceService -import org.broadinstitute.dsde.workbench.client.sam.model.FilteredHierarchicalResource +import org.broadinstitute.dsde.workbench.client.sam.model.FilteredFlatResource import org.scalatest.RecoverMethods.recoverToExceptionIf class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with MockitoTestUtils with SprayJsonSupport { @@ -1410,19 +1410,19 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki doReturn( Future.successful( Seq( - new FilteredHierarchicalResource( + new FilteredFlatResource( ).resourceId( "workspace1Billing1" ), - new FilteredHierarchicalResource( + new FilteredFlatResource( ).resourceId( "workspace2Billing1" ), - new FilteredHierarchicalResource( + new FilteredFlatResource( ).resourceId( "workspace1Billing2" ), - new FilteredHierarchicalResource( + new FilteredFlatResource( ).resourceId( "workspace1Billing3" ) @@ -1623,8 +1623,8 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki when(samDAO.listResourcesWithActions(any(), any(), any())).thenReturn( Future.successful( List( - new FilteredHierarchicalResource().resourceId(UUID.randomUUID().toString), - new FilteredHierarchicalResource().resourceId(UUID.randomUUID().toString) + new FilteredFlatResource().resourceId(UUID.randomUUID().toString), + new FilteredFlatResource().resourceId(UUID.randomUUID().toString) ) ) )