Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

first draft WIP POC #3056

Draft
wants to merge 1 commit into
base: develop
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -419,7 +419,8 @@ object Boot extends IOApp with LazyLogging {
appConfigManager.gcsConfig.map(_.getString("terraBucketWriterRole")).getOrElse("unsupported"),
new RawlsWorkspaceAclManager(samDAO),
new MultiCloudWorkspaceAclManager(workspaceManagerDAO, samDAO, billingProfileManagerDAO, slickDataSource),
fastPassServiceConstructor
fastPassServiceConstructor,
dataRepoDAO
)

val workspaceAdminServiceConstructor: RawlsRequestContext => WorkspaceAdminService =
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package org.broadinstitute.dsde.rawls.dataaccess.datarepo

import akka.http.scaladsl.model.headers.OAuth2BearerToken
import bio.terra.datarepo.model.SnapshotModel
import bio.terra.datarepo.model.{PolicyResponse, SnapshotModel}

import java.util.UUID

Expand All @@ -10,4 +10,8 @@ trait DataRepoDAO {
def getInstanceName: String

def getSnapshot(snapshotId: UUID, accessToken: OAuth2BearerToken): SnapshotModel

def retrieveSnapshotPolicies(snapshotId: UUID, accessToken: OAuth2BearerToken): PolicyResponse

def removeSnapshotPolicy(snapshotId: UUID, member: String, accessToken: OAuth2BearerToken)
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ package org.broadinstitute.dsde.rawls.dataaccess.datarepo
import akka.http.scaladsl.model.headers.OAuth2BearerToken
import bio.terra.datarepo.api.RepositoryApi
import bio.terra.datarepo.client.ApiClient
import bio.terra.datarepo.model.{ColumnModel, SnapshotModel, TableDataType}
import bio.terra.datarepo.model.{ColumnModel, PolicyResponse, SnapshotModel, TableDataType}
import org.broadinstitute.dsde.rawls.entities.datarepo.DataRepoBigQuerySupport

import java.util.stream.Collectors
Expand Down Expand Up @@ -38,4 +38,10 @@ class HttpDataRepoDAO(dataRepoInstanceName: String, dataRepoInstanceBasePath: St
}
snapshot
}

override def retrieveSnapshotPolicies(snapshotId: UUID, accessToken: OAuth2BearerToken): PolicyResponse =
getRepositoryApi(accessToken).retrieveSnapshotPolicies(snapshotId)

override def removeSnapshotPolicy(snapshotId: UUID, member: String, accessToken: OAuth2BearerToken) =
getRepositoryApi(accessToken).deleteSnapshotPolicyMember(snapshotId, "reader", member)
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package org.broadinstitute.dsde.rawls.workspace

import akka.http.scaladsl.model.{StatusCode, StatusCodes}
import akka.stream.Materializer
import bio.terra.datarepo.model.{PolicyResponse, WorkspacePolicyModel}
import bio.terra.workspace.client.ApiException
import bio.terra.workspace.model.WorkspaceDescription
import cats.implicits._
Expand Down Expand Up @@ -47,6 +48,7 @@ import spray.json._
import org.broadinstitute.dsde.rawls.metrics.MetricsHelper
import cats.effect.unsafe.implicits.global
import org.broadinstitute.dsde.rawls.billing.BillingRepository
import org.broadinstitute.dsde.rawls.dataaccess.datarepo.DataRepoDAO

import java.io.IOException
import java.util.UUID
Expand Down Expand Up @@ -81,7 +83,8 @@ object WorkspaceService {
terraBucketWriterRole: String,
rawlsWorkspaceAclManager: RawlsWorkspaceAclManager,
multiCloudWorkspaceAclManager: MultiCloudWorkspaceAclManager,
fastPassServiceConstructor: (RawlsRequestContext, SlickDataSource) => FastPassService
fastPassServiceConstructor: (RawlsRequestContext, SlickDataSource) => FastPassService,
dataRepoDao: DataRepoDAO
)(
ctx: RawlsRequestContext
)(implicit materializer: Materializer, executionContext: ExecutionContext): WorkspaceService =
Expand Down Expand Up @@ -109,6 +112,7 @@ object WorkspaceService {
rawlsWorkspaceAclManager,
multiCloudWorkspaceAclManager,
(context: RawlsRequestContext) => fastPassServiceConstructor(context, dataSource),
dataRepoDao,
new WorkspaceRepository(dataSource),
new BillingRepository(dataSource)
)
Expand Down Expand Up @@ -156,6 +160,7 @@ class WorkspaceService(
rawlsWorkspaceAclManager: RawlsWorkspaceAclManager,
multiCloudWorkspaceAclManager: MultiCloudWorkspaceAclManager,
val fastPassServiceConstructor: RawlsRequestContext => FastPassService,
val dataRepoDao: DataRepoDAO,
val workspaceRepository: WorkspaceRepository,
val billingRepository: BillingRepository
)(implicit protected val executionContext: ExecutionContext)
Expand Down Expand Up @@ -485,6 +490,11 @@ class WorkspaceService(
new RawlsExceptionWithErrorReport(ErrorReport(StatusCodes.InternalServerError, "MC workspaces not supported"))
)
}
val resourceList =
workspaceManagerDAO.enumerateDataRepoSnapshotReferences(workspaceContext.workspaceIdAsUUID, 0, 100, ctx)
val snapshotIds = resourceList.getResources.asScala.flatMap { resource =>
Option(resource.getResourceAttributes.getGcpDataRepoSnapshot).map(_.getSnapshot)
}.toList

for {
// just a simple db operation now - the extra logging is excessive
Expand All @@ -501,6 +511,32 @@ class WorkspaceService(
Future.traverse(workflowsToAbort)(wf => executionServiceCluster.abort(wf, ctx.userInfo))
)

_ <- traceFutureWithParent("getSnapshotPolicies", parentContext)(_ =>
Future
.traverse(snapshotIds) { snapshotId =>
val filteredWorkspaces =
dataRepoDao
.retrieveSnapshotPolicies(UUID.fromString(snapshotId), ctx.userInfo.accessToken)
.getWorkspaces
.asScala
.filter(wsPM => wsPM.getWorkspaceId == workspaceContext.workspaceIdAsUUID)
.toList
logger.info(s"FilteredWorkspaces: $filteredWorkspaces")

Future.successful {
filteredWorkspaces.map { workspace =>
workspace.getWorkspacePolicies.asScala.foreach { wsPolicy =>
wsPolicy.getMembers.asScala.foreach { member =>
logger.info(s"removing policyMember: $member")
dataRepoDao.removeSnapshotPolicy(UUID.fromString(snapshotId), member, ctx.userInfo.accessToken)

}
}
}
}
}
)

_ <- traceFutureWithParent("deleteFastPassGrantsTransaction", parentContext)(childContext =>
fastPassServiceConstructor(childContext).removeFastPassGrantsForWorkspace(workspaceContext)
)
Expand Down Expand Up @@ -533,7 +569,9 @@ class WorkspaceService(
// Delete workflowCollection resource in sam outside of DB transaction
_ <- traceFutureWithParent("deleteWorkflowCollectionSamResource", parentContext)(_ =>
workspaceContext.workflowCollectionName
.map(cn => samDAO.deleteResource(SamResourceTypeNames.workflowCollection, cn, ctx))
.map { cn =>
samDAO.deleteResource(SamResourceTypeNames.workflowCollection, cn, ctx)
}
.getOrElse(Future.successful(())) recoverWith {
case t: RawlsExceptionWithErrorReport if t.errorReport.statusCode.contains(StatusCodes.NotFound) =>
logger.warn(
Expand Down
Loading