Skip to content

Commit

Permalink
wip: sanction check execution.
Browse files Browse the repository at this point in the history
  • Loading branch information
Antoine Popineau committed Jan 16, 2025
1 parent 8bc794e commit 9352496
Show file tree
Hide file tree
Showing 12 changed files with 340 additions and 76 deletions.
11 changes: 11 additions & 0 deletions models/sanction_check.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
package models

type OpenSanctionCheckFilter map[string][]string

type OpenSanctionsQuery struct {
Queries OpenSanctionCheckFilter `json:"queries"`
}

type SanctionCheckResult struct {
Hits int
}
4 changes: 4 additions & 0 deletions repositories/eval_scenario_testrun.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,10 @@ type EvalScenarioRepository interface {
GetScenarioIteration(ctx context.Context, exec Executor, scenarioIterationId string) (models.ScenarioIteration, error)
}

type EvalSanctionCheckConfigRepository interface {
GetSanctionCheckConfig(ctx context.Context, exec Executor, scenarioIterationId string) (models.SanctionCheckConfig, error)
}

type EvalTestRunScenarioRepository interface {
GetTestRunIterationIdByScenarioId(ctx context.Context, exec Executor, scenarioID string) (*string, error)
}
108 changes: 108 additions & 0 deletions repositories/opensanctions_repository.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
package repositories

import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"

"github.com/checkmarble/marble-backend/models"
"github.com/checkmarble/marble-backend/utils"
"github.com/cockroachdb/errors"
"github.com/google/uuid"
)

const (
// TODO: Pull this as server configuration
DEV_YENTE_URL = "http://app.yente.orb.local"
)

type OpenSanctionsRepository struct{}

type openSanctionsRequest struct {
Queries map[string]openSanctionsRequestQuery `json:"queries"`
}

type openSanctionsRequestQuery struct {
Schema string `json:"schema"`
Properties models.OpenSanctionCheckFilter `json:"properties"`
}

type OpenSanctionsCheckResponse struct {
Responses map[string]struct {
Total struct {
Value int `json:"value"`
} `json:"total"`
Results []struct {
Id string `json:"id"`
} `json:"results"`
} `json:"responses"`
}

func (repo OpenSanctionsRepository) Search(ctx context.Context, cfg models.SanctionCheckConfig,
query models.OpenSanctionsQuery,
) (models.SanctionCheckResult, error) {
req, err := repo.searchRequest(ctx, query)
if err != nil {
return models.SanctionCheckResult{}, err
}

utils.LoggerFromContext(ctx).Debug("SANCTION CHECK: sending request...")

resp, err := http.DefaultClient.Do(req)
if err != nil {
return models.SanctionCheckResult{}, errors.Wrap(err, "could not perform sanction check")
}

if resp.StatusCode != http.StatusOK {
return models.SanctionCheckResult{}, fmt.Errorf(
"sanction check API returned status %d", resp.StatusCode)
}

var matches OpenSanctionsCheckResponse

defer resp.Body.Close()

if err := json.NewDecoder(resp.Body).Decode(&matches); err != nil {
return models.SanctionCheckResult{}, errors.Wrap(err,
"could not parse sanction check response")
}

// TODO: Replace with actual processing of responses
hits := 0

for _, response := range matches.Responses {
hits += response.Total.Value
}

result := models.SanctionCheckResult{
Hits: hits,
}

return result, nil
}

func (OpenSanctionsRepository) searchRequest(ctx context.Context, query models.OpenSanctionsQuery) (*http.Request, error) {
q := openSanctionsRequest{
Queries: make(map[string]openSanctionsRequestQuery, len(query.Queries)),
}

for key, value := range query.Queries {
q.Queries[uuid.NewString()] = openSanctionsRequestQuery{
Schema: "Thing",
Properties: map[string][]string{key: value},
}
}

var body bytes.Buffer

if err := json.NewEncoder(&body).Encode(q); err != nil {
return nil, errors.Wrap(err, "could not parse OpenSanctions response")
}

url := fmt.Sprintf("%s/match/sanctions", DEV_YENTE_URL)
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, &body)

return req, err
}
16 changes: 16 additions & 0 deletions repositories/sanction_check_repository.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package repositories

import (
"context"

"github.com/checkmarble/marble-backend/models"
"github.com/checkmarble/marble-backend/utils"
)

func (*MarbleDbRepository) InsertResults(ctx context.Context, matches models.SanctionCheckResult) (models.SanctionCheckResult, error) {
utils.LoggerFromContext(ctx).Debug("SANCTION CHECK: inserting matches in database")

return models.SanctionCheckResult{
Hits: matches.Hits,
}, nil
}
56 changes: 30 additions & 26 deletions usecases/decision_phantom/decision_phantom.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,16 @@ type evalScenarioRepository interface {
scenarioIterationId string) (models.ScenarioIteration, error)
}
type PhantomDecisionUsecase struct {
enforceSecurity security.EnforceSecurityPhantomDecision
executorFactory executor_factory.ExecutorFactory
ingestedDataReadRepository repositories.IngestedDataReadRepository
repository repositories.DecisionPhantomUsecaseRepository
testrunRepository repositories.ScenarioTestRunRepository
scenarioRepository repositories.ScenarioUsecaseRepository
evaluateAstExpression ast_eval.EvaluateAstExpression
snoozesReader evaluate_scenario.SnoozesForDecisionReader
evalScenarioRepository evalScenarioRepository
enforceSecurity security.EnforceSecurityPhantomDecision
executorFactory executor_factory.ExecutorFactory
ingestedDataReadRepository repositories.IngestedDataReadRepository
repository repositories.DecisionPhantomUsecaseRepository
testrunRepository repositories.ScenarioTestRunRepository
scenarioRepository repositories.ScenarioUsecaseRepository
evaluateAstExpression ast_eval.EvaluateAstExpression
snoozesReader evaluate_scenario.SnoozesForDecisionReader
evalScenarioRepository evalScenarioRepository
evalSanctionCheckConfigRepository repositories.EvalSanctionCheckConfigRepository
}

func NewPhantomDecisionUseCase(enforceSecurity security.EnforceSecurityPhantomDecision,
Expand All @@ -38,17 +39,19 @@ func NewPhantomDecisionUseCase(enforceSecurity security.EnforceSecurityPhantomDe
testrunRepository repositories.ScenarioTestRunRepository,
scenarioRepository repositories.ScenarioUsecaseRepository,
evalScenarioRepository evalScenarioRepository,
evalSanctionCheckConfigRepository repositories.EvalSanctionCheckConfigRepository,
) PhantomDecisionUsecase {
return PhantomDecisionUsecase{
enforceSecurity: enforceSecurity,
executorFactory: executorFactory,
ingestedDataReadRepository: ingestedDataReadRepository,
repository: repository,
scenarioRepository: scenarioRepository,
evaluateAstExpression: evaluateAstExpression,
testrunRepository: testrunRepository,
snoozesReader: snoozesReader,
evalScenarioRepository: evalScenarioRepository,
enforceSecurity: enforceSecurity,
executorFactory: executorFactory,
ingestedDataReadRepository: ingestedDataReadRepository,
repository: repository,
scenarioRepository: scenarioRepository,
evaluateAstExpression: evaluateAstExpression,
testrunRepository: testrunRepository,
snoozesReader: snoozesReader,
evalScenarioRepository: evalScenarioRepository,
evalSanctionCheckConfigRepository: evalSanctionCheckConfigRepository,
}
}

Expand All @@ -66,14 +69,15 @@ func (usecase *PhantomDecisionUsecase) CreatePhantomDecision(ctx context.Context
return models.PhantomDecision{}, err
}
evaluationRepositories := evaluate_scenario.ScenarioEvaluationRepositories{
EvalScenarioRepository: usecase.evalScenarioRepository,
EvalTestRunScenarioRepository: usecase.repository,
ScenarioTestRunRepository: usecase.testrunRepository,
ExecutorFactory: usecase.executorFactory,
IngestedDataReadRepository: usecase.ingestedDataReadRepository,
EvaluateAstExpression: usecase.evaluateAstExpression,
ScenarioRepository: usecase.scenarioRepository,
SnoozeReader: usecase.snoozesReader,
EvalScenarioRepository: usecase.evalScenarioRepository,
EvalSanctionCheckConfigRepository: usecase.evalSanctionCheckConfigRepository,
EvalTestRunScenarioRepository: usecase.repository,
ScenarioTestRunRepository: usecase.testrunRepository,
ExecutorFactory: usecase.executorFactory,
IngestedDataReadRepository: usecase.ingestedDataReadRepository,
EvaluateAstExpression: usecase.evaluateAstExpression,
ScenarioRepository: usecase.scenarioRepository,
SnoozeReader: usecase.snoozesReader,
}

// TODO remove
Expand Down
51 changes: 28 additions & 23 deletions usecases/decision_usecase.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,19 +98,21 @@ type snoozesForDecisionReader interface {
}

type DecisionUsecase struct {
enforceSecurity security.EnforceSecurityDecision
enforceSecurityScenario security.EnforceSecurityScenario
transactionFactory executor_factory.TransactionFactory
executorFactory executor_factory.ExecutorFactory
ingestedDataReadRepository repositories.IngestedDataReadRepository
dataModelRepository repositories.DataModelRepository
repository DecisionUsecaseRepository
scenarioTestRunRepository repositories.ScenarioTestRunRepository
evaluateAstExpression ast_eval.EvaluateAstExpression
decisionWorkflows decisionWorkflowsUsecase
webhookEventsSender webhookEventsUsecase
phantomUseCase decision_phantom.PhantomDecisionUsecase
snoozesReader snoozesForDecisionReader
enforceSecurity security.EnforceSecurityDecision
enforceSecurityScenario security.EnforceSecurityScenario
transactionFactory executor_factory.TransactionFactory
executorFactory executor_factory.ExecutorFactory
ingestedDataReadRepository repositories.IngestedDataReadRepository
dataModelRepository repositories.DataModelRepository
repository DecisionUsecaseRepository
sanctionCheckConfigRepository repositories.EvalSanctionCheckConfigRepository
sanctionCheckUsecase SanctionCheckUsecase
scenarioTestRunRepository repositories.ScenarioTestRunRepository
evaluateAstExpression ast_eval.EvaluateAstExpression
decisionWorkflows decisionWorkflowsUsecase
webhookEventsSender webhookEventsUsecase
phantomUseCase decision_phantom.PhantomDecisionUsecase
snoozesReader snoozesForDecisionReader
}

func (usecase *DecisionUsecase) GetDecision(ctx context.Context, decisionId string) (models.DecisionWithRuleExecutions, error) {
Expand Down Expand Up @@ -400,11 +402,13 @@ func (usecase *DecisionUsecase) CreateDecision(
}

evaluationRepositories := evaluate_scenario.ScenarioEvaluationRepositories{
EvalScenarioRepository: usecase.repository,
ExecutorFactory: usecase.executorFactory,
IngestedDataReadRepository: usecase.ingestedDataReadRepository,
EvaluateAstExpression: usecase.evaluateAstExpression,
SnoozeReader: usecase.snoozesReader,
EvalScenarioRepository: usecase.repository,
EvalSanctionCheckConfigRepository: usecase.sanctionCheckConfigRepository,
EvalSanctionCheckUsecase: usecase.sanctionCheckUsecase,
ExecutorFactory: usecase.executorFactory,
IngestedDataReadRepository: usecase.ingestedDataReadRepository,
EvaluateAstExpression: usecase.evaluateAstExpression,
SnoozeReader: usecase.snoozesReader,
}

scenarioExecution, err := evaluate_scenario.EvalScenario(ctx, evaluationParameters, evaluationRepositories)
Expand Down Expand Up @@ -553,11 +557,12 @@ func (usecase *DecisionUsecase) CreateAllDecisions(
}

evaluationRepositories := evaluate_scenario.ScenarioEvaluationRepositories{
EvalScenarioRepository: usecase.repository,
ExecutorFactory: usecase.executorFactory,
IngestedDataReadRepository: usecase.ingestedDataReadRepository,
EvaluateAstExpression: usecase.evaluateAstExpression,
SnoozeReader: usecase.snoozesReader,
EvalScenarioRepository: usecase.repository,
EvalSanctionCheckConfigRepository: usecase.sanctionCheckConfigRepository,
ExecutorFactory: usecase.executorFactory,
IngestedDataReadRepository: usecase.ingestedDataReadRepository,
EvaluateAstExpression: usecase.evaluateAstExpression,
SnoozeReader: usecase.snoozesReader,
}

type decisionAndScenario struct {
Expand Down
55 changes: 47 additions & 8 deletions usecases/evaluate_scenario/evaluate_scenario.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,10 @@ type ScenarioEvaluationParameters struct {
Pivot *models.Pivot
}

type EvalSanctionCheckUsecase interface {
Execute(context.Context, models.SanctionCheckConfig, models.OpenSanctionsQuery) (models.SanctionCheckResult, error)
}

type SnoozesForDecisionReader interface {
ListActiveRuleSnoozesForDecision(
ctx context.Context,
Expand All @@ -42,14 +46,16 @@ type SnoozesForDecisionReader interface {
}

type ScenarioEvaluationRepositories struct {
EvalScenarioRepository repositories.EvalScenarioRepository
EvalTestRunScenarioRepository repositories.EvalTestRunScenarioRepository
ScenarioTestRunRepository repositories.ScenarioTestRunRepository
ScenarioRepository repositories.ScenarioUsecaseRepository
ExecutorFactory executor_factory.ExecutorFactory
IngestedDataReadRepository repositories.IngestedDataReadRepository
EvaluateAstExpression ast_eval.EvaluateAstExpression
SnoozeReader SnoozesForDecisionReader
EvalScenarioRepository repositories.EvalScenarioRepository
EvalSanctionCheckConfigRepository repositories.EvalSanctionCheckConfigRepository
EvalSanctionCheckUsecase EvalSanctionCheckUsecase
EvalTestRunScenarioRepository repositories.EvalTestRunScenarioRepository
ScenarioTestRunRepository repositories.ScenarioTestRunRepository
ScenarioRepository repositories.ScenarioUsecaseRepository
ExecutorFactory executor_factory.ExecutorFactory
IngestedDataReadRepository repositories.IngestedDataReadRepository
EvaluateAstExpression ast_eval.EvaluateAstExpression
SnoozeReader SnoozesForDecisionReader
}

func processScenarioIteration(ctx context.Context, params ScenarioEvaluationParameters,
Expand Down Expand Up @@ -121,6 +127,19 @@ func processScenarioIteration(ctx context.Context, params ScenarioEvaluationPara
"error during concurrent rule evaluation")
}

if iteration.SanctionCheckConfig != nil {
query := models.OpenSanctionsQuery{Queries: models.OpenSanctionCheckFilter{
"name": []string{"obama"},
}}

result, err := repositories.EvalSanctionCheckUsecase.Execute(ctx, *iteration.SanctionCheckConfig, query)
if err != nil {
return models.ScenarioExecution{}, errors.Wrap(err, "could not perform sanction check")
}

logger.Debug("SANCTION CHECK: found", "matches", result.Hits)
}

// Compute outcome from score
var outcome models.Outcome

Expand Down Expand Up @@ -217,6 +236,16 @@ func EvalTestRunScenario(ctx context.Context,
return models.ScenarioExecution{}, err
}

scc, err := repositories.EvalSanctionCheckConfigRepository.GetSanctionCheckConfig(ctx, exec, testRunIteration.Id)

switch {
case err == nil:
testRunIteration.SanctionCheckConfig = &scc
case !errors.Is(err, models.NotFoundError):
return models.ScenarioExecution{}, errors.Wrap(err,
"error getting sanction check config from scenario iteration")
}

se, err = processScenarioIteration(ctx, params, testRunIteration, repositories, start, logger, exec)
if err != nil {
return models.ScenarioExecution{}, err
Expand Down Expand Up @@ -277,6 +306,16 @@ func EvalScenario(
"error getting scenario iteration in EvalScenario")
}

scc, err := repositories.EvalSanctionCheckConfigRepository.GetSanctionCheckConfig(ctx, exec, versionToRun.Id)

switch {
case err == nil:
versionToRun.SanctionCheckConfig = &scc
case !errors.Is(err, models.NotFoundError):
return models.ScenarioExecution{}, errors.Wrap(err,
"error getting sanction check config from scenario iteration")
}

se, errSe := processScenarioIteration(ctx, params, versionToRun, repositories, start, logger, exec)
if errSe != nil {
return models.ScenarioExecution{}, errors.Wrap(errSe,
Expand Down
Loading

0 comments on commit 9352496

Please sign in to comment.