From 0360e3a68655a62a3dd4a6219b8866157c7761d0 Mon Sep 17 00:00:00 2001 From: Scott Leggett Date: Tue, 30 May 2023 22:35:07 +0800 Subject: [PATCH] feat: use updated index-pattern delimiter by default This changes the default behaviour to generate the updated index patterns by default, with an option to switch back to the legacy behaviour. --- cmd/lagoon-opensearch-sync/sync.go | 18 +++-- internal/sync/indexpatterns.go | 35 ++++++--- internal/sync/indexpatterns_test.go | 107 ++++++++++++++++++++-------- internal/sync/sync.go | 5 +- 4 files changed, 117 insertions(+), 48 deletions(-) diff --git a/cmd/lagoon-opensearch-sync/sync.go b/cmd/lagoon-opensearch-sync/sync.go index 14c33ff..fb47e08 100644 --- a/cmd/lagoon-opensearch-sync/sync.go +++ b/cmd/lagoon-opensearch-sync/sync.go @@ -18,10 +18,11 @@ import ( // SyncCmd represents the `sync` command. type SyncCmd struct { - DryRun bool `kong:"env='DRY_RUN',help='Print actions that will be taken but do not persist any changes to Opensearch'"` - Once bool `kong:"default='false',help='Run the sync once instead of forever at the given period'"` - Period time.Duration `kong:"default='8m',help='Period between synchronisation polls'"` - Objects []string `kong:"enum='tenants,roles,rolesmapping,indexpatterns,indextemplates',default='tenants,roles,rolesmapping,indexpatterns,indextemplates',help='Opensearch objects which will be synchronized'"` + DryRun bool `kong:"env='DRY_RUN',help='Print actions that will be taken but do not persist any changes to Opensearch'"` + Once bool `kong:"default='false',help='Run the sync once instead of forever at the given period'"` + Period time.Duration `kong:"default='8m',help='Period between synchronisation polls'"` + Objects []string `kong:"enum='tenants,roles,rolesmapping,indexpatterns,indextemplates',default='tenants,roles,rolesmapping,indexpatterns,indextemplates',help='Opensearch objects which will be synchronized'"` + LegacyIndexPatternDelimiter bool `kong:"default='false',help='Use the legacy -* index pattern delimiter instead of -_-*'"` // lagoon DB client fields APIDBAddress string `kong:"required,env='API_DB_ADDRESS',help='Lagoon API DB Address (host[:port])'"` APIDBDatabase string `kong:"default='infrastructure',env='API_DB_DATABASE',help='Lagoon API DB Database Name'"` @@ -57,7 +58,8 @@ func (cmd *SyncCmd) Run(log *zap.Logger) error { return fmt.Errorf("couldn't init lagoon DBClient: %v", err) } // init the keycloak client - k, err := keycloak.NewClientCredentialsClient(ctx, cmd.KeycloakBaseURL, cmd.KeycloakClientID, + k, err := keycloak.NewClientCredentialsClient(ctx, cmd.KeycloakBaseURL, + cmd.KeycloakClientID, cmd.KeycloakClientSecret) if err != nil { return fmt.Errorf("couldn't init keycloak client: %v", err) @@ -75,7 +77,8 @@ func (cmd *SyncCmd) Run(log *zap.Logger) error { return fmt.Errorf("couldn't init opensearch client: %v", err) } // run sync immediately - err = sync.Sync(ctx, log, l, k, o, d, cmd.DryRun, cmd.Objects) + err = sync.Sync(ctx, log, l, k, o, d, cmd.DryRun, cmd.Objects, + cmd.LegacyIndexPatternDelimiter) if err != nil { return err } @@ -85,7 +88,8 @@ func (cmd *SyncCmd) Run(log *zap.Logger) error { // continue running in a loop tick := time.NewTicker(cmd.Period) for range tick.C { - err = sync.Sync(ctx, log, l, k, o, d, cmd.DryRun, cmd.Objects) + err = sync.Sync(ctx, log, l, k, o, d, cmd.DryRun, cmd.Objects, + cmd.LegacyIndexPatternDelimiter) if err != nil { return err } diff --git a/internal/sync/indexpatterns.go b/internal/sync/indexpatterns.go index c694926..49c227a 100644 --- a/internal/sync/indexpatterns.go +++ b/internal/sync/indexpatterns.go @@ -12,18 +12,24 @@ import ( ) var ( - indexPatternTemplates = []string{ - `application-logs-%s-*`, - `container-logs-%s-*`, - `lagoon-logs-%s-*`, - `router-logs-%s-*`, - } globalIndexPatterns = []string{ `application-logs-*`, `container-logs-*`, `lagoon-logs-*`, `router-logs-*`, } + defaultIndexPatternTemplates = []string{ + `application-logs-%s-_-*`, + `container-logs-%s-_-*`, + `lagoon-logs-%s-_-*`, + `router-logs-%s-_-*`, + } + legacyIndexPatternTemplates = []string{ + `application-logs-%s-*`, + `container-logs-%s-*`, + `lagoon-logs-%s-*`, + `router-logs-%s-*`, + } // indexNameInvalid matches characters which cannot appear in Opensearch // index names indexNameInvalid = regexp.MustCompile(`[^a-z0-9]+`) @@ -116,7 +122,7 @@ func calculateIndexPatternDiff(log *zap.Logger, // generateIndexPatternsForGroup returns a slice of index patterns for all the // projects associated with the given group. func generateIndexPatternsForGroup(log *zap.Logger, group keycloak.Group, - projectNames map[int]string) ([]string, error) { + projectNames map[int]string, legacyDelimiter bool) ([]string, error) { pids, err := projectIDsForGroup(group) if err != nil { return nil, fmt.Errorf("couldn't get project IDs for group: %v", err) @@ -129,6 +135,12 @@ func generateIndexPatternsForGroup(log *zap.Logger, group keycloak.Group, zap.Int("projectID", pid)) continue } + var indexPatternTemplates []string + if legacyDelimiter { + indexPatternTemplates = legacyIndexPatternTemplates + } else { + indexPatternTemplates = defaultIndexPatternTemplates + } for _, tpl := range indexPatternTemplates { indexPatterns = append(indexPatterns, fmt.Sprintf(tpl, name)) } @@ -143,7 +155,7 @@ func generateIndexPatternsForGroup(log *zap.Logger, group keycloak.Group, // Only regular Lagoon groups are associated with a tenant (which is where // index patterns are placed), so project groups are ignored. func generateIndexPatterns(log *zap.Logger, groups []keycloak.Group, - projectNames map[int]string) map[string]map[string]bool { + projectNames map[int]string, legacyDelimiter bool) map[string]map[string]bool { indexPatterns := map[string]map[string]bool{} var patterns []string var err error @@ -151,7 +163,8 @@ func generateIndexPatterns(log *zap.Logger, groups []keycloak.Group, if !isLagoonGroup(group) || isProjectGroup(log, group) { continue } - patterns, err = generateIndexPatternsForGroup(log, group, projectNames) + patterns, err = generateIndexPatternsForGroup(log, group, projectNames, + legacyDelimiter) if err != nil { log.Warn("couldn't generate index patterns for group", zap.String("group", group.Name), zap.Error(err)) @@ -178,7 +191,7 @@ func generateIndexPatterns(log *zap.Logger, groups []keycloak.Group, // Lagoon logging requirements. func syncIndexPatterns(ctx context.Context, log *zap.Logger, groups []keycloak.Group, projectNames map[int]string, o OpensearchService, - d DashboardsService, dryRun bool) { + d DashboardsService, dryRun, legacyDelimiter bool) { // get index patterns from Opensearch existing, err := o.IndexPatterns(ctx) if err != nil { @@ -186,7 +199,7 @@ func syncIndexPatterns(ctx context.Context, log *zap.Logger, return } // generate the index patterns required by Lagoon - required := generateIndexPatterns(log, groups, projectNames) + required := generateIndexPatterns(log, groups, projectNames, legacyDelimiter) // calculate index templates to add/remove toCreate, toDelete := calculateIndexPatternDiff(log, existing, required) for tenant, patternIDMap := range toDelete { diff --git a/internal/sync/indexpatterns_test.go b/internal/sync/indexpatterns_test.go index a33a624..aa0642d 100644 --- a/internal/sync/indexpatterns_test.go +++ b/internal/sync/indexpatterns_test.go @@ -46,18 +46,18 @@ func TestGenerateIndexPatternsForGroup(t *testing.T) { }, expect: generateIndexPatternsForGroupOutput{ indexPatterns: []string{ - "application-logs-drupal9-base-*", - "container-logs-drupal9-base-*", - "lagoon-logs-drupal9-base-*", - "router-logs-drupal9-base-*", - "application-logs-somelongerprojectname-*", - "container-logs-somelongerprojectname-*", - "lagoon-logs-somelongerprojectname-*", - "router-logs-somelongerprojectname-*", - "application-logs-drupal10-prerelease-*", - "container-logs-drupal10-prerelease-*", - "lagoon-logs-drupal10-prerelease-*", - "router-logs-drupal10-prerelease-*", + "application-logs-drupal9-base-_-*", + "container-logs-drupal9-base-_-*", + "lagoon-logs-drupal9-base-_-*", + "router-logs-drupal9-base-_-*", + "application-logs-somelongerprojectname-_-*", + "container-logs-somelongerprojectname-_-*", + "lagoon-logs-somelongerprojectname-_-*", + "router-logs-somelongerprojectname-_-*", + "application-logs-drupal10-prerelease-_-*", + "container-logs-drupal10-prerelease-_-*", + "lagoon-logs-drupal10-prerelease-_-*", + "router-logs-drupal10-prerelease-_-*", "application-logs-*", "container-logs-*", "lagoon-logs-*", @@ -86,14 +86,14 @@ func TestGenerateIndexPatternsForGroup(t *testing.T) { }, expect: generateIndexPatternsForGroupOutput{ indexPatterns: []string{ - "application-logs-drupal9-base-*", - "container-logs-drupal9-base-*", - "lagoon-logs-drupal9-base-*", - "router-logs-drupal9-base-*", - "application-logs-drupal10-prerelease-*", - "container-logs-drupal10-prerelease-*", - "lagoon-logs-drupal10-prerelease-*", - "router-logs-drupal10-prerelease-*", + "application-logs-drupal9-base-_-*", + "container-logs-drupal9-base-_-*", + "lagoon-logs-drupal9-base-_-*", + "router-logs-drupal9-base-_-*", + "application-logs-drupal10-prerelease-_-*", + "container-logs-drupal10-prerelease-_-*", + "lagoon-logs-drupal10-prerelease-_-*", + "router-logs-drupal10-prerelease-_-*", "application-logs-*", "container-logs-*", "lagoon-logs-*", @@ -106,7 +106,7 @@ func TestGenerateIndexPatternsForGroup(t *testing.T) { for name, tc := range testCases { t.Run(name, func(tt *testing.T) { indexPatterns, err := sync.GenerateIndexPatternsForGroup(log, tc.input.group, - tc.input.projectNames) + tc.input.projectNames, false) if (err == nil && tc.expect.err != nil) || (err != nil && tc.expect.err == nil) { tt.Fatalf("got err:\n%v\nexpected err:\n%v\n", err, tc.expect.err) @@ -374,17 +374,66 @@ func TestCalculateIndexPatternDiff(t *testing.T) { } } -type generateIndexPatternsInput struct { - groups []keycloak.Group - projectNames map[int]string -} - func TestGenerateIndexPatterns(t *testing.T) { + type generateIndexPatternsInput struct { + groups []keycloak.Group + projectNames map[int]string + legacyDelimiter bool + } var testCases = map[string]struct { input generateIndexPatternsInput expect map[string]map[string]bool }{ - "high-level test 0": { + "high-level test default group index patterns": { + input: generateIndexPatternsInput{ + groups: []keycloak.Group{ + { + ID: "08fef83d-cde7-43a5-8bd2-a18cf440214a", + GroupUpdateRepresentation: keycloak.GroupUpdateRepresentation{ + Name: "foocorp", + Attributes: map[string][]string{ + "group-lagoon-project-ids": {`{"foocorp":[3133,34435]}`}, + "lagoon-projects": {`3133,34435`}, + }, + }, + }, + { + ID: "9f92af94-a7ee-4759-83bb-2b983bd30142", + GroupUpdateRepresentation: keycloak.GroupUpdateRepresentation{ + Name: "project-drupal12-base", + Attributes: map[string][]string{ + "group-lagoon-project-ids": {`{"project-drupal12-base":[34435]}`}, + "lagoon-projects": {`34435`}, + "type": {`project-default-group`}, + }, + }, + }, + }, + projectNames: map[int]string{ + 34435: "drupal12-base", + }, + legacyDelimiter: false, + }, + expect: map[string]map[string]bool{ + "foocorp": { + `application-logs-drupal12-base-_-*`: true, + `container-logs-drupal12-base-_-*`: true, + `lagoon-logs-drupal12-base-_-*`: true, + `router-logs-drupal12-base-_-*`: true, + `application-logs-*`: true, + `container-logs-*`: true, + `lagoon-logs-*`: true, + `router-logs-*`: true, + }, + "global_tenant": { + `application-logs-*`: true, + `container-logs-*`: true, + `lagoon-logs-*`: true, + `router-logs-*`: true, + }, + }, + }, + "high-level test legacy group index patterns": { input: generateIndexPatternsInput{ groups: []keycloak.Group{ { @@ -412,6 +461,7 @@ func TestGenerateIndexPatterns(t *testing.T) { projectNames: map[int]string{ 34435: "drupal12-base", }, + legacyDelimiter: true, }, expect: map[string]map[string]bool{ "foocorp": { @@ -437,7 +487,8 @@ func TestGenerateIndexPatterns(t *testing.T) { for name, tc := range testCases { t.Run(name, func(tt *testing.T) { indexPatterns := sync.GenerateIndexPatterns( - log, tc.input.groups, tc.input.projectNames) + log, tc.input.groups, tc.input.projectNames, + tc.input.legacyDelimiter) if !reflect.DeepEqual(indexPatterns, tc.expect) { tt.Fatalf("got:\n%v\nexpected:\n%v\n", indexPatterns, tc.expect) } diff --git a/internal/sync/sync.go b/internal/sync/sync.go index 779fd40..5f1f1c4 100644 --- a/internal/sync/sync.go +++ b/internal/sync/sync.go @@ -54,7 +54,7 @@ type DashboardsService interface { // and then configure the OpensearchService as required. func Sync(ctx context.Context, log *zap.Logger, l LagoonDBService, k KeycloakService, o OpensearchService, d DashboardsService, dryRun bool, - objects []string) error { + objects []string, legacyIndexPatternDelimiter bool) error { // get projects from Lagoon projects, err := l.Projects(ctx) if err != nil { @@ -108,7 +108,8 @@ func Sync(ctx context.Context, log *zap.Logger, l LagoonDBService, case "rolesmapping": syncRolesMapping(ctx, log, groups, projectNames, roles, o, dryRun) case "indexpatterns": - syncIndexPatterns(ctx, log, groupsSansGlobal, projectNames, o, d, dryRun) + syncIndexPatterns(ctx, log, groupsSansGlobal, projectNames, o, d, dryRun, + legacyIndexPatternDelimiter) case "indextemplates": syncIndexTemplates(ctx, log, o, dryRun) default: