Skip to content

Commit

Permalink
Merge pull request #55 from uselagoon/legacy-index-pattern-delimiter
Browse files Browse the repository at this point in the history 
feat: use updated index-pattern delimiter by default
  • Loading branch information
@smlx
smlx authored May 31, 2023
2 parents f804561 + 0360e3a commit 6d12d5d
Show file tree
Hide file tree
Showing 4 changed files with 117 additions and 48 deletions.
18 changes: 11 additions & 7 deletions cmd/lagoon-opensearch-sync/sync.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,11 @@ import (

// SyncCmd represents the `sync` command.
type SyncCmd struct {
DryRun bool `kong:"env='DRY_RUN',help='Print actions that will be taken but do not persist any changes to Opensearch'"`
Once bool `kong:"default='false',help='Run the sync once instead of forever at the given period'"`
Period time.Duration `kong:"default='8m',help='Period between synchronisation polls'"`
Objects []string `kong:"enum='tenants,roles,rolesmapping,indexpatterns,indextemplates',default='tenants,roles,rolesmapping,indexpatterns,indextemplates',help='Opensearch objects which will be synchronized'"`
DryRun bool `kong:"env='DRY_RUN',help='Print actions that will be taken but do not persist any changes to Opensearch'"`
Once bool `kong:"default='false',help='Run the sync once instead of forever at the given period'"`
Period time.Duration `kong:"default='8m',help='Period between synchronisation polls'"`
Objects []string `kong:"enum='tenants,roles,rolesmapping,indexpatterns,indextemplates',default='tenants,roles,rolesmapping,indexpatterns,indextemplates',help='Opensearch objects which will be synchronized'"`
LegacyIndexPatternDelimiter bool `kong:"default='false',help='Use the legacy -* index pattern delimiter instead of -_-*'"`
// lagoon DB client fields
APIDBAddress string `kong:"required,env='API_DB_ADDRESS',help='Lagoon API DB Address (host[:port])'"`
APIDBDatabase string `kong:"default='infrastructure',env='API_DB_DATABASE',help='Lagoon API DB Database Name'"`
Expand Down Expand Up @@ -57,7 +58,8 @@ func (cmd *SyncCmd) Run(log *zap.Logger) error {
return fmt.Errorf("couldn't init lagoon DBClient: %v", err)
}
// init the keycloak client
k, err := keycloak.NewClientCredentialsClient(ctx, cmd.KeycloakBaseURL, cmd.KeycloakClientID,
k, err := keycloak.NewClientCredentialsClient(ctx, cmd.KeycloakBaseURL,
cmd.KeycloakClientID,
cmd.KeycloakClientSecret)
if err != nil {
return fmt.Errorf("couldn't init keycloak client: %v", err)
Expand All @@ -75,7 +77,8 @@ func (cmd *SyncCmd) Run(log *zap.Logger) error {
return fmt.Errorf("couldn't init opensearch client: %v", err)
}
// run sync immediately
err = sync.Sync(ctx, log, l, k, o, d, cmd.DryRun, cmd.Objects)
err = sync.Sync(ctx, log, l, k, o, d, cmd.DryRun, cmd.Objects,
cmd.LegacyIndexPatternDelimiter)
if err != nil {
return err
}
Expand All @@ -85,7 +88,8 @@ func (cmd *SyncCmd) Run(log *zap.Logger) error {
// continue running in a loop
tick := time.NewTicker(cmd.Period)
for range tick.C {
err = sync.Sync(ctx, log, l, k, o, d, cmd.DryRun, cmd.Objects)
err = sync.Sync(ctx, log, l, k, o, d, cmd.DryRun, cmd.Objects,
cmd.LegacyIndexPatternDelimiter)
if err != nil {
return err
}
Expand Down
35 changes: 24 additions & 11 deletions internal/sync/indexpatterns.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,18 +12,24 @@ import (
)

var (
indexPatternTemplates = []string{
`application-logs-%s-*`,
`container-logs-%s-*`,
`lagoon-logs-%s-*`,
`router-logs-%s-*`,
}
globalIndexPatterns = []string{
`application-logs-*`,
`container-logs-*`,
`lagoon-logs-*`,
`router-logs-*`,
}
defaultIndexPatternTemplates = []string{
`application-logs-%s-_-*`,
`container-logs-%s-_-*`,
`lagoon-logs-%s-_-*`,
`router-logs-%s-_-*`,
}
legacyIndexPatternTemplates = []string{
`application-logs-%s-*`,
`container-logs-%s-*`,
`lagoon-logs-%s-*`,
`router-logs-%s-*`,
}
// indexNameInvalid matches characters which cannot appear in Opensearch
// index names
indexNameInvalid = regexp.MustCompile(`[^a-z0-9]+`)
Expand Down Expand Up @@ -116,7 +122,7 @@ func calculateIndexPatternDiff(log *zap.Logger,
// generateIndexPatternsForGroup returns a slice of index patterns for all the
// projects associated with the given group.
func generateIndexPatternsForGroup(log *zap.Logger, group keycloak.Group,
projectNames map[int]string) ([]string, error) {
projectNames map[int]string, legacyDelimiter bool) ([]string, error) {
pids, err := projectIDsForGroup(group)
if err != nil {
return nil, fmt.Errorf("couldn't get project IDs for group: %v", err)
Expand All @@ -129,6 +135,12 @@ func generateIndexPatternsForGroup(log *zap.Logger, group keycloak.Group,
zap.Int("projectID", pid))
continue
}
var indexPatternTemplates []string
if legacyDelimiter {
indexPatternTemplates = legacyIndexPatternTemplates
} else {
indexPatternTemplates = defaultIndexPatternTemplates
}
for _, tpl := range indexPatternTemplates {
indexPatterns = append(indexPatterns, fmt.Sprintf(tpl, name))
}
Expand All @@ -143,15 +155,16 @@ func generateIndexPatternsForGroup(log *zap.Logger, group keycloak.Group,
// Only regular Lagoon groups are associated with a tenant (which is where
// index patterns are placed), so project groups are ignored.
func generateIndexPatterns(log *zap.Logger, groups []keycloak.Group,
projectNames map[int]string) map[string]map[string]bool {
projectNames map[int]string, legacyDelimiter bool) map[string]map[string]bool {
indexPatterns := map[string]map[string]bool{}
var patterns []string
var err error
for _, group := range groups {
if !isLagoonGroup(group) || isProjectGroup(log, group) {
continue
}
patterns, err = generateIndexPatternsForGroup(log, group, projectNames)
patterns, err = generateIndexPatternsForGroup(log, group, projectNames,
legacyDelimiter)
if err != nil {
log.Warn("couldn't generate index patterns for group",
zap.String("group", group.Name), zap.Error(err))
Expand All @@ -178,15 +191,15 @@ func generateIndexPatterns(log *zap.Logger, groups []keycloak.Group,
// Lagoon logging requirements.
func syncIndexPatterns(ctx context.Context, log *zap.Logger,
groups []keycloak.Group, projectNames map[int]string, o OpensearchService,
d DashboardsService, dryRun bool) {
d DashboardsService, dryRun, legacyDelimiter bool) {
// get index patterns from Opensearch
existing, err := o.IndexPatterns(ctx)
if err != nil {
log.Error("couldn't get index patterns from Opensearch", zap.Error(err))
return
}
// generate the index patterns required by Lagoon
required := generateIndexPatterns(log, groups, projectNames)
required := generateIndexPatterns(log, groups, projectNames, legacyDelimiter)
// calculate index templates to add/remove
toCreate, toDelete := calculateIndexPatternDiff(log, existing, required)
for tenant, patternIDMap := range toDelete {
Expand Down
107 changes: 79 additions & 28 deletions internal/sync/indexpatterns_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,18 +46,18 @@ func TestGenerateIndexPatternsForGroup(t *testing.T) {
},
expect: generateIndexPatternsForGroupOutput{
indexPatterns: []string{
"application-logs-drupal9-base-*",
"container-logs-drupal9-base-*",
"lagoon-logs-drupal9-base-*",
"router-logs-drupal9-base-*",
"application-logs-somelongerprojectname-*",
"container-logs-somelongerprojectname-*",
"lagoon-logs-somelongerprojectname-*",
"router-logs-somelongerprojectname-*",
"application-logs-drupal10-prerelease-*",
"container-logs-drupal10-prerelease-*",
"lagoon-logs-drupal10-prerelease-*",
"router-logs-drupal10-prerelease-*",
"application-logs-drupal9-base-_-*",
"container-logs-drupal9-base-_-*",
"lagoon-logs-drupal9-base-_-*",
"router-logs-drupal9-base-_-*",
"application-logs-somelongerprojectname-_-*",
"container-logs-somelongerprojectname-_-*",
"lagoon-logs-somelongerprojectname-_-*",
"router-logs-somelongerprojectname-_-*",
"application-logs-drupal10-prerelease-_-*",
"container-logs-drupal10-prerelease-_-*",
"lagoon-logs-drupal10-prerelease-_-*",
"router-logs-drupal10-prerelease-_-*",
"application-logs-*",
"container-logs-*",
"lagoon-logs-*",
Expand Down Expand Up @@ -86,14 +86,14 @@ func TestGenerateIndexPatternsForGroup(t *testing.T) {
},
expect: generateIndexPatternsForGroupOutput{
indexPatterns: []string{
"application-logs-drupal9-base-*",
"container-logs-drupal9-base-*",
"lagoon-logs-drupal9-base-*",
"router-logs-drupal9-base-*",
"application-logs-drupal10-prerelease-*",
"container-logs-drupal10-prerelease-*",
"lagoon-logs-drupal10-prerelease-*",
"router-logs-drupal10-prerelease-*",
"application-logs-drupal9-base-_-*",
"container-logs-drupal9-base-_-*",
"lagoon-logs-drupal9-base-_-*",
"router-logs-drupal9-base-_-*",
"application-logs-drupal10-prerelease-_-*",
"container-logs-drupal10-prerelease-_-*",
"lagoon-logs-drupal10-prerelease-_-*",
"router-logs-drupal10-prerelease-_-*",
"application-logs-*",
"container-logs-*",
"lagoon-logs-*",
Expand All @@ -106,7 +106,7 @@ func TestGenerateIndexPatternsForGroup(t *testing.T) {
for name, tc := range testCases {
t.Run(name, func(tt *testing.T) {
indexPatterns, err := sync.GenerateIndexPatternsForGroup(log, tc.input.group,
tc.input.projectNames)
tc.input.projectNames, false)
if (err == nil && tc.expect.err != nil) ||
(err != nil && tc.expect.err == nil) {
tt.Fatalf("got err:\n%v\nexpected err:\n%v\n", err, tc.expect.err)
Expand Down Expand Up @@ -374,17 +374,66 @@ func TestCalculateIndexPatternDiff(t *testing.T) {
}
}

type generateIndexPatternsInput struct {
groups []keycloak.Group
projectNames map[int]string
}

func TestGenerateIndexPatterns(t *testing.T) {
type generateIndexPatternsInput struct {
groups []keycloak.Group
projectNames map[int]string
legacyDelimiter bool
}
var testCases = map[string]struct {
input generateIndexPatternsInput
expect map[string]map[string]bool
}{
"high-level test 0": {
"high-level test default group index patterns": {
input: generateIndexPatternsInput{
groups: []keycloak.Group{
{
ID: "08fef83d-cde7-43a5-8bd2-a18cf440214a",
GroupUpdateRepresentation: keycloak.GroupUpdateRepresentation{
Name: "foocorp",
Attributes: map[string][]string{
"group-lagoon-project-ids": {`{"foocorp":[3133,34435]}`},
"lagoon-projects": {`3133,34435`},
},
},
},
{
ID: "9f92af94-a7ee-4759-83bb-2b983bd30142",
GroupUpdateRepresentation: keycloak.GroupUpdateRepresentation{
Name: "project-drupal12-base",
Attributes: map[string][]string{
"group-lagoon-project-ids": {`{"project-drupal12-base":[34435]}`},
"lagoon-projects": {`34435`},
"type": {`project-default-group`},
},
},
},
},
projectNames: map[int]string{
34435: "drupal12-base",
},
legacyDelimiter: false,
},
expect: map[string]map[string]bool{
"foocorp": {
`application-logs-drupal12-base-_-*`: true,
`container-logs-drupal12-base-_-*`: true,
`lagoon-logs-drupal12-base-_-*`: true,
`router-logs-drupal12-base-_-*`: true,
`application-logs-*`: true,
`container-logs-*`: true,
`lagoon-logs-*`: true,
`router-logs-*`: true,
},
"global_tenant": {
`application-logs-*`: true,
`container-logs-*`: true,
`lagoon-logs-*`: true,
`router-logs-*`: true,
},
},
},
"high-level test legacy group index patterns": {
input: generateIndexPatternsInput{
groups: []keycloak.Group{
{
Expand Down Expand Up @@ -412,6 +461,7 @@ func TestGenerateIndexPatterns(t *testing.T) {
projectNames: map[int]string{
34435: "drupal12-base",
},
legacyDelimiter: true,
},
expect: map[string]map[string]bool{
"foocorp": {
Expand All @@ -437,7 +487,8 @@ func TestGenerateIndexPatterns(t *testing.T) {
for name, tc := range testCases {
t.Run(name, func(tt *testing.T) {
indexPatterns := sync.GenerateIndexPatterns(
log, tc.input.groups, tc.input.projectNames)
log, tc.input.groups, tc.input.projectNames,
tc.input.legacyDelimiter)
if !reflect.DeepEqual(indexPatterns, tc.expect) {
tt.Fatalf("got:\n%v\nexpected:\n%v\n", indexPatterns, tc.expect)
}
Expand Down
5 changes: 3 additions & 2 deletions internal/sync/sync.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ type DashboardsService interface {
// and then configure the OpensearchService as required.
func Sync(ctx context.Context, log *zap.Logger, l LagoonDBService,
k KeycloakService, o OpensearchService, d DashboardsService, dryRun bool,
objects []string) error {
objects []string, legacyIndexPatternDelimiter bool) error {
// get projects from Lagoon
projects, err := l.Projects(ctx)
if err != nil {
Expand Down Expand Up @@ -108,7 +108,8 @@ func Sync(ctx context.Context, log *zap.Logger, l LagoonDBService,
case "rolesmapping":
syncRolesMapping(ctx, log, groups, projectNames, roles, o, dryRun)
case "indexpatterns":
syncIndexPatterns(ctx, log, groupsSansGlobal, projectNames, o, d, dryRun)
syncIndexPatterns(ctx, log, groupsSansGlobal, projectNames, o, d, dryRun,
legacyIndexPatternDelimiter)
case "indextemplates":
syncIndexTemplates(ctx, log, o, dryRun)
default:
Expand Down

0 comments on commit 6d12d5d

Please sign in to comment.