From d46072efff8af8fda4a02a904b1db07e75968253 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 31 Jan 2024 17:43:50 +0900 Subject: [PATCH 1/5] Make verify command faster by bypassing second-level data lookup --- .../VerifyImportedScoresCommand.cs | 62 +++++++++++-------- 1 file changed, 36 insertions(+), 26 deletions(-) diff --git a/osu.Server.Queues.ScoreStatisticsProcessor/Commands/Maintenance/VerifyImportedScoresCommand.cs b/osu.Server.Queues.ScoreStatisticsProcessor/Commands/Maintenance/VerifyImportedScoresCommand.cs index 964b1516..fe2cdd26 100644 --- a/osu.Server.Queues.ScoreStatisticsProcessor/Commands/Maintenance/VerifyImportedScoresCommand.cs +++ b/osu.Server.Queues.ScoreStatisticsProcessor/Commands/Maintenance/VerifyImportedScoresCommand.cs @@ -26,6 +26,12 @@ public class VerifyImportedScoresCommand [Option(CommandOptionType.SingleValue, Template = "--start-id")] public ulong? StartId { get; set; } + /// + /// The ruleset to run this verify job for. + /// + [Option(CommandOptionType.SingleValue, Template = "--ruleset-id")] + public int RulesetId { get; set; } + /// /// The number of scores to run in each batch. Setting this higher will cause larger SQL statements for insert. /// @@ -42,12 +48,14 @@ public class VerifyImportedScoresCommand public async Task OnExecuteAsync(CancellationToken cancellationToken) { + var rulesetSpecifics = LegacyDatabaseHelper.GetRulesetSpecifics(RulesetId); + ulong lastId = StartId ?? 0; int deleted = 0; int fail = 0; Console.WriteLine(); - Console.WriteLine($"Verifying scores starting from {lastId}"); + Console.WriteLine($"Verifying scores starting from {lastId} for ruleset {RulesetId}"); elasticQueueProcessor = new ElasticQueuePusher(); Console.WriteLine($"Indexing to elasticsearch queue(s) {elasticQueueProcessor.ActiveQueues}"); @@ -61,41 +69,43 @@ public async Task OnExecuteAsync(CancellationToken cancellationToken) { HashSet elasticItems = new HashSet(); - IEnumerable importedScores = await conn.QueryAsync( + IEnumerable importedScores = await conn.QueryAsync( "SELECT `id`, " + "`ruleset_id`, " + "`legacy_score_id`, " + "`legacy_total_score`, " + "`total_score`, " - + "`rank`, " - + "`pp` " - + "FROM scores " - + "WHERE id >= @lastId AND legacy_score_id IS NOT NULL ORDER BY id LIMIT @batchSize", new + + "s.`rank`, " + + "s.`pp`, " + + "h.* " + + "FROM scores s " + + $"LEFT JOIN {rulesetSpecifics.HighScoreTable} h ON (legacy_score_id = score_id)" + + "WHERE id BETWEEN @lastId AND (@lastId + @batchSize - 1) AND legacy_score_id IS NOT NULL AND ruleset_id = @rulesetId ORDER BY id", + (ComparableScore score, HighScore highScore) => + { + score.HighScore = highScore; + return score; + }, + new { lastId, + rulesetId = RulesetId, batchSize = BatchSize - }); + }, splitOn: "score_id"); - // gather high scores for each ruleset - foreach (var rulesetScores in importedScores.GroupBy(s => s.ruleset_id)) + if (!importedScores.Any()) { - var rulesetSpecifics = LegacyDatabaseHelper.GetRulesetSpecifics(rulesetScores.Key); - - var highScores = (await conn.QueryAsync( - $"SELECT * FROM {rulesetSpecifics.HighScoreTable} WHERE score_id IN ({string.Join(',', rulesetScores.Select(s => s.legacy_score_id))})")) - .ToDictionary(s => s.score_id, s => s); - - foreach (var score in rulesetScores) + if (lastId > await conn.QuerySingleAsync("SELECT MAX(id) FROM scores")) { - if (highScores.TryGetValue(score.legacy_score_id!.Value, out var highScore)) - score.HighScore = highScore; + Console.WriteLine("All done!"); + break; } - } - if (!importedScores.Any()) - { - Console.WriteLine("All done!"); - break; + lastId += (ulong)BatchSize; + + if (lastId % (ulong)BatchSize * 100 == 0) + Console.Write("."); + continue; } elasticItems.Clear(); @@ -246,10 +256,10 @@ public async Task OnExecuteAsync(CancellationToken cancellationToken) Console.WriteLine($"Queued {elasticItems.Count} items for indexing"); } - lastId = importedScores.Max(s => s.id) + 1; - Console.SetCursorPosition(0, Console.GetCursorPosition().Top); - Console.Write($"Processed up to {lastId} ({deleted} deleted {fail} failed)"); + Console.Write($"Processed up to {importedScores.Max(s => s.id)} ({deleted} deleted {fail} failed)"); + + lastId += (ulong)BatchSize; } return 0; From 1d844b522860b174860c751c2c5f58a3282e0b88 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 31 Jan 2024 21:37:21 +0900 Subject: [PATCH 2/5] Add exception for non-high scores --- .../Commands/Maintenance/VerifyImportedScoresCommand.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/osu.Server.Queues.ScoreStatisticsProcessor/Commands/Maintenance/VerifyImportedScoresCommand.cs b/osu.Server.Queues.ScoreStatisticsProcessor/Commands/Maintenance/VerifyImportedScoresCommand.cs index fe2cdd26..0b28b7e3 100644 --- a/osu.Server.Queues.ScoreStatisticsProcessor/Commands/Maintenance/VerifyImportedScoresCommand.cs +++ b/osu.Server.Queues.ScoreStatisticsProcessor/Commands/Maintenance/VerifyImportedScoresCommand.cs @@ -134,7 +134,10 @@ public async Task OnExecuteAsync(CancellationToken cancellationToken) if (importedScore.legacy_score_id == null) continue; // Score was deleted in legacy table. - if (importedScore.HighScore == null) + // + // Importantly, `legacy_score_id` of 0 implies a non-high-score (which doesn't have a matching entry). + // We should leave these. + if (importedScore.HighScore == null && importedScore.legacy_score_id > 0) { Interlocked.Increment(ref deleted); requiresIndexing = true; From c4fa03b4593f9db634aa24362b23499d19571ff6 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 31 Jan 2024 22:20:22 +0900 Subject: [PATCH 3/5] Better skipping output --- .../Commands/Maintenance/VerifyImportedScoresCommand.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/osu.Server.Queues.ScoreStatisticsProcessor/Commands/Maintenance/VerifyImportedScoresCommand.cs b/osu.Server.Queues.ScoreStatisticsProcessor/Commands/Maintenance/VerifyImportedScoresCommand.cs index 0b28b7e3..32d3a350 100644 --- a/osu.Server.Queues.ScoreStatisticsProcessor/Commands/Maintenance/VerifyImportedScoresCommand.cs +++ b/osu.Server.Queues.ScoreStatisticsProcessor/Commands/Maintenance/VerifyImportedScoresCommand.cs @@ -46,6 +46,8 @@ public class VerifyImportedScoresCommand private ElasticQueuePusher? elasticQueueProcessor; + private int skipOutput; + public async Task OnExecuteAsync(CancellationToken cancellationToken) { var rulesetSpecifics = LegacyDatabaseHelper.GetRulesetSpecifics(RulesetId); @@ -103,8 +105,8 @@ public async Task OnExecuteAsync(CancellationToken cancellationToken) lastId += (ulong)BatchSize; - if (lastId % (ulong)BatchSize * 100 == 0) - Console.Write("."); + if (++skipOutput % 100 == 0) + Console.WriteLine($"Skipped up to {lastId}..."); continue; } From b090b4c2bbf847151c37183c45adda273763d979 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 31 Jan 2024 22:34:29 +0900 Subject: [PATCH 4/5] More correctly bypass non-high-score verification --- .../VerifyImportedScoresCommand.cs | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/osu.Server.Queues.ScoreStatisticsProcessor/Commands/Maintenance/VerifyImportedScoresCommand.cs b/osu.Server.Queues.ScoreStatisticsProcessor/Commands/Maintenance/VerifyImportedScoresCommand.cs index 32d3a350..865b42e2 100644 --- a/osu.Server.Queues.ScoreStatisticsProcessor/Commands/Maintenance/VerifyImportedScoresCommand.cs +++ b/osu.Server.Queues.ScoreStatisticsProcessor/Commands/Maintenance/VerifyImportedScoresCommand.cs @@ -133,26 +133,35 @@ public async Task OnExecuteAsync(CancellationToken cancellationToken) try { + // Score was set via lazer, we have nothing to verify. if (importedScore.legacy_score_id == null) continue; // Score was deleted in legacy table. // // Importantly, `legacy_score_id` of 0 implies a non-high-score (which doesn't have a matching entry). // We should leave these. - if (importedScore.HighScore == null && importedScore.legacy_score_id > 0) + if (importedScore.HighScore == null) { - Interlocked.Increment(ref deleted); - requiresIndexing = true; - - if (!DryRun) + if (importedScore.legacy_score_id > 0) { - await conn.ExecuteAsync("DELETE FROM scores WHERE id = @id", new + Interlocked.Increment(ref deleted); + requiresIndexing = true; + + if (!DryRun) { - id = importedScore.id - }); - } + await conn.ExecuteAsync("DELETE FROM scores WHERE id = @id", new + { + id = importedScore.id + }); + } - continue; + continue; + } + else + { + // Score was sourced from the osu_scores table, and we don't really care about verifying these. + continue; + } } if (DeleteOnly) From 1ff0b819bb7427358bb8d8739d46bf84a40fcab5 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 31 Jan 2024 23:12:27 +0900 Subject: [PATCH 5/5] Skip to start of ruleset better --- .../Maintenance/VerifyImportedScoresCommand.cs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/osu.Server.Queues.ScoreStatisticsProcessor/Commands/Maintenance/VerifyImportedScoresCommand.cs b/osu.Server.Queues.ScoreStatisticsProcessor/Commands/Maintenance/VerifyImportedScoresCommand.cs index 865b42e2..428e3da0 100644 --- a/osu.Server.Queues.ScoreStatisticsProcessor/Commands/Maintenance/VerifyImportedScoresCommand.cs +++ b/osu.Server.Queues.ScoreStatisticsProcessor/Commands/Maintenance/VerifyImportedScoresCommand.cs @@ -56,6 +56,15 @@ public async Task OnExecuteAsync(CancellationToken cancellationToken) int deleted = 0; int fail = 0; + using var conn = DatabaseAccess.GetConnection(); + + lastId = await conn.QuerySingleAsync( + "SELECT id FROM scores WHERE ruleset_id = @rulesetId AND legacy_score_id = (SELECT MIN(legacy_score_id) FROM scores WHERE ruleset_id = @rulesetId AND id >= @lastId AND legacy_score_id > 0)", new + { + lastId, + rulesetId = RulesetId, + }) ?? lastId; + Console.WriteLine(); Console.WriteLine($"Verifying scores starting from {lastId} for ruleset {RulesetId}"); @@ -65,8 +74,6 @@ public async Task OnExecuteAsync(CancellationToken cancellationToken) if (DryRun) Console.WriteLine("RUNNING IN DRY RUN MODE."); - using var conn = DatabaseAccess.GetConnection(); - while (!cancellationToken.IsCancellationRequested) { HashSet elasticItems = new HashSet();