From 35f6bc6edc8b5816dda17f3d6b15e42e258005e1 Mon Sep 17 00:00:00 2001 From: Bourne Shi Date: Wed, 16 Oct 2024 10:30:13 +0800 Subject: [PATCH 01/15] feat: support terms query --- .../ElasticsearchResponseHelper.cs | 17 +++- .../IElasticsearchClientProvider.cs | 30 +++---- .../Linq/GeneratorExpressionTreeVisitor.cs | 84 +++++++++++++++++-- .../ElasticsearchRepositoryTests.cs | 79 +++++++++++++++++ 4 files changed, 189 insertions(+), 21 deletions(-) diff --git a/src/AElf.EntityMapping.Elasticsearch/ElasticsearchResponseHelper.cs b/src/AElf.EntityMapping.Elasticsearch/ElasticsearchResponseHelper.cs index ac177882..6c7b0b4c 100644 --- a/src/AElf.EntityMapping.Elasticsearch/ElasticsearchResponseHelper.cs +++ b/src/AElf.EntityMapping.Elasticsearch/ElasticsearchResponseHelper.cs @@ -6,6 +6,21 @@ public class ElasticsearchResponseHelper { public static string GetErrorMessage(IResponse response) { - return response.ServerError == null ? "Unknown error." : response.ServerError.ToString(); + if (response.ServerError == null) + { + if (response.OriginalException == null) + { + return "Unknown error."; + } + + if (response.OriginalException.InnerException == null) + { + return response.OriginalException.Message; + } + + return response.OriginalException.InnerException.Message; + } + + return response.ServerError.ToString(); } } \ No newline at end of file diff --git a/src/AElf.EntityMapping.Elasticsearch/IElasticsearchClientProvider.cs b/src/AElf.EntityMapping.Elasticsearch/IElasticsearchClientProvider.cs index 89b33684..f8e38ce7 100644 --- a/src/AElf.EntityMapping.Elasticsearch/IElasticsearchClientProvider.cs +++ b/src/AElf.EntityMapping.Elasticsearch/IElasticsearchClientProvider.cs @@ -20,21 +20,21 @@ public ElasticsearchClientProvider(IOptions options) { var uris = options.Value.Uris.ConvertAll(x => new Uri(x)); var connectionPool = new StaticConnectionPool(uris); - var settings = new ConnectionSettings(connectionPool); - // .DisableDirectStreaming(); - // .OnRequestCompleted(callDetails => - // { - // // Print Request DSL - // if (callDetails.RequestBodyInBytes != null) - // { - // Console.WriteLine($"Request JSON: {Encoding.UTF8.GetString(callDetails.RequestBodyInBytes)}"); - // } - // // // Print Response Data - // // if (callDetails.ResponseBodyInBytes != null) - // // { - // // Console.WriteLine($"Response JSON: {Encoding.UTF8.GetString(callDetails.ResponseBodyInBytes)}"); - // // } - // }); + var settings = new ConnectionSettings(connectionPool) + .DisableDirectStreaming() + .OnRequestCompleted(callDetails => + { + // Print Request DSL + if (callDetails.RequestBodyInBytes != null) + { + Console.WriteLine($"Request JSON: {Encoding.UTF8.GetString(callDetails.RequestBodyInBytes)}"); + } + // // Print Response Data + // if (callDetails.ResponseBodyInBytes != null) + // { + // Console.WriteLine($"Response JSON: {Encoding.UTF8.GetString(callDetails.ResponseBodyInBytes)}"); + // } + }); _elasticClient = new ElasticClient(settings); } diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs index 7aa1750a..43dea89c 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs @@ -1,4 +1,5 @@ using System.Linq.Expressions; +using System.Reflection; using Elasticsearch.Net; using Remotion.Linq.Clauses; using Remotion.Linq.Clauses.Expressions; @@ -202,15 +203,41 @@ protected override Expression VisitSubQuery(SubQueryExpression expression) case ContainsResultOperator containsResultOperator: Visit(containsResultOperator.Item); Visit(expression.QueryModel.MainFromClause.FromExpression); - - if (containsResultOperator.Item.Type == typeof(Guid)) + + // PropertyName = _propertyNameInferrerParser.Parser(GetFullPropertyPath(expression.QueryModel.MainFromClause.FromExpression)); + Type itemType = containsResultOperator.Item.Type; + Type nonNullableType = Nullable.GetUnderlyingType(itemType) ?? itemType; + + // Handling different types + if (itemType == typeof(Guid) || nonNullableType == typeof(Guid)) { query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); } - - if (containsResultOperator.Item.Type == typeof(Guid?)) + else if (itemType == typeof(Guid?) || nonNullableType == typeof(Guid?)) + { + query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x?.ToString())); + } + else if (nonNullableType == typeof(int) || nonNullableType == typeof(int?)) { + query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + } + else if (nonNullableType == typeof(long) || nonNullableType == typeof(long?)) { + query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + } + else if (nonNullableType == typeof(double) || nonNullableType == typeof(double?)) { + query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + } + else if (nonNullableType == typeof(DateTime) || nonNullableType == typeof(DateTime?)) { + query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString("o"))); // ISO 8601 format + } + else if (nonNullableType == typeof(bool) || nonNullableType == typeof(bool?)) { + query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + } + else if (nonNullableType == typeof(string)) { + query = new TermsNode(PropertyName, (IEnumerable)Value); + } + else { - query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + throw new NotSupportedException($"Type {nonNullableType.Name} is not supported for Terms queries."); } QueryMap[expression] = ParseQuery(query); @@ -234,6 +261,11 @@ protected override Expression VisitSubQuery(SubQueryExpression expression) QueryMap[expression].SubQueryPath = from; //from.ToLower(); QueryMap[expression].SubQueryFullPath = fullPath; // VisitBinarySetSubQuery((BinaryExpression)whereClause.Predicate, from, fullPath, true); + if (whereClause.Predicate is SubQueryExpression subQueryExpression) + { + // HandleNestedContains(subQueryExpression, tmp); + return base.VisitSubQuery(expression); + } BinaryExpression predicate = (BinaryExpression)whereClause.Predicate; if (predicate.Left is BinaryExpression) { @@ -281,6 +313,48 @@ protected override Expression VisitSubQuery(SubQueryExpression expression) return expression; } + // private void HandleNestedContains(SubQueryExpression subQueryExpression, Node parent) + // { + // if (subQueryExpression == null || parent == null) + // throw new ArgumentNullException("SubQueryExpression or parent Node cannot be null."); + // + // PropertyName = _propertyNameInferrerParser.Parser(GetFullPropertyPath(subQueryExpression)); + // PropertyType = Nullable.GetUnderlyingType(subQueryExpression.Type) ?? subQueryExpression.Type; + // Node query; + // if (PropertyType == typeof(Guid) || PropertyType == typeof(Guid)) + // { + // query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + // } + // else if (PropertyType == typeof(int) || PropertyType == typeof(int)) + // { + // query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + // } + // else if (PropertyType == typeof(string)) + // { + // query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + // } + // else + // { + // throw new NotSupportedException($"Type {PropertyType.Name} is not supported for Contains queries."); + // } + // // parent.(query); + // QueryMap[subQueryExpression] = ParseQuery(query); + // } + + private string GetMemberName(Expression expression) + { + if (expression is MemberExpression memberExpression) + return memberExpression.Member.Name; + throw new InvalidOperationException("Expression does not represent a member access."); + } + + private object GetValueFromExpression(Expression expression) + { + if (expression is ConstantExpression constantExpression) + return constantExpression.Value; + throw new InvalidOperationException("Expression is not a constant."); + } + protected override Expression VisitQuerySourceReference(QuerySourceReferenceExpression expression) { return expression; diff --git a/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs b/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs index bee149dc..b633b799 100644 --- a/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs +++ b/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs @@ -593,6 +593,85 @@ public async Task GetList_Nested_Test() filterList.Count.ShouldBe(1); } + [Fact] + public async Task GetList_Terms_Test() + { + for (int i = 1; i <= 7; i++) + { + var blockIndex = new BlockIndex + { + Id = "block" + i, + BlockHash = "BlockHash" + i, + BlockHeight = i, + BlockTime = DateTime.Now.AddDays(-10 + i), + LogEventCount = i, + ChainId = "AELF" + }; + await _elasticsearchRepository.AddAsync(blockIndex); + } + + List inputs = new List() + { + "BlockHash2", + "BlockHash3", + "BlockHash4" + }; + + var queryable = await _elasticsearchRepository.GetQueryableAsync(); + + var predicates = inputs + .Select(s => (Expression>)(info => info.BlockHash == s)) + .Aggregate((prev, next) => prev.Or(next)); + var filterList_predicate = queryable.Where(predicates).ToList(); + + var filterList = queryable.Where(item => inputs.Contains(item.BlockHash)).ToList(); + filterList.Count.ShouldBe(3); + + List heights = new List() + { + 4, 5 + }; + Expression> mustQuery = item => heights.Contains(item.BlockHeight); + var filterList_heights = queryable.Where(mustQuery).ToList(); + filterList_heights.Count.ShouldBe(2); + } + + [Fact] + public async Task GetNestedList_Terms_Test() + { + //clear data for unit test + ClearTransactionIndex("AELF", 100, 110); + + Thread.Sleep(2000); + //Unit Test 14 + var transaction_100 = MockNewTransactionEtoData(100, false, "token_contract_address", "DonateResourceToken"); + var transaction_101 = MockNewTransactionEtoData(101, false, "", ""); + var transaction_103 = MockNewTransactionEtoData(103, false, "consensus_contract_address", "UpdateValue"); + var transaction_110 = MockNewTransactionEtoData(110, true, "consensus_contract_address", "UpdateTinyBlockInformation"); + await _transactionIndexRepository.AddAsync(transaction_100); + await _transactionIndexRepository.AddAsync(transaction_101); + await _transactionIndexRepository.AddAsync(transaction_103); + await _transactionIndexRepository.AddAsync(transaction_110); + + List inputs = new List() + { + 101, + 103 + }; + + // var predicates = inputs + // .Select(s => (Expression>)(info => info.LogEvents.Any(x => x.BlockHeight == s))) + // .Aggregate((prev, next) => prev.Or(next)); + + Expression> mustQuery = item => + item.LogEvents.Any(x => inputs.Contains(x.BlockHeight)); + + var queryable = await _transactionIndexRepository.GetQueryableAsync(); + // var filterList = queryable.Where(predicates).ToList(); + var filterList = queryable.Where(mustQuery).ToList(); + filterList.Count.ShouldBe(2); + } + [Fact] public async Task SubObjectQueryTest() { From a157a53b006d46bd54c9b78d5665453d295921f5 Mon Sep 17 00:00:00 2001 From: Bourne Shi Date: Wed, 16 Oct 2024 15:28:54 +0800 Subject: [PATCH 02/15] feat: support nested terms query --- .../Linq/GeneratorExpressionTreeVisitor.cs | 172 ++++++++++-------- .../ElasticsearchRepositoryTests.cs | 10 +- 2 files changed, 98 insertions(+), 84 deletions(-) diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs index 43dea89c..03a00697 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs @@ -204,41 +204,8 @@ protected override Expression VisitSubQuery(SubQueryExpression expression) Visit(containsResultOperator.Item); Visit(expression.QueryModel.MainFromClause.FromExpression); - // PropertyName = _propertyNameInferrerParser.Parser(GetFullPropertyPath(expression.QueryModel.MainFromClause.FromExpression)); - Type itemType = containsResultOperator.Item.Type; - Type nonNullableType = Nullable.GetUnderlyingType(itemType) ?? itemType; - // Handling different types - if (itemType == typeof(Guid) || nonNullableType == typeof(Guid)) - { - query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); - } - else if (itemType == typeof(Guid?) || nonNullableType == typeof(Guid?)) - { - query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x?.ToString())); - } - else if (nonNullableType == typeof(int) || nonNullableType == typeof(int?)) { - query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); - } - else if (nonNullableType == typeof(long) || nonNullableType == typeof(long?)) { - query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); - } - else if (nonNullableType == typeof(double) || nonNullableType == typeof(double?)) { - query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); - } - else if (nonNullableType == typeof(DateTime) || nonNullableType == typeof(DateTime?)) { - query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString("o"))); // ISO 8601 format - } - else if (nonNullableType == typeof(bool) || nonNullableType == typeof(bool?)) { - query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); - } - else if (nonNullableType == typeof(string)) { - query = new TermsNode(PropertyName, (IEnumerable)Value); - } - else - { - throw new NotSupportedException($"Type {nonNullableType.Name} is not supported for Terms queries."); - } + query = GetDifferentTypesTermsQueryNode(); QueryMap[expression] = ParseQuery(query); break; @@ -254,28 +221,33 @@ protected override Expression VisitSubQuery(SubQueryExpression expression) foreach (var whereClause in whereClauses) { - Visit(whereClause.Predicate); - Node tmp = (Node)QueryMap[whereClause.Predicate].Clone(); - QueryMap[expression] = tmp; - QueryMap[expression].IsSubQuery = true; - QueryMap[expression].SubQueryPath = from; //from.ToLower(); - QueryMap[expression].SubQueryFullPath = fullPath; -// VisitBinarySetSubQuery((BinaryExpression)whereClause.Predicate, from, fullPath, true); if (whereClause.Predicate is SubQueryExpression subQueryExpression) { - // HandleNestedContains(subQueryExpression, tmp); - return base.VisitSubQuery(expression); + var parentPropertyName = PropertyName.Clone().ToString(); + HandleNestedContains(subQueryExpression, expression, from, + fullPath); } - BinaryExpression predicate = (BinaryExpression)whereClause.Predicate; - if (predicate.Left is BinaryExpression) + else { - VisitBinarySetSubQuery((BinaryExpression)predicate.Left, from, fullPath, true); - } - - if (predicate.Right is BinaryExpression) - { - VisitBinarySetSubQuery((BinaryExpression)predicate.Right, from, fullPath, true); + Visit(whereClause.Predicate); + Node tmp = (Node)QueryMap[whereClause.Predicate].Clone(); + QueryMap[expression] = tmp; + QueryMap[expression].IsSubQuery = true; + QueryMap[expression].SubQueryPath = from; //from.ToLower(); + QueryMap[expression].SubQueryFullPath = fullPath; +// VisitBinarySetSubQuery((BinaryExpression)whereClause.Predicate, from, fullPath, true); + BinaryExpression predicate = (BinaryExpression)whereClause.Predicate; + if (predicate.Left is BinaryExpression) + { + VisitBinarySetSubQuery((BinaryExpression)predicate.Left, from, fullPath, true); + } + + if (predicate.Right is BinaryExpression) + { + VisitBinarySetSubQuery((BinaryExpression)predicate.Right, from, fullPath, true); + } } + } break; @@ -313,34 +285,76 @@ protected override Expression VisitSubQuery(SubQueryExpression expression) return expression; } - // private void HandleNestedContains(SubQueryExpression subQueryExpression, Node parent) - // { - // if (subQueryExpression == null || parent == null) - // throw new ArgumentNullException("SubQueryExpression or parent Node cannot be null."); - // - // PropertyName = _propertyNameInferrerParser.Parser(GetFullPropertyPath(subQueryExpression)); - // PropertyType = Nullable.GetUnderlyingType(subQueryExpression.Type) ?? subQueryExpression.Type; - // Node query; - // if (PropertyType == typeof(Guid) || PropertyType == typeof(Guid)) - // { - // query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); - // } - // else if (PropertyType == typeof(int) || PropertyType == typeof(int)) - // { - // query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); - // } - // else if (PropertyType == typeof(string)) - // { - // query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); - // } - // else - // { - // throw new NotSupportedException($"Type {PropertyType.Name} is not supported for Contains queries."); - // } - // // parent.(query); - // QueryMap[subQueryExpression] = ParseQuery(query); - // } - + private void HandleNestedContains(SubQueryExpression subQueryExpression, Expression expression, + string subQueryPath, string subQueryFullPath) + { + if (subQueryExpression == null || expression == null) + throw new ArgumentNullException("SubQueryExpression or expression cannot be null."); + + foreach (var resultOperator in subQueryExpression.QueryModel.ResultOperators) + { + switch (resultOperator) + { + case ContainsResultOperator containsResultOperator: + Visit(containsResultOperator.Item); + Visit(subQueryExpression.QueryModel.MainFromClause.FromExpression); + break; + } + } + + Node query; + query = GetDifferentTypesTermsQueryNode(); + + QueryMap[expression] = ParseQuery(query); + QueryMap[expression].IsSubQuery = true; + QueryMap[expression].SubQueryPath = subQueryPath; //from.ToLower(); + QueryMap[expression].SubQueryFullPath = subQueryFullPath; + } + + private Node GetDifferentTypesTermsQueryNode() + { + Node query; + if (PropertyType == typeof(Guid) || PropertyType == typeof(Guid)) + { + query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + } + else if (PropertyType == typeof(Guid?) || PropertyType == typeof(Guid?)) + { + query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x?.ToString())); + } + else if (PropertyType == typeof(int) || PropertyType == typeof(int?)) + { + query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + } + else if (PropertyType == typeof(long) || PropertyType == typeof(long?)) + { + query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + } + else if (PropertyType == typeof(double) || PropertyType == typeof(double?)) + { + query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + } + else if (PropertyType == typeof(DateTime) || PropertyType == typeof(DateTime?)) + { + query = new TermsNode(PropertyName, + ((IEnumerable)Value).Select(x => x.ToString("o"))); // ISO 8601 format + } + else if (PropertyType == typeof(bool) || PropertyType == typeof(bool?)) + { + query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + } + else if (PropertyType == typeof(string)) + { + query = new TermsNode(PropertyName, (IEnumerable)Value); + } + else + { + throw new NotSupportedException($"Type {PropertyType.Name} is not supported for Terms queries."); + } + + return query; + } + private string GetMemberName(Expression expression) { if (expression is MemberExpression memberExpression) diff --git a/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs b/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs index b633b799..d9a31a26 100644 --- a/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs +++ b/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs @@ -658,16 +658,16 @@ public async Task GetNestedList_Terms_Test() 101, 103 }; - - // var predicates = inputs - // .Select(s => (Expression>)(info => info.LogEvents.Any(x => x.BlockHeight == s))) - // .Aggregate((prev, next) => prev.Or(next)); + var queryable_predicate = await _transactionIndexRepository.GetQueryableAsync(); + var predicates = inputs + .Select(s => (Expression>)(info => info.LogEvents.Any(x => x.BlockHeight == s))) + .Aggregate((prev, next) => prev.Or(next)); + var filterList_predicate = queryable_predicate.Where(predicates).ToList(); Expression> mustQuery = item => item.LogEvents.Any(x => inputs.Contains(x.BlockHeight)); var queryable = await _transactionIndexRepository.GetQueryableAsync(); - // var filterList = queryable.Where(predicates).ToList(); var filterList = queryable.Where(mustQuery).ToList(); filterList.Count.ShouldBe(2); } From 333a6c34967bf239cb5e787f7f2196b11fa82e5b Mon Sep 17 00:00:00 2001 From: Bourne Shi Date: Wed, 16 Oct 2024 15:30:49 +0800 Subject: [PATCH 03/15] feat: optimize code --- .../IElasticsearchClientProvider.cs | 30 +++++++++---------- .../Linq/GeneratorExpressionTreeVisitor.cs | 1 - .../ElasticsearchRepositoryTests.cs | 1 + 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/AElf.EntityMapping.Elasticsearch/IElasticsearchClientProvider.cs b/src/AElf.EntityMapping.Elasticsearch/IElasticsearchClientProvider.cs index f8e38ce7..1b2cfeca 100644 --- a/src/AElf.EntityMapping.Elasticsearch/IElasticsearchClientProvider.cs +++ b/src/AElf.EntityMapping.Elasticsearch/IElasticsearchClientProvider.cs @@ -20,21 +20,21 @@ public ElasticsearchClientProvider(IOptions options) { var uris = options.Value.Uris.ConvertAll(x => new Uri(x)); var connectionPool = new StaticConnectionPool(uris); - var settings = new ConnectionSettings(connectionPool) - .DisableDirectStreaming() - .OnRequestCompleted(callDetails => - { - // Print Request DSL - if (callDetails.RequestBodyInBytes != null) - { - Console.WriteLine($"Request JSON: {Encoding.UTF8.GetString(callDetails.RequestBodyInBytes)}"); - } - // // Print Response Data - // if (callDetails.ResponseBodyInBytes != null) - // { - // Console.WriteLine($"Response JSON: {Encoding.UTF8.GetString(callDetails.ResponseBodyInBytes)}"); - // } - }); + var settings = new ConnectionSettings(connectionPool); + // .DisableDirectStreaming() + // .OnRequestCompleted(callDetails => + // { + // // Print Request DSL + // if (callDetails.RequestBodyInBytes != null) + // { + // Console.WriteLine($"Request JSON: {Encoding.UTF8.GetString(callDetails.RequestBodyInBytes)}"); + // } + // // // Print Response Data + // // if (callDetails.ResponseBodyInBytes != null) + // // { + // // Console.WriteLine($"Response JSON: {Encoding.UTF8.GetString(callDetails.ResponseBodyInBytes)}"); + // // } + // }); _elasticClient = new ElasticClient(settings); } diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs index 03a00697..37dcd937 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs @@ -223,7 +223,6 @@ protected override Expression VisitSubQuery(SubQueryExpression expression) { if (whereClause.Predicate is SubQueryExpression subQueryExpression) { - var parentPropertyName = PropertyName.Clone().ToString(); HandleNestedContains(subQueryExpression, expression, from, fullPath); } diff --git a/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs b/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs index d9a31a26..62eb4322 100644 --- a/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs +++ b/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs @@ -663,6 +663,7 @@ public async Task GetNestedList_Terms_Test() .Select(s => (Expression>)(info => info.LogEvents.Any(x => x.BlockHeight == s))) .Aggregate((prev, next) => prev.Or(next)); var filterList_predicate = queryable_predicate.Where(predicates).ToList(); + filterList_predicate.Count.ShouldBe(2); Expression> mustQuery = item => item.LogEvents.Any(x => inputs.Contains(x.BlockHeight)); From 779ae77a924776c5f25ebced825cc25ae4c2dee0 Mon Sep 17 00:00:00 2001 From: Bourne Shi Date: Wed, 16 Oct 2024 17:02:23 +0800 Subject: [PATCH 04/15] fix: unit test --- .../Repositories/ElasticsearchRepositoryTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs b/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs index 62eb4322..a0ca41f3 100644 --- a/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs +++ b/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs @@ -623,6 +623,7 @@ public async Task GetList_Terms_Test() .Select(s => (Expression>)(info => info.BlockHash == s)) .Aggregate((prev, next) => prev.Or(next)); var filterList_predicate = queryable.Where(predicates).ToList(); + filterList_predicate.Count.ShouldBe(3); var filterList = queryable.Where(item => inputs.Contains(item.BlockHash)).ToList(); filterList.Count.ShouldBe(3); From e8278edf055a01b0fb36bff77b3c923d42ecb0e1 Mon Sep 17 00:00:00 2001 From: Bourne Shi Date: Thu, 17 Oct 2024 18:17:10 +0800 Subject: [PATCH 05/15] perf: optimize code --- .../Linq/GeneratorExpressionTreeVisitor.cs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs index 37dcd937..62e426c6 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs @@ -313,14 +313,10 @@ private void HandleNestedContains(SubQueryExpression subQueryExpression, Express private Node GetDifferentTypesTermsQueryNode() { Node query; - if (PropertyType == typeof(Guid) || PropertyType == typeof(Guid)) + if (PropertyType == typeof(Guid) || PropertyType == typeof(Guid?)) { query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); } - else if (PropertyType == typeof(Guid?) || PropertyType == typeof(Guid?)) - { - query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x?.ToString())); - } else if (PropertyType == typeof(int) || PropertyType == typeof(int?)) { query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); @@ -336,7 +332,7 @@ private Node GetDifferentTypesTermsQueryNode() else if (PropertyType == typeof(DateTime) || PropertyType == typeof(DateTime?)) { query = new TermsNode(PropertyName, - ((IEnumerable)Value).Select(x => x.ToString("o"))); // ISO 8601 format + ((IEnumerable)Value).Select(x => x.ToString())); } else if (PropertyType == typeof(bool) || PropertyType == typeof(bool?)) { From 9d339a4d3a3c743fa932a2c785a903838eecd013 Mon Sep 17 00:00:00 2001 From: Bourne Shi Date: Thu, 17 Oct 2024 18:27:18 +0800 Subject: [PATCH 06/15] feat: optimize property type nullable --- .../Linq/GeneratorExpressionTreeVisitor.cs | 37 ++++++++++++++++--- 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs index 62e426c6..271373d0 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs @@ -313,31 +313,56 @@ private void HandleNestedContains(SubQueryExpression subQueryExpression, Express private Node GetDifferentTypesTermsQueryNode() { Node query; - if (PropertyType == typeof(Guid) || PropertyType == typeof(Guid?)) + if (PropertyType == typeof(Guid)) { query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); } - else if (PropertyType == typeof(int) || PropertyType == typeof(int?)) + else if (PropertyType == typeof(Guid?)) + { + query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + } + else if (PropertyType == typeof(int)) { query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); } - else if (PropertyType == typeof(long) || PropertyType == typeof(long?)) + else if (PropertyType == typeof(int?)) + { + query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + } + else if (PropertyType == typeof(long)) { query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); } - else if (PropertyType == typeof(double) || PropertyType == typeof(double?)) + else if (PropertyType == typeof(long?)) + { + query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + } + else if (PropertyType == typeof(double)) { query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); } - else if (PropertyType == typeof(DateTime) || PropertyType == typeof(DateTime?)) + else if(PropertyType == typeof(double?)) + { + query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + } + else if (PropertyType == typeof(DateTime)) { query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); } - else if (PropertyType == typeof(bool) || PropertyType == typeof(bool?)) + else if (PropertyType == typeof(DateTime?)) + { + query = new TermsNode(PropertyName, + ((IEnumerable)Value).Select(x => x.ToString())); + } + else if (PropertyType == typeof(bool)) { query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); } + else if (PropertyType == typeof(bool?)) + { + query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + } else if (PropertyType == typeof(string)) { query = new TermsNode(PropertyName, (IEnumerable)Value); From a3a8b6f7ae89de61b1bd888252d0ce250ecf8ed3 Mon Sep 17 00:00:00 2001 From: Bourne Shi Date: Fri, 18 Oct 2024 11:58:30 +0800 Subject: [PATCH 07/15] feat: add terms array length limit --- .../IElasticsearchQueryableFactory.cs | 12 ++++-- .../Linq/ElasticGeneratorQueryModelVisitor.cs | 10 ++++- .../Linq/ElasticsearchQueryExecutor.cs | 9 +++- .../Linq/ElasticsearchQueryable.cs | 7 +-- .../Linq/GeneratorExpressionTreeVisitor.cs | 43 +++++++++++++------ .../Options/ElasticsearchOptions.cs | 1 + .../ElasticsearchRepositoryTests.cs | 22 +++++----- 7 files changed, 69 insertions(+), 35 deletions(-) diff --git a/src/AElf.EntityMapping.Elasticsearch/IElasticsearchQueryableFactory.cs b/src/AElf.EntityMapping.Elasticsearch/IElasticsearchQueryableFactory.cs index 73955e8b..fa16054e 100644 --- a/src/AElf.EntityMapping.Elasticsearch/IElasticsearchQueryableFactory.cs +++ b/src/AElf.EntityMapping.Elasticsearch/IElasticsearchQueryableFactory.cs @@ -1,4 +1,6 @@ using AElf.EntityMapping.Elasticsearch.Linq; +using AElf.EntityMapping.Elasticsearch.Options; +using Microsoft.Extensions.Options; using Nest; using Volo.Abp.Domain.Entities; @@ -14,14 +16,18 @@ public class ElasticsearchQueryableFactory : IElasticsearchQueryableFac where TEntity : class, IEntity { private readonly ICollectionNameProvider _collectionNameProvider; + private readonly ElasticsearchOptions _elasticsearchOptions; - public ElasticsearchQueryableFactory(ICollectionNameProvider collectionNameProvider) + public ElasticsearchQueryableFactory(ICollectionNameProvider collectionNameProvider, + IOptions elasticsearchOptions) { _collectionNameProvider = collectionNameProvider; + _elasticsearchOptions = elasticsearchOptions.Value; } - public ElasticsearchQueryable Create(IElasticClient client, string index = null) + public ElasticsearchQueryable Create(IElasticClient client, + string index = null) { - return new ElasticsearchQueryable(client, _collectionNameProvider, index); + return new ElasticsearchQueryable(client, _collectionNameProvider, index, _elasticsearchOptions); } } \ No newline at end of file diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticGeneratorQueryModelVisitor.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticGeneratorQueryModelVisitor.cs index b8562067..77ecbe7d 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticGeneratorQueryModelVisitor.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticGeneratorQueryModelVisitor.cs @@ -1,6 +1,9 @@ using System.Collections.ObjectModel; using System.Linq.Expressions; +using AElf.EntityMapping.Elasticsearch.Options; using AElf.EntityMapping.Linq; +using AElf.EntityMapping.Options; +using Microsoft.Extensions.Options; using Nest; using Remotion.Linq; using Remotion.Linq.Clauses; @@ -13,12 +16,15 @@ public class ElasticsearchGeneratorQueryModelVisitor : QueryModelVisitorBase { private readonly PropertyNameInferrerParser _propertyNameInferrerParser; private readonly INodeVisitor _nodeVisitor; + private readonly ElasticsearchOptions _elasticsearchOptions; private QueryAggregator QueryAggregator { get; set; } = new QueryAggregator(); - public ElasticsearchGeneratorQueryModelVisitor(PropertyNameInferrerParser propertyNameInferrerParser) + public ElasticsearchGeneratorQueryModelVisitor(PropertyNameInferrerParser propertyNameInferrerParser, + ElasticsearchOptions elasticsearchOptions) { _propertyNameInferrerParser = propertyNameInferrerParser; _nodeVisitor = new NodeVisitor(); + _elasticsearchOptions = elasticsearchOptions; } public QueryAggregator GenerateElasticQuery(QueryModel queryModel) @@ -48,7 +54,7 @@ public override void VisitMainFromClause(MainFromClause fromClause, QueryModel q public override void VisitWhereClause(WhereClause whereClause, QueryModel queryModel, int index) { - var tree = new GeneratorExpressionTreeVisitor(_propertyNameInferrerParser); + var tree = new GeneratorExpressionTreeVisitor(_propertyNameInferrerParser, _elasticsearchOptions); tree.Visit(whereClause.Predicate); if (QueryAggregator.Query == null) { diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryExecutor.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryExecutor.cs index efd629c2..1d5b5216 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryExecutor.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryExecutor.cs @@ -3,6 +3,7 @@ using System.Dynamic; using System.Linq.Expressions; using AElf.EntityMapping.Elasticsearch.Exceptions; +using AElf.EntityMapping.Elasticsearch.Options; using Elasticsearch.Net; using Nest; using Newtonsoft.Json; @@ -23,16 +24,20 @@ public class ElasticsearchQueryExecutor: IQueryExecutor private readonly JsonSerializerSettings _deserializerSettings; private readonly ICollectionNameProvider _collectionNameProvider; private const int ElasticQueryLimit = 10000; + private readonly ElasticsearchOptions _elasticsearchOptions; public ElasticsearchQueryExecutor(IElasticClient elasticClient, - ICollectionNameProvider collectionNameProvider, string index) + ICollectionNameProvider collectionNameProvider, string index, + ElasticsearchOptions elasticsearchOptions) { _elasticClient = elasticClient; _collectionNameProvider = collectionNameProvider; _index = index; _propertyNameInferrerParser = new PropertyNameInferrerParser(_elasticClient); + _elasticsearchOptions = elasticsearchOptions; _elasticsearchGeneratorQueryModelVisitor = - new ElasticsearchGeneratorQueryModelVisitor(_propertyNameInferrerParser); + new ElasticsearchGeneratorQueryModelVisitor(_propertyNameInferrerParser, + _elasticsearchOptions); _deserializerSettings = new JsonSerializerSettings { // Nest maps TimeSpan as a long (TimeSpan ticks) diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryable.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryable.cs index 074c5bd8..6e0cf259 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryable.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryable.cs @@ -1,4 +1,5 @@ using System.Linq.Expressions; +using AElf.EntityMapping.Elasticsearch.Options; using AElf.EntityMapping.Linq; using Nest; using Remotion.Linq; @@ -10,10 +11,10 @@ public class ElasticsearchQueryable : QueryableBase, IElasticsearchQueryab where T : class, IEntity { public ElasticsearchQueryable(IElasticClient elasticClient, ICollectionNameProvider collectionNameProvider, - string index) - : base(new DefaultQueryProvider(typeof(ElasticsearchQueryable<>), + string index, ElasticsearchOptions elasticsearchOptions) + : base(new DefaultQueryProvider(typeof(ElasticsearchQueryable<>), QueryParserFactory.Create(), - new ElasticsearchQueryExecutor(elasticClient, collectionNameProvider, index))) + new ElasticsearchQueryExecutor(elasticClient, collectionNameProvider, index, elasticsearchOptions))) { } diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs index 271373d0..34bae13a 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs @@ -1,5 +1,5 @@ using System.Linq.Expressions; -using System.Reflection; +using AElf.EntityMapping.Elasticsearch.Options; using Elasticsearch.Net; using Remotion.Linq.Clauses; using Remotion.Linq.Clauses.Expressions; @@ -11,6 +11,7 @@ namespace AElf.EntityMapping.Elasticsearch.Linq public class GeneratorExpressionTreeVisitor : ThrowingExpressionVisitor { private readonly PropertyNameInferrerParser _propertyNameInferrerParser; + private readonly ElasticsearchOptions _elasticsearchOptions; private object Value { get; set; } private string PropertyName { get; set; } @@ -20,9 +21,11 @@ public class GeneratorExpressionTreeVisitor : ThrowingExpressionVisitor public IDictionary QueryMap { get; } = new Dictionary(); - public GeneratorExpressionTreeVisitor(PropertyNameInferrerParser propertyNameInferrerParser) + public GeneratorExpressionTreeVisitor(PropertyNameInferrerParser propertyNameInferrerParser, + ElasticsearchOptions elasticsearchOptions) { _propertyNameInferrerParser = propertyNameInferrerParser; + _elasticsearchOptions = elasticsearchOptions; } protected override Expression VisitUnary(UnaryExpression expression) @@ -203,7 +206,14 @@ protected override Expression VisitSubQuery(SubQueryExpression expression) case ContainsResultOperator containsResultOperator: Visit(containsResultOperator.Item); Visit(expression.QueryModel.MainFromClause.FromExpression); - + + //Check if the number of items in the Terms query array within the Contains clause is too large. + if (expression.QueryModel.MainFromClause + .FromExpression is ConstantExpression constantExpression) + { + CheckTermsArrayLength(constantExpression); + } + // Handling different types query = GetDifferentTypesTermsQueryNode(); @@ -290,6 +300,13 @@ private void HandleNestedContains(SubQueryExpression subQueryExpression, Express if (subQueryExpression == null || expression == null) throw new ArgumentNullException("SubQueryExpression or expression cannot be null."); + //Check if the number of items in the Terms query array within the Contains clause is too large. + if (subQueryExpression.QueryModel.MainFromClause + .FromExpression is ConstantExpression constantExpression) + { + CheckTermsArrayLength(constantExpression); + } + foreach (var resultOperator in subQueryExpression.QueryModel.ResultOperators) { switch (resultOperator) @@ -375,18 +392,16 @@ private Node GetDifferentTypesTermsQueryNode() return query; } - private string GetMemberName(Expression expression) + private void CheckTermsArrayLength(ConstantExpression constantExpression) { - if (expression is MemberExpression memberExpression) - return memberExpression.Member.Name; - throw new InvalidOperationException("Expression does not represent a member access."); - } - - private object GetValueFromExpression(Expression expression) - { - if (expression is ConstantExpression constantExpression) - return constantExpression.Value; - throw new InvalidOperationException("Expression is not a constant."); + if (constantExpression.Value is System.Collections.IEnumerable objectList) + { + var count = objectList.Cast().Count(); + if (count > _elasticsearchOptions.TermsArrayMaxLength) + { + throw new Exception($"The array input for Terms query is too large, exceeding {_elasticsearchOptions.TermsArrayMaxLength} items."); + } + } } protected override Expression VisitQuerySourceReference(QuerySourceReferenceExpression expression) diff --git a/src/AElf.EntityMapping.Elasticsearch/Options/ElasticsearchOptions.cs b/src/AElf.EntityMapping.Elasticsearch/Options/ElasticsearchOptions.cs index bf6e5342..464d72b8 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Options/ElasticsearchOptions.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Options/ElasticsearchOptions.cs @@ -9,4 +9,5 @@ public class ElasticsearchOptions public int NumberOfReplicas { get; set; } = 1; public Refresh Refresh { get; set; } = Refresh.False; public int MaxResultWindow { get; set; } = 10000; + public int TermsArrayMaxLength { get; set; } = 100; } \ No newline at end of file diff --git a/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs b/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs index a0ca41f3..58080351 100644 --- a/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs +++ b/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs @@ -619,11 +619,11 @@ public async Task GetList_Terms_Test() var queryable = await _elasticsearchRepository.GetQueryableAsync(); - var predicates = inputs - .Select(s => (Expression>)(info => info.BlockHash == s)) - .Aggregate((prev, next) => prev.Or(next)); - var filterList_predicate = queryable.Where(predicates).ToList(); - filterList_predicate.Count.ShouldBe(3); + // var predicates = inputs + // .Select(s => (Expression>)(info => info.BlockHash == s)) + // .Aggregate((prev, next) => prev.Or(next)); + // var filterList_predicate = queryable.Where(predicates).ToList(); + // filterList_predicate.Count.ShouldBe(3); var filterList = queryable.Where(item => inputs.Contains(item.BlockHash)).ToList(); filterList.Count.ShouldBe(3); @@ -659,12 +659,12 @@ public async Task GetNestedList_Terms_Test() 101, 103 }; - var queryable_predicate = await _transactionIndexRepository.GetQueryableAsync(); - var predicates = inputs - .Select(s => (Expression>)(info => info.LogEvents.Any(x => x.BlockHeight == s))) - .Aggregate((prev, next) => prev.Or(next)); - var filterList_predicate = queryable_predicate.Where(predicates).ToList(); - filterList_predicate.Count.ShouldBe(2); + // var queryable_predicate = await _transactionIndexRepository.GetQueryableAsync(); + // var predicates = inputs + // .Select(s => (Expression>)(info => info.LogEvents.Any(x => x.BlockHeight == s))) + // .Aggregate((prev, next) => prev.Or(next)); + // var filterList_predicate = queryable_predicate.Where(predicates).ToList(); + // filterList_predicate.Count.ShouldBe(2); Expression> mustQuery = item => item.LogEvents.Any(x => inputs.Contains(x.BlockHeight)); From f2c063dbbd1b69c1d69d455cf410f0109253bab3 Mon Sep 17 00:00:00 2001 From: Bourne Shi Date: Fri, 18 Oct 2024 13:38:08 +0800 Subject: [PATCH 08/15] perf: optimize code --- .../ElasticsearchRepositoryTests.cs | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs b/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs index 58080351..a0ca41f3 100644 --- a/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs +++ b/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs @@ -619,11 +619,11 @@ public async Task GetList_Terms_Test() var queryable = await _elasticsearchRepository.GetQueryableAsync(); - // var predicates = inputs - // .Select(s => (Expression>)(info => info.BlockHash == s)) - // .Aggregate((prev, next) => prev.Or(next)); - // var filterList_predicate = queryable.Where(predicates).ToList(); - // filterList_predicate.Count.ShouldBe(3); + var predicates = inputs + .Select(s => (Expression>)(info => info.BlockHash == s)) + .Aggregate((prev, next) => prev.Or(next)); + var filterList_predicate = queryable.Where(predicates).ToList(); + filterList_predicate.Count.ShouldBe(3); var filterList = queryable.Where(item => inputs.Contains(item.BlockHash)).ToList(); filterList.Count.ShouldBe(3); @@ -659,12 +659,12 @@ public async Task GetNestedList_Terms_Test() 101, 103 }; - // var queryable_predicate = await _transactionIndexRepository.GetQueryableAsync(); - // var predicates = inputs - // .Select(s => (Expression>)(info => info.LogEvents.Any(x => x.BlockHeight == s))) - // .Aggregate((prev, next) => prev.Or(next)); - // var filterList_predicate = queryable_predicate.Where(predicates).ToList(); - // filterList_predicate.Count.ShouldBe(2); + var queryable_predicate = await _transactionIndexRepository.GetQueryableAsync(); + var predicates = inputs + .Select(s => (Expression>)(info => info.LogEvents.Any(x => x.BlockHeight == s))) + .Aggregate((prev, next) => prev.Or(next)); + var filterList_predicate = queryable_predicate.Where(predicates).ToList(); + filterList_predicate.Count.ShouldBe(2); Expression> mustQuery = item => item.LogEvents.Any(x => inputs.Contains(x.BlockHeight)); From 402b22dfd676541d00b6dedcd185ba7f22e66ff2 Mon Sep 17 00:00:00 2001 From: Bourne Shi Date: Sat, 19 Oct 2024 10:56:26 +0800 Subject: [PATCH 09/15] perf: optimize terms supports type --- .../Linq/GeneratorExpressionTreeVisitor.cs | 47 ++++--------------- 1 file changed, 10 insertions(+), 37 deletions(-) diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs index 34bae13a..37db570d 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs @@ -327,41 +327,22 @@ private void HandleNestedContains(SubQueryExpression subQueryExpression, Express QueryMap[expression].SubQueryFullPath = subQueryFullPath; } + private static readonly HashSet SupportedTermsTypes = new HashSet + { + typeof(Guid), typeof(Guid?), + typeof(int), typeof(int?), + typeof(long), typeof(long?), + typeof(double), typeof(double?), + typeof(bool), typeof(bool?) + }; + private Node GetDifferentTypesTermsQueryNode() { Node query; - if (PropertyType == typeof(Guid)) + if (SupportedTermsTypes.Contains(PropertyType)) { query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); } - else if (PropertyType == typeof(Guid?)) - { - query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); - } - else if (PropertyType == typeof(int)) - { - query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); - } - else if (PropertyType == typeof(int?)) - { - query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); - } - else if (PropertyType == typeof(long)) - { - query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); - } - else if (PropertyType == typeof(long?)) - { - query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); - } - else if (PropertyType == typeof(double)) - { - query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); - } - else if(PropertyType == typeof(double?)) - { - query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); - } else if (PropertyType == typeof(DateTime)) { query = new TermsNode(PropertyName, @@ -372,14 +353,6 @@ private Node GetDifferentTypesTermsQueryNode() query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); } - else if (PropertyType == typeof(bool)) - { - query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); - } - else if (PropertyType == typeof(bool?)) - { - query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); - } else if (PropertyType == typeof(string)) { query = new TermsNode(PropertyName, (IEnumerable)Value); From 88b35d338ce2eba22a0ed66c617e7b90c9c6b3f6 Mon Sep 17 00:00:00 2001 From: Bourne Shi Date: Sat, 19 Oct 2024 11:32:49 +0800 Subject: [PATCH 10/15] feat: recover code --- .../Linq/GeneratorExpressionTreeVisitor.cs | 94 +++++++++++++------ 1 file changed, 63 insertions(+), 31 deletions(-) diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs index 37db570d..68ad5142 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs @@ -133,22 +133,22 @@ protected override Expression VisitMethodCall(MethodCallExpression expression) // private string GetFullNameKey(MemberExpression memberExpression) // { - // var key = _propertyNameInferrerParser.Parser(memberExpression.Member.Name); - // while (memberExpression.Expression != null) - // { - // memberExpression = memberExpression.Expression as MemberExpression; - // if (memberExpression == null) - // { - // break; - // } - // - // key = _propertyNameInferrerParser.Parser(memberExpression.Member.Name + "." + key); - // return key; - // } - // - // return key; + // var key = _propertyNameInferrerParser.Parser(memberExpression.Member.Name); + // while (memberExpression.Expression != null) + // { + // memberExpression = memberExpression.Expression as MemberExpression; + // if (memberExpression == null) + // { + // break; + // } + // + // key = _propertyNameInferrerParser.Parser(memberExpression.Member.Name + "." + key); + // return key; // } - + // + // return key; + // } + private string GetFullPropertyPath(Expression expression) { switch (expression) @@ -166,6 +166,7 @@ private string GetFullPropertyPath(Expression expression) var collectionPath = GetFullPropertyPath(methodCallExpression.Object); return collectionPath; // Returns the path of the collection directly, without adding an index } + break; } @@ -256,7 +257,7 @@ protected override Expression VisitSubQuery(SubQueryExpression expression) VisitBinarySetSubQuery((BinaryExpression)predicate.Right, from, fullPath, true); } } - + } break; @@ -299,14 +300,14 @@ private void HandleNestedContains(SubQueryExpression subQueryExpression, Express { if (subQueryExpression == null || expression == null) throw new ArgumentNullException("SubQueryExpression or expression cannot be null."); - + //Check if the number of items in the Terms query array within the Contains clause is too large. if (subQueryExpression.QueryModel.MainFromClause .FromExpression is ConstantExpression constantExpression) { CheckTermsArrayLength(constantExpression); } - + foreach (var resultOperator in subQueryExpression.QueryModel.ResultOperators) { switch (resultOperator) @@ -320,29 +321,48 @@ private void HandleNestedContains(SubQueryExpression subQueryExpression, Express Node query; query = GetDifferentTypesTermsQueryNode(); - + QueryMap[expression] = ParseQuery(query); QueryMap[expression].IsSubQuery = true; QueryMap[expression].SubQueryPath = subQueryPath; //from.ToLower(); QueryMap[expression].SubQueryFullPath = subQueryFullPath; } - - private static readonly HashSet SupportedTermsTypes = new HashSet - { - typeof(Guid), typeof(Guid?), - typeof(int), typeof(int?), - typeof(long), typeof(long?), - typeof(double), typeof(double?), - typeof(bool), typeof(bool?) - }; private Node GetDifferentTypesTermsQueryNode() { Node query; - if (SupportedTermsTypes.Contains(PropertyType)) + if (PropertyType == typeof(Guid)) { query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); } + else if (PropertyType == typeof(Guid?)) + { + query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + } + else if (PropertyType == typeof(int)) + { + query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + } + else if (PropertyType == typeof(int?)) + { + query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + } + else if (PropertyType == typeof(long)) + { + query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + } + else if (PropertyType == typeof(long?)) + { + query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + } + else if (PropertyType == typeof(double)) + { + query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + } + else if (PropertyType == typeof(double?)) + { + query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + } else if (PropertyType == typeof(DateTime)) { query = new TermsNode(PropertyName, @@ -353,6 +373,14 @@ private Node GetDifferentTypesTermsQueryNode() query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); } + else if (PropertyType == typeof(bool)) + { + query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + } + else if (PropertyType == typeof(bool?)) + { + query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + } else if (PropertyType == typeof(string)) { query = new TermsNode(PropertyName, (IEnumerable)Value); @@ -365,6 +393,8 @@ private Node GetDifferentTypesTermsQueryNode() return query; } + + private void CheckTermsArrayLength(ConstantExpression constantExpression) { if (constantExpression.Value is System.Collections.IEnumerable objectList) @@ -372,7 +402,8 @@ private void CheckTermsArrayLength(ConstantExpression constantExpression) var count = objectList.Cast().Count(); if (count > _elasticsearchOptions.TermsArrayMaxLength) { - throw new Exception($"The array input for Terms query is too large, exceeding {_elasticsearchOptions.TermsArrayMaxLength} items."); + throw new Exception( + $"The array input for Terms query is too large, exceeding {_elasticsearchOptions.TermsArrayMaxLength} items."); } } } @@ -693,7 +724,8 @@ private object ConvertEnumValue(Type entityType, string propertyName, object val return (int)enumValue; } - protected void VisitBinarySetSubQuery(BinaryExpression expression, string path, string fullPath, bool parentIsSubQuery) + protected void VisitBinarySetSubQuery(BinaryExpression expression, string path, string fullPath, + bool parentIsSubQuery) { if (expression.Left is BinaryExpression && expression.Right is ConstantExpression) { From eb4df6744eb10e1e0e34c589f1cd60489a733520 Mon Sep 17 00:00:00 2001 From: Bourne Shi Date: Sat, 19 Oct 2024 11:42:13 +0800 Subject: [PATCH 11/15] perf: optimize get terms node code --- .../Linq/GeneratorExpressionTreeVisitor.cs | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs index 68ad5142..83e6cc49 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs @@ -333,53 +333,51 @@ private Node GetDifferentTypesTermsQueryNode() Node query; if (PropertyType == typeof(Guid)) { - query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + query = GetTermsNode(); } else if (PropertyType == typeof(Guid?)) { - query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + query = GetTermsNode(); } else if (PropertyType == typeof(int)) { - query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + query = GetTermsNode(); } else if (PropertyType == typeof(int?)) { - query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + query = GetTermsNode(); } else if (PropertyType == typeof(long)) { - query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + query = GetTermsNode(); } else if (PropertyType == typeof(long?)) { - query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + query = GetTermsNode(); } else if (PropertyType == typeof(double)) { - query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + query = GetTermsNode(); } else if (PropertyType == typeof(double?)) { - query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + query = GetTermsNode(); } else if (PropertyType == typeof(DateTime)) { - query = new TermsNode(PropertyName, - ((IEnumerable)Value).Select(x => x.ToString())); + query = GetTermsNode(); } else if (PropertyType == typeof(DateTime?)) { - query = new TermsNode(PropertyName, - ((IEnumerable)Value).Select(x => x.ToString())); + query = GetTermsNode(); } else if (PropertyType == typeof(bool)) { - query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + query = GetTermsNode(); } else if (PropertyType == typeof(bool?)) { - query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + query = GetTermsNode(); } else if (PropertyType == typeof(string)) { @@ -393,7 +391,11 @@ private Node GetDifferentTypesTermsQueryNode() return query; } - + private TermsNode GetTermsNode() + { + return new TermsNode(PropertyName, + ((IEnumerable)Value).Select(x => x.ToString())); + } private void CheckTermsArrayLength(ConstantExpression constantExpression) { From 04e581489149c149937bfa08e124b80f05a1cba6 Mon Sep 17 00:00:00 2001 From: Bourne Shi Date: Sat, 19 Oct 2024 11:52:38 +0800 Subject: [PATCH 12/15] perf: optimize exception type --- .../Linq/GeneratorExpressionTreeVisitor.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs index 83e6cc49..34dcddda 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs @@ -404,7 +404,7 @@ private void CheckTermsArrayLength(ConstantExpression constantExpression) var count = objectList.Cast().Count(); if (count > _elasticsearchOptions.TermsArrayMaxLength) { - throw new Exception( + throw new ArgumentException( $"The array input for Terms query is too large, exceeding {_elasticsearchOptions.TermsArrayMaxLength} items."); } } From f5d445e56531d951f76b0d142c0ef23d83b76dd5 Mon Sep 17 00:00:00 2001 From: Bourne Shi Date: Sat, 19 Oct 2024 19:40:39 +0800 Subject: [PATCH 13/15] fix: terms query datetime --- .../Linq/GeneratorExpressionTreeVisitor.cs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs index 34dcddda..d3d1c33b 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs @@ -363,14 +363,6 @@ private Node GetDifferentTypesTermsQueryNode() { query = GetTermsNode(); } - else if (PropertyType == typeof(DateTime)) - { - query = GetTermsNode(); - } - else if (PropertyType == typeof(DateTime?)) - { - query = GetTermsNode(); - } else if (PropertyType == typeof(bool)) { query = GetTermsNode(); @@ -379,6 +371,16 @@ private Node GetDifferentTypesTermsQueryNode() { query = GetTermsNode(); } + else if (PropertyType == typeof(DateTime)) + { + query = new TermsNode(PropertyName, + ((IEnumerable)Value).Select(x => x.ToString("o"))); + } + else if (PropertyType == typeof(DateTime?)) + { + query = new TermsNode(PropertyName, + ((IEnumerable)Value).Select(x => x.ToString("o"))); + } else if (PropertyType == typeof(string)) { query = new TermsNode(PropertyName, (IEnumerable)Value); From 47a332602de5777783c9d9003797d4fedb7a98dd Mon Sep 17 00:00:00 2001 From: Bourne Shi Date: Sat, 19 Oct 2024 20:35:42 +0800 Subject: [PATCH 14/15] feat: add terms query unit test of datetime type --- .../Repositories/ElasticsearchRepositoryTests.cs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs b/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs index a0ca41f3..98f9b5a3 100644 --- a/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs +++ b/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs @@ -596,6 +596,7 @@ public async Task GetList_Nested_Test() [Fact] public async Task GetList_Terms_Test() { + var timeNow = DateTime.Now; for (int i = 1; i <= 7; i++) { var blockIndex = new BlockIndex @@ -603,7 +604,7 @@ public async Task GetList_Terms_Test() Id = "block" + i, BlockHash = "BlockHash" + i, BlockHeight = i, - BlockTime = DateTime.Now.AddDays(-10 + i), + BlockTime = timeNow.AddDays(i), LogEventCount = i, ChainId = "AELF" }; @@ -627,7 +628,7 @@ public async Task GetList_Terms_Test() var filterList = queryable.Where(item => inputs.Contains(item.BlockHash)).ToList(); filterList.Count.ShouldBe(3); - + List heights = new List() { 4, 5 @@ -635,6 +636,14 @@ public async Task GetList_Terms_Test() Expression> mustQuery = item => heights.Contains(item.BlockHeight); var filterList_heights = queryable.Where(mustQuery).ToList(); filterList_heights.Count.ShouldBe(2); + + List times = new List() + { + DateTime.Now, timeNow.AddDays(1), timeNow.AddDays(2) + }; + Expression> termsQuery = item => times.Contains(item.BlockTime); + var filterList_times = queryable.Where(termsQuery).ToList(); + filterList_times.Count.ShouldBe(2); } [Fact] From f582f0c7956fc736987bba919cd575ebe9823917 Mon Sep 17 00:00:00 2001 From: Bourne Shi Date: Mon, 21 Oct 2024 12:03:27 +0800 Subject: [PATCH 15/15] feat: optimize code --- .../Linq/GeneratorExpressionTreeVisitor.cs | 25 ------------------- 1 file changed, 25 deletions(-) diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs index d3d1c33b..9c71ce65 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs @@ -335,52 +335,27 @@ private Node GetDifferentTypesTermsQueryNode() { query = GetTermsNode(); } - else if (PropertyType == typeof(Guid?)) - { - query = GetTermsNode(); - } else if (PropertyType == typeof(int)) { query = GetTermsNode(); } - else if (PropertyType == typeof(int?)) - { - query = GetTermsNode(); - } else if (PropertyType == typeof(long)) { query = GetTermsNode(); } - else if (PropertyType == typeof(long?)) - { - query = GetTermsNode(); - } else if (PropertyType == typeof(double)) { query = GetTermsNode(); } - else if (PropertyType == typeof(double?)) - { - query = GetTermsNode(); - } else if (PropertyType == typeof(bool)) { query = GetTermsNode(); } - else if (PropertyType == typeof(bool?)) - { - query = GetTermsNode(); - } else if (PropertyType == typeof(DateTime)) { query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString("o"))); } - else if (PropertyType == typeof(DateTime?)) - { - query = new TermsNode(PropertyName, - ((IEnumerable)Value).Select(x => x.ToString("o"))); - } else if (PropertyType == typeof(string)) { query = new TermsNode(PropertyName, (IEnumerable)Value);