diff --git a/src/main/java/com/autotune/analyzer/serviceObjects/DSMetadataAPIObject.java b/src/main/java/com/autotune/analyzer/serviceObjects/DSMetadataAPIObject.java index 9718c32e6..dab3ad790 100644 --- a/src/main/java/com/autotune/analyzer/serviceObjects/DSMetadataAPIObject.java +++ b/src/main/java/com/autotune/analyzer/serviceObjects/DSMetadataAPIObject.java @@ -23,6 +23,10 @@ public class DSMetadataAPIObject { private String version; @SerializedName(KruizeConstants.JSONKeys.DATASOURCE_NAME) private String dataSourceName; + @SerializedName(KruizeConstants.JSONKeys.METADATA_PROFILE) + private String metadataProfileName; + @SerializedName(KruizeConstants.JSONKeys.MEASUREMENT_DURATION) + private String measurement_durationMinutes; public String getVersion() { return version; @@ -36,4 +40,11 @@ public String getDataSourceName() { return dataSourceName; } + public String getMetadataProfile() { return metadataProfileName; } + + public void setMetadataProfile(String metadataProfileName) { this.metadataProfileName = metadataProfileName; } + + public String getMeasurementDurationMinutes() { return measurement_durationMinutes; } + + public void setMeasurementDurationMinutes(String measurement_durationMinutes) {this.measurement_durationMinutes = measurement_durationMinutes;} } diff --git a/src/main/java/com/autotune/analyzer/services/DSMetadataService.java b/src/main/java/com/autotune/analyzer/services/DSMetadataService.java index 762a99d7a..8703db94c 100644 --- a/src/main/java/com/autotune/analyzer/services/DSMetadataService.java +++ b/src/main/java/com/autotune/analyzer/services/DSMetadataService.java @@ -118,6 +118,9 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response) if (validationOutputData.isSuccess()) { String dataSourceName = metadataAPIObject.getDataSourceName(); + String metadataProfileName = metadataAPIObject.getMetadataProfile(); + String measurementDuration = metadataAPIObject.getMeasurementDurationMinutes(); + // fetch the DatasourceInfo object based on datasource name DataSourceInfo datasource; try { @@ -133,7 +136,8 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response) return; } - DataSourceMetadataInfo metadataInfo = dataSourceManager.importMetadataFromDataSource(datasource,"",0,0,0, new HashMap<>(), new HashMap<>()); + DataSourceMetadataInfo metadataInfo = dataSourceManager.importMetadataFromDataSource(metadataProfileName, + datasource,"",0, 0, 0, measurementDuration, new HashMap<>(), new HashMap<>()); // Validate imported metadataInfo object DataSourceMetadataValidation validationObject = new DataSourceMetadataValidation(); @@ -195,9 +199,11 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response) } + //TODO - Add measurement_duration if required private List mandatoryFields = new ArrayList<>(Arrays.asList( AnalyzerConstants.VERSION, - AnalyzerConstants.DATASOURCE_NAME + AnalyzerConstants.DATASOURCE_NAME, + AnalyzerConstants.METADATA_PROFILE )); public ValidationOutputData validateMandatoryFields(DSMetadataAPIObject metadataAPIObject) { diff --git a/src/main/java/com/autotune/analyzer/utils/AnalyzerConstants.java b/src/main/java/com/autotune/analyzer/utils/AnalyzerConstants.java index db256caaa..3586871dd 100644 --- a/src/main/java/com/autotune/analyzer/utils/AnalyzerConstants.java +++ b/src/main/java/com/autotune/analyzer/utils/AnalyzerConstants.java @@ -94,6 +94,9 @@ public class AnalyzerConstants { public static final String KRUIZE_LOCAL_DDL_SQL = "kruize_local_ddl.sql"; public static final String VERSION = "version"; public static final String DATASOURCE_NAME = "dataSourceName"; + public static final String METADATA_PROFILE = "metadataProfile"; + public static final String WORKLOAD = "workload"; + public static final String CONTAINER = "container"; private AnalyzerConstants() { diff --git a/src/main/java/com/autotune/common/data/dataSourceMetadata/DataSourceMetadataHelper.java b/src/main/java/com/autotune/common/data/dataSourceMetadata/DataSourceMetadataHelper.java index c562cf1b4..e68def44e 100644 --- a/src/main/java/com/autotune/common/data/dataSourceMetadata/DataSourceMetadataHelper.java +++ b/src/main/java/com/autotune/common/data/dataSourceMetadata/DataSourceMetadataHelper.java @@ -1,5 +1,6 @@ package com.autotune.common.data.dataSourceMetadata; +import com.autotune.common.data.metrics.Metric; import com.autotune.utils.KruizeConstants; import com.google.gson.JsonArray; import com.google.gson.JsonElement; @@ -9,6 +10,7 @@ import org.slf4j.LoggerFactory; import java.util.HashMap; +import java.util.List; /** * Utility class for handling DataSourceMetadataInfo and related metadata. @@ -445,4 +447,14 @@ public void updateContainerDataSourceMetadataInfoObject(String dataSourceName, D LOGGER.error(KruizeConstants.DataSourceConstants.DataSourceMetadataErrorMsgs.CONTAINER_METADATA_UPDATE_ERROR + e.getMessage()); } } + public String getQueryFromProfile(MetadataProfile metadataProfile, String metricName) { + List metrics = metadataProfile.getQueryVariables(); + for (Metric metric : metrics) { + String name = metric.getName(); + if (name.contains(metricName)) { + return metric.getAggregationFunctionsMap().get(KruizeConstants.JSONKeys.SUM).getQuery(); + } + } + return null; + } } diff --git a/src/main/java/com/autotune/common/data/dataSourceQueries/DataSourceQueries.java b/src/main/java/com/autotune/common/data/dataSourceQueries/DataSourceQueries.java deleted file mode 100644 index a889566c4..000000000 --- a/src/main/java/com/autotune/common/data/dataSourceQueries/DataSourceQueries.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.autotune.common.data.dataSourceQueries; - -/** - * This class contains PromQL queries as enum constants for various metrics related to Kubernetes clusters. - * TODO - Add a custom PromQL query to fetch Cluster info - * - Refactor PromQL queries into a separate YAML file utilizing Custom Resource Definitions (CRD). - */ -public class DataSourceQueries { - public enum PromQLQuery { - NAMESPACE_QUERY("sum by (namespace) ( avg_over_time(kube_namespace_status_phase{%s ADDITIONAL_LABEL}[15d]))"), - WORKLOAD_INFO_QUERY("sum by (namespace, workload, workload_type) ( avg_over_time(namespace_workload_pod:kube_pod_owner:relabel{%s ADDITIONAL_LABEL}[15d]))"), - CONTAINER_INFO_QUERY("sum by (container, image, workload, workload_type, namespace) (" + - " avg_over_time(kube_pod_container_info{%s ADDITIONAL_LABEL}[15d]) *" + - " on (pod, namespace,prometheus_replica) group_left(workload, workload_type)" + - " avg_over_time(namespace_workload_pod:kube_pod_owner:relabel{workload!~\"\" ADDITIONAL_LABEL}[15d])" + - ")"); - private final String query; - - PromQLQuery(String query) { - this.query = query; - } - - public String getQuery() { - return query; - } - } -} diff --git a/src/main/java/com/autotune/common/data/dataSourceQueries/PromQLDataSourceQueries.java b/src/main/java/com/autotune/common/data/dataSourceQueries/PromQLDataSourceQueries.java deleted file mode 100644 index b675239ae..000000000 --- a/src/main/java/com/autotune/common/data/dataSourceQueries/PromQLDataSourceQueries.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.autotune.common.data.dataSourceQueries; - -/** - * This class provides static constants for PromQL queries related to a Kubernetes cluster. - * It utilizes the queries defined in the DataSourceQueries class. - */ -public class PromQLDataSourceQueries { - public static final String NAMESPACE_QUERY = DataSourceQueries.PromQLQuery.NAMESPACE_QUERY.getQuery(); - public static final String WORKLOAD_QUERY = DataSourceQueries.PromQLQuery.WORKLOAD_INFO_QUERY.getQuery(); - public static final String CONTAINER_QUERY = DataSourceQueries.PromQLQuery.CONTAINER_INFO_QUERY.getQuery(); -} diff --git a/src/main/java/com/autotune/common/datasource/DataSourceManager.java b/src/main/java/com/autotune/common/datasource/DataSourceManager.java index ca7406e18..9d1bad923 100644 --- a/src/main/java/com/autotune/common/datasource/DataSourceManager.java +++ b/src/main/java/com/autotune/common/datasource/DataSourceManager.java @@ -68,7 +68,7 @@ public DataSourceManager() { * @param excludeResources * @return */ - public DataSourceMetadataInfo importMetadataFromDataSource(DataSourceInfo dataSourceInfo, String uniqueKey, long startTime, long endTime, int steps, Map includeResources, + public DataSourceMetadataInfo importMetadataFromDataSource(String metadataProfileName, DataSourceInfo dataSourceInfo, String uniqueKey, long startTime, long endTime, int steps, String measurementDuration, Map includeResources, Map excludeResources) throws DataSourceDoesNotExist, IOException, NoSuchAlgorithmException, KeyStoreException, KeyManagementException { String statusValue = "failure"; io.micrometer.core.instrument.Timer.Sample timerImportMetadata = Timer.start(MetricsConfig.meterRegistry()); @@ -76,7 +76,8 @@ public DataSourceMetadataInfo importMetadataFromDataSource(DataSourceInfo dataSo if (null == dataSourceInfo) { throw new DataSourceDoesNotExist(KruizeConstants.DataSourceConstants.DataSourceErrorMsgs.MISSING_DATASOURCE_INFO); } - DataSourceMetadataInfo dataSourceMetadataInfo = dataSourceMetadataOperator.createDataSourceMetadata(dataSourceInfo, uniqueKey, startTime, endTime, steps, includeResources, excludeResources); + DataSourceMetadataInfo dataSourceMetadataInfo = dataSourceMetadataOperator.createDataSourceMetadata(metadataProfileName, + dataSourceInfo, uniqueKey, startTime, endTime, steps, measurementDuration, includeResources, excludeResources); if (null == dataSourceMetadataInfo) { LOGGER.error(KruizeConstants.DataSourceConstants.DataSourceMetadataErrorMsgs.DATASOURCE_METADATA_INFO_NOT_AVAILABLE, "for datasource {}" + dataSourceInfo.getName()); return null; @@ -133,7 +134,7 @@ public DataSourceMetadataInfo getMetadataFromDataSource(DataSourceInfo dataSourc * @param dataSourceMetadataInfo The existing DataSourceMetadataInfo object containing the current * metadata information of the data source. */ - public void updateMetadataFromDataSource(DataSourceInfo dataSource, DataSourceMetadataInfo dataSourceMetadataInfo) { + public void updateMetadataFromDataSource(String metadataProfileName, DataSourceInfo dataSource, DataSourceMetadataInfo dataSourceMetadataInfo, String measurementDuration) { try { if (null == dataSource) { throw new DataSourceDoesNotExist(KruizeConstants.DataSourceConstants.DataSourceErrorMsgs.MISSING_DATASOURCE_INFO); @@ -141,7 +142,7 @@ public void updateMetadataFromDataSource(DataSourceInfo dataSource, DataSourceMe if (null == dataSourceMetadataInfo) { throw new DataSourceDoesNotExist(DATASOURCE_METADATA_INFO_NOT_AVAILABLE); } - dataSourceMetadataOperator.updateDataSourceMetadata(dataSource, "", 0, 0, 0, null, null); + dataSourceMetadataOperator.updateDataSourceMetadata(metadataProfileName, dataSource, "", 0, 0, 0, measurementDuration, null, null); } catch (Exception e) { LOGGER.error(e.getMessage()); } diff --git a/src/main/java/com/autotune/common/datasource/DataSourceMetadataOperator.java b/src/main/java/com/autotune/common/datasource/DataSourceMetadataOperator.java index 496e5c414..c2f9716f0 100644 --- a/src/main/java/com/autotune/common/datasource/DataSourceMetadataOperator.java +++ b/src/main/java/com/autotune/common/datasource/DataSourceMetadataOperator.java @@ -15,8 +15,8 @@ *******************************************************************************/ package com.autotune.common.datasource; +import com.autotune.analyzer.utils.AnalyzerConstants; import com.autotune.common.data.dataSourceMetadata.*; -import com.autotune.common.data.dataSourceQueries.PromQLDataSourceQueries; import com.autotune.utils.GenericRestApiClient; import com.autotune.utils.KruizeConstants; import com.google.gson.Gson; @@ -73,10 +73,11 @@ public static DataSourceMetadataOperator getInstance() { * @param includeResources * @param excludeResources */ - public DataSourceMetadataInfo createDataSourceMetadata(DataSourceInfo dataSourceInfo, String uniqueKey, long startTime, - long endTime, int steps, Map includeResources, + public DataSourceMetadataInfo createDataSourceMetadata(String metadataProfileName, DataSourceInfo dataSourceInfo, String uniqueKey, long startTime, + long endTime, int steps, String measurementDuration, Map includeResources, Map excludeResources) throws IOException, NoSuchAlgorithmException, KeyStoreException, KeyManagementException { - return processQueriesAndPopulateDataSourceMetadataInfo(dataSourceInfo, uniqueKey, startTime, endTime, steps, includeResources, excludeResources); + return processQueriesAndPopulateDataSourceMetadataInfo(metadataProfileName, dataSourceInfo, uniqueKey, startTime, + endTime, steps, measurementDuration, includeResources, excludeResources); } /** @@ -118,10 +119,11 @@ public DataSourceMetadataInfo getDataSourceMetadataInfo(DataSourceInfo dataSourc * TODO - Currently Create and Update functions have identical functionalities, based on UI workflow and requirements * need to further enhance updateDataSourceMetadata() to support namespace, workload level granular updates */ - public DataSourceMetadataInfo updateDataSourceMetadata(DataSourceInfo dataSourceInfo, String uniqueKey, long startTime, - long endTime, int steps, Map includeResources, + public DataSourceMetadataInfo updateDataSourceMetadata(String metadataProfileName,DataSourceInfo dataSourceInfo, String uniqueKey, long startTime, + long endTime, int steps, String measurementDuration, Map includeResources, Map excludeResources) throws Exception { - return processQueriesAndPopulateDataSourceMetadataInfo(dataSourceInfo, uniqueKey, startTime, endTime, steps, includeResources, excludeResources); + return processQueriesAndPopulateDataSourceMetadataInfo(metadataProfileName, dataSourceInfo, uniqueKey, startTime, + endTime, steps, measurementDuration, includeResources, excludeResources); } /** @@ -163,8 +165,8 @@ public void deleteDataSourceMetadata(DataSourceInfo dataSourceInfo) { * @return DataSourceMetadataInfo object with populated metadata fields * todo rename processQueriesAndFetchClusterMetadataInfo */ - public DataSourceMetadataInfo processQueriesAndPopulateDataSourceMetadataInfo(DataSourceInfo dataSourceInfo, String uniqueKey, - long startTime, long endTime, int steps, + public DataSourceMetadataInfo processQueriesAndPopulateDataSourceMetadataInfo(String metadataProfileName, DataSourceInfo dataSourceInfo, String uniqueKey, + long startTime, long endTime, int steps, String measurementDuration, Map includeResources, Map excludeResources) throws IOException, NoSuchAlgorithmException, KeyStoreException, KeyManagementException { DataSourceMetadataHelper dataSourceDetailsHelper = new DataSourceMetadataHelper(); @@ -192,7 +194,7 @@ public DataSourceMetadataInfo processQueriesAndPopulateDataSourceMetadataInfo(Da String includeRegex = includeResources.getOrDefault(field + "Regex", ""); String excludeRegex = excludeResources.getOrDefault(field + "Regex", ""); String filter = constructDynamicFilter(field, includeRegex, excludeRegex); - String queryTemplate = getQueryTemplate(field); // Helper to map fields to PromQL queries + String queryTemplate = getQueryTemplate(field, metadataProfileName); // Helper to map fields to PromQL queries queries.put(field, String.format(queryTemplate, filter)); }); @@ -204,17 +206,32 @@ public DataSourceMetadataInfo processQueriesAndPopulateDataSourceMetadataInfo(Da String dataSourceName = dataSourceInfo.getName(); if (null != uniqueKey && !uniqueKey.isEmpty()) { LOGGER.debug("uniquekey: {}", uniqueKey); - namespaceQuery = namespaceQuery.replace(KruizeConstants.KRUIZE_BULK_API.ADDITIONAL_LABEL, "," + uniqueKey); - workloadQuery = workloadQuery.replace(KruizeConstants.KRUIZE_BULK_API.ADDITIONAL_LABEL, "," + uniqueKey); - containerQuery = containerQuery.replace(KruizeConstants.KRUIZE_BULK_API.ADDITIONAL_LABEL, "," + uniqueKey); + namespaceQuery = namespaceQuery. + replace(KruizeConstants.KRUIZE_BULK_API.ADDITIONAL_LABEL, "," + uniqueKey); + workloadQuery = workloadQuery. + replace(KruizeConstants.KRUIZE_BULK_API.ADDITIONAL_LABEL, "," + uniqueKey); + containerQuery = containerQuery. + replace(KruizeConstants.KRUIZE_BULK_API.ADDITIONAL_LABEL, "," + uniqueKey); } else { - namespaceQuery = namespaceQuery.replace(KruizeConstants.KRUIZE_BULK_API.ADDITIONAL_LABEL, ""); - workloadQuery = workloadQuery.replace(KruizeConstants.KRUIZE_BULK_API.ADDITIONAL_LABEL, ""); - containerQuery = containerQuery.replace(KruizeConstants.KRUIZE_BULK_API.ADDITIONAL_LABEL, ""); + namespaceQuery = namespaceQuery. + replace(KruizeConstants.KRUIZE_BULK_API.ADDITIONAL_LABEL, ""); + workloadQuery = workloadQuery. + replace(KruizeConstants.KRUIZE_BULK_API.ADDITIONAL_LABEL, ""); + containerQuery = containerQuery. + replace(KruizeConstants.KRUIZE_BULK_API.ADDITIONAL_LABEL, ""); } + + namespaceQuery = namespaceQuery.replace(AnalyzerConstants.MEASUREMENT_DURATION_IN_MIN_VARAIBLE, measurementDuration); + workloadQuery = workloadQuery.replace(AnalyzerConstants.MEASUREMENT_DURATION_IN_MIN_VARAIBLE, measurementDuration); + containerQuery = containerQuery.replace(AnalyzerConstants.MEASUREMENT_DURATION_IN_MIN_VARAIBLE, measurementDuration); + LOGGER.info("namespaceQuery: {}", namespaceQuery); LOGGER.info("workloadQuery: {}", workloadQuery); LOGGER.info("containerQuery: {}", containerQuery); + LOGGER.info("startTime: {}", startTime); + LOGGER.info("endTime: {}", endTime); + LOGGER.info("steps: {}", steps); + JsonArray namespacesDataResultArray = fetchQueryResults(dataSourceInfo, namespaceQuery, startTime, endTime, steps); LOGGER.debug("namespacesDataResultArray: {}", namespacesDataResultArray); @@ -275,11 +292,13 @@ public DataSourceMetadataInfo processQueriesAndPopulateDataSourceMetadataInfo(Da } // Helper function to map fields to query templates - private String getQueryTemplate(String field) { + private String getQueryTemplate(String field, String metadataProfileName) { + DataSourceMetadataHelper dataSourceDetailsHelper = new DataSourceMetadataHelper(); + MetadataProfile metadataProfile = MetadataProfileCollection.getInstance().getMetadataProfileCollection().get(metadataProfileName); return switch (field) { - case "namespace" -> PromQLDataSourceQueries.NAMESPACE_QUERY; - case "workload" -> PromQLDataSourceQueries.WORKLOAD_QUERY; - case "container" -> PromQLDataSourceQueries.CONTAINER_QUERY; + case "namespace" -> dataSourceDetailsHelper.getQueryFromProfile(metadataProfile, AnalyzerConstants.NAMESPACE); + case "workload" -> dataSourceDetailsHelper.getQueryFromProfile(metadataProfile, AnalyzerConstants.WORKLOAD); + case "container" -> dataSourceDetailsHelper.getQueryFromProfile(metadataProfile, AnalyzerConstants.CONTAINER); default -> throw new IllegalArgumentException("Unknown field: " + field); }; } diff --git a/src/main/java/com/autotune/utils/KruizeConstants.java b/src/main/java/com/autotune/utils/KruizeConstants.java index ff0b52592..b8ca5ff43 100644 --- a/src/main/java/com/autotune/utils/KruizeConstants.java +++ b/src/main/java/com/autotune/utils/KruizeConstants.java @@ -271,6 +271,7 @@ public static final class JSONKeys { public static final String UUID = "UUID"; public static final String DEVICE = "device"; public static final String MODEL_NAME = "modelName"; + public static final String METADATA_PROFILE = "metadataProfile"; private JSONKeys() { }