Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Integrate MetadataProfile with DSMetadata API #1444

Draft
wants to merge 2 commits into
base: mvp_demo
Choose a base branch
from
Draft

Integrate MetadataProfile with DSMetadata API #1444

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8dd7326
Integrate metadataProfile with dsmetadata API
shreyabiradar07 Dec 27, 2024
51893e4
Remove hardcoded metadata queries
shreyabiradar07 Dec 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions src/main/java/com/autotune/analyzer/serviceObjects/DSMetadataAPIObject.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,10 @@ public class DSMetadataAPIObject {
private String version;
@SerializedName(KruizeConstants.JSONKeys.DATASOURCE_NAME)
private String dataSourceName;
@SerializedName(KruizeConstants.JSONKeys.METADATA_PROFILE)
private String metadataProfileName;
@SerializedName(KruizeConstants.JSONKeys.MEASUREMENT_DURATION)
private String measurement_durationMinutes;

public String getVersion() {
return version;
Expand All @@ -36,4 +40,11 @@ public String getDataSourceName() {
return dataSourceName;
}

public String getMetadataProfile() { return metadataProfileName; }

public void setMetadataProfile(String metadataProfileName) { this.metadataProfileName = metadataProfileName; }

public String getMeasurementDurationMinutes() { return measurement_durationMinutes; }

public void setMeasurementDurationMinutes(String measurement_durationMinutes) {this.measurement_durationMinutes = measurement_durationMinutes;}
}
10 changes: 8 additions & 2 deletions src/main/java/com/autotune/analyzer/services/DSMetadataService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,9 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response)
if (validationOutputData.isSuccess()) {

String dataSourceName = metadataAPIObject.getDataSourceName();
String metadataProfileName = metadataAPIObject.getMetadataProfile();
String measurementDuration = metadataAPIObject.getMeasurementDurationMinutes();

// fetch the DatasourceInfo object based on datasource name
DataSourceInfo datasource;
try {
Expand All @@ -133,7 +136,8 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response)
return;
}

DataSourceMetadataInfo metadataInfo = dataSourceManager.importMetadataFromDataSource(datasource,"",0,0,0, new HashMap<>(), new HashMap<>());
DataSourceMetadataInfo metadataInfo = dataSourceManager.importMetadataFromDataSource(metadataProfileName,
datasource,"",0, 0, 0, measurementDuration, new HashMap<>(), new HashMap<>());

// Validate imported metadataInfo object
DataSourceMetadataValidation validationObject = new DataSourceMetadataValidation();
Expand Down Expand Up @@ -195,9 +199,11 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response)

}

//TODO - Add measurement_duration if required
private List<String> mandatoryFields = new ArrayList<>(Arrays.asList(
AnalyzerConstants.VERSION,
AnalyzerConstants.DATASOURCE_NAME
AnalyzerConstants.DATASOURCE_NAME,
AnalyzerConstants.METADATA_PROFILE
));

public ValidationOutputData validateMandatoryFields(DSMetadataAPIObject metadataAPIObject) {
Expand Down
3 changes: 3 additions & 0 deletions src/main/java/com/autotune/analyzer/utils/AnalyzerConstants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,9 @@ public class AnalyzerConstants {
public static final String KRUIZE_LOCAL_DDL_SQL = "kruize_local_ddl.sql";
public static final String VERSION = "version";
public static final String DATASOURCE_NAME = "dataSourceName";
public static final String METADATA_PROFILE = "metadataProfile";
public static final String WORKLOAD = "workload";
public static final String CONTAINER = "container";


private AnalyzerConstants() {
Expand Down
12 changes: 12 additions & 0 deletions src/main/java/com/autotune/common/data/dataSourceMetadata/DataSourceMetadataHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.autotune.common.data.dataSourceMetadata;

import com.autotune.common.data.metrics.Metric;
import com.autotune.utils.KruizeConstants;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
Expand All @@ -9,6 +10,7 @@
import org.slf4j.LoggerFactory;

import java.util.HashMap;
import java.util.List;

/**
* Utility class for handling DataSourceMetadataInfo and related metadata.
Expand Down Expand Up @@ -445,4 +447,14 @@ public void updateContainerDataSourceMetadataInfoObject(String dataSourceName, D
LOGGER.error(KruizeConstants.DataSourceConstants.DataSourceMetadataErrorMsgs.CONTAINER_METADATA_UPDATE_ERROR + e.getMessage());
}
}
public String getQueryFromProfile(MetadataProfile metadataProfile, String metricName) {
List<Metric> metrics = metadataProfile.getQueryVariables();
for (Metric metric : metrics) {
String name = metric.getName();
if (name.contains(metricName)) {
return metric.getAggregationFunctionsMap().get(KruizeConstants.JSONKeys.SUM).getQuery();
}
}
return null;
}
}
27 changes: 0 additions & 27 deletions src/main/java/com/autotune/common/data/dataSourceQueries/DataSourceQueries.java
View file
Open in desktop

This file was deleted.

11 changes: 0 additions & 11 deletions src/main/java/com/autotune/common/data/dataSourceQueries/PromQLDataSourceQueries.java
View file
Open in desktop

This file was deleted.

9 changes: 5 additions & 4 deletions src/main/java/com/autotune/common/datasource/DataSourceManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,15 +68,16 @@ public DataSourceManager() {
* @param excludeResources
* @return
*/
public DataSourceMetadataInfo importMetadataFromDataSource(DataSourceInfo dataSourceInfo, String uniqueKey, long startTime, long endTime, int steps, Map<String, String> includeResources,
public DataSourceMetadataInfo importMetadataFromDataSource(String metadataProfileName, DataSourceInfo dataSourceInfo, String uniqueKey, long startTime, long endTime, int steps, String measurementDuration, Map<String, String> includeResources,
Map<String, String> excludeResources) throws DataSourceDoesNotExist, IOException, NoSuchAlgorithmException, KeyStoreException, KeyManagementException {
String statusValue = "failure";
io.micrometer.core.instrument.Timer.Sample timerImportMetadata = Timer.start(MetricsConfig.meterRegistry());
try {
if (null == dataSourceInfo) {
throw new DataSourceDoesNotExist(KruizeConstants.DataSourceConstants.DataSourceErrorMsgs.MISSING_DATASOURCE_INFO);
}
DataSourceMetadataInfo dataSourceMetadataInfo = dataSourceMetadataOperator.createDataSourceMetadata(dataSourceInfo, uniqueKey, startTime, endTime, steps, includeResources, excludeResources);
DataSourceMetadataInfo dataSourceMetadataInfo = dataSourceMetadataOperator.createDataSourceMetadata(metadataProfileName,
dataSourceInfo, uniqueKey, startTime, endTime, steps, measurementDuration, includeResources, excludeResources);
if (null == dataSourceMetadataInfo) {
LOGGER.error(KruizeConstants.DataSourceConstants.DataSourceMetadataErrorMsgs.DATASOURCE_METADATA_INFO_NOT_AVAILABLE, "for datasource {}" + dataSourceInfo.getName());
return null;
Expand Down Expand Up @@ -133,15 +134,15 @@ public DataSourceMetadataInfo getMetadataFromDataSource(DataSourceInfo dataSourc
* @param dataSourceMetadataInfo The existing DataSourceMetadataInfo object containing the current
* metadata information of the data source.
*/
public void updateMetadataFromDataSource(DataSourceInfo dataSource, DataSourceMetadataInfo dataSourceMetadataInfo) {
public void updateMetadataFromDataSource(String metadataProfileName, DataSourceInfo dataSource, DataSourceMetadataInfo dataSourceMetadataInfo, String measurementDuration) {
try {
if (null == dataSource) {
throw new DataSourceDoesNotExist(KruizeConstants.DataSourceConstants.DataSourceErrorMsgs.MISSING_DATASOURCE_INFO);
}
if (null == dataSourceMetadataInfo) {
throw new DataSourceDoesNotExist(DATASOURCE_METADATA_INFO_NOT_AVAILABLE);
}
dataSourceMetadataOperator.updateDataSourceMetadata(dataSource, "", 0, 0, 0, null, null);
dataSourceMetadataOperator.updateDataSourceMetadata(metadataProfileName, dataSource, "", 0, 0, 0, measurementDuration, null, null);
} catch (Exception e) {
LOGGER.error(e.getMessage());
}
Expand Down
59 changes: 39 additions & 20 deletions src/main/java/com/autotune/common/datasource/DataSourceMetadataOperator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@
*******************************************************************************/
package com.autotune.common.datasource;

import com.autotune.analyzer.utils.AnalyzerConstants;
import com.autotune.common.data.dataSourceMetadata.*;
import com.autotune.common.data.dataSourceQueries.PromQLDataSourceQueries;
import com.autotune.utils.GenericRestApiClient;
import com.autotune.utils.KruizeConstants;
import com.google.gson.Gson;
Expand Down Expand Up @@ -73,10 +73,11 @@ public static DataSourceMetadataOperator getInstance() {
* @param includeResources
* @param excludeResources
*/
public DataSourceMetadataInfo createDataSourceMetadata(DataSourceInfo dataSourceInfo, String uniqueKey, long startTime,
long endTime, int steps, Map<String, String> includeResources,
public DataSourceMetadataInfo createDataSourceMetadata(String metadataProfileName, DataSourceInfo dataSourceInfo, String uniqueKey, long startTime,
long endTime, int steps, String measurementDuration, Map<String, String> includeResources,
Map<String, String> excludeResources) throws IOException, NoSuchAlgorithmException, KeyStoreException, KeyManagementException {
return processQueriesAndPopulateDataSourceMetadataInfo(dataSourceInfo, uniqueKey, startTime, endTime, steps, includeResources, excludeResources);
return processQueriesAndPopulateDataSourceMetadataInfo(metadataProfileName, dataSourceInfo, uniqueKey, startTime,
endTime, steps, measurementDuration, includeResources, excludeResources);
}

/**
Expand Down Expand Up @@ -118,10 +119,11 @@ public DataSourceMetadataInfo getDataSourceMetadataInfo(DataSourceInfo dataSourc
* TODO - Currently Create and Update functions have identical functionalities, based on UI workflow and requirements
* need to further enhance updateDataSourceMetadata() to support namespace, workload level granular updates
*/
public DataSourceMetadataInfo updateDataSourceMetadata(DataSourceInfo dataSourceInfo, String uniqueKey, long startTime,
long endTime, int steps, Map<String, String> includeResources,
public DataSourceMetadataInfo updateDataSourceMetadata(String metadataProfileName,DataSourceInfo dataSourceInfo, String uniqueKey, long startTime,
long endTime, int steps, String measurementDuration, Map<String, String> includeResources,
Map<String, String> excludeResources) throws Exception {
return processQueriesAndPopulateDataSourceMetadataInfo(dataSourceInfo, uniqueKey, startTime, endTime, steps, includeResources, excludeResources);
return processQueriesAndPopulateDataSourceMetadataInfo(metadataProfileName, dataSourceInfo, uniqueKey, startTime,
endTime, steps, measurementDuration, includeResources, excludeResources);
}

/**
Expand Down Expand Up @@ -163,8 +165,8 @@ public void deleteDataSourceMetadata(DataSourceInfo dataSourceInfo) {
* @return DataSourceMetadataInfo object with populated metadata fields
* todo rename processQueriesAndFetchClusterMetadataInfo
*/
public DataSourceMetadataInfo processQueriesAndPopulateDataSourceMetadataInfo(DataSourceInfo dataSourceInfo, String uniqueKey,
long startTime, long endTime, int steps,
public DataSourceMetadataInfo processQueriesAndPopulateDataSourceMetadataInfo(String metadataProfileName, DataSourceInfo dataSourceInfo, String uniqueKey,
long startTime, long endTime, int steps, String measurementDuration,
Map<String, String> includeResources,
Map<String, String> excludeResources) throws IOException, NoSuchAlgorithmException, KeyStoreException, KeyManagementException {
DataSourceMetadataHelper dataSourceDetailsHelper = new DataSourceMetadataHelper();
Expand Down Expand Up @@ -192,7 +194,7 @@ public DataSourceMetadataInfo processQueriesAndPopulateDataSourceMetadataInfo(Da
String includeRegex = includeResources.getOrDefault(field + "Regex", "");
String excludeRegex = excludeResources.getOrDefault(field + "Regex", "");
String filter = constructDynamicFilter(field, includeRegex, excludeRegex);
String queryTemplate = getQueryTemplate(field); // Helper to map fields to PromQL queries
String queryTemplate = getQueryTemplate(field, metadataProfileName); // Helper to map fields to PromQL queries
queries.put(field, String.format(queryTemplate, filter));
});

Expand All @@ -204,17 +206,32 @@ public DataSourceMetadataInfo processQueriesAndPopulateDataSourceMetadataInfo(Da
String dataSourceName = dataSourceInfo.getName();
if (null != uniqueKey && !uniqueKey.isEmpty()) {
LOGGER.debug("uniquekey: {}", uniqueKey);
namespaceQuery = namespaceQuery.replace(KruizeConstants.KRUIZE_BULK_API.ADDITIONAL_LABEL, "," + uniqueKey);
workloadQuery = workloadQuery.replace(KruizeConstants.KRUIZE_BULK_API.ADDITIONAL_LABEL, "," + uniqueKey);
containerQuery = containerQuery.replace(KruizeConstants.KRUIZE_BULK_API.ADDITIONAL_LABEL, "," + uniqueKey);
namespaceQuery = namespaceQuery.
replace(KruizeConstants.KRUIZE_BULK_API.ADDITIONAL_LABEL, "," + uniqueKey);
workloadQuery = workloadQuery.
replace(KruizeConstants.KRUIZE_BULK_API.ADDITIONAL_LABEL, "," + uniqueKey);
containerQuery = containerQuery.
replace(KruizeConstants.KRUIZE_BULK_API.ADDITIONAL_LABEL, "," + uniqueKey);
} else {
namespaceQuery = namespaceQuery.replace(KruizeConstants.KRUIZE_BULK_API.ADDITIONAL_LABEL, "");
workloadQuery = workloadQuery.replace(KruizeConstants.KRUIZE_BULK_API.ADDITIONAL_LABEL, "");
containerQuery = containerQuery.replace(KruizeConstants.KRUIZE_BULK_API.ADDITIONAL_LABEL, "");
namespaceQuery = namespaceQuery.
replace(KruizeConstants.KRUIZE_BULK_API.ADDITIONAL_LABEL, "");
workloadQuery = workloadQuery.
replace(KruizeConstants.KRUIZE_BULK_API.ADDITIONAL_LABEL, "");
containerQuery = containerQuery.
replace(KruizeConstants.KRUIZE_BULK_API.ADDITIONAL_LABEL, "");
}

namespaceQuery = namespaceQuery.replace(AnalyzerConstants.MEASUREMENT_DURATION_IN_MIN_VARAIBLE, measurementDuration);
workloadQuery = workloadQuery.replace(AnalyzerConstants.MEASUREMENT_DURATION_IN_MIN_VARAIBLE, measurementDuration);
containerQuery = containerQuery.replace(AnalyzerConstants.MEASUREMENT_DURATION_IN_MIN_VARAIBLE, measurementDuration);

LOGGER.info("namespaceQuery: {}", namespaceQuery);
LOGGER.info("workloadQuery: {}", workloadQuery);
LOGGER.info("containerQuery: {}", containerQuery);
LOGGER.info("startTime: {}", startTime);
LOGGER.info("endTime: {}", endTime);
LOGGER.info("steps: {}", steps);


JsonArray namespacesDataResultArray = fetchQueryResults(dataSourceInfo, namespaceQuery, startTime, endTime, steps);
LOGGER.debug("namespacesDataResultArray: {}", namespacesDataResultArray);
Expand Down Expand Up @@ -275,11 +292,13 @@ public DataSourceMetadataInfo processQueriesAndPopulateDataSourceMetadataInfo(Da
}

// Helper function to map fields to query templates
private String getQueryTemplate(String field) {
private String getQueryTemplate(String field, String metadataProfileName) {
DataSourceMetadataHelper dataSourceDetailsHelper = new DataSourceMetadataHelper();
MetadataProfile metadataProfile = MetadataProfileCollection.getInstance().getMetadataProfileCollection().get(metadataProfileName);
return switch (field) {
case "namespace" -> PromQLDataSourceQueries.NAMESPACE_QUERY;
case "workload" -> PromQLDataSourceQueries.WORKLOAD_QUERY;
case "container" -> PromQLDataSourceQueries.CONTAINER_QUERY;
case "namespace" -> dataSourceDetailsHelper.getQueryFromProfile(metadataProfile, AnalyzerConstants.NAMESPACE);
case "workload" -> dataSourceDetailsHelper.getQueryFromProfile(metadataProfile, AnalyzerConstants.WORKLOAD);
case "container" -> dataSourceDetailsHelper.getQueryFromProfile(metadataProfile, AnalyzerConstants.CONTAINER);
default -> throw new IllegalArgumentException("Unknown field: " + field);
};
}
Expand Down
1 change: 1 addition & 0 deletions src/main/java/com/autotune/utils/KruizeConstants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,7 @@ public static final class JSONKeys {
public static final String UUID = "UUID";
public static final String DEVICE = "device";
public static final String MODEL_NAME = "modelName";
public static final String METADATA_PROFILE = "metadataProfile";

private JSONKeys() {
}
Expand Down
Loading