diff --git a/ap/__init__.py b/ap/__init__.py
index 6e5bb34..f08ba67 100644
--- a/ap/__init__.py
+++ b/ap/__init__.py
@@ -21,7 +21,7 @@
from ap.common.constants import FlaskGKey, SQLITE_CONFIG_DIR, PARTITION_NUMBER, UNIVERSAL_DB_FILE, APP_DB_FILE, \
TESTING, YAML_CONFIG_VERSION, YAML_CONFIG_BASIC, YAML_CONFIG_DB, YAML_CONFIG_PROC, YAML_CONFIG_AP, \
INIT_APP_DB_FILE, INIT_BASIC_CFG_FILE, REQUEST_THREAD_ID, YAML_START_UP, LOG_LEVEL, AP_LOG_LEVEL, AppGroup, \
- AppSource
+ AppSource, appENV
from ap.common.logger import log_execution
from ap.common.services.request_time_out_handler import RequestTimeOutAPI, set_request_g_dict
from ap.common.trace_data_log import get_log_attr, TraceErrKey
@@ -118,6 +118,7 @@ def create_app(object_name=None):
from .sankey_plot import create_module as sankey_create_module
from .co_occurrence import create_module as co_occurrence_create_module
from .multiple_scatter_plot import create_module as multiple_scatter_create_module
+ from .aggregate_plot import create_module as agp_create_module
from .common.logger import bind_user_info
from .script.convert_user_setting import convert_user_setting_url
from .script.migrate_csv_datatype import migrate_csv_datatype
@@ -202,6 +203,7 @@ def create_app(object_name=None):
co_occurrence_create_module(app)
multiple_scatter_create_module(app)
tile_interface_create_module(app)
+ agp_create_module(app)
app.add_url_rule('/', endpoint='tile_interface.tile_interface')
basic_config_yaml = BasicConfigYaml(dic_yaml_config_file[YAML_CONFIG_BASIC])
@@ -272,6 +274,8 @@ def before_request_callback():
# get the last time user request
global dic_request_info
+
+
# get api request thread id
thread_id = request.form.get(REQUEST_THREAD_ID, None)
set_request_g_dict(thread_id)
@@ -294,6 +298,7 @@ def before_request_callback():
"url": "https://www.google.com/chrome/"
})
+
@app.after_request
def after_request_callback(response: Response):
if 'event-stream' in str(request.accept_mimetypes):
diff --git a/ap/aggregate_plot/__init__.py b/ap/aggregate_plot/__init__.py
new file mode 100644
index 0000000..203ed71
--- /dev/null
+++ b/ap/aggregate_plot/__init__.py
@@ -0,0 +1,4 @@
+
+def create_module(app, **kwargs):
+ from .controllers import agp_blueprint
+ app.register_blueprint(agp_blueprint)
diff --git a/ap/aggregate_plot/controllers.py b/ap/aggregate_plot/controllers.py
new file mode 100644
index 0000000..b2a74f2
--- /dev/null
+++ b/ap/aggregate_plot/controllers.py
@@ -0,0 +1,19 @@
+import os
+
+from flask import Blueprint, render_template
+
+from ap.common.services.form_env import get_common_config_data
+
+agp_blueprint = Blueprint(
+ 'agp',
+ __name__,
+ template_folder=os.path.join('..', 'templates', 'aggregate_plot'),
+ static_folder=os.path.join('..', 'static', 'aggregate_plot'),
+ url_prefix='/ap'
+)
+
+
+@agp_blueprint.route('/agp')
+def index():
+ output_dict = get_common_config_data()
+ return render_template("aggregate_plot.html", **output_dict)
diff --git a/ap/aggregate_plot/services/__init__.py b/ap/aggregate_plot/services/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/ap/aggregate_plot/services/utils.py b/ap/aggregate_plot/services/utils.py
new file mode 100644
index 0000000..e69de29
diff --git a/ap/api/__init__.py b/ap/api/__init__.py
index 499c753..bb415dd 100644
--- a/ap/api/__init__.py
+++ b/ap/api/__init__.py
@@ -12,6 +12,7 @@ def create_module(app, **kwargs):
from .heatmap.controllers import api_heatmap_blueprint
from .parallel_plot.controllers import api_paracords_blueprint
from .common.controlllers import api_common_blueprint
+ from .aggregate_plot.controllers import api_agp_blueprint
app.register_blueprint(api_setting_module_blueprint)
app.register_blueprint(api_trace_data_blueprint)
app.register_blueprint(api_table_viewer_blueprint)
@@ -25,3 +26,4 @@ def create_module(app, **kwargs):
app.register_blueprint(api_heatmap_blueprint)
app.register_blueprint(api_paracords_blueprint)
app.register_blueprint(api_common_blueprint)
+ app.register_blueprint(api_agp_blueprint)
diff --git a/ap/api/aggregate_plot/__init__.py b/ap/api/aggregate_plot/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/ap/api/aggregate_plot/controllers.py b/ap/api/aggregate_plot/controllers.py
new file mode 100644
index 0000000..6e23038
--- /dev/null
+++ b/ap/api/aggregate_plot/controllers.py
@@ -0,0 +1,96 @@
+import timeit
+from copy import deepcopy
+
+import pytz
+import simplejson
+from dateutil import tz
+from flask import Blueprint, request
+
+from ap.api.aggregate_plot.services import gen_agp_data
+from ap.api.categorical_plot.services import customize_dict_param
+from ap.api.trace_data.services.csv_export import to_csv
+from ap.common.constants import COMMON, ARRAY_FORMVAL, END_PROC, CLIENT_TIMEZONE
+from ap.common.services import http_content
+from ap.common.services.csv_content import zip_file_to_response
+from ap.common.services.form_env import parse_multi_filter_into_one, get_end_procs_param, \
+ update_data_from_multiple_dic_params, parse_request_params
+from ap.common.services.import_export_config_n_data import get_dic_form_from_debug_info, \
+ set_export_dataset_id_to_dic_param
+from ap.common.trace_data_log import save_input_data_to_file, EventType, save_draw_graph_trace, trace_log_params
+
+api_agp_blueprint = Blueprint(
+ 'api_agp',
+ __name__,
+ url_prefix='/ap/api/agp'
+)
+
+
+@api_agp_blueprint.route('/plot', methods=['POST'])
+def generate_agp():
+ """ [summary]
+ Returns:
+ [type] -- [description]
+ """
+ start = timeit.default_timer()
+ dic_form = request.form.to_dict(flat=False)
+ # save dic_form to pickle (for future debug)
+ save_input_data_to_file(dic_form, EventType.AGP)
+
+ dic_param = parse_multi_filter_into_one(dic_form)
+
+ # check if we run debug mode (import mode)
+ dic_param = get_dic_form_from_debug_info(dic_param)
+
+ customize_dict_param(dic_param)
+ org_dic_param = deepcopy(dic_param)
+ dic_params = get_end_procs_param(dic_param)
+
+ for single_dic_param in dic_params:
+ agp_data, *_ = gen_agp_data(single_dic_param)
+ org_dic_param = update_data_from_multiple_dic_params(org_dic_param, agp_data)
+
+ stop = timeit.default_timer()
+ org_dic_param['backend_time'] = stop - start
+
+ # export mode ( output for export mode )
+ set_export_dataset_id_to_dic_param(org_dic_param)
+
+ org_dic_param['dataset_id'] = save_draw_graph_trace(vals=trace_log_params(EventType.CHM))
+
+ return simplejson.dumps(org_dic_param, ensure_ascii=False, default=http_content.json_serial, ignore_nan=True)
+
+
+@api_agp_blueprint.route('/data_export/', methods=['GET'])
+def data_export(export_type):
+ """csv export
+
+ Returns:
+ [type] -- [description]
+ """
+ dic_form = parse_request_params(request)
+ dic_param = parse_multi_filter_into_one(dic_form)
+
+ # check if we run debug mode (import mode)
+ dic_param = get_dic_form_from_debug_info(dic_param)
+
+ customize_dict_param(dic_param)
+ dic_params = get_end_procs_param(dic_param)
+
+ delimiter = ',' if export_type == 'csv' else '\t'
+
+ agp_dataset = []
+ csv_list_name = []
+ for single_dic_param in dic_params:
+ agp_dat, agp_df, graph_param, dic_proc_cfgs = gen_agp_data(single_dic_param)
+ end_proc_id = int(agp_dat[ARRAY_FORMVAL][0][END_PROC])
+ proc_name = dic_proc_cfgs[end_proc_id].name
+ csv_list_name.append('{}.{}'.format(proc_name, export_type))
+
+ client_timezone = agp_dat[COMMON].get(CLIENT_TIMEZONE)
+ client_timezone = pytz.timezone(client_timezone) if client_timezone else tz.tzlocal()
+ csv_df = to_csv(agp_df, dic_proc_cfgs, graph_param, client_timezone=client_timezone, delimiter=delimiter)
+ agp_dataset.append(csv_df)
+
+ response = zip_file_to_response(agp_dataset, csv_list_name)
+ response.charset = "utf-8-sig"
+ return response
diff --git a/ap/api/aggregate_plot/services.py b/ap/api/aggregate_plot/services.py
new file mode 100644
index 0000000..be6d7c9
--- /dev/null
+++ b/ap/api/aggregate_plot/services.py
@@ -0,0 +1,375 @@
+import copy
+from datetime import timedelta
+from typing import List, Optional, Dict, Any, Tuple
+
+import pandas as pd
+from dateutil.tz import tz
+from pandas.core.groupby import DataFrameGroupBy
+from scipy.stats import iqr
+
+from ap.api.categorical_plot.services import gen_graph_param
+from ap.api.heatmap.services import agg_func_with_na
+from ap.api.heatmap.services import get_function_i18n, range_func
+from ap.api.trace_data.services.time_series_chart import customize_dic_param_for_reuse_cache, get_data_from_db, \
+ filter_cat_dict_common
+from ap.common.common_utils import gen_sql_label
+from ap.common.constants import ARRAY_PLOTDATA, HMFunction, ACTUAL_RECORD_NUMBER, UNIQUE_SERIAL, DIVIDE_FMT_COL, \
+ COLOR_NAME, AGG_FUNC, DATA, COL_DATA_TYPE, DataType, CAT_EXP_BOX, COL_DETAIL_NAME, COL_TYPE, UNIQUE_DIV, FMT, \
+ END_PROC_ID, END_COL_NAME
+from ap.common.logger import log_execution_time
+from ap.common.memoize import memoize
+from ap.common.services.request_time_out_handler import abort_process_handler, request_timeout_handling
+from ap.common.services.sse import notify_progress
+from ap.common.sigificant_digit import get_fmt_from_array
+from ap.common.timezone_utils import get_utc_offset
+from ap.common.trace_data_log import TraceErrKey, EventType, EventAction, Target, trace_log
+from ap.setting_module.models import CfgProcessColumn
+from ap.trace_data.models import Cycle
+from ap.trace_data.schemas import DicParam
+
+CHM_AGG_FUNC = [HMFunction.median.name, HMFunction.mean.name, HMFunction.std.name]
+
+# TODO: rename and move this to constant
+OTHER_KEY = 'Other'
+OTHER_COL = 'summarized_other_col'
+MAX_ALLOW_GROUPS = 9
+
+
+@log_execution_time()
+@request_timeout_handling()
+@abort_process_handler()
+@notify_progress(75)
+@trace_log((TraceErrKey.TYPE, TraceErrKey.ACTION, TraceErrKey.TARGET),
+ (EventType.AGP, EventAction.PLOT, Target.GRAPH), send_ga=True)
+@memoize(is_save_file=True)
+def gen_agp_data(dic_param: DicParam):
+ dic_param, cat_exp, cat_procs, dic_cat_filters, use_expired_cache, *_ = \
+ customize_dic_param_for_reuse_cache(dic_param)
+
+ # gen graph_param
+ graph_param, dic_proc_cfgs = gen_graph_param(dic_param, with_ct_col=True)
+ graph_param.add_agp_color_vars()
+
+ df, actual_number_records, duplicated_serials = get_data_from_db(graph_param, dic_cat_filters,
+ use_expired_cache=use_expired_cache)
+
+ df, dic_param = filter_cat_dict_common(df, dic_param, dic_cat_filters, cat_exp, cat_procs, graph_param)
+ df = convert_utc_to_local_time_and_offset(df, graph_param)
+
+ graph_param: DicParam
+ if graph_param.common.divide_format is not None:
+ df = gen_divide_format_column(df, graph_param.common.divide_calendar_dates,
+ graph_param.common.divide_calendar_labels)
+
+ export_data = df
+
+ dic_data = gen_agp_data_from_df(df, graph_param)
+ dic_param[ARRAY_PLOTDATA] = dic_data
+ dic_param[ACTUAL_RECORD_NUMBER] = actual_number_records
+ dic_param[UNIQUE_SERIAL] = duplicated_serials
+
+ return dic_param, export_data, graph_param, dic_proc_cfgs
+
+
+@log_execution_time()
+def gen_divide_format_column(df: pd.DataFrame, divide_calendar_dates: List[str],
+ divide_calendar_labels: List[str]) -> pd.DataFrame:
+ df.sort_values(Cycle.time.key, inplace=True)
+ dt = df[Cycle.time.key]
+ df[DIVIDE_FMT_COL] = None
+ divide_calendar_dates = pd.to_datetime(divide_calendar_dates, utc=True)
+ for i, label in enumerate(divide_calendar_labels):
+ start_time = divide_calendar_dates[i]
+ end_time = divide_calendar_dates[i + 1]
+ start_index, end_index = dt.searchsorted([start_time, end_time])
+ df[start_index:end_index][DIVIDE_FMT_COL] = label
+ return df
+
+
+@log_execution_time()
+def gen_agp_data_from_df(df: pd.DataFrame, graph_param: DicParam) -> List[Dict[Any, Any]]:
+ plot_data = []
+ target_vars = graph_param.common.sensor_cols
+
+ # calculate cycle_time and replace target column
+ for target_var in target_vars:
+ # find proc_id from col info
+ general_col_info = graph_param.get_col_info_by_id(target_var)
+ # list out all CT column from this process
+ dic_datetime_cols = {cfg_col.id: cfg_col for cfg_col in
+ CfgProcessColumn.get_by_data_type(general_col_info[END_PROC_ID], DataType.DATETIME)}
+ sql_label = gen_sql_label(target_var, general_col_info[END_COL_NAME])
+ if target_var in dic_datetime_cols:
+ series = pd.to_datetime(df[sql_label])
+ series.sort_values(inplace=True)
+ series = series.diff().dt.total_seconds()
+ series.sort_index(inplace=True)
+ df.sort_index(inplace=True)
+ df[sql_label] = series
+ df[sql_label].convert_dtypes()
+
+ # each target var be shown on one chart (barchart or line chart)
+ for target_var in target_vars:
+ general_col_info = graph_param.get_col_info_by_id(target_var)
+ is_real_data = general_col_info[COL_DATA_TYPE] in [DataType.REAL.name, DataType.DATETIME.name]
+ agg_func = graph_param.common.hm_function_real if is_real_data else HMFunction.count.name
+ agg_func_show = get_function_i18n(agg_func)
+
+ summarized_df, sorted_colors = summarize_redundant_groups_into_others(df, graph_param, target_var,
+ MAX_ALLOW_GROUPS, OTHER_KEY, OTHER_COL)
+ df_groupby = gen_groupby_from_target_var(summarized_df, graph_param, target_var, is_real_data)
+
+ # get unique sorted div
+ unique_div_vars = []
+ div_col_name = get_div_col_name(graph_param)
+ if div_col_name != DIVIDE_FMT_COL:
+ unique_div_vars = sorted(df[div_col_name].dropna().unique())
+
+ if general_col_info is None:
+ continue
+
+ color_show_name = graph_param.get_color_info(target_var, shown_name=True)
+
+ agp_obj = {
+ COLOR_NAME: color_show_name,
+ AGG_FUNC: agg_func_show,
+ DATA: [],
+ CAT_EXP_BOX: [],
+ UNIQUE_DIV: unique_div_vars,
+ FMT: None,
+ }
+ general_col_info.update(agp_obj)
+
+ if is_real_data:
+ target_var_label = CfgProcessColumn.gen_label_from_col_id(target_var)
+ agg_df = get_agg_lamda_func(df_groupby, target_var_label, agg_func)
+ else:
+ agg_df = df_groupby.count()
+
+ num_facets = len(graph_param.common.cat_exp)
+
+ # TODO refactor this
+ if num_facets == 0:
+ data = get_data_for_target_var_without_facets(agg_df, graph_param, is_real_data, target_var, sorted_colors)
+ modified_agp_obj = {DATA: data, CAT_EXP_BOX: [], FMT: gen_ticks_format(data)}
+ col_info = copy.deepcopy(general_col_info)
+ col_info.update(modified_agp_obj)
+ plot_data.append(col_info)
+
+ elif num_facets == 1:
+
+ # TODO: remove hard code level=0
+ facets = agg_df.index.unique(level=0)
+ for facet in facets:
+ data = get_data_for_target_var_without_facets(agg_df.xs(facet), graph_param, is_real_data,
+ target_var, sorted_colors)
+ modified_agp_obj = {
+ DATA: data,
+ CAT_EXP_BOX: [facet],
+ FMT: gen_ticks_format(data)
+ }
+ col_info = copy.deepcopy(general_col_info)
+ col_info.update(modified_agp_obj)
+ plot_data.append(col_info)
+
+ else:
+ # TODO: remove hard code level=0, level=1
+ facets1 = agg_df.index.unique(level=0)
+ facets2 = agg_df.index.unique(level=1)
+ for facet1 in facets1:
+ sub_df = agg_df.xs(facet1)
+ for facet2 in facets2:
+ try:
+ sub_sub_df = sub_df.xs(facet2)
+ data = get_data_for_target_var_without_facets(sub_sub_df, graph_param, is_real_data,
+ target_var, sorted_colors)
+ except KeyError:
+ data = []
+
+ modified_agp_obj = {
+ DATA: data,
+ CAT_EXP_BOX: [facet1, facet2],
+ FMT: gen_ticks_format(data)
+ }
+ col_info = copy.deepcopy(general_col_info)
+ col_info.update(modified_agp_obj)
+ plot_data.append(col_info)
+
+ return plot_data
+
+
+def get_div_col_name(graph_param: DicParam) -> str:
+ """Get division column name
+ Args:
+ graph_param:
+ Returns:
+ Column name which we will use to groupby final result.
+ """
+ div_col_name = DIVIDE_FMT_COL
+ if graph_param.common.divide_format is None:
+ div_col_name = CfgProcessColumn.gen_label_from_col_id(graph_param.common.div_by_cat)
+ return div_col_name
+
+
+def get_color_col_name(graph_param: DicParam, target_var: int) -> Optional[str]:
+ """Get color column name used to groupby from target variable
+ """
+ color_id = graph_param.get_color_id(target_var)
+ color_col_name = CfgProcessColumn.gen_label_from_col_id(color_id)
+ return color_col_name
+
+
+@log_execution_time()
+def need_add_other_col(df: pd.DataFrame, graph_param: DicParam, target_var: int, maximum_allowed_groups: int) -> bool:
+ color_col_name = get_color_col_name(graph_param, target_var)
+ if color_col_name is None:
+ return False
+ return df[color_col_name].nunique(dropna=True) > maximum_allowed_groups
+
+
+# TODO: write test for this
+@log_execution_time()
+def summarize_redundant_groups_into_others(df: pd.DataFrame, graph_param: DicParam, target_var: int,
+ maximum_allowed_groups: int, other_key: str, other_col: Optional[str]) -> \
+ Tuple[pd.DataFrame, List[Any]]:
+ """Summarize color value if total unique value of color columns exceed `maximum_allowed_groups`
+ """
+ color_col_name = get_color_col_name(graph_param, target_var)
+ if color_col_name is None or other_col is None:
+ return df, []
+ df, sorted_colors = replace_redundant_groups_into_others(df, color_col_name, other_col, maximum_allowed_groups,
+ other_key)
+ return df, sorted_colors
+
+
+@log_execution_time()
+def replace_redundant_groups_into_others(df: pd.DataFrame, column_name: str, new_column_name: str,
+ maximum_allowed_groups: int, other_key: str) -> Tuple[pd.DataFrame, List[Any]]:
+ """ Rename redundant value in `column_name` into `other_key`
+ - We first sort value priority based on `value_counts`.
+ - Every key have smaller `value_counts` should have lower priority
+ """
+ assert maximum_allowed_groups > 0
+
+ color_counts = df[column_name].value_counts()
+ counted_keys = color_counts.index.to_list()
+
+ if maximum_allowed_groups >= len(counted_keys):
+ counted_keys.reverse()
+ return df, counted_keys
+
+ non_redundant_groups = maximum_allowed_groups - 1
+ df[new_column_name] = df[column_name].replace(to_replace=counted_keys[non_redundant_groups:], value=other_key)
+
+ # form a new_keys, with OTHER_KEY always on tops
+ sorted_keys = [other_key] + counted_keys[:non_redundant_groups]
+ sorted_keys.reverse()
+
+ return df, sorted_keys
+
+
+@log_execution_time()
+def get_data_for_target_var_without_facets(df: pd.DataFrame, graph_param: DicParam, is_real_data: bool,
+ target_var: int, sorted_colors: List[Any]) -> List[Dict[Any, Any]]:
+ chart_type = 'lines+markers' if is_real_data else 'bar'
+
+ target_col = CfgProcessColumn.get_by_id(target_var)
+ target_col_name = CfgProcessColumn.gen_label_from_col_id(target_var)
+
+ color_id = graph_param.get_color_id(target_var)
+
+ def gen_trace_data(sub_df: pd.DataFrame, col_name: str) -> Dict[str, Any]:
+ trace_data = {
+ 'x': sub_df[target_col_name].index.tolist(),
+ 'y': list(sub_df[target_col_name].values),
+ COL_DETAIL_NAME: col_name,
+ COL_TYPE: chart_type,
+ }
+ if is_real_data:
+ # TODO: set `mode` in constant
+ trace_data['mode'] = chart_type
+ return trace_data
+
+ if color_id is None:
+ plot_data = [gen_trace_data(df, target_col.name)]
+ else:
+ plot_data = []
+ for color in sorted_colors:
+ try:
+ plot_data.append(gen_trace_data(df.xs(color), color))
+ except KeyError:
+ pass
+ return plot_data
+
+
+@log_execution_time()
+def get_agg_cols(df: pd.DataFrame, graph_param: DicParam, target_var: int) -> List[str]:
+ agg_cols = []
+
+ facet_cols_name = [CfgProcessColumn.gen_label_from_col_id(col) for col in graph_param.common.cat_exp]
+ facet_cols_name = filter(lambda x: x is not None, facet_cols_name)
+ agg_cols.extend(facet_cols_name)
+
+ color_col_name = get_color_col_name(graph_param, target_var)
+ if color_col_name is not None:
+ if need_add_other_col(df, graph_param, target_var, MAX_ALLOW_GROUPS):
+ agg_cols.append(OTHER_COL)
+ else:
+ agg_cols.append(color_col_name)
+
+ div_col_name = get_div_col_name(graph_param)
+ agg_cols.append(div_col_name)
+
+ return agg_cols
+
+
+@log_execution_time()
+def gen_groupby_from_target_var(df: pd.DataFrame, graph_param: DicParam,
+ target_var: int, is_real_data: bool) -> DataFrameGroupBy:
+ agg_cols = get_agg_cols(df, graph_param, target_var)
+ target_col_name = CfgProcessColumn.gen_label_from_col_id(target_var)
+ # remove na before apply aggregate method for real variable only
+ if is_real_data:
+ df_groupby = df.dropna(subset=[target_col_name]).groupby(agg_cols)[[target_col_name]]
+ else:
+ # count, do not remove na
+ df_groupby = df.groupby(agg_cols)[[target_col_name]]
+ return df_groupby
+
+
+@log_execution_time()
+def convert_utc_to_local_time_and_offset(df, graph_param):
+ client_timezone = graph_param.get_client_timezone()
+ # divide_offset = graph_param.common.divide_offset or 0
+ client_tz = tz.gettz(client_timezone or None) or tz.tzlocal()
+ offset = get_utc_offset(client_tz)
+ df[Cycle.time.key] = pd.to_datetime(df[Cycle.time.key]) + offset
+
+ return df
+
+
+@log_execution_time()
+def get_agg_lamda_func(df: DataFrameGroupBy, target_var, agg_func_name) -> pd.DataFrame:
+ if agg_func_name == HMFunction.std.name:
+ agg_params = {target_var: lambda x: agg_func_with_na(x, agg_func_name)}
+ agg_df = df.agg(agg_params)
+ elif agg_func_name == HMFunction.iqr.name:
+ # iqr
+ agg_df = df.agg(func=iqr, nan_policy='omit')
+ elif agg_func_name == HMFunction.range.name:
+ agg_params = {target_var: range_func}
+ agg_df = df.agg(agg_params)
+ else:
+ # median, mean, min, max
+ agg_df = df.agg(func=agg_func_name, numeric_only=False)
+ return agg_df
+
+
+@log_execution_time()
+def gen_ticks_format(data):
+ if not len(data):
+ return None
+
+ # get fmt from first list of data instead of all data
+ ticks_format = get_fmt_from_array(data[0]['y'])
+ return ticks_format
diff --git a/ap/api/analyze/services/pca.py b/ap/api/analyze/services/pca.py
index e9d28a0..7c20c85 100644
--- a/ap/api/analyze/services/pca.py
+++ b/ap/api/analyze/services/pca.py
@@ -367,12 +367,12 @@ def gen_sensor_headers(orig_graph_param):
short_names = {}
used_names = set()
for proc in orig_graph_param.array_formval:
- for col_id, col_name in zip(proc.col_ids, proc.col_names):
+ for col_id, col_name, col_show_name in zip(proc.col_ids, proc.col_names, proc.col_show_names):
name = gen_sql_label(col_id, col_name)
- dic_labels[name] = col_name
+ dic_labels[name] = col_show_name
# gen short name
- new_name = gen_abbr_name(col_name)
+ new_name = gen_abbr_name(col_show_name)
i = 1
while new_name in used_names:
new_name = f'{new_name[0:-3]}({i})'
diff --git a/ap/api/heatmap/services.py b/ap/api/heatmap/services.py
index 79f1740..3d96feb 100644
--- a/ap/api/heatmap/services.py
+++ b/ap/api/heatmap/services.py
@@ -27,9 +27,6 @@
from ap.common.trace_data_log import TraceErrKey, EventType, EventAction, Target, trace_log
from ap.setting_module.models import CfgProcess
from ap.trace_data.schemas import DicParam
-from ap.common.common_utils import DATE_FORMAT_STR, DATE_FORMAT_STR_CSV
-
-CHM_AGG_FUNC = [HMFunction.median.name, HMFunction.mean.name, HMFunction.std.name]
@log_execution_time()
@@ -206,7 +203,7 @@ def build_plot_data(df, end_col, hm_function):
}
-def get_function_i18n(hm_function): # TODO better. can be moved to frontend
+def get_function_i18n(hm_function):
""" Generate i18n aggregate function name """
return _('CHM' + hm_function.replace('_', ' ').title().replace(' ', ''))
@@ -551,7 +548,7 @@ def agg_func_with_na(group_data, func_name):
agg_func = {
HMFunction.median.name: group_data.median,
HMFunction.mean.name: group_data.mean,
- HMFunction.std.name: group_data.mean,
+ HMFunction.std.name: group_data.std,
}
return agg_func[func_name](skipna=True)
@@ -589,12 +586,8 @@ def groupby_and_aggregate(df: pd.DataFrame, hm_function: HMFunction, hm_mode, hm
step_time = (hm_step * 60) if hm_mode == 1 else (hm_step * 3600)
df[end_col] = step_time / df[end_col]
else:
- if hm_function.name not in CHM_AGG_FUNC:
- agg_func = hm_function.name
- else:
- agg_func = lambda x: agg_func_with_na(x, hm_function.name)
- agg_params = {end_col: agg_func, TIME_COL: HMFunction.first.name}
- df = df.groupby(agg_cols).agg(agg_params).reset_index()
+ agg_params = {end_col: hm_function.name, TIME_COL: HMFunction.first.name}
+ df = df.groupby(agg_cols).agg(agg_params, numeric_only=False).reset_index()
return df
@@ -682,7 +675,8 @@ def gen_heatmap_data_as_dict(graph_param, dic_param, dic_proc_cfgs, dic_cat_filt
var_agg_cols = [gen_sql_label(cfg_col.id, cfg_col.column_name) for cfg_col in cfg_facet_cols]
# get sensor data from db
- df, actual_record_number, unique_serial = get_data_from_db(graph_param, dic_cat_filters, use_expired_cache=use_expired_cache)
+ df, actual_record_number, unique_serial = get_data_from_db(graph_param, dic_cat_filters,
+ use_expired_cache=use_expired_cache)
# filter by cat
df, dic_param = filter_cat_dict_common(df, dic_param, dic_cat_filters, cat_exp, cat_procs, graph_param, True)
@@ -721,6 +715,7 @@ def gen_heatmap_data_as_dict(graph_param, dic_param, dic_proc_cfgs, dic_cat_filt
return dic_df_proc, hm_mode, hm_step, dic_col_func, df_cells, var_agg_cols, target_var_data, export_df
+
@log_execution_time()
def get_target_variable_data_from_df(df, dic_proc_cfgs, graph_param, cat_only=True):
target_data = []
@@ -815,4 +810,4 @@ def gen_sub_df_from_heatmap(heatmap_data, dic_params, dic_proc_cfgs, dic_col_fun
def to_multiple_csv(data, delimiter=',', client_timezone=None):
csv_data = []
for df in data:
- csv_data.append(df.to_csv(sep=delimiter))
\ No newline at end of file
+ csv_data.append(df.to_csv(sep=delimiter))
diff --git a/ap/api/sankey_plot/sankey_glasso/sankey_services.py b/ap/api/sankey_plot/sankey_glasso/sankey_services.py
index 7ad42ee..127bec0 100644
--- a/ap/api/sankey_plot/sankey_glasso/sankey_services.py
+++ b/ap/api/sankey_plot/sankey_glasso/sankey_services.py
@@ -164,7 +164,8 @@ def gen_graph_sankey_group_lasso(dic_param):
x_cols = {key: val for key, val in dic_id_name.items() if key != y_id}
groups = [dic_proc_cfgs.get(proc_id).name for key, proc_id in dic_col_proc_id.items() if key != y_id]
- is_classif, dic_skd, dic_bar, dic_scp, dic_tbl, idx = gen_sankey_grouplasso_plot_data(df_sensors, x_cols, y_col, groups)
+ is_classif, dic_skd, dic_bar, dic_scp, dic_tbl, idx = gen_sankey_grouplasso_plot_data(df_sensors, x_cols,
+ y_col, groups)
# get dic_scp
dic_scp = dict(**dic_scp, **dic_tbl)
@@ -242,10 +243,11 @@ def gen_sankey_grouplasso_plot_data(df: pd.DataFrame, x_cols, y_col, groups):
y_1d = df[[y_col_id]].values
# please set verbose=False if info should not be printed
- is_classif, dic_skd, dic_bar, dic_scp, dic_tbl, idx = preprocess_skdpage(x_2d, y_1d, groups, x_col_names, y_col_name,
- penalty_factors=[0.0, 0.1, 0.3, 1.0],
- max_datapoints=10000,
- verbose=True)
+ is_classif, dic_skd, dic_bar, dic_scp, dic_tbl, idx = preprocess_skdpage(x_2d, y_1d, groups, x_col_names,
+ y_col_name,
+ penalty_factors=[0.0, 0.1, 0.3, 1.0],
+ max_datapoints=10000,
+ verbose=True)
return is_classif, dic_skd, dic_bar, dic_scp, dic_tbl, idx
@@ -326,10 +328,10 @@ def get_sensors_objective_explanation(orig_graph_param):
dic_id_name = {}
dic_col_proc_id = {}
for proc in orig_graph_param.array_formval:
- for col_id, col_name in zip(proc.col_ids, proc.col_names):
+ for col_id, col_name, col_show_name in zip(proc.col_ids, proc.col_names, proc.col_show_names):
label = gen_sql_label(col_id, col_name)
dic_label_id[label] = col_id
- dic_id_name[col_id] = col_name
+ dic_id_name[col_id] = col_show_name
dic_col_proc_id[col_id] = proc.proc_id
return dic_label_id, dic_id_name, dic_col_proc_id
diff --git a/ap/api/scatter_plot/services.py b/ap/api/scatter_plot/services.py
index d81c6ee..1479de3 100644
--- a/ap/api/scatter_plot/services.py
+++ b/ap/api/scatter_plot/services.py
@@ -28,6 +28,7 @@
Y_THRESHOLD, SCALE_SETTING, CHART_INFOS, X_SERIAL, Y_SERIAL, ARRAY_PLOTDATA, IS_DATA_LIMITED, ColorOrder, \
TIME_NUMBERINGS, SORT_KEY, VAR_TRACE_TIME, IS_RESAMPLING, CYCLE_IDS, SERIALS, DATETIME, START_PROC
from ap.common.memoize import memoize
+from ap.common.services.ana_inf_data import resample_preserve_min_med_max
from ap.common.services.form_env import bind_dic_param_to_class
from ap.common.services.request_time_out_handler import abort_process_handler, request_timeout_handling
from ap.common.services.sse import notify_progress
@@ -1172,39 +1173,3 @@ def reduce_data_by_number(df, max_graph, recent_flg=None):
return df
-@log_execution_time()
-@abort_process_handler()
-def resample_preserve_min_med_max(x, n_after: int):
- """ Resample x, but preserve (minimum, median, and maximum) values
- Inputs:
- x (1D-NumpyArray or a list)
- n_after (int) Length of x after resampling. Must be < len(x)
- Return:
- x (1D-NumpyArray) Resampled data
- """
- if x.shape[0] > n_after:
- # walkaround: n_after with odd number is easier
- if n_after % 2 == 0:
- n_after += 1
-
- n = len(x)
- n_half = int((n_after - 1) / 2)
-
- # index around median
- x = np.sort(x)
- idx_med = (n + 1) / 2 - 1 # median
- idx_med_l = int(np.ceil(idx_med - 1)) # left of median
- idx_med_r = int(np.floor(idx_med + 1)) # right of median
-
- # resampled index
- idx_low = np.linspace(0, idx_med_l - 1, num=n_half, dtype=int)
- idx_upp = np.linspace(idx_med_r, n - 1, num=n_half, dtype=int)
-
- # resampling
- if n % 2 == 1:
- med = x[int(idx_med)]
- x = np.concatenate((x[idx_low], [med], x[idx_upp]))
- else:
- med = 0.5 * (x[idx_med_l] + x[idx_med_r])
- x = np.concatenate((x[idx_low], [med], x[idx_upp]))
- return x
diff --git a/ap/api/setting_module/services/csv_import.py b/ap/api/setting_module/services/csv_import.py
index 94b79ba..e290ee1 100644
--- a/ap/api/setting_module/services/csv_import.py
+++ b/ap/api/setting_module/services/csv_import.py
@@ -43,8 +43,10 @@ def import_csv_job(_job_id, _job_name, _db_id, _proc_id, _proc_name, is_user_req
_job_id {[type]} -- [description] (default: {None})
_job_name {[type]} -- [description] (default: {None})
"""
+
def _add_gen_proc_link_job(*_args, **_kwargs):
add_gen_proc_link_job(is_user_request=is_user_request, *_args, **_kwargs)
+
kwargs.pop('is_user_request', None)
gen = import_csv(*args, **kwargs)
send_processing_info(gen, JobType.CSV_IMPORT, db_code=_db_id, process_id=_proc_id, process_name=_proc_name,
@@ -201,7 +203,7 @@ def import_csv(proc_id, record_per_commit=RECORD_PER_COMMIT, is_user_request=Non
continue
default_csv_param['usecols'] = [i for i, col in enumerate(csv_cols) if col]
- use_col_names = csv_cols
+ use_col_names = [col for col in csv_cols if col]
# read csv file
default_csv_param['dtype'] = {col: 'string' for col, data_type in dic_use_cols.items() if
diff --git a/ap/api/setting_module/services/show_latest_record.py b/ap/api/setting_module/services/show_latest_record.py
index b5dc893..b5bb9fb 100644
--- a/ap/api/setting_module/services/show_latest_record.py
+++ b/ap/api/setting_module/services/show_latest_record.py
@@ -1,6 +1,9 @@
import os
from functools import lru_cache
from itertools import islice
+from typing import List, Tuple, Any
+
+import pandas as pd
from ap.api.efa.services.etl import preview_data, detect_file_delimiter
from ap.api.setting_module.services.csv_import import convert_csv_timezone
@@ -21,7 +24,6 @@
make_session, CfgProcess, crud_config
from ap.setting_module.schemas import VisualizationSchema
from ap.trace_data.models import Sensor, find_sensor_class
-import pandas as pd
def get_latest_records(data_source_id, table_name, limit):
@@ -242,6 +244,11 @@ def preview_csv_data(folder_url, etl_func, csv_delimiter, limit, return_df=False
header_names = normalize_list(header_names)
df_data_details = normalize_big_rows(data_details, header_names)
data_types = [gen_data_types(df_data_details[col]) for col in header_names]
+ df_data_details, org_headers, header_names, dupl_cols, data_types = drop_null_header_column(df_data_details,
+ org_headers,
+ header_names,
+ dupl_cols,
+ data_types)
else:
# try to get file which has data to detect data types + get col names
dic_file_info, csv_file = get_etl_good_file(sorted_files)
@@ -282,7 +289,11 @@ def preview_csv_data(folder_url, etl_func, csv_delimiter, limit, return_df=False
org_headers, header_names, dupl_cols = gen_colsname_for_duplicated(header_names)
header_names = normalize_list(header_names)
df_data_details = normalize_big_rows(data_details, header_names)
-
+ df_data_details, org_headers, header_names, dupl_cols, data_types = drop_null_header_column(df_data_details,
+ org_headers,
+ header_names,
+ dupl_cols,
+ data_types)
has_ct_col = True
dummy_datetime_idx = None
if df_data_details is not None:
@@ -294,6 +305,8 @@ def preview_csv_data(folder_url, etl_func, csv_delimiter, limit, return_df=False
validate_datetime(df_data_details, col, False, False)
convert_csv_timezone(df_data_details, col)
df_data_details.dropna(subset=[col], inplace=True)
+ # TODO: can we do this faster?
+ data_types = [gen_data_types(df_data_details[col]) for col in header_names]
df_data_details = df_data_details[0:limit]
if DataType.DATETIME.value not in data_types and DATETIME_DUMMY not in df_data_details.columns:
@@ -381,7 +394,7 @@ def get_etl_good_file(sorted_files):
dic_file_info, is_empty_file = check_result
- if dic_file_info is None:
+ if dic_file_info is None or isinstance(dic_file_info, Exception):
continue
if is_empty_file:
@@ -465,4 +478,25 @@ def gen_colsname_for_duplicated(cols_name):
def is_valid_list(df_rows):
return (isinstance(df_rows, list) and len(df_rows)) \
- or (isinstance(df_rows, pd.DataFrame) and not df_rows.empty)
\ No newline at end of file
+ or (isinstance(df_rows, pd.DataFrame) and not df_rows.empty)
+
+
+# TODO: add test for this
+def drop_null_header_column(df: pd.DataFrame, original_headers: List[str], header_names: List[str],
+ duplicated_names: List[str],
+ data_types: List[int]) -> Tuple[pd.DataFrame, List[str], List[str], List[str], List[int]]:
+ null_indexes = {i for i, col_name in enumerate(original_headers) if not col_name}
+ if not null_indexes:
+ return df, original_headers, header_names, duplicated_names, data_types
+
+ null_header_names = [col_name for i, col_name in enumerate(header_names) if i in null_indexes]
+ df = df.drop(columns=null_header_names)
+
+ def filter_non_null_indexes(arr: List[Any]) -> List[Any]:
+ return [elem for i, elem in enumerate(arr) if i not in null_indexes]
+
+ new_original_headers = filter_non_null_indexes(original_headers)
+ new_header_names = filter_non_null_indexes(header_names)
+ new_duplicated_names = filter_non_null_indexes(duplicated_names)
+ new_data_types = filter_non_null_indexes(data_types)
+ return df, new_original_headers, new_header_names, new_duplicated_names, new_data_types
diff --git a/ap/api/trace_data/services/csv_export.py b/ap/api/trace_data/services/csv_export.py
index 8a194c7..9ab061b 100644
--- a/ap/api/trace_data/services/csv_export.py
+++ b/ap/api/trace_data/services/csv_export.py
@@ -134,7 +134,10 @@ def to_csv(df: DataFrame, dic_proc_cfgs: Dict[int, CfgProcess], graph_param: Dic
suffix = '...'
dic_rename = {}
for proc in graph_param.array_formval:
- proc_cfg = dic_proc_cfgs[proc.proc_id]
+ proc_cfg = dic_proc_cfgs[proc.proc_id] if proc.proc_id in dic_proc_cfgs else None
+ if not proc_cfg:
+ continue
+
for col_id, col_name, name in zip(proc.col_ids, proc.col_names, proc.col_show_names):
old_name = gen_sql_label(col_id, col_name)
if old_name not in df.columns:
diff --git a/ap/api/trace_data/services/proc_link.py b/ap/api/trace_data/services/proc_link.py
index 8e5001e..594391e 100644
--- a/ap/api/trace_data/services/proc_link.py
+++ b/ap/api/trace_data/services/proc_link.py
@@ -4,14 +4,13 @@
from apscheduler.triggers import date, interval
from apscheduler.triggers.date import DateTrigger
-from loguru import logger
from pytz import utc
from sqlalchemy import insert
from sqlalchemy.sql.expression import literal
from ap import scheduler
from ap.common.constants import *
-from ap.common.logger import log_execution_time
+from ap.common.logger import logger, log_execution_time
from ap.common.memoize import set_all_cache_expired
from ap.common.pydn.dblib.db_common import gen_insert_col_str, gen_select_col_str, PARAM_SYMBOL, add_single_quote
from ap.common.pydn.dblib.db_proxy import DbProxy, gen_data_source_of_universal_db
diff --git a/ap/api/trace_data/services/time_series_chart.py b/ap/api/trace_data/services/time_series_chart.py
index ae73bfc..54ffff3 100644
--- a/ap/api/trace_data/services/time_series_chart.py
+++ b/ap/api/trace_data/services/time_series_chart.py
@@ -321,12 +321,21 @@ def gen_cat_label_unique(df, dic_param, graph_param, has_na=False, label_col_id=
cat_exp_list = gen_unique_data(df, dic_proc_cfgs, cat_and_div, has_na)
dic_param[CAT_EXP_BOX] = list(cat_exp_list.values())
+ color_vals = []
+ if graph_param.common.color_var:
+ color_vals.append(graph_param.common.color_var)
+ if graph_param.common.agp_color_vars:
+ agp_color_vals = list(set(int(color) for color in graph_param.common.agp_color_vars.values()))
+ color_vals += agp_color_vals
+
exclude_col_id = cat_and_div
if label_col_id:
exclude_col_id += label_col_id
+ exclude_col_id += color_vals
sensor_ids = [col for col in graph_param.common.sensor_cols if col not in exclude_col_id]
dic_param[CAT_ON_DEMAND] = list(gen_unique_data(df, dic_proc_cfgs, sensor_ids, True).values())
+ dic_param[UNIQUE_COLOR] = list(gen_unique_data(df, dic_proc_cfgs, color_vals, False).values())
dic_param = gen_group_filter_list(df, graph_param, dic_param)
@@ -1300,10 +1309,10 @@ def gen_trace_procs_sql(path, graph: TraceGraph, start_tm, end_tm, short_procs,
# calculate +-14 day for end processes
e_start_tm = convert_time(start_tm, return_string=False)
e_start_tm = add_days(e_start_tm, -14)
- e_start_tm = convert_time(e_start_tm)
+ e_start_tm = convert_time(e_start_tm, remove_ms=True)
e_end_tm = convert_time(end_tm, return_string=False)
e_end_tm = add_days(e_end_tm, 14)
- e_end_tm = convert_time(e_end_tm)
+ e_end_tm = convert_time(e_end_tm, remove_ms=True)
for from_proc, to_proc in zip(path[:-1], path[1:]):
is_trace_forward = True
edge_id = (from_proc, to_proc)
diff --git a/ap/common/assets/assets.json b/ap/common/assets/assets.json
new file mode 100644
index 0000000..7822366
--- /dev/null
+++ b/ap/common/assets/assets.json
@@ -0,0 +1,216 @@
+{
+ "all": {
+ "js": [
+ "common/js/libs/jquery.js",
+ "common/js/libs/jquery-ui.min.js",
+ "common/js/libs/datepicker.js",
+ "modules/popper/umd/popper.min.js",
+ "common/js/libs/bootstrap.min.js",
+ "common/js/libs/all.min.js",
+ "common/js/libs/jquery.ui.datepicker-ja.min.js",
+ "common/js/libs/moment-with-locales.js",
+ "common/js/libs/loadingoverlay.min.js",
+ "common/js/libs/lodash.min.js",
+ "common/js/libs/gtag.js",
+ "common/js/libs/clipboard.min.js",
+ "common/js/config_data_interface.js",
+ "common/js/libs/bootstrap-table.min.js",
+ "common/js/libs/bootstrap-table-filter-control.min.js",
+ "common/js/libs/bootstrap-table-locale-all.min.js",
+ "common/js/libs/html2canvas.min.js",
+ "common/js/take_screenshot.js",
+ "common/js/libs/plotly.min.js",
+ "common/js/libs/toastr.min.js",
+ "modules/js-datatables/lib/jquery.dataTables.min.js",
+ "common/js/libs/dataTables.fixedHeader.min.js",
+ "common/js/libs/select2.min.js",
+ "common/js/terms_of_use.js",
+ "common/js/divide_by_calendar.js",
+ "common/js/base.js",
+ "common/js/utils.js",
+ "common/js/summary_table.js",
+ "common/js/auto-update-common.js",
+ "common/js/data-finder.js",
+ "common/js/components.js",
+ "common/js/data_point_info_table.js",
+ "common/js/save_load_user_input.js",
+ "common/js/validation.js",
+ "common/js/clipboard_utils.js",
+ "common/js/libs/dragndrop.js",
+ "common/js/libs/d3-format.js",
+ "common/js/dn-custom-select.js",
+ "common/js/libs/popper.min.js",
+ "common/js/libs/shepherd.min.js",
+ "common/js/ap_tour.js",
+ "common/js/graph_nav.js",
+ "modules/jquery-ui-timepicker-addon/jquery-ui-timepicker-addon.min.js",
+ "modules/jquery-ui-timepicker-addon/i18n/jquery-ui-timepicker-ja.js",
+ "modules/date-range-picker/daterangepicker.js",
+ "modules/date-range-picker/daterangepicker-utils.js",
+ "common/js/libs/js.cookie.min.js",
+ "common/js/libs/pagination.min.js"
+ ],
+ "css": [
+ "common/custom-jquery/jquery-ui.css",
+ "common/css/select2.min.css",
+ "common/css/bootstrap.min.css",
+ "common/css/all.min.css",
+ "common/css/main.css",
+ "common/css/components.css",
+ "common/css/dragndrop.css",
+ "common/css/bootstrap-table.min.css",
+ "common/css/toastr.css",
+ "common/css/data-finder.css",
+ "modules/js-datatables/lib/jquery.dataTables.min.css",
+ "common/css/user-setting-table.css",
+ "common/css/shepherd.css",
+ "modules/jquery-ui-timepicker-addon/jquery-ui-timepicker-addon.css",
+ "modules/date-range-picker/daterangepicker.css",
+ "common/css/pagination.css",
+ "common/css/graph_nav.css"
+ ]
+ },
+ "fpp": {
+ "js": [
+ "common/js/libs/Chart.min.js",
+ "common/js/libs/chartjs-adapter-moment.min.js",
+ "common/js/libs/chartjs-plugin-annotation-latest.min.js",
+ "trace_data/js/trace_data_time_series.js",
+ "trace_data/js/trace_data_histogram.js",
+ "trace_data/js/trace_data_step_bar_chart.js",
+ "trace_data/js/trace_data_summary_table.js",
+ "trace_data/js/trace_data_cross_hair.js",
+ "trace_data/js/trace_data_categorical_table.js",
+ "trace_data/js/trace_data_scatter_plot.js",
+ "trace_data/js/trace_data_whisker_plot.js",
+ "common/js/cat_facet_label_filter_modal.js",
+ "trace_data/js/trace_data.js",
+ "trace_data/js/trace_data_histogram_with_kde.js"
+ ],
+ "css": [
+ "trace_data/css/trace_data.css"
+ ]
+ },
+ "stp": {
+ "js": [
+ "categorical_plot/js/categorical_histogram_with_density_curve.js",
+ "categorical_plot/js/categorical_plot_utils.js",
+ "categorical_plot/js/categorical_plot.js",
+ "common/js/cat_facet_label_filter_modal.js"
+ ],
+ "css": [
+ "categorical_plot/css/categorical_plot.css"
+ ]
+ },
+ "rlp": {
+ "js": [
+ "ridgeline_plot/js/rlp_template.js",
+ "ridgeline_plot/js/ridgeline_plot_utils.js",
+ "ridgeline_plot/js/ridgeline_plot.js",
+ "common/js/cat_facet_label_filter_modal.js"
+ ],
+ "css": [
+ "ridgeline_plot/css/ridgeline.css"
+ ]
+ },
+ "chm": {
+ "js": [
+ "heatmap/js/heatmap.js",
+ "heatmap/js/heatmap_plotly.js",
+ "common/js/cat_facet_label_filter_modal.js"
+ ],
+ "css": [
+ "heatmap/css/heat_map.css"
+ ]
+ },
+ "msp": {
+ "js": [
+ "multiple_scatter_plot/js/multiple_scatter_histogram.js",
+ "multiple_scatter_plot/js/multiple_scatter_contour.js",
+ "multiple_scatter_plot/js/multiple_scatter_plot.js"
+ ],
+ "css": [
+ "multiple_scatter_plot/css/multiple_scatter_plot.css"
+ ]
+ },
+ "scp": {
+ "js": [
+ "common/js/libs/Chart.bundle.min.js",
+ "scatter_plot/js/scatter_chart.js",
+ "scatter_plot/js/hist_with_density.js",
+ "scatter_plot/js/scatter_plot.js",
+ "scatter_plot/js/scp_heatmap_plot.js",
+ "scatter_plot/js/scp_violin_plot.js",
+ "common/js/cat_facet_label_filter_modal.js"
+ ],
+ "css": [
+ "scatter_plot/css/scatter_plot.css"
+ ]
+ },
+ "pcp": {
+ "js": [
+ "common/js/libs/Chart.bundle.min.js",
+ "common/js/libs/fSelect.js",
+ "parallel_plot/js/parallel_plot.js",
+ "common/js/cat_facet_label_filter_modal.js"
+ ],
+ "css": [
+ "common/css/fSelect.css",
+ "parallel_plot/css/parallel_plot.css"
+ ]
+ },
+ "skd": {
+ "js": [
+ "sankey_plot/js/sankey_plot.js",
+ "sankey_plot/js/sankey_scp.js"
+ ],
+ "css": [
+ "sankey_plot/css/sankey_plot.css"
+ ]
+ },
+ "cog": {
+ "js": [
+ "modules/sigmajs/build/sigma.min.js",
+ "modules/sigmajs/build/plugins/sigma.renderers.edgeLabels.min.js",
+ "modules/sigmajs/build/plugins/sigma.plugins.dragNodes.min.js",
+ "modules/sigmajs/build/plugins/sigma.layout.forceAtlas2.min.js",
+ "modules/sigmajs/build/plugins/sigma.plugins.animate.min.js",
+ "modules/sigmajs/build/plugins/sigma.layout.noverlap.min.js",
+ "co_occurrence/js/pareto_plot.js",
+ "co_occurrence/js/co_occurrence_csv.js"
+ ],
+ "css": [
+ "co_occurrence/css/co_occurrence_csv.css"
+ ]
+ },
+ "pca": {
+ "js": [
+ "common/js/libs/jquery.dataTables.min.js",
+ "common/js/libs/dataTables.bootstrap4.min.js",
+ "common/js/libs/dom-text.js",
+ "analyze/js/pca_toastr.js",
+ "analyze/js/generateJson.js",
+ "analyze/js/hotelling_common.js",
+ "analyze/js/hotelling_timeseries.js",
+ "analyze/js/hotelling_scatters.js",
+ "analyze/js/hotelling_biplot.js",
+ "analyze/js/hotelling_q_contribution.js",
+ "analyze/js/hotelling_t2_contribution.js",
+ "analyze/js/pca.js"
+ ],
+ "css": [
+ "modules/js-datatables/lib/jquery.dataTables.min.css",
+ "analyze/css/anomaly_detection.css"
+ ]
+ },
+ "agp": {
+ "js": [
+ "aggregate_plot/js/aggregation_chart.js",
+ "aggregate_plot/js/aggregate_plot.js",
+ "common/js/cat_facet_label_filter_modal.js"
+ ],
+ "css": [
+ "aggregate_plot/css/aggregate_plot.css"
+ ]
+ }
+}
\ No newline at end of file
diff --git a/ap/common/common_utils.py b/ap/common/common_utils.py
index 153a063..fb17b60 100644
--- a/ap/common/common_utils.py
+++ b/ap/common/common_utils.py
@@ -1,12 +1,12 @@
import copy
import csv
import fnmatch
+import json
import locale
import os
import pickle
import re
import shutil
-import socket
import sys
from collections import OrderedDict
from datetime import datetime, timedelta
@@ -20,11 +20,12 @@
from dateutil import parser
from dateutil.relativedelta import relativedelta
from flask import g
+from flask_assets import Environment, Bundle
from pandas import DataFrame
from ap.common.constants import AbsPath, DataType, YAML_AUTO_INCREMENT_COL, CsvDelimiter, R_PORTABLE, \
SQL_COL_PREFIX, FilterFunc, ENCODING_ASCII, ENCODING_UTF_8, FlaskGKey, LANGUAGES, ENCODING_SHIFT_JIS, \
- ENCODING_UTF_8_BOM
+ ENCODING_UTF_8_BOM, appENV
from ap.common.logger import logger, log_execution_time
from ap.common.services.normalization import unicode_normalize_nfkc
@@ -281,7 +282,7 @@ def universal_db_exists():
# convert time before save to database YYYY-mm-DDTHH:MM:SS.NNNNNNZ
-def convert_time(time=None, format_str=DATE_FORMAT_STR, return_string=True, only_milisecond=False):
+def convert_time(time=None, format_str=DATE_FORMAT_STR, return_string=True, only_milisecond=False, remove_ms=False):
if not time:
time = datetime.utcnow()
elif isinstance(time, str):
@@ -291,6 +292,8 @@ def convert_time(time=None, format_str=DATE_FORMAT_STR, return_string=True, only
time = time.strftime(format_str)
if only_milisecond:
time = time[:-3]
+ elif remove_ms:
+ time = time[:-8]
return time
@@ -1109,3 +1112,37 @@ class NoDataFoundException(Exception):
def __init__(self):
super().__init__()
self.code = 999
+
+def bundle_assets(_app):
+ """
+ bundle assets when application be started at the first time
+ for commnon assets (all page), and single page
+ """
+ env = os.environ.get('ANALYSIS_INTERFACE_ENV')
+ # bundle js files
+ assets_path = os.path.join('ap', 'common', 'assets', 'assets.json')
+ with open(assets_path, 'r') as f:
+ _assets = json.load(f)
+
+ assets = Environment(_app)
+ if env != appENV.PRODUCTION.value:
+ assets.debug = True
+
+ for page in _assets.keys():
+ js_assets = _assets[page].get('js') or []
+ css_assets = _assets[page].get('css') or []
+ js_asset_name = f'js_{page}'
+ css_asset_name = f'css_{page}'
+ if env != appENV.PRODUCTION.value:
+ assets.register(js_asset_name, *js_assets)
+ assets.register(css_asset_name, *css_assets)
+ else:
+ js_bundle = Bundle(*js_assets,
+ output=f'common/js/{page}.packed.js')
+ css_bundle = Bundle(*css_assets,
+ output=f'common/css/{page}.packed.css')
+ assets.register(js_asset_name, js_bundle)
+ assets.register(css_asset_name, css_bundle)
+ # build assets
+ js_bundle.build()
+ css_bundle.build()
\ No newline at end of file
diff --git a/ap/common/constants.py b/ap/common/constants.py
index 336e51d..4d4de1a 100644
--- a/ap/common/constants.py
+++ b/ap/common/constants.py
@@ -41,8 +41,11 @@
DATETIME_DUMMY = 'DatetimeDummy'
MAX_DATETIME_STEP_PER_DAY = 8640 # 10s/step -> 6steps * 60min * 24hrs
+RESAMPLING_SIZE = 10_000
+
LOG_LEVEL = 'log_level'
+
class AP_LOG_LEVEL(Enum):
DEBUG = auto()
INFO = auto()
@@ -330,6 +333,17 @@ class ErrorMsg(Enum):
SERIALS = 'serials'
DATETIME = 'datetime'
+AGP_COLOR_VARS = 'aggColorVar'
+DIVIDE_OFFSET = 'divideOffset'
+DIVIDE_FMT = 'divideFormat'
+DIVIDE_FMT_COL = 'divide_format'
+COLOR_NAME = 'color_name'
+DATA = 'data'
+SHOWN_NAME = 'shown_name'
+COL_DATA_TYPE = 'data_type'
+DIVIDE_CALENDAR_DATES = 'divDates'
+DIVIDE_CALENDAR_LABELS = 'divFormats'
+
class HMFunction(Enum):
max = auto()
@@ -414,7 +428,8 @@ class MemoizeKey(Enum):
# error message for dangling jobs
FORCED_TO_BE_FAILED = 'DANGLING JOB. FORCED_TO_BE_FAILED'
-DEFAULT_POLLING_FREQ = 180 # default is import every 3 minutes
+DEFAULT_POLLING_FREQ = 180 # default is import every 3 minutes
+
class CfgConstantType(Enum):
def __str__(self):
diff --git a/ap/common/services/ana_inf_data.py b/ap/common/services/ana_inf_data.py
index 138a1bf..406047b 100644
--- a/ap/common/services/ana_inf_data.py
+++ b/ap/common/services/ana_inf_data.py
@@ -7,6 +7,8 @@
from scipy.stats import gaussian_kde, iqr
from ap.common.constants import *
+from ap.common.logger import log_execution_time
+from ap.common.services.request_time_out_handler import abort_process_handler
from ap.common.sigificant_digit import get_fmt_from_array
from ap.common.services.statistics import convert_series_to_number
@@ -168,7 +170,7 @@ def calculate_kde_trace_data(plotdata, bins=128, height=1, full_array_y=None):
data = convert_series_to_number(data)
data = data[np.isfinite(data)]
if full_array_y and len(data) > 10_000:
- sample_data = np.random.choice(data, size=10_000).tolist()
+ sample_data = resample_preserve_min_med_max(data, RESAMPLING_SIZE - 1).tolist()
else:
sample_data = data.tolist()
@@ -351,3 +353,40 @@ def detect_abnormal_count_values(x, nmax=2, threshold=50, min_uniques=10, max_sa
print("outlier not detected")
return val_outlier
+
+@log_execution_time()
+@abort_process_handler()
+def resample_preserve_min_med_max(x, n_after: int):
+ """ Resample x, but preserve (minimum, median, and maximum) values
+ Inputs:
+ x (1D-NumpyArray or a list)
+ n_after (int) Length of x after resampling. Must be < len(x)
+ Return:
+ x (1D-NumpyArray) Resampled data
+ """
+ if x.shape[0] > n_after:
+ # walkaround: n_after with odd number is easier
+ if n_after % 2 == 0:
+ n_after += 1
+
+ n = len(x)
+ n_half = int((n_after - 1) / 2)
+
+ # index around median
+ x = np.sort(x)
+ idx_med = (n + 1) / 2 - 1 # median
+ idx_med_l = int(np.ceil(idx_med - 1)) # left of median
+ idx_med_r = int(np.floor(idx_med + 1)) # right of median
+
+ # resampled index
+ idx_low = np.linspace(0, idx_med_l - 1, num=n_half, dtype=int)
+ idx_upp = np.linspace(idx_med_r, n - 1, num=n_half, dtype=int)
+
+ # resampling
+ if n % 2 == 1:
+ med = x[int(idx_med)]
+ x = np.concatenate((x[idx_low], [med], x[idx_upp]))
+ else:
+ med = 0.5 * (x[idx_med_l] + x[idx_med_r])
+ x = np.concatenate((x[idx_low], [med], x[idx_upp]))
+ return x
diff --git a/ap/common/services/csv_content.py b/ap/common/services/csv_content.py
index 84036d0..482fb8f 100644
--- a/ap/common/services/csv_content.py
+++ b/ap/common/services/csv_content.py
@@ -1,8 +1,8 @@
# CSVコンテンツを生成するService
# read(dic_form, local_params)が外向けの関数
-import io
import csv
import decimal
+import io
# https://stackoverrun.com/ja/q/6869533
import logging
import math
@@ -42,17 +42,15 @@ def read_data(f_name, headers=None, skip_head=None, end_row=None, delimiter=',',
rows = csv.reader((line.replace('\0', '') for line in f), delimiter=delimiter)
# skip tail
- if end_row != None:
+ if end_row is not None:
rows = islice(rows, end_row + 1)
-
# skip head
- if skip_head != None:
- for _ in range(skip_head):
- next(rows)
+ if skip_head is not None:
+ rows = islice(rows, skip_head, None)
# use specify header( may be get from yaml config)
csv_headers = next(rows)
- if headers:
+ if headers is not None:
yield normalize_func(headers)
else:
yield normalize_func(csv_headers)
@@ -252,4 +250,3 @@ def zip_file_to_response(csv_data, file_names):
})
return response
-
diff --git a/ap/common/services/form_env.py b/ap/common/services/form_env.py
index 31d7741..4b65996 100644
--- a/ap/common/services/form_env.py
+++ b/ap/common/services/form_env.py
@@ -6,11 +6,11 @@
from ap.common.common_utils import as_list
from ap.common.constants import *
+from ap.common.services.jp_to_romaji_utils import to_romaji
from ap.setting_module.models import CfgProcessColumn
from ap.setting_module.services.process_config import get_all_process_no_nested, get_all_visualizations
from ap.trace_data.schemas import DicParam, CommonParam, ConditionProc, CategoryProc, EndProc, \
ConditionProcDetail
-from ap.common.services.jp_to_romaji_utils import to_romaji
logger = logging.getLogger(__name__)
@@ -25,7 +25,9 @@
CYCLIC_DIV_NUM, CYCLIC_WINDOW_LEN, CYCLIC_INTERVAL, MATRIX_COL, COLOR_ORDER,
IS_EXPORT_MODE, IS_IMPORT_MODE, VAR_TRACE_TIME, TERM_TRACE_TIME, CYCLIC_TRACE_TIME, TRACE_TIME,
EMD_TYPE, ABNORMAL_COUNT, REMOVE_OUTLIER_OBJECTIVE_VAR, REMOVE_OUTLIER_EXPLANATORY_VAR,
- DUPLICATE_SERIAL_SHOW, EXPORT_FROM, DUPLICATED_SERIALS_COUNT)
+ DUPLICATE_SERIAL_SHOW, EXPORT_FROM, DUPLICATED_SERIALS_COUNT, AGP_COLOR_VARS, DIVIDE_FMT,
+ DIVIDE_OFFSET, DIVIDE_CALENDAR_DATES, DIVIDE_CALENDAR_LABELS
+ )
conds_startwith_keys = ('filter-', 'cond_', 'machine_id_multi')
@@ -119,8 +121,10 @@ def parse_multi_filter_into_one(dic_form):
dic_cat_filters = json.loads(value)
if dic_cat_filters:
dic_parsed[COMMON][key] = {int(col): vals for col, vals in dic_cat_filters.items()}
- elif key in (TEMP_CAT_EXP, TEMP_CAT_PROCS):
+ elif key in (TEMP_CAT_EXP, TEMP_CAT_PROCS, AGP_COLOR_VARS, DIVIDE_CALENDAR_DATES, DIVIDE_CALENDAR_LABELS):
dic_parsed[COMMON][key] = json.loads(value)
+ elif key.startswith((VAR_TRACE_TIME, TRACE_TIME)):
+ dic_parsed[COMMON][TRACE_TIME] = value
else:
dic_parsed[COMMON][key] = value
@@ -387,7 +391,14 @@ def bind_dic_param_to_class(dic_param):
sensor_cols=sensor_cols,
duplicate_serial_show=dic_common.get(DUPLICATE_SERIAL_SHOW, DuplicateSerialShow.SHOW_BOTH),
is_export_mode=dic_common.get(IS_EXPORT_MODE, False),
- duplicated_serials_count=dic_common.get(DUPLICATED_SERIALS_COUNT, DuplicateSerialCount.AUTO.value)
+ duplicated_serials_count=dic_common.get(DUPLICATED_SERIALS_COUNT,
+ DuplicateSerialCount.AUTO.value),
+ divide_format=dic_common.get(DIVIDE_FMT, None),
+ divide_offset=dic_common.get(DIVIDE_OFFSET, None),
+ agp_color_vars=dic_common.get(AGP_COLOR_VARS, None),
+ is_latest=True if dic_common.get(TRACE_TIME, 'default') == 'recent' else False,
+ divide_calendar_dates=dic_common.get(DIVIDE_CALENDAR_DATES, []),
+ divide_calendar_labels=dic_common.get(DIVIDE_CALENDAR_LABELS, []),
)
# use the first end proc as start proc
@@ -496,7 +507,7 @@ def get_end_procs_param(dic_param):
def update_data_from_multiple_dic_params(orig_dic_param, dic_param):
updated_keys = [ARRAY_PLOTDATA, ACT_CELLS, UNIQUE_SERIAL, MATCHED_FILTER_IDS, UNMATCHED_FILTER_IDS, \
NOT_EXACT_MATCH_FILTER_IDS, CAT_EXP_BOX, ACTUAL_RECORD_NUMBER, IMAGES, IS_GRAPH_LIMITED, \
- EMD_TYPE, TIME_CONDS, FMT, CAT_ON_DEMAND]
+ EMD_TYPE, TIME_CONDS, FMT, CAT_ON_DEMAND, UNIQUE_COLOR]
for key in updated_keys:
if key not in dic_param:
continue
diff --git a/ap/common/trace_data_log.py b/ap/common/trace_data_log.py
index 829eca8..9af3154 100644
--- a/ap/common/trace_data_log.py
+++ b/ap/common/trace_data_log.py
@@ -68,6 +68,7 @@ class EventType(Enum):
PCP = 'PCP'
CHM = 'CHM'
SCP = 'ScP'
+ AGP = 'AGP'
class EventAction(Enum):
diff --git a/ap/config/image/AgP.png b/ap/config/image/AgP.png
new file mode 100644
index 0000000..c4b9959
Binary files /dev/null and b/ap/config/image/AgP.png differ
diff --git a/ap/config/tile_interface_analysis_platform.yml b/ap/config/tile_interface_analysis_platform.yml
index 974eb94..6843e2c 100644
--- a/ap/config/tile_interface_analysis_platform.yml
+++ b/ap/config/tile_interface_analysis_platform.yml
@@ -46,6 +46,25 @@
hover_en: 'It visualizes long-term data such as annual variations in processes and parameters that can vary depending on days of the week or shifts, etc. For example, it is possible to visualize the rate of defects or changes in production volume over a long period of time. It is possible to understand changes in specific patterns, such as a high rate of defects at the beginning of the week, or the occurrence of specific alarms soon after the lunch breaks, and so on.'
hover_ja: '工程での年間変動など長期間のデータや曜日・シフトなどで変動しうるパラメータを可視化します。\n例えば不良率や生産量の推移などを長期にわたって可視化することができ、例えば週明けが不良率が高いとか、昼休み空けに特定のアラームが発生しやすいといった、特定のパターンの変動などを把握することが可能です。'
link_address: '/ap/chm'
+ - title_en: 'Basic'
+ title_ja: 'ベーシック'
+ tiles:
+ - row: 1
+ column: 1
+ title_en: 'AgP Aggregation Plot'
+ title_ja: 'AgP 集計プロット'
+ png_path: 'AgP.png'
+ hover_en: 'You can visualize aggregate results such as production volume and number of defects in stacked bar graphs, and time-series changes in sensor data in line plots, and explore their relationships.'
+ hover_ja: '生産量、不良数などの集計結果を積み上げ棒グラフで、センサーデータの時系列変化は線分プロットで可視化し、それらの関係性を探索することができます。'
+ link_address: '/ap/agp'
+ - row: 1
+ column: 2
+ title_en: 'ScP Scatter Plot'
+ title_ja: 'ScP 散布図'
+ png_path: 'ScP.png'
+ hover_en: 'You can draw a scatter plot of the relationships between quantitative variables such as measurements. In addition, it can be decomposed by category, period, and number of data, and changes in relationships can be confirmed. For example, it is effective for changing the process window.\n Also, the relationship between categorical variables such as equipment names and quantitative variables can be drawn with a violin plot, and the relationships between categorical variables can be drawn with a heat map. The relationships between variables that do not depend on the data type and their changes can be confirmed. '
+ hover_ja: '測定値などの量的変数同士の関係性を散布図で描画します。さらに、カテゴリや期間、データ数で分解でき、関係性の変化を確認することができます。例えばプロセスウィンドウの変化に有効です。\nまた、設備名などのカテゴリ変数と量的変数の関係はバイオリンプロットで、カテゴリ変数同士の関係性はヒートマップで描画でき、データの型に寄らない変数同士の関係性およびその変化を確認することができます。'
+ link_address: '/ap/scp'
- title_en: 'Multidimensional Correlation'
title_ja: '相関/多次元相関'
tiles:
@@ -67,14 +86,6 @@
hover_ja: '目的の変数に関して、大量の変数間の相関関係を確認できます。\nそれぞれの軸(各軸が左右に平行に配置されています)で色が分かれている軸が注意すべき変数、すなわち何らかの相関がある軸と判断してください(特に相関係数順に並べている場合は両端に着目すべき色が分かれた軸が見えます。色が混ざっている軸は相関が低いと考えて結構です)。\n
バランスよく虹色になっている軸では線形の相関(カラーバーと同じ色配置[正の相関]もしくは反転した色配置[負の相関])が認められます。色が分離しているものの偏った色配置になっている場合は非線形の関係が考えられますので、何らかの関数変換を行なったり、クラスタリングし層別する必要があります(分離している変数と目的変数で散布図を用いて関係を確認することをお勧めします)。'
link_address: '/ap/pcp'
- - row: 1
- column: 3
- title_en: 'ScP Scatter Plot'
- title_ja: 'ScP 散布図'
- png_path: 'ScP.png'
- hover_en: 'You can draw a scatter plot of the relationships between quantitative variables such as measurements. In addition, it can be decomposed by category, period, and number of data, and changes in relationships can be confirmed. For example, it is effective for changing the process window.\n Also, the relationship between categorical variables such as equipment names and quantitative variables can be drawn with a violin plot, and the relationships between categorical variables can be drawn with a heat map. The relationships between variables that do not depend on the data type and their changes can be confirmed. '
- hover_ja: '測定値などの量的変数同士の関係性を散布図で描画します。さらに、カテゴリや期間、データ数で分解でき、関係性の変化を確認することができます。例えばプロセスウィンドウの変化に有効です。\nまた、設備名などのカテゴリ変数と量的変数の関係はバイオリンプロットで、カテゴリ変数同士の関係性はヒートマップで描画でき、データの型に寄らない変数同士の関係性およびその変化を確認することができます。'
- link_address: '/ap/scp'
- title_en: 'Anomaly Detection'
title_ja: '異常検出'
tiles:
diff --git a/ap/setting_module/models.py b/ap/setting_module/models.py
index 35313ca..5c0c793 100644
--- a/ap/setting_module/models.py
+++ b/ap/setting_module/models.py
@@ -9,7 +9,7 @@
from ap import Session
from ap import db
-from ap.common.common_utils import get_current_timestamp, dict_deep_merge
+from ap.common.common_utils import get_current_timestamp, dict_deep_merge, gen_sql_label
from ap.common.constants import JobStatus, FlaskGKey, CsvDelimiter, DataType, CfgConstantType, \
EFA_HEADER_FLAG, DiskUsageStatus, DEFAULT_WARNING_DISK_USAGE, DEFAULT_ERROR_DISK_USAGE
from ap.common.constants import RelationShip
@@ -458,6 +458,18 @@ def get_by_data_type(cls, proc_id, data_type: DataType):
def get_by_ids(cls, ids):
return cls.query.filter(cls.id.in_(ids)).all()
+ @classmethod
+ def get_by_id(cls, col_id: int):
+ return cls.query.get(col_id)
+
+ @classmethod
+ def gen_label_from_col_id(cls, col_id: int):
+ col = cls.get_by_id(col_id)
+ if not col:
+ return None
+ col_label = gen_sql_label(col.id, col.column_name)
+ return col_label
+
@classmethod
def get_serials(cls, proc_id):
return cls.query.filter(cls.process_id == proc_id, cls.is_serial_no == 1).all()
diff --git a/ap/static/aggregate_plot/css/aggregate_plot.css b/ap/static/aggregate_plot/css/aggregate_plot.css
new file mode 100644
index 0000000..979caea
--- /dev/null
+++ b/ap/static/aggregate_plot/css/aggregate_plot.css
@@ -0,0 +1,79 @@
+.cyclic-calender-select-option {
+ display: flex;
+}
+
+.cyclic-calender-option-item {
+ display: flex;
+ align-items: center;
+ margin-bottom: 4px;
+}
+
+.cyclic-calender-option-example {
+ display: inline-block;
+ margin-left: 5px;
+ width: calc(50% - 15px);
+ text-align: left;
+}
+
+.cyclic-calender-option-list {
+ border: 1px solid white;
+ padding-left: 8px;
+ display: flex;
+ flex-direction: column;
+ justify-content: space-between;
+}
+
+.cyclic-calender-option-list:not(:last-child) {
+ /*margin-right: 8px;*/
+}
+.y-scale-group {
+ display: flex;
+}
+
+.for-recent-cyclicCalender {
+ display: flex;
+}
+
+#cyclicCalenderShowDiv {
+ display: inline-block;
+ margin-left: 6px;
+}
+
+.chart-row {
+ background-color: #222222;
+ display: flex;
+ justify-content: space-between;
+ flex-direction: row;
+ width: 100%;
+}
+
+.chart-row:not(:first-child) {
+ margin-top: 5px;
+}
+
+.tschart-title-parent {
+ width: 60px;
+ position: relative;
+}
+
+.tschart-title {
+ transform: rotate(-90deg);
+ display: flex;
+ flex-direction: column;
+ align-items: center;
+ justify-content: flex-start;
+ transform-origin: center;
+ color: #65c5f1;
+ text-align: center;
+ font-size: 12px;
+ height: 100%;
+ white-space: nowrap;
+ overflow: hidden !important;
+ text-overflow: ellipsis;
+}
+
+.chart-area {
+ flex: 1;
+ height: 100%;
+ width: 100%;
+}
\ No newline at end of file
diff --git a/ap/static/aggregate_plot/js/aggregate_plot.js b/ap/static/aggregate_plot/js/aggregate_plot.js
new file mode 100644
index 0000000..fbfd385
--- /dev/null
+++ b/ap/static/aggregate_plot/js/aggregate_plot.js
@@ -0,0 +1,528 @@
+/* eslint-disable no-restricted-syntax,prefer-arrow-callback */
+/* eslint-disable guard-for-in */
+/* eslint-disable no-unused-vars */
+/* eslint-disable no-undef */
+/* eslint-disable no-use-before-define */
+const REQUEST_TIMEOUT = setRequestTimeOut();
+const MAX_NUMBER_OF_GRAPH = 18;
+const MAX_END_PROC = 18;
+tabID = null;
+let currentData = null;
+const graphStore = new GraphStore();
+
+const eles = {
+ endProcSelectedItem: '#end-proc-row select',
+};
+
+const i18n = {
+ selectDivRequiredMessage: $('#i18nSelectDivMessage').text(),
+ allSelection: $('#i18nAllSelection').text(),
+};
+
+
+const formElements = {
+ formID: '#traceDataForm',
+ btnAddCondProc: '#btn-add-cond-proc',
+ endProcItems: '#end-proc-row .end-proc',
+ endProcSelectedItem: '#end-proc-row select',
+ plotCardId: '#plot-cards',
+ plotCard: $('#barplot-cards'),
+ categoryForm: '#traceDataForm',
+ agpCard: '#agpCard',
+ agpScale: $('select[name=agpScale]'),
+ resultSection: $('.result-section'),
+ divideOption: $('#divideOption')
+};
+
+const calenderFormat = {
+ 1: {
+ group1: [
+ 'yyyymmddHH',
+ 'yymmddHH',
+ 'mmddHH',
+ 'ddHH',
+ 'HH',
+ 'yyyymmdd',
+ 'yymmdd',
+ 'mmdd',
+ 'dd'
+ ],
+ group2: [
+ 'yyyymm',
+ 'yymm',
+ 'mm',
+ 'yyyy',
+ 'yy'
+ ]
+ },
+ 2: {
+ group1: [
+ 'yyyy-mm-dd_HH',
+ 'yy-mm-dd_HH',
+ 'mm-dd_HH',
+ 'dd_HH',
+ 'HH',
+ 'yyyy-mm-dd',
+ 'yy-mm-dd',
+ 'mm-dd',
+ 'dd',
+ 'yyyy-mm-dd_Fri',
+ 'yy-mm-dd_Fri',
+ 'mm-dd_Fri',
+ 'dd_Fri',
+ 'Fri',
+ ],
+ group2: [
+ 'yyyymm',
+ 'yymm',
+ 'mm',
+ 'yyyy',
+ 'yy'
+ ]
+ },
+ 3: {
+ group1: [
+ "ww"
+ ]
+ },
+ 4: {
+ group1: [
+ "Www",
+ "Www_mm-dd"
+ ]
+ }
+}
+
+const AGP_EXPORT_URL = {
+ CSV: {
+ ext_name: 'csv',
+ url: '/ap/api/agp/data_export/csv',
+ },
+ TSV: {
+ ext_name: 'tsv',
+ url: '/ap/api/agp/data_export/tsv',
+ },
+};
+
+$(() => {
+ // generate tab ID
+ while (tabID === null || sessionStorage.getItem(tabID)) {
+ tabID = Math.random();
+ }
+ // hide loading screen
+ const loading = $('.loading');
+ loading.addClass('hide');
+
+ initializeDateTime();
+
+ const endProcs = genProcessDropdownData(procConfigs);
+
+ // add first end process
+ const endProcItem = addEndProcMultiSelect(endProcs.ids, endProcs.names, {
+ showDataType: true,
+ showStrColumn: true,
+ showCatExp: true,
+ isRequired: true,
+ showColor: true,
+ colorAsDropdown: true,
+ hasDiv: true,
+ });
+ endProcItem(() => {
+ onChangeDivInFacet();
+ });
+
+
+ // add first condition process
+ const condProcItem = addCondProc(endProcs.ids, endProcs.names, '', formElements.formID, 'btn-add-cond-proc');
+ condProcItem();
+
+ // click even of condition proc add button
+ $(formElements.btnAddCondProc).click(() => {
+ condProcItem();
+ });
+
+ // click even of end proc add button
+ $('#btn-add-end-proc').click(() => {
+ endProcItem(() => {
+ onChangeDivInFacet();
+ });
+ updateSelectedItems();
+ // checkAndHideStratifiedVar();
+ addAttributeToElement();
+ });
+
+ formElements.divideOption.trigger('change');
+ renderCyclicCalenderModal();
+
+
+ // validation
+ initValidation(formElements.formID);
+
+ onChangeDivideOption();
+
+ initTargetPeriod();
+ toggleDisableAllInputOfNoneDisplayEl($('#for-cyclicTerm'));
+ toggleDisableAllInputOfNoneDisplayEl($('#for-directTerm'));
+ toggleDisableAllInputOfNoneDisplayEl($('#for-category'));
+
+ // add limit after running load_user_setting
+ setTimeout(() => {
+ // add validations for target period
+ validateTargetPeriodInput();
+ keepValueEachDivision();
+ }, 2000);
+
+
+ initializeDateTimeRangePicker();
+ initializeDateTimePicker();
+});
+
+
+const showAgP = (clearOnFlyFilter = true) => {
+ requestStartedAt = performance.now();
+ const isValid = checkValidations({ max: MAX_END_PROC });
+ updateStyleOfInvalidElements();
+ if (isValid) {
+ // close sidebar
+ beforeShowGraphCommon(clearOnFlyFilter);
+
+ beforeShowAGP(clearOnFlyFilter);
+
+ queryDataAndShowAGP(clearOnFlyFilter);
+ }
+};
+
+const beforeShowAGP = (clearOnFlyFilter) => {
+ if (clearOnFlyFilter) {
+ formElements.plotCard.empty();
+ }
+};
+
+const afterShowAGP = () => {
+ formElements.resultSection.css('display', 'block');
+ const agpCard = $(`${formElements.agpCard}`);
+ agpCard.show();
+ loadingHide();
+};
+
+const collectInputAsFormData = (clearOnFlyFilter) => {
+ const traceForm = $(formElements.formID);
+ let formData = new FormData(traceForm[0]);
+
+ if (clearOnFlyFilter) {
+ resetCheckedCats();
+ formData = transformFacetParams(formData);
+ formData = bindCategoryParams(formData);
+ formData = transformColorsParams(formData);
+
+ // choose default or recent datetime
+ formData = genDatetimeRange(formData);
+ const compareType = formData.get('compareType');
+ if (compareType === divideOptions.cyclicCalender) {
+ if (!formData.get(CYCLIC_TERM.DIV_OFFSET)) {
+ const offsetH = Number(divOffset.split(':')[0]) + Number(divOffset.split(':')[1]) / 60;
+ formData.set(CYCLIC_TERM.DIV_OFFSET, offsetH.toString());
+ }
+
+ const divDate = [...divFromTo];
+ divDate.push(lastFrom);
+ formData.set('divDates', JSON.stringify(divDate));
+ formData.set('divFormats', JSON.stringify(divFormats))
+ }
+
+ if (compareType !== divideOptions.cyclicCalender) {
+ formData.delete(CYCLIC_TERM.DIV_OFFSET);
+ formData.delete('divDates');
+ formData.delete('divFormats');
+ }
+
+ // append client timezone
+ formData.set('client_timezone', detectLocalTimezone());
+ lastUsedFormData = formData;
+ } else {
+ formData = lastUsedFormData;
+ formData = transformCatFilterParams(formData);
+ }
+ return formData;
+};
+
+const transformColorsParams = (formData) => {
+ // delete colorVar because there is empty selection in value
+ formData.delete('colorVar');
+ // get colorVar from active GUI
+ const colorVars = {};
+ $('select[name=colorVar]').get().forEach(ele => {
+ const targetID = $(ele).data('target-var-id');
+ const colorVal = $(ele).val();
+ const isObjectiveVar = $(`input[name^=GET02_VALS_SELECT][value=${targetID}]`).prop('checked');
+ if (colorVal && colorVal !== '' && isObjectiveVar) {
+ colorVars[targetID] = colorVal;
+ }
+ });
+
+ formData.append('aggColorVar', JSON.stringify(colorVars));
+ return formData;
+};
+const queryDataAndShowAGP = (clearOnFlyFilter = true) => {
+ const formData = collectInputAsFormData(clearOnFlyFilter);
+
+
+
+ // validate form
+ const hasDiv = !!formData.get('div');
+ const isDivideByCat = formData.get('compareType') === CONST.CATEGORY;
+ if (!hasDiv && isDivideByCat) {
+ // did not select catExpBox as endProcCate
+ loadingHide();
+ showToastrMsg(i18n.selectDivRequiredMessage);
+ return;
+ }
+
+ showGraphCallApi('/ap/api/agp/plot', formData, REQUEST_TIMEOUT, async (res) => {
+ afterShowAGP();
+
+ currentData = res;
+ graphStore.setTraceData(_.cloneDeep(res));
+
+ drawAGP(res);
+
+ // show info table
+ showInfoTable(res);
+
+ loadGraphSetings(clearOnFlyFilter);
+
+ const {catExpBox, cat_on_demand, unique_color} = res;
+ if (clearOnFlyFilter) {
+ clearGlobalDict();
+ initGlobalDict(catExpBox);
+ initGlobalDict(cat_on_demand);
+ initGlobalDict(unique_color);
+ initDicChecked(getDicChecked());
+ initUniquePairList(res.dic_filter);
+ }
+
+
+ longPolling(formData, () => {
+ $(`input[name=${CYCLIC_TERM.DIV_CALENDER}]:checked`).trigger('change');
+ queryDataAndShowAGP(true);
+ });
+ });
+};
+
+const drawAGP = (orgData) => {
+ const data = _.cloneDeep(orgData); // if slow, change
+ if (!data) {
+ return;
+ }
+
+ const isCyclicCalender = orgData.COMMON.compareType === divideOptions.cyclicCalender;
+
+ // orgData.array_plotdata = array_plotdata;
+ renderAgPAllChart(orgData.array_plotdata, isCyclicCalender)
+
+ // implement order
+ $(formElements.agpCard).sortable({});
+
+ formElements.plotCard.empty();
+ formElements.plotCard.show();
+
+ $('html, body').animate({
+ scrollTop: $(formElements.agpCard).offset().top,
+ }, 500);
+
+ // init filter modal
+ fillDataToFilterModal(orgData.catExpBox, [], orgData.cat_on_demand,[], orgData.unique_color || [], () => {
+ queryDataAndShowAGP(false);
+ });
+};
+
+const dumpData = (exportType, dataSrc) => {
+ const formData = lastUsedFormData || collectInputAsFormData(true);
+ formData.set('export_from', dataSrc);
+ if (exportType === EXPORT_TYPE.TSV_CLIPBOARD) {
+ tsvClipBoard(AGP_EXPORT_URL.TSV.url, formData);
+ } else {
+ exportData(
+ AGP_EXPORT_URL[exportType].url,
+ AGP_EXPORT_URL[exportType].ext_name,
+ formData,
+ );
+ }
+};
+
+const handleExportData = (exportType) => {
+ // hide export menu
+ showGraphAndDumpData(exportType, dumpData);
+};
+
+const renderCyclicCalenderModal = () => {
+ const calenderList = $('.cyclic-calender-select-option');
+ calenderList.empty();
+ const renderItem = (key, format, isChecked) => `
+
+
+
+
+
+ 2022040112
+
+ `;
+ let calenderListHtml = ''
+ let index = 1;
+ for (const key of Object.keys(calenderFormat)) {
+ const groups = calenderFormat[key];
+ let groupHtml = '';
+ for (const group of Object.keys(groups)) {
+ let itemHtml = '';
+ const formatList = groups[group];
+ for (const format of formatList) {
+ const isCheck = index === 1;
+ itemHtml += renderItem(key, format, isCheck);
+ index ++;
+ }
+
+ const html = `
+