Merge branch 'devel' into 'master'

PIRA v0.3.4

See merge request tuda-sc/projects/pira!66

.gitlab-ci.yml

@@ -16,9 +16,10 @@ stages:
 
 # Setting up the environment on the Lichtenberg system
 .lb-setup: &lb-setup
+  - export LMOD_IGNORE_CACHE=1
   - eval $(ssh-agent -s)
   - ssh-add ~/.ssh/gitlab_ed25519
-  - module load gcc/8 git llvm/10.0.0 cmake python/3 openmpi
+  - module load gcc/8 git python/3.7.4 cmake llvm/10 openmpi
 
 .lb-tear-down: &lb-tear-down
   - rm -rf $(dirname $(echo $SSH_AUTH_SOCK))

lib/Analyzer.py

@@ -84,7 +84,7 @@ def analyze_local(self, target_config: TargetConfig, kwargs: dict, iterationNumb
                                    benchmark_name)
           instr_files = U.build_instr_file_path(analyzer_dir, flavor, benchmark_name)
           L.get_logger().log('Analyzer::analyzer_local: instrumentation file = ' + instr_files)
-          prev_instr_file = U.build_previous_instr_file_path(analyzer_dir, flavor, benchmark_name)
+          prev_instr_file = U.build_previous_instr_file_path(analyzer_dir, flavor, benchmark_name, iterationNumber)
 
         tracker = T.TimeTracker()
 
@@ -131,6 +131,7 @@ def run_analyzer_command(command: str, analyzer_dir: str, flavor: str, benchmark
 
     export_performance_models = InvocCfg.get_instance().is_export()
     export_runtime_only = InvocCfg.get_instance().is_export_runtime_only()
+    use_cs_instrumentation = InvocCfg.get_instance().use_cs_instrumentation()
     export_str = ' '
     if export_performance_models:
       export_str += ' --export'
@@ -147,7 +148,7 @@ def run_analyzer_command(command: str, analyzer_dir: str, flavor: str, benchmark
         # load imbalance detection mode
         load_imbalance_detection_cfg_path = InvocCfg.get_instance().get_load_imbalance_detection_cfg_path()
         L.get_logger().log('Utility::run_analyzer_command: using Load Imbalance Detection Analyzer', level='info')
-        sh_cmd = command + export_str + ' --scorep-out -c ' + cubex_file + ' --load-imbalance ' + load_imbalance_detection_cfg_path + ' --export ' + ipcg_file 
+        sh_cmd = command + export_str + ' --scorep-out -c ' + cubex_file + ' --load-imbalance ' + load_imbalance_detection_cfg_path + ' --debug 1 --export ' + ipcg_file 
 
       else:
         # vanilla PIRA version 1 runner
@@ -162,6 +163,9 @@ def run_analyzer_command(command: str, analyzer_dir: str, flavor: str, benchmark
     if hybrid_filter and not was_rebuilt:
       command += ' --model-filter'
 
+    if use_cs_instrumentation:
+      command += ' --use-cs-instrumentation'
+
     sh_cmd = command + export_str + ' --scorep-out --extrap ' + pgis_cfg_file + ' ' + ipcg_file
     L.get_logger().log('Utility::run_analyzer_command: INSTR: Run cmd: ' + sh_cmd)
     out, _ = U.shell(sh_cmd)
@@ -174,7 +178,7 @@ def run_analyzer_command_no_instr(command: str, analyzer_dir: str, flavor: str,
 
     # load imbalancee detection mode
     if InvocCfg.get_instance().is_load_imbalance_detection_enabled():
-      sh_cmd = sh_cmd + ' --load-imbalance ' + InvocCfg.get_instance().get_load_imbalance_detection_cfg_path()
+      sh_cmd = sh_cmd + ' --debug 1 --load-imbalance ' + InvocCfg.get_instance().get_load_imbalance_detection_cfg_path()
 
     sh_cmd = sh_cmd + ' ' + ipcg_file
 

lib/Configuration.py

@@ -499,6 +499,7 @@ def __init__(self, cmdline_args: Namespace):
       self._hybrid_filter_iters = cmdline_args.hybrid_filter_iters
       self._export = cmdline_args.export
       self._export_runtime_only = cmdline_args.export_runtime_only
+      self._use_call_site_instrumentation = cmdline_args.call_site_instrumentation
 
       # check whether load imbalance detection is enabled
       if cmdline_args.load_imbalance_detection == '':
@@ -521,7 +522,7 @@ def reset_to_default() -> None:
       cmdline_args = Namespace(pira_dir=U.get_default_pira_dir(), config_version=2,
                                config=U.get_default_config_file(), runtime_filter=False,
                                hybrid_filter_iters=0, iterations=4, repetitions=5, export=False,
-                               export_runtime_only=False, load_imbalance_detection='')
+                               export_runtime_only=False, load_imbalance_detection='', call_site_instrumentation=False)
       InvocationConfig(cmdline_args)
 
     else:
@@ -537,6 +538,7 @@ def reset_to_default() -> None:
       instance._export = False
       instance._export_runtime_only = False
       instance._load_imbalance_detection_cfg_path = None
+      instance._use_call_site_instrumentation = False
 
   @staticmethod
   def create_from_kwargs(args: dict) -> None:
@@ -576,6 +578,9 @@ def create_from_kwargs(args: dict) -> None:
 
     if args.get('export_runtime_only') != None:
       instance._export_runtime_only = args['export_runtime_only']
+
+    if args.get('use_cs_instrumentation') != None:
+      instance._use_call_site_instrumentation = args['use_cs_instrumentation']
 
 
   def get_pira_dir(self) -> str:
@@ -614,6 +619,9 @@ def is_load_imbalance_detection_enabled(self) -> bool:
   def get_load_imbalance_detection_cfg_path(self) -> str:
     return self._load_imbalance_detection_cfg_path
 
+  def use_cs_instrumentation(self) -> bool:
+    return self._use_call_site_instrumentation
+
 
 class CSVConfig:
 

lib/Exporter.py

@@ -84,3 +84,62 @@ def export(self, file_name: str, dialect='unix'):
 
       # write rows
       writer.writerows(self.rows)
+
+
+
+class PiraRuntimeExporter:
+
+  class MetaInformationProvider:
+
+    def __init__(self, str_for_average: str, str_for_median: str, str_for_stdev: str):
+      self._average = str_for_average
+      self._median = str_for_median
+      self._stdev = str_for_stdev
+
+    def get_average(self, unused_void, unused_void_2):
+      return self._average
+
+    def get_median(self, unused_void,unused_void_2):
+      return self._median
+
+    def get_stdev(self,unused_void, unused_void_2):
+      return self._stdev
+
+    def get_num_data_sets(self):
+      return 1
+
+  def __init__(self):
+    self._iteration_data = [('Data', PiraRuntimeExporter.MetaInformationProvider('Average', 'Median', 'Stdev'))]
+
+  def add_iteration_data(self, name: str, rt_info) -> None:
+    self._iteration_data.append( (name, rt_info) )
+
+  def export(self, file_name: str, dialect='unix'):
+    with open(file_name, 'w', newline='') as csvfile:
+
+      writer = csv.writer(csvfile, dialect)
+
+      writer_data = []
+      for el in self._iteration_data:
+        for nd in range(0,el[1].get_num_data_sets()):
+          writer_data.append(el[0])
+      writer.writerow(writer_data)
+
+      writer_data = []
+      for el in self._iteration_data:
+        for nd in range(0,el[1].get_num_data_sets()):
+          writer_data.append(el[1].get_average(0,nd))
+      writer.writerow(writer_data)
+
+      writer_data = []
+      for el in self._iteration_data:
+        for nd in range(0,el[1].get_num_data_sets()):
+          writer_data.append(el[1].get_median(0,nd))
+      writer.writerow(writer_data)
+
+      writer_data = []
+      for el in self._iteration_data:
+        for nd in range(0,el[1].get_num_data_sets()):
+          writer_data.append(el[1].get_stdev(0,nd))
+      writer.writerow(writer_data)
+

lib/Measurement.py

@@ -13,6 +13,7 @@
 import typing
 import os
 import re
+import statistics as stat
 
 class MeasurementSystemException(PiraException):
   """  This exception is thrown if problems in the runtime occur.  """
@@ -21,6 +22,86 @@ def __init__(self, message):
     super().__init__(message)
 
 
+class RunResultSeries:
+
+  # rt is runtime, reps is the number of repetitions for one input data set, and num_data_sets is the number of different input data sets.
+  # One RunResultSeries object per iteration / phase
+  def __init__(self, rt: float = None, reps: int = None, num_data_sets: int = 1):
+      self.rt_values = []
+      self.reps = reps
+      self.num_data_sets = num_data_sets
+
+  def is_multi_value(self):
+    return False
+
+  def add_values(self, rt: float, reps: int) ->None:
+    assert(reps == self.reps)
+    self.rt_values.append(rt)
+
+  def add_from(self, other) -> None:
+    assert(other.reps == self.reps)
+    for rt in other.rt_values:
+      self.rt_values.append(rt)
+
+  def get_num_data_sets(self) -> int:
+    return self.num_data_sets
+
+  def get_average(self, pos: int = 0, data_set: int = 0) ->float:
+    # 1 data set and three repetitions
+    # [ 1 1 1 ]
+    # 2 data sets and three repetitions
+    # [ 1 1 1, 2 2 2 ]
+    if data_set > self.num_data_sets:
+      raise RuntimeError('Trying to access out-of-bounds data set')
+    start_idx = pos + data_set * self.reps
+    end_idx = start_idx + self.reps
+    L.get_logger().log('Computing mean for values: ' + str(self.rt_values[start_idx:end_idx]) + ' [pos: ' + str(pos) + ' | data_set: ' + str(data_set) + ' => start_idx: ' + str(start_idx) + ' <> ' + str(end_idx))
+    return stat.mean(self.rt_values[start_idx:end_idx])
+
+  def get_median(self, pos: int = 0, data_set: int = 0) -> float:
+    start_idx = pos + data_set * self.reps
+    end_idx = start_idx + self.reps
+    L.get_logger().log('Computing median for values: ' + str(self.rt_values[start_idx:end_idx]))
+    return stat.median(self.rt_values[start_idx:end_idx])
+
+  def get_stdev(self, pos: int = 0, data_set: int = 0) -> float:
+    start_idx = pos + data_set * self.reps
+    end_idx = start_idx + self.reps
+    L.get_logger().log('Computing stdev for values: ' + str(self.rt_values[start_idx:end_idx]))
+    return stat.stdev(self.rt_values[start_idx:end_idx])
+
+  def compute_overhead(self, base_line, pos: int = 0, data_set: int = 0) -> float:
+    L.get_logger().log('Computing overhead in RunResultSeries')
+    base_median = base_line.get_median()
+    if base_median == .0:
+      L.get_logger().log('Detected 0 seconds baseline. Setting baseline median to 1.0', level='warn')
+      base_median = 1.0
+    return (self.get_median(pos) / base_median) - 1
+
+  def get_all_averages(self) -> typing.List[float]:
+    if len(self.rt_values) % reps != 0:
+      raise RuntimeError('number of runtime values must be cleanly divisable by num reps.')
+    num_averages = len(self.rt_values) / reps
+
+    intermediate_averages = []
+    for i in range(0, len(self.rt_values), self.reps):
+      intermediate_averages.append(self.get_average(i))
+
+    return intermediate_averages
+
+  def compute_all_overheads(self) -> typing.List[float]:
+    assert(False)
+
+  def get_accumulated_runtime(self):
+    accu_rt = .0
+    for rt in self.rt_values:
+      accu_rt += rt
+    return accu_rt
+
+  def get_nr_of_repetitions(self):
+    return self.reps
+
+
 class RunResult:
   """  Holds the result of a measurement execution with potentially multiple iterations.  """
 
@@ -306,9 +387,18 @@ def prepare_MPI_filtering(cls, filter_file: str) -> None:
 
     MPI_functions_to_filter = []
     file_content = U.read_file(filter_file).split('\n')
-    # We always want to measure MPI_Init and MPI_Finalize
-    file_content.append('MPI_Init')
-    file_content.append('MPI_Finalize')
+    # We always want to measure some functions to ensure Score-P works correctly
+    always_measure = [
+      'MPI_Init',
+      'MPI_Finalize',
+      'MPI_Comm_group',
+      'MPI_Comm_dup',
+      'MPI_Comm_create_group',
+      'MPI_Comm_split',
+      'MPI_Comm_free',
+      'MPI_Group_free'
+    ]
+    file_content.extend(always_measure)
     for l in file_content:
       # Match MPI functions which have been marked for instrumentation
       # Example: (MPI_Barrier is representative for all MPI functions here)

lib/Pira.py

@@ -32,7 +32,8 @@ def execute_with_config(runner: Runner, analyzer: A, target_config: TargetConfig
     instrument = False
     was_rebuilt = True
 
-    rr_exporter = E.RunResultExporter()
+    #rr_exporter = E.RunResultExporter()
+    rr_exporter = E.PiraRuntimeExporter()
 
     # Build without any instrumentation
     L.get_logger().log('Building vanilla version for baseline measurements', level='info')
@@ -49,7 +50,7 @@ def execute_with_config(runner: Runner, analyzer: A, target_config: TargetConfig
     instr_file = ''
 
     if (csv_config.should_export()):
-      rr_exporter.add_row('Vanilla', vanilla_rr)
+      rr_exporter.add_iteration_data('Vanilla', vanilla_rr)
 
     for iteration in range(0,InvocationConfig.get_instance().get_pira_iters()):
       L.get_logger().log('Running instrumentation iteration ' + str(iteration), level='info')
@@ -76,19 +77,20 @@ def execute_with_config(runner: Runner, analyzer: A, target_config: TargetConfig
       L.get_logger().log('Running profiling measurements', level='info')
       instr_rr = runner.do_profile_run(target_config, iteration)
       if(csv_config.should_export()):
-        rr_exporter.add_row('Instrumented ' + str(iteration), instr_rr)
+        rr_exporter.add_iteration_data('Instrumented ' + str(iteration), instr_rr)
 
       # Compute overhead of instrumentation
       ovh_percentage = instr_rr.compute_overhead(vanilla_rr)
       L.get_logger().log('[RUNTIME] $' + str(iteration) + '$ ' + str(instr_rr.get_average()), level='perf')
       L.get_logger().log('[OVERHEAD] $' + str(iteration) + '$ ' + str(ovh_percentage), level='perf')
+      L.get_logger().log('[REPETITION SUM] $' + str(iteration) + '$ ' + str(instr_rr.get_accumulated_runtime()), level='perf')
 
       iteration_tracker.stop()
       user_time, system_time = iteration_tracker.get_time()
       L.get_logger().log('[ITERTIME] $' + str(iteration) + '$ ' + str(user_time) + ', ' + str(system_time), level='perf')
 
     if(csv_config.should_export()):
-      file_name = target_config.get_target() + '_' +  target_config.get_flavor() + '.csv'
+      file_name = target_config.get_target() + '-' +  target_config.get_flavor() + '.csv'
       csv_file = os.path.join(csv_config.get_csv_dir(), file_name)
       try:
         U.make_dir(csv_config.get_csv_dir())
@@ -195,6 +197,7 @@ def main(cmdline_args) -> None:
       # A build/place is a top-level directory
       for build in configuration.get_builds():
         L.get_logger().log('Build: ' + str(build))
+        total_time = T.TimeTracker()
         app_tuple = (U.generate_random_string(), build, '', '')
         dbm.insert_data_application(app_tuple)
 
@@ -221,6 +224,10 @@ def main(cmdline_args) -> None:
             L.get_logger().log('In this version of PIRA it is not yet implemented', level='error')
             assert (False)
 
+        total_time.stop()
+        L.get_logger().log('PIRA total runtime: {}'.format(total_time.get_time()), level='perf')
+
+
     U.change_cwd(home_dir)
 
   except RuntimeError as rt_err: