apps/radar/src/Ts2NetCDF - added ARCHIVE mode with start and end times

codebase/apps/radar/src/Qucid/data_init.cc

@@ -47,6 +47,17 @@ void init_data_links(const char *param_buf, long param_buf_len, long line_no,
     total_len = 0;
     start_ptr = param_buf;
 
+    cerr << "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" << endl;
+    cerr << "fields_n: " << tdrpParams.fields_n << endl;
+    for (int ii = 0; ii < tdrpParams.fields_n; ii++) {
+      Params::field_t &fld = tdrpParams._fields[ii];
+      cerr << "  button label: " << fld.button_label << endl;
+      cerr << "  legend label: " << fld.legend_label << endl;
+      cerr << "  contour_low: " << fld.contour_low << endl;
+    }
+    cerr << "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" << endl;
+
+
     // read all the lines in the data information buffer
     while((end_ptr = strchr(start_ptr,'\n')) != NULL && (total_len < param_buf_len)) {
 	// Skip over blank, short or commented lines
@@ -200,7 +211,6 @@ void init_data_links(const char *param_buf, long param_buf_len, long line_no,
 
     // set the tdrp params for the fields
 
-
     cerr << "aaaaaaaaaaaaaaaaaaaaa num_datafields: " << gd.num_datafields << endl;
     cerr << "bbbbbbbbbbbbbbbbbbbbbb fields_n: " << tdrpParams.fields_n << endl;
 
@@ -231,4 +241,14 @@ void init_data_links(const char *param_buf, long param_buf_len, long line_no,
 
     }
 
+    cerr << "yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy" << endl;
+    for (int ii = 0; ii < tdrpParams.fields_n; ii++) {
+      Params::field_t &fld = tdrpParams._fields[ii];
+      cerr << "  button label: " << fld.button_label << endl;
+      cerr << "  legend label: " << fld.legend_label << endl;
+      cerr << "  contour_low: " << fld.contour_low << endl;
+    }
+    cerr << "yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy" << endl;
+
+
 }

codebase/apps/radar/src/Ts2NetCDF/Args.cc

@@ -133,6 +133,17 @@ int Args::parse(int argc, char **argv, string &prog_name)
 	iret = -1;
       }
 
+    } else if (!strcmp(argv[i], "-input_dir")) {
+
+      if (i < argc - 1) {
+	sprintf(tmp_str, "input_dir = %s;", argv[++i]);
+	TDRP_add_override(&override, tmp_str);
+        sprintf(tmp_str, "input_mode = TS_ARCHIVE_INPUT;");
+        TDRP_add_override(&override, tmp_str);
+      } else {
+	iret = -1;
+      }
+
     } else if (!strcmp(argv[i], "-fmq")) {
 
       if (i < argc - 1) {
@@ -215,6 +226,36 @@ int Args::parse(int argc, char **argv, string &prog_name)
 	iret = -1;
       }
 
+    } else if (!strcmp(argv[i], "-start")) {
+
+      if (i < argc - 1) {
+        char *timeArg = argv[++i];
+	startTime.set(timeArg);
+	if (!startTime.isValid()) {
+	  iret = -1;
+	} else {
+	  sprintf(tmp_str, "input_mode = TS_ARCHIVE_INPUT;");
+	  TDRP_add_override(&override, tmp_str);
+	}
+      } else {
+	iret = -1;
+      }
+
+    } else if (!strcmp(argv[i], "-end")) {
+
+      if (i < argc - 1) {
+        char *timeArg = argv[++i];
+	endTime.set(timeArg);
+	if (!endTime.isValid())	{
+	  iret = -1;
+	} else {
+	  sprintf(tmp_str, "input_mode = TS_ARCHIVE_INPUT;");
+	  TDRP_add_override(&override, tmp_str);
+	}
+      } else {
+	iret = -1;
+      }
+
     } else if (!strcmp(argv[i], "-f")) {
 
       if (i < argc - 1) {
@@ -258,11 +299,14 @@ void Args::_usage(string &prog_name, ostream &out)
       << "options:\n"
       << "       [ --, -h, -help, -man ] produce this list.\n"
       << "       [ -d, -debug ] print debug messages\n"
+      << "       [ -end \"yyyy mm dd hh mm ss\"] sets end time\n"
+      << "          sets input_mode to TS_ARCHIVE_INPUT\n"
       << "       [ -f files ] specify input tsarchive file list.\n"
       << "         Sets input_mode to TS_FILE_INPUT.\n"
       << "       [ -fmq ? ] name of input fmq.\n"
       << "         Sets input_mode to TS_FMQ_INPUT.\n"
       << "       [ -from_start ] read from start of FMQ\n"
+      << "       [ -input_dir ?] sets input_dir for TS_ARCHIVE_INPUT mode\n"
       << "       [ -instance ?] instance for registering with procmap\n"
       << "       [ -max_pulses ? ] limit number of pulses per file)\n"
       << "       [ -n_gates_save ? ] only save out rays\n"
@@ -271,13 +315,15 @@ void Args::_usage(string &prog_name, ostream &out)
       << "       [ -outdir ? ] specify output directory.\n"
       << "       [ -pad_ngates_to_max] option to pad the number of gates out\n"
       << "         to the max number of gates in the file.\n"
-     << "         Not compatible with n_gates_save\n"
+      << "         Not compatible with n_gates_save\n"
       << "       [ -preserve_fname ] use the same file name on output\n"
       << "         A '.nc' extension will be appended to the file name\n"
       << "       [ -rvp8_legacy ] RVP8 data is in legacy packing\n"
       << "       [ -second ] save data from second geom found.\n"
       << "         By default, first geom is saved.\n"
       << "       [ -sectors ? ] save files in sectors of given width (deg)\n"
+      << "       [ -start \"yyyy mm dd hh mm ss\"] sets start time\n"
+      << "          sets input_mode to TS_ARCHIVE_INPUT\n"
       << "       [ -tcp_host ? ] specify host for tcp server\n"
       << "         Sets input_mode to TS_TCP_INPUT.\n"
       << "       [ -tcp_port ? ] specify port for tcp server\n"

codebase/apps/radar/src/Ts2NetCDF/Args.hh

@@ -37,6 +37,7 @@
 #include <vector>
 #include <iostream>
 #include <tdrp/tdrp.h>
+#include <toolsa/DateTime.hh>
 using namespace std;
 
 class Args {
@@ -59,6 +60,7 @@ public:
 
   tdrp_override_t override;
   vector<string> inputFileList;
+  DateTime startTime, endTime;
 
 protected:
 

codebase/apps/radar/src/Ts2NetCDF/Params.cc

@@ -636,10 +636,10 @@
     tt->ptype = ENUM_TYPE;
     tt->param_name = tdrpStrDup("input_mode");
     tt->descr = tdrpStrDup("Input mode - files or time series API.");
-    tt->help = tdrpStrDup("If TS_FMQ_INPUT, the application will read the time series from a file message queue and process the pulses as they come in. If TS_FILE_INPUT, the application will process all the files specified on the command line.");
+    tt->help = tdrpStrDup("TS_FILE_INPUT: the application will process all the files specified on the command line.\n\nTS_ARCHIVE_INPUT: reads data files from input_dir, between start_time and end_time.\n\nTS_FMQ_INPUT: realtime, the application will read the time series from a file message queue and process the pulses as they come in.\n\nTS_TCP_INPUT: realtime, the application will read the time series from a TCP socket as they come in.");
     tt->val_offset = (char *) &input_mode - &_start_;
     tt->enum_def.name = tdrpStrDup("input_mode_t");
-    tt->enum_def.nfields = 3;
+    tt->enum_def.nfields = 4;
     tt->enum_def.fields = (enum_field_t *)
         tdrpMalloc(tt->enum_def.nfields * sizeof(enum_field_t));
       tt->enum_def.fields[0].name = tdrpStrDup("TS_FILE_INPUT");
@@ -648,9 +648,23 @@
       tt->enum_def.fields[1].val = TS_FMQ_INPUT;
       tt->enum_def.fields[2].name = tdrpStrDup("TS_TCP_INPUT");
       tt->enum_def.fields[2].val = TS_TCP_INPUT;
+      tt->enum_def.fields[3].name = tdrpStrDup("TS_ARCHIVE_INPUT");
+      tt->enum_def.fields[3].val = TS_ARCHIVE_INPUT;
     tt->single_val.e = TS_FILE_INPUT;
     tt++;
 
+    // Parameter 'input_dir'
+    // ctype is 'char*'
+
+    memset(tt, 0, sizeof(TDRPtable));
+    tt->ptype = STRING_TYPE;
+    tt->param_name = tdrpStrDup("input_dir");
+    tt->descr = tdrpStrDup("Dir for input data files.");
+    tt->help = tdrpStrDup("This is used in REALTIME and ARCHIVE modes. In FILELIST and SIMULATE modes, the file paths are specified on the command line.");
+    tt->val_offset = (char *) &input_dir - &_start_;
+    tt->single_val.s = tdrpStrDup("./input");
+    tt++;
+
     // Parameter 'input_fmq_name'
     // ctype is 'char*'
 

codebase/apps/radar/src/Ts2NetCDF/Params.hh

@@ -77,7 +77,8 @@ public:
   typedef enum {
     TS_FILE_INPUT = 0,
     TS_FMQ_INPUT = 1,
-    TS_TCP_INPUT = 2
+    TS_TCP_INPUT = 2,
+    TS_ARCHIVE_INPUT = 3
   } input_mode_t;
 
   ///////////////////////////
@@ -382,6 +383,8 @@ public:
 
   input_mode_t input_mode;
 
+  char* input_dir;
+
   char* input_fmq_name;
 
   tdrp_bool_t seek_to_end_of_input;
@@ -431,7 +434,7 @@ private:
 
   void _init();
 
-  mutable TDRPtable _table[32];
+  mutable TDRPtable _table[33];
 
   const char *_className;
 

codebase/apps/radar/src/Ts2NetCDF/Ts2NetCDF.cc

@@ -139,6 +139,11 @@ Ts2NetCDF::Ts2NetCDF(int argc, char **argv)
     _pulseReader = new IwrfTsReaderTcp(_params.tcp_server_host,
                                        _params.tcp_server_port,
 				       iwrfDebug);
+  } else if (_params.input_mode == Params::TS_ARCHIVE_INPUT) {
+    _pulseReader = new IwrfTsReaderFile(_params.input_dir,
+                                        _args.startTime.utime(),
+                                        _args.endTime.utime(),
+                                        iwrfDebug);
   } else {
     _pulseReader = new IwrfTsReaderFile(_args.inputFileList, iwrfDebug);
   }

codebase/apps/radar/src/Ts2NetCDF/Ts2NetCDF.test

@@ -64,21 +64,42 @@ instance = "test";
 //
 // Input mode - files or time series API.
 //
-// If TS_FMQ_INPUT, the application will read the time series from a 
-//   file message queue and process the pulses as they come in. If 
-//   TS_FILE_INPUT, the application will process all the files specified 
+// TS_FILE_INPUT: the application will process all the files specified 
 //   on the command line.
 //
+// TS_ARCHIVE_INPUT: reads data files from input_dir, between start_time 
+//   and end_time.
+//
+// TS_FMQ_INPUT: realtime, the application will read the time series 
+//   from a file message queue and process the pulses as they come in.
+//
+// TS_TCP_INPUT: realtime, the application will read the time series 
+//   from a TCP socket as they come in.
+//
 //
 // Type: enum
 // Options:
 //     TS_FILE_INPUT
 //     TS_FMQ_INPUT
 //     TS_TCP_INPUT
+//     TS_ARCHIVE_INPUT
 //
 
 input_mode = TS_FILE_INPUT;
 
+///////////// input_dir ///////////////////////////////
+//
+// Dir for input data files.
+//
+// This is used in REALTIME and ARCHIVE modes. In FILELIST and SIMULATE 
+//   modes, the file paths are specified on the command line.
+//
+//
+// Type: string
+//
+
+input_dir = "/scr/sleet3/rsfdata/projects/spicule/hcr/time_series/wband/save";
+
 ///////////// input_fmq_name //////////////////////////
 //
 // FMQ name. For TS_FMQ_INPUT only.
@@ -216,7 +237,7 @@ chan0_is_h_or_copolar = TRUE;
 // Type: string
 //
 
-output_dir = "$(HOME)/data/time_series/nc/test";
+output_dir = "/tmp/nc";
 
 ///////////// preserve_file_name //////////////////////
 //

codebase/apps/radar/src/Ts2NetCDF/paramdef.Ts2NetCDF

@@ -41,15 +41,21 @@ commentdef {
 };
 
 typedef enum {
-  TS_FILE_INPUT, TS_FMQ_INPUT, TS_TCP_INPUT
+  TS_FILE_INPUT, TS_FMQ_INPUT, TS_TCP_INPUT, TS_ARCHIVE_INPUT
 } input_mode_t;
 
 paramdef enum input_mode_t {
   p_default = TS_FILE_INPUT;
   p_descr = "Input mode - files or time series API.";
-  p_help = "If TS_FMQ_INPUT, the application will read the time series from a file message queue and process the pulses as they come in. If TS_FILE_INPUT, the application will process all the files specified on the command line.";
+  p_help = "TS_FILE_INPUT: the application will process all the files specified on the command line.\n\nTS_ARCHIVE_INPUT: reads data files from input_dir, between start_time and end_time.\n\nTS_FMQ_INPUT: realtime, the application will read the time series from a file message queue and process the pulses as they come in.\n\nTS_TCP_INPUT: realtime, the application will read the time series from a TCP socket as they come in.";
 } input_mode;
 
+paramdef string {
+  p_default = "./input";
+  p_descr = "Dir for input data files.";
+  p_help = "This is used in REALTIME and ARCHIVE modes. In FILELIST and SIMULATE modes, the file paths are specified on the command line.";
+} input_dir;
+
 paramdef string {
   p_default = "/tmp/fmq/ts";
   p_descr = "FMQ name. For TS_FMQ_INPUT only.";

codebase/libs/radar/src/include/radar/IwrfTsReader.hh

@@ -249,7 +249,15 @@ public:
                    bool use_ldata_info = TRUE,
                    IwrfDebug_t debug = IWRF_DEBUG_OFF);
 
-  // ARCHIVE mode - specify list of files to be read
+  // ARCHIVE mode, read files from input_dir
+  // between start time and end time.
+
+  IwrfTsReaderFile(const char *input_dir,
+                   time_t start_time,
+                   time_t end_time,
+                   IwrfDebug_t debug = IWRF_DEBUG_OFF);
+
+  // FILELIST mode - specify list of files to be read
 
   IwrfTsReaderFile(const vector<string> &fileList,
                    IwrfDebug_t debug = IWRF_DEBUG_OFF);

codebase/libs/radar/src/iwrf/IwrfTsReader.cc

@@ -252,7 +252,7 @@ IwrfTsReaderFile::IwrfTsReaderFile(const char *input_dir,
 
 }
 
-// ARCHIVE mode - specify list of files to be read
+// FILELIST mode - specify list of files to be read
 
 IwrfTsReaderFile::IwrfTsReaderFile(const vector<string> &fileList,
 				   IwrfDebug_t debug) :
@@ -274,6 +274,29 @@ IwrfTsReaderFile::IwrfTsReaderFile(const vector<string> &fileList,
 
 }
 
+// ARCHIVE mode - specify input_dir and start and end times
+
+IwrfTsReaderFile::IwrfTsReaderFile(const char *input_dir,
+                                   time_t start_time,
+                                   time_t end_time,
+				   IwrfDebug_t debug) :
+        IwrfTsReader(debug)
+
+{
+
+  _input = new DsInputPath("IwrfTsReaderFile", debug, input_dir, start_time, end_time);
+  _fileList = _input->getPathList();
+  _in = NULL;
+  _fileIsRvp8Type = false;
+  if (_debug) {
+    cerr << "INFO - IwrfTsReaderFile" << endl;
+    for (size_t ii = 0; ii < _fileList.size(); ii++) {
+      cerr << "  using file: " << _fileList[ii] << endl;
+    }
+  }
+
+}
+
 //////////////////////////////////////////////////////////////////
 // destructor