Merge branch 'feature/egg_metadata' into develop

Source/Data/CMakeLists.txt

@@ -36,6 +36,7 @@ set (DATA_HEADERFILES
     EventAnalysis/KTSparseWaterfallCandidateData.hh
     EventAnalysis/KTSpectrumCollectionData.hh
     EventAnalysis/KTWaterfallCandidateData.hh
+    Time/KTArbitraryMetadata.hh
     Time/KTDigitizerTestData.hh
     Time/KTEggHeader.hh
     Time/KTProcSummary.hh
@@ -101,6 +102,7 @@ set (DATA_SOURCEFILES
     EventAnalysis/KTSparseWaterfallCandidateData.cc
     EventAnalysis/KTSpectrumCollectionData.cc
     EventAnalysis/KTWaterfallCandidateData.cc
+    Time/KTArbitraryMetadata.cc
     Time/KTDigitizerTestData.cc
     Time/KTEggHeader.cc
     Time/KTProcSummary.cc

Source/Data/Time/KTArbitraryMetadata.cc

@@ -0,0 +1,25 @@
+/*
+ * KTArbitraryMetadata.cc
+ *
+ *  Created on: Sept 20, 2021
+ *      Author: N.S. Oblath
+ */
+
+#include "KTArbitraryMetadata.hh"
+
+namespace Katydid
+{
+    const std::string KTArbitraryMetadata::sName("arbitrary-metadata");
+
+    KTArbitraryMetadata::KTArbitraryMetadata() :
+            KTExtensibleData< KTArbitraryMetadata >(),
+            fMetadata(nullptr)
+    {
+    }
+
+    KTArbitraryMetadata::~KTArbitraryMetadata()
+    {
+        delete fMetadata;
+    }
+
+} /* namespace Katydid */

Source/Data/Time/KTArbitraryMetadata.hh

@@ -0,0 +1,40 @@
+/*
+ * KTArbitraryMetadata.hh
+ *
+ *  Created on: Sept 20, 2021
+ *      Author: N.S. Oblath
+ */
+
+#ifndef KTARBITRARY_METADATA_HH_
+#define KTARBITRARY_METADATA_HH_
+
+#include "KTData.hh"
+
+#include "param.hh"
+
+
+namespace Katydid
+{
+    class KTArbitraryMetadata : public Nymph::KTExtensibleData< KTArbitraryMetadata >
+    {
+        public:
+            KTArbitraryMetadata();
+            virtual ~KTArbitraryMetadata();
+
+            void SetMetadata( scarab::param* metadata );
+            MEMBERVARIABLE_NOSET(scarab::param*, Metadata);
+
+        public:
+            static const std::string sName;
+
+    };
+
+    inline void KTArbitraryMetadata::SetMetadata( scarab::param* metadata )
+    {
+        delete fMetadata;
+        fMetadata = metadata;
+        return;
+    }
+
+} /* namespace Katydid */
+#endif /* KTARBITRARY_METADATA_HH_ */

Source/Data/Time/KTEggHeader.cc

@@ -85,6 +85,7 @@ namespace Katydid
     KTEggHeader::KTEggHeader() :
             KTExtensibleData< KTEggHeader >(),
             fFilename(),
+            fMetadataFilename(),
             fAcquisitionMode(1),
             fRunDuration(0),
             fAcquisitionRate(0),
@@ -99,6 +100,7 @@ namespace Katydid
 
     KTEggHeader::KTEggHeader(const KTEggHeader& orig) :
             fFilename(orig.fFilename),
+            fMetadataFilename(orig.fMetadataFilename),
             fAcquisitionMode(orig.fAcquisitionMode),
             fRunDuration(orig.fRunDuration),
             fAcquisitionRate(orig.fAcquisitionRate),
@@ -122,6 +124,7 @@ namespace Katydid
     KTEggHeader& KTEggHeader::operator=(const KTEggHeader& rhs)
     {
         fFilename = rhs.fFilename;
+        fMetadataFilename = rhs.fMetadataFilename;
         fAcquisitionMode = rhs.fAcquisitionMode;
         fRunDuration = rhs.fRunDuration;
         fAcquisitionRate = rhs.fAcquisitionRate;
@@ -182,6 +185,7 @@ namespace Katydid
     {
         out << "File header:\n"
                 << "\tFilename: " << header.GetFilename() << '\n'
+                << "\tMetadata Filename: " << header.GetMetadataFilename() << '\n'
                 << "\tAcquisition Mode: " << header.GetAcquisitionMode() << '\n'
                 << "\tNumber of Channels: " << header.GetNChannels() << '\n'
                 << "\tRun Duration: " << header.GetRunDuration() << " ms\n"

Source/Data/Time/KTEggHeader.hh

@@ -64,6 +64,7 @@ namespace Katydid
             KTEggHeader& operator=(const KTEggHeader& rhs);
 
             MEMBERVARIABLEREF(std::string, Filename);
+            MEMBERVARIABLEREF(std::string, MetadataFilename);
             MEMBERVARIABLE(unsigned, AcquisitionMode);
             MEMBERVARIABLE(unsigned, RunDuration);  /// in ms
             MEMBERVARIABLE(double, AcquisitionRate); /// in Hz

Source/IO/TerminalWriter/KTTerminalTypeWriterTime.cc

@@ -7,6 +7,7 @@
 
 #include "KTTerminalTypeWriterTime.hh"
 
+#include "KTArbitraryMetadata.hh"
 #include "KTDigitizerTestData.hh"
 #include "KTEggHeader.hh"
 #include "KTTIFactory.hh"
@@ -43,6 +44,7 @@ namespace Katydid
     void KTTerminalTypeWriterTime::RegisterSlots()
     {
         fWriter->RegisterSlot("header", this, &KTTerminalTypeWriterTime::WriteEggHeader);
+        fWriter->RegisterSlot("metadata", this, &KTTerminalTypeWriterTime::WriteArbitraryMetadata);
         fWriter->RegisterSlot("ts", this, &KTTerminalTypeWriterTime::WriteTimeSeriesData);
         fWriter->RegisterSlot("dig-test", this, &KTTerminalTypeWriterTime::WriteDigitizerTestData);
         fWriter->RegisterSlot("summary", this, &KTTerminalTypeWriterTime::WriteProcSummary);
@@ -64,6 +66,22 @@ namespace Katydid
     }
 
 
+    void KTTerminalTypeWriterTime::WriteArbitraryMetadata(Nymph::KTDataPtr data)
+    {
+        if (! data) return;
+
+        scarab::param* metadata = data->Of< KTArbitraryMetadata >().GetMetadata();
+        if (metadata == nullptr)
+        {
+            KTPROG(termlog, "Metadata is empty");
+            return;
+        }
+
+        KTPROG(termlog, "Arbitrary Metadata:\n" << *metadata);
+        return;
+    }
+
+
     //*****************
     // Time Series Data
     //*****************

Source/IO/TerminalWriter/KTTerminalTypeWriterTime.hh

@@ -27,6 +27,7 @@ namespace Katydid
 
         public:
             void WriteEggHeader(Nymph::KTDataPtr data);
+            void WriteArbitraryMetadata(Nymph::KTDataPtr data);
 
             void WriteTimeSeriesData(Nymph::KTDataPtr data);
             void WriteDigitizerTestData(Nymph::KTDataPtr data);

Source/Time/KTEgg1Reader.cc

@@ -67,12 +67,12 @@ namespace Katydid
         {
             KTWARN(eggreadlog, "Egg1 reader is only setup to handle a single file; multiple files have been specified and all but the first one will be skipped");
         }
-        KTINFO(eggreadlog, "Opening egg file <" << filenames[0] << ">")
-        fEggStream.open(filenames[0].c_str(), ifstream::in|ifstream::binary);
+        KTINFO(eggreadlog, "Opening egg file <" << filenames[0].first << ">")
+        fEggStream.open(filenames[0].first.c_str(), ifstream::in|ifstream::binary);
 
         if (! fEggStream.is_open())
         {
-            KTERROR(eggreadlog, "Egg filestream did not open (file: " << filenames[0] << ")");
+            KTERROR(eggreadlog, "Egg filestream did not open (file: " << filenames[0].first << ")");
             return Nymph::KTDataPtr();
         }
 
@@ -223,7 +223,8 @@ namespace Katydid
 
         Nymph::KTDataPtr eggHeaderPtr(new Nymph::KTData());
         KTEggHeader& eggHeader = eggHeaderPtr->Of< KTEggHeader >();
-        eggHeader.SetFilename(filenames[0].native());
+        eggHeader.SetFilename(filenames[0].first.native());
+        eggHeader.SetMetadataFilename(filenames[0].second.native());
         eggHeader.SetAcquisitionMode(1);
         eggHeader.SetRunDuration(fHeaderInfo.fRunLength * 1000); // conversion from s to ms
         eggHeader.SetAcquisitionRate(fHeaderInfo.fSampleRate * fHeaderInfo.fHertzPerSampleRateUnit);

Source/Time/KTEgg2Reader.cc

@@ -103,10 +103,10 @@ namespace Katydid
         {
             KTWARN(eggreadlog, "Egg2 reader is only setup to handle a single file; multiple files have been specified and all but the first one will be skipped");
         }
-        KTINFO(eggreadlog, "Opening egg file <" << filenames[0] << ">");
+        KTINFO(eggreadlog, "Opening egg file <" << filenames[0].first << ">");
         try
         {
-            fMonarch = Monarch2::OpenForReading(filenames[0].native());
+            fMonarch = Monarch2::OpenForReading(filenames[0].first.native());
         }
         catch (M2Exception& e)
         {
@@ -130,6 +130,7 @@ namespace Katydid
             return Nymph::KTDataPtr();
         }
         CopyHeaderInformation(fMonarch->GetHeader());
+        fHeader.SetMetadataFilename(filenames[0].second.native())
 
         KTDEBUG(eggreadlog, "Parsed header:\n" << fHeader);
 

Source/Time/KTEgg3Reader.cc

@@ -7,6 +7,7 @@
 
 #include "KTEgg3Reader.hh"
 
+#include "KTArbitraryMetadata.hh"
 #include "KTEggHeader.hh"
 #include "KTEggProcessor.hh"
 #include "KTLogger.hh"
@@ -20,6 +21,8 @@
 #include "M3DataInterface.hh"
 #include "M3Exception.hh"
 
+#include "param.hh"
+#include "param_codec.hh"
 #include "scarab_version.hh"
 
 using namespace monarch3;
@@ -40,6 +43,7 @@ namespace Katydid
             fStride(0),
             fStartTime(0.),
             fStartRecord(0),
+            fRequireMetadata(false),
             //fHatchNextSlicePtr(NULL),
             fFilenames(),
             fCurrentFileIt(),
@@ -50,12 +54,12 @@ namespace Katydid
             fHeader(fHeaderPtr->Of< KTEggHeader >()),
             fMasterSliceHeader(),
             fReadState(),
-            fGetTimeInRun(&KTEgg3Reader::GetTimeInRunFromMonarch),
-            fT0Offset(0),
-            fAcqTimeInRun(0),
             fSampleRateUnitsInHz(1.e6),
             fRecordSize(0),
             fBinWidth(0.),
+            fGetTimeInRun(&KTEgg3Reader::GetTimeInRunFromMonarch),
+            fT0Offset(0),
+            fAcqTimeInRun(0),
             fSliceNumber(0),
             fRecordsProcessed(0)
     {
@@ -80,6 +84,7 @@ namespace Katydid
         SetStride(eggProc.GetStride());
         SetStartTime(eggProc.GetStartTime());
         SetStartRecord(eggProc.GetStartRecord());
+        SetRequireMetadata(eggProc.GetRequireMetadata());
         return true;
     }
 
@@ -97,10 +102,10 @@ namespace Katydid
         fCurrentFileIt = fFilenames.begin();
 
         // open the file
-        KTINFO(eggreadlog, "Opening egg file <" << fFilenames[0] << ">");
+        KTINFO(eggreadlog, "Opening egg file <" << fFilenames[0].first << ">");
         try
         {
-            fMonarch = Monarch3::OpenForReading(fFilenames[0].native());
+            fMonarch = Monarch3::OpenForReading(fFilenames[0].first.native());
         }
         catch (M3Exception& e)
         {
@@ -151,6 +156,15 @@ namespace Katydid
         fM3StreamHeader = &(fMonarch->GetHeader()->GetStreamHeaders()[streamNum]);
 
         CopyHeader(fMonarch->GetHeader());
+        fHeader.SetMetadataFilename(fFilenames[0].second.native());
+        AddMetadata();
+        if (fRequireMetadata && ! MetadataIsPresent())
+        {
+            KTERROR(eggreadlog, "Metadata is required but not present");
+            delete fMonarch;
+            fMonarch = nullptr;
+            return Nymph::KTDataPtr();
+        }
 
         KTDEBUG(eggreadlog, "Parsed header:\n" << fHeader);
 
@@ -562,10 +576,10 @@ namespace Katydid
         }
 
         // open the next file
-        KTINFO(eggreadlog, "Opening next egg file <" << *fCurrentFileIt << ">");
+        KTINFO(eggreadlog, "Opening next egg file <" << fCurrentFileIt->first << ">");
         try
         {
-            fMonarch = Monarch3::OpenForReading(fCurrentFileIt->native());
+            fMonarch = Monarch3::OpenForReading(fCurrentFileIt->first.native());
         }
         catch (M3Exception& e)
         {
@@ -687,6 +701,38 @@ namespace Katydid
         return;
     }
 
+    void KTEgg3Reader::AddMetadata()
+    {
+        if (fHeader.GetMetadataFilename().empty())
+        {
+            fHeader.Of< KTArbitraryMetadata >().SetMetadata(nullptr);
+            return;
+        }
+
+        scarab::param_translator translator;
+        scarab::param* metadata = translator.read_file(fHeader.GetMetadataFilename());
+        if (metadata == nullptr)
+        {
+            KTWARN(eggreadlog, "Metadata file was not present or could not be converted to a param object");
+        }
+#ifndef NDEBUG
+        if (metadata != nullptr)
+        {
+            KTDEBUG(eggreadlog, "Adding metadata from <" << fHeader.GetMetadataFilename() << ">");
+            KTDEBUG(eggreadlog, *metadata);
+        }
+#endif
+        fHeader.Of< KTArbitraryMetadata >().SetMetadata(metadata);
+
+        return;
+    }
+
+    bool KTEgg3Reader::MetadataIsPresent() const
+    {
+        if (! fHeader.Has< KTArbitraryMetadata >() ) return false;
+        return fHeader.Of< KTArbitraryMetadata >().GetMetadata() != nullptr;
+    }
+
 
     uint32_t ConvertMonarch3DataFormat( uint32_t m3DataFormat )
     {