Merge remote-tracking branch 'origin/develop'

project8 · Feb 23, 2021 · cc8faf3 · cc8faf3
2 parents c6d52c9 + 1c8f9d9
commit cc8faf3
Show file tree

Hide file tree

Showing 11 changed files with 233 additions and 111 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -6,7 +6,7 @@ cmake_minimum_required( VERSION 3.1 )
 
 # Define the project
 cmake_policy( SET CMP0048 NEW ) # version in project()
-project( locust_mc VERSION 2.1.7)
+project( locust_mc VERSION 2.2.0)
 
 list( APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/Scarab/cmake )
 include( PackageBuilder )

diff --git a/Dockerfile b/Dockerfile
@@ -3,7 +3,7 @@ FROM project8/p8compute_dependencies:v0.9.0 as locust_common
 ARG build_type=Release
 ENV LOCUST_BUILD_TYPE=$build_type
 
-ENV LOCUST_TAG=v2.1.7
+ENV LOCUST_TAG=v2.2.0
 ENV LOCUST_BUILD_PREFIX=/usr/local/p8/locust/$LOCUST_TAG
 
 RUN mkdir -p $LOCUST_BUILD_PREFIX &&\

diff --git a/README.md b/README.md
@@ -16,7 +16,7 @@ After running Katydid, the processed fft spectra will have a span from 0 to the
 
 ### Tutorial
 
-Tutorials with examples are located in the locust-tutorial repository, [here](https://github.com/project8/locust-tutorial). 
+Tutorial documents with examples are located in the locust-tutorial repository, [here](https://github.com/project8/locust-tutorial). 
 
 Issues should be posted via [GitHub](https://github.com/project8/locust_mc/issues).
 
@@ -110,6 +110,16 @@ The following steps will build locust from scratch.  In the terminal:
     $ source /path/to/locust_mc/build/bin/kasperenv.sh 
     ```
   If you have another kassiopeia installation on your computer, you should ensure that locust is not using the environmental variables of the independent kassiopeia installation by removing this line from your .bashrc.
+  
+### Exit codes
+
+When running Locust with Kassiopeia in a shared cluster environment, 2 CPUs may be required unless hyperthreading is enabled on the cluster.  If the level of activity on the cluster is high, thread signals between the separate CPUs can occasionally be missed and an exception will be thrown.  Locust terminates with an integer exception in these three cases:
+
+*  ```exit(1)``` if the digitizer range was saturated.
+*  ```exit(2)``` if the Kassiopeia thread did not start.
+*  ```exit(3)``` if a digitizer sample was skipped.  
+
+Immediately after Locust has finished, the exit status can be checked from a bash prompt with ```echo #?```.  If the returned value is zero, then no exception was thrown.  Returned values of ```1```, ```2```, or ```3``` correspond to the exceptions above.  Output egg files are not written if an exception is thrown.  Exit codes ```2``` or ```3``` can typically be immediately requeued.  An exit code of ```1``` should be either reconfigured to unsaturate the digitizer, or if the digitizer does not appear saturated, reported as an issue.
 
 ### Docker container
 

diff --git a/Source/Applications/LocustSim.cc b/Source/Applications/LocustSim.cc
@@ -73,6 +73,11 @@ int main( int argc, char** argv )
         LERROR( lmclog, "Exception caught: " << e.what() );
         return -1;
     }
+    catch( int &e )
+    {
+        LERROR( lmclog, "Exception caught: " << e );
+        return e;
+    }
 
     return 0;
 }
diff --git a/Source/Generators/LMCArraySignalGenerator.cc b/Source/Generators/LMCArraySignalGenerator.cc
@@ -33,7 +33,7 @@ namespace locust
 		fTextFileWriting( 0 ),
         fphiLO(0.),
 		fNPreEventSamples( 150000 ),
-		fThreadCheckTime(10000),
+		fThreadCheckTime(200),
         EFieldBuffer( 1 ),
         EPhaseBuffer( 1 ),
         EAmplitudeBuffer( 1 ),
@@ -653,9 +653,9 @@ namespace locust
                     		{
                                 PreEventCounter = 0; // reset
                     		}
-                    		else if (!DriveAntenna(fp, startingIndex, index, aSignal, nfilterbins, dtfilter))
+                    		else
                     		{
-                    			LERROR(lmclog,"The antenna did not respond correctly after two tries.  Exiting.\n");
+                    			LERROR(lmclog,"The antenna did not respond correctly.  Exiting.\n");
                     			fSkippedSamples = true;
                                 tLock.unlock();
                     			break;
@@ -692,8 +692,18 @@ namespace locust
 			fInterface->fWaitBeforeEvent = false;
             WakeBeforeEvent();
             tKassiopeia.join();  // finish thread
-            if (fKassNeverStarted == true) return false;
-            if (fSkippedSamples == true) return false;
+
+            if (fKassNeverStarted == true)
+            {
+            	throw 2;
+            	return false;
+            }
+            if (fSkippedSamples == true)
+            {
+            	throw 3;
+            	return false;
+            }
+
 
 
         }  // fTransmitter->IsKassiopeia()

diff --git a/Source/Generators/LMCDigitizer.cc b/Source/Generators/LMCDigitizer.cc
@@ -21,6 +21,8 @@ namespace locust
     MT_REGISTER_GENERATOR(Digitizer, "digitizer");
 
     Digitizer::Digitizer( const std::string& aName ) :
+    		fRange( 2.e-8 ),
+			fOffset( -1.e-8 ),
             Generator( aName ),
             fADCValuesSigned( false )
     {
@@ -39,11 +41,11 @@ namespace locust
 
         unsigned bitDepth = aNode.get_value( "bit-depth", fParams.bit_depth );
         unsigned dataTypeSize = aNode.get_value( "data-type-size", fParams.data_type_size );
-        double vRange = aNode.get_value( "v-range", fParams.v_range );
-        double vMin = aNode.get_value( "v-offset", fParams.v_offset );
+        fRange = aNode.get_value( "v-range", fParams.v_range );
+        fOffset = aNode.get_value( "v-offset", fParams.v_offset );
 
 
-        get_calib_params( bitDepth, dataTypeSize, vMin, vRange, false, &fParams );
+        get_calib_params( bitDepth, dataTypeSize, fOffset, fRange, false, &fParams );
 
         LDEBUG( lmclog, "Digitizer calibration parameters set" );
 
@@ -90,6 +92,15 @@ namespace locust
                 digitizedData[2*ch*signalSize + index*2 ] = a2d< double, int8_t >( aSignal->SignalTimeComplex()[ch*signalSize + index ][0], &fParams );
                 digitizedData[2*ch*signalSize + index*2+1 ] = a2d< double, int8_t >( aSignal->SignalTimeComplex()[ch*signalSize + index ][1], &fParams );
 
+                if ((int(digitizedData[ 2*ch*signalSize + index*2 ]) == 0 ) || ( int(digitizedData[ 2*ch*signalSize + index*2 ] == 255)))
+                {
+                    LERROR(lmclog,"Digitizer range limit.\n");
+                    printf("Analog data at index %d channel %d is %g\n", index, ch, aSignal->SignalTimeComplex()[ch*signalSize + index ][0]);
+                    printf("Digitized data at index %d channel %d is %d\n", index, ch, digitizedData[2*ch*signalSize + index*2 ]);
+                	throw 1;
+                	return false;
+                }
+
                 if( index < 10 )
                 {
 
@@ -114,6 +125,14 @@ namespace locust
                     digitizedData[2*ch*signalSize + index*2 ] = a2d< double, uint8_t >( aSignal->SignalTimeComplex()[ch*signalSize + index ][0], &fParams );
                     digitizedData[2*ch*signalSize + index*2+1 ] = a2d< double, uint8_t >( aSignal->SignalTimeComplex()[ch*signalSize + index ][1], &fParams );
 
+                    if ((int(digitizedData[ 2*ch*signalSize + index*2 ]) == 0 ) || ( int(digitizedData[ 2*ch*signalSize + index*2 ] == 255)))
+                    {
+                        LERROR(lmclog,"Digitizer range limit.\n");
+                        printf("Analog data at index %d channel %d is %g\n", index, ch, aSignal->SignalTimeComplex()[ch*signalSize + index ][0]);
+                        printf("Digitized data at index %d channel %d is %d\n", index, ch, digitizedData[2*ch*signalSize + index*2 ]);
+                    	throw 1;
+                    	return false;
+                    }
 
                     if( index < 20 )
                     {

diff --git a/Source/Generators/LMCDigitizer.hh b/Source/Generators/LMCDigitizer.hh
@@ -9,8 +9,8 @@
 #define LMCDIGITIZER_HH_
 
 #include "LMCGenerator.hh"
-
 #include "digital.hh"
+#include "LMCException.hh"
 
 
 
@@ -57,6 +57,10 @@ namespace locust
             struct scarab::dig_calib_params fParams;
 
             bool fADCValuesSigned;
+
+        private:
+            double fRange;
+            double fOffset;
     };
 
     inline void Digitizer::SetADCValuesSigned( bool aFlag )

diff --git a/Source/Generators/LMCKassSignalGenerator.cc b/Source/Generators/LMCKassSignalGenerator.cc
@@ -32,8 +32,10 @@ namespace locust
 			fPhiLO_t(0.),
 			fNPreEventSamples( 15000 ),
 			fEventStartTime(-99.),
-			fThreadCheckTime(1000),
+			fThreadCheckTime(200),
 			fEventToFile( false ),
+			fKassNeverStarted( false ),
+			fSkippedSamples( false ),
 			fInterface( new KassLocustInterface() )
 
     {
@@ -145,7 +147,7 @@ namespace locust
 
     }
 
-    void* KassSignalGenerator::DriveAntenna(int PreEventCounter, unsigned index, Signal* aSignal, FILE *fp)
+    bool KassSignalGenerator::DriveAntenna(int PreEventCounter, unsigned index, Signal* aSignal, FILE *fp)
     {
 
         double tDopplerFrequencyAntenna = 0.;  // Doppler shifted cyclotron frequency in Hz.
@@ -245,7 +247,7 @@ namespace locust
 
         fInterface->fTOld += 1./(fAcquisitionRate*1.e6*aSignal->DecimationFactor());  // advance time here instead of in step modifier.  This preserves the freefield sampling.
 
-        return 0;
+        return true;
     }
 
     //Return index of fParticleHistory particle closest to the time we are evaluating
@@ -304,71 +306,98 @@ namespace locust
         fInterface->fKassTimeStep = 1./(fAcquisitionRate*1.e6*aSignal->DecimationFactor());
         std::thread tKassiopeia (&KassSignalGenerator::KassiopeiaInit, this, gxml_filename); // spawn new thread
 
+
         for( unsigned index = 0; index < aSignal->DecimationFactor()*aSignal->TimeSize(); ++index )
-        {
-            if ((!fInterface->fEventInProgress) && (!fInterface->fPreEventInProgress))
-            {
-            	if (ReceivedKassReady()) fInterface->fPreEventInProgress = true;
-            	else
-            	{
-            		printf("breaking\n");
-            		break;
-            	}
-
-            	LPROG( lmclog, "LMC ReceivedKassReady()" );
-
-            }
-
-            if (fInterface->fPreEventInProgress)
-            {
-                PreEventCounter += 1;
-
-                if (((!fTruth)&&(PreEventCounter > fNPreEventSamples))||((fTruth)&&(PreEventCounter > fNPreEventSamples)&&(index%(8192*aSignal->DecimationFactor())==0)  ))// finished pre-samples.  Start event.
-                {
-                    fInterface->fPreEventInProgress = false;  // reset.
-                    fInterface->fEventInProgress = true;
-                    LPROG( lmclog, "LMC about to WakeBeforeEvent()" );
-                    WakeBeforeEvent();  // trigger Kass event.
-                }
-            }
-
-            if (fInterface->fEventInProgress)  // fEventInProgress
-            {
-                std::unique_lock< std::mutex >tLock( fInterface->fMutexDigitizer, std::defer_lock );
-            	if (!fInterface->fKassEventReady)  // Kass confirms event is underway.
-            	{
-                    tLock.lock();
-                    fInterface->fDigitizerCondition.wait( tLock );
-                    if (fInterface->fEventInProgress)
-                    {
-                        DriveAntenna(PreEventCounter, index, aSignal, fp);
-                        PreEventCounter = 0; // reset
-                    }
-                    tLock.unlock();
-            	}
-            	else  // either Kass thread fell behind, or it has stopped generating events.
-            	{
-                    tLock.lock();
-                    std::this_thread::sleep_for(std::chrono::milliseconds(fThreadCheckTime));
-                    if (!fInterface->fKassEventReady)  // Kass event did start.  Continue but skip this sample.
-                    {
-                    	tLock.unlock();
-                    }
-                    else  // no Kass event ever started, unlock and break out of signal loop entirely.
-                    {
-                    	tLock.unlock();
-                    	break;
-                    }
-            	}
-            }
-        }  // for loop
-
-
-        fInterface->fDoneWithSignalGeneration = true;
-        fclose(fp);
-        LPROG( lmclog, "Finished signal loop." );
-        WakeBeforeEvent();
-        tKassiopeia.join();  // finish thread
+         {
+             if ((!fInterface->fEventInProgress) && (!fInterface->fPreEventInProgress))
+             {
+             	if (ReceivedKassReady()) fInterface->fPreEventInProgress = true;
+             	else
+             	{
+             		printf("breaking\n");
+             		break;
+             	}
+
+             	LPROG( lmclog, "LMC ReceivedKassReady" );
+
+             }
+
+             if (fInterface->fPreEventInProgress)  // Locust keeps sampling until Kass event.
+             {
+                 PreEventCounter += 1;
+
+                 if (((!fTruth)&&(PreEventCounter > fNPreEventSamples))||((fTruth)&&(PreEventCounter > fNPreEventSamples)&&(index%(8192*aSignal->DecimationFactor())==0)  ))// finished pre-samples.  Start event.
+                 {
+                     fInterface->fPreEventInProgress = false;  // reset.
+                     fInterface->fEventInProgress = true;
+                     LPROG( lmclog, "LMC about to WakeBeforeEvent()" );
+                     WakeBeforeEvent();  // trigger Kass event.
+                 }
+             }
+
+             if (fInterface->fEventInProgress)  // fEventInProgress
+             {
+                 std::unique_lock< std::mutex >tLock( fInterface->fMutexDigitizer, std::defer_lock );
+             	if (!fInterface->fKassEventReady)  // Kass confirms event is underway.
+             	{
+                     tLock.lock();
+                     fInterface->fDigitizerCondition.wait( tLock );
+                     if (fInterface->fEventInProgress)
+                     {
+                 		if (DriveAntenna(PreEventCounter, index, aSignal, fp))
+                 		{
+                             PreEventCounter = 0; // reset
+                 		}
+                 		else
+                 		{
+                 			LERROR(lmclog,"The antenna did not respond correctly.  Exiting.\n");
+                 			fSkippedSamples = true;
+                            tLock.unlock();
+                 			break;
+                 		}
+                     }
+                     tLock.unlock();
+             	}
+             	else  // either Kass thread fell behind, or it has stopped generating events.
+             	{
+                     tLock.lock();
+                     std::this_thread::sleep_for(std::chrono::milliseconds(fThreadCheckTime));
+                     if (!fInterface->fKassEventReady)  // Kass event did start.  Continue but skip this sample.
+                     {
+                     	tLock.unlock();
+                     }
+                     else  // Kass event has not started, unlock and exit.
+                     {
+                     	if ( index < fNPreEventSamples+1 )
+                     	{
+                 			LERROR(lmclog,"Kass thread is unresponsive.  Exiting.\n");
+                     		fKassNeverStarted = true;
+                     	}
+                     	tLock.unlock();
+                     	break;
+                     }
+             	}
+             }
+         }  // for loop
+
+
+         fInterface->fDoneWithSignalGeneration = true;
+         LPROG( lmclog, "Finished signal loop." );
+			fInterface->fWaitBeforeEvent = false;
+         WakeBeforeEvent();
+         tKassiopeia.join();  // finish thread
+
+         if (fKassNeverStarted == true)
+         {
+         	throw 2;
+         	return false;
+         }
+         if (fSkippedSamples == true)
+         {
+         	throw 3;
+         	return false;
+         }
+
 
         return true;
     }

diff --git a/Source/Generators/LMCKassSignalGenerator.hh b/Source/Generators/LMCKassSignalGenerator.hh
@@ -11,9 +11,7 @@
 #include "LMCGenerator.hh"
 
 #include "LMCKassLocustInterface.hh"
-//#include "LMCConst.hh"
-//#include "LMCThreeVector.hh"
-
+#include "LMCException.hh"
 #include <vector>
 using std::vector;
 
@@ -56,7 +54,7 @@ namespace locust
             bool ReceivedKassReady();
 
             bool DoGenerate( Signal* aSignal );
-            void* DriveAntenna(int PreEventCounter, unsigned index, Signal* aSignal, FILE *fp);
+            bool DriveAntenna(int PreEventCounter, unsigned index, Signal* aSignal, FILE *fp);
             int FindNode(double tNew) const;
             double TE11ModeExcitation() const;
             double TE10ModeExcitation() const;
@@ -75,6 +73,8 @@ namespace locust
             mutable int fPreviousRetardedIndex;
             double fEventStartTime;
             bool fEventToFile;
+            bool fKassNeverStarted;
+            bool fSkippedSamples;
             kl_interface_ptr_t fInterface;