diff --git a/CMakeLists.txt b/CMakeLists.txt index 481d376..fc6ba1f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -74,14 +74,13 @@ endif() # Dependencies # ============================================================================= -find_package(HDF5) if(SONATA_REPORT_ENABLE_MPI) find_package(MPI REQUIRED) if (MPI_FOUND) # With MPI we prefer HDF5 parallel set(HDF5_PREFER_PARALLEL "ON") endif() - + find_package(HDF5) if (MPI_FOUND) if (HDF5_FOUND) if (HDF5_IS_PARALLEL) @@ -102,6 +101,7 @@ if(SONATA_REPORT_ENABLE_MPI) endif() endif() else() + find_package(HDF5) if (HDF5_IS_PARALLEL) message(FATAL_ERROR "HDF5 parallel IO found without MPI") else() diff --git a/include/bbp/sonata/reports.h b/include/bbp/sonata/reports.h index c1657ee..943cd47 100644 --- a/include/bbp/sonata/reports.h +++ b/include/bbp/sonata/reports.h @@ -118,6 +118,15 @@ int sonata_record_node_data(double step, */ int sonata_record_data(double step); +/** + * \brief Write steps_to_write steps given a pre-buffered data + * \return -3 if the Sonata report doesn't exist, -1 if the report name doesn't exist, 0 otherwise + */ +int sonata_write_buffered_data(const char* report_name, + const float* buffered_data, + size_t buffered_data_size, + uint32_t steps_to_write); + /** * \brief Check status of the recordings/buffers and flush if necessary * \return 0 diff --git a/src/data/sonata_data.cpp b/src/data/sonata_data.cpp index 6515e58..05ec123 100644 --- a/src/data/sonata_data.cpp +++ b/src/data/sonata_data.cpp @@ -33,8 +33,8 @@ SonataData::SonataData(const std::string& report_name, time_ = {round(tstart / dt) * dt, tend, dt}; reporting_period_ = static_cast(dt / SonataReport::atomic_step_); - last_step_recorded_ = tstart / SonataReport::atomic_step_; - last_step_ = tend / SonataReport::atomic_step_; + previous_step_recorded_ = tstart / SonataReport::atomic_step_; + final_step_ = tend / SonataReport::atomic_step_; } SonataData::SonataData(const std::string& report_name) @@ -96,13 +96,13 @@ void SonataData::prepare_buffer(size_t max_buffer_size) { bool SonataData::is_due_to_report(double step) const noexcept { // Dont record data if current step < tstart - if (step < last_step_recorded_) { + if (step < previous_step_recorded_) { return false; // Dont record data if current step > tend - } else if (step > last_step_) { + } else if (step > final_step_) { return false; // Dont record data if is not a reporting step (step%period) - } else if (static_cast(step - last_step_recorded_) % reporting_period_ != 0) { + } else if (static_cast(step - previous_step_recorded_) % reporting_period_ != 0) { return false; } return true; @@ -110,16 +110,16 @@ bool SonataData::is_due_to_report(double step) const noexcept { void SonataData::record_data(double step, const std::vector& node_ids) { // Calculate the offset to write into the buffer - uint32_t offset = static_cast((step - last_step_recorded_) / reporting_period_); - uint32_t local_position = last_position_ + total_elements_ * offset; + uint32_t offset = static_cast((step - previous_step_recorded_) / reporting_period_); + uint32_t local_position = previous_position_ + total_elements_ * offset; if (SonataReport::rank_ == 0) { logger->trace( - "RANK={} Recording data for step={} last_step_recorded={} first node_id={} " + "RANK={} Recording data for step={} previous_step_recorded={} first node_id={} " "buffer_size={} " "and offset={}", SonataReport::rank_, step, - last_step_recorded_, + previous_step_recorded_, node_ids[0], report_buffer_.size(), local_position); @@ -145,13 +145,14 @@ void SonataData::record_data(double step, const std::vector& node_ids) } void SonataData::record_data(double step) { - uint32_t local_position = last_position_; + uint32_t local_position = previous_position_; if (SonataReport::rank_ == 0) { logger->trace( - "RANK={} Recording data for step={} last_step_recorded={} buffer_size={} and offset={}", + "RANK={} Recording data for step={} previous_step_recorded={} buffer_size={} and " + "offset={}", SonataReport::rank_, step, - last_step_recorded_, + previous_step_recorded_, report_buffer_.size(), local_position); } @@ -160,11 +161,11 @@ void SonataData::record_data(double step) { local_position += kv.second->get_num_elements(); } current_step_++; - last_position_ += total_elements_; - last_step_recorded_ += reporting_period_; + previous_position_ += total_elements_; + previous_step_recorded_ += reporting_period_; if (current_step_ == steps_to_write_) { - write_data(); + flush(); } } @@ -177,8 +178,8 @@ void SonataData::check_and_write(double timestep) { logger->trace("Updating timestep t={}", timestep); } current_step_ += steps_recorded_; - last_position_ += total_elements_ * steps_recorded_; - last_step_recorded_ += reporting_period_ * steps_recorded_; + previous_position_ += total_elements_ * steps_recorded_; + previous_step_recorded_ += reporting_period_ * steps_recorded_; nodes_recorded_.clear(); // Write when buffer is full, finish all remaining recordings or when record several steps in a // row @@ -194,7 +195,7 @@ void SonataData::check_and_write(double timestep) { remaining_steps_, steps_recorded_); } - write_data(); + flush(); } steps_recorded_ = 0; } @@ -303,24 +304,28 @@ void SonataData::write_spike_populations() { } } -void SonataData::write_data() { +void SonataData::write_data(const std::vector& buffered_data, uint32_t steps_to_write) { if (remaining_steps_ <= 0) { // Nothing left to write return; } - if (current_step_ >= remaining_steps_) { // Avoid writing out of bounds - current_step_ = remaining_steps_; + if (steps_to_write >= remaining_steps_) { // Avoid writing out of bounds + steps_to_write = remaining_steps_; } - hdf5_writer_->write_2D(report_buffer_, current_step_, total_elements_); - remaining_steps_ -= current_step_; + hdf5_writer_->write_2D(buffered_data, steps_to_write, total_elements_); + remaining_steps_ -= steps_to_write; if (SonataReport::rank_ == 0) { logger->debug("Writing timestep data to file {}", report_name_); - logger->debug("-Steps written: {}", current_step_); + logger->debug("-Steps written: {}", steps_to_write); logger->debug("-Remaining steps: {}", remaining_steps_); } - last_position_ = 0; + previous_position_ = 0; current_step_ = 0; } +void SonataData::flush() { + write_data(report_buffer_, current_step_); +} + void SonataData::close() { hdf5_writer_->close(); } diff --git a/src/data/sonata_data.h b/src/data/sonata_data.h index bb6ab72..4f7cdf0 100644 --- a/src/data/sonata_data.h +++ b/src/data/sonata_data.h @@ -72,7 +72,8 @@ class SonataData void write_spike_populations(); void add_population(std::unique_ptr&& population); - void write_data(); + void write_data(const std::vector& buffered_data, uint32_t steps_to_write); + void flush(); void close(); bool is_due_to_report(double step) const noexcept; @@ -106,11 +107,11 @@ class SonataData uint32_t steps_to_write_ = 0; uint32_t current_step_ = 0; uint32_t steps_recorded_ = 0; - uint32_t last_position_ = 0; + uint32_t previous_position_ = 0; uint32_t remaining_steps_ = 0; uint32_t reporting_period_ = 0; - double last_step_recorded_ = 0.; - double last_step_ = 0.; + double previous_step_recorded_ = 0.; + double final_step_ = 0.; std::vector node_ids_; std::vector index_pointers_; diff --git a/src/library/report.cpp b/src/library/report.cpp index c789c28..65294b9 100644 --- a/src/library/report.cpp +++ b/src/library/report.cpp @@ -94,6 +94,12 @@ void Report::record_data(double step) { } } +void Report::write_buffered_data(const std::vector& buffered_data, uint32_t steps_to_write) { + for (const auto& sonata_data : sonata_populations_) { + sonata_data->write_data(buffered_data, steps_to_write); + } +} + void Report::check_and_flush(double timestep) { for (const auto& sonata_data : sonata_populations_) { sonata_data->check_and_write(timestep); @@ -115,7 +121,7 @@ void Report::flush(double time) { } for (const auto& sonata_data : sonata_populations_) { // Write if there are any remaining steps to write - sonata_data->write_data(); + sonata_data->flush(); if (time - tend_ + dt_ / 2 > 1e-6) { if (!report_is_closed_) { sonata_data->close(); diff --git a/src/library/report.h b/src/library/report.h index 0a56815..7c380e5 100644 --- a/src/library/report.h +++ b/src/library/report.h @@ -45,6 +45,8 @@ class Report virtual void record_data(double step, const std::vector& node_ids); virtual void record_data(double step); + virtual void write_buffered_data(const std::vector& buffered_data, + uint32_t steps_to_write); virtual void check_and_flush(double timestep); virtual void flush(double time); void refresh_pointers(std::function refresh_function); diff --git a/src/library/reports.cpp b/src/library/reports.cpp index f603b8e..8cb7811 100644 --- a/src/library/reports.cpp +++ b/src/library/reports.cpp @@ -104,6 +104,22 @@ int sonata_record_data(double step) { return 0; } +int sonata_write_buffered_data(const char* report_name, + const float* buffered_data, + size_t buffered_data_size, + uint32_t steps_to_write) { + if (sonata_report.is_empty()) { + return -3; + } + if (!sonata_report.report_exists(report_name)) { + return -1; + } + const std::vector data(buffered_data, buffered_data + buffered_data_size); + auto report = sonata_report.get_report(report_name); + report->write_buffered_data(data, steps_to_write); + return 0; +} + int sonata_check_and_flush(double timestep) { auto functor = std::mem_fn(&bbp::sonata::Report::check_and_flush); sonata_report.apply_all(functor, timestep); diff --git a/tests/integration/integration_test.cpp b/tests/integration/integration_test.cpp index 7a95833..8b16da0 100644 --- a/tests/integration/integration_test.cpp +++ b/tests/integration/integration_test.cpp @@ -12,7 +12,7 @@ #include #include -struct Neuron { +struct Cell { uint64_t node_id; std::string kind; // soma / element std::vector voltages; @@ -40,19 +40,19 @@ void generate_spikes(const std::vector& nodeids, } } -void generate_elements(Neuron& neuron, int seed) { +void generate_elements(Cell& cell, int seed) { // 50+-5 elements size_t num_elements = 50 + ((seed % 10) - 5); - if (neuron.kind == "soma") { + if (cell.kind == "soma") { num_elements = 1; } - neuron.voltages.reserve(num_elements); + cell.voltages.reserve(num_elements); for (size_t j = 0; j < num_elements; j++) { - neuron.voltages.push_back(seed % 10); + cell.voltages.push_back(seed % 10); } } -std::vector generate_data(std::vector& neurons, +std::vector generate_data(std::vector& cells, const std::string& kind, int seed) { std::vector nodeids; @@ -60,19 +60,19 @@ std::vector generate_data(std::vector& neurons, // 1053...) uint64_t next_nodeid = 1000 + 1 + seed * 10; - // 5+-5 neurons - uint32_t num_neurons = 5 + ((2 + (seed % 10)) - 5); - nodeids.reserve(num_neurons); - for (uint32_t i = 0; i < num_neurons; i++) { - Neuron tmp_neuron; - tmp_neuron.kind = kind; + // 5+-5 cells + uint32_t num_cells = 5 + ((2 + (seed % 10)) - 5); + nodeids.reserve(num_cells); + for (uint32_t i = 0; i < num_cells; i++) { + Cell tmp_cell; + tmp_cell.kind = kind; nodeids.push_back(next_nodeid); - tmp_neuron.node_id = next_nodeid++; + tmp_cell.node_id = next_nodeid++; // element or soma - generate_elements(tmp_neuron, seed); - neurons.push_back(tmp_neuron); + generate_elements(tmp_cell, seed); + cells.push_back(tmp_cell); } return nodeids; } @@ -83,36 +83,37 @@ void init(const char* report_name, double tstart, double tstop, double dt, - std::vector& neurons, + std::vector& cells, const std::string& kind, const std::string& units) { // logic for registering soma and element reports with reportinglib sonata_create_report(report_name, tstart, tstop, dt, units.c_str(), kind.c_str()); - for (auto& neuron : neurons) { - sonata_add_node(report_name, population_name, population_offset, neuron.node_id); - int element_id = neuron.node_id * 1000; + for (auto& cell : cells) { + sonata_add_node(report_name, population_name, population_offset, cell.node_id); + int element_id = cell.node_id * 1000; - for (auto& element : neuron.voltages) { - sonata_add_element(report_name, population_name, neuron.node_id, element_id, &element); + for (auto& element : cell.voltages) { + sonata_add_element(report_name, population_name, cell.node_id, element_id, &element); ++element_id; } } } -void change_data(std::vector& neurons) { +void change_data(std::vector& cells, std::vector& buffered_data) { // Increment in 1 per timestep every voltage - for (auto& neuron : neurons) { - for (auto& element : neuron.voltages) { + for (auto& cell : cells) { + for (auto& element : cell.voltages) { + buffered_data.push_back(element); element++; } } } -void print_data(std::vector& neurons) { - for (auto& neuron : neurons) { - std::cout << "++NEURON node_id: " << neuron.node_id << "\nelements:\n"; - std::copy(neuron.voltages.begin(), - neuron.voltages.end(), +void print_data(std::vector& cells) { + for (auto& cell : cells) { + std::cout << "++NEURON node_id: " << cell.node_id << "\nelements:\n"; + std::copy(cell.voltages.begin(), + cell.voltages.end(), std::ostream_iterator(std::cout, ", ")); std::cout << "\n\n"; } @@ -135,16 +136,16 @@ int main() { const double tstart = 0.0; const double tstop = 0.3; - std::vector element_neurons; - std::vector soma_neurons; + std::vector element_cells; + std::vector soma_cells; std::vector element_nodeids; std::vector soma_nodeids; std::vector spike_timestamps; std::vector spike_node_ids; // Each rank will get different number of nodes (some even 0, so will be idle ranks) - element_nodeids = generate_data(element_neurons, "compartment", global_rank); - soma_nodeids = generate_data(soma_neurons, "soma", global_rank); + element_nodeids = generate_data(element_cells, "compartment", global_rank); + soma_nodeids = generate_data(soma_cells, "soma", global_rank); generate_spikes( soma_nodeids, spike_timestamps, spike_node_ids, tstart, tstop, global_rank, global_size); @@ -159,6 +160,8 @@ int main() { const char* population_name = "All"; const char* units = "mV"; uint64_t population_offset = 0; + std::vector element_buffered_data = {}; + std::vector soma_buffered_data = {}; init(element_report, population_name, @@ -166,7 +169,7 @@ int main() { tstart, tstop, dt, - element_neurons, + element_cells, "compartment", units); init(soma_report, @@ -175,7 +178,7 @@ int main() { tstart, tstop, dt, - soma_neurons, + soma_cells, "soma", units); sonata_set_max_buffer_size_hint(20); @@ -206,10 +209,34 @@ int main() { sonata_check_and_flush(t); t += dt; // Change data every timestep - change_data(element_neurons); - change_data(soma_neurons); + change_data(element_cells, element_buffered_data); + change_data(soma_cells, soma_buffered_data); } sonata_flush(t); + sonata_clear(); + + // Write pre-buffered data (GPU usecase) + if (global_rank == 0) { + logger->info("GPU usecase: Writting prebuffered steps"); + } + const char* buffered_soma_report = "buffered_soma_report"; + const char* buffered_population_name = "buffered"; + + init(buffered_soma_report, + buffered_population_name, + population_offset, + tstart, + tstop, + dt, + soma_cells, + "soma", + units); + + sonata_setup_communicators(); + sonata_prepare_datasets(); + sonata_write_buffered_data(buffered_soma_report, soma_buffered_data.data(), soma_buffered_data.size(), num_steps); + sonata_clear(); + const std::string output_dir = "."; // Create a spike file diff --git a/tests/integration/integration_test.sh.in b/tests/integration/integration_test.sh.in index 5e76a3f..eeb8669 100644 --- a/tests/integration/integration_test.sh.in +++ b/tests/integration/integration_test.sh.in @@ -27,6 +27,7 @@ fi #TODO: adapt comparison to any population name (for now All) h5diff -c compartment_report.h5 $ref_compartment /report/All/data > diff_compartment.dat 2>&1 h5diff -c soma_report.h5 $ref_soma /report/All/data > diff_soma.dat 2>&1 +h5diff -c buffered_soma_report.h5 $ref_soma /report/buffered/data /report/All/data > diff_soma_buffered.dat 2>&1 h5diff -c out.h5 $ref_spikes /spikes/NodeA/timestamps > diff_spikes.dat 2>&1 h5diff -c out.h5 $ref_spikes /spikes/NodeB/node_ids > diff_spikes.dat 2>&1 @@ -40,6 +41,11 @@ then echo "Soma results are different, check the file diff_soma.dat. Test failed!" cat diff_soma.dat exit 1 +elif [ -s diff_soma_buffered.dat ] +then + echo "Soma buffered results are different, check the file diff_soma_buffered.dat. Test failed!" + cat diff_soma_buffered.dat + exit 1 elif [ -s diff_spikes.dat ] then echo "Spike results are different, check the file diff_spikes.dat. Test failed!"