diff --git a/BUILD.md b/BUILD.md new file mode 100644 index 0000000..348685b --- /dev/null +++ b/BUILD.md @@ -0,0 +1,3 @@ +``` +CALIPER_ROOT_DIR=/Caliper-Adiak/install CC=cc CXX=CC cmake -Dadiak_DIR=/Adiak/install/lib/cmake/adiak/ -Dcaliper_DIR=/Caliper-Adiak/install/share/cmake/caliper/ -DMETIS_ROOT_DIR==/new/metis/install -DCMAKE_BUILD_TYPE=Release -DWITH_ADIAK=ON ../src +``` diff --git a/src/config.h.in b/src/config.h.in index 39f0f57..0ab21ec 100644 --- a/src/config.h.in +++ b/src/config.h.in @@ -62,6 +62,7 @@ #endif #ifdef caliper_FOUND +#include #include #endif @@ -97,6 +98,11 @@ inline void Insist(const bool pass, const std::string &message) { #ifdef caliper_FOUND +void cali_init() { + MPI_Comm comm = MPI_COMM_WORLD; + adiak_init(&comm); + adiak_collect_all(); +} void wrapped_cali_mark_begin(const char *timer_name) { CALI_MARK_BEGIN(timer_name); @@ -106,19 +112,19 @@ void wrapped_cali_mark_end(const char *timer_name) { } #else -void wrapped_cali_mark_begin(const char *timer_name) { +void cali_init() { int rank; MPI_Comm_rank(MPI_COMM_WORLD, &rank); if (rank == 0) { - std::cout<<"Branson built without caliper, not starting timer: "< #include #include - +#include "config.h" #include "RNG.h" #include "constants.h" #include "input.h" #include "message_counter.h" #include "photon.h" #include "cell.h" + #include //============================================================================== @@ -295,6 +296,7 @@ class IMC_State { m_step++; } + // wrapped_cali_mark_begin("transport diagnostics"); //! Set pre-transport census energy (diagnostic) void set_pre_census_E(double _pre_census_E) { pre_census_E = _pre_census_E; } @@ -340,6 +342,7 @@ class IMC_State { step_receives_posted = mctr.n_receives_posted; step_receives_completed = mctr.n_receives_completed; } + // wrapped_cali_mark_end("transport diagnostics"); //! Set the number of cells requested in mesh passing method this timestep void set_step_cells_requested(uint64_t _step_cells_requested) { diff --git a/src/main.cc b/src/main.cc index 1a6f193..b39b0b5 100644 --- a/src/main.cc +++ b/src/main.cc @@ -36,7 +36,7 @@ using std::vector; int main(int argc, char **argv) { MPI_Init(&argc, &argv); - + cali_init(); // check to see if number of arguments is correct if (argc != 2) { cout << "Usage: BRANSON " << endl; @@ -103,11 +103,15 @@ int main(int argc, char **argv) { timers.start_timer("Total non-setup"); - if (input.get_dd_mode() == PARTICLE_PASS) + if (input.get_dd_mode() == PARTICLE_PASS) { + wrapped_cali_mark_begin("imc particle pass driver"); imc_particle_pass_driver(mesh, imc_state, imc_p, mpi_types, mpi_info); - else if (input.get_dd_mode() == REPLICATED) + wrapped_cali_mark_end("imc particle pass driver"); + } else if (input.get_dd_mode() == REPLICATED) { + wrapped_cali_mark_begin("imc replicated driver"); imc_replicated_driver(mesh, imc_state, imc_p, mpi_types, mpi_info); - else { + wrapped_cali_mark_end("imc replicated driver"); + } else { cout << "Driver for DD transport method currently not supported" << endl; exit(EXIT_FAILURE); } diff --git a/src/particle_pass_driver.h b/src/particle_pass_driver.h index 2c74921..81fafd8 100644 --- a/src/particle_pass_driver.h +++ b/src/particle_pass_driver.h @@ -64,7 +64,9 @@ void imc_particle_pass_driver(Mesh &mesh, IMC_State &imc_state, mctr.reset_counters(); //set opacity, Fleck factor, all energy to source + wrapped_cali_mark_begin("calculate photon energy"); mesh.calculate_photon_energy(imc_state, n_user_photons); + wrapped_cali_mark_end("calculate photon energy"); // all reduce to get total source energy to make correct number of // particles on each rank @@ -72,31 +74,46 @@ void imc_particle_pass_driver(Mesh &mesh, IMC_State &imc_state, MPI_Allreduce(MPI_IN_PLACE, &global_source_energy, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + wrapped_cali_mark_begin("calculate pre census energy"); imc_state.set_pre_census_E(get_photon_list_E(census_photons)); + wrapped_cali_mark_end("calculate pre census energy"); // make gpu setup object, may want to source on GPU later so make it before sourcing here GPU_Setup gpu_setup(rank, n_ranks, imc_parameters.get_use_gpu_transporter_flag(), mesh.get_cells()); // setup source - if (imc_state.get_step() == 1) + if (imc_state.get_step() == 1) { + wrapped_cali_mark_begin("make initial census photons"); census_photons = make_initial_census_photons(imc_state.get_dt(), mesh, rank, seed, n_user_photons, global_source_energy); + wrapped_cali_mark_end("make initial census photons"); + } imc_state.set_pre_census_E(get_photon_list_E(census_photons)); MPI_Barrier(MPI_COMM_WORLD); // make emission and source photons + wrapped_cali_mark_begin("make emission and source photons"); auto all_photons = make_photons(imc_state.get_dt(), mesh, rank, imc_state.get_step(), seed, n_user_photons, global_source_energy); + wrapped_cali_mark_end("make emission and source photons"); // add the census photons + wrapped_cali_mark_begin("add the census photons"); all_photons.insert(all_photons.end(), census_photons.begin(), census_photons.end()); + wrapped_cali_mark_end("add the census photons"); + wrapped_cali_mark_begin("set transported particles"); imc_state.set_transported_particles(all_photons.size()); - + wrapped_cali_mark_end("set transported particles"); + imc_state.print_memory_estimate(rank, n_ranks, mesh.get_n_local_cells(), all_photons.size()); // add barrier here to make sure the transport timer starts at roughly the same time MPI_Barrier(MPI_COMM_WORLD); + wrapped_cali_mark_begin("transport photons"); census_photons = particle_pass_transport(mesh, gpu_setup, imc_parameters, mpi_info, mpi_types, imc_state, mctr, abs_E, track_E, all_photons, imc_parameters.get_n_omp_threads()); + wrapped_cali_mark_end("transport photons"); + wrapped_cali_mark_begin("update temperature"); mesh.update_temperature(abs_E, track_E, imc_state); + wrapped_cali_mark_end("update temperature"); // update time for next step imc_state.print_conservation(imc_parameters.get_dd_mode()); @@ -118,7 +135,10 @@ void imc_particle_pass_driver(Mesh &mesh, IMC_State &imc_state, mem_record.read_meminfo(meminfo_string); #endif + wrapped_cali_mark_begin("update time for next step"); imc_state.next_time_step(); + wrapped_cali_mark_end("update time for next step"); + } #ifdef USE_MEMORY_RECORD diff --git a/src/replicated_driver.h b/src/replicated_driver.h index d86d8b3..fbcd51b 100644 --- a/src/replicated_driver.h +++ b/src/replicated_driver.h @@ -50,14 +50,17 @@ void imc_replicated_driver(Mesh &mesh, IMC_State &imc_state, mctr.reset_counters(); // set opacity, Fleck factor, all energy to source + wrapped_cali_mark_begin("calculate photon energy"); mesh.calculate_photon_energy(imc_state, n_user_photons); + wrapped_cali_mark_end("calculate photon energy"); // all reduce to get total source energy to make correct number of articles on each rank double global_source_energy = mesh.get_total_photon_E(); MPI_Allreduce(MPI_IN_PLACE, &global_source_energy, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); - + wrapped_cali_mark_begin("calculate pre census energy"); imc_state.set_pre_census_E(get_photon_list_E(census_photons)); + wrapped_cali_mark_end("calculate pre census energy"); // make gpu setup object, may want to source on GPU later so make it before sourcing here GPU_Setup gpu_setup(rank, n_ranks, imc_parameters.get_use_gpu_transporter_flag(), mesh.get_cells()); @@ -65,26 +68,38 @@ void imc_replicated_driver(Mesh &mesh, IMC_State &imc_state, // setup source Timer t_source; t_source.start_timer("source"); - if (imc_state.get_step() == 1) + if (imc_state.get_step() == 1) { + wrapped_cali_mark_begin("make initial census photons"); census_photons = make_initial_census_photons(imc_state.get_dt(), mesh, rank, seed, n_user_photons, global_source_energy); + wrapped_cali_mark_end("make initial census photons"); + } imc_state.set_pre_census_E(get_photon_list_E(census_photons)); // make emission and source photons + wrapped_cali_mark_begin("make emission and source photons"); auto all_photons = make_photons(imc_state.get_dt(), mesh, rank, imc_state.get_step(), seed, n_user_photons, global_source_energy); + wrapped_cali_mark_end("make emission and source photons"); // add the census photons + wrapped_cali_mark_begin("add the census photons"); all_photons.insert(all_photons.end(), census_photons.begin(), census_photons.end()); + wrapped_cali_mark_end("add the census photons"); + t_source.stop_timer("source"); if (rank ==0) std::cout<<"source time: "< replicated_transport( uint32_t rank_cell_offset{0}; // no offset in replicated mesh if(gpu_setup.use_gpu_transporter() && gpu_available ) { t_transport.start_timer("gpu transport"); + wrapped_cali_mark_begin("gpu transport photons"); gpu_transport_photons(rank_cell_offset, all_photons, gpu_setup.get_device_cells_ptr(), cell_tallies); + wrapped_cali_mark_end("gpu transport photons"); t_transport.stop_timer("gpu transport"); + std::cout<<"gpu transport time: "<