My Project
Loading...
Searching...
No Matches
FlowMain.hpp
1/*
2 Copyright 2013, 2014, 2015 SINTEF ICT, Applied Mathematics.
3 Copyright 2014 Dr. Blatt - HPC-Simulation-Software & Services
4 Copyright 2015 IRIS AS
5 Copyright 2014 STATOIL ASA.
6
7 This file is part of the Open Porous Media project (OPM).
8
9 OPM is free software: you can redistribute it and/or modify
10 it under the terms of the GNU General Public License as published by
11 the Free Software Foundation, either version 3 of the License, or
12 (at your option) any later version.
13
14 OPM is distributed in the hope that it will be useful,
15 but WITHOUT ANY WARRANTY; without even the implied warranty of
16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 GNU General Public License for more details.
18
19 You should have received a copy of the GNU General Public License
20 along with OPM. If not, see <http://www.gnu.org/licenses/>.
21*/
22#ifndef OPM_FLOW_MAIN_HEADER_INCLUDED
23#define OPM_FLOW_MAIN_HEADER_INCLUDED
24
25#include <opm/input/eclipse/EclipseState/EclipseState.hpp>
26#include <opm/input/eclipse/EclipseState/IOConfig/IOConfig.hpp>
27#include <opm/input/eclipse/EclipseState/InitConfig/InitConfig.hpp>
28
30
31#include <opm/simulators/flow/Banners.hpp>
32#include <opm/simulators/flow/FlowUtils.hpp>
33#include <opm/simulators/flow/NlddReporting.hpp>
34#include <opm/simulators/flow/SimulatorFullyImplicitBlackoil.hpp>
35
36#if HAVE_MPI
37#define RESERVOIR_COUPLING_ENABLED
38#endif
39#if HAVE_DUNE_FEM
40#include <dune/fem/misc/mpimanager.hh>
41#else
42#include <dune/common/parallel/mpihelper.hh>
43#endif
44
45#ifdef _OPENMP
46#include <omp.h>
47#endif
48
49#include <charconv>
50#include <cstddef>
51#include <memory>
52
53namespace Opm::Parameters {
54
55// Do not merge parallel output files or warn about them
56struct EnableLoggingFalloutWarning { static constexpr bool value = false; };
57struct OutputInterval { static constexpr int value = 1; };
58
59} // namespace Opm::Parameters
60
61namespace Opm {
62
63 class Deck;
64
65 // The FlowMain class is the black-oil simulator.
66 template <class TypeTag>
68 {
69 public:
77
79
80 FlowMain(int argc, char **argv, bool output_cout, bool output_files )
81 : argc_{argc}, argv_{argv},
82 output_cout_{output_cout}, output_files_{output_files}
83 {
84
85 }
86
87 // Read the command line parameters. Throws an exception if something goes wrong.
88 static int setupParameters_(int argc, char** argv, Parallel::Communication comm)
89 {
90 if (!Parameters::IsRegistrationOpen()) {
91 // We have already successfully run setupParameters_().
92 // For the dynamically chosen runs (as from the main flow
93 // executable) we must run this function again with the
94 // real typetag to be used, as the first time was with the
95 // "FlowEarlyBird" typetag. However, for the static ones (such
96 // as 'flow_onephase_energy') it has already been run with the
97 // correct typetag.
98 return EXIT_SUCCESS;
99 }
100 // register the flow specific parameters
101 Parameters::Register<Parameters::OutputInterval>
102 ("Specify the number of report steps between two consecutive writes of restart data");
103 Parameters::Register<Parameters::EnableLoggingFalloutWarning>
104 ("Developer option to see whether logging was on non-root processors. "
105 "In that case it will be appended to the *.DBG or *.PRT files");
106
107 // register the base parameters
108 registerAllParameters_<TypeTag>(/*finalizeRegistration=*/false);
109
110 Simulator::registerParameters();
111
112 detail::hideUnusedParameters<Scalar>();
113
114 Parameters::endRegistration();
115
116 int mpiRank = comm.rank();
117
118 // read in the command line parameters
120 const_cast<const char**>(argv),
121 /*doRegistration=*/false,
122 /*allowUnused=*/true,
123 /*handleHelp=*/(mpiRank==0));
124 if (status == 0) {
125
126 // deal with unknown parameters.
127
128 int unknownKeyWords = 0;
129 if (mpiRank == 0) {
130 unknownKeyWords = Parameters::printUnused(std::cerr);
131 }
134 if ( unknownKeyWords )
135 {
136 if ( mpiRank == 0 )
137 {
138 std::string msg = "Aborting simulation due to unknown "
139 "parameters. Please query \"flow --help\" for "
140 "supported command line parameters.";
141 if (OpmLog::hasBackend("STREAMLOG"))
142 {
143 OpmLog::error(msg);
144 }
145 else {
146 std::cerr << msg << std::endl;
147 }
148 }
149 return EXIT_FAILURE;
150 }
151
152 // deal with --print-parameters and unknown parameters.
153 if (Parameters::Get<Parameters::PrintParameters>() == 1) {
154 if (mpiRank == 0) {
155 Parameters::printValues(std::cout);
156 }
157 return -1;
158 }
159 }
160
161 // set the maximum limit on OMP threads
162 setMaxThreads();
163
164 return status;
165 }
166
171 {
172 return execute_(&FlowMain::runSimulator, /*cleanup=*/true);
173 }
174
175 int executeInitStep()
176 {
177 return execute_(&FlowMain::runSimulatorInit, /*cleanup=*/false);
178 }
179
180 // Returns true unless "EXIT" was encountered in the schedule
181 // section of the input datafile.
182 int executeStep()
183 {
184 return simulator_->runStep(*simtimer_);
185 }
186
187 // Called from Python to cleanup after having executed the last
188 // executeStep()
189 int executeStepsCleanup()
190 {
191 SimulatorReport report = simulator_->finalize();
192 runSimulatorAfterSim_(report);
193 return report.success.exit_status;
194 }
195
196 ModelSimulator* getSimulatorPtr()
197 {
198 return modelSimulator_.get();
199 }
200
201 SimulatorTimer* getSimTimer()
202 {
203 return simtimer_.get();
204 }
205
208 {
209 return simtimer_->stepLengthTaken();
210 }
211
212 private:
213 // called by execute() or executeInitStep()
214 int execute_(int (FlowMain::* runOrInitFunc)(), bool cleanup)
215 {
216 auto logger = [this](const std::exception& e, const std::string& message_start) {
217 std::ostringstream message;
218 message << message_start << e.what();
219
220 if (this->output_cout_) {
221 // in some cases exceptions are thrown before the logging system is set
222 // up.
223 if (OpmLog::hasBackend("STREAMLOG")) {
224 OpmLog::error(message.str());
225 }
226 else {
227 std::cout << message.str() << "\n";
228 }
229 }
230 detail::checkAllMPIProcesses();
231 return EXIT_FAILURE;
232 };
233
234 try {
235 // deal with some administrative boilerplate
236
237 Dune::Timer setupTimerAfterReadingDeck;
239
240 int status = setupParameters_(this->argc_, this->argv_, FlowGenericVanguard::comm());
241 if (status) {
242 return status;
243 }
244
245 setupParallelism();
246 setupModelSimulator();
248
249 this->deck_read_time_ = modelSimulator_->vanguard().setupTime();
250 this->total_setup_time_ = setupTimerAfterReadingDeck.elapsed() + this->deck_read_time_;
251
252 // if run, do the actual work, else just initialize
253 int exitCode = (this->*runOrInitFunc)();
254 if (cleanup) {
255 executeCleanup_();
256 }
257 return exitCode;
258 }
259 catch (const TimeSteppingBreakdown& e) {
260 auto exitCode = logger(e, "Simulation aborted: ");
261 executeCleanup_();
262 return exitCode;
263 }
264 catch (const std::exception& e) {
265 auto exitCode = logger(e, "Simulation aborted as program threw an unexpected exception: ");
266 executeCleanup_();
267 return exitCode;
268 }
269 }
270
271 void executeCleanup_() {
272 // clean up
273 mergeParallelLogFiles();
274 }
275
276 protected:
277 void setupParallelism()
278 {
279 // determine the rank of the current process and the number of processes
280 // involved in the simulation. MPI must have already been initialized
281 // here. (yes, the name of this method is misleading.)
282 auto comm = FlowGenericVanguard::comm();
283 mpi_rank_ = comm.rank();
284 mpi_size_ = comm.size();
285
286 setMaxThreads();
287 }
288
289 static void setMaxThreads()
290 {
291#if _OPENMP
292 // If openMP is available, default to 2 threads per process unless
293 // OMP_NUM_THREADS is set or command line --threads-per-process used.
294 // Issue a warning if both OMP_NUM_THREADS and --threads-per-process are set,
295 // but let the environment variable take precedence.
296 constexpr int default_threads = 2;
297 const bool isSet = Parameters::IsSet<Parameters::ThreadsPerProcess>();
298 const int requested_threads = Parameters::Get<Parameters::ThreadsPerProcess>();
300
301 const char* env_var = getenv("OMP_NUM_THREADS");
302 if (env_var) {
303 int omp_num_threads = -1;
304 auto result = std::from_chars(env_var, env_var + std::strlen(env_var), omp_num_threads);
305 if (result.ec == std::errc() && omp_num_threads > 0) {
306 // Set threads to omp_num_threads if it was successfully parsed and is positive
308 if (isSet) {
309 OpmLog::warning("Environment variable OMP_NUM_THREADS takes precedence over the --threads-per-process cmdline argument.");
310 }
311 } else {
312 OpmLog::warning("Invalid value for OMP_NUM_THREADS environment variable.");
313 }
314 }
315
316 // Requesting -1 thread will let OMP automatically deduce the number
317 // but setting OMP_NUM_THREADS takes precedence.
318 if (env_var || !(isSet && requested_threads == -1)) {
320 }
321#endif
322
324 TM::init(false);
325 }
326
327 void mergeParallelLogFiles()
328 {
329 // force closing of all log files.
330 OpmLog::removeAllBackends();
331
332 if (mpi_rank_ != 0 || mpi_size_ < 2 || !this->output_files_ || !modelSimulator_) {
333 return;
334 }
335
336 detail::mergeParallelLogFiles(eclState().getIOConfig().getOutputDir(),
337 Parameters::Get<Parameters::EclDeckFileName>(),
338 Parameters::Get<Parameters::EnableLoggingFalloutWarning>());
339 }
340
341 void setupModelSimulator()
342 {
343 modelSimulator_ = std::make_unique<ModelSimulator>(FlowGenericVanguard::comm(), /*verbose=*/false);
344 modelSimulator_->executionTimer().start();
345 modelSimulator_->model().applyInitialSolution();
346 }
347
348 const EclipseState& eclState() const
349 { return modelSimulator_->vanguard().eclState(); }
350
351 EclipseState& eclState()
352 { return modelSimulator_->vanguard().eclState(); }
353
354 const Schedule& schedule() const
355 { return modelSimulator_->vanguard().schedule(); }
356
357 // Run the simulator.
358 int runSimulator()
359 {
360 return runSimulatorInitOrRun_(&FlowMain::runSimulatorRunCallback_);
361 }
362
363 int runSimulatorInit()
364 {
365 return runSimulatorInitOrRun_(&FlowMain::runSimulatorInitCallback_);
366 }
367
368 private:
369 // Callback that will be called from runSimulatorInitOrRun_().
370 int runSimulatorRunCallback_()
371 {
372#ifdef RESERVOIR_COUPLING_ENABLED
373 SimulatorReport report = simulator_->run(*simtimer_, this->argc_, this->argv_);
374#else
375 SimulatorReport report = simulator_->run(*simtimer_);
376#endif
377 runSimulatorAfterSim_(report);
378 return report.success.exit_status;
379 }
380
381 // Callback that will be called from runSimulatorInitOrRun_().
382 int runSimulatorInitCallback_()
383 {
384#ifdef RESERVOIR_COUPLING_ENABLED
385 simulator_->init(*simtimer_, this->argc_, this->argv_);
386#else
387 simulator_->init(*simtimer_);
388#endif
389 return EXIT_SUCCESS;
390 }
391
392 // Output summary after simulation has completed
393 void runSimulatorAfterSim_(SimulatorReport &report)
394 {
395 if (simulator_->model().hasNlddSolver()) {
396 const auto& odir = eclState().getIOConfig().getOutputDir();
397 // Write the number of nonlinear iterations per cell to a file in ResInsight compatible format
398 simulator_->model().writeNonlinearIterationsPerCell(odir);
399 // Write the NLDD statistics to the DBG file
400 reportNlddStatistics(simulator_->model().domainAccumulatedReports(),
401 simulator_->model().localAccumulatedReports(),
402 this->output_cout_,
404 }
405
406 if (! this->output_cout_) {
407 return;
408 }
409
410 const int threads
411#if !defined(_OPENMP) || !_OPENMP
412 = 1;
413#else
415#endif
416
417 printFlowTrailer(mpi_size_, threads, total_setup_time_, deck_read_time_, report);
418
419 detail::handleExtraConvergenceOutput(report,
420 Parameters::Get<Parameters::OutputExtraConvergenceInfo>(),
421 R"(OutputExtraConvergenceInfo (--output-extra-convergence-info))",
422 eclState().getIOConfig().getOutputDir(),
423 eclState().getIOConfig().getBaseName());
424 }
425
426 // Run the simulator.
427 int runSimulatorInitOrRun_(int (FlowMain::* initOrRunFunc)())
428 {
429
430 const auto& schedule = this->schedule();
431 auto& ioConfig = eclState().getIOConfig();
432 simtimer_ = std::make_unique<SimulatorTimer>();
433
434 // initialize variables
435 const auto& initConfig = eclState().getInitConfig();
436 simtimer_->init(schedule, static_cast<std::size_t>(initConfig.getRestartStep()));
437
438 if (this->output_cout_) {
439 std::ostringstream oss;
440
441 // This allows a user to catch typos and misunderstandings in the
442 // use of simulator parameters.
443 if (Parameters::printUnused(oss)) {
444 std::cout << "----------------- Unrecognized parameters: -----------------\n";
445 std::cout << oss.str();
446 std::cout << "----------------------------------------------------------------" << std::endl;
447 }
448 }
449
450 if (!ioConfig.initOnly()) {
451 if (this->output_cout_) {
452 std::string msg;
453 msg = "\n\n================ Starting main simulation loop ===============\n";
454 OpmLog::info(msg);
455 }
456
457 return (this->*initOrRunFunc)();
458 }
459 else {
460 if (this->output_cout_) {
461 std::cout << "\n\n================ Simulation turned off ===============\n" << std::flush;
462 }
463 return EXIT_SUCCESS;
464 }
465 }
466
467 protected:
468
470 // Create simulator instance.
471 // Writes to:
472 // simulator_
474 {
475 // Create the simulator instance.
476 simulator_ = std::make_unique<Simulator>(*modelSimulator_);
477 }
478
479 Grid& grid()
480 { return modelSimulator_->vanguard().grid(); }
481
482 private:
483 std::unique_ptr<ModelSimulator> modelSimulator_;
484 int mpi_rank_ = 0;
485 int mpi_size_ = 1;
486 std::any parallel_information_;
487 std::unique_ptr<Simulator> simulator_;
488 std::unique_ptr<SimulatorTimer> simtimer_;
489 int argc_;
490 char **argv_;
491 bool output_cout_;
492 bool output_files_;
493 double total_setup_time_ = 0.0;
494 double deck_read_time_ = 0.0;
495 };
496
497} // namespace Opm
498
499#endif // OPM_FLOW_MAIN_HEADER_INCLUDED
static Parallel::Communication & comm()
Obtain global communicator.
Definition FlowGenericVanguard.hpp:332
Definition FlowMain.hpp:68
double getPreviousReportStepSize()
Get the size of the previous report step.
Definition FlowMain.hpp:207
int execute()
This is the main function of Flow.
Definition FlowMain.hpp:170
void createSimulator()
This is the main function of Flow.
Definition FlowMain.hpp:473
a simulator for the blackoil model
Definition SimulatorFullyImplicitBlackoil.hpp:93
This file contains a set of helper functions used by VFPProd / VFPInj.
Definition blackoilboundaryratevector.hh:37
constexpr auto getPropValue()
get the value data member of a property
Definition propertysystem.hh:242
void reportNlddStatistics(const std::vector< SimulatorReport > &domain_reports, const SimulatorReport &local_report, const bool output_cout, const Parallel::Communication &comm)
Reports NLDD statistics after simulation.
Definition NlddReporting.cpp:44
typename Properties::Detail::GetPropImpl< TypeTag, Property >::type::type GetPropType
get the type alias defined in the property (equivalent to old macro GET_PROP_TYPE(....
Definition propertysystem.hh:235
typename Properties::Detail::GetPropImpl< TypeTag, Property >::type GetProp
get the type of a property (equivalent to old macro GET_PROP(...))
Definition propertysystem.hh:226
Provides convenience routines to bring up the simulation at runtime.
Definition FlowMain.hpp:57