From 2cfce547bdb7b002a8c58f09188e1d756266d9de Mon Sep 17 00:00:00 2001 From: Kengo Tomida Date: Sun, 19 Nov 2023 15:01:06 +0900 Subject: [PATCH 01/33] Added a flag to control red-black iteration or not. Also removed unused flags. --- src/gravity/gravity.cpp | 2 +- src/gravity/gravity.hpp | 2 - src/gravity/mg_gravity.cpp | 1 + src/multigrid/multigrid.hpp | 2 +- src/multigrid/multigrid_driver.cpp | 46 ++++++----- src/task_list/mg_task_list.cpp | 120 +++++++++++++++++------------ 6 files changed, 101 insertions(+), 72 deletions(-) diff --git a/src/gravity/gravity.cpp b/src/gravity/gravity.cpp index 52279b8c0c..20c7ab0f14 100644 --- a/src/gravity/gravity.cpp +++ b/src/gravity/gravity.cpp @@ -35,7 +35,7 @@ //! \brief Gravity constructor Gravity::Gravity(MeshBlock *pmb, ParameterInput *pin) : pmy_block(pmb), phi(pmb->ncells3, pmb->ncells2, pmb->ncells1), - coarse_phi(NHYDRO, pmb->ncc3, pmb->ncc2, pmb->ncc1, + coarse_phi(pmb->ncc3, pmb->ncc2, pmb->ncc1, (pmb->pmy_mesh->multilevel ? AthenaArray::DataStatus::allocated : AthenaArray::DataStatus::empty)), empty_flux{AthenaArray(), AthenaArray(), AthenaArray()}, diff --git a/src/gravity/gravity.hpp b/src/gravity/gravity.hpp index cd9876d7af..f8a75901db 100644 --- a/src/gravity/gravity.hpp +++ b/src/gravity/gravity.hpp @@ -54,8 +54,6 @@ class Gravity { private: MGGravity *pmg; - bool gravity_tensor_momentum_; - bool gravity_tensor_energy_; AthenaArray fbuf_[6]; }; diff --git a/src/gravity/mg_gravity.cpp b/src/gravity/mg_gravity.cpp index 7e606a0582..db7a2d69db 100644 --- a/src/gravity/mg_gravity.cpp +++ b/src/gravity/mg_gravity.cpp @@ -45,6 +45,7 @@ MGGravityDriver::MGGravityDriver(Mesh *pm, ParameterInput *pin) eps_ = pin->GetOrAddReal("gravity", "threshold", -1.0); niter_ = pin->GetOrAddInteger("gravity", "niteration", -1); ffas_ = pin->GetOrAddBoolean("gravity", "fas", ffas_); + redblack_ = true; std::string m = pin->GetOrAddString("gravity", "mgmode", "none"); std::transform(m.begin(), m.end(), m.begin(), ::tolower); if (m == "fmg") { diff --git a/src/multigrid/multigrid.hpp b/src/multigrid/multigrid.hpp index 5edf40be5c..f69368871b 100644 --- a/src/multigrid/multigrid.hpp +++ b/src/multigrid/multigrid.hpp @@ -236,7 +236,7 @@ class MultigridDriver { Mesh *pmy_mesh_; std::vector vmg_; Multigrid *mgroot_; - bool fsubtract_average_, ffas_, needinit_; + bool fsubtract_average_, ffas_, redblack_, needinit_; Real last_ave_; Real eps_; int niter_; diff --git a/src/multigrid/multigrid_driver.cpp b/src/multigrid/multigrid_driver.cpp index 25447f2b46..0b44c14b02 100644 --- a/src/multigrid/multigrid_driver.cpp +++ b/src/multigrid/multigrid_driver.cpp @@ -43,9 +43,9 @@ MultigridDriver::MultigridDriver(Mesh *pm, MGBoundaryFunc *MGBoundary, nvar_(invar), mode_(0), // 0: FMG V(1,1) + iterative, 1: V(1,1) iterative maxreflevel_(pm->multilevel?pm->max_level-pm->root_level:0), nrbx1_(pm->nrbx1), nrbx2_(pm->nrbx2), nrbx3_(pm->nrbx3), srcmask_(MGSourceMask), - pmy_mesh_(pm), fsubtract_average_(false), ffas_(pm->multilevel), needinit_(true), - eps_(-1.0), niter_(-1), coffset_(0), mporder_(-1), nmpcoeff_(0), mpo_(3), - autompo_(false), nodipole_(false), nb_rank_(0) { + pmy_mesh_(pm), fsubtract_average_(false), ffas_(pm->multilevel), + redblack_(true), needinit_(true), eps_(-1.0), niter_(-1), coffset_(0), mporder_(-1), + nmpcoeff_(0), mpo_(3), autompo_(false), nodipole_(false), nb_rank_(0) { std::cout << std::scientific << std::setprecision(15); if (pmy_mesh_->mesh_size.nx2==1 || pmy_mesh_->mesh_size.nx3==1) { @@ -604,9 +604,11 @@ void MultigridDriver::OneStepToFiner(int nsmooth) { current_level_++; for (int n = 0; n < nsmooth; ++n) { SetBoundariesOctets(false, false); - SmoothOctets(0); - SetBoundariesOctets(false, false); - SmoothOctets(1); + SmoothOctets(coffset_); + if (redblack_) { + SetBoundariesOctets(false, false); + SmoothOctets(1-coffset_); + } } } else { // root grid mgroot_->pmgbval->ApplyPhysicalBoundaries(); @@ -614,9 +616,11 @@ void MultigridDriver::OneStepToFiner(int nsmooth) { current_level_++; for (int n = 0; n < nsmooth; ++n) { mgroot_->pmgbval->ApplyPhysicalBoundaries(); - mgroot_->SmoothBlock(0); - mgroot_->pmgbval->ApplyPhysicalBoundaries(); - mgroot_->SmoothBlock(1); + mgroot_->SmoothBlock(coffset_); + if (redblack_) { + mgroot_->pmgbval->ApplyPhysicalBoundaries(); + mgroot_->SmoothBlock(1-coffset_); + } } } @@ -648,10 +652,12 @@ void MultigridDriver::OneStepToCoarser(int nsmooth) { CalculateFASRHSOctets(); } for (int n=0; nCalculateFASRHSBlock(); } for (int n = 0; n < nsmooth; ++n) { - mgroot_->SmoothBlock(0); - mgroot_->pmgbval->ApplyPhysicalBoundaries(); - mgroot_->SmoothBlock(1); + mgroot_->SmoothBlock(coffset_); mgroot_->pmgbval->ApplyPhysicalBoundaries(); + if (redblack_) { + mgroot_->SmoothBlock(1-coffset_); + mgroot_->pmgbval->ApplyPhysicalBoundaries(); + } } mgroot_->RestrictBlock(); } @@ -802,10 +810,12 @@ void MultigridDriver::SolveCoarsestGrid() { mgroot_->CalculateFASRHSBlock(); } for (int i = 0; i < ni; ++i) { // iterate ni times - mgroot_->SmoothBlock(0); - mgroot_->pmgbval->ApplyPhysicalBoundaries(); - mgroot_->SmoothBlock(1); + mgroot_->SmoothBlock(coffset_); mgroot_->pmgbval->ApplyPhysicalBoundaries(); + if (redblack_) { + mgroot_->SmoothBlock(1-coffset_); + mgroot_->pmgbval->ApplyPhysicalBoundaries(); + } } if (fsubtract_average_) { Real vol=(mgroot_->size_.x1max-mgroot_->size_.x1min) diff --git a/src/task_list/mg_task_list.cpp b/src/task_list/mg_task_list.cpp index 1c87cc8e4d..b283eda360 100644 --- a/src/task_list/mg_task_list.cpp +++ b/src/task_list/mg_task_list.cpp @@ -318,18 +318,22 @@ void MultigridTaskList::SetMGTaskListToFiner(int nsmooth, int ngh, int flag) { AddMultigridTask(MG_PHYSBND1R, MG_SENDBND1R|MG_RECVBND1R); } AddMultigridTask(MG_SMOOTH1R, MG_PHYSBND1R); - AddMultigridTask(MG_CLEARBND1R, MG_SMOOTH1R); - AddMultigridTask(MG_STARTRECV1B, MG_CLEARBND1R); - AddMultigridTask(MG_SENDBND1B, MG_STARTRECV1B); - AddMultigridTask(MG_RECVBND1B, MG_STARTRECV1B); - if (multilevel) { - AddMultigridTask(MG_PRLNGFC1B, MG_SENDBND1B|MG_RECVBND1B); - AddMultigridTask(MG_PHYSBND1B, MG_PRLNGFC1B); + if (pmy_mgdriver_->redblack_) { + AddMultigridTask(MG_CLEARBND1R, MG_SMOOTH1R); + AddMultigridTask(MG_STARTRECV1B, MG_CLEARBND1R); + AddMultigridTask(MG_SENDBND1B, MG_STARTRECV1B); + AddMultigridTask(MG_RECVBND1B, MG_STARTRECV1B); + if (multilevel) { + AddMultigridTask(MG_PRLNGFC1B, MG_SENDBND1B|MG_RECVBND1B); + AddMultigridTask(MG_PHYSBND1B, MG_PRLNGFC1B); + } else { + AddMultigridTask(MG_PHYSBND1B, MG_SENDBND1B|MG_RECVBND1B); + } + AddMultigridTask(MG_SMOOTH1B, MG_PHYSBND1B); + AddMultigridTask(MG_CLEARBND1B, MG_SMOOTH1B); } else { - AddMultigridTask(MG_PHYSBND1B, MG_SENDBND1B|MG_RECVBND1B); + AddMultigridTask(MG_CLEARBND1B, MG_SMOOTH1R); } - AddMultigridTask(MG_SMOOTH1B, MG_PHYSBND1B); - AddMultigridTask(MG_CLEARBND1B, MG_SMOOTH1B); } if (nsmooth==2) { AddMultigridTask(MG_STARTRECV2R, MG_CLEARBND1B); @@ -342,18 +346,22 @@ void MultigridTaskList::SetMGTaskListToFiner(int nsmooth, int ngh, int flag) { AddMultigridTask(MG_PHYSBND2R, MG_SENDBND2R|MG_RECVBND2R); } AddMultigridTask(MG_SMOOTH2R, MG_PHYSBND2R); - AddMultigridTask(MG_CLEARBND2R, MG_SMOOTH2R); - AddMultigridTask(MG_STARTRECV2B, MG_CLEARBND2R); - AddMultigridTask(MG_SENDBND2B, MG_STARTRECV2B); - AddMultigridTask(MG_RECVBND2B, MG_STARTRECV2B); - if (multilevel) { - AddMultigridTask(MG_PRLNGFC2B, MG_SENDBND2B|MG_RECVBND2B); - AddMultigridTask(MG_PHYSBND2B, MG_PRLNGFC2B); + if (pmy_mgdriver_->redblack_) { + AddMultigridTask(MG_CLEARBND2R, MG_SMOOTH2R); + AddMultigridTask(MG_STARTRECV2B, MG_CLEARBND2R); + AddMultigridTask(MG_SENDBND2B, MG_STARTRECV2B); + AddMultigridTask(MG_RECVBND2B, MG_STARTRECV2B); + if (multilevel) { + AddMultigridTask(MG_PRLNGFC2B, MG_SENDBND2B|MG_RECVBND2B); + AddMultigridTask(MG_PHYSBND2B, MG_PRLNGFC2B); + } else { + AddMultigridTask(MG_PHYSBND2B, MG_SENDBND2B|MG_RECVBND2B); + } + AddMultigridTask(MG_SMOOTH2B, MG_PHYSBND2B); + AddMultigridTask(MG_CLEARBND2B, MG_SMOOTH2B); } else { - AddMultigridTask(MG_PHYSBND2B, MG_SENDBND2B|MG_RECVBND2B); + AddMultigridTask(MG_CLEARBND2B, MG_SMOOTH2R); } - AddMultigridTask(MG_SMOOTH2B, MG_PHYSBND2B); - AddMultigridTask(MG_CLEARBND2B, MG_SMOOTH2B); } if (flag==2) { // last if (nsmooth==0) @@ -417,18 +425,22 @@ void MultigridTaskList::SetMGTaskListToCoarser(int nsmooth, int ngh) { } else { AddMultigridTask(MG_SMOOTH1R, MG_PHYSBND1R); } - AddMultigridTask(MG_CLEARBND1R, MG_SMOOTH1R); - AddMultigridTask(MG_STARTRECV1B, MG_CLEARBND1R); - AddMultigridTask(MG_SENDBND1B, MG_STARTRECV1B); - AddMultigridTask(MG_RECVBND1B, MG_STARTRECV1B); - if (multilevel) { - AddMultigridTask(MG_PRLNGFC1B, MG_SENDBND1B|MG_RECVBND1B); - AddMultigridTask(MG_PHYSBND1B, MG_PRLNGFC1B); + if (pmy_mgdriver_->redblack_) { + AddMultigridTask(MG_CLEARBND1R, MG_SMOOTH1R); + AddMultigridTask(MG_STARTRECV1B, MG_CLEARBND1R); + AddMultigridTask(MG_SENDBND1B, MG_STARTRECV1B); + AddMultigridTask(MG_RECVBND1B, MG_STARTRECV1B); + if (multilevel) { + AddMultigridTask(MG_PRLNGFC1B, MG_SENDBND1B|MG_RECVBND1B); + AddMultigridTask(MG_PHYSBND1B, MG_PRLNGFC1B); + } else { + AddMultigridTask(MG_PHYSBND1B, MG_SENDBND1B|MG_RECVBND1B); + } + AddMultigridTask(MG_SMOOTH1B, MG_PHYSBND1B); + AddMultigridTask(MG_CLEARBND1B, MG_SMOOTH1B); } else { - AddMultigridTask(MG_PHYSBND1B, MG_SENDBND1B|MG_RECVBND1B); + AddMultigridTask(MG_CLEARBND1B, MG_SMOOTH1R); } - AddMultigridTask(MG_SMOOTH1B, MG_PHYSBND1B); - AddMultigridTask(MG_CLEARBND1B, MG_SMOOTH1B); AddMultigridTask(MG_STARTRECV0 , MG_CLEARBND1B); AddMultigridTask(MG_SENDBND0, MG_STARTRECV0); @@ -457,18 +469,22 @@ void MultigridTaskList::SetMGTaskListToCoarser(int nsmooth, int ngh) { } else { AddMultigridTask(MG_SMOOTH1R, MG_PHYSBND1R); } - AddMultigridTask(MG_CLEARBND1R, MG_SMOOTH1R); - AddMultigridTask(MG_STARTRECV1B, MG_CLEARBND1R); - AddMultigridTask(MG_SENDBND1B, MG_STARTRECV1B); - AddMultigridTask(MG_RECVBND1B, MG_STARTRECV1B); - if (multilevel) { - AddMultigridTask(MG_PRLNGFC1B, MG_SENDBND1B|MG_RECVBND1B); - AddMultigridTask(MG_PHYSBND1B, MG_PRLNGFC1B); + if (pmy_mgdriver_->redblack_) { + AddMultigridTask(MG_CLEARBND1R, MG_SMOOTH1R); + AddMultigridTask(MG_STARTRECV1B, MG_CLEARBND1R); + AddMultigridTask(MG_SENDBND1B, MG_STARTRECV1B); + AddMultigridTask(MG_RECVBND1B, MG_STARTRECV1B); + if (multilevel) { + AddMultigridTask(MG_PRLNGFC1B, MG_SENDBND1B|MG_RECVBND1B); + AddMultigridTask(MG_PHYSBND1B, MG_PRLNGFC1B); + } else { + AddMultigridTask(MG_PHYSBND1B, MG_SENDBND1B|MG_RECVBND1B); + } + AddMultigridTask(MG_SMOOTH1B, MG_PHYSBND1B); + AddMultigridTask(MG_CLEARBND1B, MG_SMOOTH1B); } else { - AddMultigridTask(MG_PHYSBND1B, MG_SENDBND1B|MG_RECVBND1B); + AddMultigridTask(MG_CLEARBND1B, MG_SMOOTH1R); } - AddMultigridTask(MG_SMOOTH1B, MG_PHYSBND1B); - AddMultigridTask(MG_CLEARBND1B, MG_SMOOTH1B); AddMultigridTask(MG_STARTRECV2R, MG_CLEARBND1B); AddMultigridTask(MG_SENDBND2R, MG_STARTRECV2R); AddMultigridTask(MG_RECVBND2R, MG_STARTRECV2R); @@ -479,18 +495,22 @@ void MultigridTaskList::SetMGTaskListToCoarser(int nsmooth, int ngh) { AddMultigridTask(MG_PHYSBND2R, MG_SENDBND2R|MG_RECVBND2R); } AddMultigridTask(MG_SMOOTH2R, MG_PHYSBND2R); - AddMultigridTask(MG_CLEARBND2R, MG_SMOOTH2R); - AddMultigridTask(MG_STARTRECV2B, MG_CLEARBND2R); - AddMultigridTask(MG_SENDBND2B, MG_STARTRECV2B); - AddMultigridTask(MG_RECVBND2B, MG_STARTRECV2B); - if (multilevel) { - AddMultigridTask(MG_PRLNGFC2B, MG_SENDBND2B|MG_RECVBND2B); - AddMultigridTask(MG_PHYSBND2B, MG_PRLNGFC2B); + if (pmy_mgdriver_->redblack_) { + AddMultigridTask(MG_CLEARBND2R, MG_SMOOTH2R); + AddMultigridTask(MG_STARTRECV2B, MG_CLEARBND2R); + AddMultigridTask(MG_SENDBND2B, MG_STARTRECV2B); + AddMultigridTask(MG_RECVBND2B, MG_STARTRECV2B); + if (multilevel) { + AddMultigridTask(MG_PRLNGFC2B, MG_SENDBND2B|MG_RECVBND2B); + AddMultigridTask(MG_PHYSBND2B, MG_PRLNGFC2B); + } else { + AddMultigridTask(MG_PHYSBND2B, MG_SENDBND2B|MG_RECVBND2B); + } + AddMultigridTask(MG_SMOOTH2B, MG_PHYSBND2B); + AddMultigridTask(MG_CLEARBND2B, MG_SMOOTH2B); } else { - AddMultigridTask(MG_PHYSBND2B, MG_SENDBND2B|MG_RECVBND2B); + AddMultigridTask(MG_CLEARBND2B, MG_SMOOTH2R); } - AddMultigridTask(MG_SMOOTH2B, MG_PHYSBND2B); - AddMultigridTask(MG_CLEARBND2B, MG_SMOOTH2B); AddMultigridTask(MG_STARTRECV0, MG_CLEARBND2B); AddMultigridTask(MG_SENDBND0, MG_STARTRECV0); AddMultigridTask(MG_RECVBND0, MG_STARTRECV0); From bf23edce068d39240840a41c6949ec3e8d2360fa Mon Sep 17 00:00:00 2001 From: Kengo Tomida Date: Mon, 20 Nov 2023 23:50:56 +0900 Subject: [PATCH 02/33] **incomplete commit** implementing MG CR Diffusion --- Makefile.in | 1 + configure.py | 14 + src/athena.hpp | 4 +- src/bvals/bvals.hpp | 17 +- ...ne_gravity.cpp => bvals_refine_postmg.cpp} | 40 +- src/bvals/cc/mg/bvals_mg.cpp | 356 ++-------------- src/bvals/cc/mg/bvals_mg.hpp | 14 +- src/bvals/cc/mg/bvals_mg_gravity.cpp | 393 ++++++++++++++++++ src/crdiffusion/crdiffusion.cpp | 371 +++++++++++++++++ src/crdiffusion/crdiffusion.hpp | 63 +++ src/crdiffusion/mg_crdiffusion.cpp | 318 ++++++++++++++ src/crdiffusion/mg_crdiffusion.hpp | 69 +++ src/defs.hpp.in | 3 + src/gravity/mg_gravity.cpp | 4 +- src/mesh/mesh.cpp | 10 +- src/mesh/mesh.hpp | 5 + src/mesh/meshblock.cpp | 21 + src/multigrid/multigrid.hpp | 8 +- src/multigrid/multigrid_driver.cpp | 14 + src/outputs/athena_hdf5.cpp | 2 + src/outputs/outputs.cpp | 24 ++ src/outputs/restart.cpp | 6 + src/pgen/cr_diffusion_mg.cpp | 121 ++++++ src/task_list/crdiffusion_task_list.cpp | 125 ++++++ src/task_list/crdiffusion_task_list.hpp | 58 +++ src/task_list/grav_task_list.cpp | 7 +- src/utils/show_config.cpp | 5 + 27 files changed, 1703 insertions(+), 370 deletions(-) rename src/bvals/{bvals_refine_gravity.cpp => bvals_refine_postmg.cpp} (75%) create mode 100644 src/bvals/cc/mg/bvals_mg_gravity.cpp create mode 100644 src/crdiffusion/crdiffusion.cpp create mode 100644 src/crdiffusion/crdiffusion.hpp create mode 100644 src/crdiffusion/mg_crdiffusion.cpp create mode 100644 src/crdiffusion/mg_crdiffusion.hpp create mode 100644 src/pgen/cr_diffusion_mg.cpp create mode 100644 src/task_list/crdiffusion_task_list.cpp create mode 100644 src/task_list/crdiffusion_task_list.hpp diff --git a/Makefile.in b/Makefile.in index b8730a4df3..31bbdc6465 100644 --- a/Makefile.in +++ b/Makefile.in @@ -52,6 +52,7 @@ SRC_FILES := $(wildcard src/*.cpp) \ $(wildcard src/nr_radiation/implicit/*.cpp) \ $(wildcard src/cr/*.cpp) \ $(wildcard src/cr/integrators/*.cpp) \ + $(wildcard src/crdiffusion/*.cpp) \ src/hydro/rsolvers/$(RSOLVER_DIR)$(RSOLVER_FILE) \ $(wildcard src/inputs/*.cpp) \ $(wildcard src/mesh/*.cpp) \ diff --git a/configure.py b/configure.py index bbf88bf9c9..07fa73c7b0 100755 --- a/configure.py +++ b/configure.py @@ -40,6 +40,7 @@ # -nr_radiation turn on non-relativistic radiation transport # -implicit_radiation implicit radiation transport module # -cr enable cosmic ray transport +# -crdiff enable cosmic ray transport # ---------------------------------------------------------------------------------------- # Modules @@ -226,6 +227,12 @@ default=False, help='enable cosmic ray transport') +# -cosmic ray diffusion argument +parser.add_argument('-crdiff', + action='store_true', + default=False, + help='enable implicit cosmic ray diffusion') + # The main choices for --cxx flag, using "ctype[-suffix]" formatting, where "ctype" is the # major family/suite/group of compilers and "suffix" may represent variants of the # compiler version and/or predefined sets of compiler options. The C++ compiler front ends @@ -497,6 +504,12 @@ def c_to_cpp(arg): else: definitions['CR_ENABLED'] = '0' +# -crdiff argument +if args['crdiff']: + definitions['CRDIFFUSION_ENABLED'] = '1' +else: + definitions['CRDIFFUSION_ENABLED'] = '0' + # --cxx=[name] argument if args['cxx'] == 'g++': @@ -893,6 +906,7 @@ def output_config(opt_descr, opt_choice, filehandle=None): output_config('Radiative Transfer', ('ON' if args['nr_radiation'] else 'OFF'), flog) output_config('Implicit Radiation', ('ON' if args['implicit_radiation'] else 'OFF'), flog) output_config('Cosmic Ray Transport', ('ON' if args['cr'] else 'OFF'), flog) +output_config('Cosmic Ray Diffusion', ('ON' if args['crdiff'] else 'OFF'), flog) output_config('Frame transformations', ('ON' if args['t'] else 'OFF'), flog) output_config('Self-Gravity', self_grav_string, flog) output_config('Super-Time-Stepping', ('ON' if args['sts'] else 'OFF'), flog) diff --git a/src/athena.hpp b/src/athena.hpp index 28bfae80c8..53f6719e03 100644 --- a/src/athena.hpp +++ b/src/athena.hpp @@ -166,8 +166,8 @@ enum CoordinateDirection {X1DIR=0, X2DIR=1, X3DIR=2}; // KGF: Except for the 2x MG* enums, these may be unnessary w/ the new class inheritance // Now, only passed to BoundaryVariable::InitBoundaryData(); could replace w/ bool switch // TODO(tomo-ono): consider necessity of orbita_cc and orbital_fc -enum class BoundaryQuantity {cc, fc, cc_flcor, fc_flcor, mggrav, - mggrav_f, orbital_cc, orbital_fc}; +enum class BoundaryQuantity {cc, fc, cc_flcor, fc_flcor, mg, mg_faceonly, + orbital_cc, orbital_fc}; enum class HydroBoundaryQuantity {cons, prim}; enum class BoundaryCommSubset {mesh_init, gr_amr, all, orbital, radiation, radhydro}; // TODO(felker): consider generalizing/renaming to QuantityFormulation diff --git a/src/bvals/bvals.hpp b/src/bvals/bvals.hpp index fef2db1e81..c0d4822e6a 100644 --- a/src/bvals/bvals.hpp +++ b/src/bvals/bvals.hpp @@ -124,8 +124,8 @@ class BoundaryValues : public BoundaryBase, //public BoundaryPhysics, std::vector bvars_main_int; //! subset of bvars that are exchanged in the SuperTimeStepTaskList std::vector bvars_sts; - //! Pointer to the Gravity Boundary Variable - CellCenteredBoundaryVariable *pgbvar; + //! Pointer to the Gravity and CRDiffusion Boundary Variable + CellCenteredBoundaryVariable *pgbvar, *pcrbvar; // inherited functions (interface shared with BoundaryVariable objects): // ------ @@ -150,8 +150,8 @@ class BoundaryValues : public BoundaryBase, //public BoundaryPhysics, void ProlongateBoundaries(const Real time, const Real dt, std::vector bvars_subset); - // temporary workaround for self-gravity - void ProlongateGravityBoundaries(const Real time, const Real dt); + // temporary workaround for Multigrid + void ProlongateBoundariesPostMG(CellCenteredBoundaryVariable* pbvar); // compute the shear at each integrator stage //! \todo (felker): @@ -212,10 +212,11 @@ class BoundaryValues : public BoundaryBase, //public BoundaryPhysics, void ProlongateGhostCells(const NeighborBlock& nb, int si, int ei, int sj, int ej, int sk, int ek); - // temporary workaround for self-gravity - void RestrictGravityGhostCellsOnSameLevel(const NeighborBlock& nb, - int nk, int nj, int ni); - void ProlongateGravityGhostCells(int si, int ei, int sj, int ej, int sk, int ek); + // temporary workaround for Multigrid + void RestrictGhostCellsOnSameLevelPostMG(CellCenteredBoundaryVariable* pbvar, + const NeighborBlock& nb, int nk, int nj, int ni); + void ProlongateGhostCellsPostMG(CellCenteredBoundaryVariable* pbvar, + int si, int ei, int sj, int ej, int sk, int ek); void DispatchBoundaryFunctions( MeshBlock *pmb, Coordinates *pco, Real time, Real dt, diff --git a/src/bvals/bvals_refine_gravity.cpp b/src/bvals/bvals_refine_postmg.cpp similarity index 75% rename from src/bvals/bvals_refine_gravity.cpp rename to src/bvals/bvals_refine_postmg.cpp index ccc11f53a8..c0149a3268 100644 --- a/src/bvals/bvals_refine_gravity.cpp +++ b/src/bvals/bvals_refine_postmg.cpp @@ -24,13 +24,14 @@ #include "bvals.hpp" -class CellCenteredBoundaryVariable; +class CellCenteredCellCenteredBoundaryVariable; //---------------------------------------------------------------------------------------- -//! \fn void BoundaryValues::ProlongateGravityBoundaries(const Real time, const Real dt) -//! \brief Prolongate boundaries +//! \fn void BoundaryValues::ProlongateBoundariesPostMG( +//! CellCenteredBoundaryVariable* pbvar) +//! \brief Prolongate boundaries after Multigrid assuming physical boundaries are filled -void BoundaryValues::ProlongateGravityBoundaries(const Real time, const Real dt) { +void BoundaryValues::ProlongateBoundariesPostMG(CellCenteredBoundaryVariable* pbvar) { MeshBlock *pmb = pmy_block_; int &mylevel = loc.level; @@ -69,7 +70,7 @@ void BoundaryValues::ProlongateGravityBoundaries(const Real time, const Real dt) // this neighbor block is on the same level // and needs to be restricted for prolongation - RestrictGravityGhostCellsOnSameLevel(nb, nk, nj, ni); + RestrictGhostCellsOnSameLevelPostMG(pbvar, nb, nk, nj, ni); } } } @@ -108,18 +109,20 @@ void BoundaryValues::ProlongateGravityBoundaries(const Real time, const Real dt) // Step 3. Finally, the ghost-ghost zones are ready for prolongation: - ProlongateGravityGhostCells(si, ei, sj, ej, sk, ek); + ProlongateGhostCellsPostMG(pbvar, si, ei, sj, ej, sk, ek); } // end loop over nneighbor return; } //---------------------------------------------------------------------------------------- -//! \fn void BoundaryValues::RestrictGravityGhostCellsOnSameLevel(const NeighborBlock& nb, -//! int nk, int nj, int ni) -//! \brief Restrict ghost cells on same level -void BoundaryValues::RestrictGravityGhostCellsOnSameLevel(const NeighborBlock& nb, - int nk, int nj, int ni) { +//! \fn void BoundaryValues::RestrictGhostCellsOnSameLevelPostMG( +//! CellCenteredBoundaryVariable* pbvar, const NeighborBlock& nb, +//! int nk, int nj, int ni) +//! \brief Restrict ghost cells on same level after Multigrid +void BoundaryValues::RestrictGhostCellsOnSameLevelPostMG( + CellCenteredBoundaryVariable* pbvar, const NeighborBlock& nb, + int nk, int nj, int ni) { MeshBlock *pmb = pmy_block_; MeshRefinement *pmr = pmb->pmr; @@ -152,7 +155,7 @@ void BoundaryValues::RestrictGravityGhostCellsOnSameLevel(const NeighborBlock& n rks = pmb->cks - 1, rke = pmb->cks - 1; } - pmb->pmr->RestrictCellCenteredValues(*(pgbvar->var_cc), *(pgbvar->coarse_buf), 0, 0, + pmb->pmr->RestrictCellCenteredValues(*(pbvar->var_cc), *(pbvar->coarse_buf), 0, 0, ris, rie, rjs, rje, rks, rke); return; @@ -160,14 +163,15 @@ void BoundaryValues::RestrictGravityGhostCellsOnSameLevel(const NeighborBlock& n //---------------------------------------------------------------------------------------- -//! \fn void BoundaryValues::ProlongateGravityGhostCells(int si, int ei, int sj, int ej, -//! int sk, int ek) -//! \brief Prolongate gravity ghost cells +//! \fn void BoundaryValues::ProlongateGhostCellsPostMG( +//! CellCenteredBoundaryVariable* pbvar, +//! int si, int ei, int sj, int ej, int sk, int ek) +//! \brief Prolongate ghost cells after Multigrid -void BoundaryValues::ProlongateGravityGhostCells(int si, int ei, int sj, int ej, - int sk, int ek) { +void BoundaryValues::ProlongateGhostCellsPostMG(CellCenteredBoundaryVariable* pbvar, + int si, int ei, int sj, int ej, int sk, int ek) { MeshRefinement *pmr = pmy_block_->pmr; - pmr->ProlongateCellCenteredValues(*(pgbvar->coarse_buf), *(pgbvar->var_cc), 0, 0, + pmr->ProlongateCellCenteredValues(*(pbvar->coarse_buf), *(pbvar->var_cc), 0, 0, si, ei, sj, ej, sk, ek); return; } diff --git a/src/bvals/cc/mg/bvals_mg.cpp b/src/bvals/cc/mg/bvals_mg.cpp index 09b31c1b04..d514d12312 100644 --- a/src/bvals/cc/mg/bvals_mg.cpp +++ b/src/bvals/cc/mg/bvals_mg.cpp @@ -51,7 +51,7 @@ MGBoundaryValues::MGBoundaryValues(Multigrid *pmg, BoundaryFlag *input_bcs) for (int i = 0; i < 6; ++i) MGBoundaryFunction_[i] = pmg->pmy_driver_->MGBoundaryFunction_[i]; if (pmy_mg_->pmy_block_ != nullptr) { - InitBoundaryData(BoundaryQuantity::mggrav); + InitBoundaryData(BoundaryQuantity::mg); int nc = block_size_.nx1 + 2*pmy_mg_->ngh_; cbuf_.NewAthenaArray(pmy_mg_->nvar_, nc, nc, nc); cbufold_.NewAthenaArray(pmy_mg_->nvar_, nc, nc, nc); @@ -98,7 +98,7 @@ void MGBoundaryValues::InitBoundaryData(BoundaryQuantity type) { // Allocate buffers // calculate the buffer size switch (type) { - case BoundaryQuantity::mggrav: { + case BoundaryQuantity::mg: { int ngh = pmy_mg_->ngh_; int nc = block_size_.nx1; if (BoundaryValues::ni[n].type == NeighborConnect::face) @@ -384,7 +384,7 @@ void MGBoundaryValues::StartReceivingMultigrid(BoundaryQuantity type, bool foldd for (int n = 0; n < nneighbor; ++n) { NeighborBlock& nb = neighbor[n]; if (nb.snb.rank!=Globals::my_rank) { - if (type == BoundaryQuantity::mggrav_f && nb.snb.level > loc.level + if (type == BoundaryQuantity::mg_faceonly && nb.snb.level > loc.level && nb.ni.type != NeighborConnect::face) continue; int size = 0; if (nb.snb.level == loc.level) { // same @@ -429,7 +429,7 @@ void MGBoundaryValues::ClearBoundaryMultigrid(BoundaryQuantity type) { bdata_[bcolor_].sflag[nb.bufid] = BoundaryStatus::waiting; #ifdef MPI_PARALLEL if (nb.snb.rank != Globals::my_rank) { - if (!(type == BoundaryQuantity::mggrav_f + if (!(type == BoundaryQuantity::mg_faceonly && nb.ni.type != NeighborConnect::face && nb.snb.level < loc.level)) MPI_Wait(&(bdata_[bcolor_].req_send[nb.bufid]),MPI_STATUS_IGNORE); } @@ -638,7 +638,7 @@ bool MGBoundaryValues::SendMultigridBoundaryBuffers(BoundaryQuantity type, for (int n = 0; n < nneighbor; ++n) { NeighborBlock& nb = neighbor[n]; if (bdata_[bcolor_].sflag[nb.bufid] == BoundaryStatus::completed) continue; - if (type == BoundaryQuantity::mggrav_f && nb.snb.level < loc.level + if (type == BoundaryQuantity::mg_faceonly && nb.snb.level < loc.level && nb.ni.type != NeighborConnect::face) continue; int ssize = 0; if (nb.snb.rank == Globals::my_rank) { @@ -653,14 +653,14 @@ bool MGBoundaryValues::SendMultigridBoundaryBuffers(BoundaryQuantity type, ssize = LoadMultigridBoundaryBufferSameLevel(bdata_[bcolor_].send[nb.bufid], nb, folddata); } else if (nb.snb.level < loc.level) { - if (type == BoundaryQuantity::mggrav_f) + if (type == BoundaryQuantity::mg_faceonly) ssize = LoadMultigridBoundaryBufferToCoarserFluxCons( bdata_[bcolor_].send[nb.bufid], nb); else ssize = LoadMultigridBoundaryBufferToCoarser(bdata_[bcolor_].send[nb.bufid], nb, folddata); } else { - if (type == BoundaryQuantity::mggrav_f) + if (type == BoundaryQuantity::mg_faceonly) ssize = LoadMultigridBoundaryBufferToFinerFluxCons( bdata_[bcolor_].send[nb.bufid], nb); else @@ -870,7 +870,7 @@ bool MGBoundaryValues::ReceiveMultigridBoundaryBuffers(BoundaryQuantity type, for (int n = 0; n < nneighbor; ++n) { NeighborBlock& nb = neighbor[n]; if (bdata_[bcolor_].flag[nb.bufid] == BoundaryStatus::completed) continue; - if (type == BoundaryQuantity::mggrav_f && nb.snb.level > loc.level + if (type == BoundaryQuantity::mg_faceonly && nb.snb.level > loc.level && nb.ni.type != NeighborConnect::face) continue; if (bdata_[bcolor_].flag[nb.bufid] == BoundaryStatus::waiting) { if (nb.snb.rank == Globals::my_rank) {// on the same process @@ -893,12 +893,12 @@ bool MGBoundaryValues::ReceiveMultigridBoundaryBuffers(BoundaryQuantity type, if (nb.snb.level == loc.level) { SetMultigridBoundarySameLevel(bdata_[bcolor_].recv[nb.bufid], nb, folddata); } else if (nb.snb.level < loc.level) { - if (type == BoundaryQuantity::mggrav_f) + if (type == BoundaryQuantity::mg_faceonly) SetMultigridBoundaryFromCoarserFluxCons(bdata_[bcolor_].recv[nb.bufid], nb); else SetMultigridBoundaryFromCoarser(bdata_[bcolor_].recv[nb.bufid], nb, folddata); } else { - if (type == BoundaryQuantity::mggrav_f) + if (type == BoundaryQuantity::mg_faceonly) SetMultigridBoundaryFromFinerFluxCons(bdata_[bcolor_].recv[nb.bufid], nb); else SetMultigridBoundaryFromFiner(bdata_[bcolor_].recv[nb.bufid], nb, folddata); @@ -1076,224 +1076,51 @@ void MGBoundaryValues::ProlongateMultigridBoundaries(bool folddata) { //---------------------------------------------------------------------------------------- -//! \fn int MGGravityBoundaryValues::LoadMultigridBoundaryBufferToCoarserFluxCons( +//! \fn int MGBoundaryValues::LoadMultigridBoundaryBufferToCoarserFluxCons( //! Real *buf, const NeighborBlock& nb) //! \brief Set Multigrid boundary buffers for sending to a block on the coarser level -//! using the Mass Conservation formula for gravity +//! with flux-conservation for the base class (just call the normal version) -int MGGravityBoundaryValues::LoadMultigridBoundaryBufferToCoarserFluxCons(Real *buf, +int MGBoundaryValues::LoadMultigridBoundaryBufferToCoarserFluxCons(Real *buf, const NeighborBlock& nb) { - const AthenaArray &u = pmy_mg_->GetCurrentData(); - int nc = pmy_mg_->GetCurrentNumberOfCells(); - int ngh = pmy_mg_->ngh_, nvar = pmy_mg_->nvar_; - int cn = ngh, fs = ngh, cs = cn, fe = fs + nc - 1, ce = cs + nc/2 - 1; - int p = 0; - - // take averages of four cells contacting the interface and pack - if (nb.ni.ox1 != 0) { // x1 face - int fi; - if (nb.ni.ox1 < 0) fi = fs; - else fi = fe; - for (int v=0; v &u = pmy_mg_->GetCurrentData(); - int nc = pmy_mg_->GetCurrentNumberOfCells(); - int ngh = pmy_mg_->ngh_, nvar = pmy_mg_->nvar_; - int cn = ngh - 1, fs = ngh, fe = fs + nc - 1; - int si = (nb.ni.ox1 > 0) ? (fe - cn) : fs; - int ei = (nb.ni.ox1 < 0) ? (fs + cn) : fe; - int sj = (nb.ni.ox2 > 0) ? (fe - cn) : fs; - int ej = (nb.ni.ox2 < 0) ? (fs + cn) : fe; - int sk = (nb.ni.ox3 > 0) ? (fe - cn) : fs; - int ek = (nb.ni.ox3 < 0) ? (fs + cn) : fe; - - int nred = nc/2 - ngh; - if (nb.ni.ox1 == 0) { - if (nb.ni.fi1 == 1) si += nred; - else ei -= nred; - } - if (nb.ni.ox2 == 0) { - if (nb.ni.ox1 != 0) { - if (nb.ni.fi1 == 1) sj += nred; - else ej -= nred; - } else { - if (nb.ni.fi2 == 1) sj += nred; - else ej -= nred; - } - } - if (nb.ni.ox3 == 0) { - if (nb.ni.ox1 != 0 && nb.ni.ox2 != 0) { - if (nb.ni.fi1 == 1) sk += nred; - else ek -= nred; - } else { - if (nb.ni.fi2 == 1) sk += nred; - else ek -= nred; - } - } +//! with flux-conservation for the base class (just call the normal version) - int p = 0; - BufferUtility::PackData(u, buf, 0, nvar-1, si, ei, sj, ej, sk, ek, p); - - return p; +int MGBoundaryValues::LoadMultigridBoundaryBufferToFinerFluxCons(Real *buf, + const NeighborBlock& nb) { + return LoadMultigridBoundaryBufferToFiner(buf, nb, false); } //---------------------------------------------------------------------------------------- -//! \fn void MGGravityBoundaryValues::SetMultigridBoundaryFromCoarserFluxCons( +//! \fn void MGBoundaryValues::SetMultigridBoundaryFromCoarserFluxCons( //! const Real *buf, const NeighborBlock& nb) -//! \brief Set hydro boundary received from a block on the same level +//! \brief Set Multigrid boundary received from a block from the coarser level +//! with flux-conservation for the base class (just call the normal version) -void MGGravityBoundaryValues::SetMultigridBoundaryFromCoarserFluxCons(const Real *buf, +void MGBoundaryValues::SetMultigridBoundaryFromCoarserFluxCons(const Real *buf, const NeighborBlock& nb) { - int nc = pmy_mg_->GetCurrentNumberOfCells(); - int ngh = pmy_mg_->ngh_, nvar = pmy_mg_->nvar_; - int si, sj, sk, ei, ej, ek; - int cng = 1; - int cs = cng, ce = cng + nc/2 -1; - - if (nb.ni.ox1 == 0) { - si = cs, ei = ce; - if ((loc.lx1 & 1LL) == 0LL) ei += cng; - else si -= cng; - } else if (nb.ni.ox1 > 0) { - si = ce + 1, ei = ce + cng; - } else { - si = cs - cng, ei = cs - 1; - } - if (nb.ni.ox2 == 0) { - sj = cs, ej = ce; - if ((loc.lx2 & 1LL) == 0LL) ej += cng; - else sj -= cng; - } else if (nb.ni.ox2 > 0) { - sj = ce + 1, ej = ce + cng; - } else { - sj = cs - cng, ej = cs - 1; - } - if (nb.ni.ox3 == 0) { - sk = cs, ek = ce; - if ((loc.lx3 & 1LL) == 0LL) ek += cng; - else sk -= cng; - } else if (nb.ni.ox3 > 0) { - sk = ce + 1, ek = ce + cng; - } else { - sk = cs - cng, ek = cs - 1; - } - - int p = 0; - BufferUtility::UnpackData(buf, cbuf_, 0, nvar-1, si, ei, sj, ej, sk, ek, p); - + SetMultigridBoundaryFromCoarser(buf, nb, false); return; } //---------------------------------------------------------------------------------------- -//! \fn void MGGravityBoundaryValues::SetMultigridBoundaryFromFinerFluxCons( +//! \fn void MGBoundaryValues::SetMultigridBoundaryFromFinerFluxCons( //! const Real *buf, const NeighborBlock& nb) +//! \brief Set Multigrid boundary received from a block from the finer level +//! with flux-conservation for the base class (just call the normal version) -void MGGravityBoundaryValues::SetMultigridBoundaryFromFinerFluxCons(const Real *buf, +void MGBoundaryValues::SetMultigridBoundaryFromFinerFluxCons(const Real *buf, const NeighborBlock& nb) { - AthenaArray &dst = pmy_mg_->GetCurrentData(); - int nc = pmy_mg_->GetCurrentNumberOfCells(); - int ngh = pmy_mg_->ngh_, nvar = pmy_mg_->nvar_; - int fs = ngh, fe = fs + nc - 1; - int si, sj, sk, ei, ej, ek; - int oi = 0, oj = 0, ok = 0; - - // could reuse the index calculation for normal boundaries - // but in general this depends on physics - if (nb.ni.ox1 == 0) { - si = fs, ei = fe; - if (nb.ni.fi1 == 1) si += nc/2; - else ei -= nc/2; - } else if (nb.ni.ox1 > 0) { - si = fe + 1, ei = fe + ngh, oi = -1; - } else { - si = fs - ngh, ei = fs - 1, oi = 1; - } - if (nb.ni.ox2 == 0) { - sj = fs, ej = fe; - if (nb.ni.ox1 != 0) { - if (nb.ni.fi1 == 1) sj += nc/2; - else ej -= nc/2; - } else { - if (nb.ni.fi2 == 1) sj += nc/2; - else ej -= nc/2; - } - } else if (nb.ni.ox2 > 0) { - sj = fe + 1, ej = fe + ngh, oj = -1; - } else { - sj = fs - ngh, ej = fs - 1, oj = 1; - } - if (nb.ni.ox3 == 0) { - sk = fs, ek = fe; - if (nb.ni.ox1 != 0 && nb.ni.ox2 != 0) { - if (nb.ni.fi1 == 1) sk += nc/2; - else ek -= nc/2; - } else { - if (nb.ni.fi2 == 1) sk += nc/2; - else ek -= nc/2; - } - } else if (nb.ni.ox3 > 0) { - sk = fe + 1, ek = fe + ngh, ok = -1; - } else { - sk = fs - ngh, ek = fs - 1, ok = 1; - } - - constexpr Real ot = 1.0/3.0; - int p = 0; - - // correct the ghost values using the mass conservation formula - for (int v=0; v &u = pmy_mg_->GetCurrentData(); - AthenaArray &dst = pmy_mg_->GetCurrentData(); - int nc = pmy_mg_->GetCurrentNumberOfCells(); - int ngh = pmy_mg_->ngh_, nvar = pmy_mg_->nvar_; - Real time = pmy_mesh_->time; - int lev = pmy_mg_->GetCurrentLevel(); - int ll = pmy_mg_->nlevel_ - 1 - lev; - MGCoordinates &coord = pmy_mg_->ccoord_[lev]; - constexpr Real ot = 1.0 / 3.0; - int cn = 1, cs = cn, ce = cs + nc/2 -1; - int fs = ngh, fe = fs + nc - 1; - int flim = ngh + nc; - - // x1face - for (int ox1=-1; ox1<=1; ox1+=2) { - if (nblevel[1][1][ox1+1] == loc.level - 1) { - int i, fi, fig; - if (ox1 > 0) i = ce + 1, fi = fe, fig = fe + 1; - else i = cs - 1, fi = fs, fig = fs - 1; - if (apply_bndry_fn_[BoundaryFace::inner_x2]) - DispatchBoundaryFunction(BoundaryFace::inner_x2, cbuf_, time, nvar, - i, i, cs, ce, cs, ce, ngh, coord); - if (apply_bndry_fn_[BoundaryFace::outer_x2]) - DispatchBoundaryFunction(BoundaryFace::outer_x2, cbuf_, time, nvar, - i, i, cs, ce, cs, ce, ngh, coord); - if (apply_bndry_fn_[BoundaryFace::inner_x3]) - DispatchBoundaryFunction(BoundaryFace::inner_x3, cbuf_, time, nvar, - i, i, cs, ce, cs, ce, ngh, coord); - if (apply_bndry_fn_[BoundaryFace::outer_x3]) - DispatchBoundaryFunction(BoundaryFace::outer_x3, cbuf_, time, nvar, - i, i, cs, ce, cs, ce, ngh, coord); - for (int v=0; v 0) j = ce + 1, fj = fe, fjg = fe + 1; - else j = cs - 1, fj = fs, fjg = fs - 1; - if (apply_bndry_fn_[BoundaryFace::inner_x1]) - DispatchBoundaryFunction(BoundaryFace::inner_x1, cbuf_, time, nvar, - cs, ce, j, j, cs, ce, ngh, coord); - if (apply_bndry_fn_[BoundaryFace::outer_x1]) - DispatchBoundaryFunction(BoundaryFace::outer_x1, cbuf_, time, nvar, - cs, ce, j, j, cs, ce, ngh, coord); - if (apply_bndry_fn_[BoundaryFace::inner_x3]) - DispatchBoundaryFunction(BoundaryFace::inner_x3, cbuf_, time, nvar, - cs, ce, j, j, cs, ce, ngh, coord); - if (apply_bndry_fn_[BoundaryFace::outer_x3]) - DispatchBoundaryFunction(BoundaryFace::outer_x3, cbuf_, time, nvar, - cs, ce, j, j, cs, ce, ngh, coord); - for (int v=0; v 0) k = ce + 1, fk = fe, fkg = fe + 1; - else k = cs - 1, fk = fs, fkg = fs - 1; - if (apply_bndry_fn_[BoundaryFace::inner_x1]) - DispatchBoundaryFunction(BoundaryFace::inner_x1, cbuf_, time, nvar, - cs, ce, cs, ce, k, k, ngh, coord); - if (apply_bndry_fn_[BoundaryFace::outer_x1]) - DispatchBoundaryFunction(BoundaryFace::outer_x1, cbuf_, time, nvar, - cs, ce, cs, ce, k, k, ngh, coord); - if (apply_bndry_fn_[BoundaryFace::inner_x2]) - DispatchBoundaryFunction(BoundaryFace::inner_x2, cbuf_, time, nvar, - cs, ce, cs, ce, k, k, ngh, coord); - if (apply_bndry_fn_[BoundaryFace::outer_x2]) - DispatchBoundaryFunction(BoundaryFace::outer_x2, cbuf_, time, nvar, - cs, ce, cs, ce, k, k, ngh, coord); - for (int v=0; v &dst, Real time, int nvar, int is, int ie, int js, int je, int ks, int ke, int ngh, @@ -82,13 +82,13 @@ class MGBoundaryValues : public BoundaryBase { //!@{ //! functions specific to physics virtual int LoadMultigridBoundaryBufferToCoarserFluxCons(Real *buf, - const NeighborBlock& nb) = 0; + const NeighborBlock& nb); virtual int LoadMultigridBoundaryBufferToFinerFluxCons(Real *buf, - const NeighborBlock& nb) = 0; + const NeighborBlock& nb); virtual void SetMultigridBoundaryFromCoarserFluxCons(const Real *buf, - const NeighborBlock& nb) = 0; + const NeighborBlock& nb); virtual void SetMultigridBoundaryFromFinerFluxCons(const Real *buf, - const NeighborBlock& nb) = 0; + const NeighborBlock& nb); //!@} friend class Multigrid; friend class MultigridDriver; @@ -102,7 +102,9 @@ class MGBoundaryValues : public BoundaryBase { class MGGravityBoundaryValues : public MGBoundaryValues { public: MGGravityBoundaryValues(Multigrid *pmg, BoundaryFlag *input_bcs) - : MGBoundaryValues(pmg, input_bcs) {} + : MGBoundaryValues(pmg, input_bcs) {}; + void ProlongateMultigridBoundariesFluxCons(); + private: int LoadMultigridBoundaryBufferToCoarserFluxCons(Real *buf, const NeighborBlock& nb) final; diff --git a/src/bvals/cc/mg/bvals_mg_gravity.cpp b/src/bvals/cc/mg/bvals_mg_gravity.cpp new file mode 100644 index 0000000000..b26488fd79 --- /dev/null +++ b/src/bvals/cc/mg/bvals_mg_gravity.cpp @@ -0,0 +1,393 @@ +//======================================================================================== +// Athena++ astrophysical MHD code +// Copyright(C) 2014 James M. Stone and other code contributors +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +//! \file bvals_mg_gravity.cpp +//! \brief + +// C headers + +// C++ headers +#include +#include +#include +#include // memcpy() +#include +#include // endl +#include // stringstream +#include // runtime_error +#include // c_str() + +// Athena++ headers +#include "../../../athena.hpp" +#include "../../../athena_arrays.hpp" +#include "../../../coordinates/coordinates.hpp" +#include "../../../globals.hpp" +#include "../../../mesh/mesh.hpp" +#include "../../../multigrid/multigrid.hpp" +#include "../../../parameter_input.hpp" +#include "../../../utils/buffer_utils.hpp" +#include "bvals_mg.hpp" + +// MPI header +#ifdef MPI_PARALLEL +#include +#endif + +class Multigrid; +class MultigridDriver; +class MGBoundaryValues; + +//---------------------------------------------------------------------------------------- +//! \fn int MGGravityBoundaryValues::LoadMultigridBoundaryBufferToCoarserFluxCons( +//! Real *buf, const NeighborBlock& nb) +//! \brief Set Multigrid boundary buffers for sending to a block on the coarser level +//! using the Mass Conservation formula for gravity + +int MGGravityBoundaryValues::LoadMultigridBoundaryBufferToCoarserFluxCons(Real *buf, + const NeighborBlock& nb) { + const AthenaArray &u = pmy_mg_->GetCurrentData(); + int nc = pmy_mg_->GetCurrentNumberOfCells(); + int ngh = pmy_mg_->ngh_, nvar = pmy_mg_->nvar_; + int cn = ngh, fs = ngh, cs = cn, fe = fs + nc - 1, ce = cs + nc/2 - 1; + int p = 0; + + // take averages of four cells contacting the interface and pack + if (nb.ni.ox1 != 0) { // x1 face + int fi; + if (nb.ni.ox1 < 0) fi = fs; + else fi = fe; + for (int v=0; v &u = pmy_mg_->GetCurrentData(); + int nc = pmy_mg_->GetCurrentNumberOfCells(); + int ngh = pmy_mg_->ngh_, nvar = pmy_mg_->nvar_; + int cn = ngh - 1, fs = ngh, fe = fs + nc - 1; + int si = (nb.ni.ox1 > 0) ? (fe - cn) : fs; + int ei = (nb.ni.ox1 < 0) ? (fs + cn) : fe; + int sj = (nb.ni.ox2 > 0) ? (fe - cn) : fs; + int ej = (nb.ni.ox2 < 0) ? (fs + cn) : fe; + int sk = (nb.ni.ox3 > 0) ? (fe - cn) : fs; + int ek = (nb.ni.ox3 < 0) ? (fs + cn) : fe; + + int nred = nc/2 - ngh; + if (nb.ni.ox1 == 0) { + if (nb.ni.fi1 == 1) si += nred; + else ei -= nred; + } + if (nb.ni.ox2 == 0) { + if (nb.ni.ox1 != 0) { + if (nb.ni.fi1 == 1) sj += nred; + else ej -= nred; + } else { + if (nb.ni.fi2 == 1) sj += nred; + else ej -= nred; + } + } + if (nb.ni.ox3 == 0) { + if (nb.ni.ox1 != 0 && nb.ni.ox2 != 0) { + if (nb.ni.fi1 == 1) sk += nred; + else ek -= nred; + } else { + if (nb.ni.fi2 == 1) sk += nred; + else ek -= nred; + } + } + + int p = 0; + BufferUtility::PackData(u, buf, 0, nvar-1, si, ei, sj, ej, sk, ek, p); + + return p; +} + + +//---------------------------------------------------------------------------------------- +//! \fn void MGGravityBoundaryValues::SetMultigridBoundaryFromCoarserFluxCons( +//! const Real *buf, const NeighborBlock& nb) +//! \brief Set hydro boundary received from a block on the same level + +void MGGravityBoundaryValues::SetMultigridBoundaryFromCoarserFluxCons(const Real *buf, + const NeighborBlock& nb) { + int nc = pmy_mg_->GetCurrentNumberOfCells(); + int ngh = pmy_mg_->ngh_, nvar = pmy_mg_->nvar_; + int si, sj, sk, ei, ej, ek; + int cng = 1; + int cs = cng, ce = cng + nc/2 -1; + + if (nb.ni.ox1 == 0) { + si = cs, ei = ce; + if ((loc.lx1 & 1LL) == 0LL) ei += cng; + else si -= cng; + } else if (nb.ni.ox1 > 0) { + si = ce + 1, ei = ce + cng; + } else { + si = cs - cng, ei = cs - 1; + } + if (nb.ni.ox2 == 0) { + sj = cs, ej = ce; + if ((loc.lx2 & 1LL) == 0LL) ej += cng; + else sj -= cng; + } else if (nb.ni.ox2 > 0) { + sj = ce + 1, ej = ce + cng; + } else { + sj = cs - cng, ej = cs - 1; + } + if (nb.ni.ox3 == 0) { + sk = cs, ek = ce; + if ((loc.lx3 & 1LL) == 0LL) ek += cng; + else sk -= cng; + } else if (nb.ni.ox3 > 0) { + sk = ce + 1, ek = ce + cng; + } else { + sk = cs - cng, ek = cs - 1; + } + + int p = 0; + BufferUtility::UnpackData(buf, cbuf_, 0, nvar-1, si, ei, sj, ej, sk, ek, p); + + return; +} + + +//---------------------------------------------------------------------------------------- +//! \fn void MGGravityBoundaryValues::SetMultigridBoundaryFromFinerFluxCons( +//! const Real *buf, const NeighborBlock& nb) + +void MGGravityBoundaryValues::SetMultigridBoundaryFromFinerFluxCons(const Real *buf, + const NeighborBlock& nb) { + AthenaArray &dst = pmy_mg_->GetCurrentData(); + int nc = pmy_mg_->GetCurrentNumberOfCells(); + int ngh = pmy_mg_->ngh_, nvar = pmy_mg_->nvar_; + int fs = ngh, fe = fs + nc - 1; + int si, sj, sk, ei, ej, ek; + int oi = 0, oj = 0, ok = 0; + + // could reuse the index calculation for normal boundaries + // but in general this depends on physics + if (nb.ni.ox1 == 0) { + si = fs, ei = fe; + if (nb.ni.fi1 == 1) si += nc/2; + else ei -= nc/2; + } else if (nb.ni.ox1 > 0) { + si = fe + 1, ei = fe + ngh, oi = -1; + } else { + si = fs - ngh, ei = fs - 1, oi = 1; + } + if (nb.ni.ox2 == 0) { + sj = fs, ej = fe; + if (nb.ni.ox1 != 0) { + if (nb.ni.fi1 == 1) sj += nc/2; + else ej -= nc/2; + } else { + if (nb.ni.fi2 == 1) sj += nc/2; + else ej -= nc/2; + } + } else if (nb.ni.ox2 > 0) { + sj = fe + 1, ej = fe + ngh, oj = -1; + } else { + sj = fs - ngh, ej = fs - 1, oj = 1; + } + if (nb.ni.ox3 == 0) { + sk = fs, ek = fe; + if (nb.ni.ox1 != 0 && nb.ni.ox2 != 0) { + if (nb.ni.fi1 == 1) sk += nc/2; + else ek -= nc/2; + } else { + if (nb.ni.fi2 == 1) sk += nc/2; + else ek -= nc/2; + } + } else if (nb.ni.ox3 > 0) { + sk = fe + 1, ek = fe + ngh, ok = -1; + } else { + sk = fs - ngh, ek = fs - 1, ok = 1; + } + + constexpr Real ot = 1.0/3.0; + int p = 0; + + // correct the ghost values using the mass conservation formula + for (int v=0; v &u = pmy_mg_->GetCurrentData(); + AthenaArray &dst = pmy_mg_->GetCurrentData(); + int nc = pmy_mg_->GetCurrentNumberOfCells(); + int ngh = pmy_mg_->ngh_, nvar = pmy_mg_->nvar_; + Real time = pmy_mesh_->time; + int lev = pmy_mg_->GetCurrentLevel(); + int ll = pmy_mg_->nlevel_ - 1 - lev; + MGCoordinates &coord = pmy_mg_->ccoord_[lev]; + constexpr Real ot = 1.0 / 3.0; + int cn = 1, cs = cn, ce = cs + nc/2 -1; + int fs = ngh, fe = fs + nc - 1; + int flim = ngh + nc; + + // x1face + for (int ox1=-1; ox1<=1; ox1+=2) { + if (nblevel[1][1][ox1+1] == loc.level - 1) { + int i, fi, fig; + if (ox1 > 0) i = ce + 1, fi = fe, fig = fe + 1; + else i = cs - 1, fi = fs, fig = fs - 1; + if (apply_bndry_fn_[BoundaryFace::inner_x2]) + DispatchBoundaryFunction(BoundaryFace::inner_x2, cbuf_, time, nvar, + i, i, cs, ce, cs, ce, ngh, coord); + if (apply_bndry_fn_[BoundaryFace::outer_x2]) + DispatchBoundaryFunction(BoundaryFace::outer_x2, cbuf_, time, nvar, + i, i, cs, ce, cs, ce, ngh, coord); + if (apply_bndry_fn_[BoundaryFace::inner_x3]) + DispatchBoundaryFunction(BoundaryFace::inner_x3, cbuf_, time, nvar, + i, i, cs, ce, cs, ce, ngh, coord); + if (apply_bndry_fn_[BoundaryFace::outer_x3]) + DispatchBoundaryFunction(BoundaryFace::outer_x3, cbuf_, time, nvar, + i, i, cs, ce, cs, ce, ngh, coord); + for (int v=0; v 0) j = ce + 1, fj = fe, fjg = fe + 1; + else j = cs - 1, fj = fs, fjg = fs - 1; + if (apply_bndry_fn_[BoundaryFace::inner_x1]) + DispatchBoundaryFunction(BoundaryFace::inner_x1, cbuf_, time, nvar, + cs, ce, j, j, cs, ce, ngh, coord); + if (apply_bndry_fn_[BoundaryFace::outer_x1]) + DispatchBoundaryFunction(BoundaryFace::outer_x1, cbuf_, time, nvar, + cs, ce, j, j, cs, ce, ngh, coord); + if (apply_bndry_fn_[BoundaryFace::inner_x3]) + DispatchBoundaryFunction(BoundaryFace::inner_x3, cbuf_, time, nvar, + cs, ce, j, j, cs, ce, ngh, coord); + if (apply_bndry_fn_[BoundaryFace::outer_x3]) + DispatchBoundaryFunction(BoundaryFace::outer_x3, cbuf_, time, nvar, + cs, ce, j, j, cs, ce, ngh, coord); + for (int v=0; v 0) k = ce + 1, fk = fe, fkg = fe + 1; + else k = cs - 1, fk = fs, fkg = fs - 1; + if (apply_bndry_fn_[BoundaryFace::inner_x1]) + DispatchBoundaryFunction(BoundaryFace::inner_x1, cbuf_, time, nvar, + cs, ce, cs, ce, k, k, ngh, coord); + if (apply_bndry_fn_[BoundaryFace::outer_x1]) + DispatchBoundaryFunction(BoundaryFace::outer_x1, cbuf_, time, nvar, + cs, ce, cs, ce, k, k, ngh, coord); + if (apply_bndry_fn_[BoundaryFace::inner_x2]) + DispatchBoundaryFunction(BoundaryFace::inner_x2, cbuf_, time, nvar, + cs, ce, cs, ce, k, k, ngh, coord); + if (apply_bndry_fn_[BoundaryFace::outer_x2]) + DispatchBoundaryFunction(BoundaryFace::outer_x2, cbuf_, time, nvar, + cs, ce, cs, ce, k, k, ngh, coord); + for (int v=0; v and other code contributors +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +//! \file crdiffusion.cpp +//! \brief implementation of functions in class CRDiffusion + +// C headers + +// C++ headers +#include +#include // sstream +#include // runtime_error +#include // c_str() +#include + +// Athena++ headers +#include "../athena.hpp" +#include "../athena_arrays.hpp" +#include "../bvals/bvals.hpp" +#include "../bvals/bvals_interfaces.hpp" +#include "../bvals/cc/bvals_cc.hpp" +#include "../coordinates/coordinates.hpp" +#include "../mesh/mesh.hpp" +#include "../parameter_input.hpp" +#include "../utils/buffer_utils.hpp" +#include "crdiffusion.hpp" +#include "mg_crdiffusion.hpp" + + +//---------------------------------------------------------------------------------------- +//! \fn CRDiffusion::CRDiffusion(MeshBlock *pmb, ParameterInput *pin) +//! \brief CRDiffusion constructor +CRDiffusion::CRDiffusion(MeshBlock *pmb, ParameterInput *pin) : + pmy_block(pmb), ecr(pmb->ncells3, pmb->ncells2, pmb->ncells1), + zeta(pmb->ncells3, pmb->ncells2, pmb->ncells1), + coarse_ecr(pmb->ncc3, pmb->ncc2, pmb->ncc1, + (pmb->pmy_mesh->multilevel ? AthenaArray::DataStatus::allocated : + AthenaArray::DataStatus::empty)), + empty_flux{AthenaArray(), AthenaArray(), AthenaArray()}, + D(NCOEFF, pmb->ncells3, pmb->ncells2, pmb->ncells1), + nlambda(pmb->ncells3, pmb->ncells2, pmb->ncells1), + output_defect(false), crbvar(pmb, &ecr, &coarse_ecr, empty_flux, false), + refinement_idx_(), Dpara_(), Dperp_(), Lambda_() { + Dpara_ = pin->GetReal("crdiffusion", "Dpara"); + Dperp_ = pin->GetReal("crdiffusion", "Dperp"); + Lambda_ = pin->GetReal("crdiffusion", "Lambda"); + + output_defect = pin->GetOrAddBoolean("crdiffusion", "output_defect", false); + if (output_defect) + def.NewAthenaArray(pmb->ncells3, pmb->ncells2, pmb->ncells1); + + pmb->RegisterMeshBlockData(ecr); + // "Enroll" in S/AMR by adding to vector of tuples of pointers in MeshRefinement class + if (pmb->pmy_mesh->multilevel) + refinement_idx_ = pmy_block->pmr->AddToRefinement(&ecr, &coarse_ecr); + +// pmg = new MGCRDiffusion(pmb->pmy_mesh->pmgcrd, pmb); + + // Enroll CellCenteredBoundaryVariable object + crbvar.bvar_index = pmb->pbval->bvars.size(); + pmb->pbval->bvars.push_back(&crbvar); + pmb->pbval->pcrbvar = &crbvar; +} + + +//---------------------------------------------------------------------------------------- +//! \fn CRDiffusion::~CRDiffusion() +//! \brief CRDiffusion destructor +CRDiffusion::~CRDiffusion() { + delete pmg; +} + + +//---------------------------------------------------------------------------------------- +//! \fn void CRDiffusion::CalculateCoefficients() +//! \brief Calculate coefficients required for CR calculation +void CRDiffusion::CalculateCoefficients(const AthenaArray &w, + const AthenaArray &bcc) { + int il = pmy_block->is - NGHOST, iu = pmy_block->ie + NGHOST; + int jl = pmy_block->js, ju = pmy_block->je; + int kl = pmy_block->ks, ku = pmy_block->ke; + if (pmy_block->pmy_mesh->f2) + jl -= NGHOST, ju += NGHOST; + if (pmy_block->pmy_mesh->f3) + kl -= NGHOST, ku += NGHOST; + constexpr Real Dunit = 1.0, nlunit = 1.0; + Real Dpara = Dpara_ * Dunit, Dperp = Dperp_ * Dunit, Lambda = Lambda_ * nlunit; + + if (MAGNETIC_FIELDS_ENABLED) { + for (int k = kl; k <= ku; ++k) { + for (int j = jl; j <= ju; ++j) { + for (int i = il; i <= iu; ++i) { + const Real &bx = bcc(IB1,k,j,i); + const Real &by = bcc(IB2,k,j,i); + const Real &bz = bcc(IB3,k,j,i); + Real ba = std::sqrt(SQR(bx) + SQR(by) + SQR(bz) + TINY_NUMBER); + Real nx = bx / ba, ny = by / ba, nz = bz / ba; + D(XX,k,j,i) = Dperp + (Dpara - Dperp) * nx * nx; + D(XY,k,j,i) = (Dpara - Dperp) * nx * ny; + D(XZ,k,j,i) = (Dpara - Dperp) * nx * nz; + D(YY,k,j,i) = Dperp + (Dpara - Dperp) * ny * ny; + D(YZ,k,j,i) = (Dpara - Dperp) * ny * nz; + D(ZZ,k,j,i) = Dperp + (Dpara - Dperp) * nz * nz; + nlambda(k,j,i) = Lambda * w(IDN,k,j,i); + } + } + } + } else { + for (int k = kl; k <= ku; ++k) { + for (int j = jl; j <= ju; ++j) { + for (int i = il; i <= iu; ++i) { + D(XX,k,j,i) = D(XY,k,j,i) = D(XZ,k,j,i) = D(YY,k,j,i) + = D(YZ,k,j,i) = D(ZZ,k,j,i) = Dpara; + nlambda(k,j,i) = Lambda * w(IDN,k,j,i); + } + } + } + } +} + +//---------------------------------------------------------------------------------------- +//! \fn CRDiffusion::ExpandPhysicalBoundaries() +//! \brief Expand physical boundary values to NGHOST = 2 and to edges/corners. +void CRDiffusion::ExpandPhysicalBoundaries() { + int is = pmy_block->is, ie = pmy_block->ie, + js = pmy_block->js, je = pmy_block->je, + ks = pmy_block->ks, ke = pmy_block->ke; + + // push face boundary values + if (pmy_block->pbval->nblevel[1][1][0] < 0) { + for (int k = ks; k <= ke; k++) { + for (int j = js; j <= je; j++) + ecr(k, j, is-2) = ecr(k, j, is-1); + } + } + if (pmy_block->pbval->nblevel[1][1][2] < 0) { + for (int k = ks; k <= ke; k++) { + for (int j = js; j <= je; j++) + ecr(k, j, ie+2) = ecr(k, j, ie+1); + } + } + if (pmy_block->pbval->nblevel[1][0][1] < 0) { + for (int k = ks; k <= ke; k++) { + for (int i = is; i <= ie; i++) + ecr(k, js-2, i) = ecr(k, js-1, i); + } + } + if (pmy_block->pbval->nblevel[1][2][1] < 0) { + for (int k = ks; k <= ke; k++) { + for (int i = is; i <= ie; i++) + ecr(k, je+2, i) = ecr(k, je+1, i); + } + } + if (pmy_block->pbval->nblevel[0][1][1] < 0) { + for (int j = js; j <= je; j++) { + for (int i = is; i <= ie; i++) + ecr(ks-2, j, i) = ecr(ks-1, j, i); + } + } + if (pmy_block->pbval->nblevel[2][1][1] < 0) { + for (int j = js; j <= je; j++) { + for (int i = is; i <= ie; i++) + ecr(ke+2, j, i) = ecr(ke+1, j, i); + } + } + + // fill edges + if (pmy_block->pbval->nblevel[1][0][0] < 0) { + for (int k = ks; k <= ke; k++) { + Real p = 0.5*(ecr(k, js-1, is) + ecr(k, js, is-1)); + ecr(k, js-1, is-1) = p; + ecr(k, js-1, is-2) = p; + ecr(k, js-2, is-1) = p; + ecr(k, js-2, is-2) = p; + } + } + if (pmy_block->pbval->nblevel[1][0][2] < 0) { + for (int k = ks; k <= ke; k++) { + Real p = 0.5*(ecr(k, js-1, ie) + ecr(k, js, ie+1)); + ecr(k, js-1, ie+1) = p; + ecr(k, js-1, ie+2) = p; + ecr(k, js-2, ie+1) = p; + ecr(k, js-2, ie+2) = p; + } + } + if (pmy_block->pbval->nblevel[1][2][0] < 0) { + for (int k = ks; k <= ke; k++) { + Real p = 0.5*(ecr(k, je+1, is) + ecr(k, je, is-1)); + ecr(k, je+1, is-1) = p; + ecr(k, je+1, is-2) = p; + ecr(k, je+2, is-1) = p; + ecr(k, je+2, is-2) = p; + } + } + if (pmy_block->pbval->nblevel[1][2][2] < 0) { + for (int k = ks; k <= ke; k++) { + Real p = 0.5*(ecr(k, je+1, ie) + ecr(k, je, ie+1)); + ecr(k, je+1, ie+1) = p; + ecr(k, je+1, ie+2) = p; + ecr(k, je+2, ie+1) = p; + ecr(k, je+2, ie+2) = p; + } + } + if (pmy_block->pbval->nblevel[0][1][0] < 0) { + for (int j = js; j <= je; j++) { + Real p = 0.5*(ecr(ks-1, j, is) + ecr(ks, j, is-1)); + ecr(ks-1, j, is-1) = p; + ecr(ks-1, j, is-2) = p; + ecr(ks-2, j, is-1) = p; + ecr(ks-2, j, is-2) = p; + } + } + if (pmy_block->pbval->nblevel[0][1][2] < 0) { + for (int j = js; j <= je; j++) { + Real p = 0.5*(ecr(ks-1, j, ie) + ecr(ks, j, ie+1)); + ecr(ks-1, j, ie+1) = p; + ecr(ks-1, j, ie+2) = p; + ecr(ks-2, j, ie+1) = p; + ecr(ks-2, j, ie+2) = p; + } + } + if (pmy_block->pbval->nblevel[2][1][0] < 0) { + for (int j = js; j <= je; j++) { + Real p = 0.5*(ecr(ke+1, j, is) + ecr(ke, j, is-1)); + ecr(ke+1, j, is-1) = p; + ecr(ke+1, j, is-2) = p; + ecr(ke+2, j, is-1) = p; + ecr(ke+2, j, is-2) = p; + } + } + if (pmy_block->pbval->nblevel[2][1][2] < 0) { + for (int j = js; j <= je; j++) { + Real p = 0.5*(ecr(ke+1, j, ie) + ecr(ke, j, ie+1)); + ecr(ke+1, j, ie+1) = p; + ecr(ke+1, j, ie+2) = p; + ecr(ke+2, j, ie+1) = p; + ecr(ke+2, j, ie+2) = p; + } + } + if (pmy_block->pbval->nblevel[0][0][1] < 0) { + for (int i = is; i <= ie; i++) { + Real p = 0.5*(ecr(ks-1, js, i) + ecr(ks, js-1, i)); + ecr(ks-1, js-1, i) = p; + ecr(ks-1, js-2, i) = p; + ecr(ks-2, js-1, i) = p; + ecr(ks-2, js-2, i) = p; + } + } + if (pmy_block->pbval->nblevel[0][2][1] < 0) { + for (int i = is; i <= ie; i++) { + Real p = 0.5*(ecr(ks-1, je, i) + ecr(ks, je+1, i)); + ecr(ks-1, je+1, i) = p; + ecr(ks-1, je+2, i) = p; + ecr(ks-2, je+1, i) = p; + ecr(ks-2, je+2, i) = p; + } + } + if (pmy_block->pbval->nblevel[2][0][1] < 0) { + for (int i = is; i <= ie; i++) { + Real p = 0.5*(ecr(ke+1, js, i) + ecr(ke, js-1, i)); + ecr(ke+1, js-1, i) = p; + ecr(ke+1, js-2, i) = p; + ecr(ke+2, js-1, i) = p; + ecr(ke+2, js-2, i) = p; + } + } + if (pmy_block->pbval->nblevel[2][2][1] < 0) { + for (int i = is; i <= ie; i++) { + Real p = 0.5*(ecr(ke+1, je, i) + ecr(ke, je+1, i)); + ecr(ke+1, je+1, i) = p; + ecr(ke+1, je+2, i) = p; + ecr(ke+2, je+1, i) = p; + ecr(ke+2, je+2, i) = p; + } + } + + // fill corners + if (pmy_block->pbval->nblevel[0][0][0] < 0) { + Real p = (ecr(ks, js-1, is-1) + ecr(ks-1, js, is-1) + ecr(ks-1, js-1, is))/3.0; + ecr(ks-1, js-1, is-1) = p; + ecr(ks-1, js-1, is-2) = p; + ecr(ks-1, js-2, is-1) = p; + ecr(ks-1, js-2, is-2) = p; + ecr(ks-2, js-1, is-1) = p; + ecr(ks-2, js-1, is-2) = p; + ecr(ks-2, js-2, is-1) = p; + ecr(ks-2, js-2, is-2) = p; + } + if (pmy_block->pbval->nblevel[0][0][2] < 0) { + Real p = (ecr(ks, js-1, ie+1) + ecr(ks-1, js, ie+1) + ecr(ks-1, js-1, ie))/3.0; + ecr(ks-1, js-1, ie+1) = p; + ecr(ks-1, js-1, ie+2) = p; + ecr(ks-1, js-2, ie+1) = p; + ecr(ks-1, js-2, ie+2) = p; + ecr(ks-2, js-1, ie+1) = p; + ecr(ks-2, js-1, ie+2) = p; + ecr(ks-2, js-2, ie+1) = p; + ecr(ks-2, js-2, ie+2) = p; + } + if (pmy_block->pbval->nblevel[0][2][0] < 0) { + Real p = (ecr(ks, je+1, is-1) + ecr(ks-1, je, is-1) + ecr(ks-1, je+1, is))/3.0; + ecr(ks-1, je+1, is-1) = p; + ecr(ks-1, je+1, is-2) = p; + ecr(ks-1, je+2, is-1) = p; + ecr(ks-1, je+2, is-2) = p; + ecr(ks-2, je+1, is-1) = p; + ecr(ks-2, je+1, is-2) = p; + ecr(ks-2, je+2, is-1) = p; + ecr(ks-2, je+2, is-2) = p; + } + if (pmy_block->pbval->nblevel[2][0][0] < 0) { + Real p = (ecr(ke, js-1, is-1) + ecr(ke+1, js, is-1) + ecr(ke+1, js-1, is))/3.0; + ecr(ke+1, js-1, is-1) = p; + ecr(ke+1, js-1, is-2) = p; + ecr(ke+1, js-2, is-1) = p; + ecr(ke+1, js-2, is-2) = p; + ecr(ke+2, js-1, is-1) = p; + ecr(ke+2, js-1, is-2) = p; + ecr(ke+2, js-2, is-1) = p; + ecr(ke+2, js-2, is-2) = p; + } + if (pmy_block->pbval->nblevel[0][2][2] < 0) { + Real p = (ecr(ks, je+1, ie+1) + ecr(ks-1, je, ie+1) + ecr(ks-1, je+1, ie))/3.0; + ecr(ks-1, je+1, ie+1) = p; + ecr(ks-1, je+1, ie+2) = p; + ecr(ks-1, je+2, ie+1) = p; + ecr(ks-1, je+2, ie+2) = p; + ecr(ks-2, je+1, ie+1) = p; + ecr(ks-2, je+1, ie+2) = p; + ecr(ks-2, je+2, ie+1) = p; + ecr(ks-2, je+2, ie+2) = p; + } + if (pmy_block->pbval->nblevel[2][0][2] < 0) { + Real p = (ecr(ke, js-1, ie+1) + ecr(ke+1, js, ie+1) + ecr(ke+1, js-1, ie))/3.0; + ecr(ke+1, js-1, ie+1) = p; + ecr(ke+1, js-1, ie+2) = p; + ecr(ke+1, js-2, ie+1) = p; + ecr(ke+1, js-2, ie+2) = p; + ecr(ke+2, js-1, ie+1) = p; + ecr(ke+2, js-1, ie+2) = p; + ecr(ke+2, js-2, ie+1) = p; + ecr(ke+2, js-2, ie+2) = p; + } + if (pmy_block->pbval->nblevel[2][2][0] < 0) { + Real p = (ecr(ke, je+1, is-1) + ecr(ke+1, je, is-1) + ecr(ke+1, je+1, is))/3.0; + ecr(ke+1, je+1, is-1) = p; + ecr(ke+1, je+1, is-2) = p; + ecr(ke+1, je+2, is-1) = p; + ecr(ke+1, je+2, is-2) = p; + ecr(ke+2, je+1, is-1) = p; + ecr(ke+2, je+1, is-2) = p; + ecr(ke+2, je+2, is-1) = p; + ecr(ke+2, je+2, is-2) = p; + } + if (pmy_block->pbval->nblevel[2][2][2] < 0) { + Real p = (ecr(ke, je+1, ie+1) + ecr(ke+1, je, ie+1) + ecr(ke+1, je+1, ie))/3.0; + ecr(ke+1, je+1, ie+1) = p; + ecr(ke+1, je+1, ie+2) = p; + ecr(ke+1, je+2, ie+1) = p; + ecr(ke+1, je+2, ie+2) = p; + ecr(ke+2, je+1, ie+1) = p; + ecr(ke+2, je+1, ie+2) = p; + ecr(ke+2, je+2, ie+1) = p; + ecr(ke+2, je+2, ie+2) = p; + } + + return; +} + diff --git a/src/crdiffusion/crdiffusion.hpp b/src/crdiffusion/crdiffusion.hpp new file mode 100644 index 0000000000..d1ac48cbcc --- /dev/null +++ b/src/crdiffusion/crdiffusion.hpp @@ -0,0 +1,63 @@ +#ifndef CRDIFFUSION_CRDIFFUSION_HPP_ +#define CRDIFFUSION_CRDIFFUSION_HPP_ +//======================================================================================== +// Athena++ astrophysical MHD code +// Copyright(C) 2014 James M. Stone and other code contributors +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +//! \file crdiffusion.hpp +//! \brief defines CRDiffusion class which implements data and functions for the implicit +//! Cosmic-Ray diffusion solver + +// C headers + +// C++ headers + +// Athena++ headers +#include "../athena.hpp" +#include "../athena_arrays.hpp" +#include "../bvals/bvals.hpp" +#include "../bvals/cc/bvals_cc.hpp" +#include "mg_crdiffusion.hpp" + +class MeshBlock; +class ParameterInput; +class Coordinates; +class CRDiffusionBoundaryValues; +class MGCRDiffusion; +class MGCRDiffusionDriver; + +constexpr int NCOEFF = 6; +enum DCoeffIndex {XX = 0, XY = 1, YX = 1, XZ = 2, ZX = 2, YY = 3, YZ=4, ZZ=5}; + +//! \class CRDiffusion +//! \brief gravitational potential data and functions + +class CRDiffusion { + public: + CRDiffusion(MeshBlock *pmb, ParameterInput *pin); + ~CRDiffusion(); + + MeshBlock* pmy_block; + MGCRDiffusion *pmg; + AthenaArray ecr, zeta; + AthenaArray coarse_ecr, empty_flux[3]; + AthenaArray D, nlambda; + AthenaArray def; // defect from the Multigrid solver + bool output_defect; + + CellCenteredBoundaryVariable crbvar; + + void CalculateCoefficients(const AthenaArray &w, + const AthenaArray &bcc); + void Solve(); + void ExpandPhysicalBoundaries(); + + friend class MGCRDiffusuionDriver; + + private: + int refinement_idx_; + Real Dpara_, Dperp_, Lambda_; +}; + +#endif // CRDIFFUSION_CRDIFFUSION_HPP_ diff --git a/src/crdiffusion/mg_crdiffusion.cpp b/src/crdiffusion/mg_crdiffusion.cpp new file mode 100644 index 0000000000..0f56e9672a --- /dev/null +++ b/src/crdiffusion/mg_crdiffusion.cpp @@ -0,0 +1,318 @@ +//======================================================================================== +// Athena++ astrophysical MHD code +// Copyright(C) 2014 James M. Stone and other code contributors +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +//! \file mg_crdiffusion.cpp +//! \brief create multigrid solver for gravity + +// C headers + +// C++ headers +#include +#include +#include // sstream +#include // runtime_error +#include // c_str() + +// Athena++ headers +#include "../athena.hpp" +#include "../athena_arrays.hpp" +#include "../coordinates/coordinates.hpp" +#include "../globals.hpp" +#include "../hydro/hydro.hpp" +#include "../mesh/mesh.hpp" +#include "../multigrid/multigrid.hpp" +#include "../parameter_input.hpp" +#include "../task_list/crdiffusion_task_list.hpp" +#include "mg_crdiffusion.hpp" + +#ifdef MPI_PARALLEL +#include +#endif + +class MeshBlock; + +//---------------------------------------------------------------------------------------- +//! \fn MGCRDiffusionDriver::MGCRDiffusionDriver(Mesh *pm, ParameterInput *pin) +//! \brief MGCRDiffusionDriver constructor + +MGCRDiffusionDriver::MGCRDiffusionDriver(Mesh *pm, ParameterInput *pin) + : MultigridDriver(pm, pm->MGCRDiffusionBoundaryFunction_, + pm->MGCRDiffusionSourceMaskFunction_, 1) { + eps_ = pin->GetOrAddReal("crdiffusion", "threshold", -1.0); + niter_ = pin->GetOrAddInteger("crdiffusion", "niteration", -1); + ffas_ = pin->GetOrAddBoolean("crdiffusion", "fas", ffas_); + std::string m = pin->GetOrAddString("crdiffusion", "mgmode", "none"); + std::transform(m.begin(), m.end(), m.begin(), ::tolower); + if (m == "fmg") { + mode_ = 0; + } else if (m == "mgi") { + mode_ = 1; // Iterative + } else { + std::stringstream msg; + msg << "### FATAL ERROR in MGCRDiffusionDriver::MGCRDiffusionDriver" << std::endl + << "The \"mgmode\" parameter in the block is invalid." << std::endl + << "FMG: Full Multigrid + Multigrid iteration (default)" << std::endl + << "MGI: Multigrid Iteration" << std::endl; + ATHENA_ERROR(msg); + } + if (eps_ < 0.0 && niter_ < 0) { + std::stringstream msg; + msg << "### FATAL ERROR in MGCRDiffusionDriver::MGCRDiffusionDriver" << std::endl + << "Either \"threshold\" or \"niteration\" parameter must be set " + << "in the block." << std::endl + << "When both parameters are specified, \"niteration\" is ignored." << std::endl + << "Set \"threshold = 0.0\" for automatic convergence control." << std::endl; + ATHENA_ERROR(msg); + } + mg_mesh_bcs_[inner_x1] = + GetMGBoundaryFlag(pin->GetOrAddString("crdiffusion", "ix1_bc", "none")); + mg_mesh_bcs_[outer_x1] = + GetMGBoundaryFlag(pin->GetOrAddString("crdiffusion", "ox1_bc", "none")); + mg_mesh_bcs_[inner_x2] = + GetMGBoundaryFlag(pin->GetOrAddString("crdiffusion", "ix2_bc", "none")); + mg_mesh_bcs_[outer_x2] = + GetMGBoundaryFlag(pin->GetOrAddString("crdiffusion", "ox2_bc", "none")); + mg_mesh_bcs_[inner_x3] = + GetMGBoundaryFlag(pin->GetOrAddString("crdiffusion", "ix3_bc", "none")); + mg_mesh_bcs_[outer_x3] = + GetMGBoundaryFlag(pin->GetOrAddString("crdiffusion", "ox3_bc", "none")); + CheckBoundaryFunctions(); + + mgtlist_ = new MultigridTaskList(this); + + // Allocate the root multigrid + mgroot_ = new MGCRDiffusion(this, nullptr); + + crtlist_ = new CRDiffusionBoundaryTaskList(pin, pm); +} + + +//---------------------------------------------------------------------------------------- +//! \fn MGCRDiffusionDriver::~MGCRDiffusionDriver() +//! \brief MGCRDiffusionDriver destructor + +MGCRDiffusionDriver::~MGCRDiffusionDriver() { + delete crtlist_; + delete mgroot_; + delete mgtlist_; +} + + +//---------------------------------------------------------------------------------------- +//! \fn MGCRDiffusion::MGCRDiffusion(MultigridDriver *pmd, MeshBlock *pmb) +//! \brief MGCRDiffusion constructor + +MGCRDiffusion::MGCRDiffusion(MultigridDriver *pmd, MeshBlock *pmb) + : Multigrid(pmd, pmb, 1, 1) { + for (int l = 0; l < nlevel_; l++) { + int ll=nlevel_-1-l; + int ncx=(size_.nx1>>ll)+2*ngh_; + int ncy=(size_.nx2>>ll)+2*ngh_; + int ncz=(size_.nx3>>ll)+2*ngh_; + D_[l].NewAthenaArray(NCOEFF,ncz,ncy,ncx); + nlambda_[l].NewAthenaArray(ncz,ncy,ncx); + } + + btype = btypef = BoundaryQuantity::mg; + pmgbval = new MGBoundaryValues(this, mg_block_bcs_); +} + + +//---------------------------------------------------------------------------------------- +//! \fn MGCRDiffusion::~MGCRDiffusion() +//! \brief MGCRDiffusion deconstructor + +MGCRDiffusion::~MGCRDiffusion() { + delete [] D_; + delete [] nlambda_; + delete pmgbval; +} + + +//---------------------------------------------------------------------------------------- +//! \fn void MGCRDiffusionDriver::Solve(int stage) +//! \brief load the data and solve + +void MGCRDiffusionDriver::Solve(int stage) { + // Construct the Multigrid array + vmg_.clear(); + for (int i = 0; i < pmy_mesh_->nblocal; ++i) + vmg_.push_back(pmy_mesh_->my_blocks(i)->pcrdiff->pmg); + + // load the source +#pragma omp parallel for num_threads(nthreads_) + for (auto itr = vmg_.begin(); itr < vmg_.end(); itr++) { + MGCRDiffusion *pmg = static_cast(*itr); + // assume all the data are located on the same node + pmg->LoadSource(pmg->pmy_block_->pcrdiff->ecr, 0, NGHOST, 1.0); + if (mode_ == 1) // use the previous timestep data as the initial guess + pmg->LoadFinestData(pmg->pmy_block_->pcrdiff->ecr, 0, NGHOST); + pmg->LoadCoefficients(pmg->pmy_block_->pcrdiff->D, + pmg->pmy_block_->pcrdiff->nlambda, NGHOST); + } + + SetupMultigrid(); + RestrictCoefficients(); + + if (mode_ == 0) { + SolveFMGCycle(); + } else { + if (eps_ >= 0.0) + SolveIterative(); + else + SolveIterativeFixedTimes(); + } + + // Return the result +#pragma omp parallel for num_threads(nthreads_) + for (auto itr = vmg_.begin(); itr < vmg_.end(); itr++) { + Multigrid *pmg = *itr; + CRDiffusion *pcrdiff = pmg->pmy_block_->pcrdiff; + pmg->RetrieveResult(pcrdiff->ecr, 0, NGHOST); + if(pcrdiff->output_defect) + pmg->RetrieveDefect(pcrdiff->def, 0, NGHOST); + } + + crtlist_->DoTaskListOneStage(pmy_mesh_, stage); + + return; +} + + +//---------------------------------------------------------------------------------------- +//! \fn void MGCRDiffusion::Smooth(AthenaArray &u, const AthenaArray &src, +//! int rlev, int il, int iu, int jl, int ju, int kl, int ku, int color, bool th) +//! \brief Implementation of the Red-Black Gauss-Seidel Smoother +//! rlev = relative level from the finest level of this Multigrid block + +void MGCRDiffusion::Smooth(AthenaArray &u, const AthenaArray &src, int rlev, + int il, int iu, int jl, int ju, int kl, int ku, int color, bool th) { + Real dx; + if (rlev <= 0) dx = rdx_*static_cast(1<<(-rlev)); + else dx = rdx_/static_cast(1<coffset_; + if (th == true && (ku-kl) >= minth_) { +#pragma omp parallel for num_threads(pmy_driver_->nthreads_) + for (int k=kl; k<=ku; k++) { + for (int j=jl; j<=ju; j++) { + int c = (color + k + j) & 1; +#pragma ivdep + for (int i=il+c; i<=iu; i+=2); + } + } + } else { + for (int k=kl; k<=ku; k++) { + for (int j=jl; j<=ju; j++) { + int c = (color + k + j) & 1; +#pragma ivdep + for (int i=il+c; i<=iu; i+=2); + } + } + } + return; +} + + +//---------------------------------------------------------------------------------------- +//! \fn void MGCRDiffusion::CalculateDefect(AthenaArray &def, +//! const AthenaArray &u, const AthenaArray &src, int rlev, +//! int il, int iu, int jl, int ju, int kl, int ku, bool th) +//! \brief Implementation of the Defect calculation +//! rlev = relative level from the finest level of this Multigrid block + +void MGCRDiffusion::CalculateDefect(AthenaArray &def, const AthenaArray &u, + const AthenaArray &src, int rlev, + int il, int iu, int jl, int ju, int kl, int ku, bool th) { + Real dx; + if (rlev <= 0) dx = rdx_*static_cast(1<<(-rlev)); + else dx = rdx_/static_cast(1<= minth_) { +#pragma omp parallel for num_threads(pmy_driver_->nthreads_) + for (int k=kl; k<=ku; k++) { + for (int j=jl; j<=ju; j++) { +#pragma omp simd + for (int i=il; i<=iu; i++); + } + } + } else { + for (int k=kl; k<=ku; k++) { + for (int j=jl; j<=ju; j++) { +#pragma omp simd + for (int i=il; i<=iu; i++); + } + } + } + + return; +} + + +//---------------------------------------------------------------------------------------- +//! \fn void MGCRDiffusion::CalculateFASRHS(AthenaArray &src, +//! const AthenaArray &u, int rlev, +//! int il, int iu, int jl, int ju, int kl, int ku, bool th) +//! \brief Implementation of the RHS calculation for FAS +//! rlev = relative level from the finest level of this Multigrid block + +void MGCRDiffusion::CalculateFASRHS(AthenaArray &src, const AthenaArray &u, + int rlev, int il, int iu, int jl, int ju, int kl, int ku, bool th) { + Real dx; + if (rlev <= 0) dx = rdx_*static_cast(1<<(-rlev)); + else dx = rdx_/static_cast(1<= minth_) { +#pragma omp parallel for num_threads(pmy_driver_->nthreads_) + for (int k=kl; k<=ku; k++) { + for (int j=jl; j<=ju; j++) { +#pragma omp simd + for (int i=il; i<=iu; i++); + } + } + } else { + for (int k=kl; k<=ku; k++) { + for (int j=jl; j<=ju; j++) { +#pragma omp simd + for (int i=il; i<=iu; i++); + } + } + } + return; +} + +//---------------------------------------------------------------------------------------- +//! \fn void MGCRDiffusion::LoadCoefficients(const AthenaArray &D, +//! const AthenaArray &nlambda, int ngh) +//! \brief Load coefficients of the diffusion and source terms + +void MGCRDiffusion::LoadCoefficients(const AthenaArray &D, + const AthenaArray &nlambda, int ngh) { + AthenaArray &Dm=D_[nlevel_-1]; + AthenaArray &nl=nlambda_[nlevel_-1]; + int is, ie, js, je, ks, ke; + is=js=ks=ngh_; + ie=is+size_.nx1-1, je=js+size_.nx2-1, ke=ks+size_.nx3-1; + for (int mk=ks; mk<=ke; ++mk) { + int k = mk - ks + ngh; + for (int mj=js; mj<=je; ++mj) { + int j = mj - js + ngh; +#pragma omp simd + for (int mi=is; mi<=ie; ++mi) { + int i = mi - is + ngh; + Dm(XX,mk,mj,mi) = D(XX,k,j,i); + Dm(XY,mk,mj,mi) = D(XY,k,j,i); + Dm(XZ,mk,mj,mi) = D(XZ,k,j,i); + Dm(YY,mk,mj,mi) = D(YY,k,j,i); + Dm(YZ,mk,mj,mi) = D(YZ,k,j,i); + Dm(ZZ,mk,mj,mi) = D(ZZ,k,j,i); + nl(mk,mj,mi) = nlambda(k,j,i); + } + } + } + return; +} + diff --git a/src/crdiffusion/mg_crdiffusion.hpp b/src/crdiffusion/mg_crdiffusion.hpp new file mode 100644 index 0000000000..b29c8ff55d --- /dev/null +++ b/src/crdiffusion/mg_crdiffusion.hpp @@ -0,0 +1,69 @@ +#ifndef CRDIFFUSION_MG_CRDIFFUSION_HPP_ +#define CRDIFFUSION_MG_CRDIFFUSION_HPP_ +//======================================================================================== +// Athena++ astrophysical MHD code +// Copyright(C) 2014 James M. Stone and other code contributors +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +//! \file mg_crdiffusion.hpp +//! \brief defines MGCRDiffusion class + +// C headers + +// C++ headers + +// Athena++ headers +#include "../athena.hpp" +#include "../athena_arrays.hpp" +#include "../multigrid/multigrid.hpp" +#include "crdiffusion.hpp" + +class MeshBlock; +class ParameterInput; +class Coordinates; +class Multigrid; +class CRDiffusionBoundaryTaskList; + +//! \class MGCRDiffusion +//! \brief Multigrid CR Diffusion solver for each block + +class MGCRDiffusion : public Multigrid { + public: + MGCRDiffusion(MultigridDriver *pmd, MeshBlock *pmb); + ~MGCRDiffusion(); + + void Smooth(AthenaArray &dst, const AthenaArray &src, int rlev, + int il, int iu, int jl, int ju, int kl, int ku, int color, bool th) final; + void CalculateDefect(AthenaArray &def, const AthenaArray &u, + const AthenaArray &src, int rlev, + int il, int iu, int jl, int ju, int kl, int ku, bool th) final; + void CalculateFASRHS(AthenaArray &def, const AthenaArray &src, + int rlev, int il, int iu, int jl, int ju, int kl, int ku, bool th) final; + + void LoadCoefficients(const AthenaArray &D, + const AthenaArray &nlambda, int ngh); + + friend class MGCRDiffusionDriver; + + private: + AthenaArray *D_, *nlambda_; + + static constexpr Real omega_ = 1.15; +}; + + +//! \class MGCRDiffusionDriver +//! \brief Multigrid CR Diffusion solver + +class MGCRDiffusionDriver : public MultigridDriver { + public: + MGCRDiffusionDriver(Mesh *pm, ParameterInput *pin); + ~MGCRDiffusionDriver(); + void Solve(int stage) final; + void RestrictCoefficients(); + + private: + CRDiffusionBoundaryTaskList *crtlist_; +}; + +#endif // CRDIFFUSION_MG_CRDIFFUSION_HPP_ diff --git a/src/defs.hpp.in b/src/defs.hpp.in index 565b272dba..3edc1cf46d 100644 --- a/src/defs.hpp.in +++ b/src/defs.hpp.in @@ -56,6 +56,9 @@ // include cosmic ray transport? default=0 (false) #define CR_ENABLED @CR_ENABLED@ +// include cosmic ray diffusion? default=0 (false) +#define CRDIFFUSION_ENABLED @CRDIFFUSION_ENABLED@ + // enable special or general relativity? default=0 (false) #define RELATIVISTIC_DYNAMICS @RELATIVISTIC_DYNAMICS@ diff --git a/src/gravity/mg_gravity.cpp b/src/gravity/mg_gravity.cpp index db7a2d69db..ec9eace0af 100644 --- a/src/gravity/mg_gravity.cpp +++ b/src/gravity/mg_gravity.cpp @@ -145,8 +145,8 @@ MGGravityDriver::~MGGravityDriver() { //! \brief MGGravity constructor MGGravity::MGGravity(MultigridDriver *pmd, MeshBlock *pmb) : Multigrid(pmd, pmb, 1, 1) { - btype = BoundaryQuantity::mggrav; - btypef = BoundaryQuantity::mggrav_f; + btype = BoundaryQuantity::mg; + btypef = BoundaryQuantity::mg_faceonly; pmgbval = new MGGravityBoundaryValues(this, mg_block_bcs_); } diff --git a/src/mesh/mesh.cpp b/src/mesh/mesh.cpp index 5b4430e1eb..71b1c58c7b 100644 --- a/src/mesh/mesh.cpp +++ b/src/mesh/mesh.cpp @@ -31,6 +31,7 @@ #include "../bvals/bvals.hpp" #include "../coordinates/coordinates.hpp" #include "../cr/cr.hpp" +#include "../crdiffusion/crdiffusion.hpp" #include "../eos/eos.hpp" #include "../fft/athena_fft.hpp" #include "../fft/turbulence.hpp" @@ -1533,6 +1534,9 @@ void Mesh::Initialize(int res_flag, ParameterInput *pin) { if (CR_ENABLED) { pmb->pcr->cr_bvar.SendBoundaryBuffers(); } + if (CRDIFFUSION_ENABLED) { + pmb->pcrdiff->crbvar.SendBoundaryBuffers(); + } } // wait to receive conserved variables #pragma omp for private(pmb,pbval) @@ -1548,6 +1552,8 @@ void Mesh::Initialize(int res_flag, ParameterInput *pin) { pmb->pnrrad->rad_bvar.ReceiveAndSetBoundariesWithWait(); if (CR_ENABLED) pmb->pcr->cr_bvar.ReceiveAndSetBoundariesWithWait(); + if (CRDIFFUSION_ENABLED) + pmb->pcrdiff->crbvar.ReceiveAndSetBoundariesWithWait(); if (shear_periodic && orbital_advection==0) { pmb->phydro->hbvar.AddHydroShearForInit(); @@ -2068,7 +2074,7 @@ void Mesh::ReserveMeshBlockPhysIDs() { ReserveTagPhysIDs(FaceCenteredBoundaryVariable::max_phys_id); } if (SELF_GRAVITY_ENABLED) { - ReserveTagPhysIDs(CellCenteredBoundaryVariable::max_phys_id); + ReserveTagPhysIDs(1); } if (NSCALARS > 0) { ReserveTagPhysIDs(CellCenteredBoundaryVariable::max_phys_id); @@ -2078,6 +2084,8 @@ void Mesh::ReserveMeshBlockPhysIDs() { ReserveTagPhysIDs(RadBoundaryVariable::max_phys_id); if (CR_ENABLED) ReserveTagPhysIDs(CellCenteredBoundaryVariable::max_phys_id); + if (CRDIFFUSION_ENABLED) + ReserveTagPhysIDs(1); #endif return; diff --git a/src/mesh/mesh.hpp b/src/mesh/mesh.hpp index cd10539bb8..06af14baa1 100644 --- a/src/mesh/mesh.hpp +++ b/src/mesh/mesh.hpp @@ -48,6 +48,7 @@ class Hydro; class NRRadiation; class IMRadiation; class CosmicRay; +class CRDiffusion; class Field; class Particles; class PassiveScalars; @@ -120,6 +121,7 @@ class MeshBlock { Hydro *phydro; NRRadiation *pnrrad; CosmicRay *pcr; + CRDiffusion *pcrdiff; Field *pfield; Gravity *pgrav; PassiveScalars *pscalars; @@ -215,6 +217,7 @@ class Mesh { friend class TurbulenceDriver; friend class MultigridDriver; friend class MGGravityDriver; + friend class MGCRDiffusionDriver; friend class Gravity; friend class HydroDiffusion; friend class FieldDiffusion; @@ -345,7 +348,9 @@ class Mesh { FieldDiffusionCoeffFunc FieldDiffusivity_; OrbitalVelocityFunc OrbitalVelocity_, OrbitalVelocityDerivative_[2]; MGBoundaryFunc MGGravityBoundaryFunction_[6]; + MGBoundaryFunc MGCRDiffusionBoundaryFunction_[6]; MGSourceMaskFunc MGGravitySourceMaskFunction_; + MGSourceMaskFunc MGCRDiffusionSourceMaskFunction_; void AllocateRealUserMeshDataField(int n); void AllocateIntUserMeshDataField(int n); diff --git a/src/mesh/meshblock.cpp b/src/mesh/meshblock.cpp index a538f0d9e5..2171fe0fab 100644 --- a/src/mesh/meshblock.cpp +++ b/src/mesh/meshblock.cpp @@ -25,6 +25,7 @@ #include "../bvals/bvals.hpp" #include "../coordinates/coordinates.hpp" #include "../cr/cr.hpp" +#include "../crdiffusion/crdiffusion.hpp" #include "../eos/eos.hpp" #include "../fft/athena_fft.hpp" #include "../field/field.hpp" @@ -249,6 +250,11 @@ MeshBlock::MeshBlock(int igid, int ilid, LogicalLocation iloc, RegionSize input_ pbval->AdvanceCounterPhysID(CellCenteredBoundaryVariable::max_phys_id); } + if (CRDIFFUSION_ENABLED) { + pcrdiff = new CRDiffusion(this, pin); + pbval->AdvanceCounterPhysID(CellCenteredBoundaryVariable::max_phys_id); + } + // OrbitalAdvection: constructor depends on Coordinates, Hydro, Field, PassiveScalars. porb = new OrbitalAdvection(this, pin); @@ -440,6 +446,10 @@ MeshBlock::MeshBlock(int igid, int ilid, Mesh *pm, ParameterInput *pin, pbval->AdvanceCounterPhysID(CellCenteredBoundaryVariable::max_phys_id); } + if (CRDIFFUSION_ENABLED) { + pcrdiff = new CRDiffusion(this, pin); + pbval->AdvanceCounterPhysID(CellCenteredBoundaryVariable::max_phys_id); + } // OrbitalAdvection: constructor depends on Coordinates, Hydro, Field, PassiveScalars. porb = new OrbitalAdvection(this, pin); @@ -516,6 +526,12 @@ MeshBlock::MeshBlock(int igid, int ilid, Mesh *pm, ParameterInput *pin, std::memcpy(pcr->u_cr1.data(), &(mbdata[os]), pcr->u_cr1.GetSizeInBytes()); os += pcr->u_cr.GetSizeInBytes(); } + + if (CRDIFFUSION_ENABLED) { + std::memcpy(pcrdiff->ecr.data(), &(mbdata[os]), pcrdiff->ecr.GetSizeInBytes()); + os += pcrdiff->ecr.GetSizeInBytes(); + } + // (conserved variable) Passive scalars: if (NSCALARS > 0) { std::memcpy(pscalars->s.data(), &(mbdata[os]), pscalars->s.GetSizeInBytes()); @@ -552,6 +568,7 @@ MeshBlock::~MeshBlock() { if (NR_RADIATION_ENABLED || IM_RADIATION_ENABLED) delete pnrrad; if (CR_ENABLED) delete pcr; + if (CRDIFFUSION_ENABLED) delete pcrdiff; // BoundaryValues should be destructed AFTER all BoundaryVariable objects are destroyed delete pbval; @@ -655,6 +672,8 @@ std::size_t MeshBlock::GetBlockSizeInBytes() { size += pnrrad->ir.GetSizeInBytes(); if (CR_ENABLED) size += pcr->u_cr.GetSizeInBytes(); + if (CRDIFFUSION_ENABLED) + size += pcrdiff->ecr.GetSizeInBytes(); // calculate user MeshBlock data size for (int n=0; nu_cr.GetSizeInBytes(); + if (CRDIFFUSION_ENABLED) + size += pcrdiff->ecr.GetSizeInBytes(); // calculate user MeshBlock data size diff --git a/src/multigrid/multigrid.hpp b/src/multigrid/multigrid.hpp index f69368871b..4026910ac9 100644 --- a/src/multigrid/multigrid.hpp +++ b/src/multigrid/multigrid.hpp @@ -70,9 +70,9 @@ class Multigrid { virtual ~Multigrid(); MGBoundaryValues *pmgbval; - // KGF: Both btype=BoundaryQuantity::mggrav and btypef=BoundaryQuantity::mggrav_f (face + // KGF: Both btype=BoundaryQuantity::mg and btypef=BoundaryQuantity::mg_faceonly (face // neighbors only) are passed to comm function calls in mg_task_list.cpp Only - // BoundaryQuantity::mggrav is handled in a case in InitBoundaryData(). Passed directly + // BoundaryQuantity::mg is handled in a case in InitBoundaryData(). Passed directly // (not through btype) in MGBoundaryValues() ctor BoundaryQuantity btype, btypef; @@ -131,6 +131,7 @@ class Multigrid { friend class MGBoundaryValues; friend class MGGravityBoundaryValues; friend class MGGravityDriver; + friend class MGCRDiffusionDriver; protected: MultigridDriver *pmy_driver_; @@ -166,6 +167,7 @@ class MultigridDriver { friend class MGGravity; friend class MGBoundaryValues; friend class MGGravityBoundaryValues; + friend class MGCRDiffusion; protected: void CheckBoundaryFunctions(); @@ -223,7 +225,7 @@ class MultigridDriver { // pure virtual functions virtual void ProlongateOctetBoundariesFluxCons(AthenaArray &dst, - AthenaArray &cbuf, const AthenaArray &ncoarse) = 0; + AthenaArray &cbuf, const AthenaArray &ncoarse); int nranks_, nthreads_, nbtotal_, nvar_, mode_; int locrootlevel_, nrootlevel_, nmblevel_, ntotallevel_, nreflevel_, maxreflevel_; diff --git a/src/multigrid/multigrid_driver.cpp b/src/multigrid/multigrid_driver.cpp index 0b44c14b02..9f026d9db1 100644 --- a/src/multigrid/multigrid_driver.cpp +++ b/src/multigrid/multigrid_driver.cpp @@ -1862,6 +1862,20 @@ void MultigridDriver::ProlongateOctetBoundaries(AthenaArray &u, } +//---------------------------------------------------------------------------------------- +//! \fn void MultigridDriver::ProlongateOctetBoundariesFluxCons(AthenaArray &dst, +// AthenaArray &cbuf, const AthenaArray &ncoarse) +//! \brief prolongate octet boundaries using the flux conservation formula +// for the base class (just call the normal version) + +void MultigridDriver::ProlongateOctetBoundariesFluxCons(AthenaArray &dst, + AthenaArray &cbuf, const AthenaArray &ncoarse) { + ProlongateOctetBoundaries(dst,dst,cbuf,cbuf,ncoarse,false); + return; +} + + + //---------------------------------------------------------------------------------------- //! \fn void MultigridDriver::AllocateMultipoleCoefficients() // \brief Allocate arrays for multipole expansion coefficients diff --git a/src/outputs/athena_hdf5.cpp b/src/outputs/athena_hdf5.cpp index db6ff2a698..2cc5b732cb 100644 --- a/src/outputs/athena_hdf5.cpp +++ b/src/outputs/athena_hdf5.cpp @@ -143,6 +143,8 @@ void ATHDF5Output::WriteOutputFile(Mesh *pm, ParameterInput *pin, bool flag) { if(CR_ENABLED) num_variables[n_dataset] += 13; + if(CRDIFFUSION_ENABLED) + num_variables[n_dataset] += 2; // Passive scalars: if (NSCALARS > 0) diff --git a/src/outputs/outputs.cpp b/src/outputs/outputs.cpp index 5ec4278060..403b89f545 100644 --- a/src/outputs/outputs.cpp +++ b/src/outputs/outputs.cpp @@ -91,6 +91,7 @@ #include "../athena_arrays.hpp" #include "../coordinates/coordinates.hpp" #include "../cr/cr.hpp" +#include "../crdiffusion/crdiffusion.hpp" #include "../field/field.hpp" #include "../gravity/gravity.hpp" #include "../hydro/hydro.hpp" @@ -350,6 +351,7 @@ void OutputType::LoadOutputData(MeshBlock *pmb) { Field *pfld = pmb->pfield; NRRadiation *prad=pmb->pnrrad; CosmicRay *pcr=pmb->pcr; + CRDiffusion *pcrdiff=pmb->pcrdiff; PassiveScalars *psclr = pmb->pscalars; Gravity *pgrav = pmb->pgrav; OrbitalAdvection *porb = pmb->porb; @@ -1046,6 +1048,28 @@ void OutputType::LoadOutputData(MeshBlock *pmb) { } }// end Cosmic Rays + if (CRDIFFUSION_ENABLED) { + if (ContainVariable(output_params.variable, "ecr") || + ContainVariable(output_params.variable, "prim") || + ContainVariable(output_params.variable, "cons")) { + pod = new OutputData; + pod->type = "SCALARS"; + pod->name = "ecr"; + pod->data.InitWithShallowSlice(pcrdiff->ecr,4,0,1); + AppendOutputDataNode(pod); + num_vars_++; + } + if (ContainVariable(output_params.variable, "zeta") || + ContainVariable(output_params.variable, "prim") || + ContainVariable(output_params.variable, "cons")) { + pod = new OutputData; + pod->type = "SCALARS"; + pod->name = "ecr"; + pod->data.InitWithShallowSlice(pcrdiff->zeta,4,0,1); + AppendOutputDataNode(pod); + num_vars_++; + } + } diff --git a/src/outputs/restart.cpp b/src/outputs/restart.cpp index 302f50635d..86a0941fd3 100644 --- a/src/outputs/restart.cpp +++ b/src/outputs/restart.cpp @@ -22,6 +22,7 @@ #include "../athena.hpp" #include "../athena_arrays.hpp" #include "../cr/cr.hpp" +#include "../crdiffusion/crdiffusion.hpp" #include "../field/field.hpp" #include "../globals.hpp" #include "../hydro/hydro.hpp" @@ -189,6 +190,11 @@ void RestartOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, bool force_wr pdata += pmb->pcr->u_cr.GetSizeInBytes(); } + if (CRDIFFUSION_ENABLED) { + std::memcpy(pdata,pmb->pcrdiff->ecr.data(),pmb->pcrdiff->ecr.GetSizeInBytes()); + pdata += pmb->pcrdiff->ecr.GetSizeInBytes(); + } + // (conserved variable) Passive scalars: if (NSCALARS > 0) { AthenaArray &s = pmb->pscalars->s; diff --git a/src/pgen/cr_diffusion_mg.cpp b/src/pgen/cr_diffusion_mg.cpp new file mode 100644 index 0000000000..ecece80f81 --- /dev/null +++ b/src/pgen/cr_diffusion_mg.cpp @@ -0,0 +1,121 @@ +//====================================================================================== +/* Athena++ astrophysical MHD code + * Copyright (C) 2014 James M. Stone + * + * This program is free software: you can redistribute and/or modify it under the terms + * of the GNU General Public License (GPL) as published by the Free Software Foundation, + * either version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * You should have received a copy of GNU GPL in the file LICENSE included in the code + * distribution. If not see . + *====================================================================================*/ + +// C++ headers +#include // min +#include // sqrt +#include +#include // endl +#include // stringstream +#include // runtime_error +#include // c_str() + +// Athena++ headers +#include "../athena.hpp" +#include "../athena_arrays.hpp" +#include "../bvals/bvals.hpp" +#include "../coordinates/coordinates.hpp" +#include "../crdiffusion/crdiffusion.hpp" +#include "../crdiffusion/mg_crdiffusion.hpp" +#include "../eos/eos.hpp" +#include "../field/field.hpp" +#include "../globals.hpp" +#include "../hydro/hydro.hpp" +#include "../hydro/srcterms/hydro_srcterms.hpp" +#include "../mesh/mesh.hpp" +#include "../parameter_input.hpp" + + +#if !CRDIFFUSION_ENABLED +#error "The implicit CR diffusion solver must be enabled (-crdiff)." +#endif + +//====================================================================================== +//! \fn void MeshBlock::ProblemGenerator(ParameterInput *pin) +// \brief cosmic ray diffusion test +//====================================================================================== + +void MeshBlock::ProblemGenerator(ParameterInput *pin) { + Real gamma = peos->GetGamma(); + + for(int k=ks; k<=ke; ++k) { + Real x3 = pcoord->x3v(k); + for (int j=js; j<=je; ++j) { + Real x2 = pcoord->x2v(j); + for (int i=is; i<=ie; ++i) { + Real x1 = pcoord->x1v(i); + + phydro->u(IDN,k,j,i) = 1.0; + phydro->u(IM1,k,j,i) = 0.0; + phydro->u(IM2,k,j,i) = 0.0; + phydro->u(IM3,k,j,i) = 0.0; + if (NON_BAROTROPIC_EOS) + phydro->u(IEN,k,j,i) = phydro->u(IDN,k,j,i)/(gamma-1.0); + } + } + } + if (MAGNETIC_FIELDS_ENABLED) { + for (int k=ks; k<=ke; ++k) { + for (int j=js; j<=je; ++j) { + for (int i=is; i<=ie+1; ++i) { + pfield->b.x1f(k,j,i) = 1.0; + } + } + } + if (block_size.nx2 > 1) { + for (int k=ks; k<=ke; ++k) { + for (int j=js; j<=je+1; ++j) { + for (int i=is; i<=ie; ++i) { + pfield->b.x2f(k,j,i) = 1.0; + } + } + } + } + if (block_size.nx3 > 1) { + for (int k=ks; k<=ke+1; ++k) { + for (int j=js; j<=je; ++j) { + for (int i=is; i<=ie; ++i) { + pfield->b.x3f(k,j,i) = 0.0; + } + } + } + } + + pfield->CalculateCellCenteredField(pfield->b,pfield->bcc,pcoord,is,ie,js,je,ks,ke); + + for(int k=ks; k<=ke; ++k) { + for(int j=js; j<=je; ++j) { + for(int i=is; i<=ie; ++i) { + phydro->u(IEN,k,j,i) += + 0.5*(SQR((pfield->bcc(IB1,k,j,i))) + + SQR((pfield->bcc(IB2,k,j,i))) + + SQR((pfield->bcc(IB3,k,j,i)))); + } + } + } + } + + + for(int k=ks; k<=ke; ++k) { + for(int j=js; j<=je; ++j) { + for(int i=is; i<=ie; ++i) + pcrdiff->ecr(k,j,i) = 1.0; + } + } + + return; +} + diff --git a/src/task_list/crdiffusion_task_list.cpp b/src/task_list/crdiffusion_task_list.cpp new file mode 100644 index 0000000000..fba4d75e68 --- /dev/null +++ b/src/task_list/crdiffusion_task_list.cpp @@ -0,0 +1,125 @@ +//======================================================================================== +// Athena++ astrophysical MHD code +// Copyright(C) 2014 James M. Stone and other code contributors +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +//! \file grav_task_list.cpp +//! \brief function implementation for CRDiffusionBoundaryTaskList + +// C headers + +// C++ headers +#include // endl +#include // sstream +#include // runtime_error +#include // c_str() + +// Athena++ headers +#include "../athena.hpp" +#include "../crdiffusion/crdiffusion.hpp" +#include "../mesh/mesh.hpp" +#include "../parameter_input.hpp" +#include "crdiffusion_task_list.hpp" +#include "task_list.hpp" + +//---------------------------------------------------------------------------------------- +//! CRDiffusionBoundaryTaskList constructor + +CRDiffusionBoundaryTaskList::CRDiffusionBoundaryTaskList(ParameterInput *pin, Mesh *pm) { + // Now assemble list of tasks for each stage of time integrator + {using namespace CRDiffusionBoundaryTaskNames; // NOLINT (build/namespace) + AddTask(SEND_CRDIFF_BND,NONE); + AddTask(RECV_CRDIFF_BND,NONE); + AddTask(SETB_CRDIFF_BND,(RECV_CRDIFF_BND|SEND_CRDIFF_BND)); + if (pm->multilevel) { + AddTask(PROLONG_CRDIFF_BND,SETB_CRDIFF_BND); + AddTask(CRDIFF_PHYS_BND,PROLONG_CRDIFF_BND); + } else { + AddTask(CRDIFF_PHYS_BND,SETB_CRDIFF_BND); + } + AddTask(CLEAR_CRDIFF, CRDIFF_PHYS_BND); + } // end of using namespace block +} + +//---------------------------------------------------------------------------------------- +//! \fn void CRDiffusionBoundaryTaskList::AddTask(const TaskID& id, const TaskID& dep) +//! \brief Sets id and dependency for "ntask" member of task_list_ array, then iterates +//! value of ntask. + +void CRDiffusionBoundaryTaskList::AddTask(const TaskID& id, const TaskID& dep) { + task_list_[ntasks].task_id=id; + task_list_[ntasks].dependency=dep; + + using namespace CRDiffusionBoundaryTaskNames; // NOLINT (build/namespace) + if (id == CLEAR_CRDIFF) { + task_list_[ntasks].TaskFunc= + static_cast + (&CRDiffusionBoundaryTaskList::ClearCRDiffusionBoundary); + } else if (id == SEND_CRDIFF_BND) { + task_list_[ntasks].TaskFunc= + static_cast + (&CRDiffusionBoundaryTaskList::SendCRDiffusionBoundary); + } else if (id == RECV_CRDIFF_BND) { + task_list_[ntasks].TaskFunc= + static_cast + (&CRDiffusionBoundaryTaskList::ReceiveCRDiffusionBoundary); + } else if (id == SETB_CRDIFF_BND) { + task_list_[ntasks].TaskFunc= + static_cast + (&CRDiffusionBoundaryTaskList::SetCRDiffusionBoundary); + } else if (id == PROLONG_CRDIFF_BND) { + task_list_[ntasks].TaskFunc= + static_cast + (&CRDiffusionBoundaryTaskList::ProlongateCRDiffusionBoundary); + } else if (id == CRDIFF_PHYS_BND) { + task_list_[ntasks].TaskFunc= + static_cast + (&CRDiffusionBoundaryTaskList::PhysicalBoundary); + } else { + std::stringstream msg; + msg << "### FATAL ERROR in CRDiffusionBoundaryTaskList::AddTask" << std::endl + << "Invalid Task is specified" << std::endl; + ATHENA_ERROR(msg); + } + ntasks++; + return; +} + +void CRDiffusionBoundaryTaskList::StartupTaskList(MeshBlock *pmb, int stage) { + pmb->pcrdiff->crbvar.StartReceiving(BoundaryCommSubset::all); + return; +} + +TaskStatus CRDiffusionBoundaryTaskList::ClearCRDiffusionBoundary(MeshBlock *pmb, int stage) { + pmb->pcrdiff->crbvar.ClearBoundary(BoundaryCommSubset::all); + return TaskStatus::success; +} + +TaskStatus CRDiffusionBoundaryTaskList::SendCRDiffusionBoundary(MeshBlock *pmb, int stage) { + pmb->pcrdiff->crbvar.SendBoundaryBuffers(); + return TaskStatus::success; +} + +TaskStatus CRDiffusionBoundaryTaskList::ReceiveCRDiffusionBoundary(MeshBlock *pmb, + int stage) { + bool ret = pmb->pcrdiff->crbvar.ReceiveBoundaryBuffers(); + if (!ret) + return TaskStatus::fail; + return TaskStatus::success; +} + +TaskStatus CRDiffusionBoundaryTaskList::SetCRDiffusionBoundary(MeshBlock *pmb, int stage) { + pmb->pcrdiff->crbvar.SetBoundaries(); + return TaskStatus::success; +} + +TaskStatus CRDiffusionBoundaryTaskList::ProlongateCRDiffusionBoundary(MeshBlock *pmb, + int stage) { + pmb->pbval->ProlongateBoundariesPostMG(&(pmb->pcrdiff->crbvar)); + return TaskStatus::success; +} + +TaskStatus CRDiffusionBoundaryTaskList::PhysicalBoundary(MeshBlock *pmb, int stage) { + pmb->pcrdiff->ExpandPhysicalBoundaries(); + return TaskStatus::next; +} diff --git a/src/task_list/crdiffusion_task_list.hpp b/src/task_list/crdiffusion_task_list.hpp new file mode 100644 index 0000000000..a5b68d894e --- /dev/null +++ b/src/task_list/crdiffusion_task_list.hpp @@ -0,0 +1,58 @@ +#ifndef TASK_LIST_CRDIFFUSION_TASK_LIST_HPP_ +#define TASK_LIST_CRDIFFUSION_TASK_LIST_HPP_ +//======================================================================================== +// Athena++ astrophysical MHD code +// Copyright(C) 2014 James M. Stone and other code contributors +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +//! \file grav_task_list.hpp +//! \brief define CRDiffusionBoundaryTaskList + +// C headers + +// C++ headers +#include // std::uint64_t + +// Athena++ headers +#include "../athena.hpp" +#include "task_list.hpp" + +// forward declarations +class Mesh; +class MeshBlock; + +//---------------------------------------------------------------------------------------- +//! \class CRDiffusionBoundaryTaskList +//! \brief data and function definitions for CRDiffusionBoundaryTaskList derived class + +class CRDiffusionBoundaryTaskList : public TaskList { + public: + CRDiffusionBoundaryTaskList(ParameterInput *pin, Mesh *pm); + + // functions + TaskStatus ClearCRDiffusionBoundary(MeshBlock *pmb, int stage); + TaskStatus SendCRDiffusionBoundary(MeshBlock *pmb, int stage); + TaskStatus ReceiveCRDiffusionBoundary(MeshBlock *pmb, int stage); + TaskStatus SetCRDiffusionBoundary(MeshBlock *pmb, int stage); + TaskStatus ProlongateCRDiffusionBoundary(MeshBlock *pmb, int stage); + TaskStatus PhysicalBoundary(MeshBlock *pmb, int stage); + + private: + void AddTask(const TaskID& id, const TaskID& dep) override; + void StartupTaskList(MeshBlock *pmb, int stage) override; +}; + + +//---------------------------------------------------------------------------------------- +//! 64-bit integers with "1" in different bit positions used to ID each hydro task. +namespace CRDiffusionBoundaryTaskNames { +const TaskID NONE(0); +const TaskID CLEAR_CRDIFF(1); + +const TaskID SEND_CRDIFF_BND(2); +const TaskID RECV_CRDIFF_BND(3); +const TaskID SETB_CRDIFF_BND(4); +const TaskID PROLONG_CRDIFF_BND(5); +const TaskID CRDIFF_PHYS_BND(6); +} +#endif // TASK_LIST_CRDIFFUSION_TASK_LIST_HPP_ diff --git a/src/task_list/grav_task_list.cpp b/src/task_list/grav_task_list.cpp index c444664e3a..32786866da 100644 --- a/src/task_list/grav_task_list.cpp +++ b/src/task_list/grav_task_list.cpp @@ -16,11 +16,7 @@ // Athena++ headers #include "../athena.hpp" -#include "../eos/eos.hpp" -#include "../field/field.hpp" #include "../gravity/gravity.hpp" -#include "../hydro/hydro.hpp" -#include "../hydro/srcterms/hydro_srcterms.hpp" #include "../mesh/mesh.hpp" #include "../parameter_input.hpp" #include "grav_task_list.hpp" @@ -32,7 +28,6 @@ GravityBoundaryTaskList::GravityBoundaryTaskList(ParameterInput *pin, Mesh *pm) { // Now assemble list of tasks for each stage of time integrator {using namespace GravityBoundaryTaskNames; // NOLINT (build/namespace) - // compute hydro fluxes, integrate hydro variables AddTask(SEND_GRAV_BND,NONE); AddTask(RECV_GRAV_BND,NONE); AddTask(SETB_GRAV_BND,(RECV_GRAV_BND|SEND_GRAV_BND)); @@ -122,7 +117,7 @@ TaskStatus GravityBoundaryTaskList::SetGravityBoundary(MeshBlock *pmb, int stage TaskStatus GravityBoundaryTaskList::ProlongateGravityBoundary(MeshBlock *pmb, int stage) { - pmb->pbval->ProlongateGravityBoundaries(pmb->pmy_mesh->time, 0.0); + pmb->pbval->ProlongateBoundariesPostMG(&(pmb->pgrav->gbvar)); return TaskStatus::success; } diff --git a/src/utils/show_config.cpp b/src/utils/show_config.cpp index f2dae42369..17f3adb919 100644 --- a/src/utils/show_config.cpp +++ b/src/utils/show_config.cpp @@ -56,6 +56,11 @@ void ShowConfig() { } else { std::cout<<" Cosmic Ray Transport: OFF" << std::endl; } + if (CRDIFFUSION_ENABLED) { + std::cout<<" Cosmic Ray Diffusion: ON" << std::endl; + } else { + std::cout<<" Cosmic Ray Diffusion: OFF" << std::endl; + } // configure.py output: "Frame transformations" if (SELF_GRAVITY_ENABLED == 1) { From 733016084898c85ef3cc200dcc6dbb1a38e35bbf Mon Sep 17 00:00:00 2001 From: Kengo Tomida Date: Tue, 21 Nov 2023 23:47:25 +0900 Subject: [PATCH 03/33] *** incomplete commit *** working on multigrid CR diffusion This also includes refactoring of the original multigrid --- src/crdiffusion/mg_crdiffusion.cpp | 62 ++++++++++++++++ src/crdiffusion/mg_crdiffusion.hpp | 1 + src/multigrid/multigrid.cpp | 17 +++-- src/multigrid/multigrid.hpp | 15 +++- src/multigrid/multigrid_driver.cpp | 110 +++++++++++++++++------------ 5 files changed, 149 insertions(+), 56 deletions(-) diff --git a/src/crdiffusion/mg_crdiffusion.cpp b/src/crdiffusion/mg_crdiffusion.cpp index 0f56e9672a..97c538e594 100644 --- a/src/crdiffusion/mg_crdiffusion.cpp +++ b/src/crdiffusion/mg_crdiffusion.cpp @@ -131,6 +131,49 @@ MGCRDiffusion::~MGCRDiffusion() { } +//---------------------------------------------------------------------------------------- +//! \fn void MGCRDiffusionDriver::RestrictCoefficientsOctets() +//! \brief restrict coefficients in Octets for the CR diffusion equation + +void MGCRDiffusionDriver::RestrictCoefficients() { + if (nreflevel_ > 0) { + const int &ngh = mgroot_->ngh_; + for (int l = nreflevel_ - 1; l >= 1; --l) { // fine octets to coarse octets +#pragma omp parallel for num_threads(nthreads_) + for (int o = 0; o < noctets_[l]; ++o) { + const LogicalLocation &loc = octets_[l][o].loc; + LogicalLocation cloc; + cloc.lx1 = (loc.lx1 >> 1); + cloc.lx2 = (loc.lx2 >> 1); + cloc.lx3 = (loc.lx3 >> 1); + cloc.level = loc.level - 1; + int oid = octetmap_[l-1][cloc]; + int oi = (static_cast(loc.lx1) & 1) + ngh; + int oj = (static_cast(loc.lx2) & 1) + ngh; + int ok = (static_cast(loc.lx3) & 1) + ngh; + +/* for (int v = 0; v < NCOEFF; ++v) + octets_[l-1][oid].D_(v, ok, oj, oi) = RestrictOne(octets_[l][o].D_, + v, ngh, ngh, ngh); + octets_[l-1][oid].nlambda_(ok, oj, oi) = RestrictOne(octets_[l][o].nlambda_, + 0, ngh, ngh, ngh);*/ + + } + } +#pragma omp parallel for num_threads(nthreads_) + for (int o = 0; o < noctets_[0]; ++o) { // octets to the root grid + const LogicalLocation &loc = octets_[0][o].loc; + +// for (int v = 0; v < nvar_; ++v) +// Real r = RestrictOne(octets_[0][o].D_, v, ngh, ngh, ngh);//****** + } + } + + return; + +} + + //---------------------------------------------------------------------------------------- //! \fn void MGCRDiffusionDriver::Solve(int stage) //! \brief load the data and solve @@ -181,6 +224,25 @@ void MGCRDiffusionDriver::Solve(int stage) { } +//---------------------------------------------------------------------------------------- +//! \fn void MGCRDiffusion::RestrictCoefficients() +//! \brief restrict coefficients within a MGCRDiffusion object + +void MGCRDiffusion::RestrictCoefficients() { + int is, ie, js, je, ks, ke; + is=js=ks=ngh_; + for (current_level_=nlevel_-1; current_level_>0; current_level_--) { + int ll=nlevel_-current_level_; + ie=is+(size_.nx1>>ll)-1, je=js+(size_.nx2>>ll)-1, ke=ks+(size_.nx3>>ll)-1; + Restrict(D_[current_level_-1], D_[current_level_], + NCOEFF, is, ie, js, je, ks, ke, false); + Restrict(nlambda_[current_level_-1], nlambda_[current_level_], + 1, is, ie, js, je, ks, ke, false); + } + return; +} + + //---------------------------------------------------------------------------------------- //! \fn void MGCRDiffusion::Smooth(AthenaArray &u, const AthenaArray &src, //! int rlev, int il, int iu, int jl, int ju, int kl, int ku, int color, bool th) diff --git a/src/crdiffusion/mg_crdiffusion.hpp b/src/crdiffusion/mg_crdiffusion.hpp index b29c8ff55d..3fe8ae3509 100644 --- a/src/crdiffusion/mg_crdiffusion.hpp +++ b/src/crdiffusion/mg_crdiffusion.hpp @@ -42,6 +42,7 @@ class MGCRDiffusion : public Multigrid { void LoadCoefficients(const AthenaArray &D, const AthenaArray &nlambda, int ngh); + void RestrictCoefficients(); friend class MGCRDiffusionDriver; diff --git a/src/multigrid/multigrid.cpp b/src/multigrid/multigrid.cpp index 4770ab4c35..ef9a58ecfe 100644 --- a/src/multigrid/multigrid.cpp +++ b/src/multigrid/multigrid.cpp @@ -256,7 +256,8 @@ void Multigrid::RestrictFMGSource() { for (current_level_=nlevel_-1; current_level_>0; current_level_--) { int ll=nlevel_-current_level_; ie=is+(size_.nx1>>ll)-1, je=js+(size_.nx2>>ll)-1, ke=ks+(size_.nx3>>ll)-1; - Restrict(src_[current_level_-1], src_[current_level_], is, ie, js, je, ks, ke, false); + Restrict(src_[current_level_-1], src_[current_level_], + nvar_, is, ie, js, je, ks, ke, false); } return; } @@ -341,11 +342,13 @@ void Multigrid::RestrictBlock() { is=js=ks=ngh_; ie=is+(size_.nx1>>ll)-1, je=js+(size_.nx2>>ll)-1, ke=ks+(size_.nx3>>ll)-1; - Restrict(src_[current_level_-1], def_[current_level_], is, ie, js, je, ks, ke, th); + Restrict(src_[current_level_-1], def_[current_level_], + nvar_, is, ie, js, je, ks, ke, th); // Full Approximation Scheme - restrict the variable itself if (pmy_driver_->ffas_) - Restrict(u_[current_level_-1], u_[current_level_], is, ie, js, je, ks, ke, th); + Restrict(u_[current_level_-1], u_[current_level_], + nvar_, is, ie, js, je, ks, ke, th); current_level_--; @@ -678,13 +681,13 @@ void Multigrid::SetData(MGVariable type, int n, int k, int j, int i, Real v) { //---------------------------------------------------------------------------------------- //! \fn void Multigrid::Restrict(AthenaArray &dst, const AthenaArray &src, -// int il, int iu, int jl, int ju, int kl, int ku, bool th) +// int nvar, int il, int iu, int jl, int ju, int kl, int ku, bool th) // \brief Actual implementation of prolongation and correction void Multigrid::Restrict(AthenaArray &dst, const AthenaArray &src, - int il, int iu, int jl, int ju, int kl, int ku, bool th) { + int nvar, int il, int iu, int jl, int ju, int kl, int ku, bool th) { if (th == true && (ku-kl) >= minth_) { - for (int v=0; vnthreads_) for (int k=kl; k<=ku; ++k) { int fk = 2*k - kl; @@ -702,7 +705,7 @@ void Multigrid::Restrict(AthenaArray &dst, const AthenaArray &src, } } } else { - for (int v=0; v &dst, const AthenaArray &src, - int il, int iu, int jl, int ju, int kl, int ku, bool th); + int nvar, int il, int iu, int jl, int ju, int kl, int ku, bool th); void ProlongateAndCorrect(AthenaArray &dst, const AthenaArray &src, int il, int iu, int jl, int ju, int kl, int ku, int fil, int fjl, int fkl, bool th); void FMGProlongate(AthenaArray &dst, const AthenaArray &src, @@ -283,18 +283,27 @@ struct MGCoordinates { }; -//! \struct MGOctet +//! \class MGOctet // \brief structure containing eight (+ ghost) cells for Mesh Refinement -struct MGOctet { +class MGOctet { public: LogicalLocation loc; bool fleaf; AthenaArray u, def, src, uold; MGCoordinates coord, ccoord; + + virtual void AllocateOctet(int nvar, int ncoct, int nccoct, bool ffas); }; +inline Real RestrictOne(const AthenaArray &src, int v, int fi, int fj, int fk) { + return 0.125*(src(v, fk, fj, fi)+src(v, fk, fj, fi+1) + +src(v, fk, fj+1, fi)+src(v, fk, fj+1, fi+1) + +src(v, fk+1, fj, fi)+src(v, fk+1, fj, fi+1) + +src(v, fk+1, fj+1, fi)+src(v, fk+1, fj+1, fi+1)); +} + // Multigrid Boundary functions diff --git a/src/multigrid/multigrid_driver.cpp b/src/multigrid/multigrid_driver.cpp index 9f026d9db1..d25a2b5890 100644 --- a/src/multigrid/multigrid_driver.cpp +++ b/src/multigrid/multigrid_driver.cpp @@ -91,6 +91,19 @@ MultigridDriver::MultigridDriver(Mesh *pm, MGBoundaryFunc *MGBoundary, octetbflag_ = new std::vector[maxreflevel_]; noctets_ = new int[maxreflevel_](); pmaxnoct_ = new int[maxreflevel_](); + + int nth = 1; +#ifdef OPENMP_PARALLEL + nth = omp_get_max_threads(); +#endif + cbuf_ = new AthenaArray[nth]; + cbufold_ = new AthenaArray[nth]; + ncoarse_ = new AthenaArray[nth]; + for (int n = 0; n < nth; ++n) { + cbuf_[n].NewAthenaArray(nvar_,3,3,3); + cbufold_[n].NewAthenaArray(nvar_,3,3,3); + ncoarse_[n].NewAthenaArray(3,3,3); + } } } @@ -111,6 +124,9 @@ MultigridDriver::~MultigridDriver() { delete [] octetbflag_; delete [] noctets_; delete [] pmaxnoct_; + delete [] cbuf_; + delete [] cbufold_; + delete [] ncoarse_; } if (mporder_ > 0) delete [] mpcoeff_; @@ -372,15 +388,8 @@ void MultigridDriver::SetupMultigrid() { } pmy_mesh_->tree.GetMGOctetList(octets_, octetmap_, noctets_); for (int l = 0; l < nreflevel_; ++l) { - for (int o = pmaxnoct_[l]; o < noctets_[l]; ++o) { - octets_[l][o].u.NewAthenaArray(nvar_, ncoct, ncoct, ncoct); - octets_[l][o].def.NewAthenaArray(nvar_, ncoct, ncoct, ncoct); - octets_[l][o].src.NewAthenaArray(nvar_, ncoct, ncoct, ncoct); - octets_[l][o].coord.AllocateMGCoordinates(ncoct, ncoct, ncoct); - octets_[l][o].ccoord.AllocateMGCoordinates(nccoct, nccoct, nccoct); - if (ffas_) - octets_[l][o].uold.NewAthenaArray(nvar_, ncoct, ncoct, ncoct); - } + for (int o = pmaxnoct_[l]; o < noctets_[l]; ++o) + octets_[l][o].AllocateOctet(nvar_, ncoct, nccoct, ffas_); } } @@ -944,8 +953,6 @@ void MultigridDriver::RestrictFMGSourceOctets() { for (int l = nreflevel_ - 1; l >= 1; --l) { // fine octets to coarse octets #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[l]; ++o) { - AthenaArray cbuf; - cbuf.NewAthenaArray(nvar_, 3, 3, 3); const LogicalLocation &loc = octets_[l][o].loc; LogicalLocation cloc; cloc.lx1 = (loc.lx1 >> 1); @@ -956,21 +963,18 @@ void MultigridDriver::RestrictFMGSourceOctets() { int oi = (static_cast(loc.lx1) & 1) + ngh; int oj = (static_cast(loc.lx2) & 1) + ngh; int ok = (static_cast(loc.lx3) & 1) + ngh; - mgroot_->Restrict(cbuf, octets_[l][o].src, ngh, ngh, ngh, ngh, ngh, ngh, false); for (int v = 0; v < nvar_; ++v) - octets_[l-1][oid].src(v, ok, oj, oi) = cbuf(v, ngh, ngh, ngh); + octets_[l-1][oid].src(v, ok, oj, oi) = RestrictOne(octets_[l][o].src, + v, ngh, ngh, ngh); } } #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[0]; ++o) { // octets to the root grid - AthenaArray cbuf; - cbuf.NewAthenaArray(nvar_, 3, 3, 3); const LogicalLocation &loc = octets_[0][o].loc; - mgroot_->Restrict(cbuf, octets_[0][o].src, ngh, ngh, ngh, ngh, ngh, ngh, false); for (int v = 0; v < nvar_; ++v) mgroot_->SetData(MGVariable::src, v, static_cast(loc.lx3), static_cast(loc.lx2), static_cast(loc.lx1), - cbuf(v, ngh, ngh, ngh)); + RestrictOne(octets_[0][o].src, v, ngh, ngh, ngh)); } } @@ -989,8 +993,6 @@ void MultigridDriver::RestrictOctets() { if (lev >= 1) { // fine octets to coarse octets #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[lev]; ++o) { - AthenaArray cbuf; - cbuf.NewAthenaArray(nvar_, 3, 3, 3); const LogicalLocation &loc = octets_[lev][o].loc; LogicalLocation cloc; cloc.lx1 = (loc.lx1 >> 1); @@ -1003,33 +1005,31 @@ void MultigridDriver::RestrictOctets() { int ok = (static_cast(loc.lx3) & 1) + ngh; mgroot_->CalculateDefect(octets_[lev][o].def, octets_[lev][o].u, octets_[lev][o].src, lev+1, os_, oe_, os_, oe_, os_, oe_, false); - mgroot_->Restrict(cbuf, octets_[lev][o].def, ngh, ngh, ngh, ngh, ngh, ngh, false); for (int v = 0; v < nvar_; ++v) - octets_[lev-1][oid].src(v, ok, oj, oi) = cbuf(v, ngh, ngh, ngh); + octets_[lev-1][oid].src(v, ok, oj, oi) = RestrictOne(octets_[lev][o].def, + v, ngh, ngh, ngh); if (ffas_) { - mgroot_->Restrict(cbuf, octets_[lev][o].u, ngh, ngh, ngh, ngh, ngh, ngh, false); for (int v = 0; v < nvar_; ++v) - octets_[lev-1][oid].u(v, ok, oj, oi) = cbuf(v, ngh, ngh, ngh); + octets_[lev-1][oid].u(v, ok, oj, oi) = RestrictOne(octets_[lev][o].u, + v, ngh, ngh, ngh); } } } else { // octets to the root grid #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[0]; ++o) { - AthenaArray cbuf; - cbuf.NewAthenaArray(nvar_, 3, 3, 3); const LogicalLocation &loc = octets_[0][o].loc; int ri = static_cast(loc.lx1); int rj = static_cast(loc.lx2); int rk = static_cast(loc.lx3); mgroot_->CalculateDefect(octets_[0][o].def, octets_[0][o].u, octets_[0][o].src, 1, os_, oe_, os_, oe_, os_, oe_, false); - mgroot_->Restrict(cbuf, octets_[0][o].def, ngh, ngh, ngh, ngh, ngh, ngh, false); for (int v = 0; v < nvar_; ++v) - mgroot_->SetData(MGVariable::src, v, rk, rj, ri, cbuf(v, ngh, ngh, ngh)); + mgroot_->SetData(MGVariable::src, v, rk, rj, ri, + RestrictOne(octets_[0][o].def, v, ngh, ngh, ngh)); if (ffas_) { - mgroot_->Restrict(cbuf, octets_[0][o].u, ngh, ngh, ngh, ngh, ngh, ngh, false); for (int v = 0; v < nvar_; ++v) - mgroot_->SetData(MGVariable::u, v, rk, rj, ri, cbuf(v, ngh, ngh, ngh)); + mgroot_->SetData(MGVariable::u, v, rk, rj, ri, + RestrictOne(octets_[0][o].u, v, ngh, ngh, ngh)); } } } @@ -1114,8 +1114,11 @@ void MultigridDriver::ProlongateAndCorrectOctets() { const AthenaArray &uold = mgroot_->GetCurrentOldData(); #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[0]; ++o) { - AthenaArray cbuf; - cbuf.NewAthenaArray(nvar_, 3, 3, 3); + int th = 0; +#ifdef OPENMP_PARALLEL + th = omp_get_thread_num(); +#endif + AthenaArray &cbuf = cbuf_[th]; const LogicalLocation &loc = octets_[0][o].loc; int ri = static_cast(loc.lx1) + ngh - 1; int rj = static_cast(loc.lx2) + ngh - 1; @@ -1140,8 +1143,11 @@ void MultigridDriver::ProlongateAndCorrectOctets() { } else { // from coarse octets to fine octets #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[flev]; ++o) { - AthenaArray cbuf; - cbuf.NewAthenaArray(nvar_, 3, 3, 3); + int th = 0; +#ifdef OPENMP_PARALLEL + th = omp_get_thread_num(); +#endif + AthenaArray &cbuf = cbuf_[th]; const LogicalLocation &loc = octets_[flev][o].loc; LogicalLocation cloc; cloc.lx1 = (loc.lx1 >> 1); @@ -1227,11 +1233,14 @@ void MultigridDriver::SetBoundariesOctets(bool fprolong, bool folddata) { #pragma omp parallel for num_threads(nthreads_) schedule(dynamic,1) for (int o = 0; o < noctets_[lev]; ++o) { if (fprolong && octets_[lev][o].fleaf == true) continue; - AthenaArray cbuf, cbufold; - AthenaArray ncoarse; - cbuf.NewAthenaArray(nvar_, 3, 3, 3); - cbufold.NewAthenaArray(nvar_, 3, 3, 3); - ncoarse.NewAthenaArray(3, 3, 3); + int th = 0; +#ifdef OPENMP_PARALLEL + th = omp_get_thread_num(); +#endif + AthenaArray &cbuf = cbuf_[th]; + AthenaArray &cbufold = cbufold_[th]; + AthenaArray &ncoarse = ncoarse_[th]; + ncoarse.ZeroClear(); const LogicalLocation &loc = octets_[lev][o].loc; AthenaArray &u = octets_[lev][o].u; AthenaArray &uold = octets_[lev][o].uold; @@ -1584,8 +1593,6 @@ void MultigridDriver::RestrictOctetsBeforeTransfer() { for (int l = nreflevel_ - 1; l >= 1; --l) { // fine octets to coarse octets #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[l]; ++o) { - AthenaArray cbuf; - cbuf.NewAthenaArray(nvar_, 3, 3, 3); const LogicalLocation &loc = octets_[l][o].loc; LogicalLocation cloc; cloc.lx1 = (loc.lx1 >> 1); @@ -1596,21 +1603,18 @@ void MultigridDriver::RestrictOctetsBeforeTransfer() { int oi = (static_cast(loc.lx1) & 1) + ngh; int oj = (static_cast(loc.lx2) & 1) + ngh; int ok = (static_cast(loc.lx3) & 1) + ngh; - mgroot_->Restrict(cbuf, octets_[l][o].u, ngh, ngh, ngh, ngh, ngh, ngh, false); for (int v = 0; v < nvar_; ++v) - octets_[l-1][oid].u(v, ok, oj, oi) = cbuf(v, ngh, ngh, ngh); + octets_[l-1][oid].u(v, ok, oj, oi) = RestrictOne(octets_[l][o].u, + v, ngh, ngh, ngh); } } #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[0]; ++o) { // octets to the root grid - AthenaArray cbuf; - cbuf.NewAthenaArray(nvar_, 3, 3, 3); const LogicalLocation &loc = octets_[0][o].loc; - mgroot_->Restrict(cbuf, octets_[0][o].u, ngh, ngh, ngh, ngh, ngh, ngh, false); for (int v = 0; v < nvar_; ++v) mgroot_->SetData(MGVariable::u, v, static_cast(loc.lx3), static_cast(loc.lx2), static_cast(loc.lx1), - cbuf(v, ngh, ngh, ngh)); + RestrictOne(octets_[0][o].u, v, ngh, ngh, ngh)); } return; @@ -2008,3 +2012,17 @@ void MultigridDriver::CalculateCenterOfMass() { return; } + + +//---------------------------------------------------------------------------------------- +//! \fn void MGOctet::AllocateOctet(int nvar, int ncoct, int nccoct, bool ffas) +// \brief MGOctet construcor - allocate arrays and coordinates +void MGOctet::AllocateOctet(int nvar, int ncoct, int nccoct, bool ffas) { + u.NewAthenaArray(nvar, ncoct, ncoct, ncoct); + def.NewAthenaArray(nvar, ncoct, ncoct, ncoct); + src.NewAthenaArray(nvar, ncoct, ncoct, ncoct); + coord.AllocateMGCoordinates(ncoct, ncoct, ncoct); + ccoord.AllocateMGCoordinates(nccoct, nccoct, nccoct); + if (ffas) + uold.NewAthenaArray(nvar, ncoct, ncoct, ncoct); +} From a038ba0218c43a8dd29df18167fb788c37282758 Mon Sep 17 00:00:00 2001 From: Kengo Tomida Date: Wed, 22 Nov 2023 22:10:57 +0900 Subject: [PATCH 04/33] *** incomplete commit *** implemented RestrictCoefficients Also refactored octets --- src/crdiffusion/crdiffusion.hpp | 1 - src/crdiffusion/mg_crdiffusion.cpp | 45 ++++---- src/crdiffusion/mg_crdiffusion.hpp | 9 ++ src/mesh/meshblock_tree.cpp | 8 +- src/mesh/meshblock_tree.hpp | 2 +- src/multigrid/multigrid.cpp | 4 +- src/multigrid/multigrid.hpp | 16 ++- src/multigrid/multigrid_driver.cpp | 162 +++++++++++++++-------------- 8 files changed, 132 insertions(+), 115 deletions(-) diff --git a/src/crdiffusion/crdiffusion.hpp b/src/crdiffusion/crdiffusion.hpp index d1ac48cbcc..5c2afdf9e8 100644 --- a/src/crdiffusion/crdiffusion.hpp +++ b/src/crdiffusion/crdiffusion.hpp @@ -18,7 +18,6 @@ #include "../athena_arrays.hpp" #include "../bvals/bvals.hpp" #include "../bvals/cc/bvals_cc.hpp" -#include "mg_crdiffusion.hpp" class MeshBlock; class ParameterInput; diff --git a/src/crdiffusion/mg_crdiffusion.cpp b/src/crdiffusion/mg_crdiffusion.cpp index 97c538e594..6ad9605295 100644 --- a/src/crdiffusion/mg_crdiffusion.cpp +++ b/src/crdiffusion/mg_crdiffusion.cpp @@ -83,7 +83,7 @@ MGCRDiffusionDriver::MGCRDiffusionDriver(Mesh *pm, ParameterInput *pin) mgtlist_ = new MultigridTaskList(this); // Allocate the root multigrid - mgroot_ = new MGCRDiffusion(this, nullptr); + mgroot_ = mgcrroot_ = new MGCRDiffusion(this, nullptr); crtlist_ = new CRDiffusionBoundaryTaskList(pin, pm); } @@ -107,10 +107,10 @@ MGCRDiffusionDriver::~MGCRDiffusionDriver() { MGCRDiffusion::MGCRDiffusion(MultigridDriver *pmd, MeshBlock *pmb) : Multigrid(pmd, pmb, 1, 1) { for (int l = 0; l < nlevel_; l++) { - int ll=nlevel_-1-l; - int ncx=(size_.nx1>>ll)+2*ngh_; - int ncy=(size_.nx2>>ll)+2*ngh_; - int ncz=(size_.nx3>>ll)+2*ngh_; + int ll = nlevel_-1-l; + int ncx = (size_.nx1>>ll) + 2*ngh_; + int ncy = (size_.nx2>>ll) + 2*ngh_; + int ncz = (size_.nx3>>ll) + 2*ngh_; D_[l].NewAthenaArray(NCOEFF,ncz,ncy,ncx); nlambda_[l].NewAthenaArray(ncz,ncy,ncx); } @@ -136,12 +136,17 @@ MGCRDiffusion::~MGCRDiffusion() { //! \brief restrict coefficients in Octets for the CR diffusion equation void MGCRDiffusionDriver::RestrictCoefficients() { + for (auto itr = vmg_.begin(); itr < vmg_.end(); itr++) { + MGCRDiffusion *pmg = static_cast(*itr); + pmg->RestrictCoefficients(); + } if (nreflevel_ > 0) { - const int &ngh = mgroot_->ngh_; + const int &ngh = mgcrroot_->ngh_; for (int l = nreflevel_ - 1; l >= 1; --l) { // fine octets to coarse octets #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[l]; ++o) { - const LogicalLocation &loc = octets_[l][o].loc; + MGCROctet *foct = static_cast(octets_[l][o]); + const LogicalLocation &loc = foct->loc; LogicalLocation cloc; cloc.lx1 = (loc.lx1 >> 1); cloc.lx2 = (loc.lx2 >> 1); @@ -151,24 +156,28 @@ void MGCRDiffusionDriver::RestrictCoefficients() { int oi = (static_cast(loc.lx1) & 1) + ngh; int oj = (static_cast(loc.lx2) & 1) + ngh; int ok = (static_cast(loc.lx3) & 1) + ngh; - -/* for (int v = 0; v < NCOEFF; ++v) - octets_[l-1][oid].D_(v, ok, oj, oi) = RestrictOne(octets_[l][o].D_, - v, ngh, ngh, ngh); - octets_[l-1][oid].nlambda_(ok, oj, oi) = RestrictOne(octets_[l][o].nlambda_, - 0, ngh, ngh, ngh);*/ + MGCROctet *coct = static_cast(octets_[l-1][oid]); + for (int v = 0; v < NCOEFF; ++v) + coct->D(v, ok, oj, oi) = RestrictOne(foct->D, v, ngh, ngh, ngh); + coct->nlambda(ok, oj, oi) = RestrictOne(foct->nlambda, 0, ngh, ngh, ngh); } } #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[0]; ++o) { // octets to the root grid - const LogicalLocation &loc = octets_[0][o].loc; - -// for (int v = 0; v < nvar_; ++v) -// Real r = RestrictOne(octets_[0][o].D_, v, ngh, ngh, ngh);//****** + const LogicalLocation &loc = octets_[0][o]->loc; + MGCROctet *oct = static_cast(octets_[0][o]); + int lx1 = static_cast(loc.lx1) + mgcrroot_->ngh_; + int lx2 = static_cast(loc.lx2) + mgcrroot_->ngh_; + int lx3 = static_cast(loc.lx3) + mgcrroot_->ngh_; + for (int v = 0; v < NCOEFF; ++v) + mgcrroot_->D_[mgcrroot_->nlevel_-1](v, lx3, lx2, lx1) + = RestrictOne(oct->D, v, ngh, ngh, ngh); + mgcrroot_->nlambda_[mgcrroot_->nlevel_-1](lx3, lx2, lx1) + = RestrictOne(oct->nlambda, 0, ngh, ngh, ngh); } } - + mgcrroot_->RestrictCoefficients(); return; } diff --git a/src/crdiffusion/mg_crdiffusion.hpp b/src/crdiffusion/mg_crdiffusion.hpp index 3fe8ae3509..90e1f9c9b3 100644 --- a/src/crdiffusion/mg_crdiffusion.hpp +++ b/src/crdiffusion/mg_crdiffusion.hpp @@ -24,6 +24,7 @@ class Coordinates; class Multigrid; class CRDiffusionBoundaryTaskList; + //! \class MGCRDiffusion //! \brief Multigrid CR Diffusion solver for each block @@ -64,7 +65,15 @@ class MGCRDiffusionDriver : public MultigridDriver { void RestrictCoefficients(); private: + MGCRDiffusion *mgcrroot_; CRDiffusionBoundaryTaskList *crtlist_; }; +class MGCROctet : public MGOctet { + public: + MGCROctet(int nvar, int ncoct, int nccoct) : MGOctet(nvar, ncoct, nccoct), + D(NCOEFF,ncoct, ncoct, ncoct), nlambda(ncoct, ncoct, ncoct) {}; + AthenaArray D, nlambda; +}; + #endif // CRDIFFUSION_MG_CRDIFFUSION_HPP_ diff --git a/src/mesh/meshblock_tree.cpp b/src/mesh/meshblock_tree.cpp index 6db77bb6ac..f9dc8fc582 100644 --- a/src/mesh/meshblock_tree.cpp +++ b/src/mesh/meshblock_tree.cpp @@ -505,7 +505,7 @@ void MeshBlockTree::CountMGOctets(int *noct) { //! std::unordered_map *octmap, int *noct) //! \brief construct lists of octets Multigrid with mesh refinement -void MeshBlockTree::GetMGOctetList(std::vector *oct, +void MeshBlockTree::GetMGOctetList(std::vector *oct, std::unordered_map *octmap, int *noct) { if (pleaf_ == nullptr) return; @@ -513,15 +513,15 @@ void MeshBlockTree::GetMGOctetList(std::vector *oct, int oid = 0; if (lev >= 0) { oid = noct[lev]; - oct[lev][oid].loc = loc_; + oct[lev][oid]->loc = loc_; + oct[lev][oid]->fleaf = true; octmap[lev][loc_] = oid; - oct[lev][oid].fleaf = true; noct[lev]++; } for (int n=0; npleaf_ != nullptr) { - if (lev >= 0) oct[lev][oid].fleaf = false; + if (lev >= 0) oct[lev][oid]->fleaf = false; pleaf_[n]->GetMGOctetList(oct, octmap, noct); } } diff --git a/src/mesh/meshblock_tree.hpp b/src/mesh/meshblock_tree.hpp index 72f4fb18ae..4abe5a9b33 100644 --- a/src/mesh/meshblock_tree.hpp +++ b/src/mesh/meshblock_tree.hpp @@ -56,7 +56,7 @@ class MeshBlockTree { MeshBlockTree* FindNeighbor(LogicalLocation myloc, int ox1, int ox2, int ox3, BoundaryFlag *bcs, bool amrflag=false); void CountMGOctets(int *noct); - void GetMGOctetList(std::vector *oct, + void GetMGOctetList(std::vector *oct, std::unordered_map *octmap, int *noct); private: diff --git a/src/multigrid/multigrid.cpp b/src/multigrid/multigrid.cpp index ef9a58ecfe..808e2b8689 100644 --- a/src/multigrid/multigrid.cpp +++ b/src/multigrid/multigrid.cpp @@ -522,7 +522,7 @@ void Multigrid::SetFromRootGrid(bool folddata) { int ci = (static_cast(loc_.lx1)&1); int cj = (static_cast(loc_.lx2)&1); int ck = (static_cast(loc_.lx3)&1); - const AthenaArray &src = pmy_driver_->octets_[olev][oid].u; + const AthenaArray &src = pmy_driver_->octets_[olev][oid]->u; for (int v=0; v &odst = uold_[current_level_]; - const AthenaArray &osrc = pmy_driver_->octets_[olev][oid].uold; + const AthenaArray &osrc = pmy_driver_->octets_[olev][oid]->uold; for (int v=0; v &ncoarse, bool folddata); void SetOctetBoundariesBeforeTransfer(bool folddata); void RestrictOctetsBeforeTransfer(); + virtual void ProlongateOctetBoundariesFluxCons(AthenaArray &dst, + AthenaArray &cbuf, const AthenaArray &ncoarse); // for multipole expansion void AllocateMultipoleCoefficients(); @@ -223,10 +226,6 @@ class MultigridDriver { // small functions int GetNumMultigrids() { return nblist_[Globals::my_rank]; } - // pure virtual functions - virtual void ProlongateOctetBoundariesFluxCons(AthenaArray &dst, - AthenaArray &cbuf, const AthenaArray &ncoarse); - int nranks_, nthreads_, nbtotal_, nvar_, mode_; int locrootlevel_, nrootlevel_, nmblevel_, ntotallevel_, nreflevel_, maxreflevel_; int current_level_, fmglevel_; @@ -246,7 +245,7 @@ class MultigridDriver { int coffset_; // for mesh refinement - std::vector *octets_; + std::vector *octets_; std::unordered_map *octetmap_; std::vector *octetbflag_; int *noctets_, *pmaxnoct_; @@ -271,10 +270,10 @@ class MultigridDriver { }; -//! \struct MGCoordinates +//! \class MGCoordinates // \brief Minimum set of coordinate arrays for Multigrid -struct MGCoordinates { +class MGCoordinates { public: void AllocateMGCoordinates(int nx, int ny, int nz); void CalculateMGCoordinates(const RegionSize &size, int ll, int ngh); @@ -288,12 +287,11 @@ struct MGCoordinates { class MGOctet { public: + MGOctet(int nvar, int ncoct, int nccoct); LogicalLocation loc; bool fleaf; AthenaArray u, def, src, uold; MGCoordinates coord, ccoord; - - virtual void AllocateOctet(int nvar, int ncoct, int nccoct, bool ffas); }; diff --git a/src/multigrid/multigrid_driver.cpp b/src/multigrid/multigrid_driver.cpp index d25a2b5890..82de585f70 100644 --- a/src/multigrid/multigrid_driver.cpp +++ b/src/multigrid/multigrid_driver.cpp @@ -85,7 +85,7 @@ MultigridDriver::MultigridDriver(Mesh *pm, MGBoundaryFunc *MGBoundary, #endif if (maxreflevel_ > 0) { // SMR / AMR - octets_ = new std::vector[maxreflevel_]; + octets_ = new std::vector[maxreflevel_]; octetmap_ = new std::unordered_map[maxreflevel_]; octetbflag_ = new std::vector[maxreflevel_]; @@ -136,6 +136,14 @@ MultigridDriver::~MultigridDriver() { } +//---------------------------------------------------------------------------------------- +//! \fn MGOctet* MultigridDriver::AllocateOctet() +// \brief Allocate a MGOctet object +MGOctet* MultigridDriver::AllocateOctet() { + int ncoct = mgroot_->ngh_*2 + 2, nccoct = mgroot_->ngh_*2 + 1; + return new MGOctet(nvar_, ncoct, nccoct); +} + //---------------------------------------------------------------------------------------- //! \fn void MultigridDriver::CheckBoundaryFunctions() // \brief check boundary functions and set some internal flags. @@ -363,7 +371,6 @@ void MultigridDriver::SetupMultigrid() { nreflevel_ = pmy_mesh_->current_level - locrootlevel_; ntotallevel_ = nrootlevel_ + nmblevel_ + nreflevel_ - 1; fmglevel_ = current_level_ = ntotallevel_ - 1; - int ncoct = mgroot_->ngh_*2 + 2, nccoct = mgroot_->ngh_*2 + 1; os_ = mgroot_->ngh_; oe_ = os_+1; @@ -384,13 +391,11 @@ void MultigridDriver::SetupMultigrid() { octetmap_[l].reserve(noctets_[l]); octetbflag_[l].resize(noctets_[l]); } + for (int i = pmaxnoct_[l]; i < noctets_[l]; ++i) + octets_[l][i] = AllocateOctet(); noctets_[l] = 0; } pmy_mesh_->tree.GetMGOctetList(octets_, octetmap_, noctets_); - for (int l = 0; l < nreflevel_; ++l) { - for (int o = pmaxnoct_[l]; o < noctets_[l]; ++o) - octets_[l][o].AllocateOctet(nvar_, ncoct, nccoct, ffas_); - } } if (needinit_) { @@ -526,10 +531,10 @@ void MultigridDriver::TransferFromBlocksToRoot(bool initflag) { int oj = (j&1) + ngh; int ok = (k&1) + ngh; for (int v = 0; v < nvar_; ++v) - octets_[olev][oid].src(v,ok,oj,oi) = rootbuf_[n*nv+v]; + octets_[olev][oid]->src(v,ok,oj,oi) = rootbuf_[n*nv+v]; if (ffas_ && !initflag) { for (int v = 0; v < nvar_; ++v) - octets_[olev][oid].u(v,ok,oj,oi) = rootbuf_[n*nv+nvar_+v]; + octets_[olev][oid]->u(v,ok,oj,oi) = rootbuf_[n*nv+nvar_+v]; } } } @@ -903,7 +908,7 @@ void MultigridDriver::CalculateOctetCoordinates() { #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[0]; ++o) { MGCoordinates &coord = mgroot_->coord_[mgroot_->nlevel_-1]; - LogicalLocation &loc = octets_[0][o].loc; + LogicalLocation &loc = octets_[0][o]->loc; int i = static_cast(loc.lx1) + ngh; int j = static_cast(loc.lx2) + ngh; int k = static_cast(loc.lx3) + ngh; @@ -913,20 +918,20 @@ void MultigridDriver::CalculateOctetCoordinates() { size.x2max = csize.x2max = coord.x2f(j+1); size.x3min = csize.x3min = coord.x3f(k); size.x3max = csize.x3max = coord.x3f(k+1); - octets_[0][o].coord.CalculateMGCoordinates(size, 0, ngh); - octets_[0][o].ccoord.CalculateMGCoordinates(csize, 0, ngh); + octets_[0][o]->coord.CalculateMGCoordinates(size, 0, ngh); + octets_[0][o]->ccoord.CalculateMGCoordinates(csize, 0, ngh); } for (int l = 1; l < nreflevel_; l++) { #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[l]; ++o) { - LogicalLocation &loc = octets_[l][o].loc; + LogicalLocation &loc = octets_[l][o]->loc; LogicalLocation cloc; cloc.lx1 = (loc.lx1 >> 1); cloc.lx2 = (loc.lx2 >> 1); cloc.lx3 = (loc.lx3 >> 1); cloc.level = loc.level - 1; int oid = octetmap_[l-1][cloc]; - MGCoordinates &coord = octets_[l-1][oid].coord; + MGCoordinates &coord = octets_[l-1][oid]->coord; int i = static_cast(loc.lx1&1) + ngh; int j = static_cast(loc.lx2&1) + ngh; int k = static_cast(loc.lx3&1) + ngh; @@ -936,8 +941,8 @@ void MultigridDriver::CalculateOctetCoordinates() { size.x2max = csize.x2max = coord.x2f(j+1); size.x3min = csize.x3min = coord.x3f(k); size.x3max = csize.x3max = coord.x3f(k+1); - octets_[l][o].coord.CalculateMGCoordinates(size, 0, ngh); - octets_[l][o].ccoord.CalculateMGCoordinates(csize, 0, ngh); + octets_[l][o]->coord.CalculateMGCoordinates(size, 0, ngh); + octets_[l][o]->ccoord.CalculateMGCoordinates(csize, 0, ngh); } } } @@ -953,7 +958,7 @@ void MultigridDriver::RestrictFMGSourceOctets() { for (int l = nreflevel_ - 1; l >= 1; --l) { // fine octets to coarse octets #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[l]; ++o) { - const LogicalLocation &loc = octets_[l][o].loc; + const LogicalLocation &loc = octets_[l][o]->loc; LogicalLocation cloc; cloc.lx1 = (loc.lx1 >> 1); cloc.lx2 = (loc.lx2 >> 1); @@ -964,17 +969,17 @@ void MultigridDriver::RestrictFMGSourceOctets() { int oj = (static_cast(loc.lx2) & 1) + ngh; int ok = (static_cast(loc.lx3) & 1) + ngh; for (int v = 0; v < nvar_; ++v) - octets_[l-1][oid].src(v, ok, oj, oi) = RestrictOne(octets_[l][o].src, - v, ngh, ngh, ngh); + octets_[l-1][oid]->src(v, ok, oj, oi) = RestrictOne(octets_[l][o]->src, + v, ngh, ngh, ngh); } } #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[0]; ++o) { // octets to the root grid - const LogicalLocation &loc = octets_[0][o].loc; + const LogicalLocation &loc = octets_[0][o]->loc; for (int v = 0; v < nvar_; ++v) mgroot_->SetData(MGVariable::src, v, static_cast(loc.lx3), static_cast(loc.lx2), static_cast(loc.lx1), - RestrictOne(octets_[0][o].src, v, ngh, ngh, ngh)); + RestrictOne(octets_[0][o]->src, v, ngh, ngh, ngh)); } } @@ -993,7 +998,7 @@ void MultigridDriver::RestrictOctets() { if (lev >= 1) { // fine octets to coarse octets #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[lev]; ++o) { - const LogicalLocation &loc = octets_[lev][o].loc; + const LogicalLocation &loc = octets_[lev][o]->loc; LogicalLocation cloc; cloc.lx1 = (loc.lx1 >> 1); cloc.lx2 = (loc.lx2 >> 1); @@ -1003,33 +1008,33 @@ void MultigridDriver::RestrictOctets() { int oi = (static_cast(loc.lx1) & 1) + ngh; int oj = (static_cast(loc.lx2) & 1) + ngh; int ok = (static_cast(loc.lx3) & 1) + ngh; - mgroot_->CalculateDefect(octets_[lev][o].def, octets_[lev][o].u, - octets_[lev][o].src, lev+1, os_, oe_, os_, oe_, os_, oe_, false); + mgroot_->CalculateDefect(octets_[lev][o]->def, octets_[lev][o]->u, + octets_[lev][o]->src, lev+1, os_, oe_, os_, oe_, os_, oe_, false); for (int v = 0; v < nvar_; ++v) - octets_[lev-1][oid].src(v, ok, oj, oi) = RestrictOne(octets_[lev][o].def, - v, ngh, ngh, ngh); + octets_[lev-1][oid]->src(v, ok, oj, oi) = RestrictOne(octets_[lev][o]->def, + v, ngh, ngh, ngh); if (ffas_) { for (int v = 0; v < nvar_; ++v) - octets_[lev-1][oid].u(v, ok, oj, oi) = RestrictOne(octets_[lev][o].u, - v, ngh, ngh, ngh); + octets_[lev-1][oid]->u(v, ok, oj, oi) = RestrictOne(octets_[lev][o]->u, + v, ngh, ngh, ngh); } } } else { // octets to the root grid #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[0]; ++o) { - const LogicalLocation &loc = octets_[0][o].loc; + const LogicalLocation &loc = octets_[0][o]->loc; int ri = static_cast(loc.lx1); int rj = static_cast(loc.lx2); int rk = static_cast(loc.lx3); - mgroot_->CalculateDefect(octets_[0][o].def, octets_[0][o].u, - octets_[0][o].src, 1, os_, oe_, os_, oe_, os_, oe_, false); + mgroot_->CalculateDefect(octets_[0][o]->def, octets_[0][o]->u, + octets_[0][o]->src, 1, os_, oe_, os_, oe_, os_, oe_, false); for (int v = 0; v < nvar_; ++v) mgroot_->SetData(MGVariable::src, v, rk, rj, ri, - RestrictOne(octets_[0][o].def, v, ngh, ngh, ngh)); + RestrictOne(octets_[0][o]->def, v, ngh, ngh, ngh)); if (ffas_) { for (int v = 0; v < nvar_; ++v) mgroot_->SetData(MGVariable::u, v, rk, rj, ri, - RestrictOne(octets_[0][o].u, v, ngh, ngh, ngh)); + RestrictOne(octets_[0][o]->u, v, ngh, ngh, ngh)); } } } @@ -1047,7 +1052,7 @@ void MultigridDriver::ZeroClearOctets() { for (int l = 0; l <= maxlevel; l++) { #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[l]; ++o) - octets_[l][o].u.ZeroClear(); + octets_[l][o]->u.ZeroClear(); } return; @@ -1063,8 +1068,8 @@ void MultigridDriver::StoreOldDataOctets() { #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[lev]; ++o) - memcpy(octets_[lev][o].uold.data(), octets_[lev][o].u.data(), - octets_[lev][o].u.GetSizeInBytes()); + memcpy(octets_[lev][o]->uold.data(), octets_[lev][o]->u.data(), + octets_[lev][o]->u.GetSizeInBytes()); return; } @@ -1078,7 +1083,7 @@ void MultigridDriver::CalculateFASRHSOctets() { #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[lev]; ++o) - mgroot_->CalculateFASRHS(octets_[lev][o].src, octets_[lev][o].u, + mgroot_->CalculateFASRHS(octets_[lev][o]->src, octets_[lev][o]->u, lev+1, os_, oe_, os_, oe_, os_, oe_, false); return; @@ -1093,7 +1098,7 @@ void MultigridDriver::SmoothOctets(int color) { #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[lev]; ++o) - mgroot_->Smooth(octets_[lev][o].u, octets_[lev][o].src, + mgroot_->Smooth(octets_[lev][o]->u, octets_[lev][o]->src, lev+1, os_, oe_, os_, oe_, os_, oe_, color, false); return; } @@ -1119,7 +1124,7 @@ void MultigridDriver::ProlongateAndCorrectOctets() { th = omp_get_thread_num(); #endif AthenaArray &cbuf = cbuf_[th]; - const LogicalLocation &loc = octets_[0][o].loc; + const LogicalLocation &loc = octets_[0][o]->loc; int ri = static_cast(loc.lx1) + ngh - 1; int rj = static_cast(loc.lx2) + ngh - 1; int rk = static_cast(loc.lx3) + ngh - 1; @@ -1133,10 +1138,10 @@ void MultigridDriver::ProlongateAndCorrectOctets() { } } } - mgroot_->ProlongateAndCorrect(octets_[0][o].u, cbuf, + mgroot_->ProlongateAndCorrect(octets_[0][o]->u, cbuf, ngh, ngh, ngh, ngh, ngh, ngh, ngh, ngh, ngh, false); } else { - mgroot_->ProlongateAndCorrect(octets_[0][o].u, u, + mgroot_->ProlongateAndCorrect(octets_[0][o]->u, u, ri+1, ri+1, rj+1, rj+1, rk+1, rk+1, ngh, ngh, ngh, false); } } @@ -1148,7 +1153,7 @@ void MultigridDriver::ProlongateAndCorrectOctets() { th = omp_get_thread_num(); #endif AthenaArray &cbuf = cbuf_[th]; - const LogicalLocation &loc = octets_[flev][o].loc; + const LogicalLocation &loc = octets_[flev][o]->loc; LogicalLocation cloc; cloc.lx1 = (loc.lx1 >> 1); cloc.lx2 = (loc.lx2 >> 1); @@ -1158,8 +1163,8 @@ void MultigridDriver::ProlongateAndCorrectOctets() { int ci = (static_cast(loc.lx1) & 1) + ngh - 1; int cj = (static_cast(loc.lx2) & 1) + ngh - 1; int ck = (static_cast(loc.lx3) & 1) + ngh - 1; - const AthenaArray &uc = octets_[clev][cid].u; - const AthenaArray &ucold = octets_[clev][cid].uold; + const AthenaArray &uc = octets_[clev][cid]->u; + const AthenaArray &ucold = octets_[clev][cid]->uold; if (ffas_) { for (int v = 0; v < nvar_; ++v) { for (int k = 0; k <= 2; ++k) { @@ -1169,10 +1174,10 @@ void MultigridDriver::ProlongateAndCorrectOctets() { } } } - mgroot_->ProlongateAndCorrect(octets_[flev][o].u, cbuf, + mgroot_->ProlongateAndCorrect(octets_[flev][o]->u, cbuf, ngh, ngh, ngh, ngh, ngh, ngh, ngh, ngh, ngh, false); } else { - mgroot_->ProlongateAndCorrect(octets_[flev][o].u, uc, + mgroot_->ProlongateAndCorrect(octets_[flev][o]->u, uc, ci+1, ci+1, cj+1, cj+1, ck+1, ck+1, ngh, ngh, ngh, false); } } @@ -1194,17 +1199,17 @@ void MultigridDriver::FMGProlongateOctets() { if (flev == 0) { // from root to octets #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[0]; ++o) { - const LogicalLocation &loc = octets_[0][o].loc; + const LogicalLocation &loc = octets_[0][o]->loc; int ri = static_cast(loc.lx1) + ngh; int rj = static_cast(loc.lx2) + ngh; int rk = static_cast(loc.lx3) + ngh; - mgroot_->FMGProlongate(octets_[0][o].u, mgroot_->GetCurrentData(), + mgroot_->FMGProlongate(octets_[0][o]->u, mgroot_->GetCurrentData(), ri, ri, rj, rj, rk, rk, ngh, ngh, ngh, false); } } else { // from coarse octets to fine octets #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[flev]; ++o) { - const LogicalLocation &loc = octets_[flev][o].loc; + const LogicalLocation &loc = octets_[flev][o]->loc; LogicalLocation cloc; cloc.lx1 = (loc.lx1 >> 1); cloc.lx2 = (loc.lx2 >> 1); @@ -1214,7 +1219,7 @@ void MultigridDriver::FMGProlongateOctets() { int ci = (static_cast(loc.lx1) & 1) + ngh; int cj = (static_cast(loc.lx2) & 1) + ngh; int ck = (static_cast(loc.lx3) & 1) + ngh; - mgroot_->FMGProlongate(octets_[flev][o].u, octets_[clev][cid].u, + mgroot_->FMGProlongate(octets_[flev][o]->u, octets_[clev][cid]->u, ci, ci, cj, cj, ck, ck, ngh, ngh, ngh, false); } } @@ -1232,7 +1237,7 @@ void MultigridDriver::SetBoundariesOctets(bool fprolong, bool folddata) { #pragma omp parallel for num_threads(nthreads_) schedule(dynamic,1) for (int o = 0; o < noctets_[lev]; ++o) { - if (fprolong && octets_[lev][o].fleaf == true) continue; + if (fprolong && octets_[lev][o]->fleaf == true) continue; int th = 0; #ifdef OPENMP_PARALLEL th = omp_get_thread_num(); @@ -1241,9 +1246,9 @@ void MultigridDriver::SetBoundariesOctets(bool fprolong, bool folddata) { AthenaArray &cbufold = cbufold_[th]; AthenaArray &ncoarse = ncoarse_[th]; ncoarse.ZeroClear(); - const LogicalLocation &loc = octets_[lev][o].loc; - AthenaArray &u = octets_[lev][o].u; - AthenaArray &uold = octets_[lev][o].uold; + const LogicalLocation &loc = octets_[lev][o]->loc; + AthenaArray &u = octets_[lev][o]->u; + AthenaArray &uold = octets_[lev][o]->uold; LogicalLocation nloc = loc; for (int ox3 = -1; ox3 <= 1; ++ox3) { nloc.lx3 = loc.lx3 + ox3; @@ -1292,8 +1297,8 @@ void MultigridDriver::SetBoundariesOctets(bool fprolong, bool folddata) { } if (octetmap_[lev].count(nloc) == 1) { // on the same level int nid = octetmap_[lev][nloc]; - const AthenaArray &un = octets_[lev][nid].u; - const AthenaArray &unold = octets_[lev][nid].uold; + const AthenaArray &un = octets_[lev][nid]->u; + const AthenaArray &unold = octets_[lev][nid]->uold; SetOctetBoundarySameLevel(u, un, uold, unold, cbuf, cbufold, ox1, ox2, ox3, folddata); } else if (!fprolong) { // on the coarser level @@ -1306,8 +1311,8 @@ void MultigridDriver::SetBoundariesOctets(bool fprolong, bool folddata) { cloc.lx3 = nloc.lx3 >> 1; cloc.level = nloc.level - 1; int cid = octetmap_[lev-1][cloc]; - const AthenaArray &un = octets_[lev-1][cid].u; - const AthenaArray &unold = octets_[lev-1][cid].uold; + const AthenaArray &un = octets_[lev-1][cid]->u; + const AthenaArray &unold = octets_[lev-1][cid]->uold; SetOctetBoundaryFromCoarser(un, unold, cbuf, cbufold, loc, ox1, ox2, ox3, false); } else { // from root @@ -1322,10 +1327,10 @@ void MultigridDriver::SetBoundariesOctets(bool fprolong, bool folddata) { } } if (!fprolong) { - ApplyPhysicalBoundariesOctet(cbuf, loc, octets_[lev][o].ccoord, true); + ApplyPhysicalBoundariesOctet(cbuf, loc, octets_[lev][o]->ccoord, true); ProlongateOctetBoundariesFluxCons(u, cbuf, ncoarse); } - ApplyPhysicalBoundariesOctet(u, loc, octets_[lev][o].coord, false); + ApplyPhysicalBoundariesOctet(u, loc, octets_[lev][o]->coord, false); } return; @@ -1593,7 +1598,7 @@ void MultigridDriver::RestrictOctetsBeforeTransfer() { for (int l = nreflevel_ - 1; l >= 1; --l) { // fine octets to coarse octets #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[l]; ++o) { - const LogicalLocation &loc = octets_[l][o].loc; + const LogicalLocation &loc = octets_[l][o]->loc; LogicalLocation cloc; cloc.lx1 = (loc.lx1 >> 1); cloc.lx2 = (loc.lx2 >> 1); @@ -1604,17 +1609,17 @@ void MultigridDriver::RestrictOctetsBeforeTransfer() { int oj = (static_cast(loc.lx2) & 1) + ngh; int ok = (static_cast(loc.lx3) & 1) + ngh; for (int v = 0; v < nvar_; ++v) - octets_[l-1][oid].u(v, ok, oj, oi) = RestrictOne(octets_[l][o].u, - v, ngh, ngh, ngh); + octets_[l-1][oid]->u(v, ok, oj, oi) = RestrictOne(octets_[l][o]->u, + v, ngh, ngh, ngh); } } #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[0]; ++o) { // octets to the root grid - const LogicalLocation &loc = octets_[0][o].loc; + const LogicalLocation &loc = octets_[0][o]->loc; for (int v = 0; v < nvar_; ++v) mgroot_->SetData(MGVariable::u, v, static_cast(loc.lx3), static_cast(loc.lx2), static_cast(loc.lx1), - RestrictOne(octets_[0][o].u, v, ngh, ngh, ngh)); + RestrictOne(octets_[0][o]->u, v, ngh, ngh, ngh)); } return; @@ -1653,8 +1658,8 @@ void MultigridDriver::SetOctetBoundariesBeforeTransfer(bool folddata) { int oid = octetmap_[lev][loc]; if (octetbflag_[lev][oid] == true) continue; octetbflag_[lev][oid] = true; - AthenaArray &u = octets_[lev][oid].u; - AthenaArray &uold = octets_[lev][oid].uold; + AthenaArray &u = octets_[lev][oid]->u; + AthenaArray &uold = octets_[lev][oid]->uold; LogicalLocation nloc = loc; for (int ox3 = -1; ox3 <= 1; ++ox3) { nloc.lx3 = loc.lx3 + ox3; @@ -1701,8 +1706,8 @@ void MultigridDriver::SetOctetBoundariesBeforeTransfer(bool folddata) { } if (octetmap_[lev].count(nloc) == 1) { // same or finer int nid = octetmap_[lev][nloc]; - const AthenaArray &un = octets_[lev][nid].u; - const AthenaArray &unold = octets_[lev][nid].uold; + const AthenaArray &un = octets_[lev][nid]->u; + const AthenaArray &unold = octets_[lev][nid]->uold; SetOctetBoundarySameLevel(u, un, uold, unold, cbuf, cbufold, ox1, ox2, ox3, folddata); } else { // coarser @@ -1714,8 +1719,8 @@ void MultigridDriver::SetOctetBoundariesBeforeTransfer(bool folddata) { cloc.lx3 = nloc.lx3 >> 1; cloc.level = nloc.level - 1; int cid = octetmap_[lev-1][cloc]; - const AthenaArray &un = octets_[lev-1][cid].u; - const AthenaArray &unold = octets_[lev-1][cid].uold; + const AthenaArray &un = octets_[lev-1][cid]->u; + const AthenaArray &unold = octets_[lev-1][cid]->uold; SetOctetBoundaryFromCoarser(un, unold, cbuf, cbufold, loc, ox1, ox2, ox3, folddata); } else { // from root @@ -1729,11 +1734,11 @@ void MultigridDriver::SetOctetBoundariesBeforeTransfer(bool folddata) { } } - ApplyPhysicalBoundariesOctet(cbuf, loc, octets_[lev][oid].ccoord, true); + ApplyPhysicalBoundariesOctet(cbuf, loc, octets_[lev][oid]->ccoord, true); if (folddata) - ApplyPhysicalBoundariesOctet(cbufold, loc, octets_[lev][oid].ccoord, true); + ApplyPhysicalBoundariesOctet(cbufold, loc, octets_[lev][oid]->ccoord, true); ProlongateOctetBoundaries(u, uold, cbuf, cbufold, ncoarse, folddata); - ApplyPhysicalBoundariesOctet(u, loc, octets_[lev][oid].coord, false); + ApplyPhysicalBoundariesOctet(u, loc, octets_[lev][oid]->coord, false); } return; } @@ -2015,14 +2020,11 @@ void MultigridDriver::CalculateCenterOfMass() { //---------------------------------------------------------------------------------------- -//! \fn void MGOctet::AllocateOctet(int nvar, int ncoct, int nccoct, bool ffas) +//! \fn MGOctet::MGOctet(int nvar, int ncoct, int nccoct) // \brief MGOctet construcor - allocate arrays and coordinates -void MGOctet::AllocateOctet(int nvar, int ncoct, int nccoct, bool ffas) { - u.NewAthenaArray(nvar, ncoct, ncoct, ncoct); - def.NewAthenaArray(nvar, ncoct, ncoct, ncoct); - src.NewAthenaArray(nvar, ncoct, ncoct, ncoct); +MGOctet::MGOctet(int nvar, int ncoct, int nccoct) : + u(nvar, ncoct, ncoct, ncoct), uold(nvar, ncoct, ncoct, ncoct), + def(nvar, ncoct, ncoct, ncoct), src(nvar, ncoct, ncoct, ncoct) { coord.AllocateMGCoordinates(ncoct, ncoct, ncoct); ccoord.AllocateMGCoordinates(nccoct, nccoct, nccoct); - if (ffas) - uold.NewAthenaArray(nvar, ncoct, ncoct, ncoct); } From e895429fe65e0d2e89c14d25480234a866ebc8f2 Mon Sep 17 00:00:00 2001 From: Kengo Tomida Date: Thu, 23 Nov 2023 16:26:08 +0900 Subject: [PATCH 05/33] *** incomplete commit *** --- src/crdiffusion/mg_crdiffusion.cpp | 70 ++++++++++++++++++++---------- src/crdiffusion/mg_crdiffusion.hpp | 41 ++++++++++++----- src/gravity/mg_gravity.cpp | 22 ++++++---- src/gravity/mg_gravity.hpp | 10 +++-- src/multigrid/multigrid.cpp | 10 +++-- src/multigrid/multigrid.hpp | 67 +++++++++++++++------------- src/multigrid/multigrid_driver.cpp | 55 +++++++++++++---------- 7 files changed, 171 insertions(+), 104 deletions(-) diff --git a/src/crdiffusion/mg_crdiffusion.cpp b/src/crdiffusion/mg_crdiffusion.cpp index 6ad9605295..ebefd409e1 100644 --- a/src/crdiffusion/mg_crdiffusion.cpp +++ b/src/crdiffusion/mg_crdiffusion.cpp @@ -106,13 +106,13 @@ MGCRDiffusionDriver::~MGCRDiffusionDriver() { MGCRDiffusion::MGCRDiffusion(MultigridDriver *pmd, MeshBlock *pmb) : Multigrid(pmd, pmb, 1, 1) { + coeff_ = new MGCRCoefficient*[nlevel_]; for (int l = 0; l < nlevel_; l++) { int ll = nlevel_-1-l; int ncx = (size_.nx1>>ll) + 2*ngh_; int ncy = (size_.nx2>>ll) + 2*ngh_; int ncz = (size_.nx3>>ll) + 2*ngh_; - D_[l].NewAthenaArray(NCOEFF,ncz,ncy,ncx); - nlambda_[l].NewAthenaArray(ncz,ncy,ncx); + coeff_[l] = new MGCRCoefficient(ncx, ncy, ncz); } btype = btypef = BoundaryQuantity::mg; @@ -125,8 +125,9 @@ MGCRDiffusion::MGCRDiffusion(MultigridDriver *pmd, MeshBlock *pmb) //! \brief MGCRDiffusion deconstructor MGCRDiffusion::~MGCRDiffusion() { - delete [] D_; - delete [] nlambda_; + for (int l = 0; l < nlevel_; l++) + delete coeff_[l]; + delete [] coeff_; delete pmgbval; } @@ -157,9 +158,11 @@ void MGCRDiffusionDriver::RestrictCoefficients() { int oj = (static_cast(loc.lx2) & 1) + ngh; int ok = (static_cast(loc.lx3) & 1) + ngh; MGCROctet *coct = static_cast(octets_[l-1][oid]); + MGCRCoefficient *ccoeff = static_cast(coct->coeff); + MGCRCoefficient *fcoeff = static_cast(foct->coeff); for (int v = 0; v < NCOEFF; ++v) - coct->D(v, ok, oj, oi) = RestrictOne(foct->D, v, ngh, ngh, ngh); - coct->nlambda(ok, oj, oi) = RestrictOne(foct->nlambda, 0, ngh, ngh, ngh); + ccoeff->D(v, ok, oj, oi) = RestrictOne(fcoeff->D, v, ngh, ngh, ngh); + ccoeff->nlambda(ok, oj, oi) = RestrictOne(fcoeff->nlambda, 0, ngh, ngh, ngh); } } @@ -170,11 +173,12 @@ void MGCRDiffusionDriver::RestrictCoefficients() { int lx1 = static_cast(loc.lx1) + mgcrroot_->ngh_; int lx2 = static_cast(loc.lx2) + mgcrroot_->ngh_; int lx3 = static_cast(loc.lx3) + mgcrroot_->ngh_; + MGCRCoefficient *coeff = static_cast(oct->coeff); for (int v = 0; v < NCOEFF; ++v) - mgcrroot_->D_[mgcrroot_->nlevel_-1](v, lx3, lx2, lx1) - = RestrictOne(oct->D, v, ngh, ngh, ngh); - mgcrroot_->nlambda_[mgcrroot_->nlevel_-1](lx3, lx2, lx1) - = RestrictOne(oct->nlambda, 0, ngh, ngh, ngh); + mgcrroot_->coeff_[mgcrroot_->nlevel_-1]->D(v, lx3, lx2, lx1) + = RestrictOne(coeff->D, v, ngh, ngh, ngh); + mgcrroot_->coeff_[mgcrroot_->nlevel_-1]->nlambda(lx3, lx2, lx1) + = RestrictOne(coeff->nlambda, 0, ngh, ngh, ngh); } } mgcrroot_->RestrictCoefficients(); @@ -182,6 +186,13 @@ void MGCRDiffusionDriver::RestrictCoefficients() { } +//---------------------------------------------------------------------------------------- +//! \fn MGOctet* MGCRDiffusionDriver::AllocateOctet() +// \brief Allocate a MGCROctet object +MGOctet* MGCRDiffusionDriver::AllocateOctet() { + int ncoct = mgroot_->ngh_*2 + 2, nccoct = mgroot_->ngh_*2 + 1; + return new MGCROctet(nvar_, ncoct, nccoct); +} //---------------------------------------------------------------------------------------- //! \fn void MGCRDiffusionDriver::Solve(int stage) @@ -233,6 +244,15 @@ void MGCRDiffusionDriver::Solve(int stage) { } +//---------------------------------------------------------------------------------------- +//! \fn void MGCRDiffusion::AllocateCoefficient(int ncx, int ncy, int ncz) +//! \brief allocate coefficients for cosmic-ray transport + +MGCoefficient* MGCRDiffusion::AllocateCoefficient(int ncx, int ncy, int ncz) { + return new MGCRCoefficient(ncx, ncy, ncz); +} + + //---------------------------------------------------------------------------------------- //! \fn void MGCRDiffusion::RestrictCoefficients() //! \brief restrict coefficients within a MGCRDiffusion object @@ -243,9 +263,9 @@ void MGCRDiffusion::RestrictCoefficients() { for (current_level_=nlevel_-1; current_level_>0; current_level_--) { int ll=nlevel_-current_level_; ie=is+(size_.nx1>>ll)-1, je=js+(size_.nx2>>ll)-1, ke=ks+(size_.nx3>>ll)-1; - Restrict(D_[current_level_-1], D_[current_level_], + Restrict(coeff_[current_level_-1]->D, coeff_[current_level_]->D, NCOEFF, is, ie, js, je, ks, ke, false); - Restrict(nlambda_[current_level_-1], nlambda_[current_level_], + Restrict(coeff_[current_level_-1]->nlambda, coeff_[current_level_]->nlambda, 1, is, ie, js, je, ks, ke, false); } return; @@ -254,12 +274,14 @@ void MGCRDiffusion::RestrictCoefficients() { //---------------------------------------------------------------------------------------- //! \fn void MGCRDiffusion::Smooth(AthenaArray &u, const AthenaArray &src, -//! int rlev, int il, int iu, int jl, int ju, int kl, int ku, int color, bool th) +//! MGCoefficient *coeff, int rlev, int il, int iu, +//! int jl, int ju, int kl, int ku, int color, bool th) //! \brief Implementation of the Red-Black Gauss-Seidel Smoother //! rlev = relative level from the finest level of this Multigrid block -void MGCRDiffusion::Smooth(AthenaArray &u, const AthenaArray &src, int rlev, - int il, int iu, int jl, int ju, int kl, int ku, int color, bool th) { +void MGCRDiffusion::Smooth(AthenaArray &u, const AthenaArray &src, + MGCoefficient *coeff, int rlev, int il, int iu, + int jl, int ju, int kl, int ku, int color, bool th) { Real dx; if (rlev <= 0) dx = rdx_*static_cast(1<<(-rlev)); else dx = rdx_/static_cast(1< &u, const AthenaArray &src, i //---------------------------------------------------------------------------------------- //! \fn void MGCRDiffusion::CalculateDefect(AthenaArray &def, -//! const AthenaArray &u, const AthenaArray &src, int rlev, -//! int il, int iu, int jl, int ju, int kl, int ku, bool th) +//! const AthenaArray &u, const AthenaArray &src, +//! MGCoefficient *coeff, int rlev, int il, int iu, +//! int jl, int ju, int kl, int ku, bool th) //! \brief Implementation of the Defect calculation //! rlev = relative level from the finest level of this Multigrid block void MGCRDiffusion::CalculateDefect(AthenaArray &def, const AthenaArray &u, - const AthenaArray &src, int rlev, - int il, int iu, int jl, int ju, int kl, int ku, bool th) { + const AthenaArray &src, MGCoefficient *coeff, int rlev, + int il, int iu, int jl, int ju, int kl, int ku, bool th) { Real dx; if (rlev <= 0) dx = rdx_*static_cast(1<<(-rlev)); else dx = rdx_/static_cast(1< &def, const AthenaArray &src, -//! const AthenaArray &u, int rlev, +//! const AthenaArray &u, MGCoefficient *coeff, int rlev, //! int il, int iu, int jl, int ju, int kl, int ku, bool th) //! \brief Implementation of the RHS calculation for FAS //! rlev = relative level from the finest level of this Multigrid block void MGCRDiffusion::CalculateFASRHS(AthenaArray &src, const AthenaArray &u, - int rlev, int il, int iu, int jl, int ju, int kl, int ku, bool th) { + MGCoefficient *coeff, int rlev, int il, int iu, + int jl, int ju, int kl, int ku, bool th) { Real dx; if (rlev <= 0) dx = rdx_*static_cast(1<<(-rlev)); else dx = rdx_/static_cast(1< &src, const AthenaArray &D, const AthenaArray &nlambda, int ngh) { - AthenaArray &Dm=D_[nlevel_-1]; - AthenaArray &nl=nlambda_[nlevel_-1]; + AthenaArray &Dm=coeff_[nlevel_-1]->D; + AthenaArray &nl=coeff_[nlevel_-1]->nlambda; int is, ie, js, je, ks, ke; is=js=ks=ngh_; ie=is+size_.nx1-1, je=js+size_.nx2-1, ke=ks+size_.nx3-1; diff --git a/src/crdiffusion/mg_crdiffusion.hpp b/src/crdiffusion/mg_crdiffusion.hpp index 90e1f9c9b3..2df68b14d5 100644 --- a/src/crdiffusion/mg_crdiffusion.hpp +++ b/src/crdiffusion/mg_crdiffusion.hpp @@ -25,6 +25,25 @@ class Multigrid; class CRDiffusionBoundaryTaskList; +//! \class MGCRCoefficient +class MGCRCoefficient : public MGCoefficient { + public: + MGCRCoefficient(int nx, int ny, int nz) : D(NCOEFF, nz, ny, nx), + nlambda(nz, ny, nx) {}; + AthenaArray D, nlambda; +}; + + +//! \class MGCROctet +//! \brief MGOctet for cosmic ray transport + +class MGCROctet : public MGOctet { + public: + MGCROctet(int nvar, int ncoct, int nccoct) : MGOctet(nvar, ncoct, nccoct) + { coeff = new MGCRCoefficient(ncoct, ncoct, ncoct);}; + ~MGCROctet() { delete coeff; }; +}; + //! \class MGCRDiffusion //! \brief Multigrid CR Diffusion solver for each block @@ -33,14 +52,17 @@ class MGCRDiffusion : public Multigrid { MGCRDiffusion(MultigridDriver *pmd, MeshBlock *pmb); ~MGCRDiffusion(); - void Smooth(AthenaArray &dst, const AthenaArray &src, int rlev, - int il, int iu, int jl, int ju, int kl, int ku, int color, bool th) final; + void Smooth(AthenaArray &dst, const AthenaArray &src, + MGCoefficient *coeff, int rlev, int il, int iu, int jl, int ju, + int kl, int ku, int color, bool th) final; void CalculateDefect(AthenaArray &def, const AthenaArray &u, - const AthenaArray &src, int rlev, + const AthenaArray &src, MGCoefficient *coeff, int rlev, int il, int iu, int jl, int ju, int kl, int ku, bool th) final; void CalculateFASRHS(AthenaArray &def, const AthenaArray &src, - int rlev, int il, int iu, int jl, int ju, int kl, int ku, bool th) final; + MGCoefficient *coeff, int rlev, int il, int iu, + int jl, int ju, int kl, int ku, bool th) final; + MGCoefficient* AllocateCoefficient(int ncx, int ncy, int ncz) final; void LoadCoefficients(const AthenaArray &D, const AthenaArray &nlambda, int ngh); void RestrictCoefficients(); @@ -48,7 +70,7 @@ class MGCRDiffusion : public Multigrid { friend class MGCRDiffusionDriver; private: - AthenaArray *D_, *nlambda_; + MGCRCoefficient **coeff_; static constexpr Real omega_ = 1.15; }; @@ -65,15 +87,10 @@ class MGCRDiffusionDriver : public MultigridDriver { void RestrictCoefficients(); private: + MGOctet* AllocateOctet() final; + MGCRDiffusion *mgcrroot_; CRDiffusionBoundaryTaskList *crtlist_; }; -class MGCROctet : public MGOctet { - public: - MGCROctet(int nvar, int ncoct, int nccoct) : MGOctet(nvar, ncoct, nccoct), - D(NCOEFF,ncoct, ncoct, ncoct), nlambda(ncoct, ncoct, ncoct) {}; - AthenaArray D, nlambda; -}; - #endif // CRDIFFUSION_MG_CRDIFFUSION_HPP_ diff --git a/src/gravity/mg_gravity.cpp b/src/gravity/mg_gravity.cpp index ec9eace0af..d1368bde5a 100644 --- a/src/gravity/mg_gravity.cpp +++ b/src/gravity/mg_gravity.cpp @@ -211,12 +211,14 @@ void MGGravityDriver::Solve(int stage) { //---------------------------------------------------------------------------------------- //! \fn void MGGravity::Smooth(AthenaArray &u, const AthenaArray &src, -//! int rlev, int il, int iu, int jl, int ju, int kl, int ku, int color, bool th) +//! MGCoefficient *coeff, int rlev, int il, int iu, int jl, int ju, +//! int kl, int ku, int color, bool th) //! \brief Implementation of the Red-Black Gauss-Seidel Smoother //! rlev = relative level from the finest level of this Multigrid block -void MGGravity::Smooth(AthenaArray &u, const AthenaArray &src, int rlev, - int il, int iu, int jl, int ju, int kl, int ku, int color, bool th) { +void MGGravity::Smooth(AthenaArray &u, const AthenaArray &src, + MGCoefficient *coeff, int rlev, int il, int iu, int jl, int ju, + int kl, int ku, int color, bool th) { Real dx; if (rlev <= 0) dx = rdx_*static_cast(1<<(-rlev)); else dx = rdx_/static_cast(1< &u, const AthenaArray &src, int r //---------------------------------------------------------------------------------------- //! \fn void MGGravity::CalculateDefect(AthenaArray &def, -//! const AthenaArray &u, const AthenaArray &src, int rlev, -//! int il, int iu, int jl, int ju, int kl, int ku, bool th) +//! const AthenaArray &u, const AthenaArray &src, +//! MGCoefficient *coeff, int rlev, int il, int iu, +//! int jl, int ju, int kl, int ku, bool th) //! \brief Implementation of the Defect calculation //! rlev = relative level from the finest level of this Multigrid block void MGGravity::CalculateDefect(AthenaArray &def, const AthenaArray &u, - const AthenaArray &src, int rlev, - int il, int iu, int jl, int ju, int kl, int ku, bool th) { + const AthenaArray &src, MGCoefficient *coeff, int rlev, + int il, int iu, int jl, int ju, int kl, int ku, bool th) { Real dx; if (rlev <= 0) dx = rdx_*static_cast(1<<(-rlev)); else dx = rdx_/static_cast(1< &def, const AthenaArray //---------------------------------------------------------------------------------------- //! \fn void MGGravity::CalculateFASRHS(AthenaArray &src, -//! const AthenaArray &u, int rlev, +//! const AthenaArray &u, MGCoefficient *coeff, int rlev, //! int il, int iu, int jl, int ju, int kl, int ku, bool th) //! \brief Implementation of the RHS calculation for FAS //! rlev = relative level from the finest level of this Multigrid block void MGGravity::CalculateFASRHS(AthenaArray &src, const AthenaArray &u, - int rlev, int il, int iu, int jl, int ju, int kl, int ku, bool th) { + MGCoefficient *coeff, int rlev, int il, int iu, + int jl, int ju, int kl, int ku, bool th) { Real dx; if (rlev <= 0) dx = rdx_*static_cast(1<<(-rlev)); else dx = rdx_/static_cast(1< &dst, const AthenaArray &src, int rlev, - int il, int iu, int jl, int ju, int kl, int ku, int color, bool th) final; + void Smooth(AthenaArray &dst, const AthenaArray &src, + MGCoefficient *coeff, int rlev, int il, int iu, int jl, int ju, + int kl, int ku, int color, bool th) final; void CalculateDefect(AthenaArray &def, const AthenaArray &u, - const AthenaArray &src, int rlev, + const AthenaArray &src, MGCoefficient *coeff, int rlev, int il, int iu, int jl, int ju, int kl, int ku, bool th) final; void CalculateFASRHS(AthenaArray &def, const AthenaArray &src, - int rlev, int il, int iu, int jl, int ju, int kl, int ku, bool th) final; + MGCoefficient *coeff, int rlev, int il, int iu, int jl, int ju, + int kl, int ku, bool th) final; private: static constexpr Real omega_ = 1.15; diff --git a/src/multigrid/multigrid.cpp b/src/multigrid/multigrid.cpp index 808e2b8689..df89cc159e 100644 --- a/src/multigrid/multigrid.cpp +++ b/src/multigrid/multigrid.cpp @@ -120,6 +120,7 @@ Multigrid::Multigrid(MultigridDriver *pmd, MeshBlock *pmb, int invar, int nghost def_ = new AthenaArray[nlevel_]; coord_ = new MGCoordinates[nlevel_]; ccoord_ = new MGCoordinates[nlevel_]; + coeff_ = new MGCoefficient*[nlevel_]; if (pmy_block_ == nullptr) uold_ = new AthenaArray[nlevel_]; else @@ -136,6 +137,7 @@ Multigrid::Multigrid(MultigridDriver *pmd, MeshBlock *pmb, int invar, int nghost uold_[l].NewAthenaArray(nvar_,ncz,ncy,ncx); coord_[l].AllocateMGCoordinates(ncx,ncy,ncz); coord_[l].CalculateMGCoordinates(size_, ll, ngh_); + coeff_[l] = AllocateCoefficient(ncx,ncy,ncz); ncx=(size_.nx1>>(ll+1))+2*ngh_; ncy=(size_.nx2>>(ll+1))+2*ngh_; ncz=(size_.nx3>>(ll+1))+2*ngh_; @@ -425,7 +427,7 @@ void Multigrid::SmoothBlock(int color) { is = js = ks = ngh_; ie = is+(size_.nx1>>ll)-1, je = js+(size_.nx2>>ll)-1, ke = ks+(size_.nx3>>ll)-1; - Smooth(u_[current_level_], src_[current_level_], + Smooth(u_[current_level_], src_[current_level_], coeff_[current_level_], -ll, is, ie, js, je, ks, ke, color, th); return; @@ -448,7 +450,7 @@ void Multigrid::CalculateDefectBlock() { ie = is+(size_.nx1>>ll)-1, je = js+(size_.nx2>>ll)-1, ke = ks+(size_.nx3>>ll)-1; CalculateDefect(def_[current_level_], u_[current_level_], src_[current_level_], - -ll, is, ie, js, je, ks, ke, th); + coeff_[current_level_], -ll, is, ie, js, je, ks, ke, th); return; } @@ -469,7 +471,7 @@ void Multigrid::CalculateFASRHSBlock() { is = js = ks = ngh_; ie = is+(size_.nx1>>ll)-1, je = js+(size_.nx2>>ll)-1, ke = ks+(size_.nx3>>ll)-1; - CalculateFASRHS(src_[current_level_], u_[current_level_], + CalculateFASRHS(src_[current_level_], u_[current_level_], coeff_[current_level_], -ll, is, ie, js, je, ks, ke, th); return; @@ -565,7 +567,7 @@ Real Multigrid::CalculateDefectNorm(MGNormType nrm, int n) { dz=rdz_*static_cast(1< x1f, x2f, x3f, x1v, x2v, x3v; +}; + + +//! \class MGOctet +// \brief structure containing eight (+ ghost) cells for Mesh Refinement + +class MGOctet { + public: + MGOctet(int nvar, int ncoct, int nccoct); + LogicalLocation loc; + bool fleaf; + AthenaArray u, def, src, uold; + MGCoordinates coord, ccoord; + MGCoefficient* coeff; +}; + + //! \class Multigrid // \brief Multigrid object containing each MeshBlock and/or the root block @@ -117,14 +148,16 @@ class Multigrid { int il, int iu, int jl, int ju, int kl, int ku, int fil, int fjl, int fkl, bool th); // physics-dependent virtual functions + virtual MGCoefficient* AllocateCoefficient(int ncx, int ncy, int ncz) { return nullptr;}; virtual void Smooth(AthenaArray &dst, const AthenaArray &src, - int rlev, int il, int iu, int jl, int ju, int kl, int ku, - int color, bool th) = 0; + MGCoefficient *coeff, int rlev, int il, int iu, int jl, int ju, + int kl, int ku, int color, bool th) = 0; virtual void CalculateDefect(AthenaArray &def, const AthenaArray &u, - const AthenaArray &src, int rlev, + const AthenaArray &src, MGCoefficient *coeff, int rlev, int il, int iu, int jl, int ju, int kl, int ku, bool th) = 0; virtual void CalculateFASRHS(AthenaArray &def, const AthenaArray &src, - int rlev, int il, int iu, int jl, int ju, int kl, int ku, bool th) = 0; + MGCoefficient *coeff, int rlev, int il, int iu, + int jl, int ju, int kl, int ku, bool th) = 0; friend class MultigridDriver; friend class MultigridTaskList; @@ -144,6 +177,7 @@ class Multigrid { Real defscale_; AthenaArray *u_, *def_, *src_, *uold_; MGCoordinates *coord_, *ccoord_; + MGCoefficient **coeff_; private: TaskStates ts_; @@ -270,31 +304,6 @@ class MultigridDriver { }; -//! \class MGCoordinates -// \brief Minimum set of coordinate arrays for Multigrid - -class MGCoordinates { - public: - void AllocateMGCoordinates(int nx, int ny, int nz); - void CalculateMGCoordinates(const RegionSize &size, int ll, int ngh); - - AthenaArray x1f, x2f, x3f, x1v, x2v, x3v; -}; - - -//! \class MGOctet -// \brief structure containing eight (+ ghost) cells for Mesh Refinement - -class MGOctet { - public: - MGOctet(int nvar, int ncoct, int nccoct); - LogicalLocation loc; - bool fleaf; - AthenaArray u, def, src, uold; - MGCoordinates coord, ccoord; -}; - - inline Real RestrictOne(const AthenaArray &src, int v, int fi, int fj, int fk) { return 0.125*(src(v, fk, fj, fi)+src(v, fk, fj, fi+1) +src(v, fk, fj+1, fi)+src(v, fk, fj+1, fi+1) diff --git a/src/multigrid/multigrid_driver.cpp b/src/multigrid/multigrid_driver.cpp index 82de585f70..dc50315d26 100644 --- a/src/multigrid/multigrid_driver.cpp +++ b/src/multigrid/multigrid_driver.cpp @@ -119,6 +119,10 @@ MultigridDriver::~MultigridDriver() { delete [] nvslisti_; delete [] rootbuf_; if (maxreflevel_ > 0) { + for (int l = 0; l < nreflevel_; ++l) { + for (int o = pmaxnoct_[l]; o < noctets_[l]; ++o) + delete octets_[l][o]; + } delete [] octets_; delete [] octetmap_; delete [] octetbflag_; @@ -391,8 +395,8 @@ void MultigridDriver::SetupMultigrid() { octetmap_[l].reserve(noctets_[l]); octetbflag_[l].resize(noctets_[l]); } - for (int i = pmaxnoct_[l]; i < noctets_[l]; ++i) - octets_[l][i] = AllocateOctet(); + for (int o = pmaxnoct_[l]; o < noctets_[l]; ++o) + octets_[l][o] = AllocateOctet(); noctets_[l] = 0; } pmy_mesh_->tree.GetMGOctetList(octets_, octetmap_, noctets_); @@ -998,7 +1002,8 @@ void MultigridDriver::RestrictOctets() { if (lev >= 1) { // fine octets to coarse octets #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[lev]; ++o) { - const LogicalLocation &loc = octets_[lev][o]->loc; + MGOctet *foct = octets_[lev][o]; + const LogicalLocation &loc = foct->loc; LogicalLocation cloc; cloc.lx1 = (loc.lx1 >> 1); cloc.lx2 = (loc.lx2 >> 1); @@ -1008,33 +1013,33 @@ void MultigridDriver::RestrictOctets() { int oi = (static_cast(loc.lx1) & 1) + ngh; int oj = (static_cast(loc.lx2) & 1) + ngh; int ok = (static_cast(loc.lx3) & 1) + ngh; - mgroot_->CalculateDefect(octets_[lev][o]->def, octets_[lev][o]->u, - octets_[lev][o]->src, lev+1, os_, oe_, os_, oe_, os_, oe_, false); + MGOctet *coct = octets_[lev-1][oid]; + mgroot_->CalculateDefect(foct->def, foct->u, foct->src, foct->coeff, + lev+1, os_, oe_, os_, oe_, os_, oe_, false); for (int v = 0; v < nvar_; ++v) - octets_[lev-1][oid]->src(v, ok, oj, oi) = RestrictOne(octets_[lev][o]->def, - v, ngh, ngh, ngh); + coct->src(v, ok, oj, oi) = RestrictOne(foct->def, v, ngh, ngh, ngh); if (ffas_) { for (int v = 0; v < nvar_; ++v) - octets_[lev-1][oid]->u(v, ok, oj, oi) = RestrictOne(octets_[lev][o]->u, - v, ngh, ngh, ngh); + coct->u(v, ok, oj, oi) = RestrictOne(foct->u, v, ngh, ngh, ngh); } } } else { // octets to the root grid #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[0]; ++o) { - const LogicalLocation &loc = octets_[0][o]->loc; + MGOctet *oct = octets_[0][o]; + const LogicalLocation &loc = oct->loc; int ri = static_cast(loc.lx1); int rj = static_cast(loc.lx2); int rk = static_cast(loc.lx3); - mgroot_->CalculateDefect(octets_[0][o]->def, octets_[0][o]->u, - octets_[0][o]->src, 1, os_, oe_, os_, oe_, os_, oe_, false); + mgroot_->CalculateDefect(oct->def, oct->u, oct->src, oct->coeff, + 1, os_, oe_, os_, oe_, os_, oe_, false); for (int v = 0; v < nvar_; ++v) mgroot_->SetData(MGVariable::src, v, rk, rj, ri, - RestrictOne(octets_[0][o]->def, v, ngh, ngh, ngh)); + RestrictOne(oct->def, v, ngh, ngh, ngh)); if (ffas_) { for (int v = 0; v < nvar_; ++v) mgroot_->SetData(MGVariable::u, v, rk, rj, ri, - RestrictOne(octets_[0][o]->u, v, ngh, ngh, ngh)); + RestrictOne(oct->u, v, ngh, ngh, ngh)); } } } @@ -1067,9 +1072,10 @@ void MultigridDriver::StoreOldDataOctets() { int lev = current_level_ - nrootlevel_; #pragma omp parallel for num_threads(nthreads_) - for (int o = 0; o < noctets_[lev]; ++o) - memcpy(octets_[lev][o]->uold.data(), octets_[lev][o]->u.data(), - octets_[lev][o]->u.GetSizeInBytes()); + for (int o = 0; o < noctets_[lev]; ++o) { + MGOctet *oct = octets_[lev][o]; + memcpy(oct->uold.data(), oct->u.data(), oct->u.GetSizeInBytes()); + } return; } @@ -1082,10 +1088,11 @@ void MultigridDriver::CalculateFASRHSOctets() { int lev = current_level_ - nrootlevel_; #pragma omp parallel for num_threads(nthreads_) - for (int o = 0; o < noctets_[lev]; ++o) - mgroot_->CalculateFASRHS(octets_[lev][o]->src, octets_[lev][o]->u, + for (int o = 0; o < noctets_[lev]; ++o) { + MGOctet *oct = octets_[lev][o]; + mgroot_->CalculateFASRHS(oct->src, oct->u, octets_[lev][o]->coeff, lev+1, os_, oe_, os_, oe_, os_, oe_, false); - + } return; } @@ -1097,9 +1104,11 @@ void MultigridDriver::SmoothOctets(int color) { int lev = current_level_ - nrootlevel_; #pragma omp parallel for num_threads(nthreads_) - for (int o = 0; o < noctets_[lev]; ++o) - mgroot_->Smooth(octets_[lev][o]->u, octets_[lev][o]->src, + for (int o = 0; o < noctets_[lev]; ++o) { + MGOctet *oct = octets_[lev][o]; + mgroot_->Smooth(oct->u, oct->src, oct->coeff, lev+1, os_, oe_, os_, oe_, os_, oe_, color, false); + } return; } @@ -2024,7 +2033,7 @@ void MultigridDriver::CalculateCenterOfMass() { // \brief MGOctet construcor - allocate arrays and coordinates MGOctet::MGOctet(int nvar, int ncoct, int nccoct) : u(nvar, ncoct, ncoct, ncoct), uold(nvar, ncoct, ncoct, ncoct), - def(nvar, ncoct, ncoct, ncoct), src(nvar, ncoct, ncoct, ncoct) { + def(nvar, ncoct, ncoct, ncoct), src(nvar, ncoct, ncoct, ncoct), coeff(nullptr) { coord.AllocateMGCoordinates(ncoct, ncoct, ncoct); ccoord.AllocateMGCoordinates(nccoct, nccoct, nccoct); } From 3e85e54db88a4735b57577e18846197958547c16 Mon Sep 17 00:00:00 2001 From: Kengo Tomida Date: Thu, 23 Nov 2023 18:58:12 +0900 Subject: [PATCH 06/33] *** incomplete commit *** changed MG coefficients to AthenaArray. This is much simpler. --- src/bvals/cc/bvals_cc.cpp | 263 +++++++++++++++++++++++ src/bvals/cc/bvals_cc.hpp | 3 + src/crdiffusion/crdiffusion.cpp | 272 +----------------------- src/crdiffusion/crdiffusion.hpp | 8 +- src/crdiffusion/mg_crdiffusion.cpp | 162 ++------------ src/crdiffusion/mg_crdiffusion.hpp | 40 +--- src/gravity/gravity.cpp | 250 ---------------------- src/gravity/gravity.hpp | 1 - src/gravity/mg_gravity.cpp | 20 +- src/gravity/mg_gravity.hpp | 12 +- src/mesh/meshblock_tree.cpp | 8 +- src/mesh/meshblock_tree.hpp | 2 +- src/multigrid/multigrid.cpp | 63 +++++- src/multigrid/multigrid.hpp | 38 ++-- src/multigrid/multigrid_driver.cpp | 249 +++++++++++++--------- src/task_list/crdiffusion_task_list.cpp | 2 +- src/task_list/grav_task_list.cpp | 2 +- 17 files changed, 545 insertions(+), 850 deletions(-) diff --git a/src/bvals/cc/bvals_cc.cpp b/src/bvals/cc/bvals_cc.cpp index eb6f603918..222fb398be 100644 --- a/src/bvals/cc/bvals_cc.cpp +++ b/src/bvals/cc/bvals_cc.cpp @@ -859,3 +859,266 @@ void CellCenteredBoundaryVariable::ClearBoundary(BoundaryCommSubset phase) { } return; } + + +//---------------------------------------------------------------------------------------- +//! \fn void CellCenteredBoundaryVariable::ExpandPhysicalBoundaries(AthenaArray &u, +//! int nvar) +//! \brief Expand physical boundary values to NGHOST = 2 and to edges/corners. +void CellCenteredBoundaryVariable::ExpandPhysicalBoundaries(AthenaArray &u, + int nvar) { + int is = pmy_block_->is, ie = pmy_block_->ie, + js = pmy_block_->js, je = pmy_block_->je, + ks = pmy_block_->ks, ke = pmy_block_->ke; + + // push face boundary values + for (int n = 0; n < nvar; ++n) { + if (pmy_block_->pbval->nblevel[1][1][0] < 0) { + for (int k = ks; k <= ke; k++) { + for (int j = js; j <= je; j++) + u(nvar, k, j, is-2) = u(nvar, k, j, is-1); + } + } + if (pmy_block_->pbval->nblevel[1][1][2] < 0) { + for (int k = ks; k <= ke; k++) { + for (int j = js; j <= je; j++) + u(nvar, k, j, ie+2) = u(nvar, k, j, ie+1); + } + } + if (pmy_block_->pbval->nblevel[1][0][1] < 0) { + for (int k = ks; k <= ke; k++) { + for (int i = is; i <= ie; i++) + u(nvar, k, js-2, i) = u(nvar, k, js-1, i); + } + } + if (pmy_block_->pbval->nblevel[1][2][1] < 0) { + for (int k = ks; k <= ke; k++) { + for (int i = is; i <= ie; i++) + u(nvar, k, je+2, i) = u(nvar, k, je+1, i); + } + } + if (pmy_block_->pbval->nblevel[0][1][1] < 0) { + for (int j = js; j <= je; j++) { + for (int i = is; i <= ie; i++) + u(nvar, ks-2, j, i) = u(nvar, ks-1, j, i); + } + } + if (pmy_block_->pbval->nblevel[2][1][1] < 0) { + for (int j = js; j <= je; j++) { + for (int i = is; i <= ie; i++) + u(nvar, ke+2, j, i) = u(nvar, ke+1, j, i); + } + } + + // fill edges + if (pmy_block_->pbval->nblevel[1][0][0] < 0) { + for (int k = ks; k <= ke; k++) { + Real p = 0.5*(u(nvar, k, js-1, is) + u(nvar, k, js, is-1)); + u(nvar, k, js-1, is-1) = p; + u(nvar, k, js-1, is-2) = p; + u(nvar, k, js-2, is-1) = p; + u(nvar, k, js-2, is-2) = p; + } + } + if (pmy_block_->pbval->nblevel[1][0][2] < 0) { + for (int k = ks; k <= ke; k++) { + Real p = 0.5*(u(nvar, k, js-1, ie) + u(nvar, k, js, ie+1)); + u(nvar, k, js-1, ie+1) = p; + u(nvar, k, js-1, ie+2) = p; + u(nvar, k, js-2, ie+1) = p; + u(nvar, k, js-2, ie+2) = p; + } + } + if (pmy_block_->pbval->nblevel[1][2][0] < 0) { + for (int k = ks; k <= ke; k++) { + Real p = 0.5*(u(nvar, k, je+1, is) + u(nvar, k, je, is-1)); + u(nvar, k, je+1, is-1) = p; + u(nvar, k, je+1, is-2) = p; + u(nvar, k, je+2, is-1) = p; + u(nvar, k, je+2, is-2) = p; + } + } + if (pmy_block_->pbval->nblevel[1][2][2] < 0) { + for (int k = ks; k <= ke; k++) { + Real p = 0.5*(u(nvar, k, je+1, ie) + u(nvar, k, je, ie+1)); + u(nvar, k, je+1, ie+1) = p; + u(nvar, k, je+1, ie+2) = p; + u(nvar, k, je+2, ie+1) = p; + u(nvar, k, je+2, ie+2) = p; + } + } + if (pmy_block_->pbval->nblevel[0][1][0] < 0) { + for (int j = js; j <= je; j++) { + Real p = 0.5*(u(nvar, ks-1, j, is) + u(nvar, ks, j, is-1)); + u(nvar, ks-1, j, is-1) = p; + u(nvar, ks-1, j, is-2) = p; + u(nvar, ks-2, j, is-1) = p; + u(nvar, ks-2, j, is-2) = p; + } + } + if (pmy_block_->pbval->nblevel[0][1][2] < 0) { + for (int j = js; j <= je; j++) { + Real p = 0.5*(u(nvar, ks-1, j, ie) + u(nvar, ks, j, ie+1)); + u(nvar, ks-1, j, ie+1) = p; + u(nvar, ks-1, j, ie+2) = p; + u(nvar, ks-2, j, ie+1) = p; + u(nvar, ks-2, j, ie+2) = p; + } + } + if (pmy_block_->pbval->nblevel[2][1][0] < 0) { + for (int j = js; j <= je; j++) { + Real p = 0.5*(u(nvar, ke+1, j, is) + u(nvar, ke, j, is-1)); + u(nvar, ke+1, j, is-1) = p; + u(nvar, ke+1, j, is-2) = p; + u(nvar, ke+2, j, is-1) = p; + u(nvar, ke+2, j, is-2) = p; + } + } + if (pmy_block_->pbval->nblevel[2][1][2] < 0) { + for (int j = js; j <= je; j++) { + Real p = 0.5*(u(nvar, ke+1, j, ie) + u(nvar, ke, j, ie+1)); + u(nvar, ke+1, j, ie+1) = p; + u(nvar, ke+1, j, ie+2) = p; + u(nvar, ke+2, j, ie+1) = p; + u(nvar, ke+2, j, ie+2) = p; + } + } + if (pmy_block_->pbval->nblevel[0][0][1] < 0) { + for (int i = is; i <= ie; i++) { + Real p = 0.5*(u(nvar, ks-1, js, i) + u(nvar, ks, js-1, i)); + u(nvar, ks-1, js-1, i) = p; + u(nvar, ks-1, js-2, i) = p; + u(nvar, ks-2, js-1, i) = p; + u(nvar, ks-2, js-2, i) = p; + } + } + if (pmy_block_->pbval->nblevel[0][2][1] < 0) { + for (int i = is; i <= ie; i++) { + Real p = 0.5*(u(nvar, ks-1, je, i) + u(nvar, ks, je+1, i)); + u(nvar, ks-1, je+1, i) = p; + u(nvar, ks-1, je+2, i) = p; + u(nvar, ks-2, je+1, i) = p; + u(nvar, ks-2, je+2, i) = p; + } + } + if (pmy_block_->pbval->nblevel[2][0][1] < 0) { + for (int i = is; i <= ie; i++) { + Real p = 0.5*(u(nvar, ke+1, js, i) + u(nvar, ke, js-1, i)); + u(nvar, ke+1, js-1, i) = p; + u(nvar, ke+1, js-2, i) = p; + u(nvar, ke+2, js-1, i) = p; + u(nvar, ke+2, js-2, i) = p; + } + } + if (pmy_block_->pbval->nblevel[2][2][1] < 0) { + for (int i = is; i <= ie; i++) { + Real p = 0.5*(u(nvar, ke+1, je, i) + u(nvar, ke, je+1, i)); + u(nvar, ke+1, je+1, i) = p; + u(nvar, ke+1, je+2, i) = p; + u(nvar, ke+2, je+1, i) = p; + u(nvar, ke+2, je+2, i) = p; + } + } + + // fill corners + if (pmy_block_->pbval->nblevel[0][0][0] < 0) { + Real p = (u(nvar, ks, js-1, is-1) + u(nvar, ks-1, js, is-1) + + u(nvar, ks-1, js-1, is))/3.0; + u(nvar, ks-1, js-1, is-1) = p; + u(nvar, ks-1, js-1, is-2) = p; + u(nvar, ks-1, js-2, is-1) = p; + u(nvar, ks-1, js-2, is-2) = p; + u(nvar, ks-2, js-1, is-1) = p; + u(nvar, ks-2, js-1, is-2) = p; + u(nvar, ks-2, js-2, is-1) = p; + u(nvar, ks-2, js-2, is-2) = p; + } + if (pmy_block_->pbval->nblevel[0][0][2] < 0) { + Real p = (u(nvar, ks, js-1, ie+1) + u(nvar, ks-1, js, ie+1) + + u(nvar, ks-1, js-1, ie))/3.0; + u(nvar, ks-1, js-1, ie+1) = p; + u(nvar, ks-1, js-1, ie+2) = p; + u(nvar, ks-1, js-2, ie+1) = p; + u(nvar, ks-1, js-2, ie+2) = p; + u(nvar, ks-2, js-1, ie+1) = p; + u(nvar, ks-2, js-1, ie+2) = p; + u(nvar, ks-2, js-2, ie+1) = p; + u(nvar, ks-2, js-2, ie+2) = p; + } + if (pmy_block_->pbval->nblevel[0][2][0] < 0) { + Real p = (u(nvar, ks, je+1, is-1) + u(nvar, ks-1, je, is-1) + + u(nvar, ks-1, je+1, is))/3.0; + u(nvar, ks-1, je+1, is-1) = p; + u(nvar, ks-1, je+1, is-2) = p; + u(nvar, ks-1, je+2, is-1) = p; + u(nvar, ks-1, je+2, is-2) = p; + u(nvar, ks-2, je+1, is-1) = p; + u(nvar, ks-2, je+1, is-2) = p; + u(nvar, ks-2, je+2, is-1) = p; + u(nvar, ks-2, je+2, is-2) = p; + } + if (pmy_block_->pbval->nblevel[2][0][0] < 0) { + Real p = (u(nvar, ke, js-1, is-1) + u(nvar, ke+1, js, is-1) + + u(nvar, ke+1, js-1, is))/3.0; + u(nvar, ke+1, js-1, is-1) = p; + u(nvar, ke+1, js-1, is-2) = p; + u(nvar, ke+1, js-2, is-1) = p; + u(nvar, ke+1, js-2, is-2) = p; + u(nvar, ke+2, js-1, is-1) = p; + u(nvar, ke+2, js-1, is-2) = p; + u(nvar, ke+2, js-2, is-1) = p; + u(nvar, ke+2, js-2, is-2) = p; + } + if (pmy_block_->pbval->nblevel[0][2][2] < 0) { + Real p = (u(nvar, ks, je+1, ie+1) + u(nvar, ks-1, je, ie+1) + + u(nvar, ks-1, je+1, ie))/3.0; + u(nvar, ks-1, je+1, ie+1) = p; + u(nvar, ks-1, je+1, ie+2) = p; + u(nvar, ks-1, je+2, ie+1) = p; + u(nvar, ks-1, je+2, ie+2) = p; + u(nvar, ks-2, je+1, ie+1) = p; + u(nvar, ks-2, je+1, ie+2) = p; + u(nvar, ks-2, je+2, ie+1) = p; + u(nvar, ks-2, je+2, ie+2) = p; + } + if (pmy_block_->pbval->nblevel[2][0][2] < 0) { + Real p = (u(nvar, ke, js-1, ie+1) + u(nvar, ke+1, js, ie+1) + + u(nvar, ke+1, js-1, ie))/3.0; + u(nvar, ke+1, js-1, ie+1) = p; + u(nvar, ke+1, js-1, ie+2) = p; + u(nvar, ke+1, js-2, ie+1) = p; + u(nvar, ke+1, js-2, ie+2) = p; + u(nvar, ke+2, js-1, ie+1) = p; + u(nvar, ke+2, js-1, ie+2) = p; + u(nvar, ke+2, js-2, ie+1) = p; + u(nvar, ke+2, js-2, ie+2) = p; + } + if (pmy_block_->pbval->nblevel[2][2][0] < 0) { + Real p = (u(nvar, ke, je+1, is-1) + u(nvar, ke+1, je, is-1) + + u(nvar, ke+1, je+1, is))/3.0; + u(nvar, ke+1, je+1, is-1) = p; + u(nvar, ke+1, je+1, is-2) = p; + u(nvar, ke+1, je+2, is-1) = p; + u(nvar, ke+1, je+2, is-2) = p; + u(nvar, ke+2, je+1, is-1) = p; + u(nvar, ke+2, je+1, is-2) = p; + u(nvar, ke+2, je+2, is-1) = p; + u(nvar, ke+2, je+2, is-2) = p; + } + if (pmy_block_->pbval->nblevel[2][2][2] < 0) { + Real p = (u(nvar, ke, je+1, ie+1) + u(nvar, ke+1, je, ie+1) + + u(nvar, ke+1, je+1, ie))/3.0; + u(nvar, ke+1, je+1, ie+1) = p; + u(nvar, ke+1, je+1, ie+2) = p; + u(nvar, ke+1, je+2, ie+1) = p; + u(nvar, ke+1, je+2, ie+2) = p; + u(nvar, ke+2, je+1, ie+1) = p; + u(nvar, ke+2, je+1, ie+2) = p; + u(nvar, ke+2, je+2, ie+1) = p; + u(nvar, ke+2, je+2, ie+2) = p; + } + } + + return; +} + diff --git a/src/bvals/cc/bvals_cc.hpp b/src/bvals/cc/bvals_cc.hpp index 3c959bcf2d..e3764de8b3 100644 --- a/src/bvals/cc/bvals_cc.hpp +++ b/src/bvals/cc/bvals_cc.hpp @@ -92,6 +92,9 @@ class CellCenteredBoundaryVariable : public BoundaryVariable { void SetFluxShearingBoxBoundaryBuffers(); //!@} + // for Multigrid + void ExpandPhysicalBoundaries(AthenaArray &u, int nvar); + //!@{ //! BoundaryPhysics: void ReflectInnerX1(Real time, Real dt, diff --git a/src/crdiffusion/crdiffusion.cpp b/src/crdiffusion/crdiffusion.cpp index 3b9c3e0dcf..dcd69a5766 100644 --- a/src/crdiffusion/crdiffusion.cpp +++ b/src/crdiffusion/crdiffusion.cpp @@ -35,12 +35,11 @@ CRDiffusion::CRDiffusion(MeshBlock *pmb, ParameterInput *pin) : pmy_block(pmb), ecr(pmb->ncells3, pmb->ncells2, pmb->ncells1), zeta(pmb->ncells3, pmb->ncells2, pmb->ncells1), + coeff(NCOEFF, pmb->ncells3, pmb->ncells2, pmb->ncells1), coarse_ecr(pmb->ncc3, pmb->ncc2, pmb->ncc1, (pmb->pmy_mesh->multilevel ? AthenaArray::DataStatus::allocated : AthenaArray::DataStatus::empty)), empty_flux{AthenaArray(), AthenaArray(), AthenaArray()}, - D(NCOEFF, pmb->ncells3, pmb->ncells2, pmb->ncells1), - nlambda(pmb->ncells3, pmb->ncells2, pmb->ncells1), output_defect(false), crbvar(pmb, &ecr, &coarse_ecr, empty_flux, false), refinement_idx_(), Dpara_(), Dperp_(), Lambda_() { Dpara_ = pin->GetReal("crdiffusion", "Dpara"); @@ -97,13 +96,13 @@ void CRDiffusion::CalculateCoefficients(const AthenaArray &w, const Real &bz = bcc(IB3,k,j,i); Real ba = std::sqrt(SQR(bx) + SQR(by) + SQR(bz) + TINY_NUMBER); Real nx = bx / ba, ny = by / ba, nz = bz / ba; - D(XX,k,j,i) = Dperp + (Dpara - Dperp) * nx * nx; - D(XY,k,j,i) = (Dpara - Dperp) * nx * ny; - D(XZ,k,j,i) = (Dpara - Dperp) * nx * nz; - D(YY,k,j,i) = Dperp + (Dpara - Dperp) * ny * ny; - D(YZ,k,j,i) = (Dpara - Dperp) * ny * nz; - D(ZZ,k,j,i) = Dperp + (Dpara - Dperp) * nz * nz; - nlambda(k,j,i) = Lambda * w(IDN,k,j,i); + coeff(DXX,k,j,i) = Dperp + (Dpara - Dperp) * nx * nx; + coeff(DXY,k,j,i) = (Dpara - Dperp) * nx * ny; + coeff(DXZ,k,j,i) = (Dpara - Dperp) * nx * nz; + coeff(DYY,k,j,i) = Dperp + (Dpara - Dperp) * ny * ny; + coeff(DYZ,k,j,i) = (Dpara - Dperp) * ny * nz; + coeff(DZZ,k,j,i) = Dperp + (Dpara - Dperp) * nz * nz; + coeff(NLAMBDA, k,j,i) = Lambda * w(IDN,k,j,i); } } } @@ -111,261 +110,12 @@ void CRDiffusion::CalculateCoefficients(const AthenaArray &w, for (int k = kl; k <= ku; ++k) { for (int j = jl; j <= ju; ++j) { for (int i = il; i <= iu; ++i) { - D(XX,k,j,i) = D(XY,k,j,i) = D(XZ,k,j,i) = D(YY,k,j,i) - = D(YZ,k,j,i) = D(ZZ,k,j,i) = Dpara; - nlambda(k,j,i) = Lambda * w(IDN,k,j,i); + coeff(DXX,k,j,i) = coeff(DXY,k,j,i) = coeff(DXZ,k,j,i) = coeff(DYY,k,j,i) + = coeff(DYZ,k,j,i) = coeff(DZZ,k,j,i) = Dpara; + coeff(NLAMBDA, k,j,i) = Lambda * w(IDN,k,j,i); } } } } } -//---------------------------------------------------------------------------------------- -//! \fn CRDiffusion::ExpandPhysicalBoundaries() -//! \brief Expand physical boundary values to NGHOST = 2 and to edges/corners. -void CRDiffusion::ExpandPhysicalBoundaries() { - int is = pmy_block->is, ie = pmy_block->ie, - js = pmy_block->js, je = pmy_block->je, - ks = pmy_block->ks, ke = pmy_block->ke; - - // push face boundary values - if (pmy_block->pbval->nblevel[1][1][0] < 0) { - for (int k = ks; k <= ke; k++) { - for (int j = js; j <= je; j++) - ecr(k, j, is-2) = ecr(k, j, is-1); - } - } - if (pmy_block->pbval->nblevel[1][1][2] < 0) { - for (int k = ks; k <= ke; k++) { - for (int j = js; j <= je; j++) - ecr(k, j, ie+2) = ecr(k, j, ie+1); - } - } - if (pmy_block->pbval->nblevel[1][0][1] < 0) { - for (int k = ks; k <= ke; k++) { - for (int i = is; i <= ie; i++) - ecr(k, js-2, i) = ecr(k, js-1, i); - } - } - if (pmy_block->pbval->nblevel[1][2][1] < 0) { - for (int k = ks; k <= ke; k++) { - for (int i = is; i <= ie; i++) - ecr(k, je+2, i) = ecr(k, je+1, i); - } - } - if (pmy_block->pbval->nblevel[0][1][1] < 0) { - for (int j = js; j <= je; j++) { - for (int i = is; i <= ie; i++) - ecr(ks-2, j, i) = ecr(ks-1, j, i); - } - } - if (pmy_block->pbval->nblevel[2][1][1] < 0) { - for (int j = js; j <= je; j++) { - for (int i = is; i <= ie; i++) - ecr(ke+2, j, i) = ecr(ke+1, j, i); - } - } - - // fill edges - if (pmy_block->pbval->nblevel[1][0][0] < 0) { - for (int k = ks; k <= ke; k++) { - Real p = 0.5*(ecr(k, js-1, is) + ecr(k, js, is-1)); - ecr(k, js-1, is-1) = p; - ecr(k, js-1, is-2) = p; - ecr(k, js-2, is-1) = p; - ecr(k, js-2, is-2) = p; - } - } - if (pmy_block->pbval->nblevel[1][0][2] < 0) { - for (int k = ks; k <= ke; k++) { - Real p = 0.5*(ecr(k, js-1, ie) + ecr(k, js, ie+1)); - ecr(k, js-1, ie+1) = p; - ecr(k, js-1, ie+2) = p; - ecr(k, js-2, ie+1) = p; - ecr(k, js-2, ie+2) = p; - } - } - if (pmy_block->pbval->nblevel[1][2][0] < 0) { - for (int k = ks; k <= ke; k++) { - Real p = 0.5*(ecr(k, je+1, is) + ecr(k, je, is-1)); - ecr(k, je+1, is-1) = p; - ecr(k, je+1, is-2) = p; - ecr(k, je+2, is-1) = p; - ecr(k, je+2, is-2) = p; - } - } - if (pmy_block->pbval->nblevel[1][2][2] < 0) { - for (int k = ks; k <= ke; k++) { - Real p = 0.5*(ecr(k, je+1, ie) + ecr(k, je, ie+1)); - ecr(k, je+1, ie+1) = p; - ecr(k, je+1, ie+2) = p; - ecr(k, je+2, ie+1) = p; - ecr(k, je+2, ie+2) = p; - } - } - if (pmy_block->pbval->nblevel[0][1][0] < 0) { - for (int j = js; j <= je; j++) { - Real p = 0.5*(ecr(ks-1, j, is) + ecr(ks, j, is-1)); - ecr(ks-1, j, is-1) = p; - ecr(ks-1, j, is-2) = p; - ecr(ks-2, j, is-1) = p; - ecr(ks-2, j, is-2) = p; - } - } - if (pmy_block->pbval->nblevel[0][1][2] < 0) { - for (int j = js; j <= je; j++) { - Real p = 0.5*(ecr(ks-1, j, ie) + ecr(ks, j, ie+1)); - ecr(ks-1, j, ie+1) = p; - ecr(ks-1, j, ie+2) = p; - ecr(ks-2, j, ie+1) = p; - ecr(ks-2, j, ie+2) = p; - } - } - if (pmy_block->pbval->nblevel[2][1][0] < 0) { - for (int j = js; j <= je; j++) { - Real p = 0.5*(ecr(ke+1, j, is) + ecr(ke, j, is-1)); - ecr(ke+1, j, is-1) = p; - ecr(ke+1, j, is-2) = p; - ecr(ke+2, j, is-1) = p; - ecr(ke+2, j, is-2) = p; - } - } - if (pmy_block->pbval->nblevel[2][1][2] < 0) { - for (int j = js; j <= je; j++) { - Real p = 0.5*(ecr(ke+1, j, ie) + ecr(ke, j, ie+1)); - ecr(ke+1, j, ie+1) = p; - ecr(ke+1, j, ie+2) = p; - ecr(ke+2, j, ie+1) = p; - ecr(ke+2, j, ie+2) = p; - } - } - if (pmy_block->pbval->nblevel[0][0][1] < 0) { - for (int i = is; i <= ie; i++) { - Real p = 0.5*(ecr(ks-1, js, i) + ecr(ks, js-1, i)); - ecr(ks-1, js-1, i) = p; - ecr(ks-1, js-2, i) = p; - ecr(ks-2, js-1, i) = p; - ecr(ks-2, js-2, i) = p; - } - } - if (pmy_block->pbval->nblevel[0][2][1] < 0) { - for (int i = is; i <= ie; i++) { - Real p = 0.5*(ecr(ks-1, je, i) + ecr(ks, je+1, i)); - ecr(ks-1, je+1, i) = p; - ecr(ks-1, je+2, i) = p; - ecr(ks-2, je+1, i) = p; - ecr(ks-2, je+2, i) = p; - } - } - if (pmy_block->pbval->nblevel[2][0][1] < 0) { - for (int i = is; i <= ie; i++) { - Real p = 0.5*(ecr(ke+1, js, i) + ecr(ke, js-1, i)); - ecr(ke+1, js-1, i) = p; - ecr(ke+1, js-2, i) = p; - ecr(ke+2, js-1, i) = p; - ecr(ke+2, js-2, i) = p; - } - } - if (pmy_block->pbval->nblevel[2][2][1] < 0) { - for (int i = is; i <= ie; i++) { - Real p = 0.5*(ecr(ke+1, je, i) + ecr(ke, je+1, i)); - ecr(ke+1, je+1, i) = p; - ecr(ke+1, je+2, i) = p; - ecr(ke+2, je+1, i) = p; - ecr(ke+2, je+2, i) = p; - } - } - - // fill corners - if (pmy_block->pbval->nblevel[0][0][0] < 0) { - Real p = (ecr(ks, js-1, is-1) + ecr(ks-1, js, is-1) + ecr(ks-1, js-1, is))/3.0; - ecr(ks-1, js-1, is-1) = p; - ecr(ks-1, js-1, is-2) = p; - ecr(ks-1, js-2, is-1) = p; - ecr(ks-1, js-2, is-2) = p; - ecr(ks-2, js-1, is-1) = p; - ecr(ks-2, js-1, is-2) = p; - ecr(ks-2, js-2, is-1) = p; - ecr(ks-2, js-2, is-2) = p; - } - if (pmy_block->pbval->nblevel[0][0][2] < 0) { - Real p = (ecr(ks, js-1, ie+1) + ecr(ks-1, js, ie+1) + ecr(ks-1, js-1, ie))/3.0; - ecr(ks-1, js-1, ie+1) = p; - ecr(ks-1, js-1, ie+2) = p; - ecr(ks-1, js-2, ie+1) = p; - ecr(ks-1, js-2, ie+2) = p; - ecr(ks-2, js-1, ie+1) = p; - ecr(ks-2, js-1, ie+2) = p; - ecr(ks-2, js-2, ie+1) = p; - ecr(ks-2, js-2, ie+2) = p; - } - if (pmy_block->pbval->nblevel[0][2][0] < 0) { - Real p = (ecr(ks, je+1, is-1) + ecr(ks-1, je, is-1) + ecr(ks-1, je+1, is))/3.0; - ecr(ks-1, je+1, is-1) = p; - ecr(ks-1, je+1, is-2) = p; - ecr(ks-1, je+2, is-1) = p; - ecr(ks-1, je+2, is-2) = p; - ecr(ks-2, je+1, is-1) = p; - ecr(ks-2, je+1, is-2) = p; - ecr(ks-2, je+2, is-1) = p; - ecr(ks-2, je+2, is-2) = p; - } - if (pmy_block->pbval->nblevel[2][0][0] < 0) { - Real p = (ecr(ke, js-1, is-1) + ecr(ke+1, js, is-1) + ecr(ke+1, js-1, is))/3.0; - ecr(ke+1, js-1, is-1) = p; - ecr(ke+1, js-1, is-2) = p; - ecr(ke+1, js-2, is-1) = p; - ecr(ke+1, js-2, is-2) = p; - ecr(ke+2, js-1, is-1) = p; - ecr(ke+2, js-1, is-2) = p; - ecr(ke+2, js-2, is-1) = p; - ecr(ke+2, js-2, is-2) = p; - } - if (pmy_block->pbval->nblevel[0][2][2] < 0) { - Real p = (ecr(ks, je+1, ie+1) + ecr(ks-1, je, ie+1) + ecr(ks-1, je+1, ie))/3.0; - ecr(ks-1, je+1, ie+1) = p; - ecr(ks-1, je+1, ie+2) = p; - ecr(ks-1, je+2, ie+1) = p; - ecr(ks-1, je+2, ie+2) = p; - ecr(ks-2, je+1, ie+1) = p; - ecr(ks-2, je+1, ie+2) = p; - ecr(ks-2, je+2, ie+1) = p; - ecr(ks-2, je+2, ie+2) = p; - } - if (pmy_block->pbval->nblevel[2][0][2] < 0) { - Real p = (ecr(ke, js-1, ie+1) + ecr(ke+1, js, ie+1) + ecr(ke+1, js-1, ie))/3.0; - ecr(ke+1, js-1, ie+1) = p; - ecr(ke+1, js-1, ie+2) = p; - ecr(ke+1, js-2, ie+1) = p; - ecr(ke+1, js-2, ie+2) = p; - ecr(ke+2, js-1, ie+1) = p; - ecr(ke+2, js-1, ie+2) = p; - ecr(ke+2, js-2, ie+1) = p; - ecr(ke+2, js-2, ie+2) = p; - } - if (pmy_block->pbval->nblevel[2][2][0] < 0) { - Real p = (ecr(ke, je+1, is-1) + ecr(ke+1, je, is-1) + ecr(ke+1, je+1, is))/3.0; - ecr(ke+1, je+1, is-1) = p; - ecr(ke+1, je+1, is-2) = p; - ecr(ke+1, je+2, is-1) = p; - ecr(ke+1, je+2, is-2) = p; - ecr(ke+2, je+1, is-1) = p; - ecr(ke+2, je+1, is-2) = p; - ecr(ke+2, je+2, is-1) = p; - ecr(ke+2, je+2, is-2) = p; - } - if (pmy_block->pbval->nblevel[2][2][2] < 0) { - Real p = (ecr(ke, je+1, ie+1) + ecr(ke+1, je, ie+1) + ecr(ke+1, je+1, ie))/3.0; - ecr(ke+1, je+1, ie+1) = p; - ecr(ke+1, je+1, ie+2) = p; - ecr(ke+1, je+2, ie+1) = p; - ecr(ke+1, je+2, ie+2) = p; - ecr(ke+2, je+1, ie+1) = p; - ecr(ke+2, je+1, ie+2) = p; - ecr(ke+2, je+2, ie+1) = p; - ecr(ke+2, je+2, ie+2) = p; - } - - return; -} - diff --git a/src/crdiffusion/crdiffusion.hpp b/src/crdiffusion/crdiffusion.hpp index 5c2afdf9e8..21de98ee8a 100644 --- a/src/crdiffusion/crdiffusion.hpp +++ b/src/crdiffusion/crdiffusion.hpp @@ -26,8 +26,8 @@ class CRDiffusionBoundaryValues; class MGCRDiffusion; class MGCRDiffusionDriver; -constexpr int NCOEFF = 6; -enum DCoeffIndex {XX = 0, XY = 1, YX = 1, XZ = 2, ZX = 2, YY = 3, YZ=4, ZZ=5}; +constexpr int NCOEFF = 7; +enum CoeffIndex {DXX=0, DXY=1, DYX=1, DXZ=2, DZX=2, DYY=3, DYZ=4, DZZ=5, NLAMBDA=6}; //! \class CRDiffusion //! \brief gravitational potential data and functions @@ -39,9 +39,8 @@ class CRDiffusion { MeshBlock* pmy_block; MGCRDiffusion *pmg; - AthenaArray ecr, zeta; + AthenaArray ecr, zeta, coeff; AthenaArray coarse_ecr, empty_flux[3]; - AthenaArray D, nlambda; AthenaArray def; // defect from the Multigrid solver bool output_defect; @@ -50,7 +49,6 @@ class CRDiffusion { void CalculateCoefficients(const AthenaArray &w, const AthenaArray &bcc); void Solve(); - void ExpandPhysicalBoundaries(); friend class MGCRDiffusuionDriver; diff --git a/src/crdiffusion/mg_crdiffusion.cpp b/src/crdiffusion/mg_crdiffusion.cpp index ebefd409e1..87a400f3be 100644 --- a/src/crdiffusion/mg_crdiffusion.cpp +++ b/src/crdiffusion/mg_crdiffusion.cpp @@ -25,6 +25,7 @@ #include "../multigrid/multigrid.hpp" #include "../parameter_input.hpp" #include "../task_list/crdiffusion_task_list.hpp" +#include "crdiffusion.hpp" #include "mg_crdiffusion.hpp" #ifdef MPI_PARALLEL @@ -39,7 +40,7 @@ class MeshBlock; MGCRDiffusionDriver::MGCRDiffusionDriver(Mesh *pm, ParameterInput *pin) : MultigridDriver(pm, pm->MGCRDiffusionBoundaryFunction_, - pm->MGCRDiffusionSourceMaskFunction_, 1) { + pm->MGCRDiffusionSourceMaskFunction_, 1, NCOEFF) { eps_ = pin->GetOrAddReal("crdiffusion", "threshold", -1.0); niter_ = pin->GetOrAddInteger("crdiffusion", "niteration", -1); ffas_ = pin->GetOrAddBoolean("crdiffusion", "fas", ffas_); @@ -83,7 +84,7 @@ MGCRDiffusionDriver::MGCRDiffusionDriver(Mesh *pm, ParameterInput *pin) mgtlist_ = new MultigridTaskList(this); // Allocate the root multigrid - mgroot_ = mgcrroot_ = new MGCRDiffusion(this, nullptr); + mgroot_ = new MGCRDiffusion(this, nullptr); crtlist_ = new CRDiffusionBoundaryTaskList(pin, pm); } @@ -105,16 +106,7 @@ MGCRDiffusionDriver::~MGCRDiffusionDriver() { //! \brief MGCRDiffusion constructor MGCRDiffusion::MGCRDiffusion(MultigridDriver *pmd, MeshBlock *pmb) - : Multigrid(pmd, pmb, 1, 1) { - coeff_ = new MGCRCoefficient*[nlevel_]; - for (int l = 0; l < nlevel_; l++) { - int ll = nlevel_-1-l; - int ncx = (size_.nx1>>ll) + 2*ngh_; - int ncy = (size_.nx2>>ll) + 2*ngh_; - int ncz = (size_.nx3>>ll) + 2*ngh_; - coeff_[l] = new MGCRCoefficient(ncx, ncy, ncz); - } - + : Multigrid(pmd, pmb, 1) { btype = btypef = BoundaryQuantity::mg; pmgbval = new MGBoundaryValues(this, mg_block_bcs_); } @@ -125,75 +117,10 @@ MGCRDiffusion::MGCRDiffusion(MultigridDriver *pmd, MeshBlock *pmb) //! \brief MGCRDiffusion deconstructor MGCRDiffusion::~MGCRDiffusion() { - for (int l = 0; l < nlevel_; l++) - delete coeff_[l]; - delete [] coeff_; delete pmgbval; } -//---------------------------------------------------------------------------------------- -//! \fn void MGCRDiffusionDriver::RestrictCoefficientsOctets() -//! \brief restrict coefficients in Octets for the CR diffusion equation - -void MGCRDiffusionDriver::RestrictCoefficients() { - for (auto itr = vmg_.begin(); itr < vmg_.end(); itr++) { - MGCRDiffusion *pmg = static_cast(*itr); - pmg->RestrictCoefficients(); - } - if (nreflevel_ > 0) { - const int &ngh = mgcrroot_->ngh_; - for (int l = nreflevel_ - 1; l >= 1; --l) { // fine octets to coarse octets -#pragma omp parallel for num_threads(nthreads_) - for (int o = 0; o < noctets_[l]; ++o) { - MGCROctet *foct = static_cast(octets_[l][o]); - const LogicalLocation &loc = foct->loc; - LogicalLocation cloc; - cloc.lx1 = (loc.lx1 >> 1); - cloc.lx2 = (loc.lx2 >> 1); - cloc.lx3 = (loc.lx3 >> 1); - cloc.level = loc.level - 1; - int oid = octetmap_[l-1][cloc]; - int oi = (static_cast(loc.lx1) & 1) + ngh; - int oj = (static_cast(loc.lx2) & 1) + ngh; - int ok = (static_cast(loc.lx3) & 1) + ngh; - MGCROctet *coct = static_cast(octets_[l-1][oid]); - MGCRCoefficient *ccoeff = static_cast(coct->coeff); - MGCRCoefficient *fcoeff = static_cast(foct->coeff); - for (int v = 0; v < NCOEFF; ++v) - ccoeff->D(v, ok, oj, oi) = RestrictOne(fcoeff->D, v, ngh, ngh, ngh); - ccoeff->nlambda(ok, oj, oi) = RestrictOne(fcoeff->nlambda, 0, ngh, ngh, ngh); - - } - } -#pragma omp parallel for num_threads(nthreads_) - for (int o = 0; o < noctets_[0]; ++o) { // octets to the root grid - const LogicalLocation &loc = octets_[0][o]->loc; - MGCROctet *oct = static_cast(octets_[0][o]); - int lx1 = static_cast(loc.lx1) + mgcrroot_->ngh_; - int lx2 = static_cast(loc.lx2) + mgcrroot_->ngh_; - int lx3 = static_cast(loc.lx3) + mgcrroot_->ngh_; - MGCRCoefficient *coeff = static_cast(oct->coeff); - for (int v = 0; v < NCOEFF; ++v) - mgcrroot_->coeff_[mgcrroot_->nlevel_-1]->D(v, lx3, lx2, lx1) - = RestrictOne(coeff->D, v, ngh, ngh, ngh); - mgcrroot_->coeff_[mgcrroot_->nlevel_-1]->nlambda(lx3, lx2, lx1) - = RestrictOne(coeff->nlambda, 0, ngh, ngh, ngh); - } - } - mgcrroot_->RestrictCoefficients(); - return; - -} - -//---------------------------------------------------------------------------------------- -//! \fn MGOctet* MGCRDiffusionDriver::AllocateOctet() -// \brief Allocate a MGCROctet object -MGOctet* MGCRDiffusionDriver::AllocateOctet() { - int ncoct = mgroot_->ngh_*2 + 2, nccoct = mgroot_->ngh_*2 + 1; - return new MGCROctet(nvar_, ncoct, nccoct); -} - //---------------------------------------------------------------------------------------- //! \fn void MGCRDiffusionDriver::Solve(int stage) //! \brief load the data and solve @@ -212,12 +139,10 @@ void MGCRDiffusionDriver::Solve(int stage) { pmg->LoadSource(pmg->pmy_block_->pcrdiff->ecr, 0, NGHOST, 1.0); if (mode_ == 1) // use the previous timestep data as the initial guess pmg->LoadFinestData(pmg->pmy_block_->pcrdiff->ecr, 0, NGHOST); - pmg->LoadCoefficients(pmg->pmy_block_->pcrdiff->D, - pmg->pmy_block_->pcrdiff->nlambda, NGHOST); + pmg->LoadCoefficients(pmg->pmy_block_->pcrdiff->coeff, NGHOST); } SetupMultigrid(); - RestrictCoefficients(); if (mode_ == 0) { SolveFMGCycle(); @@ -244,43 +169,15 @@ void MGCRDiffusionDriver::Solve(int stage) { } -//---------------------------------------------------------------------------------------- -//! \fn void MGCRDiffusion::AllocateCoefficient(int ncx, int ncy, int ncz) -//! \brief allocate coefficients for cosmic-ray transport - -MGCoefficient* MGCRDiffusion::AllocateCoefficient(int ncx, int ncy, int ncz) { - return new MGCRCoefficient(ncx, ncy, ncz); -} - - -//---------------------------------------------------------------------------------------- -//! \fn void MGCRDiffusion::RestrictCoefficients() -//! \brief restrict coefficients within a MGCRDiffusion object - -void MGCRDiffusion::RestrictCoefficients() { - int is, ie, js, je, ks, ke; - is=js=ks=ngh_; - for (current_level_=nlevel_-1; current_level_>0; current_level_--) { - int ll=nlevel_-current_level_; - ie=is+(size_.nx1>>ll)-1, je=js+(size_.nx2>>ll)-1, ke=ks+(size_.nx3>>ll)-1; - Restrict(coeff_[current_level_-1]->D, coeff_[current_level_]->D, - NCOEFF, is, ie, js, je, ks, ke, false); - Restrict(coeff_[current_level_-1]->nlambda, coeff_[current_level_]->nlambda, - 1, is, ie, js, je, ks, ke, false); - } - return; -} - - //---------------------------------------------------------------------------------------- //! \fn void MGCRDiffusion::Smooth(AthenaArray &u, const AthenaArray &src, -//! MGCoefficient *coeff, int rlev, int il, int iu, +//! const AthenaArray &coeff, int rlev, int il, int iu, //! int jl, int ju, int kl, int ku, int color, bool th) //! \brief Implementation of the Red-Black Gauss-Seidel Smoother //! rlev = relative level from the finest level of this Multigrid block void MGCRDiffusion::Smooth(AthenaArray &u, const AthenaArray &src, - MGCoefficient *coeff, int rlev, int il, int iu, + const AthenaArray &coeff, int rlev, int il, int iu, int jl, int ju, int kl, int ku, int color, bool th) { Real dx; if (rlev <= 0) dx = rdx_*static_cast(1<<(-rlev)); @@ -313,14 +210,14 @@ void MGCRDiffusion::Smooth(AthenaArray &u, const AthenaArray &src, //---------------------------------------------------------------------------------------- //! \fn void MGCRDiffusion::CalculateDefect(AthenaArray &def, //! const AthenaArray &u, const AthenaArray &src, -//! MGCoefficient *coeff, int rlev, int il, int iu, +//! const AthenaArray &coeff, int rlev, int il, int iu, //! int jl, int ju, int kl, int ku, bool th) //! \brief Implementation of the Defect calculation //! rlev = relative level from the finest level of this Multigrid block void MGCRDiffusion::CalculateDefect(AthenaArray &def, const AthenaArray &u, - const AthenaArray &src, MGCoefficient *coeff, int rlev, - int il, int iu, int jl, int ju, int kl, int ku, bool th) { + const AthenaArray &src, const AthenaArray &coeff, + int rlev, int il, int iu, int jl, int ju, int kl, int ku, bool th) { Real dx; if (rlev <= 0) dx = rdx_*static_cast(1<<(-rlev)); else dx = rdx_/static_cast(1< &def, const AthenaArray &src, -//! const AthenaArray &u, MGCoefficient *coeff, int rlev, -//! int il, int iu, int jl, int ju, int kl, int ku, bool th) +//! const AthenaArray &u, const AthenaArray &coeff, +//! int rlev, int il, int iu, int jl, int ju, int kl, int ku, bool th) //! \brief Implementation of the RHS calculation for FAS //! rlev = relative level from the finest level of this Multigrid block void MGCRDiffusion::CalculateFASRHS(AthenaArray &src, const AthenaArray &u, - MGCoefficient *coeff, int rlev, int il, int iu, - int jl, int ju, int kl, int ku, bool th) { + const AthenaArray &coeff, int rlev, int il, int iu, + int jl, int ju, int kl, int ku, bool th) { Real dx; if (rlev <= 0) dx = rdx_*static_cast(1<<(-rlev)); else dx = rdx_/static_cast(1< &src, const AthenaArray &D, -//! const AthenaArray &nlambda, int ngh) -//! \brief Load coefficients of the diffusion and source terms - -void MGCRDiffusion::LoadCoefficients(const AthenaArray &D, - const AthenaArray &nlambda, int ngh) { - AthenaArray &Dm=coeff_[nlevel_-1]->D; - AthenaArray &nl=coeff_[nlevel_-1]->nlambda; - int is, ie, js, je, ks, ke; - is=js=ks=ngh_; - ie=is+size_.nx1-1, je=js+size_.nx2-1, ke=ks+size_.nx3-1; - for (int mk=ks; mk<=ke; ++mk) { - int k = mk - ks + ngh; - for (int mj=js; mj<=je; ++mj) { - int j = mj - js + ngh; -#pragma omp simd - for (int mi=is; mi<=ie; ++mi) { - int i = mi - is + ngh; - Dm(XX,mk,mj,mi) = D(XX,k,j,i); - Dm(XY,mk,mj,mi) = D(XY,k,j,i); - Dm(XZ,mk,mj,mi) = D(XZ,k,j,i); - Dm(YY,mk,mj,mi) = D(YY,k,j,i); - Dm(YZ,mk,mj,mi) = D(YZ,k,j,i); - Dm(ZZ,mk,mj,mi) = D(ZZ,k,j,i); - nl(mk,mj,mi) = nlambda(k,j,i); - } - } - } - return; -} diff --git a/src/crdiffusion/mg_crdiffusion.hpp b/src/crdiffusion/mg_crdiffusion.hpp index 2df68b14d5..f347f9c949 100644 --- a/src/crdiffusion/mg_crdiffusion.hpp +++ b/src/crdiffusion/mg_crdiffusion.hpp @@ -25,25 +25,6 @@ class Multigrid; class CRDiffusionBoundaryTaskList; -//! \class MGCRCoefficient -class MGCRCoefficient : public MGCoefficient { - public: - MGCRCoefficient(int nx, int ny, int nz) : D(NCOEFF, nz, ny, nx), - nlambda(nz, ny, nx) {}; - AthenaArray D, nlambda; -}; - - -//! \class MGCROctet -//! \brief MGOctet for cosmic ray transport - -class MGCROctet : public MGOctet { - public: - MGCROctet(int nvar, int ncoct, int nccoct) : MGOctet(nvar, ncoct, nccoct) - { coeff = new MGCRCoefficient(ncoct, ncoct, ncoct);}; - ~MGCROctet() { delete coeff; }; -}; - //! \class MGCRDiffusion //! \brief Multigrid CR Diffusion solver for each block @@ -53,25 +34,18 @@ class MGCRDiffusion : public Multigrid { ~MGCRDiffusion(); void Smooth(AthenaArray &dst, const AthenaArray &src, - MGCoefficient *coeff, int rlev, int il, int iu, int jl, int ju, - int kl, int ku, int color, bool th) final; + const AthenaArray &coeff, int rlev, int il, int iu, + int jl, int ju, int kl, int ku, int color, bool th) final; void CalculateDefect(AthenaArray &def, const AthenaArray &u, - const AthenaArray &src, MGCoefficient *coeff, int rlev, - int il, int iu, int jl, int ju, int kl, int ku, bool th) final; + const AthenaArray &src, const AthenaArray &coeff, + int rlev, int il, int iu, int jl, int ju, int kl, int ku, bool th) final; void CalculateFASRHS(AthenaArray &def, const AthenaArray &src, - MGCoefficient *coeff, int rlev, int il, int iu, + const AthenaArray &coeff, int rlev, int il, int iu, int jl, int ju, int kl, int ku, bool th) final; - MGCoefficient* AllocateCoefficient(int ncx, int ncy, int ncz) final; - void LoadCoefficients(const AthenaArray &D, - const AthenaArray &nlambda, int ngh); - void RestrictCoefficients(); - friend class MGCRDiffusionDriver; private: - MGCRCoefficient **coeff_; - static constexpr Real omega_ = 1.15; }; @@ -84,12 +58,8 @@ class MGCRDiffusionDriver : public MultigridDriver { MGCRDiffusionDriver(Mesh *pm, ParameterInput *pin); ~MGCRDiffusionDriver(); void Solve(int stage) final; - void RestrictCoefficients(); private: - MGOctet* AllocateOctet() final; - - MGCRDiffusion *mgcrroot_; CRDiffusionBoundaryTaskList *crtlist_; }; diff --git a/src/gravity/gravity.cpp b/src/gravity/gravity.cpp index 20c7ab0f14..1967189e22 100644 --- a/src/gravity/gravity.cpp +++ b/src/gravity/gravity.cpp @@ -180,253 +180,3 @@ void Gravity::RestoreFaceBoundaries() { return; } - -//---------------------------------------------------------------------------------------- -//! \fn Gravity::ExpandPhysicalBoundaries() -//! \brief Expand physical boundary values to NGHOST = 2 and to edges/corners. -void Gravity::ExpandPhysicalBoundaries() { - int is = pmy_block->is, ie = pmy_block->ie, - js = pmy_block->js, je = pmy_block->je, - ks = pmy_block->ks, ke = pmy_block->ke; - - // push face boundary values - if (pmy_block->pbval->nblevel[1][1][0] < 0) { - for (int k = ks; k <= ke; k++) { - for (int j = js; j <= je; j++) - phi(k, j, is-2) = phi(k, j, is-1); - } - } - if (pmy_block->pbval->nblevel[1][1][2] < 0) { - for (int k = ks; k <= ke; k++) { - for (int j = js; j <= je; j++) - phi(k, j, ie+2) = phi(k, j, ie+1); - } - } - if (pmy_block->pbval->nblevel[1][0][1] < 0) { - for (int k = ks; k <= ke; k++) { - for (int i = is; i <= ie; i++) - phi(k, js-2, i) = phi(k, js-1, i); - } - } - if (pmy_block->pbval->nblevel[1][2][1] < 0) { - for (int k = ks; k <= ke; k++) { - for (int i = is; i <= ie; i++) - phi(k, je+2, i) = phi(k, je+1, i); - } - } - if (pmy_block->pbval->nblevel[0][1][1] < 0) { - for (int j = js; j <= je; j++) { - for (int i = is; i <= ie; i++) - phi(ks-2, j, i) = phi(ks-1, j, i); - } - } - if (pmy_block->pbval->nblevel[2][1][1] < 0) { - for (int j = js; j <= je; j++) { - for (int i = is; i <= ie; i++) - phi(ke+2, j, i) = phi(ke+1, j, i); - } - } - - // fill edges - if (pmy_block->pbval->nblevel[1][0][0] < 0) { - for (int k = ks; k <= ke; k++) { - Real p = 0.5*(phi(k, js-1, is) + phi(k, js, is-1)); - phi(k, js-1, is-1) = p; - phi(k, js-1, is-2) = p; - phi(k, js-2, is-1) = p; - phi(k, js-2, is-2) = p; - } - } - if (pmy_block->pbval->nblevel[1][0][2] < 0) { - for (int k = ks; k <= ke; k++) { - Real p = 0.5*(phi(k, js-1, ie) + phi(k, js, ie+1)); - phi(k, js-1, ie+1) = p; - phi(k, js-1, ie+2) = p; - phi(k, js-2, ie+1) = p; - phi(k, js-2, ie+2) = p; - } - } - if (pmy_block->pbval->nblevel[1][2][0] < 0) { - for (int k = ks; k <= ke; k++) { - Real p = 0.5*(phi(k, je+1, is) + phi(k, je, is-1)); - phi(k, je+1, is-1) = p; - phi(k, je+1, is-2) = p; - phi(k, je+2, is-1) = p; - phi(k, je+2, is-2) = p; - } - } - if (pmy_block->pbval->nblevel[1][2][2] < 0) { - for (int k = ks; k <= ke; k++) { - Real p = 0.5*(phi(k, je+1, ie) + phi(k, je, ie+1)); - phi(k, je+1, ie+1) = p; - phi(k, je+1, ie+2) = p; - phi(k, je+2, ie+1) = p; - phi(k, je+2, ie+2) = p; - } - } - if (pmy_block->pbval->nblevel[0][1][0] < 0) { - for (int j = js; j <= je; j++) { - Real p = 0.5*(phi(ks-1, j, is) + phi(ks, j, is-1)); - phi(ks-1, j, is-1) = p; - phi(ks-1, j, is-2) = p; - phi(ks-2, j, is-1) = p; - phi(ks-2, j, is-2) = p; - } - } - if (pmy_block->pbval->nblevel[0][1][2] < 0) { - for (int j = js; j <= je; j++) { - Real p = 0.5*(phi(ks-1, j, ie) + phi(ks, j, ie+1)); - phi(ks-1, j, ie+1) = p; - phi(ks-1, j, ie+2) = p; - phi(ks-2, j, ie+1) = p; - phi(ks-2, j, ie+2) = p; - } - } - if (pmy_block->pbval->nblevel[2][1][0] < 0) { - for (int j = js; j <= je; j++) { - Real p = 0.5*(phi(ke+1, j, is) + phi(ke, j, is-1)); - phi(ke+1, j, is-1) = p; - phi(ke+1, j, is-2) = p; - phi(ke+2, j, is-1) = p; - phi(ke+2, j, is-2) = p; - } - } - if (pmy_block->pbval->nblevel[2][1][2] < 0) { - for (int j = js; j <= je; j++) { - Real p = 0.5*(phi(ke+1, j, ie) + phi(ke, j, ie+1)); - phi(ke+1, j, ie+1) = p; - phi(ke+1, j, ie+2) = p; - phi(ke+2, j, ie+1) = p; - phi(ke+2, j, ie+2) = p; - } - } - if (pmy_block->pbval->nblevel[0][0][1] < 0) { - for (int i = is; i <= ie; i++) { - Real p = 0.5*(phi(ks-1, js, i) + phi(ks, js-1, i)); - phi(ks-1, js-1, i) = p; - phi(ks-1, js-2, i) = p; - phi(ks-2, js-1, i) = p; - phi(ks-2, js-2, i) = p; - } - } - if (pmy_block->pbval->nblevel[0][2][1] < 0) { - for (int i = is; i <= ie; i++) { - Real p = 0.5*(phi(ks-1, je, i) + phi(ks, je+1, i)); - phi(ks-1, je+1, i) = p; - phi(ks-1, je+2, i) = p; - phi(ks-2, je+1, i) = p; - phi(ks-2, je+2, i) = p; - } - } - if (pmy_block->pbval->nblevel[2][0][1] < 0) { - for (int i = is; i <= ie; i++) { - Real p = 0.5*(phi(ke+1, js, i) + phi(ke, js-1, i)); - phi(ke+1, js-1, i) = p; - phi(ke+1, js-2, i) = p; - phi(ke+2, js-1, i) = p; - phi(ke+2, js-2, i) = p; - } - } - if (pmy_block->pbval->nblevel[2][2][1] < 0) { - for (int i = is; i <= ie; i++) { - Real p = 0.5*(phi(ke+1, je, i) + phi(ke, je+1, i)); - phi(ke+1, je+1, i) = p; - phi(ke+1, je+2, i) = p; - phi(ke+2, je+1, i) = p; - phi(ke+2, je+2, i) = p; - } - } - - // fill corners - if (pmy_block->pbval->nblevel[0][0][0] < 0) { - Real p = (phi(ks, js-1, is-1) + phi(ks-1, js, is-1) + phi(ks-1, js-1, is))/3.0; - phi(ks-1, js-1, is-1) = p; - phi(ks-1, js-1, is-2) = p; - phi(ks-1, js-2, is-1) = p; - phi(ks-1, js-2, is-2) = p; - phi(ks-2, js-1, is-1) = p; - phi(ks-2, js-1, is-2) = p; - phi(ks-2, js-2, is-1) = p; - phi(ks-2, js-2, is-2) = p; - } - if (pmy_block->pbval->nblevel[0][0][2] < 0) { - Real p = (phi(ks, js-1, ie+1) + phi(ks-1, js, ie+1) + phi(ks-1, js-1, ie))/3.0; - phi(ks-1, js-1, ie+1) = p; - phi(ks-1, js-1, ie+2) = p; - phi(ks-1, js-2, ie+1) = p; - phi(ks-1, js-2, ie+2) = p; - phi(ks-2, js-1, ie+1) = p; - phi(ks-2, js-1, ie+2) = p; - phi(ks-2, js-2, ie+1) = p; - phi(ks-2, js-2, ie+2) = p; - } - if (pmy_block->pbval->nblevel[0][2][0] < 0) { - Real p = (phi(ks, je+1, is-1) + phi(ks-1, je, is-1) + phi(ks-1, je+1, is))/3.0; - phi(ks-1, je+1, is-1) = p; - phi(ks-1, je+1, is-2) = p; - phi(ks-1, je+2, is-1) = p; - phi(ks-1, je+2, is-2) = p; - phi(ks-2, je+1, is-1) = p; - phi(ks-2, je+1, is-2) = p; - phi(ks-2, je+2, is-1) = p; - phi(ks-2, je+2, is-2) = p; - } - if (pmy_block->pbval->nblevel[2][0][0] < 0) { - Real p = (phi(ke, js-1, is-1) + phi(ke+1, js, is-1) + phi(ke+1, js-1, is))/3.0; - phi(ke+1, js-1, is-1) = p; - phi(ke+1, js-1, is-2) = p; - phi(ke+1, js-2, is-1) = p; - phi(ke+1, js-2, is-2) = p; - phi(ke+2, js-1, is-1) = p; - phi(ke+2, js-1, is-2) = p; - phi(ke+2, js-2, is-1) = p; - phi(ke+2, js-2, is-2) = p; - } - if (pmy_block->pbval->nblevel[0][2][2] < 0) { - Real p = (phi(ks, je+1, ie+1) + phi(ks-1, je, ie+1) + phi(ks-1, je+1, ie))/3.0; - phi(ks-1, je+1, ie+1) = p; - phi(ks-1, je+1, ie+2) = p; - phi(ks-1, je+2, ie+1) = p; - phi(ks-1, je+2, ie+2) = p; - phi(ks-2, je+1, ie+1) = p; - phi(ks-2, je+1, ie+2) = p; - phi(ks-2, je+2, ie+1) = p; - phi(ks-2, je+2, ie+2) = p; - } - if (pmy_block->pbval->nblevel[2][0][2] < 0) { - Real p = (phi(ke, js-1, ie+1) + phi(ke+1, js, ie+1) + phi(ke+1, js-1, ie))/3.0; - phi(ke+1, js-1, ie+1) = p; - phi(ke+1, js-1, ie+2) = p; - phi(ke+1, js-2, ie+1) = p; - phi(ke+1, js-2, ie+2) = p; - phi(ke+2, js-1, ie+1) = p; - phi(ke+2, js-1, ie+2) = p; - phi(ke+2, js-2, ie+1) = p; - phi(ke+2, js-2, ie+2) = p; - } - if (pmy_block->pbval->nblevel[2][2][0] < 0) { - Real p = (phi(ke, je+1, is-1) + phi(ke+1, je, is-1) + phi(ke+1, je+1, is))/3.0; - phi(ke+1, je+1, is-1) = p; - phi(ke+1, je+1, is-2) = p; - phi(ke+1, je+2, is-1) = p; - phi(ke+1, je+2, is-2) = p; - phi(ke+2, je+1, is-1) = p; - phi(ke+2, je+1, is-2) = p; - phi(ke+2, je+2, is-1) = p; - phi(ke+2, je+2, is-2) = p; - } - if (pmy_block->pbval->nblevel[2][2][2] < 0) { - Real p = (phi(ke, je+1, ie+1) + phi(ke+1, je, ie+1) + phi(ke+1, je+1, ie))/3.0; - phi(ke+1, je+1, ie+1) = p; - phi(ke+1, je+1, ie+2) = p; - phi(ke+1, je+2, ie+1) = p; - phi(ke+1, je+2, ie+2) = p; - phi(ke+2, je+1, ie+1) = p; - phi(ke+2, je+1, ie+2) = p; - phi(ke+2, je+2, ie+1) = p; - phi(ke+2, je+2, ie+2) = p; - } - - return; -} - diff --git a/src/gravity/gravity.hpp b/src/gravity/gravity.hpp index f8a75901db..ad0c05e7c6 100644 --- a/src/gravity/gravity.hpp +++ b/src/gravity/gravity.hpp @@ -48,7 +48,6 @@ class Gravity { void SaveFaceBoundaries(); void RestoreFaceBoundaries(); - void ExpandPhysicalBoundaries(); friend class MGGravityDriver; diff --git a/src/gravity/mg_gravity.cpp b/src/gravity/mg_gravity.cpp index d1368bde5a..fa5211a57f 100644 --- a/src/gravity/mg_gravity.cpp +++ b/src/gravity/mg_gravity.cpp @@ -40,7 +40,7 @@ class MeshBlock; MGGravityDriver::MGGravityDriver(Mesh *pm, ParameterInput *pin) : MultigridDriver(pm, pm->MGGravityBoundaryFunction_, - pm->MGGravitySourceMaskFunction_, 1) { + pm->MGGravitySourceMaskFunction_, 1, 0) { four_pi_G_ = pmy_mesh_->four_pi_G_; eps_ = pin->GetOrAddReal("gravity", "threshold", -1.0); niter_ = pin->GetOrAddInteger("gravity", "niteration", -1); @@ -144,7 +144,7 @@ MGGravityDriver::~MGGravityDriver() { //! \fn MGGravity::MGGravity(MultigridDriver *pmd, MeshBlock *pmb) //! \brief MGGravity constructor -MGGravity::MGGravity(MultigridDriver *pmd, MeshBlock *pmb) : Multigrid(pmd, pmb, 1, 1) { +MGGravity::MGGravity(MultigridDriver *pmd, MeshBlock *pmb) : Multigrid(pmd, pmb, 1) { btype = BoundaryQuantity::mg; btypef = BoundaryQuantity::mg_faceonly; pmgbval = new MGGravityBoundaryValues(this, mg_block_bcs_); @@ -211,14 +211,14 @@ void MGGravityDriver::Solve(int stage) { //---------------------------------------------------------------------------------------- //! \fn void MGGravity::Smooth(AthenaArray &u, const AthenaArray &src, -//! MGCoefficient *coeff, int rlev, int il, int iu, int jl, int ju, +//! const AthenaArray &coeff, int rlev, int il, int iu, int jl, int ju, //! int kl, int ku, int color, bool th) //! \brief Implementation of the Red-Black Gauss-Seidel Smoother //! rlev = relative level from the finest level of this Multigrid block void MGGravity::Smooth(AthenaArray &u, const AthenaArray &src, - MGCoefficient *coeff, int rlev, int il, int iu, int jl, int ju, - int kl, int ku, int color, bool th) { + const AthenaArray &coeff, int rlev, int il, int iu, + int jl, int ju, int kl, int ku, int color, bool th) { Real dx; if (rlev <= 0) dx = rdx_*static_cast(1<<(-rlev)); else dx = rdx_/static_cast(1< &u, const AthenaArray &src, //---------------------------------------------------------------------------------------- //! \fn void MGGravity::CalculateDefect(AthenaArray &def, //! const AthenaArray &u, const AthenaArray &src, -//! MGCoefficient *coeff, int rlev, int il, int iu, +//! const AthenaArray &coeff, int rlev, int il, int iu, //! int jl, int ju, int kl, int ku, bool th) //! \brief Implementation of the Defect calculation //! rlev = relative level from the finest level of this Multigrid block void MGGravity::CalculateDefect(AthenaArray &def, const AthenaArray &u, - const AthenaArray &src, MGCoefficient *coeff, int rlev, + const AthenaArray &src, const AthenaArray &coeff, int rlev, int il, int iu, int jl, int ju, int kl, int ku, bool th) { Real dx; if (rlev <= 0) dx = rdx_*static_cast(1<<(-rlev)); @@ -317,13 +317,13 @@ void MGGravity::CalculateDefect(AthenaArray &def, const AthenaArray //---------------------------------------------------------------------------------------- //! \fn void MGGravity::CalculateFASRHS(AthenaArray &src, -//! const AthenaArray &u, MGCoefficient *coeff, int rlev, -//! int il, int iu, int jl, int ju, int kl, int ku, bool th) +//! const AthenaArray &u, const AthenaArray &coeff, +//! int rlev, int il, int iu, int jl, int ju, int kl, int ku, bool th) //! \brief Implementation of the RHS calculation for FAS //! rlev = relative level from the finest level of this Multigrid block void MGGravity::CalculateFASRHS(AthenaArray &src, const AthenaArray &u, - MGCoefficient *coeff, int rlev, int il, int iu, + const AthenaArray &coeff, int rlev, int il, int iu, int jl, int ju, int kl, int ku, bool th) { Real dx; if (rlev <= 0) dx = rdx_*static_cast(1<<(-rlev)); diff --git a/src/gravity/mg_gravity.hpp b/src/gravity/mg_gravity.hpp index a41e9496f3..bb851be335 100644 --- a/src/gravity/mg_gravity.hpp +++ b/src/gravity/mg_gravity.hpp @@ -32,14 +32,14 @@ class MGGravity : public Multigrid { ~MGGravity(); void Smooth(AthenaArray &dst, const AthenaArray &src, - MGCoefficient *coeff, int rlev, int il, int iu, int jl, int ju, - int kl, int ku, int color, bool th) final; + const AthenaArray &coeff, int rlev, int il, int iu, + int jl, int ju, int kl, int ku, int color, bool th) final; void CalculateDefect(AthenaArray &def, const AthenaArray &u, - const AthenaArray &src, MGCoefficient *coeff, int rlev, - int il, int iu, int jl, int ju, int kl, int ku, bool th) final; + const AthenaArray &src, const AthenaArray &coeff, int rlev, + int il, int iu, int jl, int ju, int kl, int ku, bool th) final; void CalculateFASRHS(AthenaArray &def, const AthenaArray &src, - MGCoefficient *coeff, int rlev, int il, int iu, int jl, int ju, - int kl, int ku, bool th) final; + const AthenaArray &coeff, int rlev, int il, int iu, + int jl, int ju, int kl, int ku, bool th) final; private: static constexpr Real omega_ = 1.15; diff --git a/src/mesh/meshblock_tree.cpp b/src/mesh/meshblock_tree.cpp index f9dc8fc582..2317148a7f 100644 --- a/src/mesh/meshblock_tree.cpp +++ b/src/mesh/meshblock_tree.cpp @@ -505,7 +505,7 @@ void MeshBlockTree::CountMGOctets(int *noct) { //! std::unordered_map *octmap, int *noct) //! \brief construct lists of octets Multigrid with mesh refinement -void MeshBlockTree::GetMGOctetList(std::vector *oct, +void MeshBlockTree::GetMGOctetList(std::vector *oct, std::unordered_map *octmap, int *noct) { if (pleaf_ == nullptr) return; @@ -513,15 +513,15 @@ void MeshBlockTree::GetMGOctetList(std::vector *oct, int oid = 0; if (lev >= 0) { oid = noct[lev]; - oct[lev][oid]->loc = loc_; - oct[lev][oid]->fleaf = true; + oct[lev][oid].loc = loc_; + oct[lev][oid].fleaf = true; octmap[lev][loc_] = oid; noct[lev]++; } for (int n=0; npleaf_ != nullptr) { - if (lev >= 0) oct[lev][oid]->fleaf = false; + if (lev >= 0) oct[lev][oid].fleaf = false; pleaf_[n]->GetMGOctetList(oct, octmap, noct); } } diff --git a/src/mesh/meshblock_tree.hpp b/src/mesh/meshblock_tree.hpp index 4abe5a9b33..72f4fb18ae 100644 --- a/src/mesh/meshblock_tree.hpp +++ b/src/mesh/meshblock_tree.hpp @@ -56,7 +56,7 @@ class MeshBlockTree { MeshBlockTree* FindNeighbor(LogicalLocation myloc, int ox1, int ox2, int ox3, BoundaryFlag *bcs, bool amrflag=false); void CountMGOctets(int *noct); - void GetMGOctetList(std::vector *oct, + void GetMGOctetList(std::vector *oct, std::unordered_map *octmap, int *noct); private: diff --git a/src/multigrid/multigrid.cpp b/src/multigrid/multigrid.cpp index df89cc159e..3be51b39cf 100644 --- a/src/multigrid/multigrid.cpp +++ b/src/multigrid/multigrid.cpp @@ -26,11 +26,12 @@ #include "multigrid.hpp" //---------------------------------------------------------------------------------------- -//! \fn Multigrid::Multigrid(MultigridDriver *pmd, MeshBlock *pmb, int invar, int nghost) +//! \fn Multigrid::Multigrid(MultigridDriver *pmd, MeshBlock *pmb, int nghost) // \brief Multigrid constructor -Multigrid::Multigrid(MultigridDriver *pmd, MeshBlock *pmb, int invar, int nghost) : - pmy_driver_(pmd), pmy_block_(pmb), ngh_(nghost), nvar_(invar), defscale_(1.0) { +Multigrid::Multigrid(MultigridDriver *pmd, MeshBlock *pmb, int nghost) : + pmy_driver_(pmd), pmy_block_(pmb), ngh_(nghost), nvar_(pmd->nvar_), + ncoeff_(pmd->ncoeff_), defscale_(1.0) { if (pmy_block_ != nullptr) { loc_ = pmy_block_->loc; size_ = pmy_block_->block_size; @@ -120,7 +121,7 @@ Multigrid::Multigrid(MultigridDriver *pmd, MeshBlock *pmb, int invar, int nghost def_ = new AthenaArray[nlevel_]; coord_ = new MGCoordinates[nlevel_]; ccoord_ = new MGCoordinates[nlevel_]; - coeff_ = new MGCoefficient*[nlevel_]; + coeff_ = new AthenaArray[nlevel_]; if (pmy_block_ == nullptr) uold_ = new AthenaArray[nlevel_]; else @@ -137,7 +138,8 @@ Multigrid::Multigrid(MultigridDriver *pmd, MeshBlock *pmb, int invar, int nghost uold_[l].NewAthenaArray(nvar_,ncz,ncy,ncx); coord_[l].AllocateMGCoordinates(ncx,ncy,ncz); coord_[l].CalculateMGCoordinates(size_, ll, ngh_); - coeff_[l] = AllocateCoefficient(ncx,ncy,ncz); + if (ncoeff_ > 0) + coeff_[l].NewAthenaArray(ncoeff_,ncz,ncy,ncx); ncx=(size_.nx1>>(ll+1))+2*ngh_; ncy=(size_.nx2>>(ll+1))+2*ngh_; ncz=(size_.nx3>>(ll+1))+2*ngh_; @@ -156,6 +158,7 @@ Multigrid::~Multigrid() { delete [] src_; delete [] def_; delete [] uold_; + delete [] coeff_; delete [] coord_; delete [] ccoord_; } @@ -234,6 +237,33 @@ void Multigrid::LoadSource(const AthenaArray &src, int ns, int ngh, Real f } +//---------------------------------------------------------------------------------------- +//! \fn void Multigrid::LoadCoefficients(const AthenaArray &coeff, int ngh) +//! \brief Load coefficients of the diffusion and source terms + +void Multigrid::LoadCoefficients(const AthenaArray &coeff, int ngh) { + AthenaArray &cm=coeff_[nlevel_-1]; + int is, ie, js, je, ks, ke; + is=js=ks=ngh_; + ie=is+size_.nx1-1, je=js+size_.nx2-1, ke=ks+size_.nx3-1; + for (int v = 0; v < ncoeff_; ++v) { + for (int mk=ks; mk<=ke; ++mk) { + int k = mk - ks + ngh; + for (int mj=js; mj<=je; ++mj) { + int j = mj - js + ngh; +#pragma omp simd + for (int mi=is; mi<=ie; ++mi) { + int i = mi - is + ngh; + cm(v,mk,mj,mi) = coeff(v,k,j,i); + } + } + } + } + return; +} + + + //---------------------------------------------------------------------------------------- //! \fn void Multigrid::ApplySourceMask() // \brief Apply the user-defined source mask function on the finest level @@ -265,6 +295,23 @@ void Multigrid::RestrictFMGSource() { } +//---------------------------------------------------------------------------------------- +//! \fn void Multigrid::RestrictCoefficients() +//! \brief restrict coefficients within a Multigrid object + +void Multigrid::RestrictCoefficients() { + int is, ie, js, je, ks, ke; + is=js=ks=ngh_; + for (current_level_=nlevel_-1; current_level_>0; current_level_--) { + int ll=nlevel_-current_level_; + ie=is+(size_.nx1>>ll)-1, je=js+(size_.nx2>>ll)-1, ke=ks+(size_.nx3>>ll)-1; + Restrict(coeff_[current_level_-1], coeff_[current_level_], + ncoeff_, is, ie, js, je, ks, ke, false); + } + return; +} + + //---------------------------------------------------------------------------------------- //! \fn void Multigrid::RetrieveResult(AthenaArray &dst, int ns, int ngh) // \brief Set the result, including the ghost zone @@ -524,7 +571,7 @@ void Multigrid::SetFromRootGrid(bool folddata) { int ci = (static_cast(loc_.lx1)&1); int cj = (static_cast(loc_.lx2)&1); int ck = (static_cast(loc_.lx3)&1); - const AthenaArray &src = pmy_driver_->octets_[olev][oid]->u; + const AthenaArray &src = pmy_driver_->octets_[olev][oid].u; for (int v=0; v &odst = uold_[current_level_]; - const AthenaArray &osrc = pmy_driver_->octets_[olev][oid]->uold; + const AthenaArray &osrc = pmy_driver_->octets_[olev][oid].uold; for (int v=0; v(1< u, def, src, uold; + AthenaArray u, def, src, uold, coeff; MGCoordinates coord, ccoord; - MGCoefficient* coeff; }; @@ -97,7 +91,7 @@ class MGOctet { class Multigrid { public: - Multigrid(MultigridDriver *pmd, MeshBlock *pmb, int invar, int nghost); + Multigrid(MultigridDriver *pmd, MeshBlock *pmb, int nghost); virtual ~Multigrid(); MGBoundaryValues *pmgbval; @@ -109,12 +103,14 @@ class Multigrid { void LoadFinestData(const AthenaArray &src, int ns, int ngh); void LoadSource(const AthenaArray &src, int ns, int ngh, Real fac); + void LoadCoefficients(const AthenaArray &coeff, int ngh); void ApplySourceMask(); void RestrictFMGSource(); void RetrieveResult(AthenaArray &dst, int ns, int ngh); void RetrieveDefect(AthenaArray &dst, int ns, int ngh); void ZeroClearData(); void RestrictBlock(); + void RestrictCoefficients(); void ProlongateAndCorrectBlock(); void FMGProlongateBlock(); void SmoothBlock(int color); @@ -148,15 +144,14 @@ class Multigrid { int il, int iu, int jl, int ju, int kl, int ku, int fil, int fjl, int fkl, bool th); // physics-dependent virtual functions - virtual MGCoefficient* AllocateCoefficient(int ncx, int ncy, int ncz) { return nullptr;}; virtual void Smooth(AthenaArray &dst, const AthenaArray &src, - MGCoefficient *coeff, int rlev, int il, int iu, int jl, int ju, - int kl, int ku, int color, bool th) = 0; + const AthenaArray &coeff, int rlev, int il, int iu, + int jl, int ju, int kl, int ku, int color, bool th) = 0; virtual void CalculateDefect(AthenaArray &def, const AthenaArray &u, - const AthenaArray &src, MGCoefficient *coeff, int rlev, - int il, int iu, int jl, int ju, int kl, int ku, bool th) = 0; + const AthenaArray &src, const AthenaArray &coeff, int rlev, + int il, int iu, int jl, int ju, int kl, int ku, bool th) = 0; virtual void CalculateFASRHS(AthenaArray &def, const AthenaArray &src, - MGCoefficient *coeff, int rlev, int il, int iu, + const AthenaArray &coeff, int rlev, int il, int iu, int jl, int ju, int kl, int ku, bool th) = 0; friend class MultigridDriver; @@ -172,12 +167,11 @@ class Multigrid { LogicalLocation loc_; RegionSize size_; BoundaryFlag mg_block_bcs_[6]; - int nlevel_, ngh_, nvar_, current_level_; + int nlevel_, ngh_, nvar_, ncoeff_, current_level_; Real rdx_, rdy_, rdz_; Real defscale_; - AthenaArray *u_, *def_, *src_, *uold_; + AthenaArray *u_, *def_, *src_, *uold_, *coeff_; MGCoordinates *coord_, *ccoord_; - MGCoefficient **coeff_; private: TaskStates ts_; @@ -190,7 +184,7 @@ class Multigrid { class MultigridDriver { public: MultigridDriver(Mesh *pm, MGBoundaryFunc *MGBoundary, - MGSourceMaskFunc MGSourceMask, int invar); + MGSourceMaskFunc MGSourceMask, int invar, int ncoeff); virtual ~MultigridDriver(); // pure virtual function @@ -204,7 +198,6 @@ class MultigridDriver { friend class MGCRDiffusion; protected: - virtual MGOctet* AllocateOctet(); void CheckBoundaryFunctions(); void SubtractAverage(MGVariable type); void SetupMultigrid(); @@ -226,6 +219,7 @@ class MultigridDriver { void CalculateOctetCoordinates(); void RestrictFMGSourceOctets(); void RestrictOctets(); + void RestrictCoefficients(); void ZeroClearOctets(); void StoreOldDataOctets(); void CalculateFASRHSOctets(); @@ -260,7 +254,7 @@ class MultigridDriver { // small functions int GetNumMultigrids() { return nblist_[Globals::my_rank]; } - int nranks_, nthreads_, nbtotal_, nvar_, mode_; + int nranks_, nthreads_, nbtotal_, nvar_, ncoeff_, mode_; int locrootlevel_, nrootlevel_, nmblevel_, ntotallevel_, nreflevel_, maxreflevel_; int current_level_, fmglevel_; int *nslist_, *nblist_, *nvlist_, *nvslist_, *nvlisti_, *nvslisti_, *ranklist_; @@ -279,7 +273,7 @@ class MultigridDriver { int coffset_; // for mesh refinement - std::vector *octets_; + std::vector *octets_; std::unordered_map *octetmap_; std::vector *octetbflag_; int *noctets_, *pmaxnoct_; diff --git a/src/multigrid/multigrid_driver.cpp b/src/multigrid/multigrid_driver.cpp index dc50315d26..ce485c6866 100644 --- a/src/multigrid/multigrid_driver.cpp +++ b/src/multigrid/multigrid_driver.cpp @@ -38,9 +38,9 @@ // constructor, initializes data structures and parameters MultigridDriver::MultigridDriver(Mesh *pm, MGBoundaryFunc *MGBoundary, - MGSourceMaskFunc MGSourceMask, int invar) : + MGSourceMaskFunc MGSourceMask, int invar, int ncoeff) : nranks_(Globals::nranks), nthreads_(pm->num_mesh_threads_), nbtotal_(pm->nbtotal), - nvar_(invar), mode_(0), // 0: FMG V(1,1) + iterative, 1: V(1,1) iterative + nvar_(invar), ncoeff_(ncoeff), mode_(0), // 0: FMG+V-cycle, 1: V-cycle maxreflevel_(pm->multilevel?pm->max_level-pm->root_level:0), nrbx1_(pm->nrbx1), nrbx2_(pm->nrbx2), nrbx3_(pm->nrbx3), srcmask_(MGSourceMask), pmy_mesh_(pm), fsubtract_average_(false), ffas_(pm->multilevel), @@ -85,7 +85,7 @@ MultigridDriver::MultigridDriver(Mesh *pm, MGBoundaryFunc *MGBoundary, #endif if (maxreflevel_ > 0) { // SMR / AMR - octets_ = new std::vector[maxreflevel_]; + octets_ = new std::vector[maxreflevel_]; octetmap_ = new std::unordered_map[maxreflevel_]; octetbflag_ = new std::vector[maxreflevel_]; @@ -119,10 +119,6 @@ MultigridDriver::~MultigridDriver() { delete [] nvslisti_; delete [] rootbuf_; if (maxreflevel_ > 0) { - for (int l = 0; l < nreflevel_; ++l) { - for (int o = pmaxnoct_[l]; o < noctets_[l]; ++o) - delete octets_[l][o]; - } delete [] octets_; delete [] octetmap_; delete [] octetbflag_; @@ -141,13 +137,22 @@ MultigridDriver::~MultigridDriver() { //---------------------------------------------------------------------------------------- -//! \fn MGOctet* MultigridDriver::AllocateOctet() -// \brief Allocate a MGOctet object -MGOctet* MultigridDriver::AllocateOctet() { - int ncoct = mgroot_->ngh_*2 + 2, nccoct = mgroot_->ngh_*2 + 1; - return new MGOctet(nvar_, ncoct, nccoct); +//! \fn void MGOctet::Allocate(int nvar, int ncoct, int nccoct, int ncoeff) +// \brief allocate arrays and coordinates in MGOctet + +void MGOctet::Allocate(int nvar, int ncoct, int nccoct, int ncoeff) { + u.NewAthenaArray(nvar, ncoct, ncoct, ncoct); + uold.NewAthenaArray(nvar, ncoct, ncoct, ncoct); + def.NewAthenaArray(nvar, ncoct, ncoct, ncoct); + src.NewAthenaArray(nvar, ncoct, ncoct, ncoct); + if (ncoeff > 0) + coeff.NewAthenaArray(ncoeff, ncoct, ncoct, ncoct); + coord.AllocateMGCoordinates(ncoct, ncoct, ncoct); + ccoord.AllocateMGCoordinates(nccoct, nccoct, nccoct); + return; } + //---------------------------------------------------------------------------------------- //! \fn void MultigridDriver::CheckBoundaryFunctions() // \brief check boundary functions and set some internal flags. @@ -377,6 +382,7 @@ void MultigridDriver::SetupMultigrid() { fmglevel_ = current_level_ = ntotallevel_ - 1; os_ = mgroot_->ngh_; oe_ = os_+1; + const int ncoct = 2 + 2*mgroot_->ngh_, nccoct = 1 + 2*mgroot_->ngh_; if (pmy_mesh_->amr_updated) needinit_ = true; @@ -396,7 +402,7 @@ void MultigridDriver::SetupMultigrid() { octetbflag_[l].resize(noctets_[l]); } for (int o = pmaxnoct_[l]; o < noctets_[l]; ++o) - octets_[l][o] = AllocateOctet(); + octets_[l][o].Allocate(nvar_, ncoct, nccoct, ncoeff_); noctets_[l] = 0; } pmy_mesh_->tree.GetMGOctetList(octets_, octetmap_, noctets_); @@ -462,6 +468,9 @@ void MultigridDriver::SetupMultigrid() { CalculateMultipoleCoefficients(); } + if (ncoeff_ > 0) + RestrictCoefficients(); + if (mode_ == 0) { // FMG #pragma omp parallel for num_threads(nthreads_) for (auto itr = vmg_.begin(); itr < vmg_.end(); itr++) { @@ -534,11 +543,12 @@ void MultigridDriver::TransferFromBlocksToRoot(bool initflag) { int oi = (i&1) + ngh; int oj = (j&1) + ngh; int ok = (k&1) + ngh; + MGOctet &oct = octets_[olev][oid]; for (int v = 0; v < nvar_; ++v) - octets_[olev][oid]->src(v,ok,oj,oi) = rootbuf_[n*nv+v]; + oct.src(v,ok,oj,oi) = rootbuf_[n*nv+v]; if (ffas_ && !initflag) { for (int v = 0; v < nvar_; ++v) - octets_[olev][oid]->u(v,ok,oj,oi) = rootbuf_[n*nv+nvar_+v]; + oct.u(v,ok,oj,oi) = rootbuf_[n*nv+nvar_+v]; } } } @@ -911,8 +921,9 @@ void MultigridDriver::CalculateOctetCoordinates() { csize.nx1 = 1, csize.nx2 = 1, csize.nx3 = 1; #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[0]; ++o) { + MGOctet &oct = octets_[0][o]; MGCoordinates &coord = mgroot_->coord_[mgroot_->nlevel_-1]; - LogicalLocation &loc = octets_[0][o]->loc; + LogicalLocation &loc = oct.loc; int i = static_cast(loc.lx1) + ngh; int j = static_cast(loc.lx2) + ngh; int k = static_cast(loc.lx3) + ngh; @@ -922,20 +933,21 @@ void MultigridDriver::CalculateOctetCoordinates() { size.x2max = csize.x2max = coord.x2f(j+1); size.x3min = csize.x3min = coord.x3f(k); size.x3max = csize.x3max = coord.x3f(k+1); - octets_[0][o]->coord.CalculateMGCoordinates(size, 0, ngh); - octets_[0][o]->ccoord.CalculateMGCoordinates(csize, 0, ngh); + oct.coord.CalculateMGCoordinates(size, 0, ngh); + oct.ccoord.CalculateMGCoordinates(csize, 0, ngh); } for (int l = 1; l < nreflevel_; l++) { #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[l]; ++o) { - LogicalLocation &loc = octets_[l][o]->loc; + MGOctet &oct = octets_[l][o]; + LogicalLocation &loc = oct.loc; LogicalLocation cloc; cloc.lx1 = (loc.lx1 >> 1); cloc.lx2 = (loc.lx2 >> 1); cloc.lx3 = (loc.lx3 >> 1); cloc.level = loc.level - 1; int oid = octetmap_[l-1][cloc]; - MGCoordinates &coord = octets_[l-1][oid]->coord; + MGCoordinates &coord = octets_[l-1][oid].coord; int i = static_cast(loc.lx1&1) + ngh; int j = static_cast(loc.lx2&1) + ngh; int k = static_cast(loc.lx3&1) + ngh; @@ -945,8 +957,8 @@ void MultigridDriver::CalculateOctetCoordinates() { size.x2max = csize.x2max = coord.x2f(j+1); size.x3min = csize.x3min = coord.x3f(k); size.x3max = csize.x3max = coord.x3f(k+1); - octets_[l][o]->coord.CalculateMGCoordinates(size, 0, ngh); - octets_[l][o]->ccoord.CalculateMGCoordinates(csize, 0, ngh); + oct.coord.CalculateMGCoordinates(size, 0, ngh); + oct.ccoord.CalculateMGCoordinates(csize, 0, ngh); } } } @@ -962,7 +974,8 @@ void MultigridDriver::RestrictFMGSourceOctets() { for (int l = nreflevel_ - 1; l >= 1; --l) { // fine octets to coarse octets #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[l]; ++o) { - const LogicalLocation &loc = octets_[l][o]->loc; + MGOctet &foct = octets_[l][o]; + const LogicalLocation &loc = foct.loc; LogicalLocation cloc; cloc.lx1 = (loc.lx1 >> 1); cloc.lx2 = (loc.lx2 >> 1); @@ -972,18 +985,19 @@ void MultigridDriver::RestrictFMGSourceOctets() { int oi = (static_cast(loc.lx1) & 1) + ngh; int oj = (static_cast(loc.lx2) & 1) + ngh; int ok = (static_cast(loc.lx3) & 1) + ngh; + MGOctet &coct = octets_[l-1][oid]; for (int v = 0; v < nvar_; ++v) - octets_[l-1][oid]->src(v, ok, oj, oi) = RestrictOne(octets_[l][o]->src, - v, ngh, ngh, ngh); + coct.src(v, ok, oj, oi) = RestrictOne(foct.src, v, ngh, ngh, ngh); } } #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[0]; ++o) { // octets to the root grid - const LogicalLocation &loc = octets_[0][o]->loc; + MGOctet &oct = octets_[0][o]; + const LogicalLocation &loc = oct.loc; for (int v = 0; v < nvar_; ++v) mgroot_->SetData(MGVariable::src, v, static_cast(loc.lx3), static_cast(loc.lx2), static_cast(loc.lx1), - RestrictOne(octets_[0][o]->src, v, ngh, ngh, ngh)); + RestrictOne(oct.src, v, ngh, ngh, ngh)); } } @@ -1002,8 +1016,8 @@ void MultigridDriver::RestrictOctets() { if (lev >= 1) { // fine octets to coarse octets #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[lev]; ++o) { - MGOctet *foct = octets_[lev][o]; - const LogicalLocation &loc = foct->loc; + MGOctet &foct = octets_[lev][o]; + const LogicalLocation &loc = foct.loc; LogicalLocation cloc; cloc.lx1 = (loc.lx1 >> 1); cloc.lx2 = (loc.lx2 >> 1); @@ -1013,33 +1027,33 @@ void MultigridDriver::RestrictOctets() { int oi = (static_cast(loc.lx1) & 1) + ngh; int oj = (static_cast(loc.lx2) & 1) + ngh; int ok = (static_cast(loc.lx3) & 1) + ngh; - MGOctet *coct = octets_[lev-1][oid]; - mgroot_->CalculateDefect(foct->def, foct->u, foct->src, foct->coeff, + MGOctet &coct = octets_[lev-1][oid]; + mgroot_->CalculateDefect(foct.def, foct.u, foct.src, foct.coeff, lev+1, os_, oe_, os_, oe_, os_, oe_, false); for (int v = 0; v < nvar_; ++v) - coct->src(v, ok, oj, oi) = RestrictOne(foct->def, v, ngh, ngh, ngh); + coct.src(v, ok, oj, oi) = RestrictOne(foct.def, v, ngh, ngh, ngh); if (ffas_) { for (int v = 0; v < nvar_; ++v) - coct->u(v, ok, oj, oi) = RestrictOne(foct->u, v, ngh, ngh, ngh); + coct.u(v, ok, oj, oi) = RestrictOne(foct.u, v, ngh, ngh, ngh); } } } else { // octets to the root grid #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[0]; ++o) { - MGOctet *oct = octets_[0][o]; - const LogicalLocation &loc = oct->loc; + MGOctet &oct = octets_[0][o]; + const LogicalLocation &loc = oct.loc; int ri = static_cast(loc.lx1); int rj = static_cast(loc.lx2); int rk = static_cast(loc.lx3); - mgroot_->CalculateDefect(oct->def, oct->u, oct->src, oct->coeff, + mgroot_->CalculateDefect(oct.def, oct.u, oct.src, oct.coeff, 1, os_, oe_, os_, oe_, os_, oe_, false); for (int v = 0; v < nvar_; ++v) mgroot_->SetData(MGVariable::src, v, rk, rj, ri, - RestrictOne(oct->def, v, ngh, ngh, ngh)); + RestrictOne(oct.def, v, ngh, ngh, ngh)); if (ffas_) { for (int v = 0; v < nvar_; ++v) mgroot_->SetData(MGVariable::u, v, rk, rj, ri, - RestrictOne(oct->u, v, ngh, ngh, ngh)); + RestrictOne(oct.u, v, ngh, ngh, ngh)); } } } @@ -1048,6 +1062,55 @@ void MultigridDriver::RestrictOctets() { } +//---------------------------------------------------------------------------------------- +//! \fn void MultigridDiffusionDriver::RestrictCoefficients() +//! \brief restrict coefficients in Octets + +void MultigridDriver::RestrictCoefficients() { +#pragma omp parallel for num_threads(nthreads_) + for (auto itr = vmg_.begin(); itr < vmg_.end(); itr++) { + Multigrid *pmg = *itr; + pmg->RestrictCoefficients(); + } + if (nreflevel_ > 0) { + const int &ngh = mgroot_->ngh_; + for (int l = nreflevel_ - 1; l >= 1; --l) { // fine octets to coarse octets +#pragma omp parallel for num_threads(nthreads_) + for (int o = 0; o < noctets_[l]; ++o) { + MGOctet &foct = octets_[l][o]; + const LogicalLocation &loc = foct.loc; + LogicalLocation cloc; + cloc.lx1 = (loc.lx1 >> 1); + cloc.lx2 = (loc.lx2 >> 1); + cloc.lx3 = (loc.lx3 >> 1); + cloc.level = loc.level - 1; + int oid = octetmap_[l-1][cloc]; + int oi = (static_cast(loc.lx1) & 1) + ngh; + int oj = (static_cast(loc.lx2) & 1) + ngh; + int ok = (static_cast(loc.lx3) & 1) + ngh; + MGOctet &coct = octets_[l-1][oid]; + for (int v = 0; v < ncoeff_; ++v) + coct.coeff(v, ok, oj, oi) = RestrictOne(foct.coeff, v, ngh, ngh, ngh); + } + } +#pragma omp parallel for num_threads(nthreads_) + for (int o = 0; o < noctets_[0]; ++o) { // octets to the root grid + MGOctet &oct = octets_[0][o]; + const LogicalLocation &loc = oct.loc; + int lx1 = static_cast(loc.lx1) + mgroot_->ngh_; + int lx2 = static_cast(loc.lx2) + mgroot_->ngh_; + int lx3 = static_cast(loc.lx3) + mgroot_->ngh_; + for (int v = 0; v < ncoeff_; ++v) + mgroot_->coeff_[mgroot_->nlevel_-1](v, lx3, lx2, lx1) + = RestrictOne(oct.coeff, v, ngh, ngh, ngh); + } + } + mgroot_->RestrictCoefficients(); + + return; +} + + //---------------------------------------------------------------------------------------- //! \fn void MultigridDriver::ZeroClearOctets() // \brief zero clear the data in all the octets @@ -1057,7 +1120,7 @@ void MultigridDriver::ZeroClearOctets() { for (int l = 0; l <= maxlevel; l++) { #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[l]; ++o) - octets_[l][o]->u.ZeroClear(); + octets_[l][o].u.ZeroClear(); } return; @@ -1073,8 +1136,8 @@ void MultigridDriver::StoreOldDataOctets() { #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[lev]; ++o) { - MGOctet *oct = octets_[lev][o]; - memcpy(oct->uold.data(), oct->u.data(), oct->u.GetSizeInBytes()); + MGOctet &oct = octets_[lev][o]; + memcpy(oct.uold.data(), oct.u.data(), oct.u.GetSizeInBytes()); } return; @@ -1089,8 +1152,8 @@ void MultigridDriver::CalculateFASRHSOctets() { #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[lev]; ++o) { - MGOctet *oct = octets_[lev][o]; - mgroot_->CalculateFASRHS(oct->src, oct->u, octets_[lev][o]->coeff, + MGOctet &oct = octets_[lev][o]; + mgroot_->CalculateFASRHS(oct.src, oct.u, oct.coeff, lev+1, os_, oe_, os_, oe_, os_, oe_, false); } return; @@ -1105,8 +1168,8 @@ void MultigridDriver::SmoothOctets(int color) { #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[lev]; ++o) { - MGOctet *oct = octets_[lev][o]; - mgroot_->Smooth(oct->u, oct->src, oct->coeff, + MGOctet &oct = octets_[lev][o]; + mgroot_->Smooth(oct.u, oct.src, oct.coeff, lev+1, os_, oe_, os_, oe_, os_, oe_, color, false); } return; @@ -1133,7 +1196,8 @@ void MultigridDriver::ProlongateAndCorrectOctets() { th = omp_get_thread_num(); #endif AthenaArray &cbuf = cbuf_[th]; - const LogicalLocation &loc = octets_[0][o]->loc; + MGOctet & oct = octets_[0][o]; + const LogicalLocation &loc = oct.loc; int ri = static_cast(loc.lx1) + ngh - 1; int rj = static_cast(loc.lx2) + ngh - 1; int rk = static_cast(loc.lx3) + ngh - 1; @@ -1147,10 +1211,10 @@ void MultigridDriver::ProlongateAndCorrectOctets() { } } } - mgroot_->ProlongateAndCorrect(octets_[0][o]->u, cbuf, + mgroot_->ProlongateAndCorrect(oct.u, cbuf, ngh, ngh, ngh, ngh, ngh, ngh, ngh, ngh, ngh, false); } else { - mgroot_->ProlongateAndCorrect(octets_[0][o]->u, u, + mgroot_->ProlongateAndCorrect(oct.u, u, ri+1, ri+1, rj+1, rj+1, rk+1, rk+1, ngh, ngh, ngh, false); } } @@ -1162,7 +1226,8 @@ void MultigridDriver::ProlongateAndCorrectOctets() { th = omp_get_thread_num(); #endif AthenaArray &cbuf = cbuf_[th]; - const LogicalLocation &loc = octets_[flev][o]->loc; + MGOctet &foct = octets_[flev][o]; + const LogicalLocation &loc = foct.loc; LogicalLocation cloc; cloc.lx1 = (loc.lx1 >> 1); cloc.lx2 = (loc.lx2 >> 1); @@ -1172,8 +1237,9 @@ void MultigridDriver::ProlongateAndCorrectOctets() { int ci = (static_cast(loc.lx1) & 1) + ngh - 1; int cj = (static_cast(loc.lx2) & 1) + ngh - 1; int ck = (static_cast(loc.lx3) & 1) + ngh - 1; - const AthenaArray &uc = octets_[clev][cid]->u; - const AthenaArray &ucold = octets_[clev][cid]->uold; + MGOctet &coct = octets_[clev][cid]; + const AthenaArray &uc = coct.u; + const AthenaArray &ucold = coct.uold; if (ffas_) { for (int v = 0; v < nvar_; ++v) { for (int k = 0; k <= 2; ++k) { @@ -1183,10 +1249,10 @@ void MultigridDriver::ProlongateAndCorrectOctets() { } } } - mgroot_->ProlongateAndCorrect(octets_[flev][o]->u, cbuf, + mgroot_->ProlongateAndCorrect(foct.u, cbuf, ngh, ngh, ngh, ngh, ngh, ngh, ngh, ngh, ngh, false); } else { - mgroot_->ProlongateAndCorrect(octets_[flev][o]->u, uc, + mgroot_->ProlongateAndCorrect(foct.u, uc, ci+1, ci+1, cj+1, cj+1, ck+1, ck+1, ngh, ngh, ngh, false); } } @@ -1208,17 +1274,19 @@ void MultigridDriver::FMGProlongateOctets() { if (flev == 0) { // from root to octets #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[0]; ++o) { - const LogicalLocation &loc = octets_[0][o]->loc; + MGOctet &oct = octets_[0][o]; + const LogicalLocation &loc = oct.loc; int ri = static_cast(loc.lx1) + ngh; int rj = static_cast(loc.lx2) + ngh; int rk = static_cast(loc.lx3) + ngh; - mgroot_->FMGProlongate(octets_[0][o]->u, mgroot_->GetCurrentData(), + mgroot_->FMGProlongate(oct.u, mgroot_->GetCurrentData(), ri, ri, rj, rj, rk, rk, ngh, ngh, ngh, false); } } else { // from coarse octets to fine octets #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[flev]; ++o) { - const LogicalLocation &loc = octets_[flev][o]->loc; + MGOctet &foct = octets_[flev][o]; + const LogicalLocation &loc = foct.loc; LogicalLocation cloc; cloc.lx1 = (loc.lx1 >> 1); cloc.lx2 = (loc.lx2 >> 1); @@ -1228,7 +1296,8 @@ void MultigridDriver::FMGProlongateOctets() { int ci = (static_cast(loc.lx1) & 1) + ngh; int cj = (static_cast(loc.lx2) & 1) + ngh; int ck = (static_cast(loc.lx3) & 1) + ngh; - mgroot_->FMGProlongate(octets_[flev][o]->u, octets_[clev][cid]->u, + MGOctet &coct = octets_[clev][cid]; + mgroot_->FMGProlongate(foct.u, coct.u, ci, ci, cj, cj, ck, ck, ngh, ngh, ngh, false); } } @@ -1246,7 +1315,8 @@ void MultigridDriver::SetBoundariesOctets(bool fprolong, bool folddata) { #pragma omp parallel for num_threads(nthreads_) schedule(dynamic,1) for (int o = 0; o < noctets_[lev]; ++o) { - if (fprolong && octets_[lev][o]->fleaf == true) continue; + MGOctet &oct = octets_[lev][o]; + if (fprolong && oct.fleaf == true) continue; int th = 0; #ifdef OPENMP_PARALLEL th = omp_get_thread_num(); @@ -1255,9 +1325,7 @@ void MultigridDriver::SetBoundariesOctets(bool fprolong, bool folddata) { AthenaArray &cbufold = cbufold_[th]; AthenaArray &ncoarse = ncoarse_[th]; ncoarse.ZeroClear(); - const LogicalLocation &loc = octets_[lev][o]->loc; - AthenaArray &u = octets_[lev][o]->u; - AthenaArray &uold = octets_[lev][o]->uold; + const LogicalLocation &loc = oct.loc; LogicalLocation nloc = loc; for (int ox3 = -1; ox3 <= 1; ++ox3) { nloc.lx3 = loc.lx3 + ox3; @@ -1306,9 +1374,8 @@ void MultigridDriver::SetBoundariesOctets(bool fprolong, bool folddata) { } if (octetmap_[lev].count(nloc) == 1) { // on the same level int nid = octetmap_[lev][nloc]; - const AthenaArray &un = octets_[lev][nid]->u; - const AthenaArray &unold = octets_[lev][nid]->uold; - SetOctetBoundarySameLevel(u, un, uold, unold, cbuf, cbufold, + MGOctet &noct = octets_[lev][nid]; + SetOctetBoundarySameLevel(oct.u, noct.u, oct.uold, noct.uold, cbuf, cbufold, ox1, ox2, ox3, folddata); } else if (!fprolong) { // on the coarser level // note: prolongation requires neighbors on the same level only @@ -1320,9 +1387,8 @@ void MultigridDriver::SetBoundariesOctets(bool fprolong, bool folddata) { cloc.lx3 = nloc.lx3 >> 1; cloc.level = nloc.level - 1; int cid = octetmap_[lev-1][cloc]; - const AthenaArray &un = octets_[lev-1][cid]->u; - const AthenaArray &unold = octets_[lev-1][cid]->uold; - SetOctetBoundaryFromCoarser(un, unold, cbuf, cbufold, + MGOctet &coct = octets_[lev-1][cid]; + SetOctetBoundaryFromCoarser(coct.u, coct.uold, cbuf, cbufold, loc, ox1, ox2, ox3, false); } else { // from root const AthenaArray &un = mgroot_->GetCurrentData(); @@ -1336,10 +1402,10 @@ void MultigridDriver::SetBoundariesOctets(bool fprolong, bool folddata) { } } if (!fprolong) { - ApplyPhysicalBoundariesOctet(cbuf, loc, octets_[lev][o]->ccoord, true); - ProlongateOctetBoundariesFluxCons(u, cbuf, ncoarse); + ApplyPhysicalBoundariesOctet(cbuf, loc, oct.ccoord, true); + ProlongateOctetBoundariesFluxCons(oct.u, cbuf, ncoarse); } - ApplyPhysicalBoundariesOctet(u, loc, octets_[lev][o]->coord, false); + ApplyPhysicalBoundariesOctet(oct.u, loc, oct.coord, false); } return; @@ -1607,28 +1673,30 @@ void MultigridDriver::RestrictOctetsBeforeTransfer() { for (int l = nreflevel_ - 1; l >= 1; --l) { // fine octets to coarse octets #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[l]; ++o) { - const LogicalLocation &loc = octets_[l][o]->loc; + MGOctet &foct = octets_[l][o]; + const LogicalLocation &loc = foct.loc; LogicalLocation cloc; cloc.lx1 = (loc.lx1 >> 1); cloc.lx2 = (loc.lx2 >> 1); cloc.lx3 = (loc.lx3 >> 1); cloc.level = loc.level - 1; int oid = octetmap_[l-1][cloc]; + MGOctet &coct = octets_[l-1][oid]; int oi = (static_cast(loc.lx1) & 1) + ngh; int oj = (static_cast(loc.lx2) & 1) + ngh; int ok = (static_cast(loc.lx3) & 1) + ngh; for (int v = 0; v < nvar_; ++v) - octets_[l-1][oid]->u(v, ok, oj, oi) = RestrictOne(octets_[l][o]->u, - v, ngh, ngh, ngh); + coct.u(v, ok, oj, oi) = RestrictOne(foct.u, v, ngh, ngh, ngh); } } #pragma omp parallel for num_threads(nthreads_) for (int o = 0; o < noctets_[0]; ++o) { // octets to the root grid - const LogicalLocation &loc = octets_[0][o]->loc; + MGOctet &oct = octets_[0][o]; + const LogicalLocation &loc = oct.loc; for (int v = 0; v < nvar_; ++v) mgroot_->SetData(MGVariable::u, v, static_cast(loc.lx3), static_cast(loc.lx2), static_cast(loc.lx1), - RestrictOne(octets_[0][o]->u, v, ngh, ngh, ngh)); + RestrictOne(oct.u, v, ngh, ngh, ngh)); } return; @@ -1667,8 +1735,7 @@ void MultigridDriver::SetOctetBoundariesBeforeTransfer(bool folddata) { int oid = octetmap_[lev][loc]; if (octetbflag_[lev][oid] == true) continue; octetbflag_[lev][oid] = true; - AthenaArray &u = octets_[lev][oid]->u; - AthenaArray &uold = octets_[lev][oid]->uold; + MGOctet &oct = octets_[lev][oid]; LogicalLocation nloc = loc; for (int ox3 = -1; ox3 <= 1; ++ox3) { nloc.lx3 = loc.lx3 + ox3; @@ -1715,9 +1782,8 @@ void MultigridDriver::SetOctetBoundariesBeforeTransfer(bool folddata) { } if (octetmap_[lev].count(nloc) == 1) { // same or finer int nid = octetmap_[lev][nloc]; - const AthenaArray &un = octets_[lev][nid]->u; - const AthenaArray &unold = octets_[lev][nid]->uold; - SetOctetBoundarySameLevel(u, un, uold, unold, cbuf, cbufold, + MGOctet &noct = octets_[lev][nid]; + SetOctetBoundarySameLevel(oct.u, noct.u, oct.uold, noct.uold, cbuf, cbufold, ox1, ox2, ox3, folddata); } else { // coarser ncoarse(ox3+1, ox2+1, ox1+1) = true; @@ -1728,9 +1794,8 @@ void MultigridDriver::SetOctetBoundariesBeforeTransfer(bool folddata) { cloc.lx3 = nloc.lx3 >> 1; cloc.level = nloc.level - 1; int cid = octetmap_[lev-1][cloc]; - const AthenaArray &un = octets_[lev-1][cid]->u; - const AthenaArray &unold = octets_[lev-1][cid]->uold; - SetOctetBoundaryFromCoarser(un, unold, cbuf, cbufold, + MGOctet & coct = octets_[lev-1][cid]; + SetOctetBoundaryFromCoarser(coct.u, coct.uold, cbuf, cbufold, loc, ox1, ox2, ox3, folddata); } else { // from root const AthenaArray &un = mgroot_->GetCurrentData(); @@ -1743,11 +1808,11 @@ void MultigridDriver::SetOctetBoundariesBeforeTransfer(bool folddata) { } } - ApplyPhysicalBoundariesOctet(cbuf, loc, octets_[lev][oid]->ccoord, true); + ApplyPhysicalBoundariesOctet(cbuf, loc, oct.ccoord, true); if (folddata) - ApplyPhysicalBoundariesOctet(cbufold, loc, octets_[lev][oid]->ccoord, true); - ProlongateOctetBoundaries(u, uold, cbuf, cbufold, ncoarse, folddata); - ApplyPhysicalBoundariesOctet(u, loc, octets_[lev][oid]->coord, false); + ApplyPhysicalBoundariesOctet(cbufold, loc, oct.ccoord, true); + ProlongateOctetBoundaries(oct.u, oct.uold, cbuf, cbufold, ncoarse, folddata); + ApplyPhysicalBoundariesOctet(oct.u, loc, oct.coord, false); } return; } @@ -2027,13 +2092,3 @@ void MultigridDriver::CalculateCenterOfMass() { return; } - -//---------------------------------------------------------------------------------------- -//! \fn MGOctet::MGOctet(int nvar, int ncoct, int nccoct) -// \brief MGOctet construcor - allocate arrays and coordinates -MGOctet::MGOctet(int nvar, int ncoct, int nccoct) : - u(nvar, ncoct, ncoct, ncoct), uold(nvar, ncoct, ncoct, ncoct), - def(nvar, ncoct, ncoct, ncoct), src(nvar, ncoct, ncoct, ncoct), coeff(nullptr) { - coord.AllocateMGCoordinates(ncoct, ncoct, ncoct); - ccoord.AllocateMGCoordinates(nccoct, nccoct, nccoct); -} diff --git a/src/task_list/crdiffusion_task_list.cpp b/src/task_list/crdiffusion_task_list.cpp index fba4d75e68..009ca9dd9f 100644 --- a/src/task_list/crdiffusion_task_list.cpp +++ b/src/task_list/crdiffusion_task_list.cpp @@ -120,6 +120,6 @@ TaskStatus CRDiffusionBoundaryTaskList::ProlongateCRDiffusionBoundary(MeshBlock } TaskStatus CRDiffusionBoundaryTaskList::PhysicalBoundary(MeshBlock *pmb, int stage) { - pmb->pcrdiff->ExpandPhysicalBoundaries(); + pmb->pcrdiff->crbvar.ExpandPhysicalBoundaries(pmb->pcrdiff->ecr, 1); return TaskStatus::next; } diff --git a/src/task_list/grav_task_list.cpp b/src/task_list/grav_task_list.cpp index 32786866da..9a7c5897b3 100644 --- a/src/task_list/grav_task_list.cpp +++ b/src/task_list/grav_task_list.cpp @@ -124,7 +124,7 @@ TaskStatus GravityBoundaryTaskList::ProlongateGravityBoundary(MeshBlock *pmb, TaskStatus GravityBoundaryTaskList::PhysicalBoundary(MeshBlock *pmb, int stage) { if (pmb->pgrav->fill_ghost) { pmb->pgrav->RestoreFaceBoundaries(); - pmb->pgrav->ExpandPhysicalBoundaries(); + pmb->pgrav->gbvar.ExpandPhysicalBoundaries(pmb->pgrav->phi, 1); } return TaskStatus::next; } From cea80ab9f344e0038f0b0509fbee6c75a72cf1f3 Mon Sep 17 00:00:00 2001 From: Kengo Tomida Date: Fri, 24 Nov 2023 23:45:59 +0900 Subject: [PATCH 07/33] *** incomplete commit *** save --- src/crdiffusion/crdiffusion.cpp | 24 ++++++++++++++++ src/crdiffusion/crdiffusion.hpp | 2 +- src/crdiffusion/mg_crdiffusion.cpp | 45 +++++++++++++++++++++++++++--- src/gravity/mg_gravity.cpp | 1 - src/outputs/outputs.cpp | 10 ++++++- 5 files changed, 75 insertions(+), 7 deletions(-) diff --git a/src/crdiffusion/crdiffusion.cpp b/src/crdiffusion/crdiffusion.cpp index dcd69a5766..43cc6491fd 100644 --- a/src/crdiffusion/crdiffusion.cpp +++ b/src/crdiffusion/crdiffusion.cpp @@ -117,5 +117,29 @@ void CRDiffusion::CalculateCoefficients(const AthenaArray &w, } } } + + return; } + +//---------------------------------------------------------------------------------------- +//! \fn void CRDiffusion::CalculateIonizationRate(const AthenaArray &w) +//! \brief Calculate Ionization Rate from the Cosmic-ray density +void CRDiffusion::CalculateIonizationRate(const AthenaArray &w) { + Real zeta_factor = 1.0; + int il = pmy_block->is - NGHOST, iu = pmy_block->ie + NGHOST; + int jl = pmy_block->js, ju = pmy_block->je; + int kl = pmy_block->ks, ku = pmy_block->ke; + if (pmy_block->pmy_mesh->f2) + jl -= NGHOST, ju += NGHOST; + if (pmy_block->pmy_mesh->f3) + kl -= NGHOST, ku += NGHOST; + for (int k = kl; k <= ku; ++k) { + for (int j = jl; j <= ju; ++j) { + for (int i = il; i <= iu; ++i) + zeta(k, j, i) = zeta_factor * w(IDN, k, j, i) * ecr(k, j, i); + } + } + + return; +} diff --git a/src/crdiffusion/crdiffusion.hpp b/src/crdiffusion/crdiffusion.hpp index 21de98ee8a..5c7882bf78 100644 --- a/src/crdiffusion/crdiffusion.hpp +++ b/src/crdiffusion/crdiffusion.hpp @@ -48,7 +48,7 @@ class CRDiffusion { void CalculateCoefficients(const AthenaArray &w, const AthenaArray &bcc); - void Solve(); + void CalculateIonizationRate(const AthenaArray &w); friend class MGCRDiffusuionDriver; diff --git a/src/crdiffusion/mg_crdiffusion.cpp b/src/crdiffusion/mg_crdiffusion.cpp index 87a400f3be..bf494be69e 100644 --- a/src/crdiffusion/mg_crdiffusion.cpp +++ b/src/crdiffusion/mg_crdiffusion.cpp @@ -34,6 +34,10 @@ class MeshBlock; +namespace { + AthenaArray *temp; // temporary data for the Jacobi iteration +} + //---------------------------------------------------------------------------------------- //! \fn MGCRDiffusionDriver::MGCRDiffusionDriver(Mesh *pm, ParameterInput *pin) //! \brief MGCRDiffusionDriver constructor @@ -44,6 +48,7 @@ MGCRDiffusionDriver::MGCRDiffusionDriver(Mesh *pm, ParameterInput *pin) eps_ = pin->GetOrAddReal("crdiffusion", "threshold", -1.0); niter_ = pin->GetOrAddInteger("crdiffusion", "niteration", -1); ffas_ = pin->GetOrAddBoolean("crdiffusion", "fas", ffas_); + redblack_ = false; std::string m = pin->GetOrAddString("crdiffusion", "mgmode", "none"); std::transform(m.begin(), m.end(), m.begin(), ::tolower); if (m == "fmg") { @@ -87,6 +92,17 @@ MGCRDiffusionDriver::MGCRDiffusionDriver(Mesh *pm, ParameterInput *pin) mgroot_ = new MGCRDiffusion(this, nullptr); crtlist_ = new CRDiffusionBoundaryTaskList(pin, pm); + + int nth = 1; +#ifdef OPENMP_PARALLEL + nth = omp_get_max_threads(); +#endif + temp = new AthenaArray[nth]; + int nx = std::max(pmy_mesh_->block_size.nx1, pmy_mesh_->nrbx1) + 2*mgroot_->ngh_; + int ny = std::max(pmy_mesh_->block_size.nx2, pmy_mesh_->nrbx2) + 2*mgroot_->ngh_; + int nz = std::max(pmy_mesh_->block_size.nx3, pmy_mesh_->nrbx3) + 2*mgroot_->ngh_; + for (int n = 0; n < nth; ++n) + temp[n].NewAthenaArray(nz, ny, nx); } @@ -98,6 +114,7 @@ MGCRDiffusionDriver::~MGCRDiffusionDriver() { delete crtlist_; delete mgroot_; delete mgtlist_; + delete [] temp; } @@ -165,6 +182,13 @@ void MGCRDiffusionDriver::Solve(int stage) { crtlist_->DoTaskListOneStage(pmy_mesh_, stage); + for (auto itr = vmg_.begin(); itr < vmg_.end(); itr++) { + Multigrid *pmg = *itr; + CRDiffusion *pcrdiff = pmg->pmy_block_->pcrdiff; + Hydro *phydro = pmg->pmy_block_->phydro; + pcrdiff->CalculateIonizationRate(phydro->w); + } + return; } @@ -185,21 +209,34 @@ void MGCRDiffusion::Smooth(AthenaArray &u, const AthenaArray &src, Real dx2 = SQR(dx); Real isix = omega_/6.0; color ^= pmy_driver_->coffset_; + int t = 0; + if (th == true && (ku-kl) >= minth_) { + AthenaArray &work = temp[t]; #pragma omp parallel for num_threads(pmy_driver_->nthreads_) for (int k=kl; k<=ku; k++) { for (int j=jl; j<=ju; j++) { - int c = (color + k + j) & 1; #pragma ivdep - for (int i=il+c; i<=iu; i+=2); + for (int i=il; i<=iu; i++) { + Real Md = 0.0; + Real Mt = 0.0; + work(k,j,i) = (u(k,j,i) - Mt)/Md; + } } } } else { +#ifdef OPENMP_PARALLEL + t = omp_get_thread_num(); +#endif + AthenaArray &work = temp[t]; for (int k=kl; k<=ku; k++) { for (int j=jl; j<=ju; j++) { - int c = (color + k + j) & 1; #pragma ivdep - for (int i=il+c; i<=iu; i+=2); + for (int i=il; i<=iu; i++) { + Real Md = 0.0; + Real Mt = 0.0; + work(k,j,i) = (u(k,j,i) - Mt)/Md; + } } } } diff --git a/src/gravity/mg_gravity.cpp b/src/gravity/mg_gravity.cpp index fa5211a57f..2f899ba34d 100644 --- a/src/gravity/mg_gravity.cpp +++ b/src/gravity/mg_gravity.cpp @@ -165,7 +165,6 @@ MGGravity::~MGGravity() { //! \brief load the data and solve void MGGravityDriver::Solve(int stage) { - four_pi_G_ = pmy_mesh_->four_pi_G_; // Construct the Multigrid array vmg_.clear(); for (int i = 0; i < pmy_mesh_->nblocal; ++i) diff --git a/src/outputs/outputs.cpp b/src/outputs/outputs.cpp index 403b89f545..5b34d11d27 100644 --- a/src/outputs/outputs.cpp +++ b/src/outputs/outputs.cpp @@ -1058,13 +1058,21 @@ void OutputType::LoadOutputData(MeshBlock *pmb) { pod->data.InitWithShallowSlice(pcrdiff->ecr,4,0,1); AppendOutputDataNode(pod); num_vars_++; + if (pcrdiff->output_defect) { + pod = new OutputData; + pod->type = "SCALARS"; + pod->name = "defect-phi"; + pod->data.InitWithShallowSlice(pcrdiff->def, 4, 0, 1); + AppendOutputDataNode(pod); + num_vars_++; + } } if (ContainVariable(output_params.variable, "zeta") || ContainVariable(output_params.variable, "prim") || ContainVariable(output_params.variable, "cons")) { pod = new OutputData; pod->type = "SCALARS"; - pod->name = "ecr"; + pod->name = "zeta"; pod->data.InitWithShallowSlice(pcrdiff->zeta,4,0,1); AppendOutputDataNode(pod); num_vars_++; From 5a7b26cebd342124e1b23312c413bab41a5b216a Mon Sep 17 00:00:00 2001 From: Kengo Tomida Date: Sun, 26 Nov 2023 13:13:05 +0900 Subject: [PATCH 08/33] *** incomplete commit *** save --- inputs/cosmic_ray/athinput.cr_diffusion_mg | 70 ++++++++++++ src/crdiffusion/crdiffusion.cpp | 2 +- src/crdiffusion/mg_crdiffusion.cpp | 34 ++++-- src/main.cpp | 4 + src/mesh/mesh.cpp | 41 ++++++- src/mesh/mesh.hpp | 18 ++- src/multigrid/multigrid.cpp | 7 +- src/outputs/athena_hdf5.cpp | 6 +- src/outputs/outputs.cpp | 2 +- src/pgen/cr_diffusion_mg.cpp | 124 +++++++++++++++++++-- 10 files changed, 277 insertions(+), 31 deletions(-) create mode 100644 inputs/cosmic_ray/athinput.cr_diffusion_mg diff --git a/inputs/cosmic_ray/athinput.cr_diffusion_mg b/inputs/cosmic_ray/athinput.cr_diffusion_mg new file mode 100644 index 0000000000..f931ed53ae --- /dev/null +++ b/inputs/cosmic_ray/athinput.cr_diffusion_mg @@ -0,0 +1,70 @@ + +problem = CR diffusion test with Multigrid +reference = +configure = --prob=cr_diffusion_mg -crdiff + + +problem_id = MGCRDiffusion # problem ID: basename of output filenames + + +file_type = hst # History data dump +dt = 0.01 # time increment between outputs + + +file_type = vtk # Binary data dump +variable = prim # variables to be output +dt = 0.01 # time increment between outputs + +