Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix GPU problems in pulse_MG.cpp #629

Merged
merged 3 commits into from
May 15, 2024
Merged

Fix GPU problems in pulse_MG.cpp #629

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1ef0c17
fix GPU issues in pulse_MG.cpp and turn it on for 3D problems
chongchonghe May 14, 2024
0db6462
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] May 14, 2024
f6e9a23
set max_timesteps to 30
chongchonghe May 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 5 additions & 7 deletions src/RadhydroPulseMG/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,7 @@
if (AMReX_SPACEDIM EQUAL 1)
add_executable(test_radhydro_pulse_MG test_radhydro_pulse_MG.cpp ../fextract.cpp ${QuokkaObjSources})
add_executable(test_radhydro_pulse_MG test_radhydro_pulse_MG.cpp ../fextract.cpp ${QuokkaObjSources})

if(AMReX_GPU_BACKEND MATCHES "CUDA")
setup_target_for_cuda_compilation(test_radhydro_pulse_MG)
endif(AMReX_GPU_BACKEND MATCHES "CUDA")
if(AMReX_GPU_BACKEND MATCHES "CUDA")
setup_target_for_cuda_compilation(test_radhydro_pulse_MG)
endif(AMReX_GPU_BACKEND MATCHES "CUDA")

add_test(NAME RadhydroPulseMG COMMAND test_radhydro_pulse_MG RadhydroPulse.in WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests)
endif()
add_test(NAME RadhydroPulseMG COMMAND test_radhydro_pulse_MG RadhydroPulse.in WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests)
23 changes: 12 additions & 11 deletions src/RadhydroPulseMG/test_radhydro_pulse_MG.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,12 @@ struct AdvPulseProblem {
constexpr int isconst = 0;
// constexpr int n_groups_ = 1;
// constexpr amrex::GpuArray<double, n_groups_ + 1> rad_boundaries_{0., inf};
constexpr int n_groups_ = 2;
constexpr amrex::GpuArray<double, n_groups_ + 1> rad_boundaries_{1e15, 1e17, 1e19};
// constexpr int n_groups_ = 2;
// constexpr amrex::GpuArray<double, n_groups_ + 1> rad_boundaries_{1e15, 1e17, 1e19};
constexpr int n_groups_ = 4;
constexpr amrex::GpuArray<double, n_groups_ + 1> rad_boundaries_{1e16, 1e17, 1e18, 1e19, 1e20};
// constexpr int n_groups_ = 8;
// constexpr amrex::GpuArray<double, n_groups_ + 1> rad_boundaries_{1e15, 3.16e15, 1e16, 3.16e16, 1e17, 3.16e17, 1e18, 3.16e18, 1e19};
// constexpr amrex::GpuArray<double, n_groups_ + 1> rad_boundaries_{1e16, 3.16e16, 1e17, 3.16e17, 1e18, 3.16e18, 1e19, 3.16e19, 1e20};
// constexpr int n_groups_ = 64;
// constexpr amrex::GpuArray<double, n_groups_ + 1> rad_boundaries_{1.00000000e+15, 1.15478198e+15, 1.33352143e+15, 1.53992653e+15,
// 1.77827941e+15, 2.05352503e+15, 2.37137371e+15, 2.73841963e+15,
Expand Down Expand Up @@ -57,8 +59,8 @@ constexpr double v0_nonadv = 0.; // non-advecting pulse

// static diffusion: (for single group) tau = 2e3, beta = 3e-5, beta tau = 6e-2
constexpr double v0_adv = 1.0e6; // advecting pulse
constexpr double max_time = 2.4e-5; // max_time = 1.0 * width / v1;
// constexpr double max_time = 4.8e-5; // max_time = 2.0 * width / v1;
constexpr double max_time = 4.8e-5; // max_time = 0.02 * width / v1;
constexpr int64_t max_timesteps = 30;

// dynamic diffusion: tau = 2e4, beta = 3e-3, beta tau = 60
// constexpr double kappa0 = 1000.; // cm^2 g^-1
Expand Down Expand Up @@ -146,15 +148,15 @@ auto compute_kappa(const double nu, const double Tgas) -> double
}

AMREX_GPU_HOST_DEVICE
auto compute_repres_nu() -> quokka::valarray<double, n_groups_>
auto compute_repres_nu(amrex::GpuArray<double, n_groups_ + 1> rad_boundaries) -> quokka::valarray<double, n_groups_>
{
// return the geometrical mean as the representative frequency for each group
quokka::valarray<double, n_groups_> nu_rep{};
if constexpr (n_groups_ == 1) {
if (n_groups_ == 1) {
nu_rep[0] = nu_ref;
} else {
for (int g = 0; g < n_groups_; ++g) {
nu_rep[g] = std::sqrt(rad_boundaries_[g] * rad_boundaries_[g + 1]);
nu_rep[g] = std::sqrt(rad_boundaries[g] * rad_boundaries[g + 1]);
}
}
return nu_rep;
Expand All @@ -163,7 +165,7 @@ auto compute_repres_nu() -> quokka::valarray<double, n_groups_>
template <> AMREX_GPU_HOST_DEVICE auto RadSystem<PulseProblem>::ComputePlanckOpacity(const double rho, const double Tgas) -> quokka::valarray<double, nGroups_>
{
quokka::valarray<double, nGroups_> kappaPVec{};
auto nu_rep = compute_repres_nu();
auto nu_rep = compute_repres_nu(rad_boundaries_);
for (int g = 0; g < nGroups_; ++g) {
kappaPVec[g] = compute_kappa(nu_rep[g], Tgas) / rho;
}
Expand Down Expand Up @@ -194,7 +196,7 @@ AMREX_GPU_HOST_DEVICE auto RadSystem<PulseProblem>::ComputePlanckOpacityTempDeri
-> quokka::valarray<double, nGroups_>
{
quokka::valarray<double, nGroups_> opacity_deriv{};
const auto nu_rep = compute_repres_nu();
const auto nu_rep = compute_repres_nu(rad_boundaries_);
const auto T = Tgas / T0;
for (int g = 0; g < nGroups_; ++g) {
const auto nu = nu_rep[g] / nu_ref;
Expand Down Expand Up @@ -309,7 +311,6 @@ auto problem_main() -> int
// in the diffusion limit.

// Problem parameters
const int64_t max_timesteps = 1e8;
const double CFL_number = 0.8;
// const int nx = 32;

Expand Down
20 changes: 9 additions & 11 deletions src/RadhydroPulseMGint/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,13 @@
if (AMReX_SPACEDIM EQUAL 1)
add_executable(test_radhydro_pulse_MG_int test_radhydro_pulse_MG_int.cpp ../fextract.cpp ${QuokkaObjSources})
add_executable(test_radhydro_pulse_MG_int test_radhydro_pulse_MG_int.cpp ../fextract.cpp ${QuokkaObjSources})

if(AMReX_GPU_BACKEND MATCHES "CUDA")
setup_target_for_cuda_compilation(test_radhydro_pulse_MG_int)
endif(AMReX_GPU_BACKEND MATCHES "CUDA")
if(AMReX_GPU_BACKEND MATCHES "CUDA")
setup_target_for_cuda_compilation(test_radhydro_pulse_MG_int)
endif(AMReX_GPU_BACKEND MATCHES "CUDA")

# define variable WORK_PATH = {WORKSPACE}/sims
# set(WORK_PATH "${CMAKE_SOURCE_DIR}/tests/20240408-MG-int-v8.2")
# define variable WORK_PATH = {WORKSPACE}/sims
# set(WORK_PATH "${CMAKE_SOURCE_DIR}/tests/20240408-MG-int-v8.2")

# mkdir -p WORK_PATH
# file(MAKE_DIRECTORY ${WORK_PATH})
# mkdir -p WORK_PATH
# file(MAKE_DIRECTORY ${WORK_PATH})

add_test(NAME RadhydroPulseMGint COMMAND test_radhydro_pulse_MG_int RadhydroPulse.in WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests)
endif()
add_test(NAME RadhydroPulseMGint COMMAND test_radhydro_pulse_MG_int RadhydroPulse.in WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests)
8 changes: 4 additions & 4 deletions src/RadhydroPulseMGint/test_radhydro_pulse_MG_int.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,10 @@ constexpr double h_planck = C::hplanck;
constexpr double k_B = C::k_B;

// static diffusion: (for single group) tau = 2e3, beta = 3e-5, beta tau = 6e-2
constexpr double kappa0 = 180.; // cm^2 g^-1
constexpr double v0_adv = 1.0e6; // advecting pulse
constexpr double max_time = 4.8e-6; // max_time = 0.2 * width / v1;
constexpr double kappa0 = 180.; // cm^2 g^-1
constexpr double v0_adv = 1.0e6; // advecting pulse
constexpr double max_time = 4.8e-5; // max_time = 2 * width / v1;
constexpr int64_t max_timesteps = 30; // to make 3D test run fast on GPUs

// dynamic diffusion: tau = 2e4, beta = 3e-3, beta tau = 60
// constexpr double kappa0 = 1000.; // cm^2 g^-1
Expand Down Expand Up @@ -273,7 +274,6 @@ auto problem_main() -> int
// This problem is based on the radhydro_pulse test and is a test of interpolation for variable opacity for multigroup radiation.

// Problem parameters
const int64_t max_timesteps = 1e8;
const double CFL_number = 0.8;
// const int nx = 32;

Expand Down