|
|
@@ -1,7 +1,21 @@ |
|
|
|
/** |
|
|
|
* |
|
|
|
* Implementation of an MPI_Parallel Stencil-Based Jacobi Solver for the 2D PDE |
|
|
|
* |
|
|
|
* -Du(x, y) + k^2 u(x, y) = f(x, y), (x, y) in [0, 1] x [0, 1] |
|
|
|
* |
|
|
|
* with the scalar k = 2 pi. The right hand side is |
|
|
|
* |
|
|
|
* f(x, y) = k^2 sin(2 pi x) sinh(2 pi y) |
|
|
|
* |
|
|
|
* and the Dirichlet boundary conditions |
|
|
|
* |
|
|
|
* u(0, y) = u(1, y) = u(x, 0) = 0, x, y in [0, 1] |
|
|
|
* u(x, 1) = sin(2 pi x) sinh(2 pi), x in [0, 1] |
|
|
|
* |
|
|
|
* The following programm is implemented such that it can be compiled in seriel |
|
|
|
* or MPI-parallel form by declaring the `USE_MPI` macro (see `Makefile`). |
|
|
|
*/ |
|
|
|
#define _USE_MATH_DEFINES /* enables math constants from cmath */ |
|
|
|
#define _USE_MATH_DEFINES /* enables math constants from `cmath` */ |
|
|
|
|
|
|
|
#include <stddef.h> |
|
|
|
#include <iostream> |
|
|
@@ -10,8 +24,9 @@ |
|
|
|
#include <functional> |
|
|
|
#include <chrono> |
|
|
|
#include <cmath> |
|
|
|
|
|
|
|
#ifdef USE_MPI |
|
|
|
#include <mpi.h> |
|
|
|
#include <mpi.h> |
|
|
|
#endif |
|
|
|
|
|
|
|
#include "Matrix.h" |
|
|
@@ -25,7 +40,7 @@ int main(int argn, char* argv[]) { |
|
|
|
// Initialize MPI |
|
|
|
MPI_Init(nullptr, nullptr); |
|
|
|
|
|
|
|
// Get MPI config |
|
|
|
// Get MPI configure |
|
|
|
int mpi_size; /*< MPI pool size (a.k.a. total number of processes) */ |
|
|
|
MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); |
|
|
|
|
|
|
@@ -97,13 +112,13 @@ int main(int argn, char* argv[]) { |
|
|
|
const double h = 1.0 / static_cast<double>(resolution - 1); |
|
|
|
|
|
|
|
#ifdef USE_MPI |
|
|
|
// Group processes into a cartesian communication topology. Set initial |
|
|
|
// Group processes into a Cartesian communication topology. Set initial |
|
|
|
// values for a 1D grid. |
|
|
|
int mpi_dims[2] = {mpi_size, 1}; |
|
|
|
// In case of a 2D grid, make equal partitions ob both axes (as equal as |
|
|
|
// possible. Note that `MPI_Dims_create` does not garantee "as equal as". |
|
|
|
// possible. Note that `MPI_Dims_create` does not guarantee "as equal as". |
|
|
|
// For example it was observed that for 9 processes the generated grid |
|
|
|
// was a 9 x 1, the following computes a 3 x 3). |
|
|
|
// was a 9 x 1, the following computes a 3 x 3 decomposition). |
|
|
|
if (dim == 2) { |
|
|
|
two_factors(mpi_size, mpi_dims); |
|
|
|
} |
|
|
@@ -115,7 +130,7 @@ int main(int argn, char* argv[]) { |
|
|
|
} |
|
|
|
std::cout << std::endl; |
|
|
|
|
|
|
|
// Setup a cartesian topology communicator (NON-cyclic) |
|
|
|
// Setup a Cartesian topology communicator (NON-cyclic) |
|
|
|
const int mpi_periods[2] = {false, false}; |
|
|
|
MPI_Comm mpi_comm_grid; |
|
|
|
MPI_Cart_create( |
|
|
@@ -135,53 +150,53 @@ int main(int argn, char* argv[]) { |
|
|
|
int mpi_coords[2]; |
|
|
|
MPI_Cart_coords(mpi_comm_grid, mpi_grid_rank, 2, mpi_coords); |
|
|
|
|
|
|
|
// Get direct neightbours in the communication grid |
|
|
|
struct { int north; int east; int south; int west; } mpi_neighbours; |
|
|
|
// Get X-direction (dim 0) neightbours |
|
|
|
// Get direct neighbors in the communication grid |
|
|
|
struct { int north; int east; int south; int west; } mpi_neighbors; |
|
|
|
// Get X-direction (dim 0) neighbors |
|
|
|
MPI_Cart_shift( |
|
|
|
mpi_comm_grid, // grid communicator |
|
|
|
0, // axis index (0 <-> X) |
|
|
|
1, // offset |
|
|
|
&(mpi_neighbours.west), // negated offset neightbour |
|
|
|
&(mpi_neighbours.east) // offset neightbour |
|
|
|
&(mpi_neighbors.west), // negated offset neighbor |
|
|
|
&(mpi_neighbors.east) // offset neighbor |
|
|
|
); |
|
|
|
// Get Y-direction (dim 1) neightbours |
|
|
|
// Get Y-direction (dim 1) neighbors |
|
|
|
MPI_Cart_shift( |
|
|
|
mpi_comm_grid, |
|
|
|
1, // axis index (1 <-> Y) |
|
|
|
1, |
|
|
|
&(mpi_neighbours.south), |
|
|
|
&(mpi_neighbours.north) |
|
|
|
&(mpi_neighbors.south), |
|
|
|
&(mpi_neighbors.north) |
|
|
|
); |
|
|
|
|
|
|
|
// Calc local (base) grid size (without ghost layers) |
|
|
|
// Calculate local (base) grid size (without ghost layers) |
|
|
|
size_t nx = partition(resolution, mpi_dims[0], mpi_coords[0]); |
|
|
|
size_t ny = partition(resolution, mpi_dims[1], mpi_coords[1]); |
|
|
|
// Add ghost layers for each (existing) neighbour |
|
|
|
ny += (mpi_neighbours.north != MPI_PROC_NULL); |
|
|
|
nx += (mpi_neighbours.east != MPI_PROC_NULL); |
|
|
|
ny += (mpi_neighbours.south != MPI_PROC_NULL); |
|
|
|
nx += (mpi_neighbours.west != MPI_PROC_NULL); |
|
|
|
// Add ghost layers for each (existing) neighbor |
|
|
|
ny += (mpi_neighbors.north != MPI_PROC_NULL); |
|
|
|
nx += (mpi_neighbors.east != MPI_PROC_NULL); |
|
|
|
ny += (mpi_neighbors.south != MPI_PROC_NULL); |
|
|
|
nx += (mpi_neighbors.west != MPI_PROC_NULL); |
|
|
|
|
|
|
|
// Compute local domain [xmin, xmax] x [ymin, ymax] |
|
|
|
double xmin = (mpi_neighbours.west == MPI_PROC_NULL) ? 0.0 |
|
|
|
double xmin = (mpi_neighbors.west == MPI_PROC_NULL) ? 0.0 |
|
|
|
: h * (partition_sum(resolution, mpi_dims[0], mpi_coords[0] - 1) - 1); |
|
|
|
double xmax = (mpi_neighbours.east == MPI_PROC_NULL) ? 1.0 |
|
|
|
double xmax = (mpi_neighbors.east == MPI_PROC_NULL) ? 1.0 |
|
|
|
: h * partition_sum(resolution, mpi_dims[0], mpi_coords[0]); |
|
|
|
double ymin = (mpi_neighbours.south == MPI_PROC_NULL) ? 0.0 |
|
|
|
double ymin = (mpi_neighbors.south == MPI_PROC_NULL) ? 0.0 |
|
|
|
: h * (partition_sum(resolution, mpi_dims[1], mpi_coords[1] - 1) - 1); |
|
|
|
double ymax = (mpi_neighbours.north == MPI_PROC_NULL) ? 1.0 |
|
|
|
double ymax = (mpi_neighbors.north == MPI_PROC_NULL) ? 1.0 |
|
|
|
: h * partition_sum(resolution, mpi_dims[1], mpi_coords[1]); |
|
|
|
|
|
|
|
// Create MPI vector Type (for boundary condition exchange) |
|
|
|
// Allows directly exchange of matrix rows (north/south bounds) since the |
|
|
|
// row elements are "sparce" in the sence that they are not directly aside |
|
|
|
// each other in memory (column major matrix layout) in constrast to columns. |
|
|
|
// row elements are "sparse" in the sense that they are not directly aside |
|
|
|
// each other in memory (column major matrix layout) in contrast to columns. |
|
|
|
MPI_Datatype mpi_type_row; |
|
|
|
MPI_Type_vector(ny, 1, nx, MPI_DOUBLE, &mpi_type_row); |
|
|
|
MPI_Type_commit(&mpi_type_row); |
|
|
|
#else |
|
|
|
// Discretization grid resolution |
|
|
|
// discretization grid resolution |
|
|
|
size_t nx = resolution, ny = resolution; |
|
|
|
// PDE domain borders [xmin, xmax] x [ymin, ymax] = [0, 1] x [0, 1] |
|
|
|
double xmin = 0.0, xmax = 1.0, ymin = 0.0, ymax = 1.0; |
|
|
@@ -196,7 +211,7 @@ int main(int argn, char* argv[]) { |
|
|
|
std::function<double(double)> gN; |
|
|
|
#ifdef USE_MPI |
|
|
|
// Check if local north boundary is part of the global north boundary |
|
|
|
if (mpi_neighbours.north == MPI_PROC_NULL) { |
|
|
|
if (mpi_neighbors.north == MPI_PROC_NULL) { |
|
|
|
#endif |
|
|
|
// The local north boundary is equals the global north boundary |
|
|
|
gN = [k](double x) { return sin(2 * M_PI * x) * sinh(2 * M_PI); }; |
|
|
@@ -210,41 +225,41 @@ int main(int argn, char* argv[]) { |
|
|
|
std::function<double(double)> g0 = [k](double) { return 0.0; }; |
|
|
|
|
|
|
|
/******************************* Solve PDE ********************************/ |
|
|
|
// Instanciate solver (local instance) |
|
|
|
// Instantiate solver (local instance) |
|
|
|
Solver solver(nx, ny, xmin, xmax, ymin, ymax, h, k, fun, gN, g0, g0, g0); |
|
|
|
|
|
|
|
// Run solver iterations |
|
|
|
for (size_t iter = 0; iter < iterations; ++iter) { |
|
|
|
// Perform a single stencil jacobi iteration |
|
|
|
// Perform a single stencil Jacobi iteration |
|
|
|
solver.iterate(); |
|
|
|
|
|
|
|
#ifdef USE_MPI |
|
|
|
// Non-blocking send boundary conditions to all neightbours |
|
|
|
// Non-blocking send boundary conditions to all neighbors |
|
|
|
MPI_Request mpi_requests[4]; |
|
|
|
int mpi_request_count = 0; |
|
|
|
|
|
|
|
if (mpi_neighbours.north != MPI_PROC_NULL) { |
|
|
|
if (mpi_neighbors.north != MPI_PROC_NULL) { |
|
|
|
auto bound = solver.read_boundary(Solver::Dir::North); |
|
|
|
MPI_Isend(bound.data(), bound.size(), MPI_DOUBLE, |
|
|
|
mpi_neighbours.north, iter, mpi_comm_grid, |
|
|
|
mpi_neighbors.north, iter, mpi_comm_grid, |
|
|
|
&mpi_requests[mpi_request_count++]); |
|
|
|
} |
|
|
|
if (mpi_neighbours.east != MPI_PROC_NULL) { |
|
|
|
if (mpi_neighbors.east != MPI_PROC_NULL) { |
|
|
|
auto bound = solver.read_boundary(Solver::Dir::East); |
|
|
|
MPI_Isend(bound.data(), 1, mpi_type_row, |
|
|
|
mpi_neighbours.east, iter, mpi_comm_grid, |
|
|
|
mpi_neighbors.east, iter, mpi_comm_grid, |
|
|
|
&mpi_requests[mpi_request_count++]); |
|
|
|
} |
|
|
|
if (mpi_neighbours.south != MPI_PROC_NULL) { |
|
|
|
if (mpi_neighbors.south != MPI_PROC_NULL) { |
|
|
|
auto bound = solver.read_boundary(Solver::Dir::South); |
|
|
|
MPI_Isend(bound.data(), bound.size(), MPI_DOUBLE, |
|
|
|
mpi_neighbours.south, iter, mpi_comm_grid, |
|
|
|
mpi_neighbors.south, iter, mpi_comm_grid, |
|
|
|
&mpi_requests[mpi_request_count++]); |
|
|
|
} |
|
|
|
if (mpi_neighbours.west != MPI_PROC_NULL) { |
|
|
|
if (mpi_neighbors.west != MPI_PROC_NULL) { |
|
|
|
auto bound = solver.read_boundary(Solver::Dir::West); |
|
|
|
MPI_Isend(bound.data(), 1, mpi_type_row, |
|
|
|
mpi_neighbours.west, iter, mpi_comm_grid, |
|
|
|
mpi_neighbors.west, iter, mpi_comm_grid, |
|
|
|
&mpi_requests[mpi_request_count++]); |
|
|
|
} |
|
|
|
|
|
|
@@ -254,25 +269,25 @@ int main(int argn, char* argv[]) { |
|
|
|
|
|
|
|
// Get new boundary conditions using a blocking receive |
|
|
|
MPI_Status mpi_status; |
|
|
|
if (mpi_neighbours.north != MPI_PROC_NULL) { |
|
|
|
if (mpi_neighbors.north != MPI_PROC_NULL) { |
|
|
|
auto bound = solver.write_boundary(Solver::Dir::North); |
|
|
|
MPI_Recv(bound.data(), bound.size(), MPI_DOUBLE, |
|
|
|
mpi_neighbours.north, iter, mpi_comm_grid, &mpi_status); |
|
|
|
mpi_neighbors.north, iter, mpi_comm_grid, &mpi_status); |
|
|
|
} |
|
|
|
if (mpi_neighbours.east != MPI_PROC_NULL) { |
|
|
|
if (mpi_neighbors.east != MPI_PROC_NULL) { |
|
|
|
auto bound = solver.write_boundary(Solver::Dir::East); |
|
|
|
MPI_Recv(bound.data(), 1, mpi_type_row, |
|
|
|
mpi_neighbours.east, iter, mpi_comm_grid, &mpi_status); |
|
|
|
mpi_neighbors.east, iter, mpi_comm_grid, &mpi_status); |
|
|
|
} |
|
|
|
if (mpi_neighbours.south != MPI_PROC_NULL) { |
|
|
|
if (mpi_neighbors.south != MPI_PROC_NULL) { |
|
|
|
auto bound = solver.write_boundary(Solver::Dir::South); |
|
|
|
MPI_Recv(bound.data(), bound.size(), MPI_DOUBLE, |
|
|
|
mpi_neighbours.south, iter, mpi_comm_grid, &mpi_status); |
|
|
|
mpi_neighbors.south, iter, mpi_comm_grid, &mpi_status); |
|
|
|
} |
|
|
|
if (mpi_neighbours.west != MPI_PROC_NULL) { |
|
|
|
if (mpi_neighbors.west != MPI_PROC_NULL) { |
|
|
|
auto bound = solver.write_boundary(Solver::Dir::West); |
|
|
|
MPI_Recv(bound.data(), 1, mpi_type_row, |
|
|
|
mpi_neighbours.west, iter, mpi_comm_grid, &mpi_status); |
|
|
|
mpi_neighbors.west, iter, mpi_comm_grid, &mpi_status); |
|
|
|
} |
|
|
|
#endif |
|
|
|
} |
|
|
@@ -325,7 +340,7 @@ int main(int argn, char* argv[]) { |
|
|
|
}; |
|
|
|
#endif |
|
|
|
|
|
|
|
// calculate runtime time |
|
|
|
// calculate run-time |
|
|
|
auto stop = std::chrono::high_resolution_clock::now(); |
|
|
|
auto time = std::chrono::duration_cast<std::chrono::duration<double>>(stop - start) |
|
|
|
.count(); |