linear_algebra_distribution.h

Go to the documentation of this file.

// LIC// ====================================================================
// LIC// This file forms part of oomph-lib, the object-oriented,
// LIC// multi-physics finite-element library, available
// LIC// at http://www.oomph-lib.org.
// LIC//
// LIC// Copyright (C) 2006-2022 Matthias Heil and Andrew Hazel
// LIC//
// LIC// This library is free software; you can redistribute it and/or
// LIC// modify it under the terms of the GNU Lesser General Public
// LIC// License as published by the Free Software Foundation; either
// LIC// version 2.1 of the License, or (at your option) any later version.
// LIC//
// LIC// This library is distributed in the hope that it will be useful,
// LIC// but WITHOUT ANY WARRANTY; without even the implied warranty of
// LIC// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// LIC// Lesser General Public License for more details.
// LIC//
// LIC// You should have received a copy of the GNU Lesser General Public
// LIC// License along with this library; if not, write to the Free Software
// LIC// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
// LIC// 02110-1301  USA.
// LIC//
// LIC// The authors may be contacted at oomph-lib@maths.man.ac.uk.
// LIC//
// LIC//====================================================================
#ifndef OOMPH_LINEAR_ALGEBRA_DISTRIBUTION_CLASS_HEADER
#define OOMPH_LINEAR_ALGEBRA_DISTRIBUTION_CLASS_HEADER
 
// Config header generated by autoconfig
#ifdef HAVE_CONFIG_H
#include <oomph-lib-config.h>
#endif
 
// MPI headers
#ifdef OOMPH_HAS_MPI
#include "mpi.h"
#endif
 
#include <ostream>
 
// General headers
#include "Vector.h"
#include "communicator.h"
#include "oomph_utilities.h"
 
 
namespace oomph
{
  //=============================================================================
  //=============================================================================
  class LinearAlgebraDistribution
  {
  public:
    LinearAlgebraDistribution() : Comm_pt(0) {}
 
    LinearAlgebraDistribution(const OomphCommunicator& comm,
                              const unsigned& first_row_,
                              const unsigned& n_row_local,
                              const unsigned& n_row = 0)
      : Comm_pt(0)
    {
      this->build(&comm, first_row_, n_row_local, n_row);
    };
 
    LinearAlgebraDistribution(const OomphCommunicator& comm,
                              const unsigned& n_row,
                              const bool& distributed_ = true)
      : Comm_pt(0)
    {
      this->build(&comm, n_row, distributed_);
    };
 
    LinearAlgebraDistribution(const OomphCommunicator* const comm_pt,
                              const unsigned& first_row_,
                              const unsigned& n_row_local,
                              const unsigned& n_row = 0)
      : Comm_pt(0)
    {
      this->build(comm_pt, first_row_, n_row_local, n_row);
    };
 
    LinearAlgebraDistribution(const OomphCommunicator* const comm_pt,
                              const unsigned& n_row,
                              const bool& distributed_ = true)
      : Comm_pt(0)
    {
      this->build(comm_pt, n_row, distributed_);
    };
 
    LinearAlgebraDistribution(const LinearAlgebraDistribution& old_dist)
      : Comm_pt(0)
    {
      this->build(old_dist);
    }
 
    LinearAlgebraDistribution(const LinearAlgebraDistribution* old_dist_pt)
      : Comm_pt(0)
    {
      this->build(old_dist_pt);
    }
 
    ~LinearAlgebraDistribution()
    {
      delete Comm_pt;
    }
 
    void operator=(const LinearAlgebraDistribution& old_dist)
    {
      this->build(old_dist);
    }
 
    void build(const OomphCommunicator* const comm_pt,
               const unsigned& first_row,
               const unsigned& nrow_local,
               const unsigned& nrow = 0);
 
    void build(const OomphCommunicator* const comm_pt,
               const unsigned& nrow,
               const bool& distributed = true);
 
    void build(const LinearAlgebraDistribution& new_dist);
 
    void build(const LinearAlgebraDistribution* new_dist_pt)
    {
      this->build(*new_dist_pt);
    }
 
    void clear()
    {
      // delete the communicator
      delete Comm_pt;
      Comm_pt = 0;
 
      // delete first_row and nrow_local
      First_row.clear();
      Nrow_local.clear();
 
      // zero Nrow
      Nrow = 0;
    }
 
    unsigned nrow() const
    {
      return Nrow;
    }
 
    unsigned nrow_local() const
    {
      // return the nrow_local
#ifdef OOMPH_HAS_MPI
      if (Distributed)
      {
#ifdef PARANOID
        if (Comm_pt == 0)
        {
          throw OomphLibError(
            "LinearAlgebraDistribution has not been built : Comm_pt == 0.",
            "LinearAlgebraDistribution::nrow_local()",
            OOMPH_EXCEPTION_LOCATION);
        }
#endif
 
        return Nrow_local[Comm_pt->my_rank()];
      }
      else
      {
        return Nrow;
      }
#else
      return Nrow;
#endif
    }
 
    unsigned nrow_local(const unsigned& p) const
    {
#ifdef PARANOID
      if (Comm_pt == 0)
      {
        throw OomphLibError(
          "LinearAlgebraDistribution has not been built : Comm_pt == 0.",
          OOMPH_CURRENT_FUNCTION,
          OOMPH_EXCEPTION_LOCATION);
      }
      if (p >= unsigned(Comm_pt->nproc()))
      {
        std::ostringstream error_message;
        error_message << "Requested nrow_local(" << p
                      << "), but this distribution is defined "
                      << "on " << Comm_pt->nproc() << "processors.";
        throw OomphLibError(error_message.str(),
                            OOMPH_CURRENT_FUNCTION,
                            OOMPH_EXCEPTION_LOCATION);
      }
#endif
 
      // return the nrow_local
#ifdef OOMPH_HAS_MPI
      if (Distributed)
      {
        return Nrow_local[p];
      }
      else
      {
        return Nrow;
      }
#else
      return Nrow;
#endif
    }
 
    unsigned first_row() const
    {
      // return the first row
#ifdef OOMPH_HAS_MPI
      if (Distributed)
      {
#ifdef PARANOID
        if (Comm_pt == 0)
        {
          throw OomphLibError(
            "LinearAlgebraDistribution has not been built : Comm_pt == 0.",
            OOMPH_CURRENT_FUNCTION,
            OOMPH_EXCEPTION_LOCATION);
        }
#endif
 
        return First_row[Comm_pt->my_rank()];
      }
      else
      {
        return 0;
      }
#else
      return 0;
#endif
    }
 
    unsigned first_row(const unsigned& p) const
    {
#ifdef PARANOID
      if (Comm_pt == 0)
      {
        throw OomphLibError(
          "LinearAlgebraDistribution has not been built : Comm_pt == 0.",
          OOMPH_CURRENT_FUNCTION,
          OOMPH_EXCEPTION_LOCATION);
      }
      if (p >= unsigned(Comm_pt->nproc()))
      {
        std::ostringstream error_message;
        error_message << "Requested first_row(" << p
                      << "), but this distribution is defined "
                      << "on " << Comm_pt->nproc() << "processors.";
        throw OomphLibError(error_message.str(),
                            OOMPH_CURRENT_FUNCTION,
                            OOMPH_EXCEPTION_LOCATION);
      }
 
#endif
 
      // return the first row
#ifdef OOMPH_HAS_MPI
      if (Distributed)
      {
        return First_row[p];
      }
      else
      {
        return 0;
      }
#else
      return 0;
#endif
    }
 
    bool distributed() const
    {
      return Distributed;
    }
 
    OomphCommunicator* communicator_pt() const
    {
      return Comm_pt;
    }
 
    bool built() const
    {
      if (Comm_pt == 0)
      {
        return false;
      }
      return true;
    }
 
    bool operator==(const LinearAlgebraDistribution& other_dist) const;
 
    bool operator!=(const LinearAlgebraDistribution& other_dist) const
    {
      return !(*this == other_dist);
    }
 
    friend std::ostream& operator<<(std::ostream& stream,
                                    LinearAlgebraDistribution& dist);
 
    unsigned global_to_local_row_map(const unsigned& global_i) const
    {
#ifdef PARANOID
      if (global_i >= Nrow)
      {
        throw OomphLibError(
          "Requested global row outside the number of global rows",
          OOMPH_CURRENT_FUNCTION,
          OOMPH_EXCEPTION_LOCATION);
      }
#endif
      int local_i = static_cast<int>(global_i);
      int p = 0;
      while ((int)(local_i - (int)nrow_local(p)) >= 0)
      {
        local_i -= (int)nrow_local(p);
        p++;
      }
      return (unsigned)local_i;
    }
 
    unsigned rank_of_global_row(const unsigned i) const
    {
      unsigned p = 0;
      while (i < first_row(p) || i >= first_row(p) + nrow_local(p))
      {
        p++;
      }
      return p;
    }
 
    Vector<unsigned> nrow_local_vector() const
    {
      return Nrow_local;
    }
 
    Vector<unsigned> first_row_vector() const
    {
      return First_row;
    }
 
  private:
    unsigned Nrow;
 
    Vector<unsigned> Nrow_local;
 
    Vector<unsigned> First_row;
 
    bool Distributed;
 
    OomphCommunicator* Comm_pt;
  }; // end of LinearAlgebraDistribution
 
 
  //=============================================================================
  //=============================================================================
  class DistributableLinearAlgebraObject
  {
  public:
    DistributableLinearAlgebraObject()
    {
      Distribution_pt = new LinearAlgebraDistribution;
    }
 
    DistributableLinearAlgebraObject(
      const DistributableLinearAlgebraObject& matrix) = delete;
 
    void operator=(const DistributableLinearAlgebraObject&) = delete;
 
    virtual ~DistributableLinearAlgebraObject()
    {
      delete Distribution_pt;
    }
 
    LinearAlgebraDistribution* distribution_pt() const
    {
      return Distribution_pt;
    }
 
    unsigned nrow() const
    {
      return Distribution_pt->nrow();
    }
 
    unsigned nrow_local() const
    {
      return Distribution_pt->nrow_local();
    }
 
    unsigned nrow_local(const unsigned& p) const
    {
      return Distribution_pt->nrow_local(p);
    }
 
    unsigned first_row() const
    {
      return Distribution_pt->first_row();
    }
 
    unsigned first_row(const unsigned& p) const
    {
      return Distribution_pt->first_row(p);
    }
 
    bool distributed() const
    {
      return Distribution_pt->distributed();
    }
 
    bool distribution_built() const
    {
      return Distribution_pt->built();
    }
 
    void build_distribution(const LinearAlgebraDistribution* const dist_pt)
    {
      Distribution_pt->build(dist_pt);
    }
 
    void build_distribution(const LinearAlgebraDistribution& dist)
    {
      Distribution_pt->build(dist);
    }
 
  protected:
    void clear_distribution()
    {
      Distribution_pt->clear();
    }
 
  private:
    LinearAlgebraDistribution* Distribution_pt;
  }; // end of DistributableLinearAlgebraObject
 
  //=============================================================================
  //=============================================================================
  namespace LinearAlgebraDistributionHelpers
  {
    void concatenate(
      const Vector<LinearAlgebraDistribution*>& in_distribution_pt,
      LinearAlgebraDistribution& out_distribution);
  } // namespace LinearAlgebraDistributionHelpers
} // namespace oomph
#endif