Classes | Namespaces | Macros | Functions | Variables
MatrixProduct.h File Reference
#include "MatrixProductCommon.h"
#include "../../InternalHeaderCheck.h"
#include "MatrixVectorProduct.h"

Go to the source code of this file.

Classes

struct  Eigen::internal::quad_traits< Scalar >
 
struct  Eigen::internal::quad_traits< double >
 
struct  Eigen::internal::quad_traits< bfloat16 >
 
struct  Eigen::internal::symm_pack_rhs< std::complex< float >, Index, nr, StorageOrder >
 
struct  Eigen::internal::symm_pack_lhs< std::complex< float >, Index, Pack1, Pack2_dummy, StorageOrder >
 
struct  Eigen::internal::symm_pack_rhs< std::complex< double >, Index, nr, StorageOrder >
 
struct  Eigen::internal::symm_pack_lhs< std::complex< double >, Index, Pack1, Pack2_dummy, StorageOrder >
 
struct  Eigen::internal::symm_pack_rhs< float, Index, nr, StorageOrder >
 
struct  Eigen::internal::symm_pack_lhs< float, Index, Pack1, Pack2_dummy, StorageOrder >
 
struct  Eigen::internal::symm_pack_rhs< double, Index, nr, StorageOrder >
 
struct  Eigen::internal::symm_pack_lhs< double, Index, Pack1, Pack2_dummy, StorageOrder >
 
struct  Eigen::internal::dhs_cpack< Scalar, DataMapper, Packet, PacketC, StorageOrder, Conjugate, PanelMode, UseLhs >
 
struct  Eigen::internal::dhs_pack< Scalar, DataMapper, Packet, StorageOrder, PanelMode, UseLhs >
 
struct  Eigen::internal::dhs_pack< double, DataMapper, Packet2d, StorageOrder, PanelMode, true >
 
struct  Eigen::internal::dhs_pack< double, DataMapper, Packet2d, StorageOrder, PanelMode, false >
 
struct  Eigen::internal::dhs_pack< bfloat16, DataMapper, Packet8bf, StorageOrder, PanelMode, true >
 
struct  Eigen::internal::dhs_pack< bfloat16, DataMapper, Packet8bf, StorageOrder, PanelMode, false >
 
struct  Eigen::internal::dhs_cpack< double, DataMapper, Packet, PacketC, StorageOrder, Conjugate, PanelMode, true >
 
struct  Eigen::internal::dhs_cpack< double, DataMapper, Packet, PacketC, StorageOrder, Conjugate, PanelMode, false >
 
struct  Eigen::internal::gemm_pack_lhs< double, Index, DataMapper, Pack1, Pack2, Packet, ColMajor, Conjugate, PanelMode >
 
struct  Eigen::internal::gemm_pack_lhs< double, Index, DataMapper, Pack1, Pack2, Packet, RowMajor, Conjugate, PanelMode >
 
struct  Eigen::internal::gemm_pack_rhs< double, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode >
 
struct  Eigen::internal::gemm_pack_rhs< double, Index, DataMapper, nr, RowMajor, Conjugate, PanelMode >
 
struct  Eigen::internal::gemm_pack_rhs< bfloat16, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode >
 
struct  Eigen::internal::gemm_pack_rhs< bfloat16, Index, DataMapper, nr, RowMajor, Conjugate, PanelMode >
 
struct  Eigen::internal::gemm_pack_lhs< bfloat16, Index, DataMapper, Pack1, Pack2, Packet, ColMajor, Conjugate, PanelMode >
 
struct  Eigen::internal::gemm_pack_lhs< bfloat16, Index, DataMapper, Pack1, Pack2, Packet, RowMajor, Conjugate, PanelMode >
 
struct  Eigen::internal::gemm_pack_lhs< float, Index, DataMapper, Pack1, Pack2, Packet, RowMajor, Conjugate, PanelMode >
 
struct  Eigen::internal::gemm_pack_lhs< float, Index, DataMapper, Pack1, Pack2, Packet, ColMajor, Conjugate, PanelMode >
 
struct  Eigen::internal::gemm_pack_lhs< std::complex< float >, Index, DataMapper, Pack1, Pack2, Packet, RowMajor, Conjugate, PanelMode >
 
struct  Eigen::internal::gemm_pack_lhs< std::complex< float >, Index, DataMapper, Pack1, Pack2, Packet, ColMajor, Conjugate, PanelMode >
 
struct  Eigen::internal::gemm_pack_rhs< float, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode >
 
struct  Eigen::internal::gemm_pack_rhs< float, Index, DataMapper, nr, RowMajor, Conjugate, PanelMode >
 
struct  Eigen::internal::gemm_pack_rhs< std::complex< float >, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode >
 
struct  Eigen::internal::gemm_pack_rhs< std::complex< float >, Index, DataMapper, nr, RowMajor, Conjugate, PanelMode >
 
struct  Eigen::internal::gemm_pack_lhs< std::complex< double >, Index, DataMapper, Pack1, Pack2, Packet, RowMajor, Conjugate, PanelMode >
 
struct  Eigen::internal::gemm_pack_lhs< std::complex< double >, Index, DataMapper, Pack1, Pack2, Packet, ColMajor, Conjugate, PanelMode >
 
struct  Eigen::internal::gemm_pack_rhs< std::complex< double >, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode >
 
struct  Eigen::internal::gemm_pack_rhs< std::complex< double >, Index, DataMapper, nr, RowMajor, Conjugate, PanelMode >
 
struct  Eigen::internal::gebp_kernel< float, float, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >
 
struct  Eigen::internal::gebp_kernel< std::complex< float >, std::complex< float >, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >
 
struct  Eigen::internal::gebp_kernel< float, std::complex< float >, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >
 
struct  Eigen::internal::gebp_kernel< std::complex< float >, float, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >
 
struct  Eigen::internal::gebp_kernel< double, double, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >
 
struct  Eigen::internal::gebp_kernel< std::complex< double >, std::complex< double >, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >
 
struct  Eigen::internal::gebp_kernel< std::complex< double >, double, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >
 
struct  Eigen::internal::gebp_kernel< double, std::complex< double >, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >
 
struct  Eigen::internal::gebp_kernel< bfloat16, bfloat16, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >
 

Namespaces

 Eigen
 Namespace containing all symbols from the Eigen library.
 
 Eigen::internal
 Namespace containing low-level routines from the Eigen library.
 

Macros

#define EIGEN_ALTIVEC_USE_CUSTOM_PACK   1
 
#define EIGEN_ALTIVEC_DISABLE_MMA   0
 
#define USE_P10_AND_PVIPR2_0   0
 
#define PEEL   7
 
#define PEEL_ROW   7
 
#define MICRO_UNROLL(func)   func(0) func(1) func(2) func(3) func(4) func(5) func(6) func(7)
 
#define MICRO_NORMAL_ROWS   accRows == quad_traits<Scalar>::rows || accRows == 1
 
#define MICRO_NEW_ROWS   ((MICRO_NORMAL_ROWS) ? accRows : 1)
 
#define MICRO_RHS(ptr, N)   rhs_##ptr##N
 
#define MICRO_ZERO_PEEL(peel)
 
#define MICRO_ADD(ptr, N)
 
#define MICRO_ADD_ROWS(N)   MICRO_ADD(ptr, N)
 
#define MICRO_BROADCAST1(peel, ptr, rhsV, real)
 
#define MICRO_BROADCAST(peel)   MICRO_BROADCAST1(peel, ptr, rhsV, true)
 
#define MICRO_BROADCAST_EXTRA1(ptr, rhsV, real)
 
#define MICRO_BROADCAST_EXTRA
 
#define MICRO_SRC2(ptr, N, M)
 
#define MICRO_SRC2_PTR   MICRO_SRC2(ptr, strideB, 0)
 
#define MICRO_ZERO_PEEL_ROW   MICRO_UNROLL(MICRO_ZERO_PEEL)
 
#define MICRO_WORK_PEEL(peel)
 
#define MICRO_WORK_PEEL_ROW
 
#define MICRO_ADD_PEEL(peel, sum)
 
#define MICRO_ADD_PEEL_ROW
 
#define MICRO_PREFETCHN1(ptr, N)
 
#define MICRO_PREFETCHN(N)   MICRO_PREFETCHN1(ptr, N)
 
#define MICRO_COMPLEX_PREFETCHN(N)
 
#define MICRO_EXTRA(MICRO_EXTRA_UNROLL, value, is_col)
 
#define MICRO_EXTRA_ROWS(N)
 
#define MICRO_UNROLL_WORK(func, func2, peel)
 
#define MICRO_WORK_ONE(iter, peel)
 
#define MICRO_TYPE_PEEL4(func, func2, peel)
 
#define MICRO_UNROLL_TYPE_PEEL(M, func, func1, func2)
 
#define MICRO_UNROLL_TYPE_ONE(M, func, func1, func2)
 
#define MICRO_UNROLL_TYPE(MICRO_TYPE, size)
 
#define MICRO_ONE_PEEL4   MICRO_UNROLL_TYPE(MICRO_UNROLL_TYPE_PEEL, PEEL)
 
#define MICRO_ONE4   MICRO_UNROLL_TYPE(MICRO_UNROLL_TYPE_ONE, 1)
 
#define MICRO_DST_PTR_ONE(iter)
 
#define MICRO_DST_PTR   MICRO_UNROLL(MICRO_DST_PTR_ONE)
 
#define MICRO_SRC_PTR   MICRO_UNROLL(MICRO_SRC_PTR_ONE)
 
#define MICRO_PREFETCH   MICRO_UNROLL(MICRO_PREFETCH_ONE)
 
#define MICRO_STORE_ONE(iter)
 
#define MICRO_STORE   MICRO_UNROLL(MICRO_STORE_ONE)
 
#define MICRO_UNROLL_ITER2(N, M)
 
#define MAX_UNROLL   7
 
#define MICRO_EXTRA_COLS(N)
 
#define accColsC   (accCols / 2)
 
#define advanceRows   ((LhsIsReal) ? 1 : 2)
 
#define advanceCols   ((RhsIsReal) ? 1 : 2)
 
#define PEEL_COMPLEX   3
 
#define PEEL_COMPLEX_ROW   3
 
#define MICRO_COMPLEX_UNROLL(func)   func(0) func(1) func(2) func(3)
 
#define MICRO_COMPLEX_ZERO_PEEL(peel)
 
#define MICRO_COMPLEX_ADD_ROWS(N, used)
 
#define MICRO_COMPLEX_BROADCAST(peel)
 
#define MICRO_COMPLEX_BROADCAST_EXTRA
 
#define MICRO_COMPLEX_SRC2_PTR
 
#define MICRO_COMPLEX_ZERO_PEEL_ROW   MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_ZERO_PEEL)
 
#define MICRO_COMPLEX_WORK_PEEL(peel)
 
#define MICRO_COMPLEX_ADD_COLS(size)
 
#define MICRO_COMPLEX_WORK_PEEL_ROW
 
#define MICRO_COMPLEX_ADD_PEEL(peel, sum)
 
#define MICRO_COMPLEX_ADD_PEEL_ROW    MICRO_COMPLEX_ADD_PEEL(2, 0) MICRO_COMPLEX_ADD_PEEL(3, 1) MICRO_COMPLEX_ADD_PEEL(1, 0)
 
#define MICRO_COMPLEX_EXTRA_ROWS(N)
 
#define MICRO_COMPLEX_UNROLL_WORK(func, func2, peel)
 
#define MICRO_COMPLEX_WORK_ONE4(iter, peel)
 
#define MICRO_COMPLEX_TYPE_PEEL4(func, func2, peel)
 
#define MICRO_COMPLEX_UNROLL_TYPE_PEEL(M, func, func1, func2)
 
#define MICRO_COMPLEX_UNROLL_TYPE_ONE(M, func, func1, func2)
 
#define MICRO_COMPLEX_UNROLL_TYPE(MICRO_COMPLEX_TYPE, size)
 
#define MICRO_COMPLEX_ONE_PEEL4   MICRO_COMPLEX_UNROLL_TYPE(MICRO_COMPLEX_UNROLL_TYPE_PEEL, PEEL_COMPLEX)
 
#define MICRO_COMPLEX_ONE4   MICRO_COMPLEX_UNROLL_TYPE(MICRO_COMPLEX_UNROLL_TYPE_ONE, 1)
 
#define MICRO_COMPLEX_DST_PTR_ONE(iter)
 
#define MICRO_COMPLEX_DST_PTR   MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_DST_PTR_ONE)
 
#define MICRO_COMPLEX_SRC_PTR   MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_SRC_PTR_ONE)
 
#define MICRO_COMPLEX_PREFETCH   MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_PREFETCH_ONE)
 
#define MICRO_COMPLEX_STORE_ONE(iter)
 
#define MICRO_COMPLEX_STORE   MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_STORE_ONE)
 
#define MICRO_COMPLEX_UNROLL_ITER2(N, M)
 
#define MAX_COMPLEX_UNROLL   4
 
#define MICRO_COMPLEX_EXTRA_COLS(N)
 
#define MAX_BFLOAT16_ACC_VSX   4
 

Functions

template<typename Scalar , int StorageOrder>
EIGEN_ALWAYS_INLINE std::complex< Scalar > Eigen::internal::getAdjointVal (Index i, Index j, const_blas_data_mapper< std::complex< Scalar >, Index, StorageOrder > &dt)
 
template<typename Scalar , int StorageOrder, int N>
EIGEN_STRONG_INLINE void Eigen::internal::symm_pack_complex_rhs_helper (std::complex< Scalar > *blockB, const std::complex< Scalar > *_rhs, Index rhsStride, Index rows, Index cols, Index k2)
 
template<typename Scalar , int StorageOrder>
EIGEN_STRONG_INLINE void Eigen::internal::symm_pack_complex_lhs_helper (std::complex< Scalar > *blockA, const std::complex< Scalar > *_lhs, Index lhsStride, Index cols, Index rows)
 
template<typename Scalar , int StorageOrder, int N>
EIGEN_STRONG_INLINE void Eigen::internal::symm_pack_rhs_helper (Scalar *blockB, const Scalar *_rhs, Index rhsStride, Index rows, Index cols, Index k2)
 
template<typename Scalar , int StorageOrder>
EIGEN_STRONG_INLINE void Eigen::internal::symm_pack_lhs_helper (Scalar *blockA, const Scalar *_lhs, Index lhsStride, Index cols, Index rows)
 
template<typename Scalar , typename Packet , int N>
EIGEN_ALWAYS_INLINE void Eigen::internal::storeBlock (Scalar *to, PacketBlock< Packet, N > &block)
 
template<typename Packet , bool NegativeAccumulate, int N>
EIGEN_ALWAYS_INLINE void Eigen::internal::pger_common (PacketBlock< Packet, N > *acc, const Packet &lhsV, const Packet *rhsV)
 
template<int N, typename Scalar , typename Packet , bool NegativeAccumulate>
EIGEN_ALWAYS_INLINE void Eigen::internal::pger (PacketBlock< Packet, N > *acc, const Scalar *lhs, const Packet *rhsV)
 
template<int N, typename Packet , bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
EIGEN_ALWAYS_INLINE void Eigen::internal::pgerc_common (PacketBlock< Packet, N > *accReal, PacketBlock< Packet, N > *accImag, const Packet &lhsV, Packet &lhsVi, const Packet *rhsV, const Packet *rhsVi)
 
template<int N, typename Scalar , typename Packet , bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
EIGEN_ALWAYS_INLINE void Eigen::internal::pgerc (PacketBlock< Packet, N > *accReal, PacketBlock< Packet, N > *accImag, const Scalar *lhs_ptr, const Scalar *lhs_ptr_imag, const Packet *rhsV, const Packet *rhsVi)
 
template<typename Packet >
EIGEN_ALWAYS_INLINE Packet Eigen::internal::ploadLhs (const __UNPACK_TYPE__(Packet) *lhs)
 
template<typename Packet , int N>
EIGEN_ALWAYS_INLINE void Eigen::internal::bsetzero (PacketBlock< Packet, N > &acc)
 
template<typename Packet , int N>
EIGEN_ALWAYS_INLINE void Eigen::internal::bscalec_common (PacketBlock< Packet, N > &acc, PacketBlock< Packet, N > &accZ, const Packet &pAlpha)
 
template<typename Packet , int N>
EIGEN_ALWAYS_INLINE void Eigen::internal::band (PacketBlock< Packet, N > &acc, const Packet &pMask)
 
template<typename Packet , int N, bool mask>
EIGEN_ALWAYS_INLINE void Eigen::internal::bscalec (PacketBlock< Packet, N > &aReal, PacketBlock< Packet, N > &aImag, const Packet &bReal, const Packet &bImag, PacketBlock< Packet, N > &cReal, PacketBlock< Packet, N > &cImag, const Packet &pMask)
 
template<typename DataMapper , typename Packet , const Index accCols, int StorageOrder, bool Complex, int N, bool full>
EIGEN_ALWAYS_INLINE void Eigen::internal::bload (PacketBlock< Packet, N *(Complex ? 2 :1)> &acc, const DataMapper &res, Index row, Index col)
 
template<typename DataMapper , typename Packet , int N>
EIGEN_ALWAYS_INLINE void Eigen::internal::bstore (PacketBlock< Packet, N > &acc, const DataMapper &res, Index row)
 
template<typename Packet >
EIGEN_ALWAYS_INLINE Packet Eigen::internal::bmask (const Index remaining_rows)
 
template<>
EIGEN_ALWAYS_INLINE Packet2d Eigen::internal::bmask< Packet2d > (const Index remaining_rows)
 
template<typename Packet , int N>
EIGEN_ALWAYS_INLINE void Eigen::internal::bscale (PacketBlock< Packet, N > &acc, PacketBlock< Packet, N > &accZ, const Packet &pAlpha)
 
template<typename Packet , int N, bool mask>
EIGEN_ALWAYS_INLINE void Eigen::internal::bscale (PacketBlock< Packet, N > &acc, PacketBlock< Packet, N > &accZ, const Packet &pAlpha, const Packet &pMask)
 
template<typename Packet , int N, bool real>
EIGEN_ALWAYS_INLINE void Eigen::internal::pbroadcastN (const __UNPACK_TYPE__(Packet) *ap0, const __UNPACK_TYPE__(Packet) *ap1, const __UNPACK_TYPE__(Packet) *ap2, Packet &a0, Packet &a1, Packet &a2, Packet &a3)
 
template<>
EIGEN_ALWAYS_INLINE void Eigen::internal::pbroadcastN< Packet4f, 4, true > (const float *ap0, const float *, const float *, Packet4f &a0, Packet4f &a1, Packet4f &a2, Packet4f &a3)
 
template<>
EIGEN_ALWAYS_INLINE void Eigen::internal::pbroadcastN< Packet4f, 4, false > (const float *ap0, const float *ap1, const float *ap2, Packet4f &a0, Packet4f &a1, Packet4f &a2, Packet4f &a3)
 
template<>
EIGEN_ALWAYS_INLINE void Eigen::internal::pbroadcastN< Packet2d, 4, false > (const double *ap0, const double *, const double *, Packet2d &a0, Packet2d &a1, Packet2d &a2, Packet2d &a3)
 
template<typename Packet , typename Packetc , int N, bool full>
EIGEN_ALWAYS_INLINE void Eigen::internal::bcouple_common (PacketBlock< Packet, N > &taccReal, PacketBlock< Packet, N > &taccImag, PacketBlock< Packetc, N > &acc1, PacketBlock< Packetc, N > &acc2)
 
template<typename Packet , typename Packetc , int N, bool full>
EIGEN_ALWAYS_INLINE void Eigen::internal::bcouple (PacketBlock< Packet, N > &taccReal, PacketBlock< Packet, N > &taccImag, PacketBlock< Packetc, N *2 > &tRes, PacketBlock< Packetc, N > &acc1, PacketBlock< Packetc, N > &acc2)
 
template<typename Scalar , typename Packet , const Index accRows, const Index remaining_rows>
EIGEN_ALWAYS_INLINE void Eigen::internal::MICRO_EXTRA_ROW (const Scalar *&lhs_ptr, const Scalar *&rhs_ptr0, const Scalar *&rhs_ptr1, const Scalar *&rhs_ptr2, PacketBlock< Packet, accRows > &accZero)
 
template<typename Scalar , typename Packet , typename DataMapper , const Index accRows, const Index accCols, const Index remaining_rows>
EIGEN_ALWAYS_INLINE void Eigen::internal::gemm_unrolled_row_iteration (const DataMapper &res, const Scalar *lhs_base, const Scalar *rhs_base, Index depth, Index strideA, Index offsetA, Index strideB, Index row, Index rows, const Packet &pAlpha, const Packet &pMask)
 
template<typename Scalar , typename Packet , typename DataMapper , const Index accRows, const Index accCols>
EIGEN_ALWAYS_INLINE void Eigen::internal::gemm_extra_row (const DataMapper &res, const Scalar *lhs_base, const Scalar *rhs_base, Index depth, Index strideA, Index offsetA, Index strideB, Index row, Index rows, Index remaining_rows, const Packet &pAlpha, const Packet &pMask)
 
template<int unroll_factor, typename Scalar , typename Packet , typename DataMapper , const Index accRows, const Index accCols, const Index accCols2>
EIGEN_ALWAYS_INLINE void Eigen::internal::gemm_unrolled_iteration (const DataMapper &res, const Scalar *lhs_base, const Scalar *rhs_base, Index depth, Index strideA, Index offsetA, Index strideB, Index &row, const Packet &pAlpha, const Packet &pMask)
 
template<typename Scalar , typename Packet , typename DataMapper , const Index accRows, const Index accCols>
EIGEN_ALWAYS_INLINE void Eigen::internal::gemm_cols (const DataMapper &res, const Scalar *blockA, const Scalar *blockB, Index depth, Index strideA, Index offsetA, Index strideB, Index offsetB, Index col, Index rows, Index remaining_rows, const Packet &pAlpha, const Packet &pMask)
 
template<typename Scalar , typename Packet , typename DataMapper , const Index accCols>
EIGEN_ALWAYS_INLINE void Eigen::internal::gemm_extra_cols (const DataMapper &res, const Scalar *blockA, const Scalar *blockB, Index depth, Index strideA, Index offsetA, Index strideB, Index offsetB, Index col, Index rows, Index cols, Index remaining_rows, const Packet &pAlpha, const Packet &pMask)
 
template<typename Scalar , typename Packet , typename RhsPacket , typename DataMapper , const Index accRows, const Index accCols>
EIGEN_STRONG_INLINE void Eigen::internal::gemm (const DataMapper &res, const Scalar *blockA, const Scalar *blockB, Index rows, Index depth, Index cols, Scalar alpha, Index strideA, Index strideB, Index offsetA, Index offsetB)
 
template<typename Scalar , typename Packet , const Index accRows, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal, const Index remaining_rows>
EIGEN_ALWAYS_INLINE void Eigen::internal::MICRO_COMPLEX_EXTRA_ROW (const Scalar *&lhs_ptr_real, const Scalar *&lhs_ptr_imag, const Scalar *&rhs_ptr_real0, const Scalar *&rhs_ptr_real1, const Scalar *&rhs_ptr_real2, const Scalar *&rhs_ptr_imag0, const Scalar *&rhs_ptr_imag1, const Scalar *&rhs_ptr_imag2, PacketBlock< Packet, accRows > &accReal, PacketBlock< Packet, accRows > &accImag)
 
template<typename Scalar , typename Packet , typename Packetc , typename DataMapper , const Index accRows, const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal, const Index remaining_rows>
EIGEN_ALWAYS_INLINE void Eigen::internal::gemm_unrolled_complex_row_iteration (const DataMapper &res, const Scalar *lhs_base, const Scalar *rhs_base, Index depth, Index strideA, Index offsetA, Index strideB, Index row, Index rows, const Packet &pAlphaReal, const Packet &pAlphaImag, const Packet &pMask)
 
template<typename Scalar , typename Packet , typename Packetc , typename DataMapper , const Index accRows, const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
EIGEN_ALWAYS_INLINE void Eigen::internal::gemm_complex_extra_row (const DataMapper &res, const Scalar *lhs_base, const Scalar *rhs_base, Index depth, Index strideA, Index offsetA, Index strideB, Index row, Index rows, Index remaining_rows, const Packet &pAlphaReal, const Packet &pAlphaImag, const Packet &pMask)
 
template<int unroll_factor, typename Scalar , typename Packet , typename Packetc , typename DataMapper , const Index accRows, const Index accCols, const Index accCols2, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
EIGEN_ALWAYS_INLINE void Eigen::internal::gemm_complex_unrolled_iteration (const DataMapper &res, const Scalar *lhs_base, const Scalar *rhs_base, Index depth, Index strideA, Index offsetA, Index strideB, Index &row, const Packet &pAlphaReal, const Packet &pAlphaImag, const Packet &pMask)
 
template<typename Scalar , typename Packet , typename Packetc , typename DataMapper , const Index accRows, const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
EIGEN_ALWAYS_INLINE void Eigen::internal::gemm_complex_cols (const DataMapper &res, const Scalar *blockA, const Scalar *blockB, Index depth, Index strideA, Index offsetA, Index strideB, Index offsetB, Index col, Index rows, Index remaining_rows, const Packet &pAlphaReal, const Packet &pAlphaImag, const Packet &pMask)
 
template<typename Scalar , typename Packet , typename Packetc , typename DataMapper , const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
EIGEN_ALWAYS_INLINE void Eigen::internal::gemm_complex_extra_cols (const DataMapper &res, const Scalar *blockA, const Scalar *blockB, Index depth, Index strideA, Index offsetA, Index strideB, Index offsetB, Index col, Index rows, Index cols, Index remaining_rows, const Packet &pAlphaReal, const Packet &pAlphaImag, const Packet &pMask)
 
template<typename LhsScalar , typename RhsScalar , typename Scalarc , typename Scalar , typename Packet , typename Packetc , typename RhsPacket , typename DataMapper , const Index accRows, const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
EIGEN_STRONG_INLINE void Eigen::internal::gemm_complex (const DataMapper &res, const LhsScalar *blockAc, const RhsScalar *blockBc, Index rows, Index depth, Index cols, Scalarc alpha, Index strideA, Index strideB, Index offsetA, Index offsetB)
 
EIGEN_ALWAYS_INLINE bool Eigen::internal::supportsMMA ()
 
EIGEN_ALWAYS_INLINE Packet4f Eigen::internal::loadAndMultiplyF32 (Packet4f acc, const Packet4f pAlpha, float *result)
 
template<bool lhsExtraRows>
EIGEN_ALWAYS_INLINE void Eigen::internal::storeF32 (float *&result, Packet4f result_block, Index rows, Index extra_rows)
 
template<bool rhsExtraCols, bool lhsExtraRows>
EIGEN_ALWAYS_INLINE void Eigen::internal::storeResults (Packet4f(&acc)[4], Index rows, const Packet4f pAlpha, float *result, Index extra_cols, Index extra_rows)
 
EIGEN_ALWAYS_INLINE Packet4f Eigen::internal::oneConvertBF16Hi (Packet8us data)
 
EIGEN_ALWAYS_INLINE Packet4f Eigen::internal::oneConvertBF16Lo (Packet8us data)
 
template<Index N, Index M>
EIGEN_ALWAYS_INLINE void Eigen::internal::storeConvertTwoBF16 (float *to, PacketBlock< Packet8bf,(N+7)/8 > &block, Index extra=0)
 
template<Index N>
EIGEN_ALWAYS_INLINE void Eigen::internal::storeConvertBlockBF16 (float *to, PacketBlock< Packet8bf,(N+7)/8 > &block, Index extra)
 
template<bool non_unit_stride, Index delta>
EIGEN_ALWAYS_INLINE Packet8bf Eigen::internal::loadBF16fromResult (bfloat16 *src, Index resInc)
 
EIGEN_ALWAYS_INLINE Packet4f Eigen::internal::oneConvertBF16Perm (Packet8us data, Packet16uc mask)
 
template<bool lhsExtraRows, bool odd, Index size>
EIGEN_ALWAYS_INLINE void Eigen::internal::convertArrayPointerBF16toF32DupOne (float *result, Index rows, const bfloat16 *src, Index extra_rows)
 
template<bool lhsExtraRows>
EIGEN_ALWAYS_INLINE void Eigen::internal::convertArrayPointerBF16toF32Dup (float *result, Index cols, Index rows, const bfloat16 *src, Index delta, Index extra_rows)
 
template<const Index size, bool non_unit_stride>
EIGEN_ALWAYS_INLINE void Eigen::internal::convertPointerBF16toF32 (Index &i, float *result, Index rows, bfloat16 *&src, Index resInc)
 
template<bool non_unit_stride>
EIGEN_ALWAYS_INLINE void Eigen::internal::convertArrayPointerBF16toF32 (float *result, Index cols, Index rows, bfloat16 *src, Index resInc)
 
template<Index num_acc, Index size = 4>
EIGEN_ALWAYS_INLINE void Eigen::internal::zeroAccumulators (Packet4f(&acc)[num_acc][size])
 
template<Index num_acc>
EIGEN_ALWAYS_INLINE void Eigen::internal::tranposeResults (Packet4f(&acc)[num_acc][4])
 
template<Index num_acc>
EIGEN_ALWAYS_INLINE void Eigen::internal::addResults (Packet4f(&acc)[num_acc][4])
 
template<Index num_acc, bool rhsExtraCols, bool lhsExtraRows, Index num_rhs>
EIGEN_ALWAYS_INLINE void Eigen::internal::outputResultsVSX (Packet4f(&acc)[num_acc][4], Index rows, const Packet4f pAlpha, float *result, const Index extra_cols, Index extra_rows)
 
template<bool zero>
EIGEN_ALWAYS_INLINE void Eigen::internal::loadTwoRhsFloat32 (const float *block, Index strideB, Index i, Packet4f &dhs0, Packet4f &dhs1)
 
template<Index num_acc, bool zero, bool rhsExtraCols, Index num_rhs>
EIGEN_ALWAYS_INLINE void Eigen::internal::KLoop (const float *indexA, const float *indexB, Packet4f(&acc)[num_acc][4], Index strideB, Index k, Index offsetB, Index extra_cols)
 
template<const Index num_acc, bool rhsExtraCols, bool lhsExtraRows>
EIGEN_ALWAYS_INLINE void Eigen::internal::colVSXLoopBodyIter (Index depth, Index rows, const Packet4f pAlpha, const float *indexA, const float *indexB, Index strideB, Index offsetB, float *result, const Index extra_cols, const Index extra_rows)
 
template<const Index num_acc, bool rhsExtraCols, bool lhsExtraRows>
void Eigen::internal::colVSXLoopBody (Index &col, Index depth, Index cols, Index rows, const Packet4f pAlpha, const float *indexA, const float *indexB, Index strideB, Index offsetB, float *result)
 
template<const Index num_acc, bool rhsExtraCols, bool lhsExtraRows>
EIGEN_ALWAYS_INLINE void Eigen::internal::colVSXLoopBodyExtraN (Index col, Index depth, Index cols, Index rows, const Packet4f pAlpha, const float *indexA, const float *blockB, Index strideB, Index offsetB, float *result)
 
template<bool rhsExtraCols, bool lhsExtraRows>
void Eigen::internal::colVSXLoopBodyExtra (Index col, Index depth, Index cols, Index rows, const Packet4f pAlpha, const float *indexA, const float *blockB, Index strideB, Index offsetB, float *result)
 
template<Index size, bool lhsExtraRows = false>
EIGEN_ALWAYS_INLINE void Eigen::internal::colVSXLoops (Index depth, Index cols, Index rows, const Packet4f pAlpha, const bfloat16 *indexA, const float *indexA2, const float *blockB2, Index strideA, Index strideB, Index offsetB, float *result2)
 
template<Index size>
EIGEN_ALWAYS_INLINE void Eigen::internal::calcVSXColLoops (const bfloat16 *&indexA, const float *indexA2, Index &row, Index depth, Index cols, Index rows, const Packet4f pAlpha, const float *indexB, Index strideA, Index strideB, Index offsetA, Index offsetB, Index bigSuffix, float *result)
 
template<const Index size, typename DataMapper >
EIGEN_ALWAYS_INLINE void Eigen::internal::convertBF16toF32 (Index &i, float *result, Index rows, const DataMapper &src)
 
template<typename DataMapper >
EIGEN_ALWAYS_INLINE void Eigen::internal::convertArrayBF16toF32 (float *result, Index cols, Index rows, const DataMapper &src)
 
EIGEN_ALWAYS_INLINE Packet8bf Eigen::internal::convertF32toBF16VSX (const float *res)
 
template<typename DataMapper , const Index size>
EIGEN_ALWAYS_INLINE void Eigen::internal::convertArrayF32toBF16ColVSX (float *result, Index col, Index rows, const DataMapper &res)
 
template<typename DataMapper >
EIGEN_ALWAYS_INLINE void Eigen::internal::convertArrayF32toBF16VSX (float *result, Index cols, Index rows, const DataMapper &res)
 
template<typename DataMapper >
void Eigen::internal::gemmbfloat16 (const DataMapper &res, const bfloat16 *indexA, const bfloat16 *indexB, Index rows, Index depth, Index cols, bfloat16 alpha, Index strideA, Index strideB, Index offsetA, Index offsetB)
 

Variables

static const Packet16uc Eigen::internal::p16uc_GETREAL32 = {0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27}
 
static const Packet16uc Eigen::internal::p16uc_GETIMAG32 = {4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31}
 
static const Packet16uc Eigen::internal::p16uc_GETREAL32b = {0, 1, 2, 3, 16, 17, 18, 19, 8, 9, 10, 11, 24, 25, 26, 27}
 
static const Packet16uc Eigen::internal::p16uc_GETIMAG32b = {4, 5, 6, 7, 20, 21, 22, 23, 12, 13, 14, 15, 28, 29, 30, 31}
 
static const Packet4i Eigen::internal::mask4 [4] = {{0, 0, 0, 0}, {-1, 0, 0, 0}, {-1, -1, 0, 0}, {-1, -1, -1, 0}}
 
static Packet16uc Eigen::internal::p16uc_MERGE16_32_1 = {0, 1, 16, 17, 2, 3, 18, 19, 0, 1, 16, 17, 2, 3, 18, 19}
 
static Packet16uc Eigen::internal::p16uc_MERGE16_32_2 = {4, 5, 20, 21, 6, 7, 22, 23, 4, 5, 20, 21, 6, 7, 22, 23}
 
static Packet16uc Eigen::internal::p16uc_MERGE16_32_3 = {8, 9, 24, 25, 10, 11, 26, 27, 8, 9, 24, 25, 10, 11, 26, 27}
 
static Packet16uc Eigen::internal::p16uc_MERGE16_32_4 = {12, 13, 28, 29, 14, 15, 30, 31, 12, 13, 28, 29, 14, 15, 30, 31}
 
static Packet16uc Eigen::internal::p16uc_MERGE16_32_5 = {0, 1, 16, 17, 16, 17, 16, 17, 0, 1, 16, 17, 16, 17, 16, 17}
 
static Packet16uc Eigen::internal::p16uc_MERGE16_32_6 = {2, 3, 18, 19, 18, 19, 18, 19, 2, 3, 18, 19, 18, 19, 18, 19}
 
static Packet16uc Eigen::internal::p16uc_MERGE16_32_7 = {4, 5, 20, 21, 20, 21, 20, 21, 4, 5, 20, 21, 20, 21, 20, 21}
 
static Packet16uc Eigen::internal::p16uc_MERGE16_32_8 = {6, 7, 22, 23, 22, 23, 22, 23, 6, 7, 22, 23, 22, 23, 22, 23}
 

Macro Definition Documentation

◆ accColsC

#define accColsC   (accCols / 2)

◆ advanceCols

#define advanceCols   ((RhsIsReal) ? 1 : 2)

◆ advanceRows

#define advanceRows   ((LhsIsReal) ? 1 : 2)

◆ EIGEN_ALTIVEC_DISABLE_MMA

#define EIGEN_ALTIVEC_DISABLE_MMA   0

◆ EIGEN_ALTIVEC_USE_CUSTOM_PACK

#define EIGEN_ALTIVEC_USE_CUSTOM_PACK   1

◆ MAX_BFLOAT16_ACC_VSX

#define MAX_BFLOAT16_ACC_VSX   4

◆ MAX_COMPLEX_UNROLL

#define MAX_COMPLEX_UNROLL   4

◆ MAX_UNROLL

#define MAX_UNROLL   7

◆ MICRO_ADD

#define MICRO_ADD (   ptr,
  N 
)
Value:
if (MICRO_NORMAL_ROWS) { \
MICRO_RHS(ptr, 0) += (accRows * N); \
} else { \
MICRO_RHS(ptr, 0) += N; \
MICRO_RHS(ptr, 1) += N; \
if (accRows == 3) { \
MICRO_RHS(ptr, 2) += N; \
} \
}
MICRO_NORMAL_ROWS
#define MICRO_NORMAL_ROWS
Definition: MatrixProduct.h:1795
N
@ N
Definition: constructor.cpp:22

◆ MICRO_ADD_PEEL

#define MICRO_ADD_PEEL (   peel,
  sum 
)
Value:
if (PEEL_ROW > peel) { \
for (Index i = 0; i < accRows; i++) { \
accZero##sum.packet[i] += accZero##peel.packet[i]; \
} \
}
i
int i
Definition: BiCGSTAB_step_by_step.cpp:9
PEEL_ROW
#define PEEL_ROW
Definition: MatrixProduct.h:1791
Eigen::Index
EIGEN_DEFAULT_DENSE_INDEX_TYPE Index
The Index type as used for the API.
Definition: Meta.h:83

◆ MICRO_ADD_PEEL_ROW

#define MICRO_ADD_PEEL_ROW
Value:
MICRO_ADD_PEEL(4, 0) \
MICRO_ADD_PEEL(5, 1) \
MICRO_ADD_PEEL(6, 2) MICRO_ADD_PEEL(7, 3) MICRO_ADD_PEEL(2, 0) MICRO_ADD_PEEL(3, 1) MICRO_ADD_PEEL(1, 0)
MICRO_ADD_PEEL
#define MICRO_ADD_PEEL(peel, sum)
Definition: MatrixProduct.h:1873

◆ MICRO_ADD_ROWS

#define MICRO_ADD_ROWS (   N)    MICRO_ADD(ptr, N)

◆ MICRO_BROADCAST

#define MICRO_BROADCAST (   peel)    MICRO_BROADCAST1(peel, ptr, rhsV, true)

◆ MICRO_BROADCAST1

#define MICRO_BROADCAST1 (   peel,
  ptr,
  rhsV,
  real 
)
Value:
if (MICRO_NORMAL_ROWS) { \
pbroadcastN<Packet, accRows, real>(MICRO_RHS(ptr, 0) + (accRows * peel), MICRO_RHS(ptr, 0), MICRO_RHS(ptr, 0), \
rhsV##peel[0], rhsV##peel[1], rhsV##peel[2], rhsV##peel[3]); \
} else { \
pbroadcastN<Packet, accRows, real>(MICRO_RHS(ptr, 0) + peel, MICRO_RHS(ptr, 1) + peel, MICRO_RHS(ptr, 2) + peel, \
rhsV##peel[0], rhsV##peel[1], rhsV##peel[2], rhsV##peel[3]); \
}
MICRO_RHS
#define MICRO_RHS(ptr, N)
Definition: MatrixProduct.h:1799

◆ MICRO_BROADCAST_EXTRA

#define MICRO_BROADCAST_EXTRA
Value:
Packet rhsV[4]; \
MICRO_BROADCAST_EXTRA1(ptr, rhsV, true) \
MICRO_ADD_ROWS(1)

◆ MICRO_BROADCAST_EXTRA1

#define MICRO_BROADCAST_EXTRA1 (   ptr,
  rhsV,
  real 
)
Value:
pbroadcastN<Packet, accRows, real>(MICRO_RHS(ptr, 0), MICRO_RHS(ptr, 1), MICRO_RHS(ptr, 2), rhsV[0], rhsV[1], \
rhsV[2], rhsV[3]);

◆ MICRO_COMPLEX_ADD_COLS

#define MICRO_COMPLEX_ADD_COLS (   size)
Value:
lhs_ptr_real += (remaining_rows * size); \
if (!LhsIsReal) \
lhs_ptr_imag += (remaining_rows * size); \
else \
EIGEN_UNUSED_VARIABLE(lhs_ptr_imag);
EIGEN_UNUSED_VARIABLE
#define EIGEN_UNUSED_VARIABLE(var)
Definition: Macros.h:966
size
Scalar Scalar int size
Definition: benchVecAdd.cpp:17

◆ MICRO_COMPLEX_ADD_PEEL

#define MICRO_COMPLEX_ADD_PEEL (   peel,
  sum 
)
Value:
if (PEEL_COMPLEX_ROW > peel) { \
for (Index i = 0; i < accRows; i++) { \
accReal##sum.packet[i] += accReal##peel.packet[i]; \
accImag##sum.packet[i] += accImag##peel.packet[i]; \
} \
}
PEEL_COMPLEX_ROW
#define PEEL_COMPLEX_ROW
Definition: MatrixProduct.h:2219

◆ MICRO_COMPLEX_ADD_PEEL_ROW

#define MICRO_COMPLEX_ADD_PEEL_ROW    MICRO_COMPLEX_ADD_PEEL(2, 0) MICRO_COMPLEX_ADD_PEEL(3, 1) MICRO_COMPLEX_ADD_PEEL(1, 0)

◆ MICRO_COMPLEX_ADD_ROWS

#define MICRO_COMPLEX_ADD_ROWS (   N,
  used 
)
Value:
MICRO_ADD(ptr_real, N) \
if (!RhsIsReal) { \
MICRO_ADD(ptr_imag, N) \
} else if (used) { \
EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag, 0)); \
EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag, 1)); \
EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag, 2)); \
}
MICRO_ADD
#define MICRO_ADD(ptr, N)
Definition: MatrixProduct.h:1808

◆ MICRO_COMPLEX_BROADCAST

#define MICRO_COMPLEX_BROADCAST (   peel)
Value:
MICRO_BROADCAST1(peel, ptr_real, rhsV, false) \
if (!RhsIsReal) { \
MICRO_BROADCAST1(peel, ptr_imag, rhsVi, false) \
} else { \
EIGEN_UNUSED_VARIABLE(rhsVi##peel); \
}
MICRO_BROADCAST1
#define MICRO_BROADCAST1(peel, ptr, rhsV, real)
Definition: MatrixProduct.h:1821

◆ MICRO_COMPLEX_BROADCAST_EXTRA

#define MICRO_COMPLEX_BROADCAST_EXTRA
Value:
Packet rhsV[4], rhsVi[4]; \
MICRO_BROADCAST_EXTRA1(ptr_real, rhsV, false) \
if (!RhsIsReal) { \
MICRO_BROADCAST_EXTRA1(ptr_imag, rhsVi, false) \
} else { \
EIGEN_UNUSED_VARIABLE(rhsVi); \
} \
MICRO_COMPLEX_ADD_ROWS(1, true)

◆ MICRO_COMPLEX_DST_PTR

#define MICRO_COMPLEX_DST_PTR   MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_DST_PTR_ONE)

◆ MICRO_COMPLEX_DST_PTR_ONE

#define MICRO_COMPLEX_DST_PTR_ONE (   iter)
Value:
if (unroll_factor > iter) { \
bsetzero<Packet, accRows>(accReal##iter); \
bsetzero<Packet, accRows>(accImag##iter); \
} else { \
EIGEN_UNUSED_VARIABLE(accReal##iter); \
EIGEN_UNUSED_VARIABLE(accImag##iter); \
}

◆ MICRO_COMPLEX_EXTRA_COLS

#define MICRO_COMPLEX_EXTRA_COLS (   N)
Value:
gemm_complex_cols<Scalar, Packet, Packetc, DataMapper, N, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, \
RhsIsReal>(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, \
remaining_rows, pAlphaReal, pAlphaImag, pMask);
col
m col(1)
res
cout<< "Here is the matrix m:"<< endl<< m<< endl;Matrix< ptrdiff_t, 3, 1 > res
Definition: PartialRedux_count.cpp:3
rows
int rows
Definition: Tutorial_commainit_02.cpp:1
Scalar
SCALAR Scalar
Definition: bench_gemm.cpp:45
Packet
internal::packet_traits< Scalar >::type Packet
Definition: benchmark-blocking-sizes.cpp:54
Eigen::internal::gemm_complex_cols
EIGEN_ALWAYS_INLINE void gemm_complex_cols(const DataMapper &res, const Scalar *blockA, const Scalar *blockB, Index depth, Index strideA, Index offsetA, Index strideB, Index offsetB, Index col, Index rows, Index remaining_rows, const Packet &pAlphaReal, const Packet &pAlphaImag, const Packet &pMask)
Definition: MatrixProduct.h:2525

◆ MICRO_COMPLEX_EXTRA_ROWS

#define MICRO_COMPLEX_EXTRA_ROWS (   N)
Value:
gemm_unrolled_complex_row_iteration<Scalar, Packet, Packetc, DataMapper, accRows, accCols, ConjugateLhs, \
ConjugateRhs, LhsIsReal, RhsIsReal, N>( \
res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, rows, pAlphaReal, pAlphaImag, pMask);
row
m row(1)
Eigen::internal::gemm_unrolled_complex_row_iteration
EIGEN_ALWAYS_INLINE void gemm_unrolled_complex_row_iteration(const DataMapper &res, const Scalar *lhs_base, const Scalar *rhs_base, Index depth, Index strideA, Index offsetA, Index strideB, Index row, Index rows, const Packet &pAlphaReal, const Packet &pAlphaImag, const Packet &pMask)
Definition: MatrixProduct.h:2326

◆ MICRO_COMPLEX_ONE4

#define MICRO_COMPLEX_ONE4   MICRO_COMPLEX_UNROLL_TYPE(MICRO_COMPLEX_UNROLL_TYPE_ONE, 1)

◆ MICRO_COMPLEX_ONE_PEEL4

#define MICRO_COMPLEX_ONE_PEEL4   MICRO_COMPLEX_UNROLL_TYPE(MICRO_COMPLEX_UNROLL_TYPE_PEEL, PEEL_COMPLEX)

◆ MICRO_COMPLEX_PREFETCH

#define MICRO_COMPLEX_PREFETCH   MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_PREFETCH_ONE)

◆ MICRO_COMPLEX_PREFETCHN

#define MICRO_COMPLEX_PREFETCHN (   N)
Value:
MICRO_PREFETCHN1(ptr_real, N); \
if (!RhsIsReal) { \
MICRO_PREFETCHN1(ptr_imag, N); \
}
MICRO_PREFETCHN1
#define MICRO_PREFETCHN1(ptr, N)
Definition: MatrixProduct.h:1885

◆ MICRO_COMPLEX_SRC2_PTR

#define MICRO_COMPLEX_SRC2_PTR
Value:
MICRO_SRC2(ptr_real, strideB* advanceCols, 0) \
if (!RhsIsReal) { \
MICRO_RHS(ptr_imag, 0) = rhs_base + MICRO_NEW_ROWS * strideB; \
MICRO_SRC2(ptr_imag, strideB* advanceCols, strideB) \
} else { \
EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag, 0)); \
EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag, 1)); \
EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag, 2)); \
}
advanceCols
#define advanceCols
Definition: MatrixProduct.h:2215
MICRO_NEW_ROWS
#define MICRO_NEW_ROWS
Definition: MatrixProduct.h:1797
MICRO_SRC2
#define MICRO_SRC2(ptr, N, M)
Definition: MatrixProduct.h:1841

◆ MICRO_COMPLEX_SRC_PTR

#define MICRO_COMPLEX_SRC_PTR   MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_SRC_PTR_ONE)

◆ MICRO_COMPLEX_STORE

#define MICRO_COMPLEX_STORE   MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_STORE_ONE)

◆ MICRO_COMPLEX_STORE_ONE

#define MICRO_COMPLEX_STORE_ONE (   iter)
Value:
if (unroll_factor > iter) { \
constexpr bool full = ((MICRO_NORMAL(iter)) || (accCols2 > accColsC)); \
bload<DataMapper, Packetc, accColsC, ColMajor, true, accRows, full>(tRes, res, row + iter * accCols, 0); \
bscalec<Packet, accRows, !(MICRO_NORMAL(iter))>(accReal##iter, accImag##iter, pAlphaReal, pAlphaImag, taccReal, \
taccImag, pMask); \
bcouple<Packet, Packetc, accRows, full>(taccReal, taccImag, tRes, acc0, acc1); \
bstore<DataMapper, Packetc, accRows>(acc0, res, row + iter * accCols + 0); \
if (full) { \
bstore<DataMapper, Packetc, accRows>(acc1, res, row + iter * accCols + accColsC); \
} \
}
MICRO_NORMAL
#define MICRO_NORMAL(iter)
Definition: MatrixProductCommon.h:109
accColsC
#define accColsC
Definition: MatrixProduct.h:2213

◆ MICRO_COMPLEX_TYPE_PEEL4

#define MICRO_COMPLEX_TYPE_PEEL4 (   func,
  func2,
  peel 
)
Value:
if (PEEL_COMPLEX > peel) { \
Packet lhsV0, lhsV1, lhsV2, lhsV3; \
Packet lhsVi0, lhsVi1, lhsVi2, lhsVi3; \
MICRO_COMPLEX_BROADCAST(peel) \
MICRO_COMPLEX_UNROLL_WORK(func, func2, peel) \
} else { \
EIGEN_UNUSED_VARIABLE(rhsV##peel); \
EIGEN_UNUSED_VARIABLE(rhsVi##peel); \
}
PEEL_COMPLEX
#define PEEL_COMPLEX
Definition: MatrixProduct.h:2218
func
Definition: benchGeometry.cpp:21

◆ MICRO_COMPLEX_UNROLL

#define MICRO_COMPLEX_UNROLL (   func)    func(0) func(1) func(2) func(3)

◆ MICRO_COMPLEX_UNROLL_ITER2

#define MICRO_COMPLEX_UNROLL_ITER2 (   N,
  M 
)
Value:
gemm_complex_unrolled_iteration<N + (M ? 1 : 0), Scalar, Packet, Packetc, DataMapper, accRows, accCols, \
M ? M : accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>( \
res3, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, pAlphaReal, pAlphaImag, pMask); \
if (M) return;
Eigen::Matrix
The matrix class, also used for vectors and row-vectors.
Definition: Eigen/Eigen/src/Core/Matrix.h:186
Eigen::internal::gemm_complex_unrolled_iteration
EIGEN_ALWAYS_INLINE void gemm_complex_unrolled_iteration(const DataMapper &res, const Scalar *lhs_base, const Scalar *rhs_base, Index depth, Index strideA, Index offsetA, Index strideB, Index &row, const Packet &pAlphaReal, const Packet &pAlphaImag, const Packet &pMask)
Definition: MatrixProduct.h:2482

◆ MICRO_COMPLEX_UNROLL_TYPE

#define MICRO_COMPLEX_UNROLL_TYPE (   MICRO_COMPLEX_TYPE,
  size 
)
Value:
MICRO_COMPLEX_TYPE(4, MICRO_COMPLEX_TYPE_PEEL4, MICRO_COMPLEX_WORK_ONE4, MICRO_COMPLEX_LOAD_ONE) \
MICRO_COMPLEX_ADD_ROWS(size, false)
MICRO_COMPLEX_LOAD_ONE
#define MICRO_COMPLEX_LOAD_ONE(iter)
Definition: MatrixProductCommon.h:156
MICRO_COMPLEX_TYPE_PEEL4
#define MICRO_COMPLEX_TYPE_PEEL4(func, func2, peel)
Definition: MatrixProduct.h:2421
MICRO_COMPLEX_WORK_ONE4
#define MICRO_COMPLEX_WORK_ONE4(iter, peel)
Definition: MatrixProduct.h:2415

◆ MICRO_COMPLEX_UNROLL_TYPE_ONE

#define MICRO_COMPLEX_UNROLL_TYPE_ONE (   M,
  func,
  func1,
  func2 
)
Value:
Packet rhsV0[M], rhsVi0[M]; \
func(func1, func2, 0)
M
Matrix< RealScalar, Dynamic, Dynamic > M
Definition: bench_gemm.cpp:50

◆ MICRO_COMPLEX_UNROLL_TYPE_PEEL

#define MICRO_COMPLEX_UNROLL_TYPE_PEEL (   M,
  func,
  func1,
  func2 
)
Value:
Packet rhsV0[M], rhsV1[M], rhsV2[M], rhsV3[M]; \
Packet rhsVi0[M], rhsVi1[M], rhsVi2[M], rhsVi3[M]; \
func(func1, func2, 0) func(func1, func2, 1) func(func1, func2, 2) func(func1, func2, 3)

◆ MICRO_COMPLEX_UNROLL_WORK

#define MICRO_COMPLEX_UNROLL_WORK (   func,
  func2,
  peel 
)
Value:
MICRO_COMPLEX_UNROLL(func2); \
func(0, peel) func(1, peel) func(2, peel) func(3, peel)
MICRO_COMPLEX_UNROLL
#define MICRO_COMPLEX_UNROLL(func)
Definition: MatrixProduct.h:2221
func
func(actual_m, actual_n, a, *lda, actual_b, 1, actual_c, 1, alpha)

◆ MICRO_COMPLEX_WORK_ONE4

#define MICRO_COMPLEX_WORK_ONE4 (   iter,
  peel 
)
Value:
if (unroll_factor > iter) { \
pgerc_common<accRows, Packet, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>( \
&accReal##iter, &accImag##iter, lhsV##iter, lhsVi##iter, rhsV##peel, rhsVi##peel); \
}

◆ MICRO_COMPLEX_WORK_PEEL

#define MICRO_COMPLEX_WORK_PEEL (   peel)
Value:
if (PEEL_COMPLEX_ROW > peel) { \
MICRO_COMPLEX_BROADCAST(peel) \
pgerc<accRows, Scalar, Packet, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>( \
&accReal##peel, &accImag##peel, lhs_ptr_real + (remaining_rows * peel), \
lhs_ptr_imag + (remaining_rows * peel), rhsV##peel, rhsVi##peel); \
} else { \
EIGEN_UNUSED_VARIABLE(rhsV##peel); \
EIGEN_UNUSED_VARIABLE(rhsVi##peel); \
}

◆ MICRO_COMPLEX_WORK_PEEL_ROW

#define MICRO_COMPLEX_WORK_PEEL_ROW
Value:
Packet rhsV0[4], rhsV1[4], rhsV2[4], rhsV3[4]; \
Packet rhsVi0[4], rhsVi1[4], rhsVi2[4], rhsVi3[4]; \
MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_WORK_PEEL) \
MICRO_COMPLEX_ADD_COLS(PEEL_COMPLEX_ROW) \
MICRO_COMPLEX_ADD_ROWS(PEEL_COMPLEX_ROW, false)
MICRO_COMPLEX_WORK_PEEL
#define MICRO_COMPLEX_WORK_PEEL(peel)
Definition: MatrixProduct.h:2273

◆ MICRO_COMPLEX_ZERO_PEEL

#define MICRO_COMPLEX_ZERO_PEEL (   peel)
Value:
if ((PEEL_COMPLEX_ROW > peel) && (peel != 0)) { \
bsetzero<Packet, accRows>(accReal##peel); \
bsetzero<Packet, accRows>(accImag##peel); \
} else { \
EIGEN_UNUSED_VARIABLE(accReal##peel); \
EIGEN_UNUSED_VARIABLE(accImag##peel); \
}

◆ MICRO_COMPLEX_ZERO_PEEL_ROW

#define MICRO_COMPLEX_ZERO_PEEL_ROW   MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_ZERO_PEEL)

◆ MICRO_DST_PTR

#define MICRO_DST_PTR   MICRO_UNROLL(MICRO_DST_PTR_ONE)

◆ MICRO_DST_PTR_ONE

#define MICRO_DST_PTR_ONE (   iter)
Value:
if (unroll_factor > iter) { \
bsetzero<Packet, accRows>(accZero##iter); \
} else { \
EIGEN_UNUSED_VARIABLE(accZero##iter); \
}

◆ MICRO_EXTRA

#define MICRO_EXTRA (   MICRO_EXTRA_UNROLL,
  value,
  is_col 
)
Value:
switch (value) { \
default: \
MICRO_EXTRA_UNROLL(1) \
break; \
case 2: \
if (is_col || (sizeof(Scalar) == sizeof(float))) { \
MICRO_EXTRA_UNROLL(2) \
} \
break; \
case 3: \
if (is_col || (sizeof(Scalar) == sizeof(float))) { \
MICRO_EXTRA_UNROLL(3) \
} \
break; \
}
Eigen::value
squared absolute value
Definition: GlobalFunctions.h:87

◆ MICRO_EXTRA_COLS

#define MICRO_EXTRA_COLS (   N)
Value:
gemm_cols<Scalar, Packet, DataMapper, N, accCols>(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, \
col, rows, remaining_rows, pAlpha, pMask);

◆ MICRO_EXTRA_ROWS

#define MICRO_EXTRA_ROWS (   N)
Value:
gemm_unrolled_row_iteration<Scalar, Packet, DataMapper, accRows, accCols, N>( \
res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, rows, pAlpha, pMask);

◆ MICRO_NEW_ROWS

#define MICRO_NEW_ROWS   ((MICRO_NORMAL_ROWS) ? accRows : 1)

◆ MICRO_NORMAL_ROWS

#define MICRO_NORMAL_ROWS   accRows == quad_traits<Scalar>::rows || accRows == 1

◆ MICRO_ONE4

#define MICRO_ONE4   MICRO_UNROLL_TYPE(MICRO_UNROLL_TYPE_ONE, 1)

◆ MICRO_ONE_PEEL4

#define MICRO_ONE_PEEL4   MICRO_UNROLL_TYPE(MICRO_UNROLL_TYPE_PEEL, PEEL)

◆ MICRO_PREFETCH

#define MICRO_PREFETCH   MICRO_UNROLL(MICRO_PREFETCH_ONE)

◆ MICRO_PREFETCHN

#define MICRO_PREFETCHN (   N)    MICRO_PREFETCHN1(ptr, N)

◆ MICRO_PREFETCHN1

#define MICRO_PREFETCHN1 (   ptr,
  N 
)
Value:
EIGEN_POWER_PREFETCH(MICRO_RHS(ptr, 0)); \
if (N == 2 || N == 3) { \
EIGEN_POWER_PREFETCH(MICRO_RHS(ptr, 1)); \
if (N == 3) { \
EIGEN_POWER_PREFETCH(MICRO_RHS(ptr, 2)); \
} \
}
EIGEN_POWER_PREFETCH
#define EIGEN_POWER_PREFETCH(p)
Definition: MatrixProductCommon.h:5

◆ MICRO_RHS

#define MICRO_RHS (   ptr,
  N 
)    rhs_##ptr##N

◆ MICRO_SRC2

#define MICRO_SRC2 (   ptr,
  N,
  M 
)
Value:
if (MICRO_NORMAL_ROWS) { \
EIGEN_UNUSED_VARIABLE(strideB); \
EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr, 1)); \
EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr, 2)); \
} else { \
MICRO_RHS(ptr, 1) = rhs_base + N + M; \
if (accRows == 3) { \
MICRO_RHS(ptr, 2) = rhs_base + N * 2 + M; \
} else { \
EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr, 2)); \
} \
}

◆ MICRO_SRC2_PTR

#define MICRO_SRC2_PTR   MICRO_SRC2(ptr, strideB, 0)

◆ MICRO_SRC_PTR

#define MICRO_SRC_PTR   MICRO_UNROLL(MICRO_SRC_PTR_ONE)

◆ MICRO_STORE

#define MICRO_STORE   MICRO_UNROLL(MICRO_STORE_ONE)

◆ MICRO_STORE_ONE

#define MICRO_STORE_ONE (   iter)
Value:
if (unroll_factor > iter) { \
bload<DataMapper, Packet, 0, ColMajor, false, accRows>(acc, res, row + iter * accCols, 0); \
bscale<Packet, accRows, !(MICRO_NORMAL(iter))>(acc, accZero##iter, pAlpha, pMask); \
bstore<DataMapper, Packet, accRows>(acc, res, row + iter * accCols); \
}

◆ MICRO_TYPE_PEEL4

#define MICRO_TYPE_PEEL4 (   func,
  func2,
  peel 
)
Value:
if (PEEL > peel) { \
Packet lhsV0, lhsV1, lhsV2, lhsV3, lhsV4, lhsV5, lhsV6, lhsV7; \
MICRO_BROADCAST(peel) \
MICRO_UNROLL_WORK(func, func2, peel) \
} else { \
EIGEN_UNUSED_VARIABLE(rhsV##peel); \
}
PEEL
#define PEEL
Definition: MatrixProduct.h:1790

◆ MICRO_UNROLL

#define MICRO_UNROLL (   func)    func(0) func(1) func(2) func(3) func(4) func(5) func(6) func(7)

◆ MICRO_UNROLL_ITER2

#define MICRO_UNROLL_ITER2 (   N,
  M 
)
Value:
gemm_unrolled_iteration<N + ((M) ? 1 : 0), Scalar, Packet, DataMapper, accRows, accCols, M ? M : accCols>( \
res3, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, pAlpha, pMask); \
if (M) return;

◆ MICRO_UNROLL_TYPE

#define MICRO_UNROLL_TYPE (   MICRO_TYPE,
  size 
)
Value:
MICRO_TYPE(4, MICRO_TYPE_PEEL4, MICRO_WORK_ONE, MICRO_LOAD_ONE) \
MICRO_ADD_ROWS(size)
MICRO_LOAD_ONE
#define MICRO_LOAD_ONE(iter)
Definition: MatrixProductCommon.h:154
MICRO_WORK_ONE
#define MICRO_WORK_ONE(iter, peel)
Definition: MatrixProduct.h:1992
MICRO_TYPE_PEEL4
#define MICRO_TYPE_PEEL4(func, func2, peel)
Definition: MatrixProduct.h:1997

◆ MICRO_UNROLL_TYPE_ONE

#define MICRO_UNROLL_TYPE_ONE (   M,
  func,
  func1,
  func2 
)
Value:
Packet rhsV0[M]; \
func(func1, func2, 0)

◆ MICRO_UNROLL_TYPE_PEEL

#define MICRO_UNROLL_TYPE_PEEL (   M,
  func,
  func1,
  func2 
)
Value:
Packet rhsV0[M], rhsV1[M], rhsV2[M], rhsV3[M], rhsV4[M], rhsV5[M], rhsV6[M], rhsV7[M]; \
func(func1, func2, 0) func(func1, func2, 1) func(func1, func2, 2) func(func1, func2, 3) func(func1, func2, 4) \
func(func1, func2, 5) func(func1, func2, 6) func(func1, func2, 7)

◆ MICRO_UNROLL_WORK

#define MICRO_UNROLL_WORK (   func,
  func2,
  peel 
)
Value:
MICRO_UNROLL(func2); \
func(0, peel) func(1, peel) func(2, peel) func(3, peel) func(4, peel) func(5, peel) func(6, peel) func(7, peel)
MICRO_UNROLL
#define MICRO_UNROLL(func)
Definition: MatrixProduct.h:1793

◆ MICRO_WORK_ONE

#define MICRO_WORK_ONE (   iter,
  peel 
)
Value:
if (unroll_factor > iter) { \
pger_common<Packet, false, accRows>(&accZero##iter, lhsV##iter, rhsV##peel); \
}

◆ MICRO_WORK_PEEL

#define MICRO_WORK_PEEL (   peel)
Value:
if (PEEL_ROW > peel) { \
MICRO_BROADCAST(peel) \
pger<accRows, Scalar, Packet, false>(&accZero##peel, lhs_ptr + (remaining_rows * peel), rhsV##peel); \
} else { \
EIGEN_UNUSED_VARIABLE(rhsV##peel); \
}

◆ MICRO_WORK_PEEL_ROW

#define MICRO_WORK_PEEL_ROW
Value:
Packet rhsV0[4], rhsV1[4], rhsV2[4], rhsV3[4], rhsV4[4], rhsV5[4], rhsV6[4], rhsV7[4]; \
MICRO_UNROLL(MICRO_WORK_PEEL) \
lhs_ptr += (remaining_rows * PEEL_ROW); \
MICRO_ADD_ROWS(PEEL_ROW)
MICRO_WORK_PEEL
#define MICRO_WORK_PEEL(peel)
Definition: MatrixProduct.h:1859

◆ MICRO_ZERO_PEEL

#define MICRO_ZERO_PEEL (   peel)
Value:
if ((PEEL_ROW > peel) && (peel != 0)) { \
bsetzero<Packet, accRows>(accZero##peel); \
} else { \
EIGEN_UNUSED_VARIABLE(accZero##peel); \
}

◆ MICRO_ZERO_PEEL_ROW

#define MICRO_ZERO_PEEL_ROW   MICRO_UNROLL(MICRO_ZERO_PEEL)

◆ PEEL

#define PEEL   7

◆ PEEL_COMPLEX

#define PEEL_COMPLEX   3

◆ PEEL_COMPLEX_ROW

#define PEEL_COMPLEX_ROW   3

◆ PEEL_ROW

#define PEEL_ROW   7

◆ USE_P10_AND_PVIPR2_0

#define USE_P10_AND_PVIPR2_0   0