Namespaces | Macros | Functions
MatrixProductCommon.h File Reference
#include "../../InternalHeaderCheck.h"

Go to the source code of this file.

Namespaces

 Eigen
 Namespace containing all symbols from the Eigen library.
 
 Eigen::internal
 Namespace containing low-level routines from the Eigen library.
 

Macros

#define EIGEN_POWER_PREFETCH(p)
 
#define MICRO_NORMAL(iter)   (accCols == accCols2) || (unroll_factor != (iter + 1))
 
#define MICRO_UNROLL_ITER1(func, N)
 
#define MICRO_UNROLL_ITER(func, N)   MICRO_UNROLL_ITER1(func, N)
 
#define MICRO_COMPLEX_UNROLL_ITER(func, N)   MICRO_UNROLL_ITER1(func, N)
 
#define MICRO_NORMAL_COLS(iter, a, b)   ((MICRO_NORMAL(iter)) ? a : b)
 
#define MICRO_LOAD1(lhs_ptr, iter)
 
#define MICRO_LOAD_ONE(iter)   MICRO_LOAD1(lhs_ptr, iter)
 
#define MICRO_COMPLEX_LOAD_ONE(iter)
 
#define MICRO_SRC_PTR1(lhs_ptr, advRows, iter)
 
#define MICRO_SRC_PTR_ONE(iter)   MICRO_SRC_PTR1(lhs_ptr, 1, iter)
 
#define MICRO_COMPLEX_SRC_PTR_ONE(iter)   MICRO_SRC_PTR1(lhs_ptr_real, advanceRows, iter)
 
#define MICRO_PREFETCH1(lhs_ptr, iter)
 
#define MICRO_PREFETCH_ONE(iter)   MICRO_PREFETCH1(lhs_ptr, iter)
 
#define MICRO_COMPLEX_PREFETCH_ONE(iter)   MICRO_PREFETCH1(lhs_ptr_real, iter)
 
#define MICRO_UPDATE_MASK   EIGEN_UNUSED_VARIABLE(pMask);
 
#define MICRO_UPDATE
 
#define MICRO_COMPLEX_UPDATE
 

Functions

template<typename Scalar , typename Packet , typename DataMapper , const Index accRows, const Index accCols>
EIGEN_ALWAYS_INLINE void Eigen::internal::gemm_extra_row (const DataMapper &res, const Scalar *lhs_base, const Scalar *rhs_base, Index depth, Index strideA, Index offsetA, Index strideB, Index row, Index rows, Index remaining_rows, const Packet &pAlpha, const Packet &pMask)
 
template<typename Scalar , typename Packet , typename DataMapper , const Index accCols>
EIGEN_ALWAYS_INLINE void Eigen::internal::gemm_extra_cols (const DataMapper &res, const Scalar *blockA, const Scalar *blockB, Index depth, Index strideA, Index offsetA, Index strideB, Index offsetB, Index col, Index rows, Index cols, Index remaining_rows, const Packet &pAlpha, const Packet &pMask)
 
template<typename Packet >
EIGEN_ALWAYS_INLINE Packet Eigen::internal::bmask (const Index remaining_rows)
 
template<typename Scalar , typename Packet , typename Packetc , typename DataMapper , const Index accRows, const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
EIGEN_ALWAYS_INLINE void Eigen::internal::gemm_complex_extra_row (const DataMapper &res, const Scalar *lhs_base, const Scalar *rhs_base, Index depth, Index strideA, Index offsetA, Index strideB, Index row, Index rows, Index remaining_rows, const Packet &pAlphaReal, const Packet &pAlphaImag, const Packet &pMask)
 
template<typename Scalar , typename Packet , typename Packetc , typename DataMapper , const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
EIGEN_ALWAYS_INLINE void Eigen::internal::gemm_complex_extra_cols (const DataMapper &res, const Scalar *blockA, const Scalar *blockB, Index depth, Index strideA, Index offsetA, Index strideB, Index offsetB, Index col, Index rows, Index cols, Index remaining_rows, const Packet &pAlphaReal, const Packet &pAlphaImag, const Packet &pMask)
 
template<typename DataMapper >
EIGEN_ALWAYS_INLINE void Eigen::internal::convertArrayBF16toF32 (float *result, Index cols, Index rows, const DataMapper &src)
 
template<const Index size, bool non_unit_stride, Index delta>
EIGEN_ALWAYS_INLINE void Eigen::internal::storeBF16fromResult (bfloat16 *dst, Packet8bf data, Index resInc, Index extra=0)
 
template<bool non_unit_stride>
EIGEN_ALWAYS_INLINE void Eigen::internal::convertArrayPointerBF16toF32 (float *result, Index cols, Index rows, bfloat16 *src, Index resInc)
 
template<bool rhsExtraCols, bool lhsExtraRows>
EIGEN_ALWAYS_INLINE void Eigen::internal::storeResults (Packet4f(&acc)[4], Index rows, const Packet4f pAlpha, float *result, Index extra_cols, Index extra_rows)
 
template<Index num_acc, bool extraRows, Index size = 4>
EIGEN_ALWAYS_INLINE void Eigen::internal::outputVecColResults (Packet4f(&acc)[num_acc][size], float *result, Packet4f pAlpha, Index extra_rows)
 
template<Index num_acc, Index size = 4>
EIGEN_ALWAYS_INLINE void Eigen::internal::outputVecResults (Packet4f(&acc)[num_acc][size], float *result, Packet4f pAlpha)
 
template<typename RhsMapper , bool linear>
EIGEN_ALWAYS_INLINE Packet8bf Eigen::internal::loadColData (RhsMapper &rhs, Index j)
 
template<typename Packet >
EIGEN_ALWAYS_INLINE Packet Eigen::internal::ploadLhs (const __UNPACK_TYPE__(Packet) *lhs)
 
template<typename DataMapper , typename Packet , const Index accCols, int StorageOrder, bool Complex, int N, bool full>
EIGEN_ALWAYS_INLINE void Eigen::internal::bload (PacketBlock< Packet, N *(Complex ? 2 :1)> &acc, const DataMapper &res, Index row, Index col)
 
template<typename DataMapper , typename Packet , int N>
EIGEN_ALWAYS_INLINE void Eigen::internal::bstore (PacketBlock< Packet, N > &acc, const DataMapper &res, Index row)
 
template<typename Packet , int N>
EIGEN_ALWAYS_INLINE void Eigen::internal::bscale (PacketBlock< Packet, N > &acc, PacketBlock< Packet, N > &accZ, const Packet &pAlpha)
 
template<typename Packet , int N, bool mask>
EIGEN_ALWAYS_INLINE void Eigen::internal::bscale (PacketBlock< Packet, N > &acc, PacketBlock< Packet, N > &accZ, const Packet &pAlpha, const Packet &pMask)
 
template<typename Packet , int N, bool mask>
EIGEN_ALWAYS_INLINE void Eigen::internal::bscalec (PacketBlock< Packet, N > &aReal, PacketBlock< Packet, N > &aImag, const Packet &bReal, const Packet &bImag, PacketBlock< Packet, N > &cReal, PacketBlock< Packet, N > &cImag, const Packet &pMask)
 
template<typename Packet , typename Packetc , int N, bool full>
EIGEN_ALWAYS_INLINE void Eigen::internal::bcouple (PacketBlock< Packet, N > &taccReal, PacketBlock< Packet, N > &taccImag, PacketBlock< Packetc, N *2 > &tRes, PacketBlock< Packetc, N > &acc1, PacketBlock< Packetc, N > &acc2)
 

Macro Definition Documentation

◆ EIGEN_POWER_PREFETCH

#define EIGEN_POWER_PREFETCH (   p)

◆ MICRO_COMPLEX_LOAD_ONE

#define MICRO_COMPLEX_LOAD_ONE (   iter)
Value:
if (!LhsIsReal && (unroll_factor > iter)) { \
lhsVi##iter = ploadLhs<Packet>(lhs_ptr_real##iter + MICRO_NORMAL_COLS(iter, imag_delta, imag_delta2)); \
} else { \
EIGEN_UNUSED_VARIABLE(lhsVi##iter); \
} \
MICRO_LOAD1(lhs_ptr_real, iter)
MICRO_NORMAL_COLS
#define MICRO_NORMAL_COLS(iter, a, b)
Definition: MatrixProductCommon.h:144

◆ MICRO_COMPLEX_PREFETCH_ONE

#define MICRO_COMPLEX_PREFETCH_ONE (   iter)    MICRO_PREFETCH1(lhs_ptr_real, iter)

◆ MICRO_COMPLEX_SRC_PTR_ONE

#define MICRO_COMPLEX_SRC_PTR_ONE (   iter)    MICRO_SRC_PTR1(lhs_ptr_real, advanceRows, iter)

◆ MICRO_COMPLEX_UNROLL_ITER

#define MICRO_COMPLEX_UNROLL_ITER (   func,
  N 
)    MICRO_UNROLL_ITER1(func, N)

◆ MICRO_COMPLEX_UPDATE

#define MICRO_COMPLEX_UPDATE
Value:
MICRO_UPDATE \
if (LhsIsReal || (accCols == accCols2)) { \
EIGEN_UNUSED_VARIABLE(imag_delta2); \
}
if
if(UPLO(*uplo)==INVALID) info
Definition: level3_impl.h:428

◆ MICRO_LOAD1

#define MICRO_LOAD1 (   lhs_ptr,
  iter 
)
Value:
if (unroll_factor > iter) { \
lhsV##iter = ploadLhs<Packet>(lhs_ptr##iter); \
lhs_ptr##iter += MICRO_NORMAL_COLS(iter, accCols, accCols2); \
} else { \
EIGEN_UNUSED_VARIABLE(lhsV##iter); \
}

◆ MICRO_LOAD_ONE

#define MICRO_LOAD_ONE (   iter)    MICRO_LOAD1(lhs_ptr, iter)

◆ MICRO_NORMAL

#define MICRO_NORMAL (   iter)    (accCols == accCols2) || (unroll_factor != (iter + 1))

◆ MICRO_NORMAL_COLS

#define MICRO_NORMAL_COLS (   iter,
  a,
  b 
)    ((MICRO_NORMAL(iter)) ? a : b)

◆ MICRO_PREFETCH1

#define MICRO_PREFETCH1 (   lhs_ptr,
  iter 
)
Value:
if (unroll_factor > iter) { \
EIGEN_POWER_PREFETCH(lhs_ptr##iter); \
}

◆ MICRO_PREFETCH_ONE

#define MICRO_PREFETCH_ONE (   iter)    MICRO_PREFETCH1(lhs_ptr, iter)

◆ MICRO_SRC_PTR1

#define MICRO_SRC_PTR1 (   lhs_ptr,
  advRows,
  iter 
)
Value:
if (unroll_factor > iter) { \
lhs_ptr##iter = lhs_base + (row + (iter * accCols)) * strideA * advRows - \
MICRO_NORMAL_COLS(iter, 0, (accCols - accCols2) * offsetA); \
} else { \
EIGEN_UNUSED_VARIABLE(lhs_ptr##iter); \
}
row
m row(1)

◆ MICRO_SRC_PTR_ONE

#define MICRO_SRC_PTR_ONE (   iter)    MICRO_SRC_PTR1(lhs_ptr, 1, iter)

◆ MICRO_UNROLL_ITER

#define MICRO_UNROLL_ITER (   func,
  N 
)    MICRO_UNROLL_ITER1(func, N)

◆ MICRO_UNROLL_ITER1

#define MICRO_UNROLL_ITER1 (   func,
  N 
)
Value:
switch (remaining_rows) { \
default: \
func(N, 0) break; \
case 1: \
func(N, 1) break; \
case 2: \
if (sizeof(Scalar) == sizeof(float)) { \
func(N, 2) \
} \
break; \
case 3: \
if (sizeof(Scalar) == sizeof(float)) { \
func(N, 3) \
} \
break; \
}
Scalar
SCALAR Scalar
Definition: bench_gemm.cpp:45
N
@ N
Definition: constructor.cpp:22
func
Definition: benchGeometry.cpp:21

◆ MICRO_UPDATE

#define MICRO_UPDATE
Value:
if (accCols == accCols2) { \
MICRO_UPDATE_MASK \
EIGEN_UNUSED_VARIABLE(offsetA); \
row += unroll_factor * accCols; \
}
EIGEN_UNUSED_VARIABLE
#define EIGEN_UNUSED_VARIABLE(var)
Definition: Macros.h:966

◆ MICRO_UPDATE_MASK

#define MICRO_UPDATE_MASK   EIGEN_UNUSED_VARIABLE(pMask);