Namespaces | Macros | Functions
MatrixProductMMA.h File Reference
#include "../../InternalHeaderCheck.h"
#include "MatrixProductMMAbfloat16.h"

Go to the source code of this file.

Namespaces

 Eigen
 Namespace containing all symbols from the Eigen library.
 
 Eigen::internal
 Namespace containing low-level routines from the Eigen library.
 

Macros

#define accColsC   (accCols / 2)
 
#define GEMM_MULTIPLE_COLS
 
#define PEEL_MMA   8
 
#define MICRO_MMA_UNROLL(func)   func(0) func(1) func(2) func(3) func(4) func(5) func(6) func(7)
 
#define MICRO_MMA_WORK(func, type, peel)
 
#define MICRO_MMA_WORK_ONE(iter, type, peel, left, right)
 
#define MICRO_MMA_UNROLL_ITER(func, val)
 
#define MICRO_MMA_LOAD_ONE_RHS1(peel, right)   ploadRhsMMA(rhs_ptr##right + (accRows * peel), rhsV##right[peel]);
 
#define MICRO_MMA_LOAD_ONE_RHS(peel)   MICRO_MMA_UNROLL_ITER(MICRO_MMA_LOAD_ONE_RHS1, peel)
 
#define MICRO_MMA_TYPE_PEEL(funcw, funcl, type, peel)
 
#define MICRO_MMA_UNROLL_TYPE_PEEL(funcw, funcl, type)
 
#define MICRO_MMA_UNROLL_TYPE_ONE(funcw, funcl, type)
 
#define MICRO_MMA_UPDATE_RHS1(size, right)   rhs_ptr##right += (accRows * size);
 
#define MICRO_MMA_UPDATE_RHS(size)   MICRO_MMA_UNROLL_ITER(MICRO_MMA_UPDATE_RHS1, size)
 
#define MICRO_MMA_UNROLL_TYPE(MICRO_MMA_TYPE, size)
 
#define MICRO_MMA_ONE_PEEL   MICRO_MMA_UNROLL_TYPE(MICRO_MMA_UNROLL_TYPE_PEEL, PEEL_MMA)
 
#define MICRO_MMA_ONE   MICRO_MMA_UNROLL_TYPE(MICRO_MMA_UNROLL_TYPE_ONE, 1)
 
#define MICRO_MMA_DST_PTR_ONE(iter)
 
#define MICRO_MMA_DST_PTR   MICRO_MMA_UNROLL(MICRO_MMA_DST_PTR_ONE)
 
#define MICRO_MMA_SRC_PTR   MICRO_MMA_UNROLL(MICRO_SRC_PTR_ONE)
 
#define MICRO_MMA_PREFETCH   MICRO_MMA_UNROLL(MICRO_PREFETCH_ONE)
 
#define MICRO_MMA_STORE_ONE(iter, left, right)
 
#define MICRO_MMA_ITER_UNROLL(func)
 
#define MICRO_MMA_STORE   MICRO_MMA_ITER_UNROLL(MICRO_MMA_STORE_ONE)
 
#define MICRO_MMA_EXTRA_ROWS(right)
 
#define MICRO_MMA_EXTRA_ROWS1(val, right)   MICRO_MMA_EXTRA_ROWS(right);
 
#define MICRO_MMA_UNROLL_ITER2(N, M)
 
#define MICRO_MMA_ROWS(n)
 
#define MAX_MMA_UNROLL   7
 
#define MICRO_MMA_COLS(n)
 
#define advanceRows   ((LhsIsReal) ? 1 : 2)
 
#define advanceCols   ((RhsIsReal) ? 1 : 2)
 
#define PEEL_COMPLEX_MMA   4
 
#define MICRO_COMPLEX_MMA_UNROLL(func)   func(0) func(1) func(2) func(3)
 
#define MICRO_COMPLEX_MMA_WORK(func, type, peel)
 
#define MICRO_COMPLEX_MMA_WORK_ONE(iter, type, peel, left, right)
 
#define MICRO_COMPLEX_MMA_LOAD_RHS1(peel, right)
 
#define MICRO_COMPLEX_MMA_LOAD_ONE_RHS(peel)   MICRO_MMA_UNROLL_ITER(MICRO_COMPLEX_MMA_LOAD_RHS1, peel)
 
#define MICRO_COMPLEX_MMA_TYPE_PEEL(funcw, funcl, type, peel)
 
#define MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL(funcw, funcl, type)
 
#define MICRO_COMPLEX_MMA_UNROLL_TYPE_ONE(funcw, funcl, type)
 
#define MICRO_COMPLEX_MMA_UPDATE_RHS1(size, right)
 
#define MICRO_COMPLEX_MMA_UPDATE_RHS(size)   MICRO_MMA_UNROLL_ITER(MICRO_COMPLEX_MMA_UPDATE_RHS1, size)
 
#define MICRO_COMPLEX_MMA_UNROLL_TYPE(MICRO_COMPLEX_MMA_TYPE, size)
 
#define MICRO_COMPLEX_MMA_ONE_PEEL   MICRO_COMPLEX_MMA_UNROLL_TYPE(MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL, PEEL_COMPLEX_MMA)
 
#define MICRO_COMPLEX_MMA_ONE   MICRO_COMPLEX_MMA_UNROLL_TYPE(MICRO_COMPLEX_MMA_UNROLL_TYPE_ONE, 1)
 
#define MICRO_COMPLEX_MMA_DST_PTR_ONE(iter)
 
#define MICRO_COMPLEX_MMA_DST_PTR   MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_MMA_DST_PTR_ONE)
 
#define MICRO_COMPLEX_MMA_SRC_PTR   MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_SRC_PTR_ONE)
 
#define MICRO_COMPLEX_MMA_PREFETCH   MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_PREFETCH_ONE)
 
#define MICRO_COMPLEX_MMA_STORE_ONE(iter, left, right)
 
#define MICRO_COMPLEX_MMA_ITER_UNROLL(func)
 
#define MICRO_COMPLEX_MMA_STORE   MICRO_COMPLEX_MMA_ITER_UNROLL(MICRO_COMPLEX_MMA_STORE_ONE)
 
#define MICRO_COMPLEX_MMA_EXTRA_ROWS(right)
 
#define MICRO_COMPLEX_MMA_EXTRA_ROWS1(val, right)   MICRO_COMPLEX_MMA_EXTRA_ROWS(right);
 
#define MICRO_COMPLEX_MMA_UNROLL_ITER2(N, M)
 
#define MICRO_COMPLEX_MMA_ROWS(n)
 
#define MAX_COMPLEX_MMA_UNROLL   4
 
#define MICRO_COMPLEX_MMA_COLS(n)
 

Functions

EIGEN_ALWAYS_INLINE void Eigen::internal::bsetzeroMMA (__vector_quad *acc)
 
template<typename DataMapper , typename Packet , bool full>
EIGEN_ALWAYS_INLINE void Eigen::internal::storeAccumulator (Index i, const DataMapper &data, const Packet &alpha, const Index elements, __vector_quad *acc)
 
template<typename DataMapper , typename Packet , typename Packetc , const Index accCols, const Index accCols2>
EIGEN_ALWAYS_INLINE void Eigen::internal::storeComplexAccumulator (Index i, const DataMapper &data, const Packet &alphaReal, const Packet &alphaImag, const Packet &pMask, __vector_quad *accReal, __vector_quad *accImag)
 
template<typename LhsPacket , typename RhsPacket , bool NegativeAccumulate>
EIGEN_ALWAYS_INLINE void Eigen::internal::pgerMMA (__vector_quad *acc, const RhsPacket &a, const LhsPacket &b)
 
template<typename LhsPacket , typename RhsPacket , bool NegativeAccumulate>
EIGEN_ALWAYS_INLINE void Eigen::internal::pgerMMA (__vector_quad *acc, const __vector_pair &a, const Packet2d &b)
 
template<typename Packet , typename RhsPacket , bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
EIGEN_ALWAYS_INLINE void Eigen::internal::pgercMMA (__vector_quad *accReal, __vector_quad *accImag, const Packet &lhsV, Packet &lhsVi, const RhsPacket &rhsV, RhsPacket &rhsVi)
 
template<typename Packet >
EIGEN_ALWAYS_INLINE Packet Eigen::internal::ploadRhs (const __UNPACK_TYPE__(Packet) *rhs)
 
template<typename Scalar , typename Packet >
EIGEN_ALWAYS_INLINE void Eigen::internal::ploadRhsMMA (const Scalar *rhs, Packet &rhsV)
 
template<>
EIGEN_ALWAYS_INLINE void Eigen::internal::ploadRhsMMA (const double *rhs, __vector_pair &rhsV)
 
EIGEN_ALWAYS_INLINE void Eigen::internal::ploadLhsMMA (const double *lhs, __vector_pair &lhsV)
 
template<int unroll_factor, typename Scalar , typename Packet , typename RhsPacket , typename DataMapper , const Index accRows, const Index accCols, bool full, const Index accItr>
EIGEN_ALWAYS_INLINE void Eigen::internal::gemm_unrolled_MMA_iteration (const DataMapper &res0, const DataMapper &res1, const DataMapper &res2, const DataMapper &res3, const Scalar *lhs_base, const Scalar *rhs_base, Index depth, Index strideA, Index strideB, Index offsetA, Index &row, const Packet &pAlpha, Index accCols2)
 
template<typename Scalar , typename Packet , typename RhsPacket , typename DataMapper , const Index accRows, const Index accCols, const Index accItr>
EIGEN_ALWAYS_INLINE void Eigen::internal::gemmMMA_cols (const DataMapper &res, const Scalar *blockA, const Scalar *blockB, Index depth, Index strideA, Index offsetA, Index strideB, Index offsetB, Index col, Index rows, Index remaining_rows, const Packet &pAlpha, const Packet &pMask)
 
template<typename Scalar , typename Packet , typename RhsPacket , typename DataMapper , const Index accRows, const Index accCols>
void Eigen::internal::gemmMMA (const DataMapper &res, const Scalar *blockA, const Scalar *blockB, Index rows, Index depth, Index cols, Scalar alpha, Index strideA, Index strideB, Index offsetA, Index offsetB)
 
template<int unroll_factor, typename Scalar , typename Packet , typename Packetc , typename RhsPacket , typename DataMapper , const Index accRows, const Index accCols, const Index accCols2, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal, const Index accItr>
EIGEN_ALWAYS_INLINE void Eigen::internal::gemm_complex_unrolled_MMA_iteration (const DataMapper &res0, const DataMapper &res1, const DataMapper &res2, const DataMapper &res3, const Scalar *lhs_base, const Scalar *rhs_base, Index depth, Index strideA, Index offsetA, Index strideB, Index &row, const Packet &pAlphaReal, const Packet &pAlphaImag, const Packet &pMask)
 
template<typename Scalar , typename Packet , typename Packetc , typename RhsPacket , typename DataMapper , const Index accRows, const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal, const Index accItr>
EIGEN_ALWAYS_INLINE void Eigen::internal::gemmMMA_complex_cols (const DataMapper &res, const Scalar *blockA, const Scalar *blockB, Index depth, Index strideA, Index offsetA, Index strideB, Index offsetB, Index col, Index rows, Index remaining_rows, const Packet &pAlphaReal, const Packet &pAlphaImag, const Packet &pMask)
 
template<typename LhsScalar , typename RhsScalar , typename Scalarc , typename Scalar , typename Packet , typename Packetc , typename RhsPacket , typename DataMapper , const Index accRows, const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
void Eigen::internal::gemm_complexMMA (const DataMapper &res, const LhsScalar *blockAc, const RhsScalar *blockBc, Index rows, Index depth, Index cols, Scalarc alpha, Index strideA, Index strideB, Index offsetA, Index offsetB)
 

Macro Definition Documentation

◆ accColsC

#define accColsC   (accCols / 2)

◆ advanceCols

#define advanceCols   ((RhsIsReal) ? 1 : 2)

◆ advanceRows

#define advanceRows   ((LhsIsReal) ? 1 : 2)

◆ GEMM_MULTIPLE_COLS

#define GEMM_MULTIPLE_COLS

◆ MAX_COMPLEX_MMA_UNROLL

#define MAX_COMPLEX_MMA_UNROLL   4

◆ MAX_MMA_UNROLL

#define MAX_MMA_UNROLL   7

◆ MICRO_COMPLEX_MMA_COLS

#define MICRO_COMPLEX_MMA_COLS (   n)
Value:
for (; col + n * accRows <= cols; col += n * accRows) { \
gemmMMA_complex_cols<Scalar, Packet, Packetc, RhsPacket2, DataMapper, accRows, accCols, ConjugateLhs, \
ConjugateRhs, LhsIsReal, RhsIsReal, n>(res, blockA, blockB, depth, strideA, offsetA, strideB, \
offsetB, col, rows, remaining_rows, pAlphaReal, \
pAlphaImag, pMask); \
}
n
const unsigned n
Definition: CG3DPackingUnitTest.cpp:11
col
m col(1)
res
cout<< "Here is the matrix m:"<< endl<< m<< endl;Matrix< ptrdiff_t, 3, 1 > res
Definition: PartialRedux_count.cpp:3
rows
int rows
Definition: Tutorial_commainit_02.cpp:1
cols
int cols
Definition: Tutorial_commainit_02.cpp:1
Scalar
SCALAR Scalar
Definition: bench_gemm.cpp:45
Packet
internal::packet_traits< Scalar >::type Packet
Definition: benchmark-blocking-sizes.cpp:54

◆ MICRO_COMPLEX_MMA_DST_PTR

#define MICRO_COMPLEX_MMA_DST_PTR   MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_MMA_DST_PTR_ONE)

◆ MICRO_COMPLEX_MMA_DST_PTR_ONE

#define MICRO_COMPLEX_MMA_DST_PTR_ONE (   iter)
Value:
if (unroll_factor * accItr > iter) { \
bsetzeroMMA(&accReal##iter); \
bsetzeroMMA(&accImag##iter); \
} else { \
EIGEN_UNUSED_VARIABLE(accReal##iter); \
EIGEN_UNUSED_VARIABLE(accImag##iter); \
}

◆ MICRO_COMPLEX_MMA_EXTRA_ROWS

#define MICRO_COMPLEX_MMA_EXTRA_ROWS (   right)
Value:
gemm_complex_extra_row<Scalar, Packet, Packetc, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, \
RhsIsReal>(res3##right, blockA, rhs_base + right * accRows * (RhsIsReal ? 1 : 2) * strideB, \
depth, strideA, offsetA, strideB, row, rows, remaining_rows, pAlphaReal, \
pAlphaImag, pMask);
row
m row(1)
Eigen::internal::gemm_complex_extra_row
EIGEN_ALWAYS_INLINE void gemm_complex_extra_row(const DataMapper &res, const Scalar *lhs_base, const Scalar *rhs_base, Index depth, Index strideA, Index offsetA, Index strideB, Index row, Index rows, Index remaining_rows, const Packet &pAlphaReal, const Packet &pAlphaImag, const Packet &pMask)
Definition: MatrixProduct.h:2404

◆ MICRO_COMPLEX_MMA_EXTRA_ROWS1

#define MICRO_COMPLEX_MMA_EXTRA_ROWS1 (   val,
  right 
)    MICRO_COMPLEX_MMA_EXTRA_ROWS(right);

◆ MICRO_COMPLEX_MMA_ITER_UNROLL

#define MICRO_COMPLEX_MMA_ITER_UNROLL (   func)
Value:
if (accItr == 1) { \
func(0, 0, 0) func(1, 1, 0) func(2, 2, 0) func(3, 3, 0) \
} else if (accItr == 2) { \
func(0, 0, 0) func(1, 0, 1) func(2, 1, 0) func(3, 1, 1) \
} else { \
func(0, 0, 0) func(1, 0, 1) func(2, 0, 2) func(3, 0, 3) \
}
if
if(UPLO(*uplo)==INVALID) info
Definition: level3_impl.h:428
else
else
Definition: level3_impl.h:229
func
Definition: benchGeometry.cpp:21

◆ MICRO_COMPLEX_MMA_LOAD_ONE_RHS

#define MICRO_COMPLEX_MMA_LOAD_ONE_RHS (   peel)    MICRO_MMA_UNROLL_ITER(MICRO_COMPLEX_MMA_LOAD_RHS1, peel)

◆ MICRO_COMPLEX_MMA_LOAD_RHS1

#define MICRO_COMPLEX_MMA_LOAD_RHS1 (   peel,
  right 
)
Value:
ploadRhsMMA(rhs_ptr_real##right + (accRows * peel), rhsV##right[peel]); \
if (!RhsIsReal) { \
ploadRhsMMA(rhs_ptr_imag##right + (accRows * peel), rhsVi##right[peel]); \
}
Eigen::internal::ploadRhsMMA
EIGEN_ALWAYS_INLINE void ploadRhsMMA(const double *rhs, __vector_pair &rhsV)
Definition: MatrixProductMMA.h:134

◆ MICRO_COMPLEX_MMA_ONE

#define MICRO_COMPLEX_MMA_ONE   MICRO_COMPLEX_MMA_UNROLL_TYPE(MICRO_COMPLEX_MMA_UNROLL_TYPE_ONE, 1)

◆ MICRO_COMPLEX_MMA_ONE_PEEL

#define MICRO_COMPLEX_MMA_ONE_PEEL   MICRO_COMPLEX_MMA_UNROLL_TYPE(MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL, PEEL_COMPLEX_MMA)

◆ MICRO_COMPLEX_MMA_PREFETCH

#define MICRO_COMPLEX_MMA_PREFETCH   MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_PREFETCH_ONE)

◆ MICRO_COMPLEX_MMA_ROWS

#define MICRO_COMPLEX_MMA_ROWS (   n)
Value:
while (row + n * accCols <= rows) { \
MICRO_COMPLEX_MMA_UNROLL_ITER2(n, 0); \
}

◆ MICRO_COMPLEX_MMA_SRC_PTR

#define MICRO_COMPLEX_MMA_SRC_PTR   MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_SRC_PTR_ONE)

◆ MICRO_COMPLEX_MMA_STORE

#define MICRO_COMPLEX_MMA_STORE   MICRO_COMPLEX_MMA_ITER_UNROLL(MICRO_COMPLEX_MMA_STORE_ONE)

◆ MICRO_COMPLEX_MMA_STORE_ONE

#define MICRO_COMPLEX_MMA_STORE_ONE (   iter,
  left,
  right 
)
Value:
if (unroll_factor > left) { \
storeComplexAccumulator<DataMapper, Packet, Packetc, accCols, (unroll_factor != (left + 1)) ? accCols : accCols2>( \
row + left * accCols, res##right, pAlphaReal, pAlphaImag, pMask, &accReal##iter, &accImag##iter); \
}

◆ MICRO_COMPLEX_MMA_TYPE_PEEL

#define MICRO_COMPLEX_MMA_TYPE_PEEL (   funcw,
  funcl,
  type,
  peel 
)
Value:
if (PEEL_COMPLEX_MMA > peel) { \
Packet lhsV0, lhsV1, lhsV2, lhsV3; \
Packet lhsVi0, lhsVi1, lhsVi2, lhsVi3; \
MICRO_COMPLEX_MMA_LOAD_ONE_RHS(peel) \
MICRO_COMPLEX_MMA_UNROLL(funcl) \
MICRO_COMPLEX_MMA_WORK(funcw, type, peel) \
}
PEEL_COMPLEX_MMA
#define PEEL_COMPLEX_MMA
Definition: MatrixProductMMA.h:514
compute_granudrum_aor.type
type
Definition: compute_granudrum_aor.py:141

◆ MICRO_COMPLEX_MMA_UNROLL

#define MICRO_COMPLEX_MMA_UNROLL (   func)    func(0) func(1) func(2) func(3)

◆ MICRO_COMPLEX_MMA_UNROLL_ITER2

#define MICRO_COMPLEX_MMA_UNROLL_ITER2 (   N,
  M 
)
Value:
gemm_complex_unrolled_MMA_iteration<N + (M ? 1 : 0), Scalar, Packet, Packetc, RhsPacket, DataMapper, accRows, \
accCols, M ? M : accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal, \
accItr>(res30, res31, res32, res33, lhs_base, rhs_base, depth, strideA, offsetA, \
strideB, row, pAlphaReal, pAlphaImag, pMask); \
if (M) return;
Eigen::Matrix
The matrix class, also used for vectors and row-vectors.
Definition: Eigen/Eigen/src/Core/Matrix.h:186
N
@ N
Definition: constructor.cpp:22
Eigen::internal::gemm_complex_unrolled_MMA_iteration
EIGEN_ALWAYS_INLINE void gemm_complex_unrolled_MMA_iteration(const DataMapper &res0, const DataMapper &res1, const DataMapper &res2, const DataMapper &res3, const Scalar *lhs_base, const Scalar *rhs_base, Index depth, Index strideA, Index offsetA, Index strideB, Index &row, const Packet &pAlphaReal, const Packet &pAlphaImag, const Packet &pMask)
Definition: MatrixProductMMA.h:700

◆ MICRO_COMPLEX_MMA_UNROLL_TYPE

#define MICRO_COMPLEX_MMA_UNROLL_TYPE (   MICRO_COMPLEX_MMA_TYPE,
  size 
)
Value:
MICRO_COMPLEX_MMA_TYPE(MICRO_COMPLEX_MMA_WORK_ONE, MICRO_COMPLEX_LOAD_ONE, RhsPacket) \
MICRO_COMPLEX_MMA_UPDATE_RHS(size);
MICRO_COMPLEX_LOAD_ONE
#define MICRO_COMPLEX_LOAD_ONE(iter)
Definition: MatrixProductCommon.h:156
MICRO_COMPLEX_MMA_WORK_ONE
#define MICRO_COMPLEX_MMA_WORK_ONE(iter, type, peel, left, right)
Definition: MatrixProductMMA.h:530
size
Scalar Scalar int size
Definition: benchVecAdd.cpp:17

◆ MICRO_COMPLEX_MMA_UNROLL_TYPE_ONE

#define MICRO_COMPLEX_MMA_UNROLL_TYPE_ONE (   funcw,
  funcl,
  type 
)
Value:
type rhsV0[1], rhsVi0[1], rhsV1[1], rhsVi1[1], rhsV2[1], rhsVi2[1], rhsV3[1], rhsVi3[1]; \
MICRO_COMPLEX_MMA_TYPE_PEEL(funcw, funcl, type, 0)

◆ MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL

#define MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL (   funcw,
  funcl,
  type 
)
Value:
type rhsV0[4], rhsVi0[4], rhsV1[(accItr > 1) ? 4 : 1], rhsVi1[(accItr > 1) ? 4 : 1], rhsV2[(accItr > 2) ? 4 : 1], \
rhsVi2[(accItr > 2) ? 4 : 1], rhsV3[(accItr > 2) ? 4 : 1], rhsVi3[(accItr > 2) ? 4 : 1]; \
MICRO_COMPLEX_MMA_TYPE_PEEL(funcw, funcl, type, 0) \
MICRO_COMPLEX_MMA_TYPE_PEEL(funcw, funcl, type, 1) \
MICRO_COMPLEX_MMA_TYPE_PEEL(funcw, funcl, type, 2) MICRO_COMPLEX_MMA_TYPE_PEEL(funcw, funcl, type, 3)
MICRO_COMPLEX_MMA_TYPE_PEEL
#define MICRO_COMPLEX_MMA_TYPE_PEEL(funcw, funcl, type, peel)
Definition: MatrixProductMMA.h:571

◆ MICRO_COMPLEX_MMA_UPDATE_RHS

#define MICRO_COMPLEX_MMA_UPDATE_RHS (   size)    MICRO_MMA_UNROLL_ITER(MICRO_COMPLEX_MMA_UPDATE_RHS1, size)

◆ MICRO_COMPLEX_MMA_UPDATE_RHS1

#define MICRO_COMPLEX_MMA_UPDATE_RHS1 (   size,
  right 
)
Value:
rhs_ptr_real##right += (accRows * size); \
if (!RhsIsReal) rhs_ptr_imag##right += (accRows * size);

◆ MICRO_COMPLEX_MMA_WORK

#define MICRO_COMPLEX_MMA_WORK (   func,
  type,
  peel 
)
Value:
if (accItr == 1) { \
func(0, type, peel, 0, 0) func(1, type, peel, 1, 0) func(2, type, peel, 2, 0) func(3, type, peel, 3, 0) \
} else if (accItr == 2) { \
func(0, type, peel, 0, 0) func(1, type, peel, 0, 1) func(2, type, peel, 1, 0) func(3, type, peel, 1, 1) \
} else { \
func(0, type, peel, 0, 0) func(1, type, peel, 0, 1) func(2, type, peel, 0, 2) func(3, type, peel, 0, 3) \
}

◆ MICRO_COMPLEX_MMA_WORK_ONE

#define MICRO_COMPLEX_MMA_WORK_ONE (   iter,
  type,
  peel,
  left,
  right 
)
Value:
if (unroll_factor > left) { \
pgercMMA<Packet, type, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>( \
&accReal##iter, &accImag##iter, lhsV##left, lhsVi##left, rhsV##right[peel], rhsVi##right[peel]); \
}

◆ MICRO_MMA_COLS

#define MICRO_MMA_COLS (   n)
Value:
for (; col + n * accRows <= cols; col += n * accRows) { \
gemmMMA_cols<Scalar, Packet, RhsPacket2, DataMapper, accRows, accCols, n>( \
res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, remaining_rows, pAlpha, pMask); \
}

◆ MICRO_MMA_DST_PTR

#define MICRO_MMA_DST_PTR   MICRO_MMA_UNROLL(MICRO_MMA_DST_PTR_ONE)

◆ MICRO_MMA_DST_PTR_ONE

#define MICRO_MMA_DST_PTR_ONE (   iter)
Value:
if (unroll_factor * accItr > iter) { \
bsetzeroMMA(&accZero##iter); \
} else { \
EIGEN_UNUSED_VARIABLE(accZero##iter); \
}

◆ MICRO_MMA_EXTRA_ROWS

#define MICRO_MMA_EXTRA_ROWS (   right)
Value:
gemm_extra_row<Scalar, Packet, DataMapper, accRows, accCols>( \
res3##right, blockA, rhs_base + right * accRows * strideB, depth, strideA, offsetA, strideB, row, rows, \
remaining_rows, pAlpha, pMask);

◆ MICRO_MMA_EXTRA_ROWS1

#define MICRO_MMA_EXTRA_ROWS1 (   val,
  right 
)    MICRO_MMA_EXTRA_ROWS(right);

◆ MICRO_MMA_ITER_UNROLL

#define MICRO_MMA_ITER_UNROLL (   func)
Value:
if (accItr == 1) { \
func(0, 0, 0) func(1, 1, 0) func(2, 2, 0) func(3, 3, 0) func(4, 4, 0) func(5, 5, 0) func(6, 6, 0) func(7, 7, 0) \
} else if (accItr == 2) { \
func(0, 0, 0) func(1, 0, 1) func(2, 1, 0) func(3, 1, 1) func(4, 2, 0) func(5, 2, 1) func(6, 3, 0) func(7, 3, 1) \
} else { \
func(0, 0, 0) func(1, 0, 1) func(2, 0, 2) func(3, 0, 3) func(4, 1, 0) func(5, 1, 1) func(6, 1, 2) func(7, 1, 3) \
}

◆ MICRO_MMA_LOAD_ONE_RHS

#define MICRO_MMA_LOAD_ONE_RHS (   peel)    MICRO_MMA_UNROLL_ITER(MICRO_MMA_LOAD_ONE_RHS1, peel)

◆ MICRO_MMA_LOAD_ONE_RHS1

#define MICRO_MMA_LOAD_ONE_RHS1 (   peel,
  right 
)    ploadRhsMMA(rhs_ptr##right + (accRows * peel), rhsV##right[peel]);

◆ MICRO_MMA_ONE

#define MICRO_MMA_ONE   MICRO_MMA_UNROLL_TYPE(MICRO_MMA_UNROLL_TYPE_ONE, 1)

◆ MICRO_MMA_ONE_PEEL

#define MICRO_MMA_ONE_PEEL   MICRO_MMA_UNROLL_TYPE(MICRO_MMA_UNROLL_TYPE_PEEL, PEEL_MMA)

◆ MICRO_MMA_PREFETCH

#define MICRO_MMA_PREFETCH   MICRO_MMA_UNROLL(MICRO_PREFETCH_ONE)

◆ MICRO_MMA_ROWS

#define MICRO_MMA_ROWS (   n)
Value:
while (row + n * accCols <= rows) { \
MICRO_MMA_UNROLL_ITER2(n, 0); \
}

◆ MICRO_MMA_SRC_PTR

#define MICRO_MMA_SRC_PTR   MICRO_MMA_UNROLL(MICRO_SRC_PTR_ONE)

◆ MICRO_MMA_STORE

#define MICRO_MMA_STORE   MICRO_MMA_ITER_UNROLL(MICRO_MMA_STORE_ONE)

◆ MICRO_MMA_STORE_ONE

#define MICRO_MMA_STORE_ONE (   iter,
  left,
  right 
)
Value:
if (unroll_factor > left) { \
storeAccumulator<DataMapper, Packet, MICRO_NORMAL_PARTIAL(left)>(row + left * accCols, res##right, pAlpha, \
accCols2, &accZero##iter); \
}

◆ MICRO_MMA_TYPE_PEEL

#define MICRO_MMA_TYPE_PEEL (   funcw,
  funcl,
  type,
  peel 
)
Value:
if (PEEL_MMA > peel) { \
Packet lhsV0, lhsV1, lhsV2, lhsV3, lhsV4, lhsV5, lhsV6, lhsV7; \
MICRO_MMA_LOAD_ONE_RHS(peel) \
MICRO_MMA_UNROLL(funcl) \
MICRO_MMA_WORK(funcw, type, peel) \
}
PEEL_MMA
#define PEEL_MMA
Definition: MatrixProductMMA.h:156

◆ MICRO_MMA_UNROLL

#define MICRO_MMA_UNROLL (   func)    func(0) func(1) func(2) func(3) func(4) func(5) func(6) func(7)

◆ MICRO_MMA_UNROLL_ITER

#define MICRO_MMA_UNROLL_ITER (   func,
  val 
)
Value:
func(val, 0) if (accItr > 1) { \
func(val, 1) if (accItr > 2) { func(val, 2) func(val, 3) } \
}
func
func(actual_m, actual_n, a, *lda, actual_b, 1, actual_c, 1, alpha)
calibrate.val
val
Definition: calibrate.py:119

◆ MICRO_MMA_UNROLL_ITER2

#define MICRO_MMA_UNROLL_ITER2 (   N,
  M 
)
Value:
gemm_unrolled_MMA_iteration<N + (M ? 1 : 0), Scalar, Packet, RhsPacket, DataMapper, accRows, accCols, !M, accItr>( \
res30, res31, res32, res33, lhs_base, rhs_base, depth, strideA, strideB, offsetA, row, pAlpha, \
M ? remaining_rows : accCols); \
if (M) return;

◆ MICRO_MMA_UNROLL_TYPE

#define MICRO_MMA_UNROLL_TYPE (   MICRO_MMA_TYPE,
  size 
)
Value:
MICRO_MMA_TYPE(MICRO_MMA_WORK_ONE, MICRO_LOAD_ONE, RhsPacket) \
MICRO_MMA_UPDATE_RHS(size)
MICRO_LOAD_ONE
#define MICRO_LOAD_ONE(iter)
Definition: MatrixProductCommon.h:154
MICRO_MMA_WORK_ONE
#define MICRO_MMA_WORK_ONE(iter, type, peel, left, right)
Definition: MatrixProductMMA.h:180

◆ MICRO_MMA_UNROLL_TYPE_ONE

#define MICRO_MMA_UNROLL_TYPE_ONE (   funcw,
  funcl,
  type 
)
Value:
type rhsV0[1], rhsV1[1], rhsV2[1], rhsV3[1]; \
MICRO_MMA_TYPE_PEEL(funcw, funcl, type, 0)

◆ MICRO_MMA_UNROLL_TYPE_PEEL

#define MICRO_MMA_UNROLL_TYPE_PEEL (   funcw,
  funcl,
  type 
)
Value:
type rhsV0[8], rhsV1[(accItr > 1) ? 8 : 1], rhsV2[(accItr > 2) ? 8 : 1], rhsV3[(accItr > 2) ? 8 : 1]; \
MICRO_MMA_TYPE_PEEL(funcw, funcl, type, 0) \
MICRO_MMA_TYPE_PEEL(funcw, funcl, type, 1) \
MICRO_MMA_TYPE_PEEL(funcw, funcl, type, 2) \
MICRO_MMA_TYPE_PEEL(funcw, funcl, type, 3) \
MICRO_MMA_TYPE_PEEL(funcw, funcl, type, 4) \
MICRO_MMA_TYPE_PEEL(funcw, funcl, type, 5) \
MICRO_MMA_TYPE_PEEL(funcw, funcl, type, 6) MICRO_MMA_TYPE_PEEL(funcw, funcl, type, 7)
MICRO_MMA_TYPE_PEEL
#define MICRO_MMA_TYPE_PEEL(funcw, funcl, type, peel)
Definition: MatrixProductMMA.h:220

◆ MICRO_MMA_UPDATE_RHS

#define MICRO_MMA_UPDATE_RHS (   size)    MICRO_MMA_UNROLL_ITER(MICRO_MMA_UPDATE_RHS1, size)

◆ MICRO_MMA_UPDATE_RHS1

#define MICRO_MMA_UPDATE_RHS1 (   size,
  right 
)    rhs_ptr##right += (accRows * size);

◆ MICRO_MMA_WORK

#define MICRO_MMA_WORK (   func,
  type,
  peel 
)
Value:
if (accItr == 1) { \
func(0, type, peel, 0, 0) func(1, type, peel, 1, 0) func(2, type, peel, 2, 0) func(3, type, peel, 3, 0) \
func(4, type, peel, 4, 0) func(5, type, peel, 5, 0) func(6, type, peel, 6, 0) func(7, type, peel, 7, 0) \
} else if (accItr == 2) { \
func(0, type, peel, 0, 0) func(1, type, peel, 0, 1) func(2, type, peel, 1, 0) func(3, type, peel, 1, 1) \
func(4, type, peel, 2, 0) func(5, type, peel, 2, 1) func(6, type, peel, 3, 0) func(7, type, peel, 3, 1) \
} else { \
func(0, type, peel, 0, 0) func(1, type, peel, 0, 1) func(2, type, peel, 0, 2) func(3, type, peel, 0, 3) \
func(4, type, peel, 1, 0) func(5, type, peel, 1, 1) func(6, type, peel, 1, 2) func(7, type, peel, 1, 3) \
}

◆ MICRO_MMA_WORK_ONE

#define MICRO_MMA_WORK_ONE (   iter,
  type,
  peel,
  left,
  right 
)
Value:
if (unroll_factor > left) { \
pgerMMA<Packet, type, false>(&accZero##iter, rhsV##right[peel], lhsV##left); \
}

◆ PEEL_COMPLEX_MMA

#define PEEL_COMPLEX_MMA   4

◆ PEEL_MMA

#define PEEL_MMA   8