Eigen::internal Namespace Reference

Namespace containing low-level routines from the Eigen library. More...

Namespaces
	detail
	group_theory
	imklfft
	lapacke_helpers
	Implementation details and helper functions for the lapacke glue code.
	std_fallback
	test_detail
	tuple_impl
	unary_pow

Classes
struct	band_solve_triangular_selector
struct	band_solve_triangular_selector< Index, Mode, LhsScalar, ConjLhs, RhsScalar, RowMajor >
struct	band_solve_triangular_selector< Index, Mode, LhsScalar, ConjLhs, RhsScalar, ColMajor >
struct	general_rank1_update
struct	general_rank1_update< Scalar, Index, ColMajor, ConjLhs, ConjRhs >
struct	general_rank1_update< Scalar, Index, RowMajor, ConjLhs, ConjRhs >
struct	functor_traits< scalar_norm1_op >
struct	selfadjoint_packed_rank1_update
struct	selfadjoint_packed_rank1_update< Scalar, Index, ColMajor, UpLo, ConjLhs, ConjRhs >
struct	selfadjoint_packed_rank1_update< Scalar, Index, RowMajor, UpLo, ConjLhs, ConjRhs >
struct	packed_triangular_matrix_vector_product
struct	packed_triangular_matrix_vector_product< Index, Mode, LhsScalar, ConjLhs, RhsScalar, ConjRhs, ColMajor >
struct	packed_triangular_matrix_vector_product< Index, Mode, LhsScalar, ConjLhs, RhsScalar, ConjRhs, RowMajor >
struct	packed_triangular_solve_vector
struct	packed_triangular_solve_vector< LhsScalar, RhsScalar, Index, OnTheLeft, Mode, Conjugate, RowMajor >
struct	packed_triangular_solve_vector< LhsScalar, RhsScalar, Index, OnTheLeft, Mode, Conjugate, ColMajor >
struct	packed_triangular_solve_vector< LhsScalar, RhsScalar, Index, OnTheRight, Mode, Conjugate, StorageOrder >
struct	rank2_update_selector
struct	packed_rank2_update_selector
struct	traits< MatrixReplacement >
struct	generic_product_impl< MatrixReplacement, Rhs, SparseShape, DenseShape, GemvProduct >
struct	AccelFactorizationDeleter
struct	SparseTypesTraitBase
struct	SparseTypesTrait
struct	SparseTypesTrait< double >
struct	SparseTypesTrait< float >
struct	traits< LDLT< MatrixType_, UpLo_ > >
struct	LDLT_Traits
struct	ldlt_inplace
struct	ldlt_inplace< Lower >
struct	ldlt_inplace< Upper >
struct	LDLT_Traits< MatrixType, Lower >
struct	LDLT_Traits< MatrixType, Upper >
struct	traits< LLT< MatrixType_, UpLo_ > >
struct	LLT_Traits
struct	llt_inplace
struct	llt_inplace< Scalar, Lower >
struct	llt_inplace< Scalar, Upper >
struct	LLT_Traits< MatrixType, Lower >
struct	LLT_Traits< MatrixType, Upper >
struct	cholmod_configure_matrix
struct	cholmod_configure_matrix< double >
struct	cholmod_configure_matrix< std::complex< double > >
struct	Packet2cf
struct	packet_traits< std::complex< float > >
struct	unpacket_traits< Packet2cf >
struct	quad_traits
struct	quad_traits< double >
struct	quad_traits< bfloat16 >
struct	symm_pack_rhs< std::complex< float >, Index, nr, StorageOrder >
struct	symm_pack_lhs< std::complex< float >, Index, Pack1, Pack2_dummy, StorageOrder >
struct	symm_pack_rhs< std::complex< double >, Index, nr, StorageOrder >
struct	symm_pack_lhs< std::complex< double >, Index, Pack1, Pack2_dummy, StorageOrder >
struct	symm_pack_rhs< float, Index, nr, StorageOrder >
struct	symm_pack_lhs< float, Index, Pack1, Pack2_dummy, StorageOrder >
struct	symm_pack_rhs< double, Index, nr, StorageOrder >
struct	symm_pack_lhs< double, Index, Pack1, Pack2_dummy, StorageOrder >
struct	dhs_cpack
struct	dhs_pack
struct	dhs_pack< double, DataMapper, Packet2d, StorageOrder, PanelMode, true >
struct	dhs_pack< double, DataMapper, Packet2d, StorageOrder, PanelMode, false >
struct	dhs_pack< bfloat16, DataMapper, Packet8bf, StorageOrder, PanelMode, true >
struct	dhs_pack< bfloat16, DataMapper, Packet8bf, StorageOrder, PanelMode, false >
struct	dhs_cpack< double, DataMapper, Packet, PacketC, StorageOrder, Conjugate, PanelMode, true >
struct	dhs_cpack< double, DataMapper, Packet, PacketC, StorageOrder, Conjugate, PanelMode, false >
struct	gemm_pack_lhs< double, Index, DataMapper, Pack1, Pack2, Packet, ColMajor, Conjugate, PanelMode >
struct	gemm_pack_lhs< double, Index, DataMapper, Pack1, Pack2, Packet, RowMajor, Conjugate, PanelMode >
struct	gemm_pack_rhs< double, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode >
struct	gemm_pack_rhs< double, Index, DataMapper, nr, RowMajor, Conjugate, PanelMode >
struct	gemm_pack_rhs< bfloat16, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode >
struct	gemm_pack_rhs< bfloat16, Index, DataMapper, nr, RowMajor, Conjugate, PanelMode >
struct	gemm_pack_lhs< bfloat16, Index, DataMapper, Pack1, Pack2, Packet, ColMajor, Conjugate, PanelMode >
struct	gemm_pack_lhs< bfloat16, Index, DataMapper, Pack1, Pack2, Packet, RowMajor, Conjugate, PanelMode >
struct	gemm_pack_lhs< float, Index, DataMapper, Pack1, Pack2, Packet, RowMajor, Conjugate, PanelMode >
struct	gemm_pack_lhs< float, Index, DataMapper, Pack1, Pack2, Packet, ColMajor, Conjugate, PanelMode >
struct	gemm_pack_lhs< std::complex< float >, Index, DataMapper, Pack1, Pack2, Packet, RowMajor, Conjugate, PanelMode >
struct	gemm_pack_lhs< std::complex< float >, Index, DataMapper, Pack1, Pack2, Packet, ColMajor, Conjugate, PanelMode >
struct	gemm_pack_rhs< float, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode >
struct	gemm_pack_rhs< float, Index, DataMapper, nr, RowMajor, Conjugate, PanelMode >
struct	gemm_pack_rhs< std::complex< float >, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode >
struct	gemm_pack_rhs< std::complex< float >, Index, DataMapper, nr, RowMajor, Conjugate, PanelMode >
struct	gemm_pack_lhs< std::complex< double >, Index, DataMapper, Pack1, Pack2, Packet, RowMajor, Conjugate, PanelMode >
struct	gemm_pack_lhs< std::complex< double >, Index, DataMapper, Pack1, Pack2, Packet, ColMajor, Conjugate, PanelMode >
struct	gemm_pack_rhs< std::complex< double >, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode >
struct	gemm_pack_rhs< std::complex< double >, Index, DataMapper, nr, RowMajor, Conjugate, PanelMode >
struct	gebp_kernel< float, float, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >
struct	gebp_kernel< std::complex< float >, std::complex< float >, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >
struct	gebp_kernel< float, std::complex< float >, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >
struct	gebp_kernel< std::complex< float >, float, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >
struct	gebp_kernel< double, double, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >
struct	gebp_kernel< std::complex< double >, std::complex< double >, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >
struct	gebp_kernel< std::complex< double >, double, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >
struct	gebp_kernel< double, std::complex< double >, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >
struct	gebp_kernel< bfloat16, bfloat16, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >
struct	UseMMAStride
struct	UseMMAStride< RhsMapper, LhsMapper, std::enable_if_t< std::is_member_function_pointer< decltype(&RhsMapper::stride)>::value > >
struct	packet_traits< float >
struct	packet_traits< bfloat16 >
struct	packet_traits< int >
struct	packet_traits< short int >
struct	packet_traits< unsigned short int >
struct	packet_traits< signed char >
struct	packet_traits< unsigned char >
struct	unpacket_traits< Packet4f >
struct	unpacket_traits< Packet4i >
struct	unpacket_traits< Packet8s >
struct	unpacket_traits< Packet8us >
struct	unpacket_traits< Packet16c >
struct	unpacket_traits< Packet16uc >
struct	unpacket_traits< Packet8bf >
struct	type_casting_traits< float, int >
struct	type_casting_traits< int, float >
struct	type_casting_traits< bfloat16, unsigned short int >
struct	type_casting_traits< unsigned short int, bfloat16 >
struct	type_casting_traits< bfloat16, float >
struct	type_casting_traits< float, bfloat16 >
struct	Packet4cf
struct	unpacket_traits< Packet4cf >
struct	Packet2cd
struct	packet_traits< std::complex< double > >
struct	unpacket_traits< Packet2cd >
struct	is_arithmetic< __m256 >
struct	is_arithmetic< __m256i >
struct	is_arithmetic< __m256d >
struct	is_arithmetic< Packet8i >
struct	is_arithmetic< Packet8ui >
struct	is_arithmetic< Packet8h >
struct	is_arithmetic< Packet8bf >
struct	packet_traits< double >
struct	packet_traits< Eigen::half >
struct	packet_traits< uint32_t >
struct	scalar_div_cost< float, true >
struct	scalar_div_cost< double, true >
struct	unpacket_traits< Packet8f >
struct	unpacket_traits< Packet4d >
struct	unpacket_traits< Packet8i >
struct	unpacket_traits< Packet8ui >
struct	unpacket_traits< Packet8h >
struct	type_casting_traits< float, bool >
struct	type_casting_traits< bool, float >
struct	type_casting_traits< float, double >
struct	type_casting_traits< double, float >
struct	type_casting_traits< double, int >
struct	type_casting_traits< int, double >
struct	type_casting_traits< half, float >
struct	type_casting_traits< float, half >
struct	Packet8cf
struct	unpacket_traits< Packet8cf >
struct	Packet4cd
struct	unpacket_traits< Packet4cd >
class	gemm_class
class	gebp_traits< float, float, ConjLhs_, ConjRhs_, Architecture::Target, PacketSize_ >
class	gebp_traits< double, double, ConjLhs_, ConjRhs_, Architecture::Target, PacketSize_ >
struct	gemm_pack_rhs< Scalar, Index, DataMapper, 8, ColMajor, Conjugate, PanelMode >
struct	gemm_pack_rhs< Scalar, Index, DataMapper, 8, RowMajor, Conjugate, PanelMode >
struct	gebp_kernel< Scalar, Scalar, Index, DataMapper, mr, 8, ConjugateLhs, ConjugateRhs >
struct	is_arithmetic< __m512 >
struct	is_arithmetic< __m512i >
struct	is_arithmetic< __m512d >
struct	is_arithmetic< Packet8l >
struct	is_arithmetic< Packet16h >
struct	packet_traits< half >
struct	packet_traits< int64_t >
struct	unpacket_traits< Packet16f >
struct	unpacket_traits< Packet8d >
struct	unpacket_traits< Packet16i >
struct	unpacket_traits< Packet8l >
struct	unpacket_traits< Packet16h >
struct	is_arithmetic< Packet16bf >
struct	unpacket_traits< Packet16bf >
struct	unpacket_traits< Packet32h >
struct	trsmKernelR
struct	trsmKernelR< float, Index, Mode, false, TriStorageOrder, 1, true >
struct	trsmKernelR< double, Index, Mode, false, TriStorageOrder, 1, true >
struct	trsmKernelL
struct	trsmKernelL< float, Index, Mode, false, TriStorageOrder, 1, true >
struct	trsmKernelL< double, Index, Mode, false, TriStorageOrder, 1, true >
struct	type_casting_traits< double, int64_t >
struct	type_casting_traits< int64_t, double >
struct	is_arithmetic< bfloat16 >
struct	random_impl< bfloat16 >
struct	conj_if
struct	conj_if< true >
struct	conj_if< false >
struct	conj_helper
struct	conj_helper< LhsScalar, RhsScalar, true, true >
struct	conj_helper< Packet, Packet, ConjLhs, ConjRhs >
struct	conj_helper< Packet, Packet, true, true >
struct	make_integer
struct	make_integer< float >
struct	make_integer< double >
struct	make_integer< half >
struct	make_integer< bfloat16 >
struct	ppolevl
struct	ppolevl< Packet, 0 >
struct	pchebevl
struct	patan_reduced
struct	psign_impl< Packet, std::enable_if_t<!NumTraits< typename unpacket_traits< Packet >::type >::IsComplex &&!NumTraits< typename unpacket_traits< Packet >::type >::IsInteger > >
struct	psign_impl< Packet, std::enable_if_t<!NumTraits< typename unpacket_traits< Packet >::type >::IsComplex &&NumTraits< typename unpacket_traits< Packet >::type >::IsSigned &&NumTraits< typename unpacket_traits< Packet >::type >::IsInteger > >
struct	psign_impl< Packet, std::enable_if_t<!NumTraits< typename unpacket_traits< Packet >::type >::IsComplex &&!NumTraits< typename unpacket_traits< Packet >::type >::IsSigned &&NumTraits< typename unpacket_traits< Packet >::type >::IsInteger > >
struct	psign_impl< Packet, std::enable_if_t< NumTraits< typename unpacket_traits< Packet >::type >::IsComplex &&unpacket_traits< Packet >::vectorizable > >
struct	accurate_log2
struct	accurate_log2< float >
struct	accurate_log2< double >
struct	unary_pow_impl
struct	unary_pow_impl< Packet, ScalarExponent, false, false, ExponentIsSigned >
struct	unary_pow_impl< Packet, ScalarExponent, false, true, ExponentIsSigned >
struct	unary_pow_impl< Packet, ScalarExponent, true, true, true >
struct	unary_pow_impl< Packet, ScalarExponent, true, true, false >
struct	nearest_integer_packetop_impl< Packet, false, false >
struct	nearest_integer_packetop_impl< Packet, false, true >
struct	is_arithmetic< half >
struct	random_impl< half >
struct	cast_impl< float, half >
struct	cast_impl< int, half >
struct	cast_impl< half, float >
struct	Packet1cd
struct	unpacket_traits< Packet1cd >
struct	gebp_traits< float, float, false, false, Architecture::LSX, GEBPPacketFull >
struct	gebp_traits< double, double, false, false, Architecture::LSX, GEBPPacketFull >
struct	is_arithmetic< __m128 >
struct	is_arithmetic< __m128i >
struct	is_arithmetic< __m128d >
struct	is_arithmetic< Packet16c >
struct	is_arithmetic< Packet8s >
struct	is_arithmetic< Packet4i >
struct	is_arithmetic< Packet2l >
struct	is_arithmetic< Packet16uc >
struct	is_arithmetic< Packet8us >
struct	is_arithmetic< Packet4ui >
struct	is_arithmetic< Packet2ul >
struct	packet_traits< int8_t >
struct	packet_traits< int16_t >
struct	packet_traits< int32_t >
struct	packet_traits< uint8_t >
struct	packet_traits< uint16_t >
struct	packet_traits< uint64_t >
struct	unpacket_traits< Packet2l >
struct	unpacket_traits< Packet4ui >
struct	unpacket_traits< Packet2ul >
struct	unpacket_traits< Packet2d >
struct	Packet1cf
struct	unpacket_traits< Packet1cf >
struct	unpacket_traits< Packet2f >
struct	unpacket_traits< Packet4c >
struct	unpacket_traits< Packet8c >
struct	unpacket_traits< Packet4uc >
struct	unpacket_traits< Packet8uc >
struct	unpacket_traits< Packet4s >
struct	unpacket_traits< Packet4us >
struct	unpacket_traits< Packet2i >
struct	unpacket_traits< Packet2ui >
struct	is_arithmetic< Packet4bf >
struct	unpacket_traits< Packet4bf >
struct	type_casting_traits< float, numext::int64_t >
struct	type_casting_traits< float, numext::uint64_t >
struct	type_casting_traits< float, numext::int32_t >
struct	type_casting_traits< float, numext::uint32_t >
struct	type_casting_traits< float, numext::int16_t >
struct	type_casting_traits< float, numext::uint16_t >
struct	type_casting_traits< float, numext::int8_t >
struct	type_casting_traits< float, numext::uint8_t >
struct	type_casting_traits< numext::int8_t, float >
struct	type_casting_traits< numext::int8_t, numext::int64_t >
struct	type_casting_traits< numext::int8_t, numext::uint64_t >
struct	type_casting_traits< numext::int8_t, numext::int32_t >
struct	type_casting_traits< numext::int8_t, numext::uint32_t >
struct	type_casting_traits< numext::int8_t, numext::int16_t >
struct	type_casting_traits< numext::int8_t, numext::uint16_t >
struct	type_casting_traits< numext::uint8_t, float >
struct	type_casting_traits< numext::uint8_t, numext::uint64_t >
struct	type_casting_traits< numext::uint8_t, numext::int64_t >
struct	type_casting_traits< numext::uint8_t, numext::uint32_t >
struct	type_casting_traits< numext::uint8_t, numext::int32_t >
struct	type_casting_traits< numext::uint8_t, numext::uint16_t >
struct	type_casting_traits< numext::uint8_t, numext::int16_t >
struct	type_casting_traits< numext::int16_t, float >
struct	type_casting_traits< numext::int16_t, numext::int64_t >
struct	type_casting_traits< numext::int16_t, numext::uint64_t >
struct	type_casting_traits< numext::int16_t, numext::int32_t >
struct	type_casting_traits< numext::int16_t, numext::uint32_t >
struct	type_casting_traits< numext::int16_t, numext::int8_t >
struct	type_casting_traits< numext::int16_t, numext::uint8_t >
struct	type_casting_traits< numext::uint16_t, float >
struct	type_casting_traits< numext::uint16_t, numext::uint64_t >
struct	type_casting_traits< numext::uint16_t, numext::int64_t >
struct	type_casting_traits< numext::uint16_t, numext::uint32_t >
struct	type_casting_traits< numext::uint16_t, numext::int32_t >
struct	type_casting_traits< numext::uint16_t, numext::uint8_t >
struct	type_casting_traits< numext::uint16_t, numext::int8_t >
struct	type_casting_traits< numext::int32_t, float >
struct	type_casting_traits< numext::int32_t, numext::int64_t >
struct	type_casting_traits< numext::int32_t, numext::uint64_t >
struct	type_casting_traits< numext::int32_t, numext::int16_t >
struct	type_casting_traits< numext::int32_t, numext::uint16_t >
struct	type_casting_traits< numext::int32_t, numext::int8_t >
struct	type_casting_traits< numext::int32_t, numext::uint8_t >
struct	type_casting_traits< numext::uint32_t, float >
struct	type_casting_traits< numext::uint32_t, numext::uint64_t >
struct	type_casting_traits< numext::uint32_t, numext::int64_t >
struct	type_casting_traits< numext::uint32_t, numext::uint16_t >
struct	type_casting_traits< numext::uint32_t, numext::int16_t >
struct	type_casting_traits< numext::uint32_t, numext::uint8_t >
struct	type_casting_traits< numext::uint32_t, numext::int8_t >
struct	type_casting_traits< numext::int64_t, float >
struct	type_casting_traits< numext::int64_t, numext::int32_t >
struct	type_casting_traits< numext::int64_t, numext::uint32_t >
struct	type_casting_traits< numext::int64_t, numext::int16_t >
struct	type_casting_traits< numext::int64_t, numext::uint16_t >
struct	type_casting_traits< numext::int64_t, numext::int8_t >
struct	type_casting_traits< numext::int64_t, numext::uint8_t >
struct	type_casting_traits< numext::uint64_t, float >
struct	type_casting_traits< numext::uint64_t, numext::uint32_t >
struct	type_casting_traits< numext::uint64_t, numext::int32_t >
struct	type_casting_traits< numext::uint64_t, numext::uint16_t >
struct	type_casting_traits< numext::uint64_t, numext::int16_t >
struct	type_casting_traits< numext::uint64_t, numext::uint8_t >
struct	type_casting_traits< numext::uint64_t, numext::int8_t >
struct	is_arithmetic< Packet16b >
struct	shuffle_mask
struct	packet_traits< bool >
struct	unpacket_traits< Packet16b >
struct	sve_packet_size_selector
struct	packet_traits< numext::int32_t >
struct	unpacket_traits< PacketXi >
struct	unpacket_traits< PacketXf >
struct	sycl_packet_traits
union	Packet
struct	cleanup_seq_incr
struct	traits< Array< Scalar_, Rows_, Cols_, Options_, MaxRows_, MaxCols_ > >
struct	traits< ArrayWrapper< ExpressionType > >
struct	traits< MatrixWrapper< ExpressionType > >
class	vml_assign_traits
struct	copy_using_evaluator_traits
struct	copy_using_evaluator_DefaultTraversal_CompleteUnrolling
struct	copy_using_evaluator_DefaultTraversal_CompleteUnrolling< Kernel, Stop, Stop >
struct	copy_using_evaluator_DefaultTraversal_InnerUnrolling
struct	copy_using_evaluator_DefaultTraversal_InnerUnrolling< Kernel, Stop, Stop >
struct	copy_using_evaluator_LinearTraversal_CompleteUnrolling
struct	copy_using_evaluator_LinearTraversal_CompleteUnrolling< Kernel, Stop, Stop >
struct	copy_using_evaluator_innervec_CompleteUnrolling
struct	copy_using_evaluator_innervec_CompleteUnrolling< Kernel, Stop, Stop >
struct	copy_using_evaluator_innervec_InnerUnrolling
struct	copy_using_evaluator_innervec_InnerUnrolling< Kernel, Stop, Stop, SrcAlignment, DstAlignment >
struct	dense_assignment_loop
struct	dense_assignment_loop< Kernel, AllAtOnceTraversal, Unrolling >
struct	dense_assignment_loop< Kernel, DefaultTraversal, NoUnrolling >
struct	dense_assignment_loop< Kernel, DefaultTraversal, CompleteUnrolling >
struct	dense_assignment_loop< Kernel, DefaultTraversal, InnerUnrolling >
struct	unaligned_dense_assignment_loop
struct	unaligned_dense_assignment_loop< false >
struct	copy_using_evaluator_linearvec_CompleteUnrolling
struct	copy_using_evaluator_linearvec_CompleteUnrolling< Kernel, Stop, Stop >
struct	dense_assignment_loop< Kernel, LinearVectorizedTraversal, NoUnrolling >
struct	dense_assignment_loop< Kernel, LinearVectorizedTraversal, CompleteUnrolling >
struct	dense_assignment_loop< Kernel, InnerVectorizedTraversal, NoUnrolling >
struct	dense_assignment_loop< Kernel, InnerVectorizedTraversal, CompleteUnrolling >
struct	dense_assignment_loop< Kernel, InnerVectorizedTraversal, InnerUnrolling >
struct	dense_assignment_loop< Kernel, LinearTraversal, NoUnrolling >
struct	dense_assignment_loop< Kernel, LinearTraversal, CompleteUnrolling >
struct	dense_assignment_loop< Kernel, SliceVectorizedTraversal, NoUnrolling >
class	generic_dense_assignment_kernel
class	restricted_packet_dense_assignment_kernel
struct	AssignmentKind
struct	Dense2Dense
struct	EigenBase2EigenBase
struct	AssignmentKind< DenseShape, DenseShape >
struct	Assignment
struct	Assignment< DstXprType, SrcXprType, Functor, Dense2Dense, Weak >
struct	Assignment< DstXprType, CwiseNullaryOp< scalar_constant_op< typename DstXprType::Scalar >, SrcPlainObject >, assign_op< typename DstXprType::Scalar, typename DstXprType::Scalar >, Dense2Dense, Weak >
struct	Assignment< DstXprType, CwiseNullaryOp< scalar_zero_op< typename DstXprType::Scalar >, SrcPlainObject >, assign_op< typename DstXprType::Scalar, typename DstXprType::Scalar >, Dense2Dense, Weak >
struct	Assignment< DstXprType, SrcXprType, Functor, EigenBase2EigenBase, Weak >
class	BandMatrixBase
struct	traits< BandMatrix< Scalar_, Rows_, Cols_, Supers_, Subs_, Options_ > >
class	BandMatrix
	Represents a rectangular matrix with a banded storage. More...
class	BandMatrixWrapper
struct	traits< BandMatrixWrapper< CoefficientsType_, Rows_, Cols_, Supers_, Subs_, Options_ > >
class	TridiagonalMatrix
	Represents a tridiagonal matrix with a compact banded storage. More...
struct	BandShape
struct	evaluator_traits< BandMatrix< Scalar_, Rows_, Cols_, Supers_, Subs_, Options_ > >
struct	evaluator_traits< BandMatrixWrapper< CoefficientsType_, Rows_, Cols_, Supers_, Subs_, Options_ > >
struct	AssignmentKind< DenseShape, BandShape >
struct	traits< Block< XprType_, BlockRows, BlockCols, InnerPanel_ > >
class	BlockImpl_dense
class	BlockImpl_dense< XprType, BlockRows, BlockCols, InnerPanel, true >
struct	rcond_compute_sign
struct	rcond_compute_sign< Vector, Vector, false >
struct	storage_kind_to_evaluator_kind
struct	storage_kind_to_shape
struct	storage_kind_to_shape< Dense >
struct	storage_kind_to_shape< SolverStorage >
struct	storage_kind_to_shape< PermutationStorage >
struct	storage_kind_to_shape< TranspositionsStorage >
struct	ternary_evaluator
struct	binary_evaluator
struct	unary_evaluator
struct	evaluator_traits_base
struct	evaluator_traits
struct	evaluator_assume_aliasing
struct	evaluator
struct	evaluator< const T >
struct	evaluator_base
class	plainobjectbase_evaluator_data
class	plainobjectbase_evaluator_data< Scalar, Dynamic >
struct	evaluator< PlainObjectBase< Derived > >
struct	evaluator< Matrix< Scalar, Rows, Cols, Options, MaxRows, MaxCols > >
struct	evaluator< Array< Scalar, Rows, Cols, Options, MaxRows, MaxCols > >
struct	unary_evaluator< Transpose< ArgType >, IndexBased >
struct	nullary_wrapper
struct	nullary_wrapper< Scalar, NullaryOp, true, false, false >
struct	nullary_wrapper< Scalar, NullaryOp, false, false, true >
struct	nullary_wrapper< Scalar, NullaryOp, false, true, false >
struct	nullary_wrapper< Scalar, NullaryOp, false, false, false >
struct	evaluator< CwiseNullaryOp< NullaryOp, PlainObjectType > >
struct	unary_evaluator< CwiseUnaryOp< UnaryOp, ArgType >, IndexBased >
struct	unary_evaluator< CwiseUnaryOp< core_cast_op< SrcType, DstType >, ArgType >, IndexBased >
struct	evaluator< CwiseTernaryOp< TernaryOp, Arg1, Arg2, Arg3 > >
struct	ternary_evaluator< CwiseTernaryOp< TernaryOp, Arg1, Arg2, Arg3 >, IndexBased, IndexBased >
struct	evaluator< CwiseTernaryOp< scalar_boolean_select_op< Scalar, Scalar, bool >, Arg1, Arg2, CwiseBinaryOp< scalar_cmp_op< Scalar, Scalar, cmp, false >, CmpLhsType, CmpRhsType > > >
struct	evaluator< CwiseBinaryOp< BinaryOp, Lhs, Rhs > >
struct	binary_evaluator< CwiseBinaryOp< BinaryOp, Lhs, Rhs >, IndexBased, IndexBased >
struct	unary_evaluator< CwiseUnaryView< UnaryOp, ArgType, StrideType >, IndexBased >
struct	mapbase_evaluator
struct	evaluator< Map< PlainObjectType, MapOptions, StrideType > >
struct	evaluator< Ref< PlainObjectType, RefOptions, StrideType > >
struct	block_evaluator
struct	evaluator< Block< ArgType, BlockRows, BlockCols, InnerPanel > >
struct	block_evaluator< ArgType, BlockRows, BlockCols, InnerPanel, false >
struct	unary_evaluator< Block< ArgType, BlockRows, BlockCols, InnerPanel >, IndexBased >
struct	block_evaluator< ArgType, BlockRows, BlockCols, InnerPanel, true >
struct	evaluator< Select< ConditionMatrixType, ThenMatrixType, ElseMatrixType > >
struct	unary_evaluator< Replicate< ArgType, RowFactor, ColFactor > >
struct	evaluator_wrapper_base
struct	unary_evaluator< MatrixWrapper< TArgType > >
struct	unary_evaluator< ArrayWrapper< TArgType > >
struct	reverse_packet_cond
struct	unary_evaluator< Reverse< ArgType, Direction > >
struct	evaluator< Diagonal< ArgType, DiagIndex > >
class	EvalToTemp
struct	traits< EvalToTemp< ArgType > >
struct	evaluator< EvalToTemp< ArgType > >
class	inner_iterator_selector
class	inner_iterator_selector< XprType, IndexBased >
class	inner_iterator_selector< XprType, IteratorBased >
struct	traits< CwiseBinaryOp< BinaryOp, Lhs, Rhs > >
struct	traits< CwiseNullaryOp< NullaryOp, PlainObjectType > >
struct	setIdentity_impl
struct	setIdentity_impl< Derived, true >
struct	traits< CwiseTernaryOp< TernaryOp, Arg1, Arg2, Arg3 > >
struct	traits< CwiseUnaryOp< UnaryOp, XprType > >
struct	traits< CwiseUnaryView< ViewOp, MatrixType, StrideType > >
class	CwiseUnaryViewImpl
class	CwiseUnaryViewImpl< ViewOp, MatrixType, StrideType, Dense, false >
class	CwiseUnaryViewImpl< ViewOp, MatrixType, StrideType, Dense, true >
struct	add_const_on_value_type_if_arithmetic
struct	first_aligned_impl
struct	first_aligned_impl< Alignment, Derived, false >
struct	inner_stride_at_compile_time
struct	inner_stride_at_compile_time< Derived, false >
struct	outer_stride_at_compile_time
struct	outer_stride_at_compile_time< Derived, false >
struct	plain_array
struct	plain_array< T, Size, MatrixOrArrayOptions, 0 >
struct	plain_array< T, 0, MatrixOrArrayOptions, Alignment >
class	DenseStorage_impl
class	DenseStorage_impl< T, Size, Dynamic, Cols, Options >
class	DenseStorage_impl< T, Size, Rows, Dynamic, Options >
class	DenseStorage_impl< T, Size, Dynamic, Dynamic, Options >
class	DenseStorage_impl< T, 0, Rows, Cols, Options >
class	DenseStorage_impl< T, 0, Dynamic, Cols, Options >
class	DenseStorage_impl< T, 0, Rows, Dynamic, Options >
class	DenseStorage_impl< T, 0, Dynamic, Dynamic, Options >
class	DenseStorage_impl< T, Dynamic, Rows, Cols, Options >
class	DenseStorage_impl< T, Dynamic, Dynamic, Cols, Options >
class	DenseStorage_impl< T, Dynamic, Rows, Dynamic, Options >
class	DenseStorage_impl< T, Dynamic, Dynamic, Dynamic, Options >
struct	use_default_move
struct	AssignmentWithDevice
struct	AssignmentWithDevice< DstXprType, Product< Lhs, Rhs, Options >, Functor, Device, Dense2Dense, Weak >
struct	AssignmentWithDevice< DstXprType, SrcXprType, Functor, Device, Dense2Dense, Weak >
struct	dense_assignment_loop_with_device
struct	traits< Diagonal< MatrixType, DiagIndex > >
struct	traits< DiagonalMatrix< Scalar_, SizeAtCompileTime, MaxSizeAtCompileTime > >
struct	traits< DiagonalWrapper< DiagonalVectorType_ > >
struct	storage_kind_to_shape< DiagonalShape >
struct	Diagonal2Dense
struct	AssignmentKind< DenseShape, DiagonalShape >
struct	Assignment< DstXprType, SrcXprType, Functor, Diagonal2Dense >
struct	squared_norm_impl
struct	squared_norm_impl< Derived, bool >
struct	lpNorm_selector
struct	lpNorm_selector< Derived, 1 >
struct	lpNorm_selector< Derived, 2 >
struct	lpNorm_selector< Derived, Infinity >
struct	eigen_fill_helper
struct	eigen_fill_helper< Matrix< Scalar, Rows, Cols, Options, MaxRows, MaxCols > >
struct	eigen_fill_helper< Array< Scalar, Rows, Cols, Options, MaxRows, MaxCols > >
struct	eigen_fill_helper< Block< Xpr, BlockRows, BlockCols, true > >
struct	eigen_fill_helper< Block< Xpr, BlockRows, BlockCols, false > >
struct	eigen_fill_helper< Map< Xpr, Options, Stride< 0, 0 > > >
struct	eigen_fill_helper< Map< Xpr, Options, Stride< OuterStride_, 0 > > >
struct	eigen_fill_helper< Map< Xpr, Options, Stride< OuterStride_, 1 > > >
struct	eigen_fill_helper< Map< Xpr, Options, InnerStride< InnerStride_ > > >
struct	eigen_fill_helper< Map< Xpr, Options, OuterStride< OuterStride_ > > >
struct	eigen_fill_impl< Xpr, false >
struct	eigen_fill_impl< Xpr, true >
struct	eigen_memset_helper
struct	eigen_zero_impl< Xpr, false >
struct	eigen_zero_impl< Xpr, true >
struct	traits< ForceAlignedAccess< ExpressionType > >
struct	assign_op
	Template functor for scalar/packet assignment. More...
struct	assign_op< DstScalar, void >
struct	functor_traits< assign_op< DstScalar, SrcScalar > >
struct	add_assign_op
	Template functor for scalar/packet assignment with addition. More...
struct	functor_traits< add_assign_op< DstScalar, SrcScalar > >
struct	sub_assign_op
	Template functor for scalar/packet assignment with subtraction. More...
struct	functor_traits< sub_assign_op< DstScalar, SrcScalar > >
struct	mul_assign_op
	Template functor for scalar/packet assignment with multiplication. More...
struct	functor_traits< mul_assign_op< DstScalar, SrcScalar > >
struct	div_assign_op
	Template functor for scalar/packet assignment with diviving. More...
struct	functor_traits< div_assign_op< DstScalar, SrcScalar > >
struct	swap_assign_op
	Template functor for scalar/packet assignment with swapping. More...
struct	functor_traits< swap_assign_op< Scalar > >
struct	binary_op_base
struct	scalar_sum_op
	Template functor to compute the sum of two scalars. More...
struct	functor_traits< scalar_sum_op< LhsScalar, RhsScalar > >
struct	scalar_product_op
	Template functor to compute the product of two scalars. More...
struct	functor_traits< scalar_product_op< LhsScalar, RhsScalar > >
struct	scalar_conj_product_op
	Template functor to compute the conjugate product of two scalars. More...
struct	functor_traits< scalar_conj_product_op< LhsScalar, RhsScalar > >
struct	scalar_min_op
	Template functor to compute the min of two scalars. More...
struct	functor_traits< scalar_min_op< LhsScalar, RhsScalar, NaNPropagation > >
struct	scalar_max_op
	Template functor to compute the max of two scalars. More...
struct	functor_traits< scalar_max_op< LhsScalar, RhsScalar, NaNPropagation > >
struct	scalar_cmp_op
	Template functors for comparison of two scalars. More...
struct	functor_traits< scalar_cmp_op< LhsScalar, RhsScalar, cmp, UseTypedComparators > >
struct	typed_cmp_helper
struct	scalar_cmp_op< LhsScalar, RhsScalar, cmp_EQ, UseTypedComparators >
struct	scalar_cmp_op< LhsScalar, RhsScalar, cmp_LT, UseTypedComparators >
struct	scalar_cmp_op< LhsScalar, RhsScalar, cmp_LE, UseTypedComparators >
struct	scalar_cmp_op< LhsScalar, RhsScalar, cmp_GT, UseTypedComparators >
struct	scalar_cmp_op< LhsScalar, RhsScalar, cmp_GE, UseTypedComparators >
struct	scalar_cmp_op< LhsScalar, RhsScalar, cmp_UNORD, UseTypedComparators >
struct	scalar_cmp_op< LhsScalar, RhsScalar, cmp_NEQ, UseTypedComparators >
struct	scalar_hypot_op< Scalar, Scalar >
	Template functor to compute the hypot of two positive and real scalars. More...
struct	functor_traits< scalar_hypot_op< Scalar, Scalar > >
struct	scalar_pow_op
	Template functor to compute the pow of two scalars See the specification of pow in https://en.cppreference.com/w/cpp/numeric/math/pow. More...
struct	functor_traits< scalar_pow_op< Scalar, Exponent > >
struct	scalar_difference_op
	Template functor to compute the difference of two scalars. More...
struct	functor_traits< scalar_difference_op< LhsScalar, RhsScalar > >
struct	maybe_raise_div_by_zero
struct	maybe_raise_div_by_zero< Packet, true >
struct	scalar_quotient_op
	Template functor to compute the quotient of two scalars. More...
struct	functor_traits< scalar_quotient_op< LhsScalar, RhsScalar > >
struct	scalar_boolean_and_op
	Template functor to compute the and of two scalars as if they were booleans. More...
struct	functor_traits< scalar_boolean_and_op< Scalar > >
struct	scalar_boolean_or_op
	Template functor to compute the or of two scalars as if they were booleans. More...
struct	functor_traits< scalar_boolean_or_op< Scalar > >
struct	scalar_boolean_xor_op
	Template functor to compute the xor of two scalars as if they were booleans. More...
struct	functor_traits< scalar_boolean_xor_op< Scalar > >
struct	bitwise_binary_impl
struct	bitwise_binary_impl< Scalar, true >
struct	scalar_bitwise_and_op
	Template functor to compute the bitwise and of two scalars. More...
struct	functor_traits< scalar_bitwise_and_op< Scalar > >
struct	scalar_bitwise_or_op
	Template functor to compute the bitwise or of two scalars. More...
struct	functor_traits< scalar_bitwise_or_op< Scalar > >
struct	scalar_bitwise_xor_op
	Template functor to compute the bitwise xor of two scalars. More...
struct	functor_traits< scalar_bitwise_xor_op< Scalar > >
struct	scalar_absolute_difference_op
	Template functor to compute the absolute difference of two scalars. More...
struct	functor_traits< scalar_absolute_difference_op< LhsScalar, RhsScalar > >
struct	scalar_atan2_op
struct	functor_traits< scalar_atan2_op< LhsScalar, RhsScalar > >
struct	bind1st_op
struct	functor_traits< bind1st_op< BinaryOp > >
struct	bind2nd_op
struct	functor_traits< bind2nd_op< BinaryOp > >
struct	scalar_constant_op
struct	functor_traits< scalar_constant_op< Scalar > >
struct	scalar_zero_op
struct	functor_traits< scalar_zero_op< Scalar > >
struct	scalar_identity_op
struct	functor_traits< scalar_identity_op< Scalar > >
struct	linspaced_op_impl
struct	linspaced_op_impl< Scalar, false >
struct	linspaced_op_impl< Scalar, true >
struct	linspaced_op
struct	functor_traits< linspaced_op< Scalar > >
struct	equalspaced_op
struct	functor_traits< equalspaced_op< Scalar > >
struct	functor_has_linear_access
struct	has_nullary_operator< scalar_constant_op< Scalar >, IndexType >
struct	has_unary_operator< scalar_constant_op< Scalar >, IndexType >
struct	has_binary_operator< scalar_constant_op< Scalar >, IndexType >
struct	has_nullary_operator< scalar_identity_op< Scalar >, IndexType >
struct	has_unary_operator< scalar_identity_op< Scalar >, IndexType >
struct	has_binary_operator< scalar_identity_op< Scalar >, IndexType >
struct	has_nullary_operator< linspaced_op< Scalar >, IndexType >
struct	has_unary_operator< linspaced_op< Scalar >, IndexType >
struct	has_binary_operator< linspaced_op< Scalar >, IndexType >
struct	has_nullary_operator< scalar_random_op< Scalar >, IndexType >
struct	has_unary_operator< scalar_random_op< Scalar >, IndexType >
struct	has_binary_operator< scalar_random_op< Scalar >, IndexType >
struct	functor_traits< std::multiplies< T > >
struct	functor_traits< std::divides< T > >
struct	functor_traits< std::plus< T > >
struct	functor_traits< std::minus< T > >
struct	functor_traits< std::negate< T > >
struct	functor_traits< std::logical_or< T > >
struct	functor_traits< std::logical_and< T > >
struct	functor_traits< std::logical_not< T > >
struct	functor_traits< std::greater< T > >
struct	functor_traits< std::less< T > >
struct	functor_traits< std::greater_equal< T > >
struct	functor_traits< std::less_equal< T > >
struct	functor_traits< std::equal_to< T > >
struct	functor_traits< std::not_equal_to< T > >
struct	functor_traits< std::unary_negate< T > >
struct	functor_traits< std::binary_negate< T > >
struct	scalar_boolean_select_op
struct	functor_traits< scalar_boolean_select_op< ThenScalar, ElseScalar, ConditionScalar > >
struct	scalar_opposite_op
	Template functor to compute the opposite of a scalar. More...
struct	functor_traits< scalar_opposite_op< Scalar > >
struct	scalar_abs_op
	Template functor to compute the absolute value of a scalar. More...
struct	functor_traits< scalar_abs_op< Scalar > >
struct	scalar_score_coeff_op
	Template functor to compute the score of a scalar, to chose a pivot. More...
struct	functor_traits< scalar_score_coeff_op< Scalar > >
struct	abs_knowing_score
struct	abs_knowing_score< Scalar, typename scalar_score_coeff_op< Scalar >::Score_is_abs >
struct	scalar_abs2_op
	Template functor to compute the squared absolute value of a scalar. More...
struct	functor_traits< scalar_abs2_op< Scalar > >
struct	squared_norm_functor
struct	squared_norm_functor< Scalar, false >
struct	functor_traits< squared_norm_functor< Scalar > >
struct	scalar_conjugate_op
	Template functor to compute the conjugate of a complex value. More...
struct	functor_traits< scalar_conjugate_op< Scalar > >
struct	scalar_arg_op
	Template functor to compute the phase angle of a complex. More...
struct	functor_traits< scalar_arg_op< Scalar > >
struct	scalar_carg_op
	Template functor to compute the complex argument, returned as a complex type. More...
struct	functor_traits< scalar_carg_op< Scalar > >
struct	scalar_cast_op
	Template functor to cast a scalar to another type. More...
struct	functor_traits< scalar_cast_op< Scalar, NewType > >
struct	core_cast_op
struct	functor_traits< core_cast_op< SrcType, DstType > >
struct	scalar_shift_right_op
	Template functor to arithmetically shift a scalar right by a number of bits. More...
struct	functor_traits< scalar_shift_right_op< Scalar, N > >
struct	scalar_shift_left_op
	Template functor to logically shift a scalar left by a number of bits. More...
struct	functor_traits< scalar_shift_left_op< Scalar, N > >
struct	scalar_real_op
	Template functor to extract the real part of a complex. More...
struct	functor_traits< scalar_real_op< Scalar > >
struct	scalar_imag_op
	Template functor to extract the imaginary part of a complex. More...
struct	functor_traits< scalar_imag_op< Scalar > >
struct	scalar_real_ref_op
	Template functor to extract the real part of a complex as a reference. More...
struct	functor_traits< scalar_real_ref_op< Scalar > >
struct	scalar_imag_ref_op
	Template functor to extract the imaginary part of a complex as a reference. More...
struct	functor_traits< scalar_imag_ref_op< Scalar > >
struct	scalar_exp_op
	Template functor to compute the exponential of a scalar. More...
struct	functor_traits< scalar_exp_op< Scalar > >
struct	scalar_exp2_op
struct	functor_traits< scalar_exp2_op< Scalar > >
struct	scalar_expm1_op
	Template functor to compute the exponential of a scalar - 1. More...
struct	functor_traits< scalar_expm1_op< Scalar > >
struct	scalar_log_op
	Template functor to compute the logarithm of a scalar. More...
struct	functor_traits< scalar_log_op< Scalar > >
struct	scalar_log1p_op
	Template functor to compute the logarithm of 1 plus a scalar value. More...
struct	functor_traits< scalar_log1p_op< Scalar > >
struct	scalar_log10_op
	Template functor to compute the base-10 logarithm of a scalar. More...
struct	functor_traits< scalar_log10_op< Scalar > >
struct	scalar_log2_op
	Template functor to compute the base-2 logarithm of a scalar. More...
struct	functor_traits< scalar_log2_op< Scalar > >
struct	scalar_sqrt_op
	Template functor to compute the square root of a scalar. More...
struct	functor_traits< scalar_sqrt_op< Scalar > >
struct	scalar_sqrt_op< bool >
struct	functor_traits< scalar_sqrt_op< bool > >
struct	scalar_cbrt_op
	Template functor to compute the cube root of a scalar. More...
struct	functor_traits< scalar_cbrt_op< Scalar > >
struct	scalar_rsqrt_op
	Template functor to compute the reciprocal square root of a scalar. More...
struct	functor_traits< scalar_rsqrt_op< Scalar > >
struct	scalar_cos_op
	Template functor to compute the cosine of a scalar. More...
struct	functor_traits< scalar_cos_op< Scalar > >
struct	scalar_sin_op
	Template functor to compute the sine of a scalar. More...
struct	functor_traits< scalar_sin_op< Scalar > >
struct	scalar_tan_op
	Template functor to compute the tan of a scalar. More...
struct	functor_traits< scalar_tan_op< Scalar > >
struct	scalar_acos_op
	Template functor to compute the arc cosine of a scalar. More...
struct	functor_traits< scalar_acos_op< Scalar > >
struct	scalar_asin_op
	Template functor to compute the arc sine of a scalar. More...
struct	functor_traits< scalar_asin_op< Scalar > >
struct	scalar_atan_op
	Template functor to compute the atan of a scalar. More...
struct	functor_traits< scalar_atan_op< Scalar > >
struct	scalar_tanh_op
	Template functor to compute the tanh of a scalar. More...
struct	functor_traits< scalar_tanh_op< Scalar > >
struct	scalar_atanh_op
	Template functor to compute the atanh of a scalar. More...
struct	functor_traits< scalar_atanh_op< Scalar > >
struct	scalar_sinh_op
	Template functor to compute the sinh of a scalar. More...
struct	functor_traits< scalar_sinh_op< Scalar > >
struct	scalar_asinh_op
	Template functor to compute the asinh of a scalar. More...
struct	functor_traits< scalar_asinh_op< Scalar > >
struct	scalar_cosh_op
	Template functor to compute the cosh of a scalar. More...
struct	functor_traits< scalar_cosh_op< Scalar > >
struct	scalar_acosh_op
	Template functor to compute the acosh of a scalar. More...
struct	functor_traits< scalar_acosh_op< Scalar > >
struct	scalar_inverse_op
	Template functor to compute the inverse of a scalar. More...
struct	functor_traits< scalar_inverse_op< Scalar > >
struct	scalar_square_op
	Template functor to compute the square of a scalar. More...
struct	functor_traits< scalar_square_op< Scalar > >
struct	scalar_square_op< bool >
struct	functor_traits< scalar_square_op< bool > >
struct	scalar_cube_op
	Template functor to compute the cube of a scalar. More...
struct	functor_traits< scalar_cube_op< Scalar > >
struct	scalar_cube_op< bool >
struct	functor_traits< scalar_cube_op< bool > >
struct	scalar_round_op
	Template functor to compute the rounded value of a scalar. More...
struct	functor_traits< scalar_round_op< Scalar > >
struct	scalar_floor_op
	Template functor to compute the floor of a scalar. More...
struct	functor_traits< scalar_floor_op< Scalar > >
struct	scalar_rint_op
	Template functor to compute the rounded (with current rounding mode) value of a scalar. More...
struct	functor_traits< scalar_rint_op< Scalar > >
struct	scalar_ceil_op
	Template functor to compute the ceil of a scalar. More...
struct	functor_traits< scalar_ceil_op< Scalar > >
struct	scalar_trunc_op
	Template functor to compute the truncation of a scalar. More...
struct	functor_traits< scalar_trunc_op< Scalar > >
struct	scalar_isnan_op
	Template functor to compute whether a scalar is NaN. More...
struct	scalar_isnan_op< Scalar, true >
struct	functor_traits< scalar_isnan_op< Scalar, UseTypedPredicate > >
struct	scalar_isinf_op
	Template functor to check whether a scalar is +/-inf. More...
struct	scalar_isinf_op< Scalar, true >
struct	functor_traits< scalar_isinf_op< Scalar, UseTypedPredicate > >
struct	scalar_isfinite_op
	Template functor to check whether a scalar has a finite value. More...
struct	scalar_isfinite_op< Scalar, true >
struct	functor_traits< scalar_isfinite_op< Scalar, UseTypedPredicate > >
struct	scalar_boolean_not_op
	Template functor to compute the logical not of a scalar as if it were a boolean. More...
struct	functor_traits< scalar_boolean_not_op< Scalar > >
struct	bitwise_unary_impl
struct	bitwise_unary_impl< Scalar, true >
struct	scalar_bitwise_not_op
	Template functor to compute the bitwise not of a scalar. More...
struct	functor_traits< scalar_bitwise_not_op< Scalar > >
struct	scalar_sign_op
	Template functor to compute the signum of a scalar. More...
struct	functor_traits< scalar_sign_op< Scalar > >
struct	scalar_logistic_op_impl
struct	scalar_logistic_op_impl< T, std::enable_if_t< NumTraits< T >::IsComplex > >
struct	scalar_logistic_op
	Template functor to compute the logistic function of a scalar. More...
struct	scalar_logistic_op< float >
	Template specialization of the logistic function for float. Computes S(x) = exp(x) / (1 + exp(x)), where exp(x) is implemented using an algorithm partly adopted from the implementation of pexp_float. See the individual steps described in the code below. Note that compared to pexp, we use an additional outer multiplicative range reduction step using the identity exp(x) = exp(x/2)^2. This prevert us from having to call ldexp on values that could produce a denormal result, which allows us to call the faster implementation in pldexp_fast_impl<Packet>::run(p, m). The final squaring, however, doubles the error bound on the final approximation. Exhaustive testing shows that we have a worst case error of 4.5 ulps (compared to computing S(x) in double precision), which is acceptable. More...
struct	functor_traits< scalar_logistic_op< T > >
struct	scalar_unary_pow_op
struct	is_floating_exactly_representable
struct	scalar_unary_pow_op< Scalar, ExponentScalar, false, false, false, false >
struct	scalar_unary_pow_op< Scalar, ExponentScalar, BaseIsInteger, true, false, false >
struct	functor_traits< scalar_unary_pow_op< Scalar, ExponentScalar > >
struct	isApprox_selector
struct	isApprox_selector< Derived, OtherDerived, true >
struct	isMuchSmallerThan_object_selector
struct	isMuchSmallerThan_object_selector< Derived, OtherDerived, true >
struct	isMuchSmallerThan_scalar_selector
struct	isMuchSmallerThan_scalar_selector< Derived, true >
struct	product_type_selector
struct	product_size_category
struct	product_type
struct	product_type_selector< M, N, 1 >
struct	product_type_selector< M, 1, 1 >
struct	product_type_selector< 1, N, 1 >
struct	product_type_selector< 1, 1, Depth >
struct	product_type_selector< 1, 1, 1 >
struct	product_type_selector< Small, 1, Small >
struct	product_type_selector< 1, Small, Small >
struct	product_type_selector< Small, Small, Small >
struct	product_type_selector< Small, Small, 1 >
struct	product_type_selector< Small, Large, 1 >
struct	product_type_selector< Large, Small, 1 >
struct	product_type_selector< 1, Large, Small >
struct	product_type_selector< 1, Large, Large >
struct	product_type_selector< 1, Small, Large >
struct	product_type_selector< Large, 1, Small >
struct	product_type_selector< Large, 1, Large >
struct	product_type_selector< Small, 1, Large >
struct	product_type_selector< Small, Small, Large >
struct	product_type_selector< Large, Small, Large >
struct	product_type_selector< Small, Large, Large >
struct	product_type_selector< Large, Large, Large >
struct	product_type_selector< Large, Small, Small >
struct	product_type_selector< Small, Large, Small >
struct	product_type_selector< Large, Large, Small >
struct	gemv_dense_selector
struct	gemv_static_vector_if
struct	gemv_static_vector_if< Scalar, Size, MaxSize, false >
struct	gemv_static_vector_if< Scalar, Size, Dynamic, true >
struct	gemv_static_vector_if< Scalar, Size, MaxSize, true >
struct	gemv_dense_selector< OnTheLeft, StorageOrder, BlasCompatible >
struct	gemv_dense_selector< OnTheRight, ColMajor, true >
struct	gemv_dense_selector< OnTheRight, RowMajor, true >
struct	gemv_dense_selector< OnTheRight, ColMajor, false >
struct	gemv_dense_selector< OnTheRight, RowMajor, false >
struct	default_packet_traits
struct	packet_traits
struct	packet_traits< const T >
struct	unpacket_traits
struct	unpacket_traits< const T >
struct	is_scalar
struct	is_degenerate_helper
struct	is_degenerate_helper< int8_t, uint8_t, true >
struct	is_degenerate_helper< int16_t, uint16_t, true >
struct	is_degenerate_helper< int32_t, uint32_t, true >
struct	is_degenerate_helper< int64_t, uint64_t, true >
struct	is_degenerate_helper< SrcPacket, TgtPacket, false >
struct	is_degenerate
struct	is_half
struct	type_casting_traits
struct	vectorized_type_casting_traits
struct	eigen_packet_wrapper
struct	preinterpret_generic
struct	preinterpret_generic< Target, Packet, false >
struct	preinterpret_generic< Packet, Packet, true >
struct	pcast_generic
struct	pcast_generic< SrcPacket, TgtPacket, false, false >
struct	pcast_generic< Packet, Packet, true, false >
struct	pcast_generic< SrcPacket, TgtPacket, true, TgtIsHalf >
struct	pcast_generic< SrcPacket, TgtPacket, false, true >
struct	ptrue_impl
struct	ptrue_impl< bool, void >
struct	ptrue_impl< T, std::enable_if_t< is_scalar< T >::value &&NumTraits< T >::RequireInitialization > >
struct	pzero_impl
struct	pzero_impl< T, std::enable_if_t< is_scalar< T >::value > >
struct	bit_and
struct	bit_or
struct	bit_xor
struct	bit_not
struct	bit_and< bool >
struct	bit_or< bool >
struct	bit_xor< bool >
struct	bit_not< bool >
struct	operator_bitwise_helper
struct	bytewise_bitwise_helper
struct	bitwise_helper
struct	bitwise_helper< T, typename std::enable_if_t< is_scalar< T >::value &&(NumTraits< T >::IsInteger||NumTraits< T >::RequireInitialization)> >
struct	pselect_impl
struct	pselect_impl< Packet, std::enable_if_t< is_scalar< Packet >::value > >
struct	pminmax_impl
struct	pminmax_impl< PropagateNaN >
struct	pminmax_impl< PropagateNumbers >
struct	nearest_integer_packetop_impl
struct	psign_impl
struct	PacketBlock
struct	Selector
struct	psignbit_impl
struct	psignbit_impl< Packet, true, IsInteger >
struct	psignbit_impl< Packet, false, false >
struct	psignbit_impl< Packet, false, true >
struct	traits< IndexedView< XprType, RowIndices, ColIndices > >
class	IndexedViewImpl
class	IndexedViewImpl< XprType, RowIndices, ColIndices, StorageKind, true >
struct	unary_evaluator< IndexedView< ArgType, RowIndices, ColIndices >, IndexBased >
struct	find_inner_product_packet_helper
struct	find_inner_product_packet_helper< Scalar, Size, Packet, false >
struct	find_inner_product_packet_helper< Scalar, Size, Packet, true >
struct	find_inner_product_packet
struct	find_inner_product_packet< Scalar, Dynamic >
struct	inner_product_assert
struct	inner_product_evaluator
struct	inner_product_impl
struct	inner_product_impl< Evaluator, false >
struct	inner_product_impl< Evaluator, true >
struct	conditional_conj
struct	conditional_conj< Scalar, true >
struct	conditional_conj< Scalar, false >
struct	scalar_inner_product_op
struct	scalar_inner_product_op< Scalar, Scalar, Conj >
struct	default_inner_product_impl
struct	dot_impl
struct	traits< Inverse< XprType > >
struct	unary_evaluator< Inverse< ArgType > >
	Default evaluator for Inverse expression. More...
struct	significant_decimals_impl
struct	traits< Map< PlainObjectType, MapOptions, StrideType > >
class	global_math_functions_filtering_base
struct	always_void
struct	global_math_functions_filtering_base< T, typename always_void< typename T::Eigen_BaseClassForSpecializationOfGlobalMathFuncImpl >::type >
struct	real_default_impl
struct	real_default_impl< Scalar, true >
struct	real_impl
struct	real_retval
struct	imag_default_impl
struct	imag_default_impl< Scalar, true >
struct	imag_impl
struct	imag_retval
struct	real_ref_impl
struct	real_ref_retval
struct	imag_ref_default_impl
struct	imag_ref_default_impl< Scalar, false >
struct	imag_ref_impl
struct	imag_ref_retval
struct	conj_default_impl
struct	conj_default_impl< Scalar, true >
struct	conj_impl
struct	conj_retval
struct	abs2_impl_default
struct	abs2_impl_default< Scalar, true >
struct	abs2_impl
struct	abs2_retval
struct	sqrt_impl
struct	sqrt_impl< std::complex< T > >
struct	sqrt_retval
struct	rsqrt_impl
struct	rsqrt_impl< std::complex< T > >
struct	rsqrt_retval
struct	norm1_default_impl
struct	norm1_default_impl< Scalar, true >
struct	norm1_default_impl< Scalar, false >
struct	norm1_impl
struct	norm1_retval
struct	hypot_impl
struct	hypot_retval
struct	cast_impl
struct	cast_impl< OldType, bool >
struct	cast_impl< OldType, NewType, typename std::enable_if_t<!NumTraits< OldType >::IsComplex &&NumTraits< NewType >::IsComplex > >
struct	arg_default_impl
struct	arg_default_impl< Scalar, true >
struct	arg_default_impl< Scalar, false >
struct	arg_impl
struct	arg_retval
struct	expm1_impl
struct	expm1_retval
struct	log_impl
struct	log_impl< std::complex< Scalar > >
struct	log1p_impl
struct	log1p_impl< std::complex< RealScalar > >
struct	log1p_retval
struct	pow_impl
struct	pow_impl< ScalarX, ScalarY, true >
struct	meta_floor_log2_selector
struct	meta_floor_log2
struct	meta_floor_log2< n, lower, upper, meta_floor_log2_move_down >
struct	meta_floor_log2< n, lower, upper, meta_floor_log2_move_up >
struct	meta_floor_log2< n, lower, upper, meta_floor_log2_terminate >
struct	meta_floor_log2< n, lower, upper, meta_floor_log2_bogus >
struct	count_bits_impl
struct	log_2_impl
struct	sign_impl
struct	sign_impl< Scalar, false, false >
struct	sign_impl< Scalar, true, IsInteger >
struct	sign_impl< bool, false, true >
struct	sign_retval
struct	negate_impl
struct	negate_impl< Scalar, true >
struct	negate_retval
struct	nearest_integer_impl
struct	nearest_integer_impl< Scalar, true >
struct	scalar_fuzzy_default_impl
struct	scalar_fuzzy_default_impl< Scalar, false, false >
struct	scalar_fuzzy_default_impl< Scalar, false, true >
struct	scalar_fuzzy_default_impl< Scalar, true, false >
struct	scalar_fuzzy_impl
struct	scalar_fuzzy_impl< bool >
struct	expm1_impl< std::complex< RealScalar > >
struct	generic_reciprocal_newton_step
struct	generic_reciprocal_newton_step< Packet, 0 >
struct	generic_rsqrt_newton_step
struct	generic_rsqrt_newton_step< Packet, 0 >
struct	generic_sqrt_newton_step
struct	traits< Matrix< Scalar_, Rows_, Cols_, Options_, MaxRows_, MaxCols_ > >
struct	traits< NestByValue< ExpressionType > >
struct	evaluator< NestByValue< ArgType > >
struct	default_digits_impl
struct	default_digits_impl< T, false, false >
struct	default_digits_impl< T, false, true >
struct	default_digits10_impl
struct	default_digits10_impl< T, false, false >
struct	default_digits10_impl< T, false, true >
struct	default_max_digits10_impl
struct	default_max_digits10_impl< T, false, false >
struct	default_max_digits10_impl< T, false, true >
struct	packetwise_redux_traits
struct	packetwise_redux_impl
struct	packetwise_redux_impl< Func, Evaluator, CompleteUnrolling >
struct	redux_vec_unroller< Func, Evaluator, Start, 0 >
struct	packetwise_redux_impl< Func, Evaluator, NoUnrolling >
struct	evaluator< PartialReduxExpr< ArgType, MemberOp, Direction > >
struct	traits< PermutationMatrix< SizeAtCompileTime, MaxSizeAtCompileTime, StorageIndex_ > >
struct	traits< Map< PermutationMatrix< SizeAtCompileTime, MaxSizeAtCompileTime, StorageIndex_ >, PacketAccess_ > >
struct	traits< PermutationWrapper< IndicesType_ > >
struct	AssignmentKind< DenseShape, PermutationShape >
struct	check_rows_cols_for_overflow
struct	check_rows_cols_for_overflow< Dynamic, MaxRowsAtCompileTime, Dynamic >
struct	check_rows_cols_for_overflow< Dynamic, Dynamic, MaxColsAtCompileTime >
struct	check_rows_cols_for_overflow< Dynamic, Dynamic, Dynamic >
struct	conservative_resize_like_impl
struct	matrix_swap_impl
struct	conservative_resize_like_impl< Derived, OtherDerived, true >
struct	matrix_swap_impl< MatrixTypeA, MatrixTypeB, true >
struct	traits< Product< Lhs, Rhs, Option > >
struct	TransposeProductEnum
struct	TransposeKind
struct	TransposeProductKind
struct	product_transpose_helper
struct	product_transpose_helper< Lhs, Rhs, Option, TransposeProductEnum::MatrixMatrix >
struct	product_transpose_helper< Lhs, Rhs, Option, TransposeProductEnum::PermutationMatrix >
struct	product_transpose_helper< Lhs, Rhs, Option, TransposeProductEnum::MatrixPermutation >
class	dense_product_base
class	dense_product_base< Lhs, Rhs, Option, InnerProduct >
struct	evaluator< Product< Lhs, Rhs, Options > >
struct	evaluator_assume_aliasing< CwiseBinaryOp< internal::scalar_product_op< Scalar1, Scalar2 >, const CwiseNullaryOp< internal::scalar_constant_op< Scalar1 >, Plain1 >, const Product< Lhs, Rhs, DefaultProduct > > >
struct	evaluator< CwiseBinaryOp< internal::scalar_product_op< Scalar1, Scalar2 >, const CwiseNullaryOp< internal::scalar_constant_op< Scalar1 >, Plain1 >, const Product< Lhs, Rhs, DefaultProduct > > >
struct	evaluator< Diagonal< const Product< Lhs, Rhs, DefaultProduct >, DiagIndex > >
struct	generic_product_impl
struct	evaluator_assume_aliasing< Product< Lhs, Rhs, DefaultProduct > >
struct	product_evaluator< Product< Lhs, Rhs, Options >, ProductTag, LhsShape, RhsShape >
struct	Assignment< DstXprType, Product< Lhs, Rhs, Options >, internal::assign_op< Scalar, Scalar >, Dense2Dense, std::enable_if_t<(Options==DefaultProduct||Options==AliasFreeProduct)> >
struct	Assignment< DstXprType, Product< Lhs, Rhs, Options >, internal::add_assign_op< Scalar, Scalar >, Dense2Dense, std::enable_if_t<(Options==DefaultProduct||Options==AliasFreeProduct)> >
struct	Assignment< DstXprType, Product< Lhs, Rhs, Options >, internal::sub_assign_op< Scalar, Scalar >, Dense2Dense, std::enable_if_t<(Options==DefaultProduct||Options==AliasFreeProduct)> >
struct	Assignment< DstXprType, CwiseBinaryOp< internal::scalar_product_op< ScalarBis, Scalar >, const CwiseNullaryOp< internal::scalar_constant_op< ScalarBis >, Plain >, const Product< Lhs, Rhs, DefaultProduct > >, AssignFunc, Dense2Dense >
struct	evaluator_assume_aliasing< CwiseBinaryOp< internal::scalar_sum_op< typename OtherXpr::Scalar, typename Product< Lhs, Rhs, DefaultProduct >::Scalar >, const OtherXpr, const Product< Lhs, Rhs, DefaultProduct > >, DenseShape >
struct	evaluator_assume_aliasing< CwiseBinaryOp< internal::scalar_difference_op< typename OtherXpr::Scalar, typename Product< Lhs, Rhs, DefaultProduct >::Scalar >, const OtherXpr, const Product< Lhs, Rhs, DefaultProduct > >, DenseShape >
struct	assignment_from_xpr_op_product
struct	generic_product_impl< Lhs, Rhs, DenseShape, DenseShape, InnerProduct >
struct	generic_product_impl< Lhs, Rhs, DenseShape, DenseShape, OuterProduct >
struct	generic_product_impl_base
struct	generic_product_impl< Lhs, Rhs, DenseShape, DenseShape, GemvProduct >
struct	generic_product_impl< Lhs, Rhs, DenseShape, DenseShape, CoeffBasedProductMode >
struct	generic_product_impl< Lhs, Rhs, DenseShape, DenseShape, LazyCoeffBasedProductMode >
struct	etor_product_coeff_impl
struct	etor_product_packet_impl
struct	product_evaluator< Product< Lhs, Rhs, LazyProduct >, ProductTag, DenseShape, DenseShape >
struct	product_evaluator< Product< Lhs, Rhs, DefaultProduct >, LazyCoeffBasedProductMode, DenseShape, DenseShape >
struct	etor_product_packet_impl< RowMajor, UnrollingIndex, Lhs, Rhs, Packet, LoadMode >
struct	etor_product_packet_impl< ColMajor, UnrollingIndex, Lhs, Rhs, Packet, LoadMode >
struct	etor_product_packet_impl< RowMajor, 1, Lhs, Rhs, Packet, LoadMode >
struct	etor_product_packet_impl< ColMajor, 1, Lhs, Rhs, Packet, LoadMode >
struct	etor_product_packet_impl< RowMajor, 0, Lhs, Rhs, Packet, LoadMode >
struct	etor_product_packet_impl< ColMajor, 0, Lhs, Rhs, Packet, LoadMode >
struct	etor_product_packet_impl< RowMajor, Dynamic, Lhs, Rhs, Packet, LoadMode >
struct	etor_product_packet_impl< ColMajor, Dynamic, Lhs, Rhs, Packet, LoadMode >
struct	triangular_product_impl
struct	generic_product_impl< Lhs, Rhs, TriangularShape, DenseShape, ProductTag >
struct	generic_product_impl< Lhs, Rhs, DenseShape, TriangularShape, ProductTag >
struct	selfadjoint_product_impl
struct	generic_product_impl< Lhs, Rhs, SelfAdjointShape, DenseShape, ProductTag >
struct	generic_product_impl< Lhs, Rhs, DenseShape, SelfAdjointShape, ProductTag >
struct	diagonal_product_evaluator_base
struct	product_evaluator< Product< Lhs, Rhs, ProductKind >, ProductTag, DiagonalShape, DenseShape >
struct	product_evaluator< Product< Lhs, Rhs, ProductKind >, ProductTag, DenseShape, DiagonalShape >
class	permutation_matrix_product
struct	permutation_matrix_product< ExpressionType, Side, Transposed, DenseShape >
struct	generic_product_impl< Lhs, Rhs, PermutationShape, MatrixShape, ProductTag >
struct	generic_product_impl< Lhs, Rhs, MatrixShape, PermutationShape, ProductTag >
struct	generic_product_impl< Inverse< Lhs >, Rhs, PermutationShape, MatrixShape, ProductTag >
struct	generic_product_impl< Lhs, Inverse< Rhs >, MatrixShape, PermutationShape, ProductTag >
class	transposition_matrix_product
struct	generic_product_impl< Lhs, Rhs, TranspositionsShape, MatrixShape, ProductTag >
struct	generic_product_impl< Lhs, Rhs, MatrixShape, TranspositionsShape, ProductTag >
struct	generic_product_impl< Transpose< Lhs >, Rhs, TranspositionsShape, MatrixShape, ProductTag >
struct	generic_product_impl< Lhs, Transpose< Rhs >, MatrixShape, TranspositionsShape, ProductTag >
struct	generic_product_impl< Lhs, Rhs, SkewSymmetricShape, MatrixShape, ProductTag >
struct	generic_product_impl< Lhs, Rhs, MatrixShape, SkewSymmetricShape, ProductTag >
struct	generic_product_impl< Lhs, Rhs, SkewSymmetricShape, SkewSymmetricShape, ProductTag >
class	gebp_traits
struct	CacheSizes
struct	RhsPanelHelper
struct	QuadPacket
struct	packet_conditional
struct	packet_conditional< GEBPPacketFull, T1, T2, T3 >
struct	packet_conditional< GEBPPacketHalf, T1, T2, T3 >
class	gebp_traits< std::complex< RealScalar >, RealScalar, ConjLhs_, false, Arch, PacketSize_ >
struct	DoublePacket
struct	unpacket_traits< DoublePacket< Packet > >
class	gebp_traits< std::complex< RealScalar >, std::complex< RealScalar >, ConjLhs_, ConjRhs_, Arch, PacketSize_ >
class	gebp_traits< RealScalar, std::complex< RealScalar >, false, ConjRhs_, Arch, PacketSize_ >
struct	gebp_kernel
struct	last_row_process_16_packets
struct	last_row_process_16_packets< LhsScalar, RhsScalar, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs, 16 >
struct	lhs_process_one_packet
struct	lhs_process_fraction_of_packet
struct	gemm_pack_lhs< Scalar, Index, DataMapper, Pack1, Pack2, Packet, ColMajor, Conjugate, PanelMode >
struct	gemm_pack_lhs< Scalar, Index, DataMapper, Pack1, Pack2, Packet, RowMajor, Conjugate, PanelMode >
struct	gemm_pack_rhs< Scalar, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode >
struct	gemm_pack_rhs< Scalar, Index, DataMapper, nr, RowMajor, Conjugate, PanelMode >
class	level3_blocking
struct	general_matrix_matrix_product< Index, LhsScalar, LhsStorageOrder, ConjugateLhs, RhsScalar, RhsStorageOrder, ConjugateRhs, RowMajor, ResInnerStride >
struct	general_matrix_matrix_product< Index, LhsScalar, LhsStorageOrder, ConjugateLhs, RhsScalar, RhsStorageOrder, ConjugateRhs, ColMajor, ResInnerStride >
struct	gemm_functor
class	gemm_blocking_space
class	gemm_blocking_space< StorageOrder, LhsScalar_, RhsScalar_, MaxRows, MaxCols, MaxDepth, KcFactor, true >
class	gemm_blocking_space< StorageOrder, LhsScalar_, RhsScalar_, MaxRows, MaxCols, MaxDepth, KcFactor, false >
struct	generic_product_impl< Lhs, Rhs, DenseShape, DenseShape, GemmProduct >
struct	tribb_kernel
struct	general_matrix_matrix_triangular_product
struct	general_matrix_matrix_triangular_product< Index, LhsScalar, LhsStorageOrder, ConjugateLhs, RhsScalar, RhsStorageOrder, ConjugateRhs, RowMajor, ResInnerStride, UpLo, Version >
struct	general_matrix_matrix_triangular_product< Index, LhsScalar, LhsStorageOrder, ConjugateLhs, RhsScalar, RhsStorageOrder, ConjugateRhs, ColMajor, ResInnerStride, UpLo, Version >
struct	general_matrix_matrix_rankupdate
struct	gemv_packet_cond
struct	gemv_packet_cond< GEMVPacketFull, T1, T2, T3 >
struct	gemv_packet_cond< GEMVPacketHalf, T1, T2, T3 >
class	gemv_traits
struct	general_matrix_vector_product< Index, LhsScalar, LhsMapper, ColMajor, ConjugateLhs, RhsScalar, RhsMapper, ConjugateRhs, Version >
struct	general_matrix_vector_product< Index, LhsScalar, LhsMapper, RowMajor, ConjugateLhs, RhsScalar, RhsMapper, ConjugateRhs, Version >
struct	general_matrix_vector_product_gemv
struct	GemmParallelInfo
struct	symm_pack_lhs
struct	symm_pack_rhs
struct	product_selfadjoint_matrix
struct	product_selfadjoint_matrix< Scalar, Index, LhsStorageOrder, LhsSelfAdjoint, ConjugateLhs, RhsStorageOrder, RhsSelfAdjoint, ConjugateRhs, RowMajor, ResInnerStride >
struct	product_selfadjoint_matrix< Scalar, Index, LhsStorageOrder, true, ConjugateLhs, RhsStorageOrder, false, ConjugateRhs, ColMajor, ResInnerStride >
struct	product_selfadjoint_matrix< Scalar, Index, LhsStorageOrder, false, ConjugateLhs, RhsStorageOrder, true, ConjugateRhs, ColMajor, ResInnerStride >
struct	selfadjoint_product_impl< Lhs, LhsMode, false, Rhs, RhsMode, false >
struct	selfadjoint_matrix_vector_product
struct	selfadjoint_product_impl< Lhs, LhsMode, false, Rhs, 0, true >
struct	selfadjoint_product_impl< Lhs, 0, true, Rhs, RhsMode, false >
struct	selfadjoint_matrix_vector_product_symv
struct	selfadjoint_rank2_update_selector
struct	selfadjoint_rank2_update_selector< Scalar, Index, UType, VType, Lower >
struct	selfadjoint_rank2_update_selector< Scalar, Index, UType, VType, Upper >
struct	product_triangular_matrix_matrix
struct	product_triangular_matrix_matrix< Scalar, Index, Mode, LhsIsTriangular, LhsStorageOrder, ConjugateLhs, RhsStorageOrder, ConjugateRhs, RowMajor, ResInnerStride, Version >
struct	product_triangular_matrix_matrix< Scalar, Index, Mode, true, LhsStorageOrder, ConjugateLhs, RhsStorageOrder, ConjugateRhs, ColMajor, ResInnerStride, Version >
struct	product_triangular_matrix_matrix< Scalar, Index, Mode, false, LhsStorageOrder, ConjugateLhs, RhsStorageOrder, ConjugateRhs, ColMajor, ResInnerStride, Version >
struct	triangular_product_impl< Mode, LhsIsTriangular, Lhs, false, Rhs, false >
struct	product_triangular_matrix_matrix_trmm
struct	triangular_matrix_vector_product
struct	triangular_matrix_vector_product< Index, Mode, LhsScalar, ConjLhs, RhsScalar, ConjRhs, ColMajor, Version >
struct	triangular_matrix_vector_product< Index, Mode, LhsScalar, ConjLhs, RhsScalar, ConjRhs, RowMajor, Version >
struct	trmv_selector
struct	triangular_product_impl< Mode, true, Lhs, false, Rhs, true >
struct	triangular_product_impl< Mode, false, Lhs, true, Rhs, false >
struct	trmv_selector< Mode, ColMajor >
struct	trmv_selector< Mode, RowMajor >
struct	triangular_matrix_vector_product_trmv
struct	triangular_solve_matrix< Scalar, Index, Side, Mode, Conjugate, TriStorageOrder, RowMajor, OtherInnerStride >
struct	triangular_solve_matrix< Scalar, Index, OnTheLeft, Mode, Conjugate, TriStorageOrder, ColMajor, OtherInnerStride >
struct	triangular_solve_matrix< Scalar, Index, OnTheRight, Mode, Conjugate, TriStorageOrder, ColMajor, OtherInnerStride >
struct	triangular_solve_vector< LhsScalar, RhsScalar, Index, OnTheRight, Mode, Conjugate, StorageOrder >
struct	triangular_solve_vector< LhsScalar, RhsScalar, Index, OnTheLeft, Mode, Conjugate, RowMajor >
struct	triangular_solve_vector< LhsScalar, RhsScalar, Index, OnTheLeft, Mode, Conjugate, ColMajor >
struct	scalar_random_op
struct	functor_traits< scalar_random_op< Scalar > >
struct	random_default_impl
struct	random_impl
struct	random_retval
struct	eigen_random_device
struct	random_bits_impl
struct	random_float_impl
struct	random_float_impl< Scalar, false >
struct	random_longdouble_impl
struct	random_longdouble_impl< false >
struct	random_float_impl< long double >
struct	random_default_impl< Scalar, false, false >
struct	random_int_impl
struct	random_int_impl< Scalar, false, true >
struct	random_int_impl< Scalar, true, true >
struct	random_int_impl< Scalar, IsSigned, false >
struct	random_default_impl< Scalar, false, true >
struct	random_impl< bool >
struct	random_default_impl< Scalar, true, false >
struct	redux_traits
struct	redux_novec_unroller
struct	redux_novec_unroller< Func, Evaluator, Start, 1 >
struct	redux_novec_unroller< Func, Evaluator, Start, 0 >
struct	redux_novec_linear_unroller
struct	redux_novec_linear_unroller< Func, Evaluator, Start, 1 >
struct	redux_novec_linear_unroller< Func, Evaluator, Start, 0 >
struct	redux_vec_unroller
struct	redux_vec_unroller< Func, Evaluator, Start, 1 >
struct	redux_vec_linear_unroller
struct	redux_vec_linear_unroller< Func, Evaluator, Start, 1 >
struct	redux_impl
struct	redux_impl< Func, Evaluator, DefaultTraversal, NoUnrolling >
struct	redux_impl< Func, Evaluator, LinearTraversal, NoUnrolling >
struct	redux_impl< Func, Evaluator, DefaultTraversal, CompleteUnrolling >
struct	redux_impl< Func, Evaluator, LinearTraversal, CompleteUnrolling >
struct	redux_impl< Func, Evaluator, LinearVectorizedTraversal, NoUnrolling >
struct	redux_impl< Func, Evaluator, SliceVectorizedTraversal, Unrolling >
struct	redux_impl< Func, Evaluator, LinearVectorizedTraversal, CompleteUnrolling >
class	redux_evaluator
struct	traits< Ref< PlainObjectType_, Options_, StrideType_ > >
struct	traits< RefBase< Derived > >
struct	traits< Replicate< MatrixType, RowFactor, ColFactor > >
struct	traits< Reshaped< XprType, Rows, Cols, Order > >
class	ReshapedImpl_dense
class	ReshapedImpl_dense< XprType, Rows, Cols, Order, false >
class	ReshapedImpl_dense< XprType, Rows, Cols, Order, true >
struct	reshaped_evaluator
struct	evaluator< Reshaped< ArgType, Rows, Cols, Order > >
struct	reshaped_evaluator< ArgType, Rows, Cols, Order, false >
struct	reshaped_evaluator< ArgType, Rows, Cols, Order, true >
struct	traits< ReturnByValue< Derived > >
struct	nested_eval< ReturnByValue< Derived >, n, PlainObject >
struct	evaluator< ReturnByValue< Derived > >
struct	traits< Reverse< MatrixType, Direction > >
struct	reverse_packet_cond< PacketType, false >
struct	vectorwise_reverse_inplace_impl
struct	vectorwise_reverse_inplace_impl< Vertical >
struct	vectorwise_reverse_inplace_impl< Horizontal >
struct	traits< Select< ConditionMatrixType, ThenMatrixType, ElseMatrixType > >
struct	traits< SelfAdjointView< MatrixType, UpLo > >
struct	evaluator_traits< SelfAdjointView< MatrixType, Mode > >
class	triangular_dense_assignment_kernel< UpLo, SelfAdjoint, SetOpposite, DstEvaluatorTypeT, SrcEvaluatorTypeT, Functor, Version >
struct	traits< SkewSymmetricMatrix3< Scalar_ > >
struct	traits< SkewSymmetricWrapper< SkewSymmetricVectorType_ > >
struct	storage_kind_to_shape< SkewSymmetricShape >
struct	SkewSymmetric2Dense
struct	AssignmentKind< DenseShape, SkewSymmetricShape >
struct	Assignment< DstXprType, SrcXprType, Functor, SkewSymmetric2Dense >
struct	solve_traits
struct	solve_traits< Decomposition, RhsType, Dense >
struct	traits< Solve< Decomposition, RhsType > >
struct	evaluator< Solve< Decomposition, RhsType > >
struct	Assignment< DstXprType, Solve< DecType, RhsType >, internal::assign_op< Scalar, Scalar >, Dense2Dense >
struct	Assignment< DstXprType, Solve< Transpose< const DecType >, RhsType >, internal::assign_op< Scalar, Scalar >, Dense2Dense >
struct	Assignment< DstXprType, Solve< CwiseUnaryOp< internal::scalar_conjugate_op< typename DecType::Scalar >, const Transpose< const DecType > >, RhsType >, internal::assign_op< Scalar, Scalar >, Dense2Dense >
struct	solve_assertion
struct	solve_assertion< Transpose< Derived > >
struct	solve_assertion< CwiseUnaryOp< Eigen::internal::scalar_conjugate_op< Scalar >, const Transpose< Derived > > >
struct	generic_xpr_base< Derived, MatrixXpr, SolverStorage >
struct	triangular_solve_vector
struct	triangular_solve_matrix
class	trsolve_traits
struct	triangular_solver_selector
struct	triangular_solver_selector< Lhs, Rhs, Side, Mode, NoUnrolling, 1 >
struct	triangular_solver_selector< Lhs, Rhs, Side, Mode, NoUnrolling, Dynamic >
struct	triangular_solver_unroller
struct	triangular_solver_unroller< Lhs, Rhs, Mode, LoopIndex, Size, false >
struct	triangular_solver_unroller< Lhs, Rhs, Mode, LoopIndex, Size, true >
struct	triangular_solver_selector< Lhs, Rhs, OnTheLeft, Mode, CompleteUnrolling, 1 >
struct	triangular_solver_selector< Lhs, Rhs, OnTheRight, Mode, CompleteUnrolling, 1 >
struct	traits< triangular_solve_retval< Side, TriangularType, Rhs > >
struct	triangular_solve_retval
struct	indexed_based_stl_iterator_traits
class	indexed_based_stl_iterator_base
class	indexed_based_stl_reverse_iterator_base
class	pointer_based_stl_iterator
struct	indexed_based_stl_iterator_traits< generic_randaccess_stl_iterator< XprType_ > >
class	generic_randaccess_stl_iterator
struct	indexed_based_stl_iterator_traits< subvector_stl_iterator< XprType_, Direction > >
class	subvector_stl_iterator
struct	indexed_based_stl_iterator_traits< subvector_stl_reverse_iterator< XprType_, Direction > >
class	subvector_stl_reverse_iterator
class	generic_dense_assignment_kernel< DstEvaluatorTypeT, SrcEvaluatorTypeT, swap_assign_op< typename DstEvaluatorTypeT::Scalar >, Specialized >
struct	traits< Transpose< MatrixType > >
struct	TransposeImpl_base
struct	TransposeImpl_base< MatrixType, false >
struct	inplace_transpose_selector
struct	inplace_transpose_selector< MatrixType, true, false >
struct	inplace_transpose_selector< MatrixType, true, true >
struct	inplace_transpose_selector< MatrixType, false, MatchPacketSize >
struct	check_transpose_aliasing_compile_time_selector
struct	check_transpose_aliasing_compile_time_selector< DestIsTransposed, CwiseBinaryOp< BinOp, DerivedA, DerivedB > >
struct	check_transpose_aliasing_run_time_selector
struct	check_transpose_aliasing_run_time_selector< Scalar, DestIsTransposed, CwiseBinaryOp< BinOp, DerivedA, DerivedB > >
struct	checkTransposeAliasing_impl
struct	checkTransposeAliasing_impl< Derived, OtherDerived, false >
struct	traits< Transpositions< SizeAtCompileTime, MaxSizeAtCompileTime, StorageIndex_ > >
struct	traits< Map< Transpositions< SizeAtCompileTime, MaxSizeAtCompileTime, StorageIndex_ >, PacketAccess_ > >
struct	traits< TranspositionsWrapper< IndicesType_ > >
struct	traits< Transpose< TranspositionsBase< Derived > > >
struct	traits< TriangularView< MatrixType, Mode_ > >
struct	evaluator_traits< TriangularView< MatrixType, Mode > >
struct	unary_evaluator< TriangularView< MatrixType, Mode >, IndexBased >
struct	Triangular2Triangular
struct	Triangular2Dense
struct	Dense2Triangular
struct	triangular_assignment_loop
class	triangular_dense_assignment_kernel
struct	AssignmentKind< TriangularShape, TriangularShape >
struct	AssignmentKind< DenseShape, TriangularShape >
struct	AssignmentKind< TriangularShape, DenseShape >
struct	Assignment< DstXprType, SrcXprType, Functor, Triangular2Triangular >
struct	Assignment< DstXprType, SrcXprType, Functor, Triangular2Dense >
struct	Assignment< DstXprType, SrcXprType, Functor, Dense2Triangular >
struct	triangular_assignment_loop< Kernel, Mode, 0, SetOpposite >
struct	triangular_assignment_loop< Kernel, Mode, Dynamic, SetOpposite >
struct	Assignment< DstXprType, Product< Lhs, Rhs, DefaultProduct >, internal::assign_op< Scalar, typename Product< Lhs, Rhs, DefaultProduct >::Scalar >, Dense2Triangular >
struct	Assignment< DstXprType, Product< Lhs, Rhs, DefaultProduct >, internal::add_assign_op< Scalar, typename Product< Lhs, Rhs, DefaultProduct >::Scalar >, Dense2Triangular >
struct	Assignment< DstXprType, Product< Lhs, Rhs, DefaultProduct >, internal::sub_assign_op< Scalar, typename Product< Lhs, Rhs, DefaultProduct >::Scalar >, Dense2Triangular >
struct	gemm_pack_rhs
struct	gemm_pack_lhs
struct	general_matrix_matrix_product
struct	general_matrix_vector_product
struct	get_factor
struct	get_factor< Scalar, typename NumTraits< Scalar >::Real >
class	BlasVectorMapper
class	BlasLinearMapper
class	BlasLinearMapper< Scalar, Index, AlignmentType >
class	blas_data_mapper
struct	PacketBlockManagement
struct	PacketBlockManagement< Index, Scalar, Packet, n, idx, RowMajor >
struct	PacketBlockManagement< Index, Scalar, Packet, n, -1, StorageOrder >
struct	PacketBlockManagement< Index, Scalar, Packet, n, -1, RowMajor >
class	blas_data_mapper< Scalar, Index, StorageOrder, AlignmentType, 1 >
class	const_blas_data_mapper
struct	blas_traits
struct	blas_traits< CwiseUnaryOp< scalar_conjugate_op< Scalar >, NestedXpr > >
struct	blas_traits< CwiseBinaryOp< scalar_product_op< Scalar >, const CwiseNullaryOp< scalar_constant_op< Scalar >, Plain >, NestedXpr > >
struct	blas_traits< CwiseBinaryOp< scalar_product_op< Scalar >, NestedXpr, const CwiseNullaryOp< scalar_constant_op< Scalar >, Plain > > >
struct	blas_traits< CwiseBinaryOp< scalar_product_op< Scalar >, const CwiseNullaryOp< scalar_constant_op< Scalar >, Plain1 >, const CwiseNullaryOp< scalar_constant_op< Scalar >, Plain2 > > >
struct	blas_traits< CwiseUnaryOp< scalar_opposite_op< Scalar >, NestedXpr > >
struct	blas_traits< Transpose< NestedXpr > >
struct	blas_traits< const T >
struct	extract_data_selector
struct	extract_data_selector< T, false >
struct	combine_scalar_factors_impl
struct	combine_scalar_factors_impl< bool, Lhs, Rhs >
struct	IndexBased
struct	IteratorBased
struct	traits
struct	traits< const T >
struct	has_direct_access
struct	accessors_level
class	kernel_retval_base
struct	kernel_retval
class	image_retval_base
struct	image_retval
struct	EnableIf
class	product_evaluator
struct	scalar_hypot_op
struct	scalar_lgamma_op
	Template functor to compute the natural log of the absolute value of Gamma of a scalar. More...
struct	scalar_digamma_op
	Template functor to compute psi, the derivative of lgamma of a scalar. More...
struct	scalar_erf_op
	Template functor to compute the error function of a scalar. More...
struct	scalar_erfc_op
	Template functor to compute the Complementary Error Function of a scalar. More...
struct	scalar_ndtri_op
	Template functor to compute the Inverse of the normal distribution function of a scalar. More...
struct	scalar_igamma_op
	Template functor to compute the incomplete gamma function igamma(a, x) More...
struct	scalar_igammac_op
	Template functor to compute the complementary incomplete gamma function igammac(a, x) More...
struct	scalar_zeta_op
	Template functor to compute the Riemann Zeta function of two arguments. More...
struct	scalar_betainc_op
	Template functor to compute the incomplete beta integral betainc(a, b, x) More...
struct	scalar_bessel_i0_op
	Template functor to compute the modified Bessel function of the first kind of order zero. More...
struct	scalar_bessel_i0e_op
	Template functor to compute the exponentially scaled modified Bessel function of the first kind of order zero. More...
struct	scalar_bessel_i1_op
	Template functor to compute the modified Bessel function of the first kind of order one. More...
struct	scalar_bessel_i1e_op
	Template functor to compute the exponentially scaled modified Bessel function of the first kind of order zero. More...
struct	scalar_bessel_j0_op
	Template functor to compute the Bessel function of the second kind of order zero. More...
struct	scalar_bessel_y0_op
	Template functor to compute the Bessel function of the second kind of order zero. More...
struct	scalar_bessel_j1_op
	Template functor to compute the Bessel function of the first kind of order one. More...
struct	scalar_bessel_y1_op
	Template functor to compute the Bessel function of the second kind of order one. More...
struct	scalar_bessel_k0_op
	Template functor to compute the modified Bessel function of the second kind of order zero. More...
struct	scalar_bessel_k0e_op
	Template functor to compute the exponentially scaled modified Bessel function of the second kind of order zero. More...
struct	scalar_bessel_k1_op
	Template functor to compute the modified Bessel function of the second kind of order one. More...
struct	scalar_bessel_k1e_op
	Template functor to compute the exponentially scaled modified Bessel function of the second kind of order one. More...
struct	inverse_impl
struct	cross_impl
struct	stem_function
struct	eigen_fill_impl
struct	eigen_zero_impl
struct	symbolic_last_tag
struct	all_t
struct	SymbolicExpressionEvaluator
struct	SymbolicExpressionEvaluator< Expr, SizeAtCompileTime, std::enable_if_t< symbolic::is_symbolic< Expr >::value > >
struct	SymbolicExpressionEvaluator< Expr, Dynamic, std::enable_if_t< symbolic::is_symbolic< Expr >::value > >
struct	SymbolicExpressionEvaluator< FixedInt< N >, SizeAtCompileTime, void >
struct	IndexedViewHelperIndicesWrapper
struct	IndexedViewHelper
class	ArithmeticSequenceRange
struct	IndexedViewHelperIndicesWrapper< ArithmeticSequence< FirstType, SizeType, IncrType >, NestedSizeAtCompileTime, void >
struct	IndexedViewHelper< ArithmeticSequenceRange< FirstAtCompileTime_, SizeAtCompileTime_, IncrAtCompileTime_ >, void >
class	SingleRange
struct	is_single_range
struct	is_single_range< SingleRange< ValueAtCompileTime > >
struct	IndexedViewHelperIndicesWrapper< SingleIndex, NestedSizeAtCompileTime, std::enable_if_t< std::is_integral< SingleIndex >::value||symbolic::is_symbolic< SingleIndex >::value > >
struct	IndexedViewHelperIndicesWrapper< FixedInt< N >, NestedSizeAtCompileTime, void >
struct	IndexedViewHelper< SingleRange< ValueAtCompileTime >, void >
class	AllRange
struct	IndexedViewHelperIndicesWrapper< all_t, NestedSizeAtCompileTime, void >
struct	IndexedViewHelper< AllRange< SizeAtCompileTime_ >, void >
struct	IndexedViewSelector
struct	IndexedViewSelector< Derived, RowIndices, ColIndices, std::enable_if_t< internal::traits< IndexedView< Derived, IvcType< RowIndices, Derived::RowsAtCompileTime >, IvcType< ColIndices, Derived::ColsAtCompileTime > > >::ReturnAsIndexedView > >
struct	IndexedViewSelector< Derived, RowIndices, ColIndices, std::enable_if_t< internal::traits< IndexedView< Derived, IvcType< RowIndices, Derived::RowsAtCompileTime >, IvcType< ColIndices, Derived::ColsAtCompileTime > > >::ReturnAsBlock > >
struct	IndexedViewSelector< Derived, RowIndices, ColIndices, std::enable_if_t< internal::traits< IndexedView< Derived, IvcType< RowIndices, Derived::RowsAtCompileTime >, IvcType< ColIndices, Derived::ColsAtCompileTime > > >::ReturnAsScalar > >
struct	VectorIndexedViewSelector
struct	VectorIndexedViewSelector< Derived, Indices, std::enable_if_t<!internal::is_single_range< IvcType< Indices, Derived::SizeAtCompileTime > >::value &&internal::IndexedViewHelper< IvcType< Indices, Derived::SizeAtCompileTime > >::IncrAtCompileTime !=1 > >
struct	VectorIndexedViewSelector< Derived, Indices, std::enable_if_t<!internal::is_single_range< IvcType< Indices, Derived::SizeAtCompileTime > >::value &&internal::IndexedViewHelper< IvcType< Indices, Derived::SizeAtCompileTime > >::IncrAtCompileTime==1 > >
struct	VectorIndexedViewSelector< Derived, Indices, std::enable_if_t< internal::is_single_range< IvcType< Indices, Derived::SizeAtCompileTime > >::value > >
class	FixedInt
class	VariableAndFixedInt
struct	get_fixed_value
struct	get_fixed_value< FixedInt< N >, Default >
struct	get_fixed_value< VariableAndFixedInt< N >, Default >
struct	get_fixed_value< variable_if_dynamic< T, N >, Default >
struct	cleanup_index_type
struct	cleanup_index_type< T, DynamicKey, std::enable_if_t< internal::is_integral< T >::value > >
struct	cleanup_index_type< VariableAndFixedInt< N >, DynamicKey >
struct	cleanup_index_type< VariableAndFixedInt< DynamicKey >, DynamicKey >
struct	cleanup_index_type< std::integral_constant< int, N >, DynamicKey >
struct	smart_copy_helper
struct	smart_copy_helper< T, true >
struct	smart_copy_helper< T, false >
struct	smart_memmove_helper
struct	smart_memmove_helper< T, true >
struct	smart_memmove_helper< T, false >
class	aligned_stack_memory_handler
class	scoped_array
struct	true_type
struct	false_type
struct	bool_constant
struct	bool_constant< true >
struct	bool_constant< false >
struct	remove_all
struct	remove_all< const T >
struct	remove_all< T const & >
struct	remove_all< T & >
struct	remove_all< T const * >
struct	remove_all< T * >
struct	is_arithmetic
struct	is_arithmetic< float >
struct	is_arithmetic< double >
struct	is_arithmetic< long double >
struct	is_arithmetic< bool >
struct	is_arithmetic< char >
struct	is_arithmetic< signed char >
struct	is_arithmetic< unsigned char >
struct	is_arithmetic< signed short >
struct	is_arithmetic< unsigned short >
struct	is_arithmetic< signed int >
struct	is_arithmetic< unsigned int >
struct	is_arithmetic< signed long >
struct	is_arithmetic< unsigned long >
struct	is_same
struct	is_same< T, T >
struct	is_void
struct	is_arithmetic< signed long long >
struct	is_arithmetic< unsigned long long >
struct	is_const
struct	is_const< T const >
struct	add_const_on_value_type
struct	add_const_on_value_type< T & >
struct	add_const_on_value_type< T * >
struct	add_const_on_value_type< T *const >
struct	add_const_on_value_type< T const *const >
class	noncopyable
struct	array_size
struct	array_size< T, std::enable_if_t<((T::SizeAtCompileTime &0)==0)> >
struct	array_size< const T(&)[N]>
struct	array_size< T(&)[N]>
struct	array_size< const std::array< T, N > >
struct	array_size< std::array< T, N > >
struct	result_of
struct	invoke_result
struct	meta_yes
struct	meta_no
struct	has_ReturnType
struct	has_nullary_operator
struct	has_unary_operator
struct	has_binary_operator
class	meta_sqrt
class	meta_sqrt< Y, InfX, SupX, true >
struct	meta_least_common_multiple
struct	meta_least_common_multiple< A, B, K, Done, false >
struct	meta_least_common_multiple< A, B, K, true, true >
struct	scalar_product_traits
struct	aligned_storage
struct	is_identically_zero_impl
struct	type_list
struct	type_list< t, tt... >
struct	numeric_list
struct	numeric_list< T, n, nn... >
struct	gen_numeric_list
struct	gen_numeric_list< T, 0, start, ii... >
struct	gen_numeric_list_reversed
struct	gen_numeric_list_reversed< T, 0, start, ii... >
struct	gen_numeric_list_swapped_pair
struct	gen_numeric_list_swapped_pair< T, 0, a, b, start, ii... >
struct	gen_numeric_list_repeated
struct	gen_numeric_list_repeated< T, 0, V, nn... >
struct	concat
struct	concat< type_list< as... >, type_list< bs... > >
struct	concat< numeric_list< T, as... >, numeric_list< T, bs... > >
struct	mconcat
struct	mconcat< a >
struct	mconcat< a, b >
struct	mconcat< a, b, cs... >
struct	take
struct	take< n, type_list< a, as... > >
struct	take< n, type_list<> >
struct	take< 0, type_list< a, as... > >
struct	take< 0, type_list<> >
struct	take< n, numeric_list< T, a, as... > >
struct	take< 0, numeric_list< T, a, as... > >
struct	take< 0, numeric_list< T > >
struct	h_skip_helper_numeric
struct	h_skip_helper_numeric< T, n, i, ii... >
struct	h_skip_helper_numeric< T, 0, i, ii... >
struct	h_skip_helper_numeric< T, n >
struct	h_skip_helper_numeric< T, 0 >
struct	h_skip_helper_type
struct	h_skip_helper_type< n, t, tt... >
struct	h_skip_helper_type< 0, t, tt... >
struct	h_skip_helper_type< n >
struct	h_skip_helper_type< 0 >
struct	h_skip
struct	skip
struct	slice
struct	get
struct	get< n, type_list< a, as... > >
struct	get< 0, type_list< a, as... > >
struct	get< n, numeric_list< T, a, as... > >
struct	get< 0, numeric_list< T, a, as... > >
struct	id_numeric
struct	id_type
struct	is_same_gf
struct	h_apply_op_helper
struct	h_apply_op_helper< true, op, additional_param, values... >
struct	h_apply_op
struct	apply_op_from_left
struct	apply_op_from_right
struct	contained_in_list
struct	contained_in_list< test, check_against, h_list, true >
struct	contained_in_list< test, check_against, type_list< a, as... >, false >
struct	contained_in_list< test, check_against, type_list< empty... >, false >
struct	contained_in_list_gf
struct	contained_in_list_gf< test, check_against, h_list, default_flags, true, last_check_flags >
struct	contained_in_list_gf< test, check_against, type_list< a, as... >, default_flags, false, last_check_flags >
struct	contained_in_list_gf< test, check_against, type_list< empty... >, default_flags, false, last_check_flags >
struct	reduce
struct	reduce< Reducer >
struct	reduce< Reducer, A >
struct	reduce< Reducer, A, Ts... >
struct	sum_op
struct	product_op
struct	logical_and_op
struct	logical_or_op
struct	equal_op
struct	not_equal_op
struct	lesser_op
struct	lesser_equal_op
struct	greater_op
struct	greater_equal_op
struct	not_op
struct	negation_op
struct	greater_equal_zero_op
struct	h_array_reduce
struct	h_array_reduce< Reducer, T, N, 0 >
struct	h_array_reduce< Reducer, T, 0 >
struct	h_repeat
struct	h_instantiate_by_c_array
struct	h_instantiate_by_c_array< InstType, ArrType, N, false, Ps... >
struct	h_instantiate_by_c_array< InstType, ArrType, N, true, Ps... >
struct	h_instantiate_by_c_array< InstType, ArrType, 0, false, Ps... >
struct	h_instantiate_by_c_array< InstType, ArrType, 0, true, Ps... >
struct	get_compiletime_reshape_size
struct	get_compiletime_reshape_size< AutoSize_t, OtherSize, TotalSize >
struct	serialize_impl
struct	serialize_impl< N, T1, Ts... >
struct	serialize_impl< 0 >
struct	convert_index_impl
struct	convert_index_impl< IndexDest, IndexSrc, true, true, true, false >
struct	convert_index_impl< IndexDest, IndexSrc, true, false, true, true >
struct	is_valid_index_type
struct	valid_indexed_view_overload
struct	promote_scalar_arg
struct	promote_scalar_arg< S, T, true >
struct	promote_scalar_arg_unsupported
struct	promote_scalar_arg< S, T, false >
struct	promote_scalar_arg_unsupported< S, T, PromotedType, true, true >
struct	promote_scalar_arg_unsupported< ExprScalar, T, PromotedType, false, true >
struct	promote_scalar_arg_unsupported< S, T, PromotedType, ConvertibleToLiteral, false >
struct	promote_scalar_arg_unsupported< S, T, S, false, true >
class	no_assignment_operator
struct	promote_index_type
class	variable_if_dynamic
class	variable_if_dynamic< T, Dynamic >
class	variable_if_dynamicindex
class	variable_if_dynamicindex< T, DynamicIndex >
struct	functor_traits
struct	nested_functor_cost
struct	nested_functor_cost< Matrix< Scalar, Rows, Cols, Options, MaxRows, MaxCols > >
struct	nested_functor_cost< Array< Scalar, Rows, Cols, Options, MaxRows, MaxCols > >
struct	nested_functor_cost< Map< PlainObjectType, MapOptions, StrideType > >
struct	nested_functor_cost< CwiseUnaryOp< Func, Xpr > >
struct	nested_functor_cost< CwiseNullaryOp< Func, Xpr > >
struct	nested_functor_cost< CwiseBinaryOp< Func, LhsXpr, RhsXpr > >
struct	nested_functor_cost< CwiseTernaryOp< Func, LhsXpr, MidXpr, RhsXpr > >
struct	functor_cost
struct	find_best_packet_helper
struct	find_best_packet_helper< Size, PacketType, true >
struct	find_best_packet_helper< Size, PacketType, false >
struct	find_best_packet
struct	find_packet_by_size_helper
struct	find_packet_by_size_helper< Size, PacketType, true >
struct	find_packet_by_size_helper< Size, PacketType, false >
struct	find_packet_by_size
struct	find_packet_by_size< T, 1 >
struct	compute_default_alignment
struct	compute_default_alignment< T, Dynamic >
class	make_proper_matrix_type
struct	size_of_xpr_at_compile_time
struct	plain_matrix_type
struct	plain_matrix_type_dense
struct	plain_matrix_type< T, Dense >
struct	plain_matrix_type< T, DiagonalShape >
struct	plain_matrix_type< T, SkewSymmetricShape >
struct	plain_matrix_type_dense< T, MatrixXpr, Flags >
struct	plain_matrix_type_dense< T, ArrayXpr, Flags >
struct	eval
struct	eval< T, Dense >
struct	eval< T, DiagonalShape >
struct	eval< T, SkewSymmetricShape >
struct	eval< Matrix< Scalar_, Rows_, Cols_, Options_, MaxRows_, MaxCols_ >, Dense >
struct	eval< Array< Scalar_, Rows_, Cols_, Options_, MaxRows_, MaxCols_ >, Dense >
struct	plain_object_eval
struct	plain_object_eval< T, Dense >
struct	plain_matrix_type_column_major
struct	plain_matrix_type_row_major
struct	ref_selector
struct	transfer_constness
struct	nested_eval
struct	dense_xpr_base
struct	dense_xpr_base< Derived, MatrixXpr >
struct	dense_xpr_base< Derived, ArrayXpr >
struct	generic_xpr_base
struct	generic_xpr_base< Derived, XprKind, Dense >
struct	cast_return_type
struct	promote_storage_type
struct	promote_storage_type< A, A >
struct	promote_storage_type< A, const A >
struct	promote_storage_type< const A, A >
struct	cwise_promote_storage_type
struct	cwise_promote_storage_type< A, A, Functor >
struct	cwise_promote_storage_type< Dense, Dense, Functor >
struct	cwise_promote_storage_type< A, Dense, Functor >
struct	cwise_promote_storage_type< Dense, B, Functor >
struct	cwise_promote_storage_type< Sparse, Dense, Functor >
struct	cwise_promote_storage_type< Dense, Sparse, Functor >
struct	cwise_promote_storage_order
struct	cwise_promote_storage_order< LhsKind, Sparse, LhsOrder, RhsOrder >
struct	cwise_promote_storage_order< Sparse, RhsKind, LhsOrder, RhsOrder >
struct	cwise_promote_storage_order< Sparse, Sparse, Order, Order >
struct	product_promote_storage_type
struct	product_promote_storage_type< A, A, ProductTag >
struct	product_promote_storage_type< Dense, Dense, ProductTag >
struct	product_promote_storage_type< A, Dense, ProductTag >
struct	product_promote_storage_type< Dense, B, ProductTag >
struct	product_promote_storage_type< A, DiagonalShape, ProductTag >
struct	product_promote_storage_type< DiagonalShape, B, ProductTag >
struct	product_promote_storage_type< Dense, DiagonalShape, ProductTag >
struct	product_promote_storage_type< DiagonalShape, Dense, ProductTag >
struct	product_promote_storage_type< A, SkewSymmetricShape, ProductTag >
struct	product_promote_storage_type< SkewSymmetricShape, B, ProductTag >
struct	product_promote_storage_type< Dense, SkewSymmetricShape, ProductTag >
struct	product_promote_storage_type< SkewSymmetricShape, Dense, ProductTag >
struct	product_promote_storage_type< SkewSymmetricShape, SkewSymmetricShape, ProductTag >
struct	product_promote_storage_type< A, PermutationStorage, ProductTag >
struct	product_promote_storage_type< PermutationStorage, B, ProductTag >
struct	product_promote_storage_type< Dense, PermutationStorage, ProductTag >
struct	product_promote_storage_type< PermutationStorage, Dense, ProductTag >
struct	plain_row_type
struct	plain_col_type
struct	plain_diag_type
struct	plain_constant_type
struct	is_lvalue
struct	is_diagonal
struct	is_diagonal< DiagonalBase< T > >
struct	is_diagonal< DiagonalWrapper< T > >
struct	is_diagonal< DiagonalMatrix< T, S > >
struct	is_identity
struct	is_identity< CwiseNullaryOp< internal::scalar_identity_op< typename T::Scalar >, T > >
struct	glue_shapes
struct	glue_shapes< DenseShape, TriangularShape >
struct	possibly_same_dense
struct	scalar_div_cost
struct	scalar_div_cost< std::complex< T >, Vectorized >
struct	scalar_div_cost< signed long, Vectorized, std::conditional_t< sizeof(long)==8, void, false_type > >
struct	scalar_div_cost< unsigned long, Vectorized, std::conditional_t< sizeof(long)==8, void, false_type > >
struct	is_block_xpr
struct	is_block_xpr< Block< XprType, BlockRows, BlockCols, InnerPanel > >
struct	is_block_xpr< const Block< XprType, BlockRows, BlockCols, InnerPanel > >
struct	block_xpr_helper
struct	block_xpr_helper< Block< XprType, BlockRows, BlockCols, InnerPanel > >
struct	block_xpr_helper< const Block< XprType, BlockRows, BlockCols, InnerPanel > >
struct	is_matrix_base_xpr
struct	is_permutation_base_xpr
struct	traits< VectorBlock< VectorType, Size > >
struct	traits< PartialReduxExpr< MatrixType, MemberOp, Direction > >
struct	member_lpnorm
struct	member_redux
struct	visitor_impl
struct	short_circuit_eval_impl
struct	short_circuit_eval_impl< Visitor, true >
struct	visitor_impl< Visitor, Derived, UnrollCount, Vectorize, false, ShortCircuitEvaluation >
struct	visitor_impl< Visitor, Derived, UnrollCount, Vectorize, true, ShortCircuitEvaluation >
struct	visitor_impl< Visitor, Derived, Dynamic, false, false, ShortCircuitEvaluation >
struct	visitor_impl< Visitor, Derived, Dynamic, true, false, ShortCircuitEvaluation >
struct	visitor_impl< Visitor, Derived, Dynamic, false, true, ShortCircuitEvaluation >
struct	visitor_impl< Visitor, Derived, Dynamic, true, true, ShortCircuitEvaluation >
class	visitor_evaluator
struct	visit_impl
struct	coeff_visitor
	Base class to implement min and max visitors. More...
struct	minmax_compare
struct	minmax_compare< Scalar, NaNPropagation, false >
struct	minmax_coeff_visitor
struct	minmax_coeff_visitor< Derived, is_min, PropagateNumbers, false >
struct	minmax_coeff_visitor< Derived, is_min, NaNPropagation, false >
struct	functor_traits< minmax_coeff_visitor< Derived, is_min, NaNPropagation > >
struct	all_visitor
struct	functor_traits< all_visitor< Scalar > >
struct	any_visitor
struct	functor_traits< any_visitor< Scalar > >
struct	count_visitor
struct	functor_traits< count_visitor< Scalar > >
struct	complex_schur_reduce_to_hessenberg
struct	complex_schur_reduce_to_hessenberg< MatrixType, false >
struct	HessenbergDecompositionMatrixHReturnType
	Expression type for return value of HessenbergDecomposition::matrixH() More...
struct	traits< HessenbergDecompositionMatrixHReturnType< MatrixType > >
struct	eigenvalues_selector
struct	eigenvalues_selector< Derived, false >
struct	direct_selfadjoint_eigenvalues
struct	direct_selfadjoint_eigenvalues< SolverType, 3, false >
struct	direct_selfadjoint_eigenvalues< SolverType, 2, false >
struct	TridiagonalizationMatrixTReturnType
	Expression type for return value of Tridiagonalization::matrixT() More...
struct	traits< TridiagonalizationMatrixTReturnType< MatrixType > >
struct	tridiagonalization_inplace_selector
struct	tridiagonalization_inplace_selector< MatrixType, 3, false >
struct	tridiagonalization_inplace_selector< MatrixType, 1, IsComplex >
struct	traits< AngleAxis< Scalar_ > >
struct	quat_product< Architecture::Target, Derived, OtherDerived, float >
struct	quat_conj< Architecture::Target, Derived, float >
struct	cross3_impl< Architecture::Target, VectorLhs, VectorRhs, float, true >
struct	traits< Homogeneous< MatrixType, Direction > >
struct	homogeneous_left_product_impl
struct	homogeneous_right_product_impl
struct	take_matrix_for_product
struct	take_matrix_for_product< Transform< Scalar, Dim, Mode, Options > >
struct	take_matrix_for_product< Transform< Scalar, Dim, Projective, Options > >
struct	traits< homogeneous_left_product_impl< Homogeneous< MatrixType, Vertical >, Lhs > >
struct	homogeneous_left_product_impl< Homogeneous< MatrixType, Vertical >, Lhs >
struct	traits< homogeneous_right_product_impl< Homogeneous< MatrixType, Horizontal >, Rhs > >
struct	homogeneous_right_product_impl< Homogeneous< MatrixType, Horizontal >, Rhs >
struct	evaluator_traits< Homogeneous< ArgType, Direction > >
struct	AssignmentKind< DenseShape, HomogeneousShape >
struct	unary_evaluator< Homogeneous< ArgType, Direction >, IndexBased >
struct	Assignment< DstXprType, Homogeneous< ArgType, Vertical >, internal::assign_op< Scalar, typename ArgType::Scalar >, Dense2Dense >
struct	Assignment< DstXprType, Homogeneous< ArgType, Horizontal >, internal::assign_op< Scalar, typename ArgType::Scalar >, Dense2Dense >
struct	generic_product_impl< Homogeneous< LhsArg, Horizontal >, Rhs, HomogeneousShape, DenseShape, ProductTag >
struct	homogeneous_right_product_refactoring_helper
struct	product_evaluator< Product< Lhs, Rhs, LazyProduct >, ProductTag, HomogeneousShape, DenseShape >
struct	generic_product_impl< Lhs, Homogeneous< RhsArg, Vertical >, DenseShape, HomogeneousShape, ProductTag >
struct	generic_product_impl< Lhs, Homogeneous< RhsArg, Vertical >, TriangularShape, HomogeneousShape, ProductTag >
struct	homogeneous_left_product_refactoring_helper
struct	product_evaluator< Product< Lhs, Rhs, LazyProduct >, ProductTag, DenseShape, HomogeneousShape >
struct	generic_product_impl< Transform< Scalar, Dim, Mode, Options >, Homogeneous< RhsArg, Vertical >, DenseShape, HomogeneousShape, ProductTag >
struct	permutation_matrix_product< ExpressionType, Side, Transposed, HomogeneousShape >
struct	cross_impl< Derived, OtherDerived, 2 >
struct	cross3_impl
struct	unitOrthogonal_selector
struct	unitOrthogonal_selector< Derived, 3 >
struct	unitOrthogonal_selector< Derived, 2 >
struct	quaternionbase_assign_impl
struct	traits< Quaternion< Scalar_, Options_ > >
struct	traits< Map< Quaternion< Scalar_ >, Options_ > >
struct	traits< Map< const Quaternion< Scalar_ >, Options_ > >
struct	quat_product
struct	quat_conj
struct	quaternionbase_assign_impl< Other, 3, 3 >
struct	quaternionbase_assign_impl< Other, 4, 1 >
struct	traits< Rotation2D< Scalar_ > >
struct	rotation_base_generic_product_selector
struct	rotation_base_generic_product_selector< RotationDerived, MatrixType, false >
struct	rotation_base_generic_product_selector< RotationDerived, DiagonalMatrix< Scalar, Dim, MaxDim >, false >
struct	rotation_base_generic_product_selector< RotationDerived, OtherVectorType, true >
struct	uniformscaling_times_affine_returntype
struct	transform_traits
struct	transform_right_product_impl
struct	transform_left_product_impl
struct	transform_transform_product_impl
struct	transform_construct_from_matrix
struct	transform_take_affine_part
struct	traits< Transform< Scalar_, Dim_, Mode_, Options_ > >
struct	transform_make_affine
struct	transform_rotation_impl
struct	transform_rotation_impl< Isometry >
struct	transform_make_affine< AffineCompact >
struct	projective_transform_inverse
struct	projective_transform_inverse< TransformType, Projective >
struct	transform_take_affine_part< Transform< Scalar, Dim, AffineCompact, Options > >
struct	transform_construct_from_matrix< Other, Mode, Options, Dim, HDim, Dim, Dim >
struct	transform_construct_from_matrix< Other, Mode, Options, Dim, HDim, Dim, HDim >
struct	transform_construct_from_matrix< Other, Mode, Options, Dim, HDim, HDim, HDim >
struct	transform_construct_from_matrix< Other, AffineCompact, Options, Dim, HDim, HDim, HDim >
struct	transform_product_result
struct	transform_right_product_impl< TransformType, MatrixType, 0, RhsCols >
struct	transform_right_product_impl< TransformType, MatrixType, 1, RhsCols >
struct	transform_right_product_impl< TransformType, MatrixType, 2, RhsCols >
struct	transform_right_product_impl< TransformType, MatrixType, 2, 1 >
struct	transform_left_product_impl< Other, Mode, Options, Dim, HDim, HDim, HDim >
struct	transform_left_product_impl< Other, AffineCompact, Options, Dim, HDim, HDim, HDim >
struct	transform_left_product_impl< Other, Mode, Options, Dim, HDim, Dim, HDim >
struct	transform_left_product_impl< Other, AffineCompact, Options, Dim, HDim, Dim, HDim >
struct	transform_left_product_impl< Other, Mode, Options, Dim, HDim, Dim, Dim >
struct	transform_transform_product_impl< Transform< Scalar, Dim, LhsMode, LhsOptions >, Transform< Scalar, Dim, RhsMode, RhsOptions >, false >
struct	transform_transform_product_impl< Transform< Scalar, Dim, LhsMode, LhsOptions >, Transform< Scalar, Dim, RhsMode, RhsOptions >, true >
struct	transform_transform_product_impl< Transform< Scalar, Dim, AffineCompact, LhsOptions >, Transform< Scalar, Dim, Projective, RhsOptions >, true >
struct	transform_transform_product_impl< Transform< Scalar, Dim, Projective, LhsOptions >, Transform< Scalar, Dim, AffineCompact, RhsOptions >, true >
struct	umeyama_transform_matrix_type
struct	decrement_size
struct	traits< HouseholderSequence< VectorsType, CoeffsType, Side > >
struct	HouseholderSequenceShape
struct	evaluator_traits< HouseholderSequence< VectorsType, CoeffsType, Side > >
struct	hseq_side_dependent_impl
struct	hseq_side_dependent_impl< VectorsType, CoeffsType, OnTheRight >
struct	matrix_type_times_scalar_type
struct	traits< BiCGSTAB< MatrixType_, Preconditioner_ > >
struct	traits< ConjugateGradient< MatrixType_, UpLo_, Preconditioner_ > >
struct	is_ref_compatible_impl
struct	is_ref_compatible
class	generic_matrix_wrapper
class	generic_matrix_wrapper< MatrixType, false >
class	generic_matrix_wrapper< MatrixType, true >
struct	traits< LeastSquaresConjugateGradient< MatrixType_, Preconditioner_ > >
struct	traits< SolveWithGuess< Decomposition, RhsType, GuessType > >
struct	evaluator< SolveWithGuess< Decomposition, RhsType, GuessType > >
struct	Assignment< DstXprType, SolveWithGuess< DecType, RhsType, GuessType >, internal::assign_op< Scalar, Scalar >, Dense2Dense >
struct	apply_rotation_in_the_plane_selector
struct	apply_rotation_in_the_plane_selector< Scalar, OtherScalar, SizeAtCompileTime, MinAlignment, true >
struct	compute_inverse_size4< Architecture::Target, float, MatrixType, ResultType >
struct	compute_inverse_size4< Architecture::Target, double, MatrixType, ResultType >
struct	determinant_impl
struct	determinant_impl< Derived, 1 >
struct	determinant_impl< Derived, 2 >
struct	determinant_impl< Derived, 3 >
struct	determinant_impl< Derived, 4 >
struct	traits< FullPivLU< MatrixType_, PermutationIndex_ > >
struct	kernel_retval< FullPivLU< MatrixType_, PermutationIndex_ > >
struct	image_retval< FullPivLU< MatrixType_, PermutationIndex_ > >
struct	Assignment< DstXprType, Inverse< FullPivLU< MatrixType, PermutationIndex > >, internal::assign_op< typename DstXprType::Scalar, typename FullPivLU< MatrixType, PermutationIndex >::Scalar >, Dense2Dense >
struct	compute_inverse
struct	compute_inverse_and_det_with_check
struct	compute_inverse< MatrixType, ResultType, 1 >
struct	compute_inverse_and_det_with_check< MatrixType, ResultType, 1 >
struct	compute_inverse< MatrixType, ResultType, 2 >
struct	compute_inverse_and_det_with_check< MatrixType, ResultType, 2 >
struct	compute_inverse< MatrixType, ResultType, 3 >
struct	compute_inverse_and_det_with_check< MatrixType, ResultType, 3 >
struct	compute_inverse_size4
struct	compute_inverse< MatrixType, ResultType, 4 >
struct	compute_inverse_and_det_with_check< MatrixType, ResultType, 4 >
struct	Assignment< DstXprType, Inverse< XprType >, internal::assign_op< typename DstXprType::Scalar, typename XprType::Scalar >, Dense2Dense >
struct	traits< PartialPivLU< MatrixType_, PermutationIndex_ > >
struct	enable_if_ref
struct	enable_if_ref< Ref< T >, Derived >
struct	partial_lu_impl
struct	Assignment< DstXprType, Inverse< PartialPivLU< MatrixType, PermutationIndex > >, internal::assign_op< typename DstXprType::Scalar, typename PartialPivLU< MatrixType, PermutationIndex >::Scalar >, Dense2Dense >
struct	traits< image_retval_base< DecompositionType > >
struct	traits< kernel_retval_base< DecompositionType > >
struct	pardiso_run_selector
struct	pardiso_run_selector< long long int >
struct	pardiso_traits
struct	pardiso_traits< PardisoLU< MatrixType_ > >
struct	pardiso_traits< PardisoLLT< MatrixType_, Options > >
struct	pardiso_traits< PardisoLDLT< MatrixType_, Options > >
struct	pastix_traits
struct	pastix_traits< PastixLU< MatrixType_ > >
struct	pastix_traits< PastixLLT< MatrixType_, Options > >
struct	pastix_traits< PastixLDLT< MatrixType_, Options > >
struct	traits< ColPivHouseholderQR< MatrixType_, PermutationIndex_ > >
struct	Assignment< DstXprType, Inverse< ColPivHouseholderQR< MatrixType, PermutationIndex > >, internal::assign_op< typename DstXprType::Scalar, typename ColPivHouseholderQR< MatrixType, PermutationIndex >::Scalar >, Dense2Dense >
struct	traits< CompleteOrthogonalDecomposition< MatrixType_, PermutationIndex_ > >
struct	traits< Inverse< CompleteOrthogonalDecomposition< MatrixType, PermutationIndex > > >
struct	Assignment< DstXprType, Inverse< CompleteOrthogonalDecomposition< MatrixType, PermutationIndex > >, internal::assign_op< typename DstXprType::Scalar, typename CompleteOrthogonalDecomposition< MatrixType, PermutationIndex >::Scalar >, Dense2Dense >
struct	traits< FullPivHouseholderQR< MatrixType_, PermutationIndex_ > >
struct	FullPivHouseholderQRMatrixQReturnType
	Expression type for return value of FullPivHouseholderQR::matrixQ() More...
struct	traits< FullPivHouseholderQRMatrixQReturnType< MatrixType, PermutationIndex > >
struct	Assignment< DstXprType, Inverse< FullPivHouseholderQR< MatrixType, PermutationIndex > >, internal::assign_op< typename DstXprType::Scalar, typename FullPivHouseholderQR< MatrixType, PermutationIndex >::Scalar >, Dense2Dense >
struct	traits< HouseholderQR< MatrixType_ > >
struct	householder_determinant
struct	householder_determinant< HCoeffs, Scalar, false >
struct	householder_qr_inplace_blocked
struct	simplicial_cholesky_grab_input
struct	simplicial_cholesky_grab_input< MatrixType, MatrixType >
struct	traits< SimplicialLLT< MatrixType_, UpLo_, Ordering_ > >
struct	traits< SimplicialLDLT< MatrixType_, UpLo_, Ordering_ > >
struct	traits< SimplicialNonHermitianLLT< MatrixType_, UpLo_, Ordering_ > >
struct	traits< SimplicialNonHermitianLDLT< MatrixType_, UpLo_, Ordering_ > >
struct	traits< SimplicialCholesky< MatrixType_, UpLo_, Ordering_ > >
struct	simpl_chol_helper
class	AmbiVector
class	CompressedStorage
struct	conservative_sparse_sparse_product_selector
struct	conservative_sparse_sparse_product_selector< Lhs, Rhs, ResultType, ColMajor, ColMajor, ColMajor >
struct	conservative_sparse_sparse_product_selector< Lhs, Rhs, ResultType, RowMajor, ColMajor, ColMajor >
struct	conservative_sparse_sparse_product_selector< Lhs, Rhs, ResultType, ColMajor, RowMajor, ColMajor >
struct	conservative_sparse_sparse_product_selector< Lhs, Rhs, ResultType, RowMajor, RowMajor, ColMajor >
struct	conservative_sparse_sparse_product_selector< Lhs, Rhs, ResultType, ColMajor, ColMajor, RowMajor >
struct	conservative_sparse_sparse_product_selector< Lhs, Rhs, ResultType, RowMajor, ColMajor, RowMajor >
struct	conservative_sparse_sparse_product_selector< Lhs, Rhs, ResultType, ColMajor, RowMajor, RowMajor >
struct	conservative_sparse_sparse_product_selector< Lhs, Rhs, ResultType, RowMajor, RowMajor, RowMajor >
struct	sparse_sparse_to_dense_product_selector
struct	sparse_sparse_to_dense_product_selector< Lhs, Rhs, ResultType, ColMajor, ColMajor >
struct	sparse_sparse_to_dense_product_selector< Lhs, Rhs, ResultType, RowMajor, ColMajor >
struct	sparse_sparse_to_dense_product_selector< Lhs, Rhs, ResultType, ColMajor, RowMajor >
struct	sparse_sparse_to_dense_product_selector< Lhs, Rhs, ResultType, RowMajor, RowMajor >
struct	storage_kind_to_evaluator_kind< Sparse >
struct	storage_kind_to_shape< Sparse >
struct	Sparse2Sparse
struct	Sparse2Dense
struct	AssignmentKind< SparseShape, SparseShape >
struct	AssignmentKind< SparseShape, SparseTriangularShape >
struct	AssignmentKind< DenseShape, SparseShape >
struct	AssignmentKind< DenseShape, SparseTriangularShape >
struct	Assignment< DstXprType, SrcXprType, Functor, Sparse2Sparse >
struct	Assignment< DstXprType, SrcXprType, Functor, Sparse2Dense, Weak >
struct	assignment_from_dense_op_sparse
struct	Assignment< DstXprType, Solve< DecType, RhsType >, internal::assign_op< Scalar, Scalar >, Sparse2Sparse >
struct	Diagonal2Sparse
struct	AssignmentKind< SparseShape, DiagonalShape >
struct	Assignment< DstXprType, SrcXprType, Functor, Diagonal2Sparse >
class	sparse_matrix_block_impl
struct	unary_evaluator< Block< ArgType, BlockRows, BlockCols, InnerPanel >, IteratorBased >
struct	unary_evaluator< Block< SparseMatrix< Scalar_, Options_, StorageIndex_ >, BlockRows, BlockCols, true >, IteratorBased >
struct	unary_evaluator< Block< const SparseMatrix< Scalar_, Options_, StorageIndex_ >, BlockRows, BlockCols, true >, IteratorBased >
struct	traits< SparseCompressedBase< Derived > >
struct	inner_sort_impl
class	StorageVal
class	StorageRef
class	CompressedStorageIterator
struct	inner_sort_impl< Derived, Comp, true >
struct	evaluator< SparseCompressedBase< Derived > >
struct	binary_sparse_evaluator
struct	binary_evaluator< CwiseBinaryOp< BinaryOp, Lhs, Rhs >, IteratorBased, IteratorBased >
struct	binary_evaluator< CwiseBinaryOp< BinaryOp, Lhs, Rhs >, IndexBased, IteratorBased >
struct	binary_evaluator< CwiseBinaryOp< BinaryOp, Lhs, Rhs >, IteratorBased, IndexBased >
struct	sparse_conjunction_evaluator
struct	binary_evaluator< CwiseBinaryOp< scalar_product_op< T1, T2 >, Lhs, Rhs >, IteratorBased, IteratorBased >
struct	binary_evaluator< CwiseBinaryOp< scalar_product_op< T1, T2 >, Lhs, Rhs >, IndexBased, IteratorBased >
struct	binary_evaluator< CwiseBinaryOp< scalar_product_op< T1, T2 >, Lhs, Rhs >, IteratorBased, IndexBased >
struct	binary_evaluator< CwiseBinaryOp< scalar_quotient_op< T1, T2 >, Lhs, Rhs >, IteratorBased, IndexBased >
struct	binary_evaluator< CwiseBinaryOp< scalar_boolean_and_op< bool >, Lhs, Rhs >, IteratorBased, IteratorBased >
struct	binary_evaluator< CwiseBinaryOp< scalar_boolean_and_op< bool >, Lhs, Rhs >, IndexBased, IteratorBased >
struct	binary_evaluator< CwiseBinaryOp< scalar_boolean_and_op< bool >, Lhs, Rhs >, IteratorBased, IndexBased >
struct	sparse_conjunction_evaluator< XprType, IteratorBased, IteratorBased >
struct	sparse_conjunction_evaluator< XprType, IndexBased, IteratorBased >
struct	sparse_conjunction_evaluator< XprType, IteratorBased, IndexBased >
struct	sparse_disjunction_evaluator
struct	sparse_disjunction_evaluator< XprType, IteratorBased, IteratorBased >
struct	sparse_disjunction_evaluator< XprType, IndexBased, IteratorBased >
struct	sparse_disjunction_evaluator< XprType, IteratorBased, IndexBased >
struct	binary_evaluator< CwiseBinaryOp< scalar_disjunction_op< DupFunc, T1, T2 >, Lhs, Rhs >, IteratorBased, IteratorBased >
struct	unary_evaluator< CwiseUnaryOp< UnaryOp, ArgType >, IteratorBased >
struct	unary_evaluator< CwiseUnaryView< ViewOp, ArgType >, IteratorBased >
struct	product_promote_storage_type< Sparse, Dense, OuterProduct >
struct	product_promote_storage_type< Dense, Sparse, OuterProduct >
struct	sparse_time_dense_product_impl
struct	sparse_time_dense_product_impl< SparseLhsType, DenseRhsType, DenseResType, typename DenseResType::Scalar, RowMajor, true >
struct	sparse_time_dense_product_impl< SparseLhsType, DenseRhsType, DenseResType, AlphaType, ColMajor, true >
struct	sparse_time_dense_product_impl< SparseLhsType, DenseRhsType, DenseResType, typename DenseResType::Scalar, RowMajor, false >
struct	sparse_time_dense_product_impl< SparseLhsType, DenseRhsType, DenseResType, typename DenseResType::Scalar, ColMajor, false >
struct	generic_product_impl< Lhs, Rhs, SparseShape, DenseShape, ProductType >
struct	generic_product_impl< Lhs, Rhs, SparseTriangularShape, DenseShape, ProductType >
struct	generic_product_impl< Lhs, Rhs, DenseShape, SparseShape, ProductType >
struct	generic_product_impl< Lhs, Rhs, DenseShape, SparseTriangularShape, ProductType >
struct	sparse_dense_outer_product_evaluator
struct	product_evaluator< Product< Lhs, Rhs, DefaultProduct >, OuterProduct, SparseShape, DenseShape >
struct	product_evaluator< Product< Lhs, Rhs, DefaultProduct >, OuterProduct, DenseShape, SparseShape >
struct	sparse_diagonal_product_evaluator
struct	product_evaluator< Product< Lhs, Rhs, DefaultProduct >, ProductTag, DiagonalShape, SparseShape >
struct	product_evaluator< Product< Lhs, Rhs, DefaultProduct >, ProductTag, SparseShape, DiagonalShape >
struct	sparse_diagonal_product_evaluator< SparseXprType, DiagonalCoeffType, SDP_AsScalarProduct >
struct	sparse_diagonal_product_evaluator< SparseXprType, DiagCoeffType, SDP_AsCwiseProduct >
struct	traits< Map< SparseMatrix< MatScalar, MatOptions, MatIndex >, Options, StrideType > >
struct	traits< Map< const SparseMatrix< MatScalar, MatOptions, MatIndex >, Options, StrideType > >
struct	evaluator< Map< SparseMatrix< MatScalar, MatOptions, MatIndex >, Options, StrideType > >
struct	evaluator< Map< const SparseMatrix< MatScalar, MatOptions, MatIndex >, Options, StrideType > >
struct	traits< SparseMatrix< Scalar_, Options_, StorageIndex_ > >
struct	traits< Diagonal< SparseMatrix< Scalar_, Options_, StorageIndex_ >, DiagIndex > >
struct	traits< Diagonal< const SparseMatrix< Scalar_, Options_, StorageIndex_ >, DiagIndex > >
struct	sparse_reserve_op
struct	functor_traits< sparse_reserve_op< Scalar > >
struct	scalar_disjunction_op
struct	functor_traits< scalar_disjunction_op< DupFunctor, LhsScalar, RhsScalar > >
struct	evaluator< SparseMatrix< Scalar_, Options_, StorageIndex_ > >
struct	XprHelper
struct	XprHelper< ExpressionType, PlainObjectType, false >
struct	PermHelper
struct	PermHelper< PermDerived, false >
struct	permutation_matrix_product< ExpressionType, Side, Transposed, SparseShape >
struct	product_promote_storage_type< Sparse, PermutationStorage, ProductTag >
struct	product_promote_storage_type< PermutationStorage, Sparse, ProductTag >
struct	product_evaluator< Product< Lhs, Rhs, AliasFreeProduct >, ProductTag, PermutationShape, SparseShape >
struct	product_evaluator< Product< Lhs, Rhs, AliasFreeProduct >, ProductTag, SparseShape, PermutationShape >
struct	generic_product_impl< Lhs, Rhs, SparseShape, SparseShape, ProductType >
struct	generic_product_impl< Lhs, Rhs, SparseShape, SparseTriangularShape, ProductType >
struct	generic_product_impl< Lhs, Rhs, SparseTriangularShape, SparseShape, ProductType >
struct	Assignment< DstXprType, Product< Lhs, Rhs, AliasFreeProduct >, internal::assign_op< typename DstXprType::Scalar, typename Product< Lhs, Rhs, AliasFreeProduct >::Scalar >, Sparse2Dense >
struct	Assignment< DstXprType, Product< Lhs, Rhs, AliasFreeProduct >, internal::add_assign_op< typename DstXprType::Scalar, typename Product< Lhs, Rhs, AliasFreeProduct >::Scalar >, Sparse2Dense >
struct	Assignment< DstXprType, Product< Lhs, Rhs, AliasFreeProduct >, internal::sub_assign_op< typename DstXprType::Scalar, typename Product< Lhs, Rhs, AliasFreeProduct >::Scalar >, Sparse2Dense >
struct	unary_evaluator< SparseView< Product< Lhs, Rhs, Options > >, IteratorBased >
class	SparseRefBase
struct	traits< Ref< SparseMatrix< MatScalar, MatOptions, MatIndex >, Options_, StrideType_ > >
struct	traits< Ref< const SparseMatrix< MatScalar, MatOptions, MatIndex >, Options_, StrideType_ > >
struct	traits< Ref< SparseVector< MatScalar, MatOptions, MatIndex >, Options_, StrideType_ > >
struct	traits< Ref< const SparseVector< MatScalar, MatOptions, MatIndex >, Options_, StrideType_ > >
struct	traits< SparseRefBase< Derived > >
struct	evaluator< Ref< SparseMatrix< MatScalar, MatOptions, MatIndex >, Options, StrideType > >
struct	evaluator< Ref< const SparseMatrix< MatScalar, MatOptions, MatIndex >, Options, StrideType > >
struct	evaluator< Ref< SparseVector< MatScalar, MatOptions, MatIndex >, Options, StrideType > >
struct	evaluator< Ref< const SparseVector< MatScalar, MatOptions, MatIndex >, Options, StrideType > >
struct	traits< SparseSelfAdjointView< MatrixType, Mode > >
struct	evaluator_traits< SparseSelfAdjointView< MatrixType, Mode > >
struct	SparseSelfAdjoint2Sparse
struct	AssignmentKind< SparseShape, SparseSelfAdjointShape >
struct	AssignmentKind< SparseSelfAdjointShape, SparseShape >
struct	Assignment< DstXprType, SrcXprType, Functor, SparseSelfAdjoint2Sparse >
struct	generic_product_impl< LhsView, Rhs, SparseSelfAdjointShape, DenseShape, ProductType >
struct	generic_product_impl< Lhs, RhsView, DenseShape, SparseSelfAdjointShape, ProductType >
struct	product_evaluator< Product< LhsView, Rhs, DefaultProduct >, ProductTag, SparseSelfAdjointShape, SparseShape >
struct	product_evaluator< Product< Lhs, RhsView, DefaultProduct >, ProductTag, SparseShape, SparseSelfAdjointShape >
struct	traits< SparseSymmetricPermutationProduct< MatrixType, Mode > >
struct	Assignment< DstXprType, SparseSymmetricPermutationProduct< MatrixType, Mode >, internal::assign_op< Scalar, typename MatrixType::Scalar >, Sparse2Sparse >
struct	sparse_sparse_product_with_pruning_selector
struct	sparse_sparse_product_with_pruning_selector< Lhs, Rhs, ResultType, ColMajor, ColMajor, ColMajor >
struct	sparse_sparse_product_with_pruning_selector< Lhs, Rhs, ResultType, ColMajor, ColMajor, RowMajor >
struct	sparse_sparse_product_with_pruning_selector< Lhs, Rhs, ResultType, RowMajor, RowMajor, RowMajor >
struct	sparse_sparse_product_with_pruning_selector< Lhs, Rhs, ResultType, RowMajor, RowMajor, ColMajor >
struct	sparse_sparse_product_with_pruning_selector< Lhs, Rhs, ResultType, ColMajor, RowMajor, RowMajor >
struct	sparse_sparse_product_with_pruning_selector< Lhs, Rhs, ResultType, RowMajor, ColMajor, RowMajor >
struct	sparse_sparse_product_with_pruning_selector< Lhs, Rhs, ResultType, ColMajor, RowMajor, ColMajor >
struct	sparse_sparse_product_with_pruning_selector< Lhs, Rhs, ResultType, RowMajor, ColMajor, ColMajor >
class	SparseTransposeImpl
class	SparseTransposeImpl< MatrixType, CompressedAccessBit >
struct	unary_evaluator< Transpose< ArgType >, IteratorBased >
struct	unary_evaluator< TriangularView< ArgType, Mode >, IteratorBased >
struct	sparse_eval
struct	eval< T, Sparse >
struct	sparse_eval< T, 1, Cols, Flags >
struct	sparse_eval< T, Rows, 1, Flags >
struct	sparse_eval< T, 1, 1, Flags >
struct	plain_matrix_type< T, Sparse >
struct	plain_object_eval< T, Sparse >
struct	solve_traits< Decomposition, RhsType, Sparse >
struct	generic_xpr_base< Derived, MatrixXpr, Sparse >
struct	SparseTriangularShape
struct	SparseSelfAdjointShape
struct	glue_shapes< SparseShape, SelfAdjointShape >
struct	glue_shapes< SparseShape, TriangularShape >
struct	LowerBoundIndex
struct	traits< SparseVector< Scalar_, Options_, StorageIndex_ > >
struct	sparse_vector_assign_selector
struct	evaluator< SparseVector< Scalar_, Options_, Index_ > >
struct	sparse_vector_assign_selector< Dest, Src, SVA_Inner >
struct	sparse_vector_assign_selector< Dest, Src, SVA_Outer >
struct	sparse_vector_assign_selector< Dest, Src, SVA_RuntimeSwitch >
struct	traits< SparseView< MatrixType > >
struct	unary_evaluator< SparseView< ArgType >, IteratorBased >
struct	unary_evaluator< SparseView< ArgType >, IndexBased >
struct	sparse_solve_triangular_selector
struct	sparse_solve_triangular_selector< Lhs, Rhs, Mode, Lower, RowMajor >
struct	sparse_solve_triangular_selector< Lhs, Rhs, Mode, Upper, RowMajor >
struct	sparse_solve_triangular_selector< Lhs, Rhs, Mode, Lower, ColMajor >
struct	sparse_solve_triangular_selector< Lhs, Rhs, Mode, Upper, ColMajor >
struct	sparse_solve_triangular_sparse_selector
struct	sparse_solve_triangular_sparse_selector< Lhs, Rhs, Mode, UpLo, ColMajor >
struct	column_dfs_traits
struct	LU_kernel_bmod
struct	LU_kernel_bmod< 1 >
struct	panel_dfs_traits
struct	LU_GlobalLU_t
struct	perfvalues
class	MappedSuperNodalMatrix
	a class to manipulate the L supernodal factor from the SparseLU factorization More...
class	SparseLUImpl
struct	traits< SparseQRMatrixQReturnType< SparseQRType > >
struct	traits< SparseQRMatrixQTransposeReturnType< SparseQRType > >
struct	traits< SparseQR_QProduct< SparseQRType, Derived > >
struct	evaluator_traits< SparseQRMatrixQReturnType< SparseQRType > >
struct	Assignment< DstXprType, SparseQRMatrixQReturnType< SparseQRType >, internal::assign_op< typename DstXprType::Scalar, typename DstXprType::Scalar >, Sparse2Sparse >
struct	Assignment< DstXprType, SparseQRMatrixQReturnType< SparseQRType >, internal::assign_op< typename DstXprType::Scalar, typename DstXprType::Scalar >, Sparse2Dense >
struct	traits< SPQRMatrixQReturnType< SPQRType > >
struct	traits< SPQRMatrixQTransposeReturnType< SPQRType > >
struct	traits< SPQR_QProduct< SPQRType, Derived > >
struct	traits< BDCSVD< MatrixType_, Options > >
struct	svd_precondition_2x2_block_to_be_real
struct	qr_preconditioner_should_do_anything
struct	qr_preconditioner_impl
class	qr_preconditioner_impl< MatrixType, Options, QRPreconditioner, Case, false >
class	qr_preconditioner_impl< MatrixType, Options, FullPivHouseholderQRPreconditioner, PreconditionIfMoreRowsThanCols, true >
class	qr_preconditioner_impl< MatrixType, Options, FullPivHouseholderQRPreconditioner, PreconditionIfMoreColsThanRows, true >
class	qr_preconditioner_impl< MatrixType, Options, ColPivHouseholderQRPreconditioner, PreconditionIfMoreRowsThanCols, true >
class	qr_preconditioner_impl< MatrixType, Options, ColPivHouseholderQRPreconditioner, PreconditionIfMoreColsThanRows, true >
class	qr_preconditioner_impl< MatrixType, Options, HouseholderQRPreconditioner, PreconditionIfMoreRowsThanCols, true >
class	qr_preconditioner_impl< MatrixType, Options, HouseholderQRPreconditioner, PreconditionIfMoreColsThanRows, true >
struct	svd_precondition_2x2_block_to_be_real< MatrixType, Options, false >
struct	svd_precondition_2x2_block_to_be_real< MatrixType, Options, true >
struct	traits< JacobiSVD< MatrixType_, Options > >
struct	traits< SVDBase< Derived > >
struct	svd_traits
class	UpperBidiagonalization
struct	cost_helper
struct	dense_assignment_loop_with_device< Kernel, CoreThreadPoolDevice, DefaultTraversal, NoUnrolling >
struct	dense_assignment_loop_with_device< Kernel, CoreThreadPoolDevice, DefaultTraversal, InnerUnrolling >
struct	dense_assignment_loop_with_device< Kernel, CoreThreadPoolDevice, InnerVectorizedTraversal, NoUnrolling >
struct	dense_assignment_loop_with_device< Kernel, CoreThreadPoolDevice, InnerVectorizedTraversal, InnerUnrolling >
struct	dense_assignment_loop_with_device< Kernel, CoreThreadPoolDevice, SliceVectorizedTraversal, NoUnrolling >
struct	dense_assignment_loop_with_device< Kernel, CoreThreadPoolDevice, LinearTraversal, NoUnrolling >
struct	dense_assignment_loop_with_device< Kernel, CoreThreadPoolDevice, LinearVectorizedTraversal, NoUnrolling >
struct	ThreadLocalNoOpInitialize
struct	ThreadLocalNoOpRelease
struct	test_signbit_op
struct	functor_traits< test_signbit_op< Scalar > >
struct	functor_traits< logical_left_shift_op< N, Scalar > >
struct	functor_traits< logical_right_shift_op< N, Scalar > >
struct	functor_traits< arithmetic_right_shift_op< N, Scalar > >
struct	scalar_dummy_op
struct	functor_traits< scalar_dummy_op< Scalar, Vectorize > >
struct	cast_impl< Real, NewType >
struct	cast_impl< Real, std::complex< Real > >
struct	extract_output_indices_helper
struct	extract_output_indices_helper< N, Idx, std::index_sequence< OutputIndices... >, T1, Ts... >
struct	extract_output_indices_helper< 0, Idx, std::index_sequence< OutputIndices... > >
struct	void_helper
struct	random_without_cast_overflow
struct	random_without_cast_overflow< SrcScalar, TgtScalar, std::enable_if_t< NumTraits< SrcScalar >::IsInteger &&NumTraits< SrcScalar >::IsSigned &&NumTraits< TgtScalar >::IsInteger &&!NumTraits< TgtScalar >::IsSigned &&(std::numeric_limits< SrcScalar >::digits< std::numeric_limits< TgtScalar >::digits||(std::numeric_limits< SrcScalar >::digits==std::numeric_limits< TgtScalar >::digits &&NumTraits< SrcScalar >::IsSigned))> >
struct	random_without_cast_overflow< SrcScalar, TgtScalar, std::enable_if_t< NumTraits< SrcScalar >::IsInteger &&!NumTraits< SrcScalar >::IsSigned &&NumTraits< TgtScalar >::IsInteger &&!NumTraits< TgtScalar >::IsSigned &&(std::numeric_limits< SrcScalar >::digits< std::numeric_limits< TgtScalar >::digits||(std::numeric_limits< SrcScalar >::digits==std::numeric_limits< TgtScalar >::digits &&NumTraits< SrcScalar >::IsSigned))> >
struct	random_without_cast_overflow< SrcScalar, TgtScalar, std::enable_if_t< NumTraits< SrcScalar >::IsInteger &&NumTraits< TgtScalar >::IsInteger &&NumTraits< TgtScalar >::IsSigned &&!NumTraits< SrcScalar >::IsSigned &&(std::numeric_limits< SrcScalar >::digits > std::numeric_limits< TgtScalar >::digits)> >
struct	random_without_cast_overflow< SrcScalar, TgtScalar, std::enable_if_t< NumTraits< SrcScalar >::IsInteger &&NumTraits< TgtScalar >::IsInteger &&!NumTraits< TgtScalar >::IsSigned &&!NumTraits< SrcScalar >::IsSigned &&(std::numeric_limits< SrcScalar >::digits > std::numeric_limits< TgtScalar >::digits)> >
struct	random_without_cast_overflow< SrcScalar, TgtScalar, std::enable_if_t< NumTraits< SrcScalar >::IsInteger &&NumTraits< TgtScalar >::IsInteger &&NumTraits< SrcScalar >::IsSigned &&(std::numeric_limits< SrcScalar >::digits > std::numeric_limits< TgtScalar >::digits)> >
struct	random_without_cast_overflow< SrcScalar, TgtScalar, std::enable_if_t< NumTraits< SrcScalar >::IsInteger &&NumTraits< TgtScalar >::IsInteger &&!NumTraits< SrcScalar >::IsSigned &&NumTraits< TgtScalar >::IsSigned &&(std::numeric_limits< SrcScalar >::digits==std::numeric_limits< TgtScalar >::digits)> >
struct	random_without_cast_overflow< SrcScalar, TgtScalar, std::enable_if_t<!NumTraits< SrcScalar >::IsInteger &&!NumTraits< SrcScalar >::IsComplex &&NumTraits< TgtScalar >::IsInteger &&(std::numeric_limits< TgtScalar >::digits<=std::numeric_limits< SrcScalar >::digits)> >
struct	random_without_cast_overflow< SrcScalar, TgtScalar, std::enable_if_t<!NumTraits< SrcScalar >::IsInteger &&!NumTraits< SrcScalar >::IsComplex &&NumTraits< TgtScalar >::IsInteger &&NumTraits< TgtScalar >::IsSigned &&(std::numeric_limits< TgtScalar >::digits > std::numeric_limits< SrcScalar >::digits)> >
struct	random_without_cast_overflow< SrcScalar, TgtScalar, std::enable_if_t<!NumTraits< SrcScalar >::IsInteger &&!NumTraits< SrcScalar >::IsComplex &&NumTraits< TgtScalar >::IsInteger &&!NumTraits< TgtScalar >::IsSigned &&(std::numeric_limits< TgtScalar >::digits > std::numeric_limits< SrcScalar >::digits)> >
struct	random_without_cast_overflow< SrcScalar, TgtScalar, std::enable_if_t< NumTraits< SrcScalar >::IsInteger &&!NumTraits< TgtScalar >::IsInteger &&!NumTraits< TgtScalar >::IsComplex > >
struct	random_without_cast_overflow< SrcScalar, TgtScalar, std::enable_if_t<!NumTraits< SrcScalar >::IsInteger &&!NumTraits< SrcScalar >::IsComplex &&!NumTraits< TgtScalar >::IsInteger &&!NumTraits< TgtScalar >::IsComplex &&(std::numeric_limits< SrcScalar >::digits > std::numeric_limits< TgtScalar >::digits)> >
struct	random_without_cast_overflow< SrcScalar, TgtScalar, std::enable_if_t< NumTraits< SrcScalar >::IsComplex &&!NumTraits< TgtScalar >::IsComplex > >
struct	random_without_cast_overflow< SrcScalar, TgtScalar, std::enable_if_t<!NumTraits< SrcScalar >::IsComplex &&NumTraits< TgtScalar >::IsComplex > >
struct	random_without_cast_overflow< SrcScalar, TgtScalar, std::enable_if_t< NumTraits< SrcScalar >::IsComplex &&NumTraits< TgtScalar >::IsComplex > >
struct	functor_traits< TrackedVisitor< T, Vectorizable > >
struct	traits< TensorIndexPairOp< XprType > >
struct	eval< TensorIndexPairOp< XprType >, Eigen::Dense >
struct	nested< TensorIndexPairOp< XprType >, 1, typename eval< TensorIndexPairOp< XprType > >::type >
struct	traits< TensorPairReducerOp< ReduceOp, Dims, XprType > >
struct	eval< TensorPairReducerOp< ReduceOp, Dims, XprType >, Eigen::Dense >
struct	nested< TensorPairReducerOp< ReduceOp, Dims, XprType >, 1, typename eval< TensorPairReducerOp< ReduceOp, Dims, XprType > >::type >
struct	traits< TensorAssignOp< LhsXprType, RhsXprType > >
struct	eval< TensorAssignOp< LhsXprType, RhsXprType >, Eigen::Dense >
struct	nested< TensorAssignOp< LhsXprType, RhsXprType >, 1, typename eval< TensorAssignOp< LhsXprType, RhsXprType > >::type >
class	TensorBlockIO
struct	TensorBlockResourceRequirements
class	TensorBlockDescriptor
class	TensorBlockMapper
class	TensorBlockScratchAllocator
class	TensorBlockNotImplemented
struct	XprScalar
struct	XprScalar< void >
class	TensorMaterializedBlock
class	TensorCwiseUnaryBlock
class	TensorCwiseBinaryBlock
class	TensorUnaryExprBlock
class	TensorTernaryExprBlock
class	StridedLinearBufferCopy
class	TensorBlockAssignment
struct	traits< TensorBroadcastingOp< Broadcast, XprType > >
struct	eval< TensorBroadcastingOp< Broadcast, XprType >, Eigen::Dense >
struct	nested< TensorBroadcastingOp< Broadcast, XprType >, 1, typename eval< TensorBroadcastingOp< Broadcast, XprType > >::type >
struct	is_input_scalar
struct	is_input_scalar< Sizes<> >
struct	is_input_scalar< Sizes< Indices... > >
struct	traits< TensorChippingOp< DimId, XprType > >
struct	eval< TensorChippingOp< DimId, XprType >, Eigen::Dense >
struct	nested< TensorChippingOp< DimId, XprType >, 1, typename eval< TensorChippingOp< DimId, XprType > >::type >
struct	DimensionId
struct	DimensionId< Dynamic >
struct	traits< TensorConcatenationOp< Axis, LhsXprType, RhsXprType > >
struct	eval< TensorConcatenationOp< Axis, LhsXprType, RhsXprType >, Eigen::Dense >
struct	nested< TensorConcatenationOp< Axis, LhsXprType, RhsXprType >, 1, typename eval< TensorConcatenationOp< Axis, LhsXprType, RhsXprType > >::type >
struct	traits< TensorContractionOp< Dimensions, LhsXprType, RhsXprType, OutputKernelType > >
struct	eval< TensorContractionOp< Dimensions, LhsXprType, RhsXprType, OutputKernelType >, Eigen::Dense >
struct	nested< TensorContractionOp< Dimensions, LhsXprType, RhsXprType, OutputKernelType >, 1, typename eval< TensorContractionOp< Dimensions, LhsXprType, RhsXprType, OutputKernelType > >::type >
struct	traits< TensorEvaluator< const TensorContractionOp< Indices_, LeftArgType_, RightArgType_, OutputKernelType_ >, Device_ > >
struct	TensorContractionBlockMemAllocator
struct	TensorContractionKernel
class	TensorContractionBlocking
struct	CoeffLoader
class	BaseTensorContractionMapper
struct	CoeffLoader< Tensor, true, MakePointer_ >
class	SimpleTensorContractionMapper
class	BaseTensorContractionMapper< Scalar, Index, side, Tensor, nocontract_t, contract_t, 1, inner_dim_contiguous, inner_dim_reordered, Alignment, MakePointer_ >
class	TensorContractionSubMapper
class	TensorContractionInputMapper
struct	TensorContractionInputMapperTrait
struct	TensorContractionInputMapperTrait< TensorContractionInputMapper< Scalar_, Index_, side_, Tensor_, nocontract_t_, contract_t_, packet_size_, inner_dim_contiguous_, inner_dim_reordered_, Alignment_, MakePointer_ > >
struct	traits< TensorConversionOp< TargetType, XprType > >
struct	eval< TensorConversionOp< TargetType, XprType >, Eigen::Dense >
struct	nested< TensorConversionOp< TargetType, XprType >, 1, typename eval< TensorConversionOp< TargetType, XprType > >::type >
struct	CoeffConv
struct	CoeffConv< SrcType, TargetType, true >
struct	PacketConv
struct	PacketConv< SrcPacket, TargetPacket, LoadMode, true, IsSameT >
struct	PacketConv< SrcPacket, TargetPacket, LoadMode, false, true >
struct	PacketConv< SrcPacket, TargetPacket, LoadMode, true, true >
class	IndexMapper
struct	traits< TensorConvolutionOp< Dimensions, InputXprType, KernelXprType > >
struct	eval< TensorConvolutionOp< Dimensions, InputXprType, KernelXprType >, Eigen::Dense >
struct	nested< TensorConvolutionOp< Dimensions, InputXprType, KernelXprType >, 1, typename eval< TensorConvolutionOp< Dimensions, InputXprType, KernelXprType > >::type >
struct	traits< TensorCustomUnaryOp< CustomUnaryFunc, XprType > >
struct	eval< TensorCustomUnaryOp< CustomUnaryFunc, XprType >, Eigen::Dense >
struct	nested< TensorCustomUnaryOp< CustomUnaryFunc, XprType > >
struct	traits< TensorCustomBinaryOp< CustomBinaryFunc, LhsXprType, RhsXprType > >
struct	eval< TensorCustomBinaryOp< CustomBinaryFunc, LhsXprType, RhsXprType >, Eigen::Dense >
struct	nested< TensorCustomBinaryOp< CustomBinaryFunc, LhsXprType, RhsXprType > >
struct	array_size< DimensionList< Index, Rank > >
struct	array_size< const DimensionList< Index, Rank > >
struct	index_known_statically_impl< DimensionList< Index, Rank > >
struct	index_known_statically_impl< const DimensionList< Index, Rank > >
struct	all_indices_known_statically_impl< DimensionList< Index, Rank > >
struct	all_indices_known_statically_impl< const DimensionList< Index, Rank > >
struct	indices_statically_known_to_increase_impl< DimensionList< Index, Rank > >
struct	indices_statically_known_to_increase_impl< const DimensionList< Index, Rank > >
struct	index_statically_eq_impl< DimensionList< Index, Rank > >
struct	index_statically_eq_impl< const DimensionList< Index, Rank > >
struct	index_statically_ne_impl< DimensionList< Index, Rank > >
struct	index_statically_ne_impl< const DimensionList< Index, Rank > >
struct	index_statically_gt_impl< DimensionList< Index, Rank > >
struct	index_statically_gt_impl< const DimensionList< Index, Rank > >
struct	index_statically_lt_impl< DimensionList< Index, Rank > >
struct	index_statically_lt_impl< const DimensionList< Index, Rank > >
struct	dget
struct	fixed_size_tensor_index_linearization_helper
struct	fixed_size_tensor_index_linearization_helper< Index, NumIndices, 0, RowMajor >
struct	fixed_size_tensor_index_extraction_helper
struct	fixed_size_tensor_index_extraction_helper< Index, 0 >
struct	tensor_index_linearization_helper
struct	tensor_index_linearization_helper< Index, NumIndices, 0, RowMajor >
struct	tensor_vsize_index_linearization_helper
struct	tensor_vsize_index_linearization_helper< Index, NumIndices, 0, RowMajor >
struct	array_size< const DSizes< DenseIndex, NumDims > >
struct	array_size< DSizes< DenseIndex, NumDims > >
struct	array_size< const Sizes< Indices... > >
struct	array_size< Sizes< Indices... > >
struct	sizes_match_below_dim
struct	sizes_match_below_dim< Dims1, Dims2, n, n >
struct	sizes_match_below_dim< Dims1, Dims2, 0, 0 >
struct	traits< TensorEvalToOp< XprType, MakePointer_ > >
struct	eval< TensorEvalToOp< XprType, MakePointer_ >, Eigen::Dense >
struct	nested< TensorEvalToOp< XprType, MakePointer_ >, 1, typename eval< TensorEvalToOp< XprType, MakePointer_ > >::type >
struct	ExpressionHasTensorBroadcastingOp
struct	ExpressionHasTensorBroadcastingOp< const TensorAssignOp< LhsXprType, RhsXprType > >
struct	ExpressionHasTensorBroadcastingOp< const TensorCwiseUnaryOp< UnaryOp, XprType > >
struct	ExpressionHasTensorBroadcastingOp< const TensorCwiseBinaryOp< BinaryOp, LhsXprType, RhsXprType > >
struct	ExpressionHasTensorBroadcastingOp< const TensorBroadcastingOp< Broadcast, XprType > >
class	TensorExecutor
class	TensorAsyncExecutor
class	TensorExecutor< Expression, DefaultDevice, true, TiledEvaluation::Off >
class	TensorExecutor< Expression, DefaultDevice, Vectorizable, TiledEvaluation::On >
struct	traits< TensorCwiseNullaryOp< NullaryOp, XprType > >
struct	traits< TensorCwiseUnaryOp< UnaryOp, XprType > >
struct	eval< TensorCwiseUnaryOp< UnaryOp, XprType >, Eigen::Dense >
struct	nested< TensorCwiseUnaryOp< UnaryOp, XprType >, 1, typename eval< TensorCwiseUnaryOp< UnaryOp, XprType > >::type >
struct	traits< TensorCwiseBinaryOp< BinaryOp, LhsXprType, RhsXprType > >
struct	eval< TensorCwiseBinaryOp< BinaryOp, LhsXprType, RhsXprType >, Eigen::Dense >
struct	nested< TensorCwiseBinaryOp< BinaryOp, LhsXprType, RhsXprType >, 1, typename eval< TensorCwiseBinaryOp< BinaryOp, LhsXprType, RhsXprType > >::type >
struct	traits< TensorCwiseTernaryOp< TernaryOp, Arg1XprType, Arg2XprType, Arg3XprType > >
struct	eval< TensorCwiseTernaryOp< TernaryOp, Arg1XprType, Arg2XprType, Arg3XprType >, Eigen::Dense >
struct	nested< TensorCwiseTernaryOp< TernaryOp, Arg1XprType, Arg2XprType, Arg3XprType >, 1, typename eval< TensorCwiseTernaryOp< TernaryOp, Arg1XprType, Arg2XprType, Arg3XprType > >::type >
struct	traits< TensorSelectOp< IfXprType, ThenXprType, ElseXprType > >
struct	eval< TensorSelectOp< IfXprType, ThenXprType, ElseXprType >, Eigen::Dense >
struct	nested< TensorSelectOp< IfXprType, ThenXprType, ElseXprType >, 1, typename eval< TensorSelectOp< IfXprType, ThenXprType, ElseXprType > >::type >
struct	traits< TensorFFTOp< FFT, XprType, FFTResultType, FFTDir > >
struct	eval< TensorFFTOp< FFT, XprType, FFTResultType, FFTDirection >, Eigen::Dense >
struct	nested< TensorFFTOp< FFT, XprType, FFTResultType, FFTDirection >, 1, typename eval< TensorFFTOp< FFT, XprType, FFTResultType, FFTDirection > >::type >
struct	traits< TensorForcedEvalOp< XprType > >
struct	eval< TensorForcedEvalOp< XprType >, Eigen::Dense >
struct	nested< TensorForcedEvalOp< XprType >, 1, typename eval< TensorForcedEvalOp< XprType > >::type >
struct	non_integral_type_placement_new
struct	non_integral_type_placement_new< Eigen::SyclDevice, CoeffReturnType >
struct	Pointer_type_promotion
struct	Pointer_type_promotion< A, A >
struct	TypeConversion
struct	IsVectorizable
struct	IsVectorizable< GpuDevice, Expression >
struct	IsTileable
struct	scalar_mod_op
	Template functor to compute the modulo between an array and a scalar. More...
struct	functor_traits< scalar_mod_op< Scalar > >
struct	scalar_mod2_op
	Template functor to compute the modulo between 2 arrays. More...
struct	functor_traits< scalar_mod2_op< Scalar > >
struct	scalar_fmod_op
struct	functor_traits< scalar_fmod_op< Scalar > >
struct	reducer_traits
struct	SumReducer
struct	reducer_traits< SumReducer< T >, Device >
struct	MeanReducer
struct	reducer_traits< MeanReducer< T >, Device >
struct	MinMaxBottomValue
struct	MinMaxBottomValue< T, true, false >
struct	MinMaxBottomValue< T, false, true >
struct	MinMaxBottomValue< T, false, false >
struct	MaxReducer
struct	reducer_traits< MaxReducer< T, NaNPropagation >, Device >
struct	MinReducer
struct	reducer_traits< MinReducer< T, NaNPropagation >, Device >
struct	ProdReducer
struct	reducer_traits< ProdReducer< T >, Device >
struct	AndReducer
struct	reducer_traits< AndReducer, Device >
struct	OrReducer
struct	reducer_traits< OrReducer, Device >
struct	ArgMaxPairReducer
struct	reducer_traits< ArgMaxPairReducer< T >, Device >
struct	ArgMinPairReducer
struct	reducer_traits< ArgMinPairReducer< T >, Device >
class	GaussianGenerator
struct	functor_traits< GaussianGenerator< T, Index, NumDims > >
struct	scalar_clamp_op
struct	functor_traits< scalar_clamp_op< Scalar > >
struct	traits< TensorGeneratorOp< Generator, XprType > >
struct	eval< TensorGeneratorOp< Generator, XprType >, Eigen::Dense >
struct	nested< TensorGeneratorOp< Generator, XprType >, 1, typename eval< TensorGeneratorOp< Generator, XprType > >::type >
struct	traits< TensorImagePatchOp< Rows, Cols, XprType > >
struct	eval< TensorImagePatchOp< Rows, Cols, XprType >, Eigen::Dense >
struct	nested< TensorImagePatchOp< Rows, Cols, XprType >, 1, typename eval< TensorImagePatchOp< Rows, Cols, XprType > >::type >
struct	ImagePatchCopyOp
struct	ImagePatchCopyOp< Self, true >
struct	ImagePatchPaddingOp
struct	is_compile_time_constant
struct	is_compile_time_constant< type2index< idx > >
struct	is_compile_time_constant< const type2index< idx > >
struct	is_compile_time_constant< type2index< idx > & >
struct	is_compile_time_constant< const type2index< idx > & >
struct	is_compile_time_constant< type2indexpair< f, s > >
struct	is_compile_time_constant< const type2indexpair< f, s > >
struct	is_compile_time_constant< type2indexpair< f, s > & >
struct	is_compile_time_constant< const type2indexpair< f, s > & >
struct	IndexTuple
struct	IndexTuple< T, O... >
struct	IndexTuple< T >
struct	IndexTupleExtractor
struct	IndexTupleExtractor< N, T, O... >
struct	IndexTupleExtractor< 0, T, O... >
struct	array_size< IndexTuple< T, O... > >
struct	array_size< const IndexTuple< T, O... > >
struct	tuple_coeff
struct	tuple_coeff< 0, ValueT >
struct	array_size< IndexList< FirstType, OtherTypes... > >
struct	array_size< const IndexList< FirstType, OtherTypes... > >
struct	array_size< IndexPairList< FirstType, OtherTypes... > >
struct	array_size< const IndexPairList< FirstType, OtherTypes... > >
struct	index_known_statically_impl
struct	index_known_statically_impl< IndexList< FirstType, OtherTypes... > >
struct	index_known_statically_impl< const IndexList< FirstType, OtherTypes... > >
struct	all_indices_known_statically_impl
struct	all_indices_known_statically_impl< IndexList< FirstType, OtherTypes... > >
struct	all_indices_known_statically_impl< const IndexList< FirstType, OtherTypes... > >
struct	indices_statically_known_to_increase_impl
struct	indices_statically_known_to_increase_impl< IndexList< FirstType, OtherTypes... > >
struct	indices_statically_known_to_increase_impl< const IndexList< FirstType, OtherTypes... > >
struct	index_statically_eq_impl
struct	index_statically_eq_impl< IndexList< FirstType, OtherTypes... > >
struct	index_statically_eq_impl< const IndexList< FirstType, OtherTypes... > >
struct	index_statically_ne_impl
struct	index_statically_ne_impl< IndexList< FirstType, OtherTypes... > >
struct	index_statically_ne_impl< const IndexList< FirstType, OtherTypes... > >
struct	index_statically_gt_impl
struct	index_statically_gt_impl< IndexList< FirstType, OtherTypes... > >
struct	index_statically_gt_impl< const IndexList< FirstType, OtherTypes... > >
struct	index_statically_lt_impl
struct	index_statically_lt_impl< IndexList< FirstType, OtherTypes... > >
struct	index_statically_lt_impl< const IndexList< FirstType, OtherTypes... > >
struct	index_pair_first_statically_eq_impl
struct	index_pair_first_statically_eq_impl< IndexPairList< FirstType, OtherTypes... > >
struct	index_pair_first_statically_eq_impl< const IndexPairList< FirstType, OtherTypes... > >
struct	index_pair_second_statically_eq_impl
struct	index_pair_second_statically_eq_impl< IndexPairList< FirstType, OtherTypes... > >
struct	index_pair_second_statically_eq_impl< const IndexPairList< FirstType, OtherTypes... > >
struct	traits< TensorInflationOp< Strides, XprType > >
struct	eval< TensorInflationOp< Strides, XprType >, Eigen::Dense >
struct	nested< TensorInflationOp< Strides, XprType >, 1, typename eval< TensorInflationOp< Strides, XprType > >::type >
struct	Initializer
struct	Initializer< Derived, 1 >
struct	Initializer< Derived, 0 >
struct	UnsignedTraits
struct	DividerTraits
struct	DividerHelper
struct	DividerHelper< 64, T >
struct	TensorIntDivisor
class	TensorIntDivisor< int32_t, true >
struct	TensorPrinter
struct	ScalarPrinter
struct	ScalarPrinter< Scalar, TensorIOFormatNumpy, std::enable_if_t< NumTraits< Scalar >::IsComplex > >
struct	ScalarPrinter< Scalar, TensorIOFormatNative, std::enable_if_t< NumTraits< Scalar >::IsComplex > >
struct	TensorPrinter< Tensor, rank, TensorIOFormatLegacy, std::enable_if_t< rank !=0 > >
struct	TensorPrinter< Tensor, 0, Format >
struct	traits< TensorLayoutSwapOp< XprType > >
struct	eval< TensorLayoutSwapOp< XprType >, Eigen::Dense >
struct	nested< TensorLayoutSwapOp< XprType >, 1, typename eval< TensorLayoutSwapOp< XprType > >::type >
struct	is_base_of
struct	traits< TensorReshapingOp< NewDimensions, XprType > >
struct	eval< TensorReshapingOp< NewDimensions, XprType >, Eigen::Dense >
struct	nested< TensorReshapingOp< NewDimensions, XprType >, 1, typename eval< TensorReshapingOp< NewDimensions, XprType > >::type >
struct	traits< TensorSlicingOp< StartIndices, Sizes, XprType > >
struct	eval< TensorSlicingOp< StartIndices, Sizes, XprType >, Eigen::Dense >
struct	nested< TensorSlicingOp< StartIndices, Sizes, XprType >, 1, typename eval< TensorSlicingOp< StartIndices, Sizes, XprType > >::type >
struct	MemcpyTriggerForSlicing
struct	traits< TensorStridingSlicingOp< StartIndices, StopIndices, Strides, XprType > >
struct	eval< TensorStridingSlicingOp< StartIndices, StopIndices, Strides, XprType >, Eigen::Dense >
struct	nested< TensorStridingSlicingOp< StartIndices, StopIndices, Strides, XprType >, 1, typename eval< TensorStridingSlicingOp< StartIndices, StopIndices, Strides, XprType > >::type >
struct	traits< TensorPaddingOp< PaddingDimensions, XprType > >
struct	eval< TensorPaddingOp< PaddingDimensions, XprType >, Eigen::Dense >
struct	nested< TensorPaddingOp< PaddingDimensions, XprType >, 1, typename eval< TensorPaddingOp< PaddingDimensions, XprType > >::type >
struct	traits< TensorPatchOp< PatchDim, XprType > >
struct	eval< TensorPatchOp< PatchDim, XprType >, Eigen::Dense >
struct	nested< TensorPatchOp< PatchDim, XprType >, 1, typename eval< TensorPatchOp< PatchDim, XprType > >::type >
class	UniformRandomGenerator
struct	functor_traits< UniformRandomGenerator< Scalar > >
class	NormalRandomGenerator
struct	functor_traits< NormalRandomGenerator< Scalar > >
struct	traits< TensorReductionOp< Op, Dims, XprType, MakePointer_ > >
struct	eval< TensorReductionOp< Op, Dims, XprType, MakePointer_ >, Eigen::Dense >
struct	nested< TensorReductionOp< Op, Dims, XprType, MakePointer_ >, 1, typename eval< TensorReductionOp< Op, Dims, XprType, MakePointer_ > >::type >
struct	DimInitializer
struct	DimInitializer< Sizes<> >
struct	are_inner_most_dims
struct	preserve_inner_most_dims
struct	are_inner_most_dims< ReducedDims, NumTensorDims, ColMajor >
struct	are_inner_most_dims< ReducedDims, NumTensorDims, RowMajor >
struct	preserve_inner_most_dims< ReducedDims, NumTensorDims, ColMajor >
struct	preserve_inner_most_dims< ReducedDims, NumTensorDims, RowMajor >
struct	GenericDimReducer
struct	GenericDimReducer< 0, Self, Op >
struct	GenericDimReducer<-1, Self, Op >
struct	InnerMostDimReducer
struct	InnerMostDimReducer< Self, Op, true, false >
struct	InnerMostDimReducer< Self, Op, false, true >
struct	InnerMostDimReducer< Self, Op, true, true >
struct	InnerMostDimPreserver
struct	InnerMostDimPreserver< DimIndex, Self, Op, true >
struct	InnerMostDimPreserver< 0, Self, Op, true >
struct	InnerMostDimPreserver<-1, Self, Op, true >
struct	FullReducer
struct	InnerReducer
struct	OuterReducer
struct	ReductionReturnType
struct	FullReducer< Self, Op, Eigen::SyclDevice, Vectorizable >
struct	OuterReducer< Self, Op, Eigen::SyclDevice >
struct	InnerReducer< Self, Op, Eigen::SyclDevice >
struct	GenericReducer< Self, Op, Eigen::SyclDevice >
class	TensorLazyBaseEvaluator
class	TensorLazyEvaluatorReadOnly
class	TensorLazyEvaluatorWritable
class	TensorLazyEvaluator
struct	traits< TensorReverseOp< ReverseDimensions, XprType > >
struct	eval< TensorReverseOp< ReverseDimensions, XprType >, Eigen::Dense >
struct	nested< TensorReverseOp< ReverseDimensions, XprType >, 1, typename eval< TensorReverseOp< ReverseDimensions, XprType > >::type >
struct	traits< TensorRollOp< RollDimensions, XprType > >
struct	eval< TensorRollOp< RollDimensions, XprType >, Eigen::Dense >
struct	nested< TensorRollOp< RollDimensions, XprType >, 1, typename eval< TensorRollOp< RollDimensions, XprType > >::type >
struct	traits< TensorScanOp< Op, XprType > >
struct	eval< TensorScanOp< Op, XprType >, Eigen::Dense >
struct	nested< TensorScanOp< Op, XprType >, 1, typename eval< TensorScanOp< Op, XprType > >::type >
struct	ReduceBlock
struct	ReduceBlock< Self, true, false >
struct	ScanLauncher
struct	ScanLauncher< Self, Reducer, Eigen::SyclDevice, vectorize >
struct	traits< TensorShufflingOp< Shuffle, XprType > >
struct	eval< TensorShufflingOp< Shuffle, XprType >, Eigen::Dense >
struct	nested< TensorShufflingOp< Shuffle, XprType >, 1, typename eval< TensorShufflingOp< Shuffle, XprType > >::type >
struct	traits< TensorStridingOp< Strides, XprType > >
struct	eval< TensorStridingOp< Strides, XprType >, Eigen::Dense >
struct	nested< TensorStridingOp< Strides, XprType >, 1, typename eval< TensorStridingOp< Strides, XprType > >::type >
struct	traits< TensorTraceOp< Dims, XprType > >
struct	eval< TensorTraceOp< Dims, XprType >, Eigen::Dense >
struct	nested< TensorTraceOp< Dims, XprType >, 1, typename eval< TensorTraceOp< Dims, XprType > >::type >
class	compute_tensor_flags
struct	traits< Tensor< Scalar_, NumIndices_, Options_, IndexType_ > >
struct	traits< TensorFixedSize< Scalar_, Dimensions, Options_, IndexType_ > >
struct	traits< TensorMap< PlainObjectType, Options_, MakePointer_ > >
struct	traits< TensorRef< PlainObjectType > >
struct	eval< Tensor< Scalar_, NumIndices_, Options, IndexType_ >, Eigen::Dense >
struct	eval< const Tensor< Scalar_, NumIndices_, Options, IndexType_ >, Eigen::Dense >
struct	eval< TensorFixedSize< Scalar_, Dimensions, Options, IndexType_ >, Eigen::Dense >
struct	eval< const TensorFixedSize< Scalar_, Dimensions, Options, IndexType_ >, Eigen::Dense >
struct	eval< TensorMap< PlainObjectType, Options, MakePointer >, Eigen::Dense >
struct	eval< const TensorMap< PlainObjectType, Options, MakePointer >, Eigen::Dense >
struct	eval< TensorRef< PlainObjectType >, Eigen::Dense >
struct	eval< const TensorRef< PlainObjectType >, Eigen::Dense >
struct	nested
struct	nested< Tensor< Scalar_, NumIndices_, Options_, IndexType_ > >
struct	nested< const Tensor< Scalar_, NumIndices_, Options_, IndexType_ > >
struct	nested< TensorFixedSize< Scalar_, Dimensions, Options, IndexType_ > >
struct	nested< const TensorFixedSize< Scalar_, Dimensions, Options, IndexType_ > >
struct	nested< TensorRef< PlainObjectType > >
struct	nested< const TensorRef< PlainObjectType > >
struct	static_val
struct	TensorUInt128
struct	traits< TensorVolumePatchOp< Planes, Rows, Cols, XprType > >
struct	eval< TensorVolumePatchOp< Planes, Rows, Cols, XprType >, Eigen::Dense >
struct	nested< TensorVolumePatchOp< Planes, Rows, Cols, XprType >, 1, typename eval< TensorVolumePatchOp< Planes, Rows, Cols, XprType > >::type >
struct	tensor_static_symgroup_permutate
struct	tensor_static_symgroup_permutate< numeric_list< int, nn... > >
struct	tensor_static_symgroup_element
struct	tensor_static_symgroup_element_ctor
struct	tensor_static_symgroup_identity_ctor
struct	tensor_static_symgroup_multiply_helper
struct	tensor_static_symgroup_multiply
struct	tensor_static_symgroup_equality
struct	tensor_static_symgroup
struct	tensor_static_symgroup_do_apply
struct	tensor_static_symgroup_do_apply< internal::type_list< first, next... > >
struct	tensor_static_symgroup_do_apply< internal::type_list< EIGEN_TPL_PP_SPEC_HACK_USE(empty)> >
class	tensor_symmetry_pre_analysis
	Pre-select whether to use a static or dynamic symmetry group. More...
struct	tensor_static_symgroup_if
struct	tensor_symmetry_calculate_flags
struct	tensor_symmetry_assign_value
struct	tensor_symmetry_num_indices
struct	tensor_symmetry_num_indices< Symmetry< One_, Two_ >, Sym... >
struct	tensor_symmetry_num_indices< AntiSymmetry< One_, Two_ >, Sym... >
struct	tensor_symmetry_num_indices< Hermiticity< One_, Two_ >, Sym... >
struct	tensor_symmetry_num_indices< AntiHermiticity< One_, Two_ >, Sym... >
struct	tensor_symmetry_pre_analysis< NumIndices >
struct	tensor_symmetry_pre_analysis< NumIndices, Gen_, Gens_... >
struct	tensor_static_symgroup_if< true, NumIndices, Gens... >
class	tensor_symmetry_value_setter
struct	auto_diff_special_op
struct	maybe_coherent_pad_helper
struct	maybe_coherent_pad_helper< DerivativeType, OtherDerivativeType, std::enable_if_t< enum_ge_not_dynamic(DerivativeType::SizeAtCompileTime, OtherDerivativeType::SizeAtCompileTime)> >
struct	auto_diff_special_op< DerivativeType, true >
struct	auto_diff_special_op< DerivativeType, false >
struct	is_identically_zero_impl< AutoDiffScalar< DerivativeType > >
struct	CoherentPadOp
struct	traits< CoherentPadOp< XprType, SizeAtCompileTime_ > >
struct	unary_evaluator< CoherentPadOp< ArgType, SizeAtCompileTime > >
struct	intersector_helper1
struct	intersector_helper2
struct	minimizer_helper1
struct	minimizer_helper2
struct	vector_int_pair
struct	get_boxes_helper
struct	get_boxes_helper< ObjectList, VolumeList, int >
struct	arpack_wrapper
struct	OP
struct	arpack_wrapper< float, float >
struct	arpack_wrapper< double, double >
struct	OP< MatrixSolver, MatrixType, Scalar, true >
struct	OP< MatrixSolver, MatrixType, Scalar, false >
struct	traits< EulerAngles< Scalar_, _System > >
struct	eulerangles_assign_impl< System, Other, 3, 3 >
struct	eulerangles_assign_impl< System, Other, 3, 1 >
struct	Abs
struct	Abs< Num, false >
struct	IsValidAxis
struct	eulerangles_assign_impl
struct	fftw_plan
struct	fftw_plan< float >
struct	fftw_plan< double >
struct	fftw_plan< long double >
struct	fftw_impl
struct	kiss_cpx_fft
struct	kissfft_impl
struct	pocketfft_impl
struct	traits< Eigen::BiCGSTABL< MatrixType_, Preconditioner_ > >
struct	traits< DGMRES< MatrixType_, Preconditioner_ > >
struct	traits< GMRES< MatrixType_, Preconditioner_ > >
struct	traits< Eigen::IDRS< MatrixType_, Preconditioner_ > >
struct	traits< IDRSTABL< MatrixType_, Preconditioner_ > >
struct	traits< MINRES< MatrixType_, UpLo_, Preconditioner_ > >
struct	traits< KroneckerProduct< Lhs_, Rhs_ > >
struct	traits< KroneckerProductSparse< Lhs_, Rhs_ > >
struct	MatrixExponentialScalingOp
	Scaling operator. More...
struct	matrix_exp_computeUV
	Compute the (17,17)-Padé approximant to the exponential. More...
struct	matrix_exp_computeUV< MatrixType, float >
struct	matrix_exp_computeUV< MatrixType, double >
struct	matrix_exp_computeUV< MatrixType, long double >
struct	is_exp_known_type
struct	is_exp_known_type< float >
struct	is_exp_known_type< double >
struct	is_exp_known_type< long double >
struct	traits< MatrixExponentialReturnValue< Derived > >
class	MatrixFunctionAtomic
	Helper class for computing matrix functions of atomic matrices. More...
struct	matrix_function_compute
	Class for computing matrix functions. More...
struct	matrix_function_compute< MatrixType, 0 >
	Partial specialization of MatrixFunction for real matrices. More...
struct	matrix_function_compute< MatrixType, 1 >
	Partial specialization of MatrixFunction for complex matrices. More...
struct	traits< MatrixFunctionReturnValue< Derived > >
struct	matrix_log_min_pade_degree
struct	matrix_log_max_pade_degree
class	MatrixLogarithmAtomic
	Helper class for computing matrix logarithm of atomic matrices. More...
struct	traits< MatrixLogarithmReturnValue< Derived > >
struct	traits< MatrixPowerParenthesesReturnValue< MatrixPowerType > >
struct	traits< MatrixPowerReturnValue< Derived > >
struct	traits< MatrixComplexPowerReturnValue< Derived > >
struct	matrix_sqrt_compute
	Helper struct for computing matrix square roots of general matrices. More...
struct	matrix_sqrt_compute< MatrixType, 0 >
struct	matrix_sqrt_compute< MatrixType, 1 >
struct	traits< MatrixSquareRootReturnValue< Derived > >
struct	decrement_if_fixed_size
class	companion
struct	traits< BlockSparseMatrix< Scalar_, _BlockAtCompileTime, Options_, Index_ > >
struct	traits< BlockSparseMatrixView< BlockSparseMatrixT > >
struct	TripletComp
struct	traits< BlockSparseTimeDenseProduct< BlockSparseMatrixT, VecType > >
struct	functor_traits< scalar_bessel_i0_op< Scalar > >
struct	functor_traits< scalar_bessel_i0e_op< Scalar > >
struct	functor_traits< scalar_bessel_i1_op< Scalar > >
struct	functor_traits< scalar_bessel_i1e_op< Scalar > >
struct	functor_traits< scalar_bessel_j0_op< Scalar > >
struct	functor_traits< scalar_bessel_y0_op< Scalar > >
struct	functor_traits< scalar_bessel_j1_op< Scalar > >
struct	functor_traits< scalar_bessel_y1_op< Scalar > >
struct	functor_traits< scalar_bessel_k0_op< Scalar > >
struct	functor_traits< scalar_bessel_k0e_op< Scalar > >
struct	functor_traits< scalar_bessel_k1_op< Scalar > >
struct	functor_traits< scalar_bessel_k1e_op< Scalar > >
struct	bessel_i0e_retval
struct	generic_i0e
struct	generic_i0e< T, float >
struct	generic_i0e< T, double >
struct	bessel_i0e_impl
struct	bessel_i0_retval
struct	generic_i0
struct	bessel_i0_impl
struct	bessel_i1e_retval
struct	generic_i1e
struct	generic_i1e< T, float >
struct	generic_i1e< T, double >
struct	bessel_i1e_impl
struct	bessel_i1_retval
struct	generic_i1
struct	bessel_i1_impl
struct	bessel_k0e_retval
struct	generic_k0e
struct	generic_k0e< T, float >
struct	generic_k0e< T, double >
struct	bessel_k0e_impl
struct	bessel_k0_retval
struct	generic_k0
struct	generic_k0< T, float >
struct	generic_k0< T, double >
struct	bessel_k0_impl
struct	bessel_k1e_retval
struct	generic_k1e
struct	generic_k1e< T, float >
struct	generic_k1e< T, double >
struct	bessel_k1e_impl
struct	bessel_k1_retval
struct	generic_k1
struct	generic_k1< T, float >
struct	generic_k1< T, double >
struct	bessel_k1_impl
struct	bessel_j0_retval
struct	generic_j0
struct	generic_j0< T, float >
struct	generic_j0< T, double >
struct	bessel_j0_impl
struct	bessel_y0_retval
struct	generic_y0
struct	generic_y0< T, float >
struct	generic_y0< T, double >
struct	bessel_y0_impl
struct	bessel_j1_retval
struct	generic_j1
struct	generic_j1< T, float >
struct	generic_j1< T, double >
struct	bessel_j1_impl
struct	bessel_y1_retval
struct	generic_y1
struct	generic_y1< T, float >
struct	generic_y1< T, double >
struct	bessel_y1_impl
struct	functor_traits< scalar_igamma_op< Scalar > >
struct	scalar_igamma_der_a_op
	Template functor to compute the derivative of the incomplete gamma function igamma_der_a(a, x) More...
struct	functor_traits< scalar_igamma_der_a_op< Scalar > >
struct	scalar_gamma_sample_der_alpha_op
	Template functor to compute the derivative of the sample of a Gamma(alpha, 1) random variable with respect to the parameter alpha gamma_sample_der_alpha(alpha, sample) More...
struct	functor_traits< scalar_gamma_sample_der_alpha_op< Scalar > >
struct	functor_traits< scalar_igammac_op< Scalar > >
struct	functor_traits< scalar_betainc_op< Scalar > >
struct	functor_traits< scalar_lgamma_op< Scalar > >
struct	functor_traits< scalar_digamma_op< Scalar > >
struct	functor_traits< scalar_zeta_op< Scalar > >
struct	scalar_polygamma_op
	Template functor to compute the polygamma function. More...
struct	functor_traits< scalar_polygamma_op< Scalar > >
struct	functor_traits< scalar_erf_op< Scalar > >
struct	functor_traits< scalar_erfc_op< Scalar > >
struct	functor_traits< scalar_ndtri_op< Scalar > >
struct	lgamma_impl
struct	lgamma_retval
struct	digamma_retval
struct	digamma_impl_maybe_poly
struct	digamma_impl_maybe_poly< float >
struct	digamma_impl_maybe_poly< double >
struct	digamma_impl
struct	generic_fast_erfc
struct	erfc_impl
struct	erfc_retval
struct	generic_fast_erf
struct	erf_impl
struct	erf_retval
struct	ndtri_retval
struct	ndtri_impl
struct	igammac_retval
struct	cephes_helper
struct	cephes_helper< float >
struct	cephes_helper< double >
struct	igammac_cf_impl
struct	igamma_series_impl
struct	igammac_impl
struct	igamma_generic_impl
struct	igamma_retval
struct	igamma_impl
struct	igamma_der_a_retval
struct	igamma_der_a_impl
struct	gamma_sample_der_alpha_retval
struct	gamma_sample_der_alpha_impl
struct	zeta_retval
struct	zeta_impl_series
struct	zeta_impl_series< float >
struct	zeta_impl_series< double >
struct	zeta_impl
struct	polygamma_retval
struct	polygamma_impl
struct	betainc_retval
struct	betainc_impl
struct	increment_if_fixed_size

Typedefs
typedef __vector float	Packet4f
typedef __vector int	Packet4i
typedef __vector unsigned int	Packet4ui
typedef __vector __bool int	Packet4bi
typedef __vector short int	Packet8s
typedef __vector unsigned short int	Packet8us
typedef __vector __bool short	Packet8bi
typedef __vector signed char	Packet16c
typedef __vector unsigned char	Packet16uc
typedef eigen_packet_wrapper< __vector unsigned short int, 0 >	Packet8bf
typedef __m256	Packet8f
typedef eigen_packet_wrapper< __m256i, 0 >	Packet8i
typedef __m256d	Packet4d
typedef eigen_packet_wrapper< __m128i, 2 >	Packet8h
typedef eigen_packet_wrapper< __m256i, 4 >	Packet8ui
typedef __m512	Packet16f
typedef __m512i	Packet16i
typedef __m512d	Packet8d
typedef eigen_packet_wrapper< __m512i, 1 >	Packet8l
typedef eigen_packet_wrapper< __m256i, 1 >	Packet16h
typedef eigen_packet_wrapper< __m256i, 2 >	Packet16bf
typedef __m512h	Packet32h
typedef Packet16f	vecFullFloat
typedef Packet8d	vecFullDouble
typedef Packet8f	vecHalfFloat
typedef Packet4d	vecHalfDouble
typedef __m128d	Packet2d
typedef eigen_packet_wrapper< __m128i, 3 >	Packet2l
typedef eigen_packet_wrapper< __m128i, 7 >	Packet2ul
typedef float32x2_t	Packet2f
typedef eigen_packet_wrapper< int32_t, 2 >	Packet4c
typedef int8x8_t	Packet8c
typedef eigen_packet_wrapper< uint32_t, 5 >	Packet4uc
typedef uint8x8_t	Packet8uc
typedef int16x4_t	Packet4s
typedef uint16x4_t	Packet4us
typedef int32x2_t	Packet2i
typedef uint32x2_t	Packet2ui
typedef eigen_packet_wrapper< uint16x4_t, 19 >	Packet4bf
typedef eigen_packet_wrapper< __m128i, 1 >	Packet16b
typedef const char *	SsePrefetchPtrType
typedef svint32_t PacketXi	__attribute__((arm_sve_vector_bits(EIGEN_ARM64_SVE_VL)))
template<typename LhsScalar , typename RhsScalar , bool UseTypedComparators>
using	cmp_return_t = typename typed_cmp_helper< LhsScalar, RhsScalar, UseTypedComparators >::type
template<bool Cond, typename T >
using	conj_expr_if = std::conditional<!Cond, const T &, CwiseUnaryOp< scalar_conjugate_op< typename traits< T >::Scalar >, T > >
template<typename Indices , int SizeAtCompileTime>
using	IvcType = typename internal::IndexedViewHelperIndicesWrapper< Indices, SizeAtCompileTime >::type
template<typename T >
using	remove_all_t = typename remove_all< T >::type
template<typename... >
using	void_t = void
template<typename T >
using	add_const_on_value_type_t = typename add_const_on_value_type< T >::type
template<bool... values>
using	reduce_all = std::is_same< std::integer_sequence< bool, values..., true >, std::integer_sequence< bool, true, values... > >
template<bool... values>
using	reduce_any = std::integral_constant< bool, !std::is_same< std::integer_sequence< bool, values..., false >, std::integer_sequence< bool, false, values... > >::value >
template<class Source , int Order>
using	WithStorageOrder = SparseMatrix< typename Source::Scalar, Order, typename Source::StorageIndex >
template<typename... Types>
using	extract_output_indices = typename extract_output_indices_helper< sizeof...(Types), 0, std::index_sequence<>, Types... >::type

Enumerations
enum	SignMatrix { PositiveSemiDef , NegativeSemiDef , ZeroSign , Indefinite }
enum	{ meta_floor_log2_terminate , meta_floor_log2_move_up , meta_floor_log2_move_down , meta_floor_log2_bogus }
enum	PermPermProduct_t { PermPermProduct }
enum	GEBPPacketSizeType { GEBPPacketFull = 0 , GEBPPacketHalf , GEBPPacketQuarter }
enum	GEMVPacketSizeType { GEMVPacketFull = 0 , GEMVPacketHalf , GEMVPacketQuarter }
enum	ComparisonName : unsigned int {   cmp_EQ = 0 , cmp_LT = 1 , cmp_LE = 2 , cmp_UNORD = 3 ,   cmp_NEQ = 4 , cmp_GT = 5 , cmp_GE = 6 }
enum	{ SDP_AsScalarProduct , SDP_AsCwiseProduct }
enum	{ SVA_RuntimeSwitch , SVA_Inner , SVA_Outer }
enum	{ LUNoMarker = 3 }
enum	{ emptyIdxLU = -1 }
enum	MemType {   LUSUP , UCOL , LSUB , USUB ,   LLVL , ULVL }
enum	{ PreconditionIfMoreColsThanRows , PreconditionIfMoreRowsThanCols }
enum	OptionsMasks { QRPreconditionerBits , ComputationOptionsBits = ComputeThinU | ComputeFullU | ComputeThinV | ComputeFullV }
enum class	TensorBlockShapeType { kUniformAllDims , kSkewedInnerDims }
enum	TensorBlockKind { kExpr , kView , kMaterializedInScratch , kMaterializedInOutput }
enum	{ ShardByRow = 0 , ShardByCol = 1 }
enum	{ Rhs = 0 , Lhs = 1 }
enum	TiledEvaluation { Off = 0 , On = 1 }
enum	IgammaComputationMode { VALUE , DERIVATIVE , SAMPLE_DERIVATIVE }

Functions
template<typename MatrixType , typename VectorType >
static Index	llt_rank_update_lower (MatrixType &mat, const VectorType &vec, const typename MatrixType::RealScalar &sigma)
template<typename StorageIndex_ >
cholmod_dense *	cm_solve (int sys, cholmod_factor &L, cholmod_dense &B, cholmod_common &Common)
template<>
cholmod_dense *	cm_solve< SuiteSparse_long > (int sys, cholmod_factor &L, cholmod_dense &B, cholmod_common &Common)
template<typename StorageIndex_ >
cholmod_sparse *	cm_spsolve (int sys, cholmod_factor &L, cholmod_sparse &B, cholmod_common &Common)
template<>
cholmod_sparse *	cm_spsolve< SuiteSparse_long > (int sys, cholmod_factor &L, cholmod_sparse &B, cholmod_common &Common)
template<typename StorageIndex_ >
int	cm_factorize_p (cholmod_sparse *A, double beta[2], StorageIndex_ *fset, std::size_t fsize, cholmod_factor *L, cholmod_common &Common)
template<>
int	cm_factorize_p< SuiteSparse_long > (cholmod_sparse *A, double beta[2], SuiteSparse_long *fset, std::size_t fsize, cholmod_factor *L, cholmod_common &Common)
template<>
EIGEN_STRONG_INLINE Packet2cf	pset1< Packet2cf > (const std::complex< float > &from)
template<>
EIGEN_STRONG_INLINE Packet2cf	pload< Packet2cf > (const std::complex< float > *from)
template<>
EIGEN_STRONG_INLINE Packet2cf	ploadu< Packet2cf > (const std::complex< float > *from)
template<>
EIGEN_ALWAYS_INLINE Packet2cf	pload_partial< Packet2cf > (const std::complex< float > *from, const Index n, const Index offset)
template<>
EIGEN_ALWAYS_INLINE Packet2cf	ploadu_partial< Packet2cf > (const std::complex< float > *from, const Index n, const Index offset)
template<>
EIGEN_STRONG_INLINE Packet2cf	ploaddup< Packet2cf > (const std::complex< float > *from)
template<>
EIGEN_STRONG_INLINE void	pstore< std::complex< float > > (std::complex< float > *to, const Packet2cf &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< std::complex< float > > (std::complex< float > *to, const Packet2cf &from)
template<>
EIGEN_ALWAYS_INLINE void	pstore_partial< std::complex< float > > (std::complex< float > *to, const Packet2cf &from, const Index n, const Index offset)
template<>
EIGEN_ALWAYS_INLINE void	pstoreu_partial< std::complex< float > > (std::complex< float > *to, const Packet2cf &from, const Index n, const Index offset)
EIGEN_STRONG_INLINE Packet2cf	pload2 (const std::complex< float > &from0, const std::complex< float > &from1)
template<>
EIGEN_ALWAYS_INLINE Packet2cf	pload_ignore< Packet2cf > (const std::complex< float > *from)
template<typename Scalar , typename Packet >
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet	pgather_complex_size2 (const Scalar *from, Index stride, const Index n=2)
template<>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet2cf	pgather< std::complex< float >, Packet2cf > (const std::complex< float > *from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet2cf	pgather_partial< std::complex< float >, Packet2cf > (const std::complex< float > *from, Index stride, const Index n)
template<typename Scalar , typename Packet >
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void	pscatter_complex_size2 (Scalar *to, const Packet &from, Index stride, const Index n=2)
template<>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void	pscatter< std::complex< float >, Packet2cf > (std::complex< float > *to, const Packet2cf &from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void	pscatter_partial< std::complex< float >, Packet2cf > (std::complex< float > *to, const Packet2cf &from, Index stride, const Index n)
template<>
EIGEN_STRONG_INLINE Packet2cf	padd< Packet2cf > (const Packet2cf &a, const Packet2cf &b)
template<>
EIGEN_STRONG_INLINE Packet2cf	psub< Packet2cf > (const Packet2cf &a, const Packet2cf &b)
template<>
EIGEN_STRONG_INLINE Packet2cf	pnegate (const Packet2cf &a)
template<>
EIGEN_STRONG_INLINE Packet2cf	pconj (const Packet2cf &a)
template<>
EIGEN_STRONG_INLINE Packet2cf	pand< Packet2cf > (const Packet2cf &a, const Packet2cf &b)
template<>
EIGEN_STRONG_INLINE Packet2cf	por< Packet2cf > (const Packet2cf &a, const Packet2cf &b)
template<>
EIGEN_STRONG_INLINE Packet2cf	pxor< Packet2cf > (const Packet2cf &a, const Packet2cf &b)
template<>
EIGEN_STRONG_INLINE Packet2cf	pandnot< Packet2cf > (const Packet2cf &a, const Packet2cf &b)
template<>
EIGEN_STRONG_INLINE void	prefetch< std::complex< float > > (const std::complex< float > *addr)
template<>
EIGEN_STRONG_INLINE std::complex< float >	pfirst< Packet2cf > (const Packet2cf &a)
template<>
EIGEN_STRONG_INLINE Packet2cf	preverse (const Packet2cf &a)
template<>
EIGEN_STRONG_INLINE std::complex< float >	predux< Packet2cf > (const Packet2cf &a)
template<>
EIGEN_STRONG_INLINE std::complex< float >	predux_mul< Packet2cf > (const Packet2cf &a)
template<>
EIGEN_STRONG_INLINE Packet2cf	pdiv< Packet2cf > (const Packet2cf &a, const Packet2cf &b)
template<>
EIGEN_STRONG_INLINE Packet2cf	pcplxflip< Packet2cf > (const Packet2cf &x)
EIGEN_STRONG_INLINE void	ptranspose (PacketBlock< Packet2cf, 2 > &kernel)
template<>
EIGEN_STRONG_INLINE Packet2cf	pcmp_eq (const Packet2cf &a, const Packet2cf &b)
template<>
EIGEN_STRONG_INLINE Packet2cf	psqrt< Packet2cf > (const Packet2cf &a)
template<>
EIGEN_STRONG_INLINE Packet2cf	plog< Packet2cf > (const Packet2cf &a)
template<>
EIGEN_STRONG_INLINE Packet2cf	pexp< Packet2cf > (const Packet2cf &a)
template<>
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet4f	psqrt< Packet4f > (const Packet4f &x)
template<typename Scalar , int StorageOrder>
EIGEN_ALWAYS_INLINE std::complex< Scalar >	getAdjointVal (Index i, Index j, const_blas_data_mapper< std::complex< Scalar >, Index, StorageOrder > &dt)
template<typename Scalar , int StorageOrder, int N>
EIGEN_STRONG_INLINE void	symm_pack_complex_rhs_helper (std::complex< Scalar > *blockB, const std::complex< Scalar > *_rhs, Index rhsStride, Index rows, Index cols, Index k2)
template<typename Scalar , int StorageOrder>
EIGEN_STRONG_INLINE void	symm_pack_complex_lhs_helper (std::complex< Scalar > *blockA, const std::complex< Scalar > *_lhs, Index lhsStride, Index cols, Index rows)
template<typename Scalar , int StorageOrder, int N>
EIGEN_STRONG_INLINE void	symm_pack_rhs_helper (Scalar *blockB, const Scalar *_rhs, Index rhsStride, Index rows, Index cols, Index k2)
template<typename Scalar , int StorageOrder>
EIGEN_STRONG_INLINE void	symm_pack_lhs_helper (Scalar *blockA, const Scalar *_lhs, Index lhsStride, Index cols, Index rows)
template<typename Scalar , typename Packet , int N>
EIGEN_ALWAYS_INLINE void	storeBlock (Scalar *to, PacketBlock< Packet, N > &block)
template<typename Packet , bool NegativeAccumulate, int N>
EIGEN_ALWAYS_INLINE void	pger_common (PacketBlock< Packet, N > *acc, const Packet &lhsV, const Packet *rhsV)
template<int N, typename Scalar , typename Packet , bool NegativeAccumulate>
EIGEN_ALWAYS_INLINE void	pger (PacketBlock< Packet, N > *acc, const Scalar *lhs, const Packet *rhsV)
template<int N, typename Packet , bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
EIGEN_ALWAYS_INLINE void	pgerc_common (PacketBlock< Packet, N > *accReal, PacketBlock< Packet, N > *accImag, const Packet &lhsV, Packet &lhsVi, const Packet *rhsV, const Packet *rhsVi)
template<int N, typename Scalar , typename Packet , bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
EIGEN_ALWAYS_INLINE void	pgerc (PacketBlock< Packet, N > *accReal, PacketBlock< Packet, N > *accImag, const Scalar *lhs_ptr, const Scalar *lhs_ptr_imag, const Packet *rhsV, const Packet *rhsVi)
template<typename Packet >
EIGEN_ALWAYS_INLINE Packet	ploadLhs (const __UNPACK_TYPE__(Packet) *lhs)
template<typename Packet , int N>
EIGEN_ALWAYS_INLINE void	bsetzero (PacketBlock< Packet, N > &acc)
template<typename Packet , int N>
EIGEN_ALWAYS_INLINE void	bscalec_common (PacketBlock< Packet, N > &acc, PacketBlock< Packet, N > &accZ, const Packet &pAlpha)
template<typename Packet , int N>
EIGEN_ALWAYS_INLINE void	band (PacketBlock< Packet, N > &acc, const Packet &pMask)
template<typename Packet , int N, bool mask>
EIGEN_ALWAYS_INLINE void	bscalec (PacketBlock< Packet, N > &aReal, PacketBlock< Packet, N > &aImag, const Packet &bReal, const Packet &bImag, PacketBlock< Packet, N > &cReal, PacketBlock< Packet, N > &cImag, const Packet &pMask)
template<typename DataMapper , typename Packet , const Index accCols, int StorageOrder, bool Complex, int N, bool full>
EIGEN_ALWAYS_INLINE void	bload (PacketBlock< Packet, N *(Complex ? 2 :1)> &acc, const DataMapper &res, Index row, Index col)
template<typename DataMapper , typename Packet , int N>
EIGEN_ALWAYS_INLINE void	bstore (PacketBlock< Packet, N > &acc, const DataMapper &res, Index row)
template<typename Packet >
EIGEN_ALWAYS_INLINE Packet	bmask (const Index remaining_rows)
template<>
EIGEN_ALWAYS_INLINE Packet2d	bmask< Packet2d > (const Index remaining_rows)
template<typename Packet , int N>
EIGEN_ALWAYS_INLINE void	bscale (PacketBlock< Packet, N > &acc, PacketBlock< Packet, N > &accZ, const Packet &pAlpha)
template<typename Packet , int N, bool mask>
EIGEN_ALWAYS_INLINE void	bscale (PacketBlock< Packet, N > &acc, PacketBlock< Packet, N > &accZ, const Packet &pAlpha, const Packet &pMask)
template<typename Packet , int N, bool real>
EIGEN_ALWAYS_INLINE void	pbroadcastN (const __UNPACK_TYPE__(Packet) *ap0, const __UNPACK_TYPE__(Packet) *ap1, const __UNPACK_TYPE__(Packet) *ap2, Packet &a0, Packet &a1, Packet &a2, Packet &a3)
template<>
EIGEN_ALWAYS_INLINE void	pbroadcastN< Packet4f, 4, true > (const float *ap0, const float *, const float *, Packet4f &a0, Packet4f &a1, Packet4f &a2, Packet4f &a3)
template<>
EIGEN_ALWAYS_INLINE void	pbroadcastN< Packet4f, 4, false > (const float *ap0, const float *ap1, const float *ap2, Packet4f &a0, Packet4f &a1, Packet4f &a2, Packet4f &a3)
template<>
EIGEN_ALWAYS_INLINE void	pbroadcastN< Packet2d, 4, false > (const double *ap0, const double *, const double *, Packet2d &a0, Packet2d &a1, Packet2d &a2, Packet2d &a3)
template<typename Packet , typename Packetc , int N, bool full>
EIGEN_ALWAYS_INLINE void	bcouple_common (PacketBlock< Packet, N > &taccReal, PacketBlock< Packet, N > &taccImag, PacketBlock< Packetc, N > &acc1, PacketBlock< Packetc, N > &acc2)
template<typename Packet , typename Packetc , int N, bool full>
EIGEN_ALWAYS_INLINE void	bcouple (PacketBlock< Packet, N > &taccReal, PacketBlock< Packet, N > &taccImag, PacketBlock< Packetc, N *2 > &tRes, PacketBlock< Packetc, N > &acc1, PacketBlock< Packetc, N > &acc2)
template<typename Scalar , typename Packet , const Index accRows, const Index remaining_rows>
EIGEN_ALWAYS_INLINE void	MICRO_EXTRA_ROW (const Scalar *&lhs_ptr, const Scalar *&rhs_ptr0, const Scalar *&rhs_ptr1, const Scalar *&rhs_ptr2, PacketBlock< Packet, accRows > &accZero)
template<typename Scalar , typename Packet , typename DataMapper , const Index accRows, const Index accCols, const Index remaining_rows>
EIGEN_ALWAYS_INLINE void	gemm_unrolled_row_iteration (const DataMapper &res, const Scalar *lhs_base, const Scalar *rhs_base, Index depth, Index strideA, Index offsetA, Index strideB, Index row, Index rows, const Packet &pAlpha, const Packet &pMask)
template<typename Scalar , typename Packet , typename DataMapper , const Index accRows, const Index accCols>
EIGEN_ALWAYS_INLINE void	gemm_extra_row (const DataMapper &res, const Scalar *lhs_base, const Scalar *rhs_base, Index depth, Index strideA, Index offsetA, Index strideB, Index row, Index rows, Index remaining_rows, const Packet &pAlpha, const Packet &pMask)
template<int unroll_factor, typename Scalar , typename Packet , typename DataMapper , const Index accRows, const Index accCols, const Index accCols2>
EIGEN_ALWAYS_INLINE void	gemm_unrolled_iteration (const DataMapper &res, const Scalar *lhs_base, const Scalar *rhs_base, Index depth, Index strideA, Index offsetA, Index strideB, Index &row, const Packet &pAlpha, const Packet &pMask)
template<typename Scalar , typename Packet , typename DataMapper , const Index accRows, const Index accCols>
EIGEN_ALWAYS_INLINE void	gemm_cols (const DataMapper &res, const Scalar *blockA, const Scalar *blockB, Index depth, Index strideA, Index offsetA, Index strideB, Index offsetB, Index col, Index rows, Index remaining_rows, const Packet &pAlpha, const Packet &pMask)
template<typename Scalar , typename Packet , typename DataMapper , const Index accCols>
EIGEN_ALWAYS_INLINE void	gemm_extra_cols (const DataMapper &res, const Scalar *blockA, const Scalar *blockB, Index depth, Index strideA, Index offsetA, Index strideB, Index offsetB, Index col, Index rows, Index cols, Index remaining_rows, const Packet &pAlpha, const Packet &pMask)
template<typename Scalar , typename Packet , typename RhsPacket , typename DataMapper , const Index accRows, const Index accCols>
EIGEN_STRONG_INLINE void	gemm (const DataMapper &res, const Scalar *blockA, const Scalar *blockB, Index rows, Index depth, Index cols, Scalar alpha, Index strideA, Index strideB, Index offsetA, Index offsetB)
template<typename Scalar , typename Packet , const Index accRows, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal, const Index remaining_rows>
EIGEN_ALWAYS_INLINE void	MICRO_COMPLEX_EXTRA_ROW (const Scalar *&lhs_ptr_real, const Scalar *&lhs_ptr_imag, const Scalar *&rhs_ptr_real0, const Scalar *&rhs_ptr_real1, const Scalar *&rhs_ptr_real2, const Scalar *&rhs_ptr_imag0, const Scalar *&rhs_ptr_imag1, const Scalar *&rhs_ptr_imag2, PacketBlock< Packet, accRows > &accReal, PacketBlock< Packet, accRows > &accImag)
template<typename Scalar , typename Packet , typename Packetc , typename DataMapper , const Index accRows, const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal, const Index remaining_rows>
EIGEN_ALWAYS_INLINE void	gemm_unrolled_complex_row_iteration (const DataMapper &res, const Scalar *lhs_base, const Scalar *rhs_base, Index depth, Index strideA, Index offsetA, Index strideB, Index row, Index rows, const Packet &pAlphaReal, const Packet &pAlphaImag, const Packet &pMask)
template<typename Scalar , typename Packet , typename Packetc , typename DataMapper , const Index accRows, const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
EIGEN_ALWAYS_INLINE void	gemm_complex_extra_row (const DataMapper &res, const Scalar *lhs_base, const Scalar *rhs_base, Index depth, Index strideA, Index offsetA, Index strideB, Index row, Index rows, Index remaining_rows, const Packet &pAlphaReal, const Packet &pAlphaImag, const Packet &pMask)
template<int unroll_factor, typename Scalar , typename Packet , typename Packetc , typename DataMapper , const Index accRows, const Index accCols, const Index accCols2, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
EIGEN_ALWAYS_INLINE void	gemm_complex_unrolled_iteration (const DataMapper &res, const Scalar *lhs_base, const Scalar *rhs_base, Index depth, Index strideA, Index offsetA, Index strideB, Index &row, const Packet &pAlphaReal, const Packet &pAlphaImag, const Packet &pMask)
template<typename Scalar , typename Packet , typename Packetc , typename DataMapper , const Index accRows, const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
EIGEN_ALWAYS_INLINE void	gemm_complex_cols (const DataMapper &res, const Scalar *blockA, const Scalar *blockB, Index depth, Index strideA, Index offsetA, Index strideB, Index offsetB, Index col, Index rows, Index remaining_rows, const Packet &pAlphaReal, const Packet &pAlphaImag, const Packet &pMask)
template<typename Scalar , typename Packet , typename Packetc , typename DataMapper , const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
EIGEN_ALWAYS_INLINE void	gemm_complex_extra_cols (const DataMapper &res, const Scalar *blockA, const Scalar *blockB, Index depth, Index strideA, Index offsetA, Index strideB, Index offsetB, Index col, Index rows, Index cols, Index remaining_rows, const Packet &pAlphaReal, const Packet &pAlphaImag, const Packet &pMask)
template<typename LhsScalar , typename RhsScalar , typename Scalarc , typename Scalar , typename Packet , typename Packetc , typename RhsPacket , typename DataMapper , const Index accRows, const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
EIGEN_STRONG_INLINE void	gemm_complex (const DataMapper &res, const LhsScalar *blockAc, const RhsScalar *blockBc, Index rows, Index depth, Index cols, Scalarc alpha, Index strideA, Index strideB, Index offsetA, Index offsetB)
EIGEN_ALWAYS_INLINE bool	supportsMMA ()
EIGEN_ALWAYS_INLINE Packet4f	loadAndMultiplyF32 (Packet4f acc, const Packet4f pAlpha, float *result)
template<bool lhsExtraRows>
EIGEN_ALWAYS_INLINE void	storeF32 (float *&result, Packet4f result_block, Index rows, Index extra_rows)
template<bool rhsExtraCols, bool lhsExtraRows>
EIGEN_ALWAYS_INLINE void	storeResults (Packet4f(&acc)[4], Index rows, const Packet4f pAlpha, float *result, Index extra_cols, Index extra_rows)
EIGEN_ALWAYS_INLINE Packet4f	oneConvertBF16Hi (Packet8us data)
EIGEN_ALWAYS_INLINE Packet4f	oneConvertBF16Lo (Packet8us data)
template<Index N, Index M>
EIGEN_ALWAYS_INLINE void	storeConvertTwoBF16 (float *to, PacketBlock< Packet8bf,(N+7)/8 > &block, Index extra=0)
template<Index N>
EIGEN_ALWAYS_INLINE void	storeConvertBlockBF16 (float *to, PacketBlock< Packet8bf,(N+7)/8 > &block, Index extra)
template<bool non_unit_stride, Index delta>
EIGEN_ALWAYS_INLINE Packet8bf	loadBF16fromResult (bfloat16 *src, Index resInc)
EIGEN_ALWAYS_INLINE Packet4f	oneConvertBF16Perm (Packet8us data, Packet16uc mask)
template<bool lhsExtraRows, bool odd, Index size>
EIGEN_ALWAYS_INLINE void	convertArrayPointerBF16toF32DupOne (float *result, Index rows, const bfloat16 *src, Index extra_rows)
template<bool lhsExtraRows>
EIGEN_ALWAYS_INLINE void	convertArrayPointerBF16toF32Dup (float *result, Index cols, Index rows, const bfloat16 *src, Index delta, Index extra_rows)
template<const Index size, bool non_unit_stride>
EIGEN_ALWAYS_INLINE void	convertPointerBF16toF32 (Index &i, float *result, Index rows, bfloat16 *&src, Index resInc)
template<bool non_unit_stride>
EIGEN_ALWAYS_INLINE void	convertArrayPointerBF16toF32 (float *result, Index cols, Index rows, bfloat16 *src, Index resInc)
template<Index num_acc, Index size = 4>
EIGEN_ALWAYS_INLINE void	zeroAccumulators (Packet4f(&acc)[num_acc][size])
template<Index num_acc>
EIGEN_ALWAYS_INLINE void	tranposeResults (Packet4f(&acc)[num_acc][4])
template<Index num_acc>
EIGEN_ALWAYS_INLINE void	addResults (Packet4f(&acc)[num_acc][4])
template<Index num_acc, bool rhsExtraCols, bool lhsExtraRows, Index num_rhs>
EIGEN_ALWAYS_INLINE void	outputResultsVSX (Packet4f(&acc)[num_acc][4], Index rows, const Packet4f pAlpha, float *result, const Index extra_cols, Index extra_rows)
template<bool zero>
EIGEN_ALWAYS_INLINE void	loadTwoRhsFloat32 (const float *block, Index strideB, Index i, Packet4f &dhs0, Packet4f &dhs1)
template<Index num_acc, bool zero, bool rhsExtraCols, Index num_rhs>
EIGEN_ALWAYS_INLINE void	KLoop (const float *indexA, const float *indexB, Packet4f(&acc)[num_acc][4], Index strideB, Index k, Index offsetB, Index extra_cols)
template<const Index num_acc, bool rhsExtraCols, bool lhsExtraRows>
EIGEN_ALWAYS_INLINE void	colVSXLoopBodyIter (Index depth, Index rows, const Packet4f pAlpha, const float *indexA, const float *indexB, Index strideB, Index offsetB, float *result, const Index extra_cols, const Index extra_rows)
template<const Index num_acc, bool rhsExtraCols, bool lhsExtraRows>
void	colVSXLoopBody (Index &col, Index depth, Index cols, Index rows, const Packet4f pAlpha, const float *indexA, const float *indexB, Index strideB, Index offsetB, float *result)
template<const Index num_acc, bool rhsExtraCols, bool lhsExtraRows>
EIGEN_ALWAYS_INLINE void	colVSXLoopBodyExtraN (Index col, Index depth, Index cols, Index rows, const Packet4f pAlpha, const float *indexA, const float *blockB, Index strideB, Index offsetB, float *result)
template<bool rhsExtraCols, bool lhsExtraRows>
void	colVSXLoopBodyExtra (Index col, Index depth, Index cols, Index rows, const Packet4f pAlpha, const float *indexA, const float *blockB, Index strideB, Index offsetB, float *result)
template<Index size, bool lhsExtraRows = false>
EIGEN_ALWAYS_INLINE void	colVSXLoops (Index depth, Index cols, Index rows, const Packet4f pAlpha, const bfloat16 *indexA, const float *indexA2, const float *blockB2, Index strideA, Index strideB, Index offsetB, float *result2)
template<Index size>
EIGEN_ALWAYS_INLINE void	calcVSXColLoops (const bfloat16 *&indexA, const float *indexA2, Index &row, Index depth, Index cols, Index rows, const Packet4f pAlpha, const float *indexB, Index strideA, Index strideB, Index offsetA, Index offsetB, Index bigSuffix, float *result)
template<const Index size, typename DataMapper >
EIGEN_ALWAYS_INLINE void	convertBF16toF32 (Index &i, float *result, Index rows, const DataMapper &src)
template<typename DataMapper >
EIGEN_ALWAYS_INLINE void	convertArrayBF16toF32 (float *result, Index cols, Index rows, const DataMapper &src)
EIGEN_ALWAYS_INLINE Packet8bf	convertF32toBF16VSX (const float *res)
template<typename DataMapper , const Index size>
EIGEN_ALWAYS_INLINE void	convertArrayF32toBF16ColVSX (float *result, Index col, Index rows, const DataMapper &res)
template<typename DataMapper >
EIGEN_ALWAYS_INLINE void	convertArrayF32toBF16VSX (float *result, Index cols, Index rows, const DataMapper &res)
template<typename DataMapper >
void	gemmbfloat16 (const DataMapper &res, const bfloat16 *indexA, const bfloat16 *indexB, Index rows, Index depth, Index cols, bfloat16 alpha, Index strideA, Index strideB, Index offsetA, Index offsetB)
template<const Index size, bool non_unit_stride, Index delta>
EIGEN_ALWAYS_INLINE void	storeBF16fromResult (bfloat16 *dst, Packet8bf data, Index resInc, Index extra=0)
template<Index num_acc, bool extraRows, Index size = 4>
EIGEN_ALWAYS_INLINE void	outputVecColResults (Packet4f(&acc)[num_acc][size], float *result, Packet4f pAlpha, Index extra_rows)
template<Index num_acc, Index size = 4>
EIGEN_ALWAYS_INLINE void	outputVecResults (Packet4f(&acc)[num_acc][size], float *result, Packet4f pAlpha)
template<typename RhsMapper , bool linear>
EIGEN_ALWAYS_INLINE Packet8bf	loadColData (RhsMapper &rhs, Index j)
EIGEN_ALWAYS_INLINE void	bsetzeroMMA (__vector_quad *acc)
template<typename DataMapper , typename Packet , bool full>
EIGEN_ALWAYS_INLINE void	storeAccumulator (Index i, const DataMapper &data, const Packet &alpha, const Index elements, __vector_quad *acc)
template<typename DataMapper , typename Packet , typename Packetc , const Index accCols, const Index accCols2>
EIGEN_ALWAYS_INLINE void	storeComplexAccumulator (Index i, const DataMapper &data, const Packet &alphaReal, const Packet &alphaImag, const Packet &pMask, __vector_quad *accReal, __vector_quad *accImag)
template<typename LhsPacket , typename RhsPacket , bool NegativeAccumulate>
EIGEN_ALWAYS_INLINE void	pgerMMA (__vector_quad *acc, const RhsPacket &a, const LhsPacket &b)
template<typename LhsPacket , typename RhsPacket , bool NegativeAccumulate>
EIGEN_ALWAYS_INLINE void	pgerMMA (__vector_quad *acc, const __vector_pair &a, const Packet2d &b)
template<typename Packet , typename RhsPacket , bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
EIGEN_ALWAYS_INLINE void	pgercMMA (__vector_quad *accReal, __vector_quad *accImag, const Packet &lhsV, Packet &lhsVi, const RhsPacket &rhsV, RhsPacket &rhsVi)
template<typename Packet >
EIGEN_ALWAYS_INLINE Packet	ploadRhs (const __UNPACK_TYPE__(Packet) *rhs)
template<typename Scalar , typename Packet >
EIGEN_ALWAYS_INLINE void	ploadRhsMMA (const Scalar *rhs, Packet &rhsV)
template<>
EIGEN_ALWAYS_INLINE void	ploadRhsMMA (const double *rhs, __vector_pair &rhsV)
EIGEN_ALWAYS_INLINE void	ploadLhsMMA (const double *lhs, __vector_pair &lhsV)
template<int unroll_factor, typename Scalar , typename Packet , typename RhsPacket , typename DataMapper , const Index accRows, const Index accCols, bool full, const Index accItr>
EIGEN_ALWAYS_INLINE void	gemm_unrolled_MMA_iteration (const DataMapper &res0, const DataMapper &res1, const DataMapper &res2, const DataMapper &res3, const Scalar *lhs_base, const Scalar *rhs_base, Index depth, Index strideA, Index strideB, Index offsetA, Index &row, const Packet &pAlpha, Index accCols2)
template<typename Scalar , typename Packet , typename RhsPacket , typename DataMapper , const Index accRows, const Index accCols, const Index accItr>
EIGEN_ALWAYS_INLINE void	gemmMMA_cols (const DataMapper &res, const Scalar *blockA, const Scalar *blockB, Index depth, Index strideA, Index offsetA, Index strideB, Index offsetB, Index col, Index rows, Index remaining_rows, const Packet &pAlpha, const Packet &pMask)
template<typename Scalar , typename Packet , typename RhsPacket , typename DataMapper , const Index accRows, const Index accCols>
void	gemmMMA (const DataMapper &res, const Scalar *blockA, const Scalar *blockB, Index rows, Index depth, Index cols, Scalar alpha, Index strideA, Index strideB, Index offsetA, Index offsetB)
template<int unroll_factor, typename Scalar , typename Packet , typename Packetc , typename RhsPacket , typename DataMapper , const Index accRows, const Index accCols, const Index accCols2, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal, const Index accItr>
EIGEN_ALWAYS_INLINE void	gemm_complex_unrolled_MMA_iteration (const DataMapper &res0, const DataMapper &res1, const DataMapper &res2, const DataMapper &res3, const Scalar *lhs_base, const Scalar *rhs_base, Index depth, Index strideA, Index offsetA, Index strideB, Index &row, const Packet &pAlphaReal, const Packet &pAlphaImag, const Packet &pMask)
template<typename Scalar , typename Packet , typename Packetc , typename RhsPacket , typename DataMapper , const Index accRows, const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal, const Index accItr>
EIGEN_ALWAYS_INLINE void	gemmMMA_complex_cols (const DataMapper &res, const Scalar *blockA, const Scalar *blockB, Index depth, Index strideA, Index offsetA, Index strideB, Index offsetB, Index col, Index rows, Index remaining_rows, const Packet &pAlphaReal, const Packet &pAlphaImag, const Packet &pMask)
template<typename LhsScalar , typename RhsScalar , typename Scalarc , typename Scalar , typename Packet , typename Packetc , typename RhsPacket , typename DataMapper , const Index accRows, const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
void	gemm_complexMMA (const DataMapper &res, const LhsScalar *blockAc, const RhsScalar *blockBc, Index rows, Index depth, Index cols, Scalarc alpha, Index strideA, Index strideB, Index offsetA, Index offsetB)
template<bool zero>
EIGEN_ALWAYS_INLINE Packet8bf	loadBfloat16 (const bfloat16 *indexA)
template<bool zero>
EIGEN_ALWAYS_INLINE Packet8bf	loadRhsBfloat16 (const bfloat16 *blockB, Index strideB, Index i)
template<Index num_acc, Index num_packets, bool zero, bool rhsExtraCols, bool lhsExtraRows, Index num_rhs, Index num_lhs>
EIGEN_ALWAYS_INLINE void	KLoop (const bfloat16 *indexA, const bfloat16 *indexB, __vector_quad(&quad_acc)[num_acc], Index strideB, Index k, Index offsetB, Index extra_cols, Index extra_rows)
template<Index num_acc>
EIGEN_ALWAYS_INLINE void	zeroAccumulators (__vector_quad(&quad_acc)[num_acc])
template<Index num_acc>
EIGEN_ALWAYS_INLINE void	disassembleAccumulators (__vector_quad(&quad_acc)[num_acc], Packet4f(&acc)[num_acc][4])
template<Index num_acc, bool rhsExtraCols, bool lhsExtraRows, Index num_rhs, Index num_lhs>
EIGEN_ALWAYS_INLINE void	outputResults (Packet4f(&acc)[num_acc][4], Index rows, const Packet4f pAlpha, float *result, const Index extra_cols, Index extra_rows)
template<const Index num_acc, const Index num_packets, bool rhsExtraCols, bool lhsExtraRows, bool multiIter = false>
EIGEN_ALWAYS_INLINE void	colLoopBodyIter (Index depth, Index rows, const Packet4f pAlpha, const bfloat16 *indexA, const bfloat16 *indexB, Index strideB, Index offsetB, float *result, const Index extra_cols, const Index extra_rows)
template<const Index num_acc, const Index num_packets, bool rhsExtraCols, bool lhsExtraRows>
void	colLoopBody (Index &col, Index depth, Index cols, Index rows, const Packet4f pAlpha, const bfloat16 *indexA, const bfloat16 *indexB, Index strideB, Index offsetB, float *result)
template<const Index num_acc, const Index num_packets, bool rhsExtraCols, bool lhsExtraRows>
EIGEN_ALWAYS_INLINE void	colLoopBodyExtraN (Index col, Index depth, Index cols, Index rows, const Packet4f pAlpha, const bfloat16 *indexA, const bfloat16 *blockB, Index strideB, Index offsetB, float *result)
template<const Index num_packets, bool rhsExtraCols, bool lhsExtraRows>
void	colLoopBodyExtra (Index col, Index depth, Index cols, Index rows, const Packet4f pAlpha, const bfloat16 *indexA, const bfloat16 *blockB, Index strideB, Index offsetB, float *result)
template<const Index num_packets, bool lhsExtraRows = false>
EIGEN_ALWAYS_INLINE void	colLoops (Index depth, Index cols, Index rows, const Packet4f pAlpha, const bfloat16 *indexA, const bfloat16 *blockB, Index strideB, Index offsetB, float *result)
EIGEN_ALWAYS_INLINE Packet8bf	convertF32toBF16 (const float *res)
template<typename DataMapper , const Index size>
EIGEN_ALWAYS_INLINE void	convertArrayF32toBF16Col (float *result, Index col, Index rows, const DataMapper &res)
template<const Index size, bool non_unit_stride = false>
EIGEN_ALWAYS_INLINE void	convertPointerF32toBF16 (Index &i, float *result, Index rows, bfloat16 *&dst, Index resInc=1)
template<bool non_unit_stride = false>
EIGEN_ALWAYS_INLINE void	convertArrayPointerF32toBF16 (float *result, Index rows, bfloat16 *dst, Index resInc=1)
template<typename DataMapper >
EIGEN_ALWAYS_INLINE void	convertArrayF32toBF16 (float *result, Index cols, Index rows, const DataMapper &res)
template<Index size>
EIGEN_ALWAYS_INLINE void	calcColLoops (const bfloat16 *&indexA, Index &row, Index depth, Index cols, Index rows, const Packet4f pAlpha, const bfloat16 *indexB, Index strideB, Index offsetA, Index offsetB, Index bigSuffix, float *result)
template<typename DataMapper >
void	gemmMMAbfloat16 (const DataMapper &res, const bfloat16 *indexA, const bfloat16 *indexB, Index rows, Index depth, Index cols, bfloat16 alpha, Index strideA, Index strideB, Index offsetA, Index offsetB)
template<Index num_acc, typename LhsMapper , bool zero>
EIGEN_ALWAYS_INLINE void	loadVecLoop (Index k, LhsMapper &lhs, Packet8bf(&a0)[num_acc], Packet8bf b1)
template<Index num_acc>
EIGEN_ALWAYS_INLINE void	multVec (__vector_quad(&quad_acc)[num_acc], Packet8bf(&a0)[num_acc], Packet8bf b0)
template<Index num_acc, typename LhsMapper , typename RhsMapper , bool zero, bool linear>
EIGEN_ALWAYS_INLINE void	vecColLoop (Index j, LhsMapper &lhs, RhsMapper &rhs, __vector_quad(&quad_acc)[num_acc])
template<const Index num_acc, typename LhsMapper , typename RhsMapper , bool extraRows, bool linear>
void	colVecColLoopBody (Index &row, Index cend, Index rows, LhsMapper &lhs, RhsMapper &rhs, const Packet4f pAlpha, float *result)
template<const Index num_acc, typename LhsMapper , typename RhsMapper , bool extraRows, bool linear>
EIGEN_ALWAYS_INLINE void	colVecColLoopBodyExtraN (Index &row, Index cend, Index rows, LhsMapper &lhs, RhsMapper &rhs, const Packet4f pAlpha, float *result)
template<typename LhsMapper , typename RhsMapper , bool extraRows, bool linear>
EIGEN_ALWAYS_INLINE void	colVecColLoopBodyExtra (Index &row, Index cend, Index rows, LhsMapper &lhs, RhsMapper &rhs, const Packet4f pAlpha, float *result)
template<typename LhsMapper , typename RhsMapper , bool linear>
EIGEN_ALWAYS_INLINE void	calcVecColLoops (Index cend, Index rows, LhsMapper &lhs, RhsMapper &rhs, const Packet4f pAlpha, float *result)
template<typename LhsMapper , typename RhsMapper >
void	gemvMMA_bfloat16_col (Index rows, Index cols, const LhsMapper &alhs, const RhsMapper &rhs, bfloat16 *res, Index resIncr, bfloat16 alpha)
template<Index num_acc>
EIGEN_ALWAYS_INLINE void	preduxVecResults2 (Packet4f(&acc)[num_acc][4], Index k)
template<Index num_acc>
EIGEN_ALWAYS_INLINE void	preduxVecResults (Packet4f(&acc)[num_acc][4])
template<Index num_acc, typename LhsMapper , typename RhsMapper , bool extra>
EIGEN_ALWAYS_INLINE void	multVecLoop (__vector_quad(&quad_acc)[num_acc], const LhsMapper &lhs, RhsMapper &rhs, Index j, Index extra_cols)
template<Index num_acc, typename LhsMapper , typename RhsMapper >
EIGEN_ALWAYS_INLINE void	vecLoop (Index cols, const LhsMapper &lhs, RhsMapper &rhs, __vector_quad(&quad_acc)[num_acc], Index extra_cols)
template<const Index num_acc, typename LhsMapper , typename RhsMapper >
void	colVecLoopBody (Index &row, Index cols, Index rows, LhsMapper &lhs, RhsMapper &rhs, const Packet4f pAlpha, float *result)
template<const Index num_acc, typename LhsMapper , typename RhsMapper >
EIGEN_ALWAYS_INLINE void	colVecLoopBodyExtraN (Index &row, Index cols, Index rows, LhsMapper &lhs, RhsMapper &rhs, const Packet4f pAlpha, float *result)
template<typename LhsMapper , typename RhsMapper >
EIGEN_ALWAYS_INLINE void	colVecLoopBodyExtra (Index &row, Index cols, Index rows, LhsMapper &lhs, RhsMapper &rhs, const Packet4f pAlpha, float *result)
template<typename LhsMapper , typename RhsMapper >
EIGEN_ALWAYS_INLINE void	calcVecLoops (Index cols, Index rows, LhsMapper &lhs, RhsMapper &rhs, const Packet4f pAlpha, float *result)
template<typename LhsMapper , typename RhsMapper >
EIGEN_STRONG_INLINE void	gemvMMA_bfloat16_row (Index rows, Index cols, const LhsMapper &alhs, const RhsMapper &rhs, bfloat16 *res, Index resIncr, bfloat16 alpha)
static	EIGEN_DECLARE_CONST_FAST_Packet4f (ZERO, 0)
static	EIGEN_DECLARE_CONST_FAST_Packet4i (ZERO, 0)
static	EIGEN_DECLARE_CONST_FAST_Packet4i (ONE, 1)
static	EIGEN_DECLARE_CONST_FAST_Packet4i (MINUS16, -16)
static	EIGEN_DECLARE_CONST_FAST_Packet4i (MINUS1, -1)
static	EIGEN_DECLARE_CONST_FAST_Packet4ui (SIGN, 0x80000000u)
static	EIGEN_DECLARE_CONST_FAST_Packet4ui (PREV0DOT5, 0x3EFFFFFFu)
static	EIGEN_DECLARE_CONST_FAST_Packet8us (ONE, 1)
std::ostream &	operator<< (std::ostream &s, const Packet16c &v)
std::ostream &	operator<< (std::ostream &s, const Packet16uc &v)
std::ostream &	operator<< (std::ostream &s, const Packet4f &v)
std::ostream &	operator<< (std::ostream &s, const Packet4i &v)
std::ostream &	operator<< (std::ostream &s, const Packet4ui &v)
template<typename Packet >
EIGEN_STRONG_INLINE Packet	pload_common (const __UNPACK_TYPE__(Packet) *from)
template<>
EIGEN_STRONG_INLINE Packet4f	pload< Packet4f > (const float *from)
template<>
EIGEN_STRONG_INLINE Packet4i	pload< Packet4i > (const int *from)
template<>
EIGEN_STRONG_INLINE Packet8s	pload< Packet8s > (const short int *from)
template<>
EIGEN_STRONG_INLINE Packet8us	pload< Packet8us > (const unsigned short int *from)
template<>
EIGEN_STRONG_INLINE Packet16c	pload< Packet16c > (const signed char *from)
template<>
EIGEN_STRONG_INLINE Packet16uc	pload< Packet16uc > (const unsigned char *from)
template<>
EIGEN_STRONG_INLINE Packet8bf	pload< Packet8bf > (const bfloat16 *from)
template<typename Packet >
EIGEN_ALWAYS_INLINE Packet	pload_ignore (const __UNPACK_TYPE__(Packet) *from)
template<>
EIGEN_ALWAYS_INLINE Packet8bf	pload_ignore< Packet8bf > (const bfloat16 *from)
template<typename Packet >
EIGEN_ALWAYS_INLINE Packet	pload_partial_common (const __UNPACK_TYPE__(Packet) *from, const Index n, const Index offset)
template<>
EIGEN_ALWAYS_INLINE Packet4f	pload_partial< Packet4f > (const float *from, const Index n, const Index offset)
template<>
EIGEN_ALWAYS_INLINE Packet4i	pload_partial< Packet4i > (const int *from, const Index n, const Index offset)
template<>
EIGEN_ALWAYS_INLINE Packet8s	pload_partial< Packet8s > (const short int *from, const Index n, const Index offset)
template<>
EIGEN_ALWAYS_INLINE Packet8us	pload_partial< Packet8us > (const unsigned short int *from, const Index n, const Index offset)
template<>
EIGEN_ALWAYS_INLINE Packet8bf	pload_partial< Packet8bf > (const bfloat16 *from, const Index n, const Index offset)
template<>
EIGEN_ALWAYS_INLINE Packet16c	pload_partial< Packet16c > (const signed char *from, const Index n, const Index offset)
template<>
EIGEN_ALWAYS_INLINE Packet16uc	pload_partial< Packet16uc > (const unsigned char *from, const Index n, const Index offset)
template<typename Packet >
EIGEN_STRONG_INLINE void	pstore_common (__UNPACK_TYPE__(Packet) *to, const Packet &from)
template<>
EIGEN_STRONG_INLINE void	pstore< float > (float *to, const Packet4f &from)
template<>
EIGEN_STRONG_INLINE void	pstore< int > (int *to, const Packet4i &from)
template<>
EIGEN_STRONG_INLINE void	pstore< short int > (short int *to, const Packet8s &from)
template<>
EIGEN_STRONG_INLINE void	pstore< unsigned short int > (unsigned short int *to, const Packet8us &from)
template<>
EIGEN_STRONG_INLINE void	pstore< bfloat16 > (bfloat16 *to, const Packet8bf &from)
template<>
EIGEN_STRONG_INLINE void	pstore< signed char > (signed char *to, const Packet16c &from)
template<>
EIGEN_STRONG_INLINE void	pstore< unsigned char > (unsigned char *to, const Packet16uc &from)
template<typename Packet >
EIGEN_ALWAYS_INLINE void	pstore_partial_common (__UNPACK_TYPE__(Packet) *to, const Packet &from, const Index n, const Index offset)
template<>
EIGEN_ALWAYS_INLINE void	pstore_partial< float > (float *to, const Packet4f &from, const Index n, const Index offset)
template<>
EIGEN_ALWAYS_INLINE void	pstore_partial< int > (int *to, const Packet4i &from, const Index n, const Index offset)
template<>
EIGEN_ALWAYS_INLINE void	pstore_partial< short int > (short int *to, const Packet8s &from, const Index n, const Index offset)
template<>
EIGEN_ALWAYS_INLINE void	pstore_partial< unsigned short int > (unsigned short int *to, const Packet8us &from, const Index n, const Index offset)
template<>
EIGEN_ALWAYS_INLINE void	pstore_partial< bfloat16 > (bfloat16 *to, const Packet8bf &from, const Index n, const Index offset)
template<>
EIGEN_ALWAYS_INLINE void	pstore_partial< signed char > (signed char *to, const Packet16c &from, const Index n, const Index offset)
template<>
EIGEN_ALWAYS_INLINE void	pstore_partial< unsigned char > (unsigned char *to, const Packet16uc &from, const Index n, const Index offset)
template<typename Packet >
EIGEN_STRONG_INLINE Packet	pset1_size4 (const __UNPACK_TYPE__(Packet) &from)
template<typename Packet >
EIGEN_STRONG_INLINE Packet	pset1_size8 (const __UNPACK_TYPE__(Packet) &from)
template<typename Packet >
EIGEN_STRONG_INLINE Packet	pset1_size16 (const __UNPACK_TYPE__(Packet) &from)
template<>
EIGEN_STRONG_INLINE Packet4f	pset1< Packet4f > (const float &from)
template<>
EIGEN_STRONG_INLINE Packet4i	pset1< Packet4i > (const int &from)
template<>
EIGEN_STRONG_INLINE Packet8s	pset1< Packet8s > (const short int &from)
template<>
EIGEN_STRONG_INLINE Packet8us	pset1< Packet8us > (const unsigned short int &from)
template<>
EIGEN_STRONG_INLINE Packet16c	pset1< Packet16c > (const signed char &from)
template<>
EIGEN_STRONG_INLINE Packet16uc	pset1< Packet16uc > (const unsigned char &from)
template<>
EIGEN_STRONG_INLINE Packet4f	pset1frombits< Packet4f > (unsigned int from)
template<>
EIGEN_STRONG_INLINE Packet8bf	pset1< Packet8bf > (const bfloat16 &from)
template<typename Packet >
EIGEN_STRONG_INLINE void	pbroadcast4_common (const __UNPACK_TYPE__(Packet) *a, Packet &a0, Packet &a1, Packet &a2, Packet &a3)
template<>
EIGEN_STRONG_INLINE void	pbroadcast4< Packet4f > (const float *a, Packet4f &a0, Packet4f &a1, Packet4f &a2, Packet4f &a3)
template<>
EIGEN_STRONG_INLINE void	pbroadcast4< Packet4i > (const int *a, Packet4i &a0, Packet4i &a1, Packet4i &a2, Packet4i &a3)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet	pgather_common (const __UNPACK_TYPE__(Packet) *from, Index stride, const Index n=unpacket_traits< Packet >::size)
template<>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet4f	pgather< float, Packet4f > (const float *from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet4i	pgather< int, Packet4i > (const int *from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet8s	pgather< short int, Packet8s > (const short int *from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet8us	pgather< unsigned short int, Packet8us > (const unsigned short int *from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet8bf	pgather< bfloat16, Packet8bf > (const bfloat16 *from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet16c	pgather< signed char, Packet16c > (const signed char *from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet16uc	pgather< unsigned char, Packet16uc > (const unsigned char *from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet4f	pgather_partial< float, Packet4f > (const float *from, Index stride, const Index n)
template<>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet4i	pgather_partial< int, Packet4i > (const int *from, Index stride, const Index n)
template<>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet8s	pgather_partial< short int, Packet8s > (const short int *from, Index stride, const Index n)
template<>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet8us	pgather_partial< unsigned short int, Packet8us > (const unsigned short int *from, Index stride, const Index n)
template<>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet8bf	pgather_partial< bfloat16, Packet8bf > (const bfloat16 *from, Index stride, const Index n)
template<>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet16c	pgather_partial< signed char, Packet16c > (const signed char *from, Index stride, const Index n)
template<>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet16uc	pgather_partial< unsigned char, Packet16uc > (const unsigned char *from, Index stride, const Index n)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void	pscatter_common (__UNPACK_TYPE__(Packet) *to, const Packet &from, Index stride, const Index n=unpacket_traits< Packet >::size)
template<>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void	pscatter< float, Packet4f > (float *to, const Packet4f &from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void	pscatter< int, Packet4i > (int *to, const Packet4i &from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void	pscatter< short int, Packet8s > (short int *to, const Packet8s &from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void	pscatter< unsigned short int, Packet8us > (unsigned short int *to, const Packet8us &from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void	pscatter< bfloat16, Packet8bf > (bfloat16 *to, const Packet8bf &from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void	pscatter< signed char, Packet16c > (signed char *to, const Packet16c &from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void	pscatter< unsigned char, Packet16uc > (unsigned char *to, const Packet16uc &from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void	pscatter_partial< float, Packet4f > (float *to, const Packet4f &from, Index stride, const Index n)
template<>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void	pscatter_partial< int, Packet4i > (int *to, const Packet4i &from, Index stride, const Index n)
template<>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void	pscatter_partial< short int, Packet8s > (short int *to, const Packet8s &from, Index stride, const Index n)
template<>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void	pscatter_partial< unsigned short int, Packet8us > (unsigned short int *to, const Packet8us &from, Index stride, const Index n)
template<>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void	pscatter_partial< bfloat16, Packet8bf > (bfloat16 *to, const Packet8bf &from, Index stride, const Index n)
template<>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void	pscatter_partial< signed char, Packet16c > (signed char *to, const Packet16c &from, Index stride, const Index n)
template<>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void	pscatter_partial< unsigned char, Packet16uc > (unsigned char *to, const Packet16uc &from, Index stride, const Index n)
template<>
EIGEN_STRONG_INLINE Packet4f	plset< Packet4f > (const float &a)
template<>
EIGEN_STRONG_INLINE Packet4i	plset< Packet4i > (const int &a)
template<>
EIGEN_STRONG_INLINE Packet8s	plset< Packet8s > (const short int &a)
template<>
EIGEN_STRONG_INLINE Packet8us	plset< Packet8us > (const unsigned short int &a)
template<>
EIGEN_STRONG_INLINE Packet16c	plset< Packet16c > (const signed char &a)
template<>
EIGEN_STRONG_INLINE Packet16uc	plset< Packet16uc > (const unsigned char &a)
template<>
EIGEN_STRONG_INLINE Packet4f	padd< Packet4f > (const Packet4f &a, const Packet4f &b)
template<>
EIGEN_STRONG_INLINE Packet4i	padd< Packet4i > (const Packet4i &a, const Packet4i &b)
template<>
EIGEN_STRONG_INLINE Packet4ui	padd< Packet4ui > (const Packet4ui &a, const Packet4ui &b)
template<>
EIGEN_STRONG_INLINE Packet8s	padd< Packet8s > (const Packet8s &a, const Packet8s &b)
template<>
EIGEN_STRONG_INLINE Packet8us	padd< Packet8us > (const Packet8us &a, const Packet8us &b)
template<>
EIGEN_STRONG_INLINE Packet16c	padd< Packet16c > (const Packet16c &a, const Packet16c &b)
template<>
EIGEN_STRONG_INLINE Packet16uc	padd< Packet16uc > (const Packet16uc &a, const Packet16uc &b)
template<>
EIGEN_STRONG_INLINE Packet4f	psub< Packet4f > (const Packet4f &a, const Packet4f &b)
template<>
EIGEN_STRONG_INLINE Packet4i	psub< Packet4i > (const Packet4i &a, const Packet4i &b)
template<>
EIGEN_STRONG_INLINE Packet8s	psub< Packet8s > (const Packet8s &a, const Packet8s &b)
template<>
EIGEN_STRONG_INLINE Packet8us	psub< Packet8us > (const Packet8us &a, const Packet8us &b)
template<>
EIGEN_STRONG_INLINE Packet16c	psub< Packet16c > (const Packet16c &a, const Packet16c &b)
template<>
EIGEN_STRONG_INLINE Packet16uc	psub< Packet16uc > (const Packet16uc &a, const Packet16uc &b)
template<>
EIGEN_STRONG_INLINE Packet4f	pnegate (const Packet4f &a)
template<>
EIGEN_STRONG_INLINE Packet16c	pnegate (const Packet16c &a)
template<>
EIGEN_STRONG_INLINE Packet8s	pnegate (const Packet8s &a)
template<>
EIGEN_STRONG_INLINE Packet4i	pnegate (const Packet4i &a)
template<>
EIGEN_STRONG_INLINE Packet4f	pconj (const Packet4f &a)
template<>
EIGEN_STRONG_INLINE Packet4i	pconj (const Packet4i &a)
template<>
EIGEN_STRONG_INLINE Packet4f	pmul< Packet4f > (const Packet4f &a, const Packet4f &b)
template<>
EIGEN_STRONG_INLINE Packet4i	pmul< Packet4i > (const Packet4i &a, const Packet4i &b)
template<>
EIGEN_STRONG_INLINE Packet8s	pmul< Packet8s > (const Packet8s &a, const Packet8s &b)
template<>
EIGEN_STRONG_INLINE Packet8us	pmul< Packet8us > (const Packet8us &a, const Packet8us &b)
template<>
EIGEN_STRONG_INLINE Packet16c	pmul< Packet16c > (const Packet16c &a, const Packet16c &b)
template<>
EIGEN_STRONG_INLINE Packet16uc	pmul< Packet16uc > (const Packet16uc &a, const Packet16uc &b)
template<>
EIGEN_STRONG_INLINE Packet4f	pdiv< Packet4f > (const Packet4f &a, const Packet4f &b)
template<>
EIGEN_STRONG_INLINE Packet4i	pdiv< Packet4i > (const Packet4i &a, const Packet4i &b)
template<>
EIGEN_STRONG_INLINE Packet4f	pmadd (const Packet4f &a, const Packet4f &b, const Packet4f &c)
template<>
EIGEN_STRONG_INLINE Packet4i	pmadd (const Packet4i &a, const Packet4i &b, const Packet4i &c)
template<>
EIGEN_STRONG_INLINE Packet8s	pmadd (const Packet8s &a, const Packet8s &b, const Packet8s &c)
template<>
EIGEN_STRONG_INLINE Packet8us	pmadd (const Packet8us &a, const Packet8us &b, const Packet8us &c)
template<>
EIGEN_STRONG_INLINE Packet4f	pmin< Packet4f > (const Packet4f &a, const Packet4f &b)
template<>
EIGEN_STRONG_INLINE Packet4i	pmin< Packet4i > (const Packet4i &a, const Packet4i &b)
template<>
EIGEN_STRONG_INLINE Packet8s	pmin< Packet8s > (const Packet8s &a, const Packet8s &b)
template<>
EIGEN_STRONG_INLINE Packet8us	pmin< Packet8us > (const Packet8us &a, const Packet8us &b)
template<>
EIGEN_STRONG_INLINE Packet16c	pmin< Packet16c > (const Packet16c &a, const Packet16c &b)
template<>
EIGEN_STRONG_INLINE Packet16uc	pmin< Packet16uc > (const Packet16uc &a, const Packet16uc &b)
template<>
EIGEN_STRONG_INLINE Packet4f	pmax< Packet4f > (const Packet4f &a, const Packet4f &b)
template<>
EIGEN_STRONG_INLINE Packet4i	pmax< Packet4i > (const Packet4i &a, const Packet4i &b)
template<>
EIGEN_STRONG_INLINE Packet8s	pmax< Packet8s > (const Packet8s &a, const Packet8s &b)
template<>
EIGEN_STRONG_INLINE Packet8us	pmax< Packet8us > (const Packet8us &a, const Packet8us &b)
template<>
EIGEN_STRONG_INLINE Packet16c	pmax< Packet16c > (const Packet16c &a, const Packet16c &b)
template<>
EIGEN_STRONG_INLINE Packet16uc	pmax< Packet16uc > (const Packet16uc &a, const Packet16uc &b)
template<>
EIGEN_STRONG_INLINE Packet4f	pcmp_le (const Packet4f &a, const Packet4f &b)
template<>
EIGEN_STRONG_INLINE Packet4f	pcmp_eq (const Packet4f &a, const Packet4f &b)
template<>
EIGEN_STRONG_INLINE Packet4f	pcmp_lt_or_nan (const Packet4f &a, const Packet4f &b)
template<>
EIGEN_STRONG_INLINE Packet4i	pcmp_lt (const Packet4i &a, const Packet4i &b)
template<>
EIGEN_STRONG_INLINE Packet4i	pcmp_eq (const Packet4i &a, const Packet4i &b)
template<>
EIGEN_STRONG_INLINE Packet8s	pcmp_lt (const Packet8s &a, const Packet8s &b)
template<>
EIGEN_STRONG_INLINE Packet8s	pcmp_eq (const Packet8s &a, const Packet8s &b)
template<>
EIGEN_STRONG_INLINE Packet8us	pcmp_lt (const Packet8us &a, const Packet8us &b)
template<>
EIGEN_STRONG_INLINE Packet8us	pcmp_eq (const Packet8us &a, const Packet8us &b)
template<>
EIGEN_STRONG_INLINE Packet16c	pcmp_lt (const Packet16c &a, const Packet16c &b)
template<>
EIGEN_STRONG_INLINE Packet16c	pcmp_eq (const Packet16c &a, const Packet16c &b)
template<>
EIGEN_STRONG_INLINE Packet16uc	pcmp_lt (const Packet16uc &a, const Packet16uc &b)
template<>
EIGEN_STRONG_INLINE Packet16uc	pcmp_eq (const Packet16uc &a, const Packet16uc &b)
template<>
EIGEN_STRONG_INLINE Packet4f	pand< Packet4f > (const Packet4f &a, const Packet4f &b)
template<>
EIGEN_STRONG_INLINE Packet4i	pand< Packet4i > (const Packet4i &a, const Packet4i &b)
template<>
EIGEN_STRONG_INLINE Packet4ui	pand< Packet4ui > (const Packet4ui &a, const Packet4ui &b)
template<>
EIGEN_STRONG_INLINE Packet8us	pand< Packet8us > (const Packet8us &a, const Packet8us &b)
template<>
EIGEN_STRONG_INLINE Packet8bf	pand< Packet8bf > (const Packet8bf &a, const Packet8bf &b)
template<>
EIGEN_STRONG_INLINE Packet4f	por< Packet4f > (const Packet4f &a, const Packet4f &b)
template<>
EIGEN_STRONG_INLINE Packet4i	por< Packet4i > (const Packet4i &a, const Packet4i &b)
template<>
EIGEN_STRONG_INLINE Packet8s	por< Packet8s > (const Packet8s &a, const Packet8s &b)
template<>
EIGEN_STRONG_INLINE Packet8us	por< Packet8us > (const Packet8us &a, const Packet8us &b)
template<>
EIGEN_STRONG_INLINE Packet8bf	por< Packet8bf > (const Packet8bf &a, const Packet8bf &b)
template<>
EIGEN_STRONG_INLINE Packet4f	pxor< Packet4f > (const Packet4f &a, const Packet4f &b)
template<>
EIGEN_STRONG_INLINE Packet4i	pxor< Packet4i > (const Packet4i &a, const Packet4i &b)
template<>
EIGEN_STRONG_INLINE Packet8us	pxor< Packet8us > (const Packet8us &a, const Packet8us &b)
template<>
EIGEN_STRONG_INLINE Packet8bf	pxor< Packet8bf > (const Packet8bf &a, const Packet8bf &b)
template<>
EIGEN_STRONG_INLINE Packet4f	pandnot< Packet4f > (const Packet4f &a, const Packet4f &b)
template<>
EIGEN_STRONG_INLINE Packet4i	pandnot< Packet4i > (const Packet4i &a, const Packet4i &b)
template<>
EIGEN_STRONG_INLINE Packet4f	pselect (const Packet4f &mask, const Packet4f &a, const Packet4f &b)
template<>
EIGEN_STRONG_INLINE Packet4f	pround< Packet4f > (const Packet4f &a)
template<>
EIGEN_STRONG_INLINE Packet4f	pceil< Packet4f > (const Packet4f &a)
template<>
EIGEN_STRONG_INLINE Packet4f	pfloor< Packet4f > (const Packet4f &a)
template<>
EIGEN_STRONG_INLINE Packet4f	ptrunc< Packet4f > (const Packet4f &a)
template<typename Packet >
EIGEN_STRONG_INLINE Packet	ploadu_common (const __UNPACK_TYPE__(Packet) *from)
template<>
EIGEN_STRONG_INLINE Packet4f	ploadu< Packet4f > (const float *from)
template<>
EIGEN_STRONG_INLINE Packet4i	ploadu< Packet4i > (const int *from)
template<>
EIGEN_STRONG_INLINE Packet8s	ploadu< Packet8s > (const short int *from)
template<>
EIGEN_STRONG_INLINE Packet8us	ploadu< Packet8us > (const unsigned short int *from)
template<>
EIGEN_STRONG_INLINE Packet8bf	ploadu< Packet8bf > (const bfloat16 *from)
template<>
EIGEN_STRONG_INLINE Packet16c	ploadu< Packet16c > (const signed char *from)
template<>
EIGEN_STRONG_INLINE Packet16uc	ploadu< Packet16uc > (const unsigned char *from)
template<typename Packet >
EIGEN_ALWAYS_INLINE Packet	ploadu_partial_common (const __UNPACK_TYPE__(Packet) *from, const Index n, const Index offset)
template<>
EIGEN_ALWAYS_INLINE Packet4f	ploadu_partial< Packet4f > (const float *from, const Index n, const Index offset)
template<>
EIGEN_ALWAYS_INLINE Packet4i	ploadu_partial< Packet4i > (const int *from, const Index n, const Index offset)
template<>
EIGEN_ALWAYS_INLINE Packet8s	ploadu_partial< Packet8s > (const short int *from, const Index n, const Index offset)
template<>
EIGEN_ALWAYS_INLINE Packet8us	ploadu_partial< Packet8us > (const unsigned short int *from, const Index n, const Index offset)
template<>
EIGEN_ALWAYS_INLINE Packet8bf	ploadu_partial< Packet8bf > (const bfloat16 *from, const Index n, const Index offset)
template<>
EIGEN_ALWAYS_INLINE Packet16c	ploadu_partial< Packet16c > (const signed char *from, const Index n, const Index offset)
template<>
EIGEN_ALWAYS_INLINE Packet16uc	ploadu_partial< Packet16uc > (const unsigned char *from, const Index n, const Index offset)
template<typename Packet >
EIGEN_STRONG_INLINE Packet	ploaddup_common (const __UNPACK_TYPE__(Packet) *from)
template<>
EIGEN_STRONG_INLINE Packet4f	ploaddup< Packet4f > (const float *from)
template<>
EIGEN_STRONG_INLINE Packet4i	ploaddup< Packet4i > (const int *from)
template<>
EIGEN_STRONG_INLINE Packet8s	ploaddup< Packet8s > (const short int *from)
template<>
EIGEN_STRONG_INLINE Packet8us	ploaddup< Packet8us > (const unsigned short int *from)
template<>
EIGEN_STRONG_INLINE Packet8s	ploadquad< Packet8s > (const short int *from)
template<>
EIGEN_STRONG_INLINE Packet8us	ploadquad< Packet8us > (const unsigned short int *from)
template<>
EIGEN_STRONG_INLINE Packet8bf	ploadquad< Packet8bf > (const bfloat16 *from)
template<>
EIGEN_STRONG_INLINE Packet16c	ploaddup< Packet16c > (const signed char *from)
template<>
EIGEN_STRONG_INLINE Packet16uc	ploaddup< Packet16uc > (const unsigned char *from)
template<>
EIGEN_STRONG_INLINE Packet16c	ploadquad< Packet16c > (const signed char *from)
template<>
EIGEN_STRONG_INLINE Packet16uc	ploadquad< Packet16uc > (const unsigned char *from)
template<typename Packet >
EIGEN_STRONG_INLINE void	pstoreu_common (__UNPACK_TYPE__(Packet) *to, const Packet &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< float > (float *to, const Packet4f &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< int > (int *to, const Packet4i &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< short int > (short int *to, const Packet8s &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< unsigned short int > (unsigned short int *to, const Packet8us &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< bfloat16 > (bfloat16 *to, const Packet8bf &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< signed char > (signed char *to, const Packet16c &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< unsigned char > (unsigned char *to, const Packet16uc &from)
template<typename Packet >
EIGEN_ALWAYS_INLINE void	pstoreu_partial_common (__UNPACK_TYPE__(Packet) *to, const Packet &from, const Index n, const Index offset)
template<>
EIGEN_ALWAYS_INLINE void	pstoreu_partial< float > (float *to, const Packet4f &from, const Index n, const Index offset)
template<>
EIGEN_ALWAYS_INLINE void	pstoreu_partial< int > (int *to, const Packet4i &from, const Index n, const Index offset)
template<>
EIGEN_ALWAYS_INLINE void	pstoreu_partial< short int > (short int *to, const Packet8s &from, const Index n, const Index offset)
template<>
EIGEN_ALWAYS_INLINE void	pstoreu_partial< unsigned short int > (unsigned short int *to, const Packet8us &from, const Index n, const Index offset)
template<>
EIGEN_ALWAYS_INLINE void	pstoreu_partial< bfloat16 > (bfloat16 *to, const Packet8bf &from, const Index n, const Index offset)
template<>
EIGEN_ALWAYS_INLINE void	pstoreu_partial< signed char > (signed char *to, const Packet16c &from, const Index n, const Index offset)
template<>
EIGEN_ALWAYS_INLINE void	pstoreu_partial< unsigned char > (unsigned char *to, const Packet16uc &from, const Index n, const Index offset)
template<>
EIGEN_STRONG_INLINE void	prefetch< float > (const float *addr)
template<>
EIGEN_STRONG_INLINE void	prefetch< int > (const int *addr)
template<>
EIGEN_STRONG_INLINE float	pfirst< Packet4f > (const Packet4f &a)
template<>
EIGEN_STRONG_INLINE int	pfirst< Packet4i > (const Packet4i &a)
template<typename Packet >
EIGEN_STRONG_INLINE	__UNPACK_TYPE__ (Packet) pfirst_common(const Packet &a)
template<>
EIGEN_STRONG_INLINE short int	pfirst< Packet8s > (const Packet8s &a)
template<>
EIGEN_STRONG_INLINE unsigned short int	pfirst< Packet8us > (const Packet8us &a)
template<>
EIGEN_STRONG_INLINE signed char	pfirst< Packet16c > (const Packet16c &a)
template<>
EIGEN_STRONG_INLINE unsigned char	pfirst< Packet16uc > (const Packet16uc &a)
template<>
EIGEN_STRONG_INLINE Packet4f	preverse (const Packet4f &a)
template<>
EIGEN_STRONG_INLINE Packet4i	preverse (const Packet4i &a)
template<>
EIGEN_STRONG_INLINE Packet8s	preverse (const Packet8s &a)
template<>
EIGEN_STRONG_INLINE Packet8us	preverse (const Packet8us &a)
template<>
EIGEN_STRONG_INLINE Packet16c	preverse (const Packet16c &a)
template<>
EIGEN_STRONG_INLINE Packet16uc	preverse (const Packet16uc &a)
template<>
EIGEN_STRONG_INLINE Packet8bf	preverse (const Packet8bf &a)
template<>
EIGEN_STRONG_INLINE Packet4f	pabs (const Packet4f &a)
template<>
EIGEN_STRONG_INLINE Packet4i	pabs (const Packet4i &a)
template<>
EIGEN_STRONG_INLINE Packet8s	pabs (const Packet8s &a)
template<>
EIGEN_STRONG_INLINE Packet8us	pabs (const Packet8us &a)
template<>
EIGEN_STRONG_INLINE Packet16c	pabs (const Packet16c &a)
template<>
EIGEN_STRONG_INLINE Packet16uc	pabs (const Packet16uc &a)
template<>
EIGEN_STRONG_INLINE Packet8bf	pabs (const Packet8bf &a)
template<>
EIGEN_STRONG_INLINE Packet8bf	psignbit (const Packet8bf &a)
template<>
EIGEN_STRONG_INLINE Packet4f	psignbit (const Packet4f &a)
template<int N>
EIGEN_STRONG_INLINE Packet4i	parithmetic_shift_right (const Packet4i &a)
template<int N>
EIGEN_STRONG_INLINE Packet4i	plogical_shift_right (const Packet4i &a)
template<int N>
EIGEN_STRONG_INLINE Packet4i	plogical_shift_left (const Packet4i &a)
template<int N>
EIGEN_STRONG_INLINE Packet4f	plogical_shift_left (const Packet4f &a)
template<int N>
EIGEN_STRONG_INLINE Packet4f	plogical_shift_right (const Packet4f &a)
template<int N>
EIGEN_STRONG_INLINE Packet4ui	plogical_shift_right (const Packet4ui &a)
template<int N>
EIGEN_STRONG_INLINE Packet4ui	plogical_shift_left (const Packet4ui &a)
template<int N>
EIGEN_STRONG_INLINE Packet8us	plogical_shift_left (const Packet8us &a)
template<int N>
EIGEN_STRONG_INLINE Packet8us	plogical_shift_right (const Packet8us &a)
EIGEN_STRONG_INLINE Packet4f	Bf16ToF32Even (const Packet8bf &bf)
EIGEN_STRONG_INLINE Packet4f	Bf16ToF32Odd (const Packet8bf &bf)
EIGEN_ALWAYS_INLINE Packet8us	pmerge (Packet4ui even, Packet4ui odd)
EIGEN_STRONG_INLINE Packet8bf	F32ToBf16Bool (Packet4f even, Packet4f odd)
EIGEN_STRONG_INLINE Packet8bf	F32ToBf16 (Packet4f p4f)
template<bool lohi>
EIGEN_ALWAYS_INLINE Packet8bf	Bf16PackLow (Packet4f hi, Packet4f lo)
template<bool lohi>
EIGEN_ALWAYS_INLINE Packet8bf	Bf16PackHigh (Packet4f hi, Packet4f lo)
template<bool lohi = true>
EIGEN_ALWAYS_INLINE Packet8bf	F32ToBf16Two (Packet4f lo, Packet4f hi)
EIGEN_STRONG_INLINE Packet8bf	F32ToBf16Both (Packet4f lo, Packet4f hi)
EIGEN_STRONG_INLINE Packet8bf	F32ToBf16 (Packet4f even, Packet4f odd)
template<>
EIGEN_STRONG_INLINE Packet8bf	padd< Packet8bf > (const Packet8bf &a, const Packet8bf &b)
template<>
EIGEN_STRONG_INLINE Packet8bf	pmul< Packet8bf > (const Packet8bf &a, const Packet8bf &b)
template<>
EIGEN_STRONG_INLINE Packet8bf	pdiv< Packet8bf > (const Packet8bf &a, const Packet8bf &b)
template<>
EIGEN_STRONG_INLINE Packet8bf	pnegate< Packet8bf > (const Packet8bf &a)
template<>
EIGEN_STRONG_INLINE Packet8bf	psub< Packet8bf > (const Packet8bf &a, const Packet8bf &b)
template<>
EIGEN_STRONG_INLINE Packet8bf	pexp< Packet8bf > (const Packet8bf &a)
template<>
EIGEN_STRONG_INLINE Packet8bf	pexp2< Packet8bf > (const Packet8bf &a)
template<>
EIGEN_STRONG_INLINE Packet4f	pldexp< Packet4f > (const Packet4f &a, const Packet4f &exponent)
template<>
EIGEN_STRONG_INLINE Packet8bf	pldexp< Packet8bf > (const Packet8bf &a, const Packet8bf &exponent)
template<>
EIGEN_STRONG_INLINE Packet4f	pfrexp< Packet4f > (const Packet4f &a, Packet4f &exponent)
template<>
EIGEN_STRONG_INLINE Packet8bf	pfrexp< Packet8bf > (const Packet8bf &a, Packet8bf &e)
template<>
EIGEN_STRONG_INLINE Packet8bf	psin< Packet8bf > (const Packet8bf &a)
template<>
EIGEN_STRONG_INLINE Packet8bf	pcos< Packet8bf > (const Packet8bf &a)
template<>
EIGEN_STRONG_INLINE Packet8bf	plog< Packet8bf > (const Packet8bf &a)
template<>
EIGEN_STRONG_INLINE Packet8bf	pfloor< Packet8bf > (const Packet8bf &a)
template<>
EIGEN_STRONG_INLINE Packet8bf	pceil< Packet8bf > (const Packet8bf &a)
template<>
EIGEN_STRONG_INLINE Packet8bf	pround< Packet8bf > (const Packet8bf &a)
template<>
EIGEN_STRONG_INLINE Packet8bf	ptrunc< Packet8bf > (const Packet8bf &a)
template<>
EIGEN_STRONG_INLINE Packet8bf	pmadd (const Packet8bf &a, const Packet8bf &b, const Packet8bf &c)
template<>
EIGEN_STRONG_INLINE Packet8bf	pmin< Packet8bf > (const Packet8bf &a, const Packet8bf &b)
template<>
EIGEN_STRONG_INLINE Packet8bf	pmax< Packet8bf > (const Packet8bf &a, const Packet8bf &b)
template<>
EIGEN_STRONG_INLINE Packet8bf	pcmp_lt (const Packet8bf &a, const Packet8bf &b)
template<>
EIGEN_STRONG_INLINE Packet8bf	pcmp_lt_or_nan (const Packet8bf &a, const Packet8bf &b)
template<>
EIGEN_STRONG_INLINE Packet8bf	pcmp_le (const Packet8bf &a, const Packet8bf &b)
template<>
EIGEN_STRONG_INLINE Packet8bf	pcmp_eq (const Packet8bf &a, const Packet8bf &b)
template<>
EIGEN_STRONG_INLINE bfloat16	pfirst (const Packet8bf &a)
template<>
EIGEN_STRONG_INLINE Packet8bf	ploaddup< Packet8bf > (const bfloat16 *from)
template<>
EIGEN_STRONG_INLINE Packet8bf	plset< Packet8bf > (const bfloat16 &a)
template<>
EIGEN_STRONG_INLINE float	predux< Packet4f > (const Packet4f &a)
template<>
EIGEN_STRONG_INLINE int	predux< Packet4i > (const Packet4i &a)
template<>
EIGEN_STRONG_INLINE bfloat16	predux< Packet8bf > (const Packet8bf &a)
template<>
EIGEN_STRONG_INLINE short int	predux< Packet8s > (const Packet8s &a)
template<>
EIGEN_STRONG_INLINE unsigned short int	predux< Packet8us > (const Packet8us &a)
template<>
EIGEN_STRONG_INLINE signed char	predux< Packet16c > (const Packet16c &a)
template<>
EIGEN_STRONG_INLINE unsigned char	predux< Packet16uc > (const Packet16uc &a)
template<>
EIGEN_STRONG_INLINE float	predux_mul< Packet4f > (const Packet4f &a)
template<>
EIGEN_STRONG_INLINE int	predux_mul< Packet4i > (const Packet4i &a)
template<>
EIGEN_STRONG_INLINE short int	predux_mul< Packet8s > (const Packet8s &a)
template<>
EIGEN_STRONG_INLINE unsigned short int	predux_mul< Packet8us > (const Packet8us &a)
template<>
EIGEN_STRONG_INLINE bfloat16	predux_mul< Packet8bf > (const Packet8bf &a)
template<>
EIGEN_STRONG_INLINE signed char	predux_mul< Packet16c > (const Packet16c &a)
template<>
EIGEN_STRONG_INLINE unsigned char	predux_mul< Packet16uc > (const Packet16uc &a)
template<>
EIGEN_STRONG_INLINE float	predux_min< Packet4f > (const Packet4f &a)
template<>
EIGEN_STRONG_INLINE int	predux_min< Packet4i > (const Packet4i &a)
template<>
EIGEN_STRONG_INLINE bfloat16	predux_min< Packet8bf > (const Packet8bf &a)
template<>
EIGEN_STRONG_INLINE short int	predux_min< Packet8s > (const Packet8s &a)
template<>
EIGEN_STRONG_INLINE unsigned short int	predux_min< Packet8us > (const Packet8us &a)
template<>
EIGEN_STRONG_INLINE signed char	predux_min< Packet16c > (const Packet16c &a)
template<>
EIGEN_STRONG_INLINE unsigned char	predux_min< Packet16uc > (const Packet16uc &a)
template<>
EIGEN_STRONG_INLINE float	predux_max< Packet4f > (const Packet4f &a)
template<>
EIGEN_STRONG_INLINE int	predux_max< Packet4i > (const Packet4i &a)
template<>
EIGEN_STRONG_INLINE bfloat16	predux_max< Packet8bf > (const Packet8bf &a)
template<>
EIGEN_STRONG_INLINE short int	predux_max< Packet8s > (const Packet8s &a)
template<>
EIGEN_STRONG_INLINE unsigned short int	predux_max< Packet8us > (const Packet8us &a)
template<>
EIGEN_STRONG_INLINE signed char	predux_max< Packet16c > (const Packet16c &a)
template<>
EIGEN_STRONG_INLINE unsigned char	predux_max< Packet16uc > (const Packet16uc &a)
template<>
EIGEN_STRONG_INLINE bool	predux_any (const Packet4f &x)
template<typename T >
EIGEN_DEVICE_FUNC void	ptranpose_common (PacketBlock< T, 4 > &kernel)
EIGEN_DEVICE_FUNC void	ptranspose (PacketBlock< Packet4f, 4 > &kernel)
EIGEN_DEVICE_FUNC void	ptranspose (PacketBlock< Packet4i, 4 > &kernel)
EIGEN_DEVICE_FUNC void	ptranspose (PacketBlock< Packet8s, 4 > &kernel)
EIGEN_DEVICE_FUNC void	ptranspose (PacketBlock< Packet8us, 4 > &kernel)
EIGEN_DEVICE_FUNC void	ptranspose (PacketBlock< Packet8bf, 4 > &kernel)
EIGEN_DEVICE_FUNC void	ptranspose (PacketBlock< Packet16c, 4 > &kernel)
EIGEN_DEVICE_FUNC void	ptranspose (PacketBlock< Packet16uc, 4 > &kernel)
EIGEN_DEVICE_FUNC void	ptranspose (PacketBlock< Packet8s, 8 > &kernel)
EIGEN_DEVICE_FUNC void	ptranspose (PacketBlock< Packet8us, 8 > &kernel)
EIGEN_DEVICE_FUNC void	ptranspose (PacketBlock< Packet8bf, 8 > &kernel)
EIGEN_DEVICE_FUNC void	ptranspose (PacketBlock< Packet16c, 16 > &kernel)
EIGEN_DEVICE_FUNC void	ptranspose (PacketBlock< Packet16uc, 16 > &kernel)
template<typename Packet >
EIGEN_STRONG_INLINE Packet	pblend4 (const Selector< 4 > &ifPacket, const Packet &thenPacket, const Packet &elsePacket)
template<>
EIGEN_STRONG_INLINE Packet4i	pblend (const Selector< 4 > &ifPacket, const Packet4i &thenPacket, const Packet4i &elsePacket)
template<>
EIGEN_STRONG_INLINE Packet4f	pblend (const Selector< 4 > &ifPacket, const Packet4f &thenPacket, const Packet4f &elsePacket)
template<>
EIGEN_STRONG_INLINE Packet8s	pblend (const Selector< 8 > &ifPacket, const Packet8s &thenPacket, const Packet8s &elsePacket)
template<>
EIGEN_STRONG_INLINE Packet8us	pblend (const Selector< 8 > &ifPacket, const Packet8us &thenPacket, const Packet8us &elsePacket)
template<>
EIGEN_STRONG_INLINE Packet8bf	pblend (const Selector< 8 > &ifPacket, const Packet8bf &thenPacket, const Packet8bf &elsePacket)
template<>
EIGEN_STRONG_INLINE Packet16c	pblend (const Selector< 16 > &ifPacket, const Packet16c &thenPacket, const Packet16c &elsePacket)
template<>
EIGEN_STRONG_INLINE Packet16uc	pblend (const Selector< 16 > &ifPacket, const Packet16uc &thenPacket, const Packet16uc &elsePacket)
template<>
EIGEN_STRONG_INLINE Packet4i	pcast< Packet4f, Packet4i > (const Packet4f &a)
template<>
EIGEN_STRONG_INLINE Packet4ui	pcast< Packet4f, Packet4ui > (const Packet4f &a)
template<>
EIGEN_STRONG_INLINE Packet4f	pcast< Packet4i, Packet4f > (const Packet4i &a)
template<>
EIGEN_STRONG_INLINE Packet4f	pcast< Packet4ui, Packet4f > (const Packet4ui &a)
template<>
EIGEN_STRONG_INLINE Packet8us	pcast< Packet8bf, Packet8us > (const Packet8bf &a)
template<>
EIGEN_STRONG_INLINE Packet8bf	pcast< Packet8us, Packet8bf > (const Packet8us &a)
template<>
EIGEN_STRONG_INLINE Packet4f	pcast< Packet8bf, Packet4f > (const Packet8bf &a)
template<>
EIGEN_STRONG_INLINE Packet8bf	pcast< Packet4f, Packet8bf > (const Packet4f &a, const Packet4f &b)
template<>
EIGEN_STRONG_INLINE Packet4i	preinterpret< Packet4i, Packet4f > (const Packet4f &a)
template<>
EIGEN_STRONG_INLINE Packet4f	preinterpret< Packet4f, Packet4i > (const Packet4i &a)
template<>
EIGEN_STRONG_INLINE Packet4cf	padd< Packet4cf > (const Packet4cf &a, const Packet4cf &b)
template<>
EIGEN_STRONG_INLINE Packet4cf	psub< Packet4cf > (const Packet4cf &a, const Packet4cf &b)
template<>
EIGEN_STRONG_INLINE Packet4cf	pnegate (const Packet4cf &a)
template<>
EIGEN_STRONG_INLINE Packet4cf	pconj (const Packet4cf &a)
template<>
EIGEN_STRONG_INLINE Packet4cf	pmul (const Packet4cf &a, const Packet4cf &b)
template<>
EIGEN_STRONG_INLINE Packet4cf	pcmp_eq (const Packet4cf &a, const Packet4cf &b)
template<>
EIGEN_STRONG_INLINE Packet4cf	ptrue< Packet4cf > (const Packet4cf &a)
template<>
EIGEN_STRONG_INLINE Packet4cf	pand< Packet4cf > (const Packet4cf &a, const Packet4cf &b)
template<>
EIGEN_STRONG_INLINE Packet4cf	por< Packet4cf > (const Packet4cf &a, const Packet4cf &b)
template<>
EIGEN_STRONG_INLINE Packet4cf	pxor< Packet4cf > (const Packet4cf &a, const Packet4cf &b)
template<>
EIGEN_STRONG_INLINE Packet4cf	pandnot< Packet4cf > (const Packet4cf &a, const Packet4cf &b)
template<>
EIGEN_STRONG_INLINE Packet4cf	pload< Packet4cf > (const std::complex< float > *from)
template<>
EIGEN_STRONG_INLINE Packet4cf	ploadu< Packet4cf > (const std::complex< float > *from)
template<>
EIGEN_STRONG_INLINE Packet4cf	pset1< Packet4cf > (const std::complex< float > &from)
template<>
EIGEN_STRONG_INLINE Packet4cf	ploaddup< Packet4cf > (const std::complex< float > *from)
template<>
EIGEN_STRONG_INLINE void	pstore< std::complex< float > > (std::complex< float > *to, const Packet4cf &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< std::complex< float > > (std::complex< float > *to, const Packet4cf &from)
template<>
EIGEN_DEVICE_FUNC Packet4cf	pgather< std::complex< float >, Packet4cf > (const std::complex< float > *from, Index stride)
template<>
EIGEN_DEVICE_FUNC void	pscatter< std::complex< float >, Packet4cf > (std::complex< float > *to, const Packet4cf &from, Index stride)
template<>
EIGEN_STRONG_INLINE std::complex< float >	pfirst< Packet4cf > (const Packet4cf &a)
template<>
EIGEN_STRONG_INLINE Packet4cf	preverse (const Packet4cf &a)
template<>
EIGEN_STRONG_INLINE std::complex< float >	predux< Packet4cf > (const Packet4cf &a)
template<>
EIGEN_STRONG_INLINE std::complex< float >	predux_mul< Packet4cf > (const Packet4cf &a)
template<>
EIGEN_STRONG_INLINE Packet4cf	pdiv< Packet4cf > (const Packet4cf &a, const Packet4cf &b)
template<>
EIGEN_STRONG_INLINE Packet4cf	pcplxflip< Packet4cf > (const Packet4cf &x)
template<>
EIGEN_STRONG_INLINE Packet2cd	padd< Packet2cd > (const Packet2cd &a, const Packet2cd &b)
template<>
EIGEN_STRONG_INLINE Packet2cd	psub< Packet2cd > (const Packet2cd &a, const Packet2cd &b)
template<>
EIGEN_STRONG_INLINE Packet2cd	pnegate (const Packet2cd &a)
template<>
EIGEN_STRONG_INLINE Packet2cd	pconj (const Packet2cd &a)
template<>
EIGEN_STRONG_INLINE Packet2cd	pmul (const Packet2cd &a, const Packet2cd &b)
template<>
EIGEN_STRONG_INLINE Packet2cd	pcmp_eq (const Packet2cd &a, const Packet2cd &b)
template<>
EIGEN_STRONG_INLINE Packet2cd	ptrue< Packet2cd > (const Packet2cd &a)
template<>
EIGEN_STRONG_INLINE Packet2cd	pand< Packet2cd > (const Packet2cd &a, const Packet2cd &b)
template<>
EIGEN_STRONG_INLINE Packet2cd	por< Packet2cd > (const Packet2cd &a, const Packet2cd &b)
template<>
EIGEN_STRONG_INLINE Packet2cd	pxor< Packet2cd > (const Packet2cd &a, const Packet2cd &b)
template<>
EIGEN_STRONG_INLINE Packet2cd	pandnot< Packet2cd > (const Packet2cd &a, const Packet2cd &b)
template<>
EIGEN_STRONG_INLINE Packet2cd	pload< Packet2cd > (const std::complex< double > *from)
template<>
EIGEN_STRONG_INLINE Packet2cd	ploadu< Packet2cd > (const std::complex< double > *from)
template<>
EIGEN_STRONG_INLINE Packet2cd	pset1< Packet2cd > (const std::complex< double > &from)
template<>
EIGEN_STRONG_INLINE Packet2cd	ploaddup< Packet2cd > (const std::complex< double > *from)
template<>
EIGEN_STRONG_INLINE void	pstore< std::complex< double > > (std::complex< double > *to, const Packet2cd &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< std::complex< double > > (std::complex< double > *to, const Packet2cd &from)
template<>
EIGEN_DEVICE_FUNC Packet2cd	pgather< std::complex< double >, Packet2cd > (const std::complex< double > *from, Index stride)
template<>
EIGEN_DEVICE_FUNC void	pscatter< std::complex< double >, Packet2cd > (std::complex< double > *to, const Packet2cd &from, Index stride)
template<>
EIGEN_STRONG_INLINE std::complex< double >	pfirst< Packet2cd > (const Packet2cd &a)
template<>
EIGEN_STRONG_INLINE Packet2cd	preverse (const Packet2cd &a)
template<>
EIGEN_STRONG_INLINE std::complex< double >	predux< Packet2cd > (const Packet2cd &a)
template<>
EIGEN_STRONG_INLINE std::complex< double >	predux_mul< Packet2cd > (const Packet2cd &a)
template<>
EIGEN_STRONG_INLINE Packet2cd	pdiv< Packet2cd > (const Packet2cd &a, const Packet2cd &b)
template<>
EIGEN_STRONG_INLINE Packet2cd	pcplxflip< Packet2cd > (const Packet2cd &x)
EIGEN_DEVICE_FUNC void	ptranspose (PacketBlock< Packet4cf, 4 > &kernel)
EIGEN_DEVICE_FUNC void	ptranspose (PacketBlock< Packet2cd, 2 > &kernel)
template<>
EIGEN_STRONG_INLINE Packet2cd	psqrt< Packet2cd > (const Packet2cd &a)
template<>
EIGEN_STRONG_INLINE Packet4cf	psqrt< Packet4cf > (const Packet4cf &a)
template<>
EIGEN_STRONG_INLINE Packet2cd	plog< Packet2cd > (const Packet2cd &a)
template<>
EIGEN_STRONG_INLINE Packet4cf	plog< Packet4cf > (const Packet4cf &a)
template<>
EIGEN_STRONG_INLINE Packet4cf	pexp< Packet4cf > (const Packet4cf &a)
template<>
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet8f	psqrt< Packet8f > (const Packet8f &_x)
template<>
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet4d	psqrt< Packet4d > (const Packet4d &_x)
template<>
EIGEN_STRONG_INLINE Packet8h	pfrexp (const Packet8h &a, Packet8h &exponent)
template<>
EIGEN_STRONG_INLINE Packet8h	pldexp (const Packet8h &a, const Packet8h &exponent)
template<>
EIGEN_STRONG_INLINE Packet8bf	pfrexp (const Packet8bf &a, Packet8bf &exponent)
template<>
EIGEN_STRONG_INLINE Packet8bf	pldexp (const Packet8bf &a, const Packet8bf &exponent)
EIGEN_STRONG_INLINE __m128i	Pack16To8 (Packet8f rf)
template<>
EIGEN_STRONG_INLINE Packet8f	pset1< Packet8f > (const float &from)
template<>
EIGEN_STRONG_INLINE Packet4d	pset1< Packet4d > (const double &from)
template<>
EIGEN_STRONG_INLINE Packet8i	pset1< Packet8i > (const int &from)
template<>
EIGEN_STRONG_INLINE Packet8ui	pset1< Packet8ui > (const uint32_t &from)
template<>
EIGEN_STRONG_INLINE Packet8f	pset1frombits< Packet8f > (unsigned int from)
template<>
EIGEN_STRONG_INLINE Packet4d	pset1frombits< Packet4d > (uint64_t from)
template<>
EIGEN_STRONG_INLINE Packet8f	pzero (const Packet8f &)
template<>
EIGEN_STRONG_INLINE Packet4d	pzero (const Packet4d &)
template<>
EIGEN_STRONG_INLINE Packet8i	pzero (const Packet8i &)
template<>
EIGEN_STRONG_INLINE Packet8ui	pzero (const Packet8ui &)
template<>
EIGEN_STRONG_INLINE Packet8f	peven_mask (const Packet8f &)
template<>
EIGEN_STRONG_INLINE Packet8i	peven_mask (const Packet8i &)
template<>
EIGEN_STRONG_INLINE Packet8ui	peven_mask (const Packet8ui &)
template<>
EIGEN_STRONG_INLINE Packet4d	peven_mask (const Packet4d &)
template<>
EIGEN_STRONG_INLINE Packet8f	pload1< Packet8f > (const float *from)
template<>
EIGEN_STRONG_INLINE Packet4d	pload1< Packet4d > (const double *from)
template<>
EIGEN_STRONG_INLINE Packet8f	padd< Packet8f > (const Packet8f &a, const Packet8f &b)
template<>
EIGEN_STRONG_INLINE Packet4d	padd< Packet4d > (const Packet4d &a, const Packet4d &b)
template<>
EIGEN_STRONG_INLINE Packet8i	padd< Packet8i > (const Packet8i &a, const Packet8i &b)
template<>
EIGEN_STRONG_INLINE Packet8ui	padd< Packet8ui > (const Packet8ui &a, const Packet8ui &b)
template<>
EIGEN_STRONG_INLINE Packet8f	plset< Packet8f > (const float &a)
template<>
EIGEN_STRONG_INLINE Packet4d	plset< Packet4d > (const double &a)
template<>
EIGEN_STRONG_INLINE Packet8i	plset< Packet8i > (const int &a)
template<>
EIGEN_STRONG_INLINE Packet8ui	plset< Packet8ui > (const uint32_t &a)
template<>
EIGEN_STRONG_INLINE Packet8f	psub< Packet8f > (const Packet8f &a, const Packet8f &b)
template<>
EIGEN_STRONG_INLINE Packet4d	psub< Packet4d > (const Packet4d &a, const Packet4d &b)
template<>
EIGEN_STRONG_INLINE Packet8i	psub< Packet8i > (const Packet8i &a, const Packet8i &b)
template<>
EIGEN_STRONG_INLINE Packet8ui	psub< Packet8ui > (const Packet8ui &a, const Packet8ui &b)
template<>
EIGEN_STRONG_INLINE Packet8f	pnegate (const Packet8f &a)
template<>
EIGEN_STRONG_INLINE Packet4d	pnegate (const Packet4d &a)
template<>
EIGEN_STRONG_INLINE Packet8i	pnegate (const Packet8i &a)
template<>
EIGEN_STRONG_INLINE Packet8f	pconj (const Packet8f &a)
template<>
EIGEN_STRONG_INLINE Packet4d	pconj (const Packet4d &a)
template<>
EIGEN_STRONG_INLINE Packet8i	pconj (const Packet8i &a)
template<>
EIGEN_STRONG_INLINE Packet8f	pmul< Packet8f > (const Packet8f &a, const Packet8f &b)
template<>
EIGEN_STRONG_INLINE Packet4d	pmul< Packet4d > (const Packet4d &a, const Packet4d &b)
template<>
EIGEN_STRONG_INLINE Packet8i	pmul< Packet8i > (const Packet8i &a, const Packet8i &b)
template<>
EIGEN_STRONG_INLINE Packet8ui	pmul< Packet8ui > (const Packet8ui &a, const Packet8ui &b)
template<>
EIGEN_STRONG_INLINE Packet8f	pdiv< Packet8f > (const Packet8f &a, const Packet8f &b)
template<>
EIGEN_STRONG_INLINE Packet4d	pdiv< Packet4d > (const Packet4d &a, const Packet4d &b)
template<>
EIGEN_STRONG_INLINE Packet8i	pdiv< Packet8i > (const Packet8i &a, const Packet8i &b)
template<>
EIGEN_STRONG_INLINE Packet8f	pcmp_le (const Packet8f &a, const Packet8f &b)
template<>
EIGEN_STRONG_INLINE Packet8f	pcmp_lt (const Packet8f &a, const Packet8f &b)
template<>
EIGEN_STRONG_INLINE Packet8f	pcmp_lt_or_nan (const Packet8f &a, const Packet8f &b)
template<>
EIGEN_STRONG_INLINE Packet8f	pcmp_eq (const Packet8f &a, const Packet8f &b)
template<>
EIGEN_STRONG_INLINE Packet8f	pisnan (const Packet8f &a)
template<>
EIGEN_STRONG_INLINE Packet4d	pcmp_le (const Packet4d &a, const Packet4d &b)
template<>
EIGEN_STRONG_INLINE Packet4d	pcmp_lt (const Packet4d &a, const Packet4d &b)
template<>
EIGEN_STRONG_INLINE Packet4d	pcmp_lt_or_nan (const Packet4d &a, const Packet4d &b)
template<>
EIGEN_STRONG_INLINE Packet4d	pcmp_eq (const Packet4d &a, const Packet4d &b)
template<>
EIGEN_STRONG_INLINE Packet8i	pcmp_le (const Packet8i &a, const Packet8i &b)
template<>
EIGEN_STRONG_INLINE Packet8i	pcmp_lt (const Packet8i &a, const Packet8i &b)
template<>
EIGEN_STRONG_INLINE Packet8i	pcmp_eq (const Packet8i &a, const Packet8i &b)
template<>
EIGEN_STRONG_INLINE Packet8ui	pcmp_eq (const Packet8ui &a, const Packet8ui &b)
template<>
EIGEN_STRONG_INLINE Packet8f	pmin< Packet8f > (const Packet8f &a, const Packet8f &b)
template<>
EIGEN_STRONG_INLINE Packet4d	pmin< Packet4d > (const Packet4d &a, const Packet4d &b)
template<>
EIGEN_STRONG_INLINE Packet8i	pmin< Packet8i > (const Packet8i &a, const Packet8i &b)
template<>
EIGEN_STRONG_INLINE Packet8ui	pmin< Packet8ui > (const Packet8ui &a, const Packet8ui &b)
template<>
EIGEN_STRONG_INLINE Packet8f	pmax< Packet8f > (const Packet8f &a, const Packet8f &b)
template<>
EIGEN_STRONG_INLINE Packet4d	pmax< Packet4d > (const Packet4d &a, const Packet4d &b)
template<>
EIGEN_STRONG_INLINE Packet8i	pmax< Packet8i > (const Packet8i &a, const Packet8i &b)
template<>
EIGEN_STRONG_INLINE Packet8ui	pmax< Packet8ui > (const Packet8ui &a, const Packet8ui &b)
template<>
EIGEN_STRONG_INLINE Packet8f	pmin< PropagateNumbers, Packet8f > (const Packet8f &a, const Packet8f &b)
template<>
EIGEN_STRONG_INLINE Packet4d	pmin< PropagateNumbers, Packet4d > (const Packet4d &a, const Packet4d &b)
template<>
EIGEN_STRONG_INLINE Packet8f	pmax< PropagateNumbers, Packet8f > (const Packet8f &a, const Packet8f &b)
template<>
EIGEN_STRONG_INLINE Packet4d	pmax< PropagateNumbers, Packet4d > (const Packet4d &a, const Packet4d &b)
template<>
EIGEN_STRONG_INLINE Packet8f	pmin< PropagateNaN, Packet8f > (const Packet8f &a, const Packet8f &b)
template<>
EIGEN_STRONG_INLINE Packet4d	pmin< PropagateNaN, Packet4d > (const Packet4d &a, const Packet4d &b)
template<>
EIGEN_STRONG_INLINE Packet8f	pmax< PropagateNaN, Packet8f > (const Packet8f &a, const Packet8f &b)
template<>
EIGEN_STRONG_INLINE Packet4d	pmax< PropagateNaN, Packet4d > (const Packet4d &a, const Packet4d &b)
template<>
EIGEN_STRONG_INLINE Packet8f	print< Packet8f > (const Packet8f &a)
template<>
EIGEN_STRONG_INLINE Packet4d	print< Packet4d > (const Packet4d &a)
template<>
EIGEN_STRONG_INLINE Packet8f	pceil< Packet8f > (const Packet8f &a)
template<>
EIGEN_STRONG_INLINE Packet4d	pceil< Packet4d > (const Packet4d &a)
template<>
EIGEN_STRONG_INLINE Packet8f	pfloor< Packet8f > (const Packet8f &a)
template<>
EIGEN_STRONG_INLINE Packet4d	pfloor< Packet4d > (const Packet4d &a)
template<>
EIGEN_STRONG_INLINE Packet8f	ptrunc< Packet8f > (const Packet8f &a)
template<>
EIGEN_STRONG_INLINE Packet4d	ptrunc< Packet4d > (const Packet4d &a)
template<>
EIGEN_STRONG_INLINE Packet8i	ptrue< Packet8i > (const Packet8i &a)
template<>
EIGEN_STRONG_INLINE Packet8f	ptrue< Packet8f > (const Packet8f &a)
template<>
EIGEN_STRONG_INLINE Packet4d	ptrue< Packet4d > (const Packet4d &a)
template<>
EIGEN_STRONG_INLINE Packet8f	pand< Packet8f > (const Packet8f &a, const Packet8f &b)
template<>
EIGEN_STRONG_INLINE Packet4d	pand< Packet4d > (const Packet4d &a, const Packet4d &b)
template<>
EIGEN_STRONG_INLINE Packet8i	pand< Packet8i > (const Packet8i &a, const Packet8i &b)
template<>
EIGEN_STRONG_INLINE Packet8ui	pand< Packet8ui > (const Packet8ui &a, const Packet8ui &b)
template<>
EIGEN_STRONG_INLINE Packet8f	por< Packet8f > (const Packet8f &a, const Packet8f &b)
template<>
EIGEN_STRONG_INLINE Packet4d	por< Packet4d > (const Packet4d &a, const Packet4d &b)
template<>
EIGEN_STRONG_INLINE Packet8i	por< Packet8i > (const Packet8i &a, const Packet8i &b)
template<>
EIGEN_STRONG_INLINE Packet8ui	por< Packet8ui > (const Packet8ui &a, const Packet8ui &b)
template<>
EIGEN_STRONG_INLINE Packet8f	pxor< Packet8f > (const Packet8f &a, const Packet8f &b)
template<>
EIGEN_STRONG_INLINE Packet4d	pxor< Packet4d > (const Packet4d &a, const Packet4d &b)
template<>
EIGEN_STRONG_INLINE Packet8i	pxor< Packet8i > (const Packet8i &a, const Packet8i &b)
template<>
EIGEN_STRONG_INLINE Packet8ui	pxor< Packet8ui > (const Packet8ui &a, const Packet8ui &b)
template<>
EIGEN_STRONG_INLINE Packet8f	pandnot< Packet8f > (const Packet8f &a, const Packet8f &b)
template<>
EIGEN_STRONG_INLINE Packet4d	pandnot< Packet4d > (const Packet4d &a, const Packet4d &b)
template<>
EIGEN_STRONG_INLINE Packet8i	pandnot< Packet8i > (const Packet8i &a, const Packet8i &b)
template<>
EIGEN_STRONG_INLINE Packet8ui	pandnot< Packet8ui > (const Packet8ui &a, const Packet8ui &b)
template<>
EIGEN_STRONG_INLINE Packet8ui	pcmp_lt (const Packet8ui &a, const Packet8ui &b)
template<>
EIGEN_STRONG_INLINE Packet8ui	pcmp_le (const Packet8ui &a, const Packet8ui &b)
template<>
EIGEN_STRONG_INLINE Packet8f	pround< Packet8f > (const Packet8f &a)
template<>
EIGEN_STRONG_INLINE Packet4d	pround< Packet4d > (const Packet4d &a)
template<>
EIGEN_STRONG_INLINE Packet8f	pselect< Packet8f > (const Packet8f &mask, const Packet8f &a, const Packet8f &b)
template<>
EIGEN_STRONG_INLINE Packet8i	pselect< Packet8i > (const Packet8i &mask, const Packet8i &a, const Packet8i &b)
template<>
EIGEN_STRONG_INLINE Packet8ui	pselect< Packet8ui > (const Packet8ui &mask, const Packet8ui &a, const Packet8ui &b)
template<>
EIGEN_STRONG_INLINE Packet4d	pselect< Packet4d > (const Packet4d &mask, const Packet4d &a, const Packet4d &b)
template<int N>
EIGEN_STRONG_INLINE Packet8i	parithmetic_shift_right (Packet8i a)
template<int N>
EIGEN_STRONG_INLINE Packet8i	plogical_shift_right (Packet8i a)
template<int N>
EIGEN_STRONG_INLINE Packet8i	plogical_shift_left (Packet8i a)
template<int N>
EIGEN_STRONG_INLINE Packet8ui	parithmetic_shift_right (Packet8ui a)
template<int N>
EIGEN_STRONG_INLINE Packet8ui	plogical_shift_right (Packet8ui a)
template<int N>
EIGEN_STRONG_INLINE Packet8ui	plogical_shift_left (Packet8ui a)
template<>
EIGEN_STRONG_INLINE Packet8f	pload< Packet8f > (const float *from)
template<>
EIGEN_STRONG_INLINE Packet4d	pload< Packet4d > (const double *from)
template<>
EIGEN_STRONG_INLINE Packet8i	pload< Packet8i > (const int *from)
template<>
EIGEN_STRONG_INLINE Packet8ui	pload< Packet8ui > (const uint32_t *from)
template<>
EIGEN_STRONG_INLINE Packet8f	ploadu< Packet8f > (const float *from)
template<>
EIGEN_STRONG_INLINE Packet4d	ploadu< Packet4d > (const double *from)
template<>
EIGEN_STRONG_INLINE Packet8i	ploadu< Packet8i > (const int *from)
template<>
EIGEN_STRONG_INLINE Packet8ui	ploadu< Packet8ui > (const uint32_t *from)
template<>
EIGEN_STRONG_INLINE Packet8f	ploadu< Packet8f > (const float *from, uint8_t umask)
template<>
EIGEN_STRONG_INLINE Packet8f	ploaddup< Packet8f > (const float *from)
template<>
EIGEN_STRONG_INLINE Packet4d	ploaddup< Packet4d > (const double *from)
template<>
EIGEN_STRONG_INLINE Packet8i	ploaddup< Packet8i > (const int *from)
template<>
EIGEN_STRONG_INLINE Packet8ui	ploaddup< Packet8ui > (const uint32_t *from)
template<>
EIGEN_STRONG_INLINE Packet8f	ploadquad< Packet8f > (const float *from)
template<>
EIGEN_STRONG_INLINE Packet8i	ploadquad< Packet8i > (const int *from)
template<>
EIGEN_STRONG_INLINE Packet8ui	ploadquad< Packet8ui > (const uint32_t *from)
template<>
EIGEN_STRONG_INLINE void	pstore< float > (float *to, const Packet8f &from)
template<>
EIGEN_STRONG_INLINE void	pstore< double > (double *to, const Packet4d &from)
template<>
EIGEN_STRONG_INLINE void	pstore< int > (int *to, const Packet8i &from)
template<>
EIGEN_STRONG_INLINE void	pstore< uint32_t > (uint32_t *to, const Packet8ui &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< float > (float *to, const Packet8f &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< double > (double *to, const Packet4d &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< int > (int *to, const Packet8i &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< uint32_t > (uint32_t *to, const Packet8ui &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< float > (float *to, const Packet8f &from, uint8_t umask)
template<>
EIGEN_DEVICE_FUNC Packet8f	pgather< float, Packet8f > (const float *from, Index stride)
template<>
EIGEN_DEVICE_FUNC Packet4d	pgather< double, Packet4d > (const double *from, Index stride)
template<>
EIGEN_DEVICE_FUNC Packet8i	pgather< int, Packet8i > (const int *from, Index stride)
template<>
EIGEN_DEVICE_FUNC Packet8ui	pgather< uint32_t, Packet8ui > (const uint32_t *from, Index stride)
template<>
EIGEN_DEVICE_FUNC void	pscatter< float, Packet8f > (float *to, const Packet8f &from, Index stride)
template<>
EIGEN_DEVICE_FUNC void	pscatter< double, Packet4d > (double *to, const Packet4d &from, Index stride)
template<>
EIGEN_DEVICE_FUNC void	pscatter< int, Packet8i > (int *to, const Packet8i &from, Index stride)
template<>
EIGEN_DEVICE_FUNC void	pscatter< uint32_t, Packet8ui > (uint32_t *to, const Packet8ui &from, Index stride)
template<>
EIGEN_STRONG_INLINE void	pstore1< Packet8f > (float *to, const float &a)
template<>
EIGEN_STRONG_INLINE void	pstore1< Packet4d > (double *to, const double &a)
template<>
EIGEN_STRONG_INLINE void	pstore1< Packet8i > (int *to, const int &a)
template<>
EIGEN_STRONG_INLINE void	prefetch< double > (const double *addr)
template<>
EIGEN_STRONG_INLINE void	prefetch< uint32_t > (const uint32_t *addr)
template<>
EIGEN_STRONG_INLINE float	pfirst< Packet8f > (const Packet8f &a)
template<>
EIGEN_STRONG_INLINE double	pfirst< Packet4d > (const Packet4d &a)
template<>
EIGEN_STRONG_INLINE int	pfirst< Packet8i > (const Packet8i &a)
template<>
EIGEN_STRONG_INLINE uint32_t	pfirst< Packet8ui > (const Packet8ui &a)
template<>
EIGEN_STRONG_INLINE Packet8f	preverse (const Packet8f &a)
template<>
EIGEN_STRONG_INLINE Packet4d	preverse (const Packet4d &a)
template<>
EIGEN_STRONG_INLINE Packet8i	preverse (const Packet8i &a)
template<>
EIGEN_STRONG_INLINE Packet8ui	preverse (const Packet8ui &a)
template<>
EIGEN_STRONG_INLINE Packet8f	pabs (const Packet8f &a)
template<>
EIGEN_STRONG_INLINE Packet4d	pabs (const Packet4d &a)
template<>
EIGEN_STRONG_INLINE Packet8i	pabs (const Packet8i &a)
template<>
EIGEN_STRONG_INLINE Packet8ui	pabs (const Packet8ui &a)
template<>
EIGEN_STRONG_INLINE Packet8h	psignbit (const Packet8h &a)
template<>
EIGEN_STRONG_INLINE Packet8f	psignbit (const Packet8f &a)
template<>
EIGEN_STRONG_INLINE Packet8ui	psignbit (const Packet8ui &)
template<>
EIGEN_STRONG_INLINE Packet8f	pfrexp< Packet8f > (const Packet8f &a, Packet8f &exponent)
template<>
EIGEN_STRONG_INLINE Packet4d	pfrexp_generic_get_biased_exponent (const Packet4d &a)
template<>
EIGEN_STRONG_INLINE Packet4d	pfrexp< Packet4d > (const Packet4d &a, Packet4d &exponent)
template<>
EIGEN_STRONG_INLINE Packet8f	pldexp< Packet8f > (const Packet8f &a, const Packet8f &exponent)
template<>
EIGEN_STRONG_INLINE Packet4d	pldexp< Packet4d > (const Packet4d &a, const Packet4d &exponent)
template<>
EIGEN_STRONG_INLINE Packet4d	pldexp_fast< Packet4d > (const Packet4d &a, const Packet4d &exponent)
template<>
EIGEN_STRONG_INLINE float	predux< Packet8f > (const Packet8f &a)
template<>
EIGEN_STRONG_INLINE double	predux< Packet4d > (const Packet4d &a)
template<>
EIGEN_STRONG_INLINE int	predux< Packet8i > (const Packet8i &a)
template<>
EIGEN_STRONG_INLINE uint32_t	predux< Packet8ui > (const Packet8ui &a)
template<>
EIGEN_STRONG_INLINE Packet4f	predux_half_dowto4< Packet8f > (const Packet8f &a)
template<>
EIGEN_STRONG_INLINE Packet4i	predux_half_dowto4< Packet8i > (const Packet8i &a)
template<>
EIGEN_STRONG_INLINE Packet4ui	predux_half_dowto4< Packet8ui > (const Packet8ui &a)
template<>
EIGEN_STRONG_INLINE float	predux_mul< Packet8f > (const Packet8f &a)
template<>
EIGEN_STRONG_INLINE double	predux_mul< Packet4d > (const Packet4d &a)
template<>
EIGEN_STRONG_INLINE float	predux_min< Packet8f > (const Packet8f &a)
template<>
EIGEN_STRONG_INLINE double	predux_min< Packet4d > (const Packet4d &a)
template<>
EIGEN_STRONG_INLINE float	predux_max< Packet8f > (const Packet8f &a)
template<>
EIGEN_STRONG_INLINE double	predux_max< Packet4d > (const Packet4d &a)
template<>
EIGEN_STRONG_INLINE bool	predux_any (const Packet8f &x)
template<>
EIGEN_STRONG_INLINE bool	predux_any (const Packet4d &x)
template<>
EIGEN_STRONG_INLINE bool	predux_any (const Packet8i &x)
template<>
EIGEN_STRONG_INLINE bool	predux_any (const Packet8ui &x)
template<>
EIGEN_STRONG_INLINE bool	predux_any (const Packet8h &x)
template<>
EIGEN_STRONG_INLINE bool	predux_any (const Packet8bf &x)
EIGEN_DEVICE_FUNC void	ptranspose (PacketBlock< Packet8f, 8 > &kernel)
EIGEN_DEVICE_FUNC void	ptranspose (PacketBlock< Packet8f, 4 > &kernel)
EIGEN_DEVICE_FUNC void	ptranspose (PacketBlock< Packet8i, 8 > &kernel)
EIGEN_DEVICE_FUNC void	ptranspose (PacketBlock< Packet8ui, 8 > &kernel)
EIGEN_DEVICE_FUNC void	ptranspose (PacketBlock< Packet8i, 4 > &kernel)
EIGEN_DEVICE_FUNC void	ptranspose (PacketBlock< Packet8ui, 4 > &kernel)
EIGEN_DEVICE_FUNC void	ptranspose (PacketBlock< Packet4d, 4 > &kernel)
EIGEN_STRONG_INLINE __m256i	avx_blend_mask (const Selector< 4 > &ifPacket)
EIGEN_STRONG_INLINE __m256i	avx_blend_mask (const Selector< 8 > &ifPacket)
template<>
EIGEN_STRONG_INLINE Packet8f	pblend (const Selector< 8 > &ifPacket, const Packet8f &thenPacket, const Packet8f &elsePacket)
template<>
EIGEN_STRONG_INLINE Packet4d	pblend (const Selector< 4 > &ifPacket, const Packet4d &thenPacket, const Packet4d &elsePacket)
template<>
EIGEN_STRONG_INLINE Packet8h	pset1< Packet8h > (const Eigen::half &from)
template<>
EIGEN_STRONG_INLINE Eigen::half	pfirst< Packet8h > (const Packet8h &from)
template<>
EIGEN_STRONG_INLINE Packet8h	pload< Packet8h > (const Eigen::half *from)
template<>
EIGEN_STRONG_INLINE Packet8h	ploadu< Packet8h > (const Eigen::half *from)
template<>
EIGEN_STRONG_INLINE void	pstore< Eigen::half > (Eigen::half *to, const Packet8h &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< Eigen::half > (Eigen::half *to, const Packet8h &from)
template<>
EIGEN_STRONG_INLINE Packet8h	ploaddup< Packet8h > (const Eigen::half *from)
template<>
EIGEN_STRONG_INLINE Packet8h	ploadquad< Packet8h > (const Eigen::half *from)
template<>
EIGEN_STRONG_INLINE Packet8h	ptrue (const Packet8h &a)
template<>
EIGEN_STRONG_INLINE Packet8h	pabs (const Packet8h &a)
EIGEN_STRONG_INLINE Packet8f	half2float (const Packet8h &a)
EIGEN_STRONG_INLINE Packet8h	float2half (const Packet8f &a)
template<>
EIGEN_STRONG_INLINE Packet8h	pmin< Packet8h > (const Packet8h &a, const Packet8h &b)
template<>
EIGEN_STRONG_INLINE Packet8h	pmax< Packet8h > (const Packet8h &a, const Packet8h &b)
template<>
EIGEN_STRONG_INLINE Packet8h	plset< Packet8h > (const half &a)
template<>
EIGEN_STRONG_INLINE Packet8h	por (const Packet8h &a, const Packet8h &b)
template<>
EIGEN_STRONG_INLINE Packet8h	pxor (const Packet8h &a, const Packet8h &b)
template<>
EIGEN_STRONG_INLINE Packet8h	pand (const Packet8h &a, const Packet8h &b)
template<>
EIGEN_STRONG_INLINE Packet8h	pandnot (const Packet8h &a, const Packet8h &b)
template<>
EIGEN_STRONG_INLINE Packet8h	pselect (const Packet8h &mask, const Packet8h &a, const Packet8h &b)
template<>
EIGEN_STRONG_INLINE Packet8h	pround< Packet8h > (const Packet8h &a)
template<>
EIGEN_STRONG_INLINE Packet8h	print< Packet8h > (const Packet8h &a)
template<>
EIGEN_STRONG_INLINE Packet8h	pceil< Packet8h > (const Packet8h &a)
template<>
EIGEN_STRONG_INLINE Packet8h	pfloor< Packet8h > (const Packet8h &a)
template<>
EIGEN_STRONG_INLINE Packet8h	ptrunc< Packet8h > (const Packet8h &a)
template<>
EIGEN_STRONG_INLINE Packet8h	pcmp_eq (const Packet8h &a, const Packet8h &b)
template<>
EIGEN_STRONG_INLINE Packet8h	pcmp_le (const Packet8h &a, const Packet8h &b)
template<>
EIGEN_STRONG_INLINE Packet8h	pcmp_lt (const Packet8h &a, const Packet8h &b)
template<>
EIGEN_STRONG_INLINE Packet8h	pcmp_lt_or_nan (const Packet8h &a, const Packet8h &b)
template<>
EIGEN_STRONG_INLINE Packet8h	pconj (const Packet8h &a)
template<>
EIGEN_STRONG_INLINE Packet8h	pnegate (const Packet8h &a)
template<>
EIGEN_STRONG_INLINE Packet8h	padd< Packet8h > (const Packet8h &a, const Packet8h &b)
template<>
EIGEN_STRONG_INLINE Packet8h	psub< Packet8h > (const Packet8h &a, const Packet8h &b)
template<>
EIGEN_STRONG_INLINE Packet8h	pmul< Packet8h > (const Packet8h &a, const Packet8h &b)
template<>
EIGEN_STRONG_INLINE Packet8h	pdiv< Packet8h > (const Packet8h &a, const Packet8h &b)
template<>
EIGEN_STRONG_INLINE Packet8h	pgather< Eigen::half, Packet8h > (const Eigen::half *from, Index stride)
template<>
EIGEN_STRONG_INLINE void	pscatter< Eigen::half, Packet8h > (Eigen::half *to, const Packet8h &from, Index stride)
template<>
EIGEN_STRONG_INLINE Eigen::half	predux< Packet8h > (const Packet8h &a)
template<>
EIGEN_STRONG_INLINE Eigen::half	predux_max< Packet8h > (const Packet8h &a)
template<>
EIGEN_STRONG_INLINE Eigen::half	predux_min< Packet8h > (const Packet8h &a)
template<>
EIGEN_STRONG_INLINE Eigen::half	predux_mul< Packet8h > (const Packet8h &a)
template<>
EIGEN_STRONG_INLINE Packet8h	preverse (const Packet8h &a)
EIGEN_STRONG_INLINE void	ptranspose (PacketBlock< Packet8h, 8 > &kernel)
EIGEN_STRONG_INLINE void	ptranspose (PacketBlock< Packet8h, 4 > &kernel)
EIGEN_STRONG_INLINE Packet8f	Bf16ToF32 (const Packet8bf &a)
EIGEN_STRONG_INLINE Packet8bf	F32ToBf16 (const Packet8f &a)
template<>
EIGEN_STRONG_INLINE bfloat16	pfirst< Packet8bf > (const Packet8bf &from)
template<>
EIGEN_STRONG_INLINE Packet8bf	ptrue (const Packet8bf &a)
template<>
EIGEN_STRONG_INLINE Packet8bf	por (const Packet8bf &a, const Packet8bf &b)
template<>
EIGEN_STRONG_INLINE Packet8bf	pxor (const Packet8bf &a, const Packet8bf &b)
template<>
EIGEN_STRONG_INLINE Packet8bf	pand (const Packet8bf &a, const Packet8bf &b)
template<>
EIGEN_STRONG_INLINE Packet8bf	pandnot (const Packet8bf &a, const Packet8bf &b)
template<>
EIGEN_STRONG_INLINE Packet8bf	pselect (const Packet8bf &mask, const Packet8bf &a, const Packet8bf &b)
template<>
EIGEN_STRONG_INLINE Packet8bf	print< Packet8bf > (const Packet8bf &a)
template<>
EIGEN_STRONG_INLINE Packet8bf	pconj (const Packet8bf &a)
template<>
EIGEN_STRONG_INLINE Packet8bf	pnegate (const Packet8bf &a)
template<>
EIGEN_STRONG_INLINE Packet16b	pcast< Packet8f, Packet16b > (const Packet8f &a, const Packet8f &b)
template<>
EIGEN_STRONG_INLINE Packet8f	pcast< Packet16b, Packet8f > (const Packet16b &a)
template<>
EIGEN_STRONG_INLINE Packet8i	pcast< Packet8f, Packet8i > (const Packet8f &a)
template<>
EIGEN_STRONG_INLINE Packet8i	pcast< Packet4d, Packet8i > (const Packet4d &a, const Packet4d &b)
template<>
EIGEN_STRONG_INLINE Packet4i	pcast< Packet4d, Packet4i > (const Packet4d &a)
template<>
EIGEN_STRONG_INLINE Packet8f	pcast< Packet8i, Packet8f > (const Packet8i &a)
template<>
EIGEN_STRONG_INLINE Packet8f	pcast< Packet4d, Packet8f > (const Packet4d &a, const Packet4d &b)
template<>
EIGEN_STRONG_INLINE Packet4f	pcast< Packet4d, Packet4f > (const Packet4d &a)
template<>
EIGEN_STRONG_INLINE Packet4d	pcast< Packet8i, Packet4d > (const Packet8i &a)
template<>
EIGEN_STRONG_INLINE Packet4d	pcast< Packet4i, Packet4d > (const Packet4i &a)
template<>
EIGEN_STRONG_INLINE Packet4d	pcast< Packet8f, Packet4d > (const Packet8f &a)
template<>
EIGEN_STRONG_INLINE Packet4d	pcast< Packet4f, Packet4d > (const Packet4f &a)
template<>
EIGEN_STRONG_INLINE Packet8i	preinterpret< Packet8i, Packet8f > (const Packet8f &a)
template<>
EIGEN_STRONG_INLINE Packet8f	preinterpret< Packet8f, Packet8i > (const Packet8i &a)
template<>
EIGEN_STRONG_INLINE Packet8ui	preinterpret< Packet8ui, Packet8i > (const Packet8i &a)
template<>
EIGEN_STRONG_INLINE Packet8i	preinterpret< Packet8i, Packet8ui > (const Packet8ui &a)
template<>
EIGEN_STRONG_INLINE Packet4f	preinterpret< Packet4f, Packet8f > (const Packet8f &a)
template<>
EIGEN_STRONG_INLINE Packet2d	preinterpret< Packet2d, Packet4d > (const Packet4d &a)
template<>
EIGEN_STRONG_INLINE Packet4i	preinterpret< Packet4i, Packet8i > (const Packet8i &a)
template<>
EIGEN_STRONG_INLINE Packet4ui	preinterpret< Packet4ui, Packet8ui > (const Packet8ui &a)
template<>
EIGEN_STRONG_INLINE Packet8f	pcast< Packet8h, Packet8f > (const Packet8h &a)
template<>
EIGEN_STRONG_INLINE Packet8f	pcast< Packet8bf, Packet8f > (const Packet8bf &a)
template<>
EIGEN_STRONG_INLINE Packet8h	pcast< Packet8f, Packet8h > (const Packet8f &a)
template<>
EIGEN_STRONG_INLINE Packet8bf	pcast< Packet8f, Packet8bf > (const Packet8f &a)
template<>
EIGEN_STRONG_INLINE Packet8cf	ptrue< Packet8cf > (const Packet8cf &a)
template<>
EIGEN_STRONG_INLINE Packet8cf	padd< Packet8cf > (const Packet8cf &a, const Packet8cf &b)
template<>
EIGEN_STRONG_INLINE Packet8cf	psub< Packet8cf > (const Packet8cf &a, const Packet8cf &b)
template<>
EIGEN_STRONG_INLINE Packet8cf	pnegate (const Packet8cf &a)
template<>
EIGEN_STRONG_INLINE Packet8cf	pconj (const Packet8cf &a)
template<>
EIGEN_STRONG_INLINE Packet8cf	pmul< Packet8cf > (const Packet8cf &a, const Packet8cf &b)
template<>
EIGEN_STRONG_INLINE Packet8cf	pand< Packet8cf > (const Packet8cf &a, const Packet8cf &b)
template<>
EIGEN_STRONG_INLINE Packet8cf	por< Packet8cf > (const Packet8cf &a, const Packet8cf &b)
template<>
EIGEN_STRONG_INLINE Packet8cf	pxor< Packet8cf > (const Packet8cf &a, const Packet8cf &b)
template<>
EIGEN_STRONG_INLINE Packet8cf	pandnot< Packet8cf > (const Packet8cf &a, const Packet8cf &b)
template<>
EIGEN_STRONG_INLINE Packet8cf	pcmp_eq (const Packet8cf &a, const Packet8cf &b)
template<>
EIGEN_STRONG_INLINE Packet8cf	pload< Packet8cf > (const std::complex< float > *from)
template<>
EIGEN_STRONG_INLINE Packet8cf	ploadu< Packet8cf > (const std::complex< float > *from)
template<>
EIGEN_STRONG_INLINE Packet8cf	pset1< Packet8cf > (const std::complex< float > &from)
template<>
EIGEN_STRONG_INLINE Packet8cf	ploaddup< Packet8cf > (const std::complex< float > *from)
template<>
EIGEN_STRONG_INLINE Packet8cf	ploadquad< Packet8cf > (const std::complex< float > *from)
template<>
EIGEN_STRONG_INLINE void	pstore< std::complex< float > > (std::complex< float > *to, const Packet8cf &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< std::complex< float > > (std::complex< float > *to, const Packet8cf &from)
template<>
EIGEN_DEVICE_FUNC Packet8cf	pgather< std::complex< float >, Packet8cf > (const std::complex< float > *from, Index stride)
template<>
EIGEN_DEVICE_FUNC void	pscatter< std::complex< float >, Packet8cf > (std::complex< float > *to, const Packet8cf &from, Index stride)
template<>
EIGEN_STRONG_INLINE std::complex< float >	pfirst< Packet8cf > (const Packet8cf &a)
template<>
EIGEN_STRONG_INLINE Packet8cf	preverse (const Packet8cf &a)
template<>
EIGEN_STRONG_INLINE std::complex< float >	predux< Packet8cf > (const Packet8cf &a)
template<>
EIGEN_STRONG_INLINE std::complex< float >	predux_mul< Packet8cf > (const Packet8cf &a)
template<>
EIGEN_STRONG_INLINE Packet4cf	predux_half_dowto4< Packet8cf > (const Packet8cf &a)
template<>
EIGEN_STRONG_INLINE Packet8cf	pdiv< Packet8cf > (const Packet8cf &a, const Packet8cf &b)
template<>
EIGEN_STRONG_INLINE Packet8cf	pcplxflip< Packet8cf > (const Packet8cf &x)
template<>
EIGEN_STRONG_INLINE Packet4cd	padd< Packet4cd > (const Packet4cd &a, const Packet4cd &b)
template<>
EIGEN_STRONG_INLINE Packet4cd	psub< Packet4cd > (const Packet4cd &a, const Packet4cd &b)
template<>
EIGEN_STRONG_INLINE Packet4cd	pnegate (const Packet4cd &a)
template<>
EIGEN_STRONG_INLINE Packet4cd	pconj (const Packet4cd &a)
template<>
EIGEN_STRONG_INLINE Packet4cd	pmul< Packet4cd > (const Packet4cd &a, const Packet4cd &b)
template<>
EIGEN_STRONG_INLINE Packet4cd	ptrue< Packet4cd > (const Packet4cd &a)
template<>
EIGEN_STRONG_INLINE Packet4cd	pand< Packet4cd > (const Packet4cd &a, const Packet4cd &b)
template<>
EIGEN_STRONG_INLINE Packet4cd	por< Packet4cd > (const Packet4cd &a, const Packet4cd &b)
template<>
EIGEN_STRONG_INLINE Packet4cd	pxor< Packet4cd > (const Packet4cd &a, const Packet4cd &b)
template<>
EIGEN_STRONG_INLINE Packet4cd	pandnot< Packet4cd > (const Packet4cd &a, const Packet4cd &b)
template<>
EIGEN_STRONG_INLINE Packet4cd	pcmp_eq (const Packet4cd &a, const Packet4cd &b)
template<>
EIGEN_STRONG_INLINE Packet4cd	pload< Packet4cd > (const std::complex< double > *from)
template<>
EIGEN_STRONG_INLINE Packet4cd	ploadu< Packet4cd > (const std::complex< double > *from)
template<>
EIGEN_STRONG_INLINE Packet4cd	pset1< Packet4cd > (const std::complex< double > &from)
template<>
EIGEN_STRONG_INLINE Packet4cd	ploaddup< Packet4cd > (const std::complex< double > *from)
template<>
EIGEN_STRONG_INLINE void	pstore< std::complex< double > > (std::complex< double > *to, const Packet4cd &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< std::complex< double > > (std::complex< double > *to, const Packet4cd &from)
template<>
EIGEN_DEVICE_FUNC Packet4cd	pgather< std::complex< double >, Packet4cd > (const std::complex< double > *from, Index stride)
template<>
EIGEN_DEVICE_FUNC void	pscatter< std::complex< double >, Packet4cd > (std::complex< double > *to, const Packet4cd &from, Index stride)
template<>
EIGEN_STRONG_INLINE std::complex< double >	pfirst< Packet4cd > (const Packet4cd &a)
template<>
EIGEN_STRONG_INLINE Packet4cd	preverse (const Packet4cd &a)
template<>
EIGEN_STRONG_INLINE std::complex< double >	predux< Packet4cd > (const Packet4cd &a)
template<>
EIGEN_STRONG_INLINE std::complex< double >	predux_mul< Packet4cd > (const Packet4cd &a)
template<>
EIGEN_STRONG_INLINE Packet4cd	pdiv< Packet4cd > (const Packet4cd &a, const Packet4cd &b)
template<>
EIGEN_STRONG_INLINE Packet4cd	pcplxflip< Packet4cd > (const Packet4cd &x)
EIGEN_DEVICE_FUNC void	ptranspose (PacketBlock< Packet8cf, 4 > &kernel)
EIGEN_DEVICE_FUNC void	ptranspose (PacketBlock< Packet8cf, 8 > &kernel)
EIGEN_DEVICE_FUNC void	ptranspose (PacketBlock< Packet4cd, 4 > &kernel)
template<>
EIGEN_STRONG_INLINE Packet4cd	psqrt< Packet4cd > (const Packet4cd &a)
template<>
EIGEN_STRONG_INLINE Packet8cf	psqrt< Packet8cf > (const Packet8cf &a)
template<>
EIGEN_STRONG_INLINE Packet4cd	plog< Packet4cd > (const Packet4cd &a)
template<>
EIGEN_STRONG_INLINE Packet8cf	plog< Packet8cf > (const Packet8cf &a)
template<>
EIGEN_STRONG_INLINE Packet8cf	pexp< Packet8cf > (const Packet8cf &a)
template<typename Scalar , int max_a_unroll, int max_b_unroll, bool is_alpha1, bool is_beta0, bool is_unit_inc>
EIGEN_DONT_INLINE void	gemm_kern_avx512 (Index m, Index n, Index k, Scalar *alpha, const Scalar *a, const Scalar *b, Scalar *c, Index ldc, Index inc=1, Index a_stride=-1, Index b_stride=-1, Index a_off=0, Index b_off=0)
template<>
EIGEN_STRONG_INLINE Packet16h	pfrexp (const Packet16h &a, Packet16h &exponent)
template<>
EIGEN_STRONG_INLINE Packet16h	pldexp (const Packet16h &a, const Packet16h &exponent)
template<>
EIGEN_STRONG_INLINE Packet16bf	pfrexp (const Packet16bf &a, Packet16bf &exponent)
template<>
EIGEN_STRONG_INLINE Packet16bf	pldexp (const Packet16bf &a, const Packet16bf &exponent)
template<>
EIGEN_STRONG_INLINE Packet16f	psqrt< Packet16f > (const Packet16f &x)
template<>
EIGEN_STRONG_INLINE Packet8d	psqrt< Packet8d > (const Packet8d &x)
template<>
EIGEN_STRONG_INLINE Packet16f	pset1< Packet16f > (const float &from)
template<>
EIGEN_STRONG_INLINE Packet8d	pset1< Packet8d > (const double &from)
template<>
EIGEN_STRONG_INLINE Packet16i	pset1< Packet16i > (const int &from)
template<>
EIGEN_STRONG_INLINE Packet8l	pset1< Packet8l > (const int64_t &from)
template<>
EIGEN_STRONG_INLINE Packet16f	pset1frombits< Packet16f > (unsigned int from)
template<>
EIGEN_STRONG_INLINE Packet8d	pset1frombits< Packet8d > (const numext::uint64_t from)
template<>
EIGEN_STRONG_INLINE Packet16f	pzero (const Packet16f &)
template<>
EIGEN_STRONG_INLINE Packet8d	pzero (const Packet8d &)
template<>
EIGEN_STRONG_INLINE Packet16i	pzero (const Packet16i &)
template<>
EIGEN_STRONG_INLINE Packet8l	pzero (const Packet8l &)
template<>
EIGEN_STRONG_INLINE Packet16f	peven_mask (const Packet16f &)
template<>
EIGEN_STRONG_INLINE Packet16i	peven_mask (const Packet16i &)
template<>
EIGEN_STRONG_INLINE Packet8d	peven_mask (const Packet8d &)
template<>
EIGEN_STRONG_INLINE Packet8l	peven_mask (const Packet8l &)
template<>
EIGEN_STRONG_INLINE Packet16f	pload1< Packet16f > (const float *from)
template<>
EIGEN_STRONG_INLINE Packet8d	pload1< Packet8d > (const double *from)
template<>
EIGEN_STRONG_INLINE Packet16f	plset< Packet16f > (const float &a)
template<>
EIGEN_STRONG_INLINE Packet8d	plset< Packet8d > (const double &a)
template<>
EIGEN_STRONG_INLINE Packet16i	plset< Packet16i > (const int &a)
template<>
EIGEN_STRONG_INLINE Packet8l	plset< Packet8l > (const int64_t &a)
template<>
EIGEN_STRONG_INLINE Packet16f	padd< Packet16f > (const Packet16f &a, const Packet16f &b)
template<>
EIGEN_STRONG_INLINE Packet8d	padd< Packet8d > (const Packet8d &a, const Packet8d &b)
template<>
EIGEN_STRONG_INLINE Packet16i	padd< Packet16i > (const Packet16i &a, const Packet16i &b)
template<>
EIGEN_STRONG_INLINE Packet8l	padd< Packet8l > (const Packet8l &a, const Packet8l &b)
template<>
EIGEN_STRONG_INLINE Packet16f	padd< Packet16f > (const Packet16f &a, const Packet16f &b, uint16_t umask)
template<>
EIGEN_STRONG_INLINE Packet8d	padd< Packet8d > (const Packet8d &a, const Packet8d &b, uint8_t umask)
template<>
EIGEN_STRONG_INLINE Packet16f	psub< Packet16f > (const Packet16f &a, const Packet16f &b)
template<>
EIGEN_STRONG_INLINE Packet8d	psub< Packet8d > (const Packet8d &a, const Packet8d &b)
template<>
EIGEN_STRONG_INLINE Packet16i	psub< Packet16i > (const Packet16i &a, const Packet16i &b)
template<>
EIGEN_STRONG_INLINE Packet8l	psub< Packet8l > (const Packet8l &a, const Packet8l &b)
template<>
EIGEN_STRONG_INLINE Packet16f	pnegate (const Packet16f &a)
template<>
EIGEN_STRONG_INLINE Packet8d	pnegate (const Packet8d &a)
template<>
EIGEN_STRONG_INLINE Packet16i	pnegate (const Packet16i &a)
template<>
EIGEN_STRONG_INLINE Packet8l	pnegate (const Packet8l &a)
template<>
EIGEN_STRONG_INLINE Packet16f	pconj (const Packet16f &a)
template<>
EIGEN_STRONG_INLINE Packet8d	pconj (const Packet8d &a)
template<>
EIGEN_STRONG_INLINE Packet16i	pconj (const Packet16i &a)
template<>
EIGEN_STRONG_INLINE Packet8l	pconj (const Packet8l &a)
template<>
EIGEN_STRONG_INLINE Packet16f	pmul< Packet16f > (const Packet16f &a, const Packet16f &b)
template<>
EIGEN_STRONG_INLINE Packet8d	pmul< Packet8d > (const Packet8d &a, const Packet8d &b)
template<>
EIGEN_STRONG_INLINE Packet16i	pmul< Packet16i > (const Packet16i &a, const Packet16i &b)
template<>
EIGEN_STRONG_INLINE Packet8l	pmul< Packet8l > (const Packet8l &a, const Packet8l &b)
template<>
EIGEN_STRONG_INLINE Packet16f	pdiv< Packet16f > (const Packet16f &a, const Packet16f &b)
template<>
EIGEN_STRONG_INLINE Packet8d	pdiv< Packet8d > (const Packet8d &a, const Packet8d &b)
template<>
EIGEN_STRONG_INLINE Packet16i	pdiv< Packet16i > (const Packet16i &a, const Packet16i &b)
template<>
EIGEN_DEVICE_FUNC Packet16f	pselect (const Packet16f &mask, const Packet16f &a, const Packet16f &b)
template<>
EIGEN_DEVICE_FUNC Packet16i	pselect (const Packet16i &mask, const Packet16i &a, const Packet16i &b)
template<>
EIGEN_DEVICE_FUNC Packet8l	pselect (const Packet8l &mask, const Packet8l &a, const Packet8l &b)
template<>
EIGEN_DEVICE_FUNC Packet8d	pselect (const Packet8d &mask, const Packet8d &a, const Packet8d &b)
template<>
EIGEN_STRONG_INLINE Packet16f	pmin< Packet16f > (const Packet16f &a, const Packet16f &b)
template<>
EIGEN_STRONG_INLINE Packet8d	pmin< Packet8d > (const Packet8d &a, const Packet8d &b)
template<>
EIGEN_STRONG_INLINE Packet16i	pmin< Packet16i > (const Packet16i &a, const Packet16i &b)
template<>
EIGEN_STRONG_INLINE Packet8l	pmin< Packet8l > (const Packet8l &a, const Packet8l &b)
template<>
EIGEN_STRONG_INLINE Packet16f	pmax< Packet16f > (const Packet16f &a, const Packet16f &b)
template<>
EIGEN_STRONG_INLINE Packet8d	pmax< Packet8d > (const Packet8d &a, const Packet8d &b)
template<>
EIGEN_STRONG_INLINE Packet16i	pmax< Packet16i > (const Packet16i &a, const Packet16i &b)
template<>
EIGEN_STRONG_INLINE Packet8l	pmax< Packet8l > (const Packet8l &a, const Packet8l &b)
template<>
EIGEN_STRONG_INLINE Packet16f	pmin< PropagateNumbers, Packet16f > (const Packet16f &a, const Packet16f &b)
template<>
EIGEN_STRONG_INLINE Packet8d	pmin< PropagateNumbers, Packet8d > (const Packet8d &a, const Packet8d &b)
template<>
EIGEN_STRONG_INLINE Packet16f	pmax< PropagateNumbers, Packet16f > (const Packet16f &a, const Packet16f &b)
template<>
EIGEN_STRONG_INLINE Packet8d	pmax< PropagateNumbers, Packet8d > (const Packet8d &a, const Packet8d &b)
template<>
EIGEN_STRONG_INLINE Packet16f	pmin< PropagateNaN, Packet16f > (const Packet16f &a, const Packet16f &b)
template<>
EIGEN_STRONG_INLINE Packet8d	pmin< PropagateNaN, Packet8d > (const Packet8d &a, const Packet8d &b)
template<>
EIGEN_STRONG_INLINE Packet16f	pmax< PropagateNaN, Packet16f > (const Packet16f &a, const Packet16f &b)
template<>
EIGEN_STRONG_INLINE Packet8d	pmax< PropagateNaN, Packet8d > (const Packet8d &a, const Packet8d &b)
template<int I_>
EIGEN_STRONG_INLINE Packet8f	extract256 (Packet16f x)
template<int I_>
EIGEN_STRONG_INLINE Packet2d	extract128 (Packet8d x)
EIGEN_STRONG_INLINE Packet16f	cat256 (Packet8f a, Packet8f b)
EIGEN_STRONG_INLINE Packet16i	cat256i (Packet8i a, Packet8i b)
EIGEN_STRONG_INLINE __m256i	Pack32To16 (Packet16f rf)
template<>
EIGEN_STRONG_INLINE Packet16f	pisnan (const Packet16f &a)
template<>
EIGEN_STRONG_INLINE Packet16f	pcmp_eq (const Packet16f &a, const Packet16f &b)
template<>
EIGEN_STRONG_INLINE Packet16f	pcmp_le (const Packet16f &a, const Packet16f &b)
template<>
EIGEN_STRONG_INLINE Packet16f	pcmp_lt (const Packet16f &a, const Packet16f &b)
template<>
EIGEN_STRONG_INLINE Packet16f	pcmp_lt_or_nan (const Packet16f &a, const Packet16f &b)
template<>
EIGEN_STRONG_INLINE Packet16i	pcmp_eq (const Packet16i &a, const Packet16i &b)
template<>
EIGEN_STRONG_INLINE Packet16i	pcmp_le (const Packet16i &a, const Packet16i &b)
template<>
EIGEN_STRONG_INLINE Packet16i	pcmp_lt (const Packet16i &a, const Packet16i &b)
template<>
EIGEN_STRONG_INLINE Packet8l	pcmp_eq (const Packet8l &a, const Packet8l &b)
template<>
EIGEN_STRONG_INLINE Packet8l	pcmp_le (const Packet8l &a, const Packet8l &b)
template<>
EIGEN_STRONG_INLINE Packet8l	pcmp_lt (const Packet8l &a, const Packet8l &b)
template<>
EIGEN_STRONG_INLINE Packet8d	pcmp_eq (const Packet8d &a, const Packet8d &b)
template<>
EIGEN_STRONG_INLINE Packet8d	pcmp_le (const Packet8d &a, const Packet8d &b)
template<>
EIGEN_STRONG_INLINE Packet8d	pcmp_lt (const Packet8d &a, const Packet8d &b)
template<>
EIGEN_STRONG_INLINE Packet8d	pcmp_lt_or_nan (const Packet8d &a, const Packet8d &b)
template<>
EIGEN_STRONG_INLINE Packet16f	print< Packet16f > (const Packet16f &a)
template<>
EIGEN_STRONG_INLINE Packet8d	print< Packet8d > (const Packet8d &a)
template<>
EIGEN_STRONG_INLINE Packet16f	pceil< Packet16f > (const Packet16f &a)
template<>
EIGEN_STRONG_INLINE Packet8d	pceil< Packet8d > (const Packet8d &a)
template<>
EIGEN_STRONG_INLINE Packet16f	pfloor< Packet16f > (const Packet16f &a)
template<>
EIGEN_STRONG_INLINE Packet8d	pfloor< Packet8d > (const Packet8d &a)
template<>
EIGEN_STRONG_INLINE Packet16f	ptrunc< Packet16f > (const Packet16f &a)
template<>
EIGEN_STRONG_INLINE Packet8d	ptrunc< Packet8d > (const Packet8d &a)
template<>
EIGEN_STRONG_INLINE Packet16i	ptrue< Packet16i > (const Packet16i &)
template<>
EIGEN_STRONG_INLINE Packet8l	ptrue< Packet8l > (const Packet8l &)
template<>
EIGEN_STRONG_INLINE Packet16f	ptrue< Packet16f > (const Packet16f &a)
template<>
EIGEN_STRONG_INLINE Packet8d	ptrue< Packet8d > (const Packet8d &a)
template<>
EIGEN_STRONG_INLINE Packet16i	pand< Packet16i > (const Packet16i &a, const Packet16i &b)
template<>
EIGEN_STRONG_INLINE Packet8l	pand< Packet8l > (const Packet8l &a, const Packet8l &b)
template<>
EIGEN_STRONG_INLINE Packet16f	pand< Packet16f > (const Packet16f &a, const Packet16f &b)
template<>
EIGEN_STRONG_INLINE Packet8d	pand< Packet8d > (const Packet8d &a, const Packet8d &b)
template<>
EIGEN_STRONG_INLINE Packet16i	por< Packet16i > (const Packet16i &a, const Packet16i &b)
template<>
EIGEN_STRONG_INLINE Packet8l	por< Packet8l > (const Packet8l &a, const Packet8l &b)
template<>
EIGEN_STRONG_INLINE Packet16f	por< Packet16f > (const Packet16f &a, const Packet16f &b)
template<>
EIGEN_STRONG_INLINE Packet8d	por< Packet8d > (const Packet8d &a, const Packet8d &b)
template<>
EIGEN_STRONG_INLINE Packet16i	pxor< Packet16i > (const Packet16i &a, const Packet16i &b)
template<>
EIGEN_STRONG_INLINE Packet8l	pxor< Packet8l > (const Packet8l &a, const Packet8l &b)
template<>
EIGEN_STRONG_INLINE Packet16f	pxor< Packet16f > (const Packet16f &a, const Packet16f &b)
template<>
EIGEN_STRONG_INLINE Packet8d	pxor< Packet8d > (const Packet8d &a, const Packet8d &b)
template<>
EIGEN_STRONG_INLINE Packet16i	pandnot< Packet16i > (const Packet16i &a, const Packet16i &b)
template<>
EIGEN_STRONG_INLINE Packet8l	pandnot< Packet8l > (const Packet8l &a, const Packet8l &b)
template<>
EIGEN_STRONG_INLINE Packet16f	pandnot< Packet16f > (const Packet16f &a, const Packet16f &b)
template<>
EIGEN_STRONG_INLINE Packet8d	pandnot< Packet8d > (const Packet8d &a, const Packet8d &b)
template<>
EIGEN_STRONG_INLINE Packet16f	pround< Packet16f > (const Packet16f &a)
template<>
EIGEN_STRONG_INLINE Packet8d	pround< Packet8d > (const Packet8d &a)
template<int N>
EIGEN_STRONG_INLINE Packet16i	parithmetic_shift_right (Packet16i a)
template<int N>
EIGEN_STRONG_INLINE Packet16i	plogical_shift_right (Packet16i a)
template<int N>
EIGEN_STRONG_INLINE Packet16i	plogical_shift_left (Packet16i a)
template<int N>
EIGEN_STRONG_INLINE Packet8l	parithmetic_shift_right (Packet8l a)
template<int N>
EIGEN_STRONG_INLINE Packet8l	plogical_shift_right (Packet8l a)
template<int N>
EIGEN_STRONG_INLINE Packet8l	plogical_shift_left (Packet8l a)
template<>
EIGEN_STRONG_INLINE Packet16f	pload< Packet16f > (const float *from)
template<>
EIGEN_STRONG_INLINE Packet8d	pload< Packet8d > (const double *from)
template<>
EIGEN_STRONG_INLINE Packet16i	pload< Packet16i > (const int *from)
template<>
EIGEN_STRONG_INLINE Packet8l	pload< Packet8l > (const int64_t *from)
template<>
EIGEN_STRONG_INLINE Packet16f	ploadu< Packet16f > (const float *from)
template<>
EIGEN_STRONG_INLINE Packet8d	ploadu< Packet8d > (const double *from)
template<>
EIGEN_STRONG_INLINE Packet16i	ploadu< Packet16i > (const int *from)
template<>
EIGEN_STRONG_INLINE Packet8l	ploadu< Packet8l > (const int64_t *from)
template<>
EIGEN_STRONG_INLINE Packet16f	ploadu< Packet16f > (const float *from, uint16_t umask)
template<>
EIGEN_STRONG_INLINE Packet8d	ploadu< Packet8d > (const double *from, uint8_t umask)
template<>
EIGEN_STRONG_INLINE Packet16f	ploaddup< Packet16f > (const float *from)
template<>
EIGEN_STRONG_INLINE Packet8d	ploaddup< Packet8d > (const double *from)
template<>
EIGEN_STRONG_INLINE Packet8l	ploaddup< Packet8l > (const int64_t *from)
template<>
EIGEN_STRONG_INLINE Packet16i	ploaddup< Packet16i > (const int *from)
template<>
EIGEN_STRONG_INLINE Packet16f	ploadquad< Packet16f > (const float *from)
template<>
EIGEN_STRONG_INLINE Packet8d	ploadquad< Packet8d > (const double *from)
template<>
EIGEN_STRONG_INLINE Packet8l	ploadquad< Packet8l > (const int64_t *from)
template<>
EIGEN_STRONG_INLINE Packet16i	ploadquad< Packet16i > (const int *from)
template<>
EIGEN_STRONG_INLINE void	pstore< float > (float *to, const Packet16f &from)
template<>
EIGEN_STRONG_INLINE void	pstore< double > (double *to, const Packet8d &from)
template<>
EIGEN_STRONG_INLINE void	pstore< int > (int *to, const Packet16i &from)
template<>
EIGEN_STRONG_INLINE void	pstore< int64_t > (int64_t *to, const Packet8l &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< float > (float *to, const Packet16f &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< double > (double *to, const Packet8d &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< int > (int *to, const Packet16i &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< int64_t > (int64_t *to, const Packet8l &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< float > (float *to, const Packet16f &from, uint16_t umask)
template<>
EIGEN_STRONG_INLINE void	pstoreu< double > (double *to, const Packet8d &from, uint8_t umask)
template<typename Scalar , typename Packet >
EIGEN_DEVICE_FUNC Packet	pgather (const Packet &src, const Scalar *from, Index stride, typename unpacket_traits< Packet >::mask_t umask)
template<>
EIGEN_DEVICE_FUNC Packet16f	pgather< float, Packet16f > (const Packet16f &src, const float *from, Index stride, uint16_t umask)
template<>
EIGEN_DEVICE_FUNC Packet8d	pgather< double, Packet8d > (const Packet8d &src, const double *from, Index stride, uint8_t umask)
template<>
EIGEN_DEVICE_FUNC Packet16f	pgather< float, Packet16f > (const float *from, Index stride)
template<>
EIGEN_DEVICE_FUNC Packet8d	pgather< double, Packet8d > (const double *from, Index stride)
template<>
EIGEN_DEVICE_FUNC Packet8l	pgather< int64_t, Packet8l > (const int64_t *from, Index stride)
template<>
EIGEN_DEVICE_FUNC Packet16i	pgather< int, Packet16i > (const int *from, Index stride)
template<typename Scalar , typename Packet >
EIGEN_DEVICE_FUNC void	pscatter (Scalar *to, const Packet &from, Index stride, typename unpacket_traits< Packet >::mask_t umask)
template<>
EIGEN_DEVICE_FUNC void	pscatter< float, Packet16f > (float *to, const Packet16f &from, Index stride, uint16_t umask)
template<>
EIGEN_DEVICE_FUNC void	pscatter< double, Packet8d > (double *to, const Packet8d &from, Index stride, uint8_t umask)
template<>
EIGEN_DEVICE_FUNC void	pscatter< float, Packet16f > (float *to, const Packet16f &from, Index stride)
template<>
EIGEN_DEVICE_FUNC void	pscatter< double, Packet8d > (double *to, const Packet8d &from, Index stride)
template<>
EIGEN_DEVICE_FUNC void	pscatter< int64_t, Packet8l > (int64_t *to, const Packet8l &from, Index stride)
template<>
EIGEN_DEVICE_FUNC void	pscatter< int, Packet16i > (int *to, const Packet16i &from, Index stride)
template<>
EIGEN_STRONG_INLINE void	pstore1< Packet16f > (float *to, const float &a)
template<>
EIGEN_STRONG_INLINE void	pstore1< Packet8d > (double *to, const double &a)
template<>
EIGEN_STRONG_INLINE void	pstore1< Packet16i > (int *to, const int &a)
template<>
EIGEN_STRONG_INLINE void	pstore1< Packet8l > (int64_t *to, const int64_t &a)
template<>
EIGEN_STRONG_INLINE float	pfirst< Packet16f > (const Packet16f &a)
template<>
EIGEN_STRONG_INLINE double	pfirst< Packet8d > (const Packet8d &a)
template<>
EIGEN_STRONG_INLINE int64_t	pfirst< Packet8l > (const Packet8l &a)
template<>
EIGEN_STRONG_INLINE int	pfirst< Packet16i > (const Packet16i &a)
template<>
EIGEN_STRONG_INLINE Packet16f	preverse (const Packet16f &a)
template<>
EIGEN_STRONG_INLINE Packet8d	preverse (const Packet8d &a)
template<>
EIGEN_STRONG_INLINE Packet16i	preverse (const Packet16i &a)
template<>
EIGEN_STRONG_INLINE Packet8l	preverse (const Packet8l &a)
template<>
EIGEN_STRONG_INLINE Packet16f	pabs (const Packet16f &a)
template<>
EIGEN_STRONG_INLINE Packet8d	pabs (const Packet8d &a)
template<>
EIGEN_STRONG_INLINE Packet16i	pabs (const Packet16i &a)
template<>
EIGEN_STRONG_INLINE Packet8l	pabs (const Packet8l &a)
template<>
EIGEN_STRONG_INLINE Packet16h	psignbit (const Packet16h &a)
template<>
EIGEN_STRONG_INLINE Packet16bf	psignbit (const Packet16bf &a)
template<>
EIGEN_STRONG_INLINE Packet16f	psignbit (const Packet16f &a)
template<>
EIGEN_STRONG_INLINE Packet8d	psignbit (const Packet8d &a)
template<>
EIGEN_STRONG_INLINE Packet16f	pfrexp< Packet16f > (const Packet16f &a, Packet16f &exponent)
template<>
EIGEN_STRONG_INLINE Packet8d	pfrexp_generic_get_biased_exponent (const Packet8d &a)
template<>
EIGEN_STRONG_INLINE Packet8d	pfrexp< Packet8d > (const Packet8d &a, Packet8d &exponent)
template<>
EIGEN_STRONG_INLINE Packet16f	pldexp< Packet16f > (const Packet16f &a, const Packet16f &exponent)
template<>
EIGEN_STRONG_INLINE Packet8d	pldexp< Packet8d > (const Packet8d &a, const Packet8d &exponent)
template<>
EIGEN_STRONG_INLINE float	predux< Packet16f > (const Packet16f &a)
template<>
EIGEN_STRONG_INLINE double	predux< Packet8d > (const Packet8d &a)
template<>
EIGEN_STRONG_INLINE int64_t	predux< Packet8l > (const Packet8l &a)
template<>
EIGEN_STRONG_INLINE int	predux< Packet16i > (const Packet16i &a)
template<>
EIGEN_STRONG_INLINE Packet8f	predux_half_dowto4< Packet16f > (const Packet16f &a)
template<>
EIGEN_STRONG_INLINE Packet4d	predux_half_dowto4< Packet8d > (const Packet8d &a)
template<>
EIGEN_STRONG_INLINE Packet8i	predux_half_dowto4< Packet16i > (const Packet16i &a)
template<>
EIGEN_STRONG_INLINE Packet4l	predux_half_dowto4< Packet8l > (const Packet8l &a)
template<>
EIGEN_STRONG_INLINE float	predux_mul< Packet16f > (const Packet16f &a)
template<>
EIGEN_STRONG_INLINE double	predux_mul< Packet8d > (const Packet8d &a)
template<>
EIGEN_STRONG_INLINE int	predux_mul< Packet16i > (const Packet16i &a)
template<>
EIGEN_STRONG_INLINE int64_t	predux_mul< Packet8l > (const Packet8l &a)
template<>
EIGEN_STRONG_INLINE float	predux_min< Packet16f > (const Packet16f &a)
template<>
EIGEN_STRONG_INLINE double	predux_min< Packet8d > (const Packet8d &a)
template<>
EIGEN_STRONG_INLINE int	predux_min< Packet16i > (const Packet16i &a)
template<>
EIGEN_STRONG_INLINE int64_t	predux_min< Packet8l > (const Packet8l &a)
template<>
EIGEN_STRONG_INLINE float	predux_max< Packet16f > (const Packet16f &a)
template<>
EIGEN_STRONG_INLINE double	predux_max< Packet8d > (const Packet8d &a)
template<>
EIGEN_STRONG_INLINE int	predux_max< Packet16i > (const Packet16i &a)
template<>
EIGEN_STRONG_INLINE int64_t	predux_max< Packet8l > (const Packet8l &a)
template<>
EIGEN_STRONG_INLINE bool	predux_any (const Packet16f &a)
template<>
EIGEN_STRONG_INLINE bool	predux_any (const Packet16i &a)
template<>
EIGEN_STRONG_INLINE bool	predux_any (const Packet8d &a)
template<>
EIGEN_STRONG_INLINE bool	predux_any (const Packet8l &a)
EIGEN_DEVICE_FUNC void	ptranspose (PacketBlock< Packet16f, 16 > &kernel)
EIGEN_DEVICE_FUNC void	ptranspose (PacketBlock< Packet16f, 8 > &kernel)
EIGEN_DEVICE_FUNC void	ptranspose (PacketBlock< Packet16f, 4 > &kernel)
EIGEN_DEVICE_FUNC void	ptranspose (PacketBlock< Packet8d, 4 > &kernel)
EIGEN_DEVICE_FUNC void	ptranspose (PacketBlock< Packet8d, 8 > &kernel)
EIGEN_DEVICE_FUNC void	ptranspose (PacketBlock< Packet8l, 4 > &kernel)
EIGEN_DEVICE_FUNC void	ptranspose (PacketBlock< Packet8l, 8 > &kernel)
EIGEN_DEVICE_FUNC void	ptranspose (PacketBlock< Packet16i, 16 > &kernel)
EIGEN_DEVICE_FUNC void	ptranspose (PacketBlock< Packet16i, 4 > &kernel)
template<size_t N>
EIGEN_STRONG_INLINE int	avx512_blend_mask (const Selector< N > &ifPacket)
template<>
EIGEN_STRONG_INLINE Packet16f	pblend (const Selector< 16 > &ifPacket, const Packet16f &thenPacket, const Packet16f &elsePacket)
template<>
EIGEN_STRONG_INLINE Packet8d	pblend (const Selector< 8 > &ifPacket, const Packet8d &thenPacket, const Packet8d &elsePacket)
template<>
EIGEN_STRONG_INLINE Packet16h	pset1< Packet16h > (const Eigen::half &from)
template<>
EIGEN_STRONG_INLINE Eigen::half	pfirst< Packet16h > (const Packet16h &from)
template<>
EIGEN_STRONG_INLINE Packet16h	pload< Packet16h > (const Eigen::half *from)
template<>
EIGEN_STRONG_INLINE Packet16h	ploadu< Packet16h > (const Eigen::half *from)
template<>
EIGEN_STRONG_INLINE void	pstore< half > (Eigen::half *to, const Packet16h &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< half > (Eigen::half *to, const Packet16h &from)
template<>
EIGEN_STRONG_INLINE Packet16h	ploaddup< Packet16h > (const Eigen::half *from)
template<>
EIGEN_STRONG_INLINE Packet16h	ploadquad (const Eigen::half *from)
EIGEN_STRONG_INLINE Packet16f	half2float (const Packet16h &a)
EIGEN_STRONG_INLINE Packet16h	float2half (const Packet16f &a)
template<>
EIGEN_STRONG_INLINE Packet16h	ptrue (const Packet16h &a)
template<>
EIGEN_STRONG_INLINE Packet16h	pabs (const Packet16h &a)
template<>
EIGEN_STRONG_INLINE Packet16h	pmin< Packet16h > (const Packet16h &a, const Packet16h &b)
template<>
EIGEN_STRONG_INLINE Packet16h	pmax< Packet16h > (const Packet16h &a, const Packet16h &b)
template<>
EIGEN_STRONG_INLINE Packet16h	plset< Packet16h > (const half &a)
template<>
EIGEN_STRONG_INLINE Packet16h	por (const Packet16h &a, const Packet16h &b)
template<>
EIGEN_STRONG_INLINE Packet16h	pxor (const Packet16h &a, const Packet16h &b)
template<>
EIGEN_STRONG_INLINE Packet16h	pand (const Packet16h &a, const Packet16h &b)
template<>
EIGEN_STRONG_INLINE Packet16h	pandnot (const Packet16h &a, const Packet16h &b)
template<>
EIGEN_STRONG_INLINE Packet16h	pselect (const Packet16h &mask, const Packet16h &a, const Packet16h &b)
template<>
EIGEN_STRONG_INLINE Packet16h	pround< Packet16h > (const Packet16h &a)
template<>
EIGEN_STRONG_INLINE Packet16h	print< Packet16h > (const Packet16h &a)
template<>
EIGEN_STRONG_INLINE Packet16h	pceil< Packet16h > (const Packet16h &a)
template<>
EIGEN_STRONG_INLINE Packet16h	pfloor< Packet16h > (const Packet16h &a)
template<>
EIGEN_STRONG_INLINE Packet16h	ptrunc< Packet16h > (const Packet16h &a)
template<>
EIGEN_STRONG_INLINE Packet16h	pcmp_eq (const Packet16h &a, const Packet16h &b)
template<>
EIGEN_STRONG_INLINE Packet16h	pcmp_le (const Packet16h &a, const Packet16h &b)
template<>
EIGEN_STRONG_INLINE Packet16h	pcmp_lt (const Packet16h &a, const Packet16h &b)
template<>
EIGEN_STRONG_INLINE Packet16h	pcmp_lt_or_nan (const Packet16h &a, const Packet16h &b)
template<>
EIGEN_STRONG_INLINE Packet16h	pconj (const Packet16h &a)
template<>
EIGEN_STRONG_INLINE Packet16h	pnegate (const Packet16h &a)
template<>
EIGEN_STRONG_INLINE Packet16h	padd< Packet16h > (const Packet16h &a, const Packet16h &b)
template<>
EIGEN_STRONG_INLINE Packet16h	psub< Packet16h > (const Packet16h &a, const Packet16h &b)
template<>
EIGEN_STRONG_INLINE Packet16h	pmul< Packet16h > (const Packet16h &a, const Packet16h &b)
template<>
EIGEN_STRONG_INLINE Packet16h	pdiv< Packet16h > (const Packet16h &a, const Packet16h &b)
template<>
EIGEN_STRONG_INLINE half	predux< Packet16h > (const Packet16h &from)
template<>
EIGEN_STRONG_INLINE Packet8h	predux_half_dowto4< Packet16h > (const Packet16h &a)
template<>
EIGEN_STRONG_INLINE Eigen::half	predux_max< Packet16h > (const Packet16h &a)
template<>
EIGEN_STRONG_INLINE Eigen::half	predux_min< Packet16h > (const Packet16h &a)
template<>
EIGEN_STRONG_INLINE half	predux_mul< Packet16h > (const Packet16h &from)
template<>
EIGEN_STRONG_INLINE Packet16h	preverse (const Packet16h &a)
template<>
EIGEN_STRONG_INLINE Packet16h	pgather< Eigen::half, Packet16h > (const Eigen::half *from, Index stride)
template<>
EIGEN_STRONG_INLINE void	pscatter< half, Packet16h > (half *to, const Packet16h &from, Index stride)
EIGEN_STRONG_INLINE void	ptranspose (PacketBlock< Packet16h, 16 > &kernel)
EIGEN_STRONG_INLINE void	ptranspose (PacketBlock< Packet16h, 8 > &kernel)
EIGEN_STRONG_INLINE void	ptranspose (PacketBlock< Packet16h, 4 > &kernel)
template<>
EIGEN_STRONG_INLINE Packet16bf	pset1< Packet16bf > (const bfloat16 &from)
template<>
EIGEN_STRONG_INLINE bfloat16	pfirst< Packet16bf > (const Packet16bf &from)
template<>
EIGEN_STRONG_INLINE Packet16bf	pload< Packet16bf > (const bfloat16 *from)
template<>
EIGEN_STRONG_INLINE Packet16bf	ploadu< Packet16bf > (const bfloat16 *from)
template<>
EIGEN_STRONG_INLINE void	pstore< bfloat16 > (bfloat16 *to, const Packet16bf &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< bfloat16 > (bfloat16 *to, const Packet16bf &from)
template<>
EIGEN_STRONG_INLINE Packet16bf	ploaddup< Packet16bf > (const bfloat16 *from)
template<>
EIGEN_STRONG_INLINE Packet16bf	ploadquad (const bfloat16 *from)
EIGEN_STRONG_INLINE Packet16f	Bf16ToF32 (const Packet16bf &a)
EIGEN_STRONG_INLINE Packet16bf	F32ToBf16 (const Packet16f &a)
template<>
EIGEN_STRONG_INLINE Packet16bf	ptrue (const Packet16bf &a)
template<>
EIGEN_STRONG_INLINE Packet16bf	por (const Packet16bf &a, const Packet16bf &b)
template<>
EIGEN_STRONG_INLINE Packet16bf	pxor (const Packet16bf &a, const Packet16bf &b)
template<>
EIGEN_STRONG_INLINE Packet16bf	pand (const Packet16bf &a, const Packet16bf &b)
template<>
EIGEN_STRONG_INLINE Packet16bf	pandnot (const Packet16bf &a, const Packet16bf &b)
template<>
EIGEN_STRONG_INLINE Packet16bf	pselect (const Packet16bf &mask, const Packet16bf &a, const Packet16bf &b)
template<>
EIGEN_STRONG_INLINE Packet16bf	pround< Packet16bf > (const Packet16bf &a)
template<>
EIGEN_STRONG_INLINE Packet16bf	print< Packet16bf > (const Packet16bf &a)
template<>
EIGEN_STRONG_INLINE Packet16bf	pceil< Packet16bf > (const Packet16bf &a)
template<>
EIGEN_STRONG_INLINE Packet16bf	pfloor< Packet16bf > (const Packet16bf &a)
template<>
EIGEN_STRONG_INLINE Packet16bf	ptrunc< Packet16bf > (const Packet16bf &a)
template<>
EIGEN_STRONG_INLINE Packet16bf	pcmp_eq (const Packet16bf &a, const Packet16bf &b)
template<>
EIGEN_STRONG_INLINE Packet16bf	pcmp_le (const Packet16bf &a, const Packet16bf &b)
template<>
EIGEN_STRONG_INLINE Packet16bf	pcmp_lt (const Packet16bf &a, const Packet16bf &b)
template<>
EIGEN_STRONG_INLINE Packet16bf	pcmp_lt_or_nan (const Packet16bf &a, const Packet16bf &b)
template<>
EIGEN_STRONG_INLINE Packet16bf	pnegate (const Packet16bf &a)
template<>
EIGEN_STRONG_INLINE Packet16bf	pconj (const Packet16bf &a)
template<>
EIGEN_STRONG_INLINE Packet16bf	pabs (const Packet16bf &a)
template<>
EIGEN_STRONG_INLINE Packet16bf	padd< Packet16bf > (const Packet16bf &a, const Packet16bf &b)
template<>
EIGEN_STRONG_INLINE Packet16bf	psub< Packet16bf > (const Packet16bf &a, const Packet16bf &b)
template<>
EIGEN_STRONG_INLINE Packet16bf	pmul< Packet16bf > (const Packet16bf &a, const Packet16bf &b)
template<>
EIGEN_STRONG_INLINE Packet16bf	pdiv< Packet16bf > (const Packet16bf &a, const Packet16bf &b)
template<>
EIGEN_STRONG_INLINE Packet16bf	pmin< Packet16bf > (const Packet16bf &a, const Packet16bf &b)
template<>
EIGEN_STRONG_INLINE Packet16bf	pmax< Packet16bf > (const Packet16bf &a, const Packet16bf &b)
template<>
EIGEN_STRONG_INLINE Packet16bf	plset< Packet16bf > (const bfloat16 &a)
template<>
EIGEN_STRONG_INLINE Packet8bf	predux_half_dowto4< Packet16bf > (const Packet16bf &a)
template<>
EIGEN_STRONG_INLINE bfloat16	predux< Packet16bf > (const Packet16bf &p)
template<>
EIGEN_STRONG_INLINE bfloat16	predux_mul< Packet16bf > (const Packet16bf &from)
template<>
EIGEN_STRONG_INLINE bfloat16	predux_min< Packet16bf > (const Packet16bf &from)
template<>
EIGEN_STRONG_INLINE bfloat16	predux_max< Packet16bf > (const Packet16bf &from)
template<>
EIGEN_STRONG_INLINE Packet16bf	preverse (const Packet16bf &a)
template<>
EIGEN_STRONG_INLINE Packet16bf	pgather< bfloat16, Packet16bf > (const bfloat16 *from, Index stride)
template<>
EIGEN_STRONG_INLINE void	pscatter< bfloat16, Packet16bf > (bfloat16 *to, const Packet16bf &from, Index stride)
EIGEN_STRONG_INLINE void	ptranspose (PacketBlock< Packet16bf, 16 > &kernel)
EIGEN_STRONG_INLINE void	ptranspose (PacketBlock< Packet16bf, 4 > &kernel)
template<>
EIGEN_STRONG_INLINE Packet32h	pset1< Packet32h > (const Eigen::half &from)
template<>
EIGEN_STRONG_INLINE Packet32h	pzero (const Packet32h &)
template<>
EIGEN_STRONG_INLINE Packet32h	pset1frombits< Packet32h > (unsigned short from)
template<>
EIGEN_STRONG_INLINE Eigen::half	pfirst< Packet32h > (const Packet32h &from)
template<>
EIGEN_STRONG_INLINE Packet32h	pload< Packet32h > (const Eigen::half *from)
template<>
EIGEN_STRONG_INLINE Packet32h	ploadu< Packet32h > (const Eigen::half *from)
template<>
EIGEN_STRONG_INLINE void	pstore< half > (Eigen::half *to, const Packet32h &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< half > (Eigen::half *to, const Packet32h &from)
template<>
EIGEN_STRONG_INLINE Packet32h	ploaddup< Packet32h > (const Eigen::half *from)
template<>
EIGEN_STRONG_INLINE Packet32h	ploadquad< Packet32h > (const Eigen::half *from)
template<>
EIGEN_STRONG_INLINE Packet32h	pabs< Packet32h > (const Packet32h &a)
template<>
EIGEN_STRONG_INLINE Packet32h	psignbit< Packet32h > (const Packet32h &a)
template<>
EIGEN_STRONG_INLINE Packet32h	pmin< Packet32h > (const Packet32h &a, const Packet32h &b)
template<>
EIGEN_STRONG_INLINE Packet32h	pmax< Packet32h > (const Packet32h &a, const Packet32h &b)
template<>
EIGEN_STRONG_INLINE Packet32h	plset< Packet32h > (const half &a)
template<>
EIGEN_STRONG_INLINE Packet32h	por (const Packet32h &a, const Packet32h &b)
template<>
EIGEN_STRONG_INLINE Packet32h	pxor (const Packet32h &a, const Packet32h &b)
template<>
EIGEN_STRONG_INLINE Packet32h	pand (const Packet32h &a, const Packet32h &b)
template<>
EIGEN_STRONG_INLINE Packet32h	pandnot (const Packet32h &a, const Packet32h &b)
template<>
EIGEN_DEVICE_FUNC Packet32h	pselect (const Packet32h &mask, const Packet32h &a, const Packet32h &b)
template<>
EIGEN_STRONG_INLINE Packet32h	pcmp_eq (const Packet32h &a, const Packet32h &b)
template<>
EIGEN_STRONG_INLINE Packet32h	pcmp_le (const Packet32h &a, const Packet32h &b)
template<>
EIGEN_STRONG_INLINE Packet32h	pcmp_lt (const Packet32h &a, const Packet32h &b)
template<>
EIGEN_STRONG_INLINE Packet32h	pcmp_lt_or_nan (const Packet32h &a, const Packet32h &b)
template<>
EIGEN_STRONG_INLINE Packet32h	padd< Packet32h > (const Packet32h &a, const Packet32h &b)
template<>
EIGEN_STRONG_INLINE Packet32h	psub< Packet32h > (const Packet32h &a, const Packet32h &b)
template<>
EIGEN_STRONG_INLINE Packet32h	pmul< Packet32h > (const Packet32h &a, const Packet32h &b)
template<>
EIGEN_STRONG_INLINE Packet32h	pdiv< Packet32h > (const Packet32h &a, const Packet32h &b)
template<>
EIGEN_STRONG_INLINE Packet32h	pround< Packet32h > (const Packet32h &a)
template<>
EIGEN_STRONG_INLINE Packet32h	print< Packet32h > (const Packet32h &a)
template<>
EIGEN_STRONG_INLINE Packet32h	pceil< Packet32h > (const Packet32h &a)
template<>
EIGEN_STRONG_INLINE Packet32h	pfloor< Packet32h > (const Packet32h &a)
template<>
EIGEN_STRONG_INLINE Packet32h	ptrunc< Packet32h > (const Packet32h &a)
template<>
EIGEN_STRONG_INLINE half	predux< Packet32h > (const Packet32h &a)
template<>
EIGEN_STRONG_INLINE Packet16h	predux_half_dowto4< Packet32h > (const Packet32h &a)
template<>
EIGEN_STRONG_INLINE Packet32h	pnegate< Packet32h > (const Packet32h &a)
template<>
EIGEN_STRONG_INLINE Packet32h	pconj< Packet32h > (const Packet32h &a)
template<>
EIGEN_STRONG_INLINE Packet32h	psqrt< Packet32h > (const Packet32h &a)
template<>
EIGEN_STRONG_INLINE Packet32h	prsqrt< Packet32h > (const Packet32h &a)
template<>
EIGEN_STRONG_INLINE Packet32h	preciprocal< Packet32h > (const Packet32h &a)
EIGEN_DEVICE_FUNC void	ptranspose (PacketBlock< Packet32h, 32 > &a)
EIGEN_DEVICE_FUNC void	ptranspose (PacketBlock< Packet32h, 4 > &a)
template<>
EIGEN_STRONG_INLINE Packet32h	preverse (const Packet32h &a)
template<>
EIGEN_STRONG_INLINE void	pscatter< half, Packet32h > (half *to, const Packet32h &from, Index stride)
template<>
EIGEN_STRONG_INLINE Packet32h	pgather< Eigen::half, Packet32h > (const Eigen::half *from, Index stride)
template<>
EIGEN_STRONG_INLINE Packet16h	pcos< Packet16h > (const Packet16h &)
template<>
EIGEN_STRONG_INLINE Packet16h	psin< Packet16h > (const Packet16h &)
template<>
EIGEN_STRONG_INLINE Packet16h	plog< Packet16h > (const Packet16h &)
template<>
EIGEN_STRONG_INLINE Packet16h	plog2< Packet16h > (const Packet16h &)
template<>
EIGEN_STRONG_INLINE Packet16h	plog1p< Packet16h > (const Packet16h &)
template<>
EIGEN_STRONG_INLINE Packet16h	pexp< Packet16h > (const Packet16h &)
template<>
EIGEN_STRONG_INLINE Packet16h	pexpm1< Packet16h > (const Packet16h &)
template<>
EIGEN_STRONG_INLINE Packet16h	ptanh< Packet16h > (const Packet16h &)
template<>
EIGEN_STRONG_INLINE Packet16h	pfrexp< Packet16h > (const Packet16h &, Packet16h &)
template<>
EIGEN_STRONG_INLINE Packet16h	pldexp< Packet16h > (const Packet16h &, const Packet16h &)
EIGEN_STRONG_INLINE Packet32h	combine2Packet16h (const Packet16h &a, const Packet16h &b)
EIGEN_STRONG_INLINE void	extract2Packet16h (const Packet32h &x, Packet16h &a, Packet16h &b)
template<>
EIGEN_STRONG_INLINE Packet32h	psin< Packet32h > (const Packet32h &a)
template<>
EIGEN_STRONG_INLINE Packet32h	pcos< Packet32h > (const Packet32h &a)
template<>
EIGEN_STRONG_INLINE Packet32h	plog< Packet32h > (const Packet32h &a)
template<>
EIGEN_STRONG_INLINE Packet32h	plog2< Packet32h > (const Packet32h &a)
template<>
EIGEN_STRONG_INLINE Packet32h	plog1p< Packet32h > (const Packet32h &a)
template<>
EIGEN_STRONG_INLINE Packet32h	pexp< Packet32h > (const Packet32h &a)
template<>
EIGEN_STRONG_INLINE Packet32h	pexpm1< Packet32h > (const Packet32h &a)
template<>
EIGEN_STRONG_INLINE Packet32h	ptanh< Packet32h > (const Packet32h &a)
template<>
EIGEN_STRONG_INLINE Packet32h	pfrexp< Packet32h > (const Packet32h &a, Packet32h &exponent)
template<>
EIGEN_STRONG_INLINE Packet32h	pldexp< Packet32h > (const Packet32h &a, const Packet32h &exponent)
template<typename Scalar , typename vec , int64_t unrollM, int64_t unrollN, bool remM, bool remN>
EIGEN_ALWAYS_INLINE void	transStoreC (PacketBlock< vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS > &zmm, Scalar *C_arr, int64_t LDC, int64_t remM_=0, int64_t remN_=0)
template<typename Scalar , bool isARowMajor, bool isCRowMajor, bool isAdd, bool handleKRem>
void	gemmKernel (Scalar *A_arr, Scalar *B_arr, Scalar *C_arr, int64_t M, int64_t N, int64_t K, int64_t LDA, int64_t LDB, int64_t LDC)
template<typename Scalar , typename vec , int64_t unrollM, bool isARowMajor, bool isFWDSolve, bool isUnitDiag>
EIGEN_ALWAYS_INLINE void	triSolveKernel (Scalar *A_arr, Scalar *B_arr, int64_t K, int64_t LDA, int64_t LDB)
template<typename Scalar , bool isARowMajor, bool isFWDSolve, bool isUnitDiag>
void	triSolveKernelLxK (Scalar *A_arr, Scalar *B_arr, int64_t M, int64_t K, int64_t LDA, int64_t LDB)
template<typename Scalar , bool toTemp = true, bool remM = false>
EIGEN_ALWAYS_INLINE void	copyBToRowMajor (Scalar *B_arr, int64_t LDB, int64_t K, Scalar *B_temp, int64_t LDB_, int64_t remM_=0)
template<typename Scalar , bool isARowMajor = true, bool isBRowMajor = true, bool isFWDSolve = true, bool isUnitDiag = false>
void	triSolve (Scalar *A_arr, Scalar *B_arr, int64_t M, int64_t numRHS, int64_t LDA, int64_t LDB)
template<>
EIGEN_STRONG_INLINE Packet16b	pcast< Packet16f, Packet16b > (const Packet16f &a)
template<>
EIGEN_STRONG_INLINE Packet16f	pcast< Packet16b, Packet16f > (const Packet16b &a)
template<>
EIGEN_STRONG_INLINE Packet16i	pcast< Packet16f, Packet16i > (const Packet16f &a)
template<>
EIGEN_STRONG_INLINE Packet8d	pcast< Packet16f, Packet8d > (const Packet16f &a)
template<>
EIGEN_STRONG_INLINE Packet8d	pcast< Packet8f, Packet8d > (const Packet8f &a)
template<>
EIGEN_STRONG_INLINE Packet8l	pcast< Packet8d, Packet8l > (const Packet8d &a)
template<>
EIGEN_STRONG_INLINE Packet16f	pcast< Packet16i, Packet16f > (const Packet16i &a)
template<>
EIGEN_STRONG_INLINE Packet8d	pcast< Packet16i, Packet8d > (const Packet16i &a)
template<>
EIGEN_STRONG_INLINE Packet8d	pcast< Packet8i, Packet8d > (const Packet8i &a)
template<>
EIGEN_STRONG_INLINE Packet8d	pcast< Packet8l, Packet8d > (const Packet8l &a)
template<>
EIGEN_STRONG_INLINE Packet16f	pcast< Packet8d, Packet16f > (const Packet8d &a, const Packet8d &b)
template<>
EIGEN_STRONG_INLINE Packet16i	pcast< Packet8d, Packet16i > (const Packet8d &a, const Packet8d &b)
template<>
EIGEN_STRONG_INLINE Packet8i	pcast< Packet8d, Packet8i > (const Packet8d &a)
template<>
EIGEN_STRONG_INLINE Packet8f	pcast< Packet8d, Packet8f > (const Packet8d &a)
template<>
EIGEN_STRONG_INLINE Packet16i	preinterpret< Packet16i, Packet16f > (const Packet16f &a)
template<>
EIGEN_STRONG_INLINE Packet16f	preinterpret< Packet16f, Packet16i > (const Packet16i &a)
template<>
EIGEN_STRONG_INLINE Packet8d	preinterpret< Packet8d, Packet16f > (const Packet16f &a)
template<>
EIGEN_STRONG_INLINE Packet8d	preinterpret< Packet8d, Packet8l > (const Packet8l &a)
template<>
EIGEN_STRONG_INLINE Packet8l	preinterpret< Packet8l, Packet8d > (const Packet8d &a)
template<>
EIGEN_STRONG_INLINE Packet16f	preinterpret< Packet16f, Packet8d > (const Packet8d &a)
template<>
EIGEN_STRONG_INLINE Packet8f	preinterpret< Packet8f, Packet16f > (const Packet16f &a)
template<>
EIGEN_STRONG_INLINE Packet4f	preinterpret< Packet4f, Packet16f > (const Packet16f &a)
template<>
EIGEN_STRONG_INLINE Packet4d	preinterpret< Packet4d, Packet8d > (const Packet8d &a)
template<>
EIGEN_STRONG_INLINE Packet2d	preinterpret< Packet2d, Packet8d > (const Packet8d &a)
template<>
EIGEN_STRONG_INLINE Packet16f	preinterpret< Packet16f, Packet8f > (const Packet8f &a)
template<>
EIGEN_STRONG_INLINE Packet16f	preinterpret< Packet16f, Packet4f > (const Packet4f &a)
template<>
EIGEN_STRONG_INLINE Packet8d	preinterpret< Packet8d, Packet4d > (const Packet4d &a)
template<>
EIGEN_STRONG_INLINE Packet8d	preinterpret< Packet8d, Packet2d > (const Packet2d &a)
template<>
EIGEN_STRONG_INLINE Packet8i	preinterpret< Packet8i, Packet16i > (const Packet16i &a)
template<>
EIGEN_STRONG_INLINE Packet4i	preinterpret< Packet4i, Packet16i > (const Packet16i &a)
template<>
EIGEN_STRONG_INLINE Packet8h	preinterpret< Packet8h, Packet16h > (const Packet16h &a)
template<>
EIGEN_STRONG_INLINE Packet8bf	preinterpret< Packet8bf, Packet16bf > (const Packet16bf &a)
template<>
EIGEN_STRONG_INLINE Packet16f	pcast< Packet16h, Packet16f > (const Packet16h &a)
template<>
EIGEN_STRONG_INLINE Packet16h	pcast< Packet16f, Packet16h > (const Packet16f &a)
template<>
EIGEN_STRONG_INLINE Packet16f	pcast< Packet16bf, Packet16f > (const Packet16bf &a)
template<>
EIGEN_STRONG_INLINE Packet16bf	pcast< Packet16f, Packet16bf > (const Packet16f &a)
template<typename Packet >
EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Packet	pfrexp_generic_get_biased_exponent (const Packet &a)
template<typename Packet >
EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Packet	pfrexp_generic (const Packet &a, Packet &exponent)
template<typename Packet >
EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Packet	pldexp_generic (const Packet &a, const Packet &exponent)
template<typename Packet >
EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Packet	pldexp_fast (const Packet &a, const Packet &exponent)
template<typename Packet , bool base2>
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	plog_impl_float (const Packet _x)
template<typename Packet >
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	plog_float (const Packet _x)
template<typename Packet >
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	plog2_float (const Packet _x)
template<typename Packet , bool base2>
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	plog_impl_double (const Packet _x)
template<typename Packet >
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	plog_double (const Packet _x)
template<typename Packet >
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	plog2_double (const Packet _x)
template<typename Packet >
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	generic_log1p (const Packet &x)
template<typename Packet >
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	generic_expm1 (const Packet &x)
template<typename Packet >
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	pexp_float (const Packet _x)
template<typename Packet >
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	pexp_double (const Packet _x)
float	trig_reduce_huge (float xf, Eigen::numext::int32_t *quadrant)
template<bool ComputeSine, typename Packet , bool ComputeBoth = false>
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	psincos_float (const Packet &_x)
template<typename Packet >
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	psin_float (const Packet &x)
template<typename Packet >
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	pcos_float (const Packet &x)
template<typename Packet >
Packet	trig_reduce_small_double (const Packet &x, const Packet &q)
template<typename Packet >
Packet	trig_reduce_medium_double (const Packet &x, const Packet &q_high, const Packet &q_low)
template<bool ComputeSine, typename Packet , bool ComputeBoth = false>
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	psincos_double (const Packet &x)
template<typename Packet >
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	psin_double (const Packet &x)
template<typename Packet >
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	pcos_double (const Packet &x)
template<typename Packet >
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	pacos_float (const Packet &x_in)
template<typename Packet >
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	pasin_float (const Packet &x_in)
template<typename Packet >
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	generic_atan (const Packet &x_in)
template<typename T >
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS T	ptanh_float (const T &a_x)
template<typename T >
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS T	ptanh_double (const T &a_x)
template<typename Packet >
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	patanh_float (const Packet &x)
template<typename Packet >
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	patanh_double (const Packet &x)
template<typename Packet >
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	pdiv_complex (const Packet &x, const Packet &y)
template<typename Packet >
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	plog_complex (const Packet &x)
template<typename Packet >
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	pexp_complex (const Packet &a)
template<typename Packet >
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	psqrt_complex (const Packet &a)
template<typename Packet >
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	phypot_complex (const Packet &a)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	absolute_split (const Packet &x, Packet &n, Packet &r)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	fast_twosum (const Packet &x, const Packet &y, Packet &s_hi, Packet &s_lo)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	veltkamp_splitting (const Packet &x, Packet &x_hi, Packet &x_lo)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	twoprod (const Packet &x, const Packet &y, Packet &p_hi, Packet &p_lo)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet	twoprod_low (const Packet &x, const Packet &y, const Packet &xy)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	twosum (const Packet &x_hi, const Packet &x_lo, const Packet &y_hi, const Packet &y_lo, Packet &s_hi, Packet &s_lo)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	fast_twosum (const Packet &x_hi, const Packet &x_lo, const Packet &y_hi, const Packet &y_lo, Packet &s_hi, Packet &s_lo)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	fast_twosum (const Packet &x, const Packet &y_hi, const Packet &y_lo, Packet &s_hi, Packet &s_lo)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	twoprod (const Packet &x_hi, const Packet &x_lo, const Packet &y, Packet &p_hi, Packet &p_lo)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	twoprod (const Packet &x_hi, const Packet &x_lo, const Packet &y_hi, const Packet &y_lo, Packet &p_hi, Packet &p_lo)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	doubleword_div_fp (const Packet &x_hi, const Packet &x_lo, const Packet &y, Packet &z_hi, Packet &z_lo)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet	generic_pow_impl (const Packet &x, const Packet &y)
template<typename Packet >
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	generic_pow (const Packet &x, const Packet &y)
template<typename Packet >
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	generic_exp2 (const Packet &_x)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet	generic_rint (const Packet &a)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet	generic_floor (const Packet &a)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet	generic_ceil (const Packet &a)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet	generic_trunc (const Packet &a)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet	generic_round (const Packet &a)
template<typename Packet >
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	ptanh_float (const Packet &x)
template<typename Packet >
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	ptanh_double (const Packet &x)
template<>
EIGEN_STRONG_INLINE Packet2cf	pmul< Packet2cf > (const Packet2cf &a, const Packet2cf &b)
template<>
EIGEN_STRONG_INLINE Packet2cf	ptrue< Packet2cf > (const Packet2cf &a)
EIGEN_STRONG_INLINE Packet2cf	pcplxflip (const Packet2cf &x)
template<>
EIGEN_STRONG_INLINE Packet2cf	pzero (const Packet2cf &)
template<>
EIGEN_STRONG_INLINE Packet2cf	pmadd< Packet2cf > (const Packet2cf &a, const Packet2cf &b, const Packet2cf &c)
template<>
EIGEN_STRONG_INLINE Packet1cd	padd< Packet1cd > (const Packet1cd &a, const Packet1cd &b)
template<>
EIGEN_STRONG_INLINE Packet1cd	psub< Packet1cd > (const Packet1cd &a, const Packet1cd &b)
template<>
EIGEN_STRONG_INLINE Packet1cd	pnegate (const Packet1cd &a)
template<>
EIGEN_STRONG_INLINE Packet1cd	pconj (const Packet1cd &a)
template<>
EIGEN_STRONG_INLINE Packet1cd	pmul< Packet1cd > (const Packet1cd &a, const Packet1cd &b)
template<>
EIGEN_STRONG_INLINE Packet1cd	ptrue< Packet1cd > (const Packet1cd &a)
template<>
EIGEN_STRONG_INLINE Packet1cd	pand< Packet1cd > (const Packet1cd &a, const Packet1cd &b)
template<>
EIGEN_STRONG_INLINE Packet1cd	por< Packet1cd > (const Packet1cd &a, const Packet1cd &b)
template<>
EIGEN_STRONG_INLINE Packet1cd	pxor< Packet1cd > (const Packet1cd &a, const Packet1cd &b)
template<>
EIGEN_STRONG_INLINE Packet1cd	pandnot< Packet1cd > (const Packet1cd &a, const Packet1cd &b)
template<>
EIGEN_STRONG_INLINE Packet1cd	pload< Packet1cd > (const std::complex< double > *from)
template<>
EIGEN_STRONG_INLINE Packet1cd	ploadu< Packet1cd > (const std::complex< double > *from)
template<>
EIGEN_STRONG_INLINE Packet1cd	pset1< Packet1cd > (const std::complex< double > &from)
template<>
EIGEN_STRONG_INLINE Packet1cd	ploaddup< Packet1cd > (const std::complex< double > *from)
template<>
EIGEN_STRONG_INLINE void	pstore< std::complex< double > > (std::complex< double > *to, const Packet1cd &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< std::complex< double > > (std::complex< double > *to, const Packet1cd &from)
template<>
EIGEN_STRONG_INLINE void	prefetch< std::complex< double > > (const std::complex< double > *addr)
template<>
EIGEN_STRONG_INLINE std::complex< double >	pfirst< Packet1cd > (const Packet1cd &a)
template<>
EIGEN_STRONG_INLINE Packet1cd	preverse (const Packet1cd &a)
template<>
EIGEN_STRONG_INLINE std::complex< double >	predux< Packet1cd > (const Packet1cd &a)
template<>
EIGEN_STRONG_INLINE std::complex< double >	predux_mul< Packet1cd > (const Packet1cd &a)
template<>
EIGEN_STRONG_INLINE Packet1cd	pdiv< Packet1cd > (const Packet1cd &a, const Packet1cd &b)
EIGEN_STRONG_INLINE Packet1cd	pcplxflip (const Packet1cd &x)
template<>
EIGEN_STRONG_INLINE Packet1cd	pcmp_eq (const Packet1cd &a, const Packet1cd &b)
template<>
EIGEN_DEVICE_FUNC Packet2cf	pselect (const Packet2cf &mask, const Packet2cf &a, const Packet2cf &b)
template<>
EIGEN_STRONG_INLINE Packet1cd	psqrt< Packet1cd > (const Packet1cd &a)
template<>
EIGEN_STRONG_INLINE Packet1cd	plog< Packet1cd > (const Packet1cd &a)
template<>
EIGEN_STRONG_INLINE Packet1cd	pzero< Packet1cd > (const Packet1cd &)
template<>
EIGEN_STRONG_INLINE Packet1cd	pmadd< Packet1cd > (const Packet1cd &a, const Packet1cd &b, const Packet1cd &c)
template<>
EIGEN_DEVICE_FUNC Packet1cd	pgather< std::complex< double >, Packet1cd > (const std::complex< double > *from, Index)
template<>
EIGEN_DEVICE_FUNC void	pscatter< std::complex< double >, Packet1cd > (std::complex< double > *to, const Packet1cd &from, Index)
EIGEN_STRONG_INLINE void	ptranspose (PacketBlock< Packet1cd, 2 > &kernel)
EIGEN_ALWAYS_INLINE Packet4f	make_packet4f (float a, float b, float c, float d)
EIGEN_STRONG_INLINE Packet4f	shuffle1 (const Packet4f &m, int mask)
template<bool interleave>
EIGEN_STRONG_INLINE Packet4f	shuffle2 (const Packet4f &m, const Packet4f &n, int mask)
template<>
EIGEN_STRONG_INLINE Packet4f	shuffle2< true > (const Packet4f &m, const Packet4f &n, int mask)
static EIGEN_STRONG_INLINE int	eigen_lsx_shuffle_mask (int p, int q, int r, int s)
EIGEN_STRONG_INLINE Packet4f	vec4f_swizzle1 (const Packet4f &a, int p, int q, int r, int s)
EIGEN_STRONG_INLINE Packet4f	vec4f_swizzle2 (const Packet4f &a, const Packet4f &b, int p, int q, int r, int s)
EIGEN_STRONG_INLINE Packet4f	vec4f_movelh (const Packet4f &a, const Packet4f &b)
EIGEN_STRONG_INLINE Packet4f	vec4f_movehl (const Packet4f &a, const Packet4f &b)
EIGEN_STRONG_INLINE Packet4f	vec4f_unpacklo (const Packet4f &a, const Packet4f &b)
EIGEN_STRONG_INLINE Packet4f	vec4f_unpackhi (const Packet4f &a, const Packet4f &b)
EIGEN_ALWAYS_INLINE Packet2d	make_packet2d (double a, double b)
EIGEN_STRONG_INLINE Packet2d	shuffle (const Packet2d &m, const Packet2d &n, int mask)
EIGEN_STRONG_INLINE Packet2d	vec2d_swizzle2 (const Packet2d &a, const Packet2d &b, int mask)
EIGEN_STRONG_INLINE Packet2d	vec2d_unpacklo (const Packet2d &a, const Packet2d &b)
EIGEN_STRONG_INLINE Packet2d	vec2d_unpackhi (const Packet2d &a, const Packet2d &b)
template<>
EIGEN_STRONG_INLINE Packet16c	pset1< Packet16c > (const int8_t &from)
template<>
EIGEN_STRONG_INLINE Packet8s	pset1< Packet8s > (const int16_t &from)
template<>
EIGEN_STRONG_INLINE Packet4i	pset1< Packet4i > (const int32_t &from)
template<>
EIGEN_STRONG_INLINE Packet2l	pset1< Packet2l > (const int64_t &from)
template<>
EIGEN_STRONG_INLINE Packet16uc	pset1< Packet16uc > (const uint8_t &from)
template<>
EIGEN_STRONG_INLINE Packet8us	pset1< Packet8us > (const uint16_t &from)
template<>
EIGEN_STRONG_INLINE Packet4ui	pset1< Packet4ui > (const uint32_t &from)
template<>
EIGEN_STRONG_INLINE Packet2ul	pset1< Packet2ul > (const uint64_t &from)
template<>
EIGEN_STRONG_INLINE Packet2d	pset1< Packet2d > (const double &from)
template<>
EIGEN_STRONG_INLINE Packet4f	pset1frombits< Packet4f > (uint32_t from)
template<>
EIGEN_STRONG_INLINE Packet2d	pset1frombits< Packet2d > (uint64_t from)
template<>
EIGEN_STRONG_INLINE Packet16c	plset< Packet16c > (const int8_t &a)
template<>
EIGEN_STRONG_INLINE Packet8s	plset< Packet8s > (const int16_t &a)
template<>
EIGEN_STRONG_INLINE Packet4i	plset< Packet4i > (const int32_t &a)
template<>
EIGEN_STRONG_INLINE Packet2l	plset< Packet2l > (const int64_t &a)
template<>
EIGEN_STRONG_INLINE Packet16uc	plset< Packet16uc > (const uint8_t &a)
template<>
EIGEN_STRONG_INLINE Packet8us	plset< Packet8us > (const uint16_t &a)
template<>
EIGEN_STRONG_INLINE Packet4ui	plset< Packet4ui > (const uint32_t &a)
template<>
EIGEN_STRONG_INLINE Packet2ul	plset< Packet2ul > (const uint64_t &a)
template<>
EIGEN_STRONG_INLINE Packet2d	plset< Packet2d > (const double &a)
template<>
EIGEN_STRONG_INLINE Packet2l	padd< Packet2l > (const Packet2l &a, const Packet2l &b)
template<>
EIGEN_STRONG_INLINE Packet2ul	padd< Packet2ul > (const Packet2ul &a, const Packet2ul &b)
template<>
EIGEN_STRONG_INLINE Packet2d	padd< Packet2d > (const Packet2d &a, const Packet2d &b)
template<>
EIGEN_STRONG_INLINE Packet2l	psub< Packet2l > (const Packet2l &a, const Packet2l &b)
template<>
EIGEN_STRONG_INLINE Packet4ui	psub< Packet4ui > (const Packet4ui &a, const Packet4ui &b)
template<>
EIGEN_STRONG_INLINE Packet2ul	psub< Packet2ul > (const Packet2ul &a, const Packet2ul &b)
template<>
EIGEN_STRONG_INLINE Packet2d	psub< Packet2d > (const Packet2d &a, const Packet2d &b)
template<>
EIGEN_STRONG_INLINE Packet4f	paddsub< Packet4f > (const Packet4f &a, const Packet4f &b)
template<>
EIGEN_STRONG_INLINE Packet2d	pxor< Packet2d > (const Packet2d &a, const Packet2d &b)
template<>
EIGEN_STRONG_INLINE Packet2d	paddsub< Packet2d > (const Packet2d &a, const Packet2d &b)
template<>
EIGEN_STRONG_INLINE Packet2d	pnegate (const Packet2d &a)
template<>
EIGEN_STRONG_INLINE Packet2l	pnegate (const Packet2l &a)
template<>
EIGEN_STRONG_INLINE Packet2d	pconj (const Packet2d &a)
template<>
EIGEN_STRONG_INLINE Packet16c	pconj (const Packet16c &a)
template<>
EIGEN_STRONG_INLINE Packet8s	pconj (const Packet8s &a)
template<>
EIGEN_STRONG_INLINE Packet2l	pconj (const Packet2l &a)
template<>
EIGEN_STRONG_INLINE Packet16uc	pconj (const Packet16uc &a)
template<>
EIGEN_STRONG_INLINE Packet8us	pconj (const Packet8us &a)
template<>
EIGEN_STRONG_INLINE Packet4ui	pconj (const Packet4ui &a)
template<>
EIGEN_STRONG_INLINE Packet2ul	pconj (const Packet2ul &a)
template<>
EIGEN_STRONG_INLINE Packet2d	pmul< Packet2d > (const Packet2d &a, const Packet2d &b)
template<>
EIGEN_STRONG_INLINE Packet2l	pmul< Packet2l > (const Packet2l &a, const Packet2l &b)
template<>
EIGEN_STRONG_INLINE Packet4ui	pmul< Packet4ui > (const Packet4ui &a, const Packet4ui &b)
template<>
EIGEN_STRONG_INLINE Packet2ul	pmul< Packet2ul > (const Packet2ul &a, const Packet2ul &b)
template<>
EIGEN_STRONG_INLINE Packet2d	pdiv< Packet2d > (const Packet2d &a, const Packet2d &b)
template<>
EIGEN_STRONG_INLINE Packet8s	pdiv< Packet8s > (const Packet8s &a, const Packet8s &b)
template<>
EIGEN_STRONG_INLINE Packet2l	pdiv< Packet2l > (const Packet2l &a, const Packet2l &b)
template<>
EIGEN_STRONG_INLINE Packet8us	pdiv< Packet8us > (const Packet8us &a, const Packet8us &b)
template<>
EIGEN_STRONG_INLINE Packet4ui	pdiv< Packet4ui > (const Packet4ui &a, const Packet4ui &b)
template<>
EIGEN_STRONG_INLINE Packet2ul	pdiv< Packet2ul > (const Packet2ul &a, const Packet2ul &b)
template<>
EIGEN_STRONG_INLINE Packet2d	pmadd (const Packet2d &a, const Packet2d &b, const Packet2d &c)
template<>
EIGEN_STRONG_INLINE Packet4f	pmsub (const Packet4f &a, const Packet4f &b, const Packet4f &c)
template<>
EIGEN_STRONG_INLINE Packet2d	pmsub (const Packet2d &a, const Packet2d &b, const Packet2d &c)
template<>
EIGEN_STRONG_INLINE Packet4f	pnmadd (const Packet4f &a, const Packet4f &b, const Packet4f &c)
template<>
EIGEN_STRONG_INLINE Packet2d	pnmadd (const Packet2d &a, const Packet2d &b, const Packet2d &c)
template<>
EIGEN_STRONG_INLINE Packet4f	pnmsub (const Packet4f &a, const Packet4f &b, const Packet4f &c)
template<>
EIGEN_STRONG_INLINE Packet2d	pnmsub (const Packet2d &a, const Packet2d &b, const Packet2d &c)
template<>
EIGEN_STRONG_INLINE Packet16c	pmadd (const Packet16c &a, const Packet16c &b, const Packet16c &c)
template<>
EIGEN_STRONG_INLINE Packet2l	pmadd (const Packet2l &a, const Packet2l &b, const Packet2l &c)
template<>
EIGEN_STRONG_INLINE Packet16uc	pmadd (const Packet16uc &a, const Packet16uc &b, const Packet16uc &c)
template<>
EIGEN_STRONG_INLINE Packet4ui	pmadd (const Packet4ui &a, const Packet4ui &b, const Packet4ui &c)
template<>
EIGEN_STRONG_INLINE Packet2ul	pmadd (const Packet2ul &a, const Packet2ul &b, const Packet2ul &c)
template<>
EIGEN_STRONG_INLINE Packet2d	pand< Packet2d > (const Packet2d &a, const Packet2d &b)
template<>
EIGEN_STRONG_INLINE Packet16c	pand< Packet16c > (const Packet16c &a, const Packet16c &b)
template<>
EIGEN_STRONG_INLINE Packet8s	pand< Packet8s > (const Packet8s &a, const Packet8s &b)
template<>
EIGEN_STRONG_INLINE Packet2l	pand< Packet2l > (const Packet2l &a, const Packet2l &b)
template<>
EIGEN_STRONG_INLINE Packet16uc	pand< Packet16uc > (const Packet16uc &a, const Packet16uc &b)
template<>
EIGEN_STRONG_INLINE Packet2ul	pand< Packet2ul > (const Packet2ul &a, const Packet2ul &b)
template<>
EIGEN_STRONG_INLINE Packet2d	por< Packet2d > (const Packet2d &a, const Packet2d &b)
template<>
EIGEN_STRONG_INLINE Packet16c	por< Packet16c > (const Packet16c &a, const Packet16c &b)
template<>
EIGEN_STRONG_INLINE Packet2l	por< Packet2l > (const Packet2l &a, const Packet2l &b)
template<>
EIGEN_STRONG_INLINE Packet16uc	por< Packet16uc > (const Packet16uc &a, const Packet16uc &b)
template<>
EIGEN_STRONG_INLINE Packet4ui	por< Packet4ui > (const Packet4ui &a, const Packet4ui &b)
template<>
EIGEN_STRONG_INLINE Packet2ul	por< Packet2ul > (const Packet2ul &a, const Packet2ul &b)
template<>
EIGEN_STRONG_INLINE Packet16c	pxor< Packet16c > (const Packet16c &a, const Packet16c &b)
template<>
EIGEN_STRONG_INLINE Packet8s	pxor< Packet8s > (const Packet8s &a, const Packet8s &b)
template<>
EIGEN_STRONG_INLINE Packet2l	pxor< Packet2l > (const Packet2l &a, const Packet2l &b)
template<>
EIGEN_STRONG_INLINE Packet16uc	pxor< Packet16uc > (const Packet16uc &a, const Packet16uc &b)
template<>
EIGEN_STRONG_INLINE Packet4ui	pxor< Packet4ui > (const Packet4ui &a, const Packet4ui &b)
template<>
EIGEN_STRONG_INLINE Packet2ul	pxor< Packet2ul > (const Packet2ul &a, const Packet2ul &b)
template<>
EIGEN_STRONG_INLINE Packet2d	pandnot< Packet2d > (const Packet2d &a, const Packet2d &b)
template<>
EIGEN_STRONG_INLINE Packet16c	pandnot< Packet16c > (const Packet16c &a, const Packet16c &b)
template<>
EIGEN_STRONG_INLINE Packet8s	pandnot< Packet8s > (const Packet8s &a, const Packet8s &b)
template<>
EIGEN_STRONG_INLINE Packet2l	pandnot< Packet2l > (const Packet2l &a, const Packet2l &b)
template<>
EIGEN_STRONG_INLINE Packet16uc	pandnot< Packet16uc > (const Packet16uc &a, const Packet16uc &b)
template<>
EIGEN_STRONG_INLINE Packet8us	pandnot< Packet8us > (const Packet8us &a, const Packet8us &b)
template<>
EIGEN_STRONG_INLINE Packet4ui	pandnot< Packet4ui > (const Packet4ui &a, const Packet4ui &b)
template<>
EIGEN_STRONG_INLINE Packet2ul	pandnot< Packet2ul > (const Packet2ul &a, const Packet2ul &b)
template<>
EIGEN_STRONG_INLINE Packet4f	pcmp_le< Packet4f > (const Packet4f &a, const Packet4f &b)
template<>
EIGEN_STRONG_INLINE Packet2d	pcmp_le< Packet2d > (const Packet2d &a, const Packet2d &b)
template<>
EIGEN_STRONG_INLINE Packet16c	pcmp_le< Packet16c > (const Packet16c &a, const Packet16c &b)
template<>
EIGEN_STRONG_INLINE Packet8s	pcmp_le< Packet8s > (const Packet8s &a, const Packet8s &b)
template<>
EIGEN_STRONG_INLINE Packet4i	pcmp_le< Packet4i > (const Packet4i &a, const Packet4i &b)
template<>
EIGEN_STRONG_INLINE Packet2l	pcmp_le< Packet2l > (const Packet2l &a, const Packet2l &b)
template<>
EIGEN_STRONG_INLINE Packet16uc	pcmp_le< Packet16uc > (const Packet16uc &a, const Packet16uc &b)
template<>
EIGEN_STRONG_INLINE Packet8us	pcmp_le< Packet8us > (const Packet8us &a, const Packet8us &b)
template<>
EIGEN_STRONG_INLINE Packet4ui	pcmp_le< Packet4ui > (const Packet4ui &a, const Packet4ui &b)
template<>
EIGEN_STRONG_INLINE Packet2ul	pcmp_le< Packet2ul > (const Packet2ul &a, const Packet2ul &b)
template<>
EIGEN_STRONG_INLINE Packet4f	pcmp_lt< Packet4f > (const Packet4f &a, const Packet4f &b)
template<>
EIGEN_STRONG_INLINE Packet2d	pcmp_lt< Packet2d > (const Packet2d &a, const Packet2d &b)
template<>
EIGEN_STRONG_INLINE Packet16c	pcmp_lt< Packet16c > (const Packet16c &a, const Packet16c &b)
template<>
EIGEN_STRONG_INLINE Packet8s	pcmp_lt< Packet8s > (const Packet8s &a, const Packet8s &b)
template<>
EIGEN_STRONG_INLINE Packet4i	pcmp_lt< Packet4i > (const Packet4i &a, const Packet4i &b)
template<>
EIGEN_STRONG_INLINE Packet2l	pcmp_lt< Packet2l > (const Packet2l &a, const Packet2l &b)
template<>
EIGEN_STRONG_INLINE Packet16uc	pcmp_lt< Packet16uc > (const Packet16uc &a, const Packet16uc &b)
template<>
EIGEN_STRONG_INLINE Packet8us	pcmp_lt< Packet8us > (const Packet8us &a, const Packet8us &b)
template<>
EIGEN_STRONG_INLINE Packet4ui	pcmp_lt< Packet4ui > (const Packet4ui &a, const Packet4ui &b)
template<>
EIGEN_STRONG_INLINE Packet2ul	pcmp_lt< Packet2ul > (const Packet2ul &a, const Packet2ul &b)
template<>
EIGEN_STRONG_INLINE Packet4f	pcmp_lt_or_nan< Packet4f > (const Packet4f &a, const Packet4f &b)
template<>
EIGEN_STRONG_INLINE Packet2d	pcmp_lt_or_nan< Packet2d > (const Packet2d &a, const Packet2d &b)
template<>
EIGEN_STRONG_INLINE Packet4f	pcmp_eq< Packet4f > (const Packet4f &a, const Packet4f &b)
template<>
EIGEN_STRONG_INLINE Packet2d	pcmp_eq< Packet2d > (const Packet2d &a, const Packet2d &b)
template<>
EIGEN_STRONG_INLINE Packet16c	pcmp_eq< Packet16c > (const Packet16c &a, const Packet16c &b)
template<>
EIGEN_STRONG_INLINE Packet8s	pcmp_eq< Packet8s > (const Packet8s &a, const Packet8s &b)
template<>
EIGEN_STRONG_INLINE Packet4i	pcmp_eq< Packet4i > (const Packet4i &a, const Packet4i &b)
template<>
EIGEN_STRONG_INLINE Packet2l	pcmp_eq< Packet2l > (const Packet2l &a, const Packet2l &b)
template<>
EIGEN_STRONG_INLINE Packet16uc	pcmp_eq< Packet16uc > (const Packet16uc &a, const Packet16uc &b)
template<>
EIGEN_STRONG_INLINE Packet8us	pcmp_eq< Packet8us > (const Packet8us &a, const Packet8us &b)
template<>
EIGEN_STRONG_INLINE Packet4ui	pcmp_eq< Packet4ui > (const Packet4ui &a, const Packet4ui &b)
template<>
EIGEN_STRONG_INLINE Packet2ul	pcmp_eq< Packet2ul > (const Packet2ul &a, const Packet2ul &b)
template<>
EIGEN_STRONG_INLINE Packet2l	pmin< Packet2l > (const Packet2l &a, const Packet2l &b)
template<>
EIGEN_STRONG_INLINE Packet4ui	pmin< Packet4ui > (const Packet4ui &a, const Packet4ui &b)
template<>
EIGEN_STRONG_INLINE Packet2ul	pmin< Packet2ul > (const Packet2ul &a, const Packet2ul &b)
template<>
EIGEN_STRONG_INLINE Packet2l	pmax< Packet2l > (const Packet2l &a, const Packet2l &b)
template<>
EIGEN_STRONG_INLINE Packet4ui	pmax< Packet4ui > (const Packet4ui &a, const Packet4ui &b)
template<>
EIGEN_STRONG_INLINE Packet2ul	pmax< Packet2ul > (const Packet2ul &a, const Packet2ul &b)
template<>
EIGEN_STRONG_INLINE Packet2d	pmin< Packet2d > (const Packet2d &a, const Packet2d &b)
template<>
EIGEN_STRONG_INLINE Packet2d	pmax< Packet2d > (const Packet2d &a, const Packet2d &b)
template<int N>
EIGEN_STRONG_INLINE Packet16c	parithmetic_shift_right (const Packet16c &a)
template<int N>
EIGEN_STRONG_INLINE Packet8s	parithmetic_shift_right (const Packet8s &a)
template<int N>
EIGEN_STRONG_INLINE Packet2l	parithmetic_shift_right (const Packet2l &a)
template<int N>
EIGEN_STRONG_INLINE Packet16uc	parithmetic_shift_right (const Packet16uc &a)
template<int N>
EIGEN_STRONG_INLINE Packet8us	parithmetic_shift_right (const Packet8us &a)
template<int N>
EIGEN_STRONG_INLINE Packet4ui	parithmetic_shift_right (const Packet4ui &a)
template<int N>
EIGEN_STRONG_INLINE Packet2ul	parithmetic_shift_right (const Packet2ul &a)
template<int N>
EIGEN_STRONG_INLINE Packet16c	plogical_shift_right (const Packet16c &a)
template<int N>
EIGEN_STRONG_INLINE Packet8s	plogical_shift_right (const Packet8s &a)
template<int N>
EIGEN_STRONG_INLINE Packet2l	plogical_shift_right (const Packet2l &a)
template<int N>
EIGEN_STRONG_INLINE Packet16uc	plogical_shift_right (const Packet16uc &a)
template<int N>
EIGEN_STRONG_INLINE Packet2ul	plogical_shift_right (const Packet2ul &a)
template<int N>
EIGEN_STRONG_INLINE Packet16c	plogical_shift_left (const Packet16c &a)
template<int N>
EIGEN_STRONG_INLINE Packet8s	plogical_shift_left (const Packet8s &a)
template<int N>
EIGEN_STRONG_INLINE Packet2l	plogical_shift_left (const Packet2l &a)
template<int N>
EIGEN_STRONG_INLINE Packet16uc	plogical_shift_left (const Packet16uc &a)
template<int N>
EIGEN_STRONG_INLINE Packet2ul	plogical_shift_left (const Packet2ul &a)
template<>
EIGEN_STRONG_INLINE Packet2d	pabs (const Packet2d &a)
template<>
EIGEN_STRONG_INLINE Packet2l	pabs (const Packet2l &a)
template<>
EIGEN_STRONG_INLINE Packet4ui	pabs (const Packet4ui &a)
template<>
EIGEN_STRONG_INLINE Packet2ul	pabs (const Packet2ul &a)
template<>
EIGEN_STRONG_INLINE Packet2d	pload< Packet2d > (const double *from)
template<>
EIGEN_STRONG_INLINE Packet16c	pload< Packet16c > (const int8_t *from)
template<>
EIGEN_STRONG_INLINE Packet8s	pload< Packet8s > (const int16_t *from)
template<>
EIGEN_STRONG_INLINE Packet4i	pload< Packet4i > (const int32_t *from)
template<>
EIGEN_STRONG_INLINE Packet2l	pload< Packet2l > (const int64_t *from)
template<>
EIGEN_STRONG_INLINE Packet16uc	pload< Packet16uc > (const uint8_t *from)
template<>
EIGEN_STRONG_INLINE Packet8us	pload< Packet8us > (const uint16_t *from)
template<>
EIGEN_STRONG_INLINE Packet4ui	pload< Packet4ui > (const uint32_t *from)
template<>
EIGEN_STRONG_INLINE Packet2ul	pload< Packet2ul > (const uint64_t *from)
template<>
EIGEN_STRONG_INLINE Packet2d	ploadu< Packet2d > (const double *from)
template<>
EIGEN_STRONG_INLINE Packet16c	ploadu< Packet16c > (const int8_t *from)
template<>
EIGEN_STRONG_INLINE Packet8s	ploadu< Packet8s > (const int16_t *from)
template<>
EIGEN_STRONG_INLINE Packet4i	ploadu< Packet4i > (const int32_t *from)
template<>
EIGEN_STRONG_INLINE Packet2l	ploadu< Packet2l > (const int64_t *from)
template<>
EIGEN_STRONG_INLINE Packet16uc	ploadu< Packet16uc > (const uint8_t *from)
template<>
EIGEN_STRONG_INLINE Packet8us	ploadu< Packet8us > (const uint16_t *from)
template<>
EIGEN_STRONG_INLINE Packet4ui	ploadu< Packet4ui > (const uint32_t *from)
template<>
EIGEN_STRONG_INLINE Packet2ul	ploadu< Packet2ul > (const uint64_t *from)
template<>
EIGEN_STRONG_INLINE Packet2d	ploaddup< Packet2d > (const double *from)
template<>
EIGEN_STRONG_INLINE Packet16c	ploaddup< Packet16c > (const int8_t *from)
template<>
EIGEN_STRONG_INLINE Packet8s	ploaddup< Packet8s > (const int16_t *from)
template<>
EIGEN_STRONG_INLINE Packet4i	ploaddup< Packet4i > (const int32_t *from)
template<>
EIGEN_STRONG_INLINE Packet2l	ploaddup< Packet2l > (const int64_t *from)
template<>
EIGEN_STRONG_INLINE Packet16uc	ploaddup< Packet16uc > (const uint8_t *from)
template<>
EIGEN_STRONG_INLINE Packet8us	ploaddup< Packet8us > (const uint16_t *from)
template<>
EIGEN_STRONG_INLINE Packet4ui	ploaddup< Packet4ui > (const uint32_t *from)
template<>
EIGEN_STRONG_INLINE Packet2ul	ploaddup< Packet2ul > (const uint64_t *from)
template<>
EIGEN_STRONG_INLINE void	pstore< double > (double *to, const Packet2d &from)
template<>
EIGEN_STRONG_INLINE void	pstore< int8_t > (int8_t *to, const Packet16c &from)
template<>
EIGEN_STRONG_INLINE void	pstore< int16_t > (int16_t *to, const Packet8s &from)
template<>
EIGEN_STRONG_INLINE void	pstore< int32_t > (int32_t *to, const Packet4i &from)
template<>
EIGEN_STRONG_INLINE void	pstore< int64_t > (int64_t *to, const Packet2l &from)
template<>
EIGEN_STRONG_INLINE void	pstore< uint8_t > (uint8_t *to, const Packet16uc &from)
template<>
EIGEN_STRONG_INLINE void	pstore< uint16_t > (uint16_t *to, const Packet8us &from)
template<>
EIGEN_STRONG_INLINE void	pstore< uint32_t > (uint32_t *to, const Packet4ui &from)
template<>
EIGEN_STRONG_INLINE void	pstore< uint64_t > (uint64_t *to, const Packet2ul &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< double > (double *to, const Packet2d &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< int8_t > (int8_t *to, const Packet16c &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< int16_t > (int16_t *to, const Packet8s &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< int32_t > (int32_t *to, const Packet4i &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< int64_t > (int64_t *to, const Packet2l &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< uint8_t > (uint8_t *to, const Packet16uc &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< uint16_t > (uint16_t *to, const Packet8us &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< uint32_t > (uint32_t *to, const Packet4ui &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< uint64_t > (uint64_t *to, const Packet2ul &from)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2d	pgather< double, Packet2d > (const double *from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16c	pgather< int8_t, Packet16c > (const int8_t *from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8s	pgather< int16_t, Packet8s > (const int16_t *from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4i	pgather< int32_t, Packet4i > (const int32_t *from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2l	pgather< int64_t, Packet2l > (const int64_t *from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16uc	pgather< uint8_t, Packet16uc > (const uint8_t *from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8us	pgather< uint16_t, Packet8us > (const uint16_t *from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4ui	pgather< uint32_t, Packet4ui > (const uint32_t *from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ul	pgather< uint64_t, Packet2ul > (const uint64_t *from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	pscatter< double, Packet2d > (double *to, const Packet2d &from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	pscatter< int8_t, Packet16c > (int8_t *to, const Packet16c &from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	pscatter< int16_t, Packet8s > (int16_t *to, const Packet8s &from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	pscatter< int32_t, Packet4i > (int32_t *to, const Packet4i &from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	pscatter< int64_t, Packet2l > (int64_t *to, const Packet2l &from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	pscatter< uint8_t, Packet16uc > (uint8_t *to, const Packet16uc &from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	pscatter< uint16_t, Packet8us > (uint16_t *to, const Packet8us &from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	pscatter< uint32_t, Packet4ui > (uint32_t *to, const Packet4ui &from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	pscatter< uint64_t, Packet2ul > (uint64_t *to, const Packet2ul &from, Index stride)
template<>
EIGEN_STRONG_INLINE void	prefetch< int8_t > (const int8_t *addr)
template<>
EIGEN_STRONG_INLINE void	prefetch< int16_t > (const int16_t *addr)
template<>
EIGEN_STRONG_INLINE void	prefetch< int32_t > (const int32_t *addr)
template<>
EIGEN_STRONG_INLINE void	prefetch< int64_t > (const int64_t *addr)
template<>
EIGEN_STRONG_INLINE void	prefetch< uint8_t > (const uint8_t *addr)
template<>
EIGEN_STRONG_INLINE void	prefetch< uint16_t > (const uint16_t *addr)
template<>
EIGEN_STRONG_INLINE void	prefetch< uint64_t > (const uint64_t *addr)
template<>
EIGEN_STRONG_INLINE double	pfirst< Packet2d > (const Packet2d &a)
template<>
EIGEN_STRONG_INLINE int64_t	pfirst< Packet2l > (const Packet2l &a)
template<>
EIGEN_STRONG_INLINE uint32_t	pfirst< Packet4ui > (const Packet4ui &a)
template<>
EIGEN_STRONG_INLINE uint64_t	pfirst< Packet2ul > (const Packet2ul &a)
template<>
EIGEN_STRONG_INLINE Packet2d	preverse (const Packet2d &a)
template<>
EIGEN_STRONG_INLINE Packet2l	preverse (const Packet2l &a)
template<>
EIGEN_STRONG_INLINE Packet4ui	preverse (const Packet4ui &a)
template<>
EIGEN_STRONG_INLINE Packet2ul	preverse (const Packet2ul &a)
template<>
EIGEN_STRONG_INLINE double	predux< Packet2d > (const Packet2d &a)
template<>
EIGEN_STRONG_INLINE int64_t	predux< Packet2l > (const Packet2l &a)
template<>
EIGEN_STRONG_INLINE uint32_t	predux< Packet4ui > (const Packet4ui &a)
template<>
EIGEN_STRONG_INLINE uint64_t	predux< Packet2ul > (const Packet2ul &a)
template<>
EIGEN_STRONG_INLINE double	predux_mul< Packet2d > (const Packet2d &a)
template<>
EIGEN_STRONG_INLINE int64_t	predux_mul< Packet2l > (const Packet2l &a)
template<>
EIGEN_STRONG_INLINE uint32_t	predux_mul< Packet4ui > (const Packet4ui &a)
template<>
EIGEN_STRONG_INLINE uint64_t	predux_mul< Packet2ul > (const Packet2ul &a)
template<>
EIGEN_STRONG_INLINE double	predux_min< Packet2d > (const Packet2d &a)
template<>
EIGEN_STRONG_INLINE int64_t	predux_min< Packet2l > (const Packet2l &a)
template<>
EIGEN_STRONG_INLINE uint32_t	predux_min< Packet4ui > (const Packet4ui &a)
template<>
EIGEN_STRONG_INLINE uint64_t	predux_min< Packet2ul > (const Packet2ul &a)
template<>
EIGEN_STRONG_INLINE double	predux_max< Packet2d > (const Packet2d &a)
template<>
EIGEN_STRONG_INLINE int64_t	predux_max< Packet2l > (const Packet2l &a)
template<>
EIGEN_STRONG_INLINE uint32_t	predux_max< Packet4ui > (const Packet4ui &a)
template<>
EIGEN_STRONG_INLINE uint64_t	predux_max< Packet2ul > (const Packet2ul &a)
template<>
EIGEN_STRONG_INLINE Packet4f	psqrt (const Packet4f &a)
template<>
EIGEN_STRONG_INLINE Packet2d	psqrt (const Packet2d &a)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	ptranspose (PacketBlock< Packet2d, 2 > &kernel)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	ptranspose (PacketBlock< Packet16c, 8 > &kernel)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	ptranspose (PacketBlock< Packet2l, 2 > &kernel)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	ptranspose (PacketBlock< Packet16uc, 8 > &kernel)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	ptranspose (PacketBlock< Packet4ui, 4 > &kernel)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	ptranspose (PacketBlock< Packet2ul, 2 > &kernel)
template<>
EIGEN_STRONG_INLINE Packet4f	prsqrt (const Packet4f &a)
template<>
EIGEN_STRONG_INLINE Packet2d	prsqrt (const Packet2d &a)
template<>
EIGEN_STRONG_INLINE Packet4f	pfloor (const Packet4f &a)
template<>
EIGEN_STRONG_INLINE Packet2d	pfloor (const Packet2d &a)
template<>
EIGEN_STRONG_INLINE Packet4f	pceil (const Packet4f &a)
template<>
EIGEN_STRONG_INLINE Packet2d	pceil (const Packet2d &a)
template<>
EIGEN_STRONG_INLINE Packet4f	pround (const Packet4f &a)
template<>
EIGEN_STRONG_INLINE Packet2d	pround (const Packet2d &a)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16c	pselect (const Packet16c &mask, const Packet16c &a, const Packet16c &b)
template<>
EIGEN_STRONG_INLINE Packet16c	ploadquad< Packet16c > (const int8_t *from)
template<>
EIGEN_STRONG_INLINE Packet16uc	ploadquad< Packet16uc > (const uint8_t *from)
template<>
EIGEN_STRONG_INLINE Packet8s	ploadquad< Packet8s > (const int16_t *from)
template<>
EIGEN_STRONG_INLINE Packet8us	ploadquad< Packet8us > (const uint16_t *from)
template<>
EIGEN_STRONG_INLINE Packet4i	ploadquad< Packet4i > (const int32_t *from)
template<>
EIGEN_STRONG_INLINE Packet4ui	ploadquad< Packet4ui > (const uint32_t *from)
template<>
EIGEN_STRONG_INLINE Packet16c	pnmsub (const Packet16c &a, const Packet16c &b, const Packet16c &c)
template<>
EIGEN_STRONG_INLINE Packet8s	pnmsub (const Packet8s &a, const Packet8s &b, const Packet8s &c)
template<>
EIGEN_STRONG_INLINE Packet4i	pnmsub (const Packet4i &a, const Packet4i &b, const Packet4i &c)
template<>
EIGEN_STRONG_INLINE Packet2l	pnmsub (const Packet2l &a, const Packet2l &b, const Packet2l &c)
template<>
EIGEN_STRONG_INLINE Packet16c	pmsub (const Packet16c &a, const Packet16c &b, const Packet16c &c)
template<>
EIGEN_STRONG_INLINE Packet8s	pmsub (const Packet8s &a, const Packet8s &b, const Packet8s &c)
template<>
EIGEN_STRONG_INLINE Packet4i	pmsub (const Packet4i &a, const Packet4i &b, const Packet4i &c)
template<>
EIGEN_STRONG_INLINE Packet2l	pmsub (const Packet2l &a, const Packet2l &b, const Packet2l &c)
template<>
EIGEN_STRONG_INLINE Packet16c	pnmadd (const Packet16c &a, const Packet16c &b, const Packet16c &c)
template<>
EIGEN_STRONG_INLINE Packet8s	pnmadd (const Packet8s &a, const Packet8s &b, const Packet8s &c)
template<>
EIGEN_STRONG_INLINE Packet4i	pnmadd (const Packet4i &a, const Packet4i &b, const Packet4i &c)
template<>
EIGEN_STRONG_INLINE Packet2l	pnmadd (const Packet2l &a, const Packet2l &b, const Packet2l &c)
template<>
EIGEN_STRONG_INLINE Packet4f	pexp (const Packet4f &_x)
template<>
EIGEN_STRONG_INLINE Packet2d	pexp (const Packet2d &_x)
template<>
EIGEN_STRONG_INLINE Packet2d	pfrexp< Packet2d > (const Packet2d &a, Packet2d &exponent)
template<>
EIGEN_STRONG_INLINE Packet4f	pzero (const Packet4f &)
template<>
EIGEN_STRONG_INLINE Packet4f	pabsdiff< Packet4f > (const Packet4f &a, const Packet4f &b)
template<>
EIGEN_STRONG_INLINE Packet4f	pmin< PropagateNaN, Packet4f > (const Packet4f &a, const Packet4f &b)
template<>
EIGEN_STRONG_INLINE Packet4f	pmax< PropagateNaN, Packet4f > (const Packet4f &a, const Packet4f &b)
template<>
EIGEN_STRONG_INLINE Packet4f	ploadquad< Packet4f > (const float *from)
template<>
EIGEN_STRONG_INLINE Packet4f	print< Packet4f > (const Packet4f &a)
template<>
EIGEN_STRONG_INLINE Packet4f	preciprocal< Packet4f > (const Packet4f &a)
template<>
EIGEN_STRONG_INLINE Packet2d	pzero (const Packet2d &)
template<>
EIGEN_STRONG_INLINE Packet2d	pmin< PropagateNaN, Packet2d > (const Packet2d &a, const Packet2d &b)
template<>
EIGEN_STRONG_INLINE Packet2d	pmax< PropagateNaN, Packet2d > (const Packet2d &a, const Packet2d &b)
template<>
EIGEN_STRONG_INLINE Packet2d	psignbit (const Packet2d &a)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2d	pselect (const Packet2d &mask, const Packet2d &a, const Packet2d &b)
template<>
EIGEN_STRONG_INLINE Packet2d	print< Packet2d > (const Packet2d &a)
template<>
EIGEN_STRONG_INLINE Packet2d	ptrunc< Packet2d > (const Packet2d &a)
template<>
EIGEN_STRONG_INLINE Packet2d	pldexp< Packet2d > (const Packet2d &a, const Packet2d &exponent)
template<>
EIGEN_STRONG_INLINE Packet16c	pabsdiff< Packet16c > (const Packet16c &a, const Packet16c &b)
template<>
EIGEN_STRONG_INLINE Packet8s	pabsdiff< Packet8s > (const Packet8s &a, const Packet8s &b)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8s	pselect (const Packet8s &mask, const Packet8s &a, const Packet8s &b)
template<>
EIGEN_STRONG_INLINE Packet4i	pabsdiff< Packet4i > (const Packet4i &a, const Packet4i &b)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4i	pselect (const Packet4i &mask, const Packet4i &a, const Packet4i &b)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2l	pselect (const Packet2l &mask, const Packet2l &a, const Packet2l &b)
template<>
EIGEN_STRONG_INLINE Packet16uc	pdiv< Packet16uc > (const Packet16uc &a, const Packet16uc &b)
template<>
EIGEN_STRONG_INLINE Packet16uc	pabsdiff< Packet16uc > (const Packet16uc &a, const Packet16uc &b)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16uc	pselect (const Packet16uc &mask, const Packet16uc &a, const Packet16uc &b)
template<>
EIGEN_STRONG_INLINE Packet16uc	psqrt (const Packet16uc &a)
template<>
EIGEN_STRONG_INLINE Packet8us	pabsdiff< Packet8us > (const Packet8us &a, const Packet8us &b)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8us	pselect (const Packet8us &mask, const Packet8us &a, const Packet8us &b)
template<>
EIGEN_STRONG_INLINE Packet8us	psqrt (const Packet8us &a)
template<>
EIGEN_STRONG_INLINE Packet4ui	pabsdiff< Packet4ui > (const Packet4ui &a, const Packet4ui &b)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4ui	pselect (const Packet4ui &mask, const Packet4ui &a, const Packet4ui &b)
template<>
EIGEN_STRONG_INLINE Packet4ui	psqrt (const Packet4ui &a)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ul	pselect (const Packet2ul &mask, const Packet2ul &a, const Packet2ul &b)
template<>
EIGEN_STRONG_INLINE Packet4f	preinterpret< Packet4f, Packet4ui > (const Packet4ui &a)
template<>
EIGEN_STRONG_INLINE Packet2d	preinterpret< Packet2d, Packet2l > (const Packet2l &a)
template<>
EIGEN_STRONG_INLINE Packet2d	preinterpret< Packet2d, Packet2ul > (const Packet2ul &a)
template<>
EIGEN_STRONG_INLINE Packet2d	preinterpret< Packet2d, Packet4i > (const Packet4i &a)
template<>
EIGEN_STRONG_INLINE Packet16c	preinterpret< Packet16c, Packet16uc > (const Packet16uc &a)
template<>
EIGEN_STRONG_INLINE Packet8s	preinterpret< Packet8s, Packet8us > (const Packet8us &a)
template<>
EIGEN_STRONG_INLINE Packet4i	preinterpret< Packet4i, Packet4ui > (const Packet4ui &a)
template<>
EIGEN_STRONG_INLINE Packet4i	preinterpret< Packet4i, Packet2d > (const Packet2d &a)
template<>
EIGEN_STRONG_INLINE Packet2l	preinterpret< Packet2l, Packet2d > (const Packet2d &a)
template<>
EIGEN_STRONG_INLINE Packet16uc	preinterpret< Packet16uc, Packet16c > (const Packet16c &a)
template<>
EIGEN_STRONG_INLINE Packet8us	preinterpret< Packet8us, Packet8s > (const Packet8s &a)
template<>
EIGEN_STRONG_INLINE Packet4ui	preinterpret< Packet4ui, Packet4f > (const Packet4f &a)
template<>
EIGEN_STRONG_INLINE Packet4ui	preinterpret< Packet4ui, Packet4i > (const Packet4i &a)
template<>
EIGEN_STRONG_INLINE Packet2ul	preinterpret< Packet2ul, Packet2d > (const Packet2d &a)
template<>
EIGEN_STRONG_INLINE Packet2ul	preinterpret< Packet2ul, Packet2l > (const Packet2l &a)
template<>
EIGEN_STRONG_INLINE Packet2l	pcast< Packet4f, Packet2l > (const Packet4f &a)
template<>
EIGEN_STRONG_INLINE Packet2ul	pcast< Packet4f, Packet2ul > (const Packet4f &a)
template<>
EIGEN_STRONG_INLINE Packet8s	pcast< Packet4f, Packet8s > (const Packet4f &a, const Packet4f &b)
template<>
EIGEN_STRONG_INLINE Packet8us	pcast< Packet4f, Packet8us > (const Packet4f &a, const Packet4f &b)
template<>
EIGEN_STRONG_INLINE Packet16c	pcast< Packet4f, Packet16c > (const Packet4f &a, const Packet4f &b, const Packet4f &c, const Packet4f &d)
template<>
EIGEN_STRONG_INLINE Packet16uc	pcast< Packet4f, Packet16uc > (const Packet4f &a, const Packet4f &b, const Packet4f &c, const Packet4f &d)
template<>
EIGEN_STRONG_INLINE Packet4f	pcast< Packet16c, Packet4f > (const Packet16c &a)
template<>
EIGEN_STRONG_INLINE Packet2l	pcast< Packet16c, Packet2l > (const Packet16c &a)
template<>
EIGEN_STRONG_INLINE Packet2ul	pcast< Packet16c, Packet2ul > (const Packet16c &a)
template<>
EIGEN_STRONG_INLINE Packet4i	pcast< Packet16c, Packet4i > (const Packet16c &a)
template<>
EIGEN_STRONG_INLINE Packet4ui	pcast< Packet16c, Packet4ui > (const Packet16c &a)
template<>
EIGEN_STRONG_INLINE Packet8s	pcast< Packet16c, Packet8s > (const Packet16c &a)
template<>
EIGEN_STRONG_INLINE Packet8us	pcast< Packet16c, Packet8us > (const Packet16c &a)
template<>
EIGEN_STRONG_INLINE Packet4f	pcast< Packet16uc, Packet4f > (const Packet16uc &a)
template<>
EIGEN_STRONG_INLINE Packet2ul	pcast< Packet16uc, Packet2ul > (const Packet16uc &a)
template<>
EIGEN_STRONG_INLINE Packet2l	pcast< Packet16uc, Packet2l > (const Packet16uc &a)
template<>
EIGEN_STRONG_INLINE Packet4ui	pcast< Packet16uc, Packet4ui > (const Packet16uc &a)
template<>
EIGEN_STRONG_INLINE Packet4i	pcast< Packet16uc, Packet4i > (const Packet16uc &a)
template<>
EIGEN_STRONG_INLINE Packet8us	pcast< Packet16uc, Packet8us > (const Packet16uc &a)
template<>
EIGEN_STRONG_INLINE Packet8s	pcast< Packet16uc, Packet8s > (const Packet16uc &a)
template<>
EIGEN_STRONG_INLINE Packet4f	pcast< Packet8s, Packet4f > (const Packet8s &a)
template<>
EIGEN_STRONG_INLINE Packet2l	pcast< Packet8s, Packet2l > (const Packet8s &a)
template<>
EIGEN_STRONG_INLINE Packet2ul	pcast< Packet8s, Packet2ul > (const Packet8s &a)
template<>
EIGEN_STRONG_INLINE Packet4i	pcast< Packet8s, Packet4i > (const Packet8s &a)
template<>
EIGEN_STRONG_INLINE Packet4ui	pcast< Packet8s, Packet4ui > (const Packet8s &a)
template<>
EIGEN_STRONG_INLINE Packet16c	pcast< Packet8s, Packet16c > (const Packet8s &a, const Packet8s &b)
template<>
EIGEN_STRONG_INLINE Packet16uc	pcast< Packet8s, Packet16uc > (const Packet8s &a, const Packet8s &b)
template<>
EIGEN_STRONG_INLINE Packet4f	pcast< Packet8us, Packet4f > (const Packet8us &a)
template<>
EIGEN_STRONG_INLINE Packet2ul	pcast< Packet8us, Packet2ul > (const Packet8us &a)
template<>
EIGEN_STRONG_INLINE Packet2l	pcast< Packet8us, Packet2l > (const Packet8us &a)
template<>
EIGEN_STRONG_INLINE Packet4ui	pcast< Packet8us, Packet4ui > (const Packet8us &a)
template<>
EIGEN_STRONG_INLINE Packet4i	pcast< Packet8us, Packet4i > (const Packet8us &a)
template<>
EIGEN_STRONG_INLINE Packet16uc	pcast< Packet8us, Packet16uc > (const Packet8us &a, const Packet8us &b)
template<>
EIGEN_STRONG_INLINE Packet16c	pcast< Packet8us, Packet16c > (const Packet8us &a, const Packet8us &b)
template<>
EIGEN_STRONG_INLINE Packet2l	pcast< Packet4i, Packet2l > (const Packet4i &a)
template<>
EIGEN_STRONG_INLINE Packet2ul	pcast< Packet4i, Packet2ul > (const Packet4i &a)
template<>
EIGEN_STRONG_INLINE Packet8s	pcast< Packet4i, Packet8s > (const Packet4i &a, const Packet4i &b)
template<>
EIGEN_STRONG_INLINE Packet8us	pcast< Packet4i, Packet8us > (const Packet4i &a, const Packet4i &b)
template<>
EIGEN_STRONG_INLINE Packet16c	pcast< Packet4i, Packet16c > (const Packet4i &a, const Packet4i &b, const Packet4i &c, const Packet4i &d)
template<>
EIGEN_STRONG_INLINE Packet16uc	pcast< Packet4i, Packet16uc > (const Packet4i &a, const Packet4i &b, const Packet4i &c, const Packet4i &d)
template<>
EIGEN_STRONG_INLINE Packet2ul	pcast< Packet4ui, Packet2ul > (const Packet4ui &a)
template<>
EIGEN_STRONG_INLINE Packet2l	pcast< Packet4ui, Packet2l > (const Packet4ui &a)
template<>
EIGEN_STRONG_INLINE Packet8us	pcast< Packet4ui, Packet8us > (const Packet4ui &a, const Packet4ui &b)
template<>
EIGEN_STRONG_INLINE Packet8s	pcast< Packet4ui, Packet8s > (const Packet4ui &a, const Packet4ui &b)
template<>
EIGEN_STRONG_INLINE Packet16uc	pcast< Packet4ui, Packet16uc > (const Packet4ui &a, const Packet4ui &b, const Packet4ui &c, const Packet4ui &d)
template<>
EIGEN_STRONG_INLINE Packet16c	pcast< Packet4ui, Packet16c > (const Packet4ui &a, const Packet4ui &b, const Packet4ui &c, const Packet4ui &d)
template<>
EIGEN_STRONG_INLINE Packet4f	pcast< Packet2l, Packet4f > (const Packet2l &a, const Packet2l &b)
template<>
EIGEN_STRONG_INLINE Packet4i	pcast< Packet2l, Packet4i > (const Packet2l &a, const Packet2l &b)
template<>
EIGEN_STRONG_INLINE Packet4ui	pcast< Packet2l, Packet4ui > (const Packet2l &a, const Packet2l &b)
template<>
EIGEN_STRONG_INLINE Packet8s	pcast< Packet2l, Packet8s > (const Packet2l &a, const Packet2l &b, const Packet2l &c, const Packet2l &d)
template<>
EIGEN_STRONG_INLINE Packet8us	pcast< Packet2l, Packet8us > (const Packet2l &a, const Packet2l &b, const Packet2l &c, const Packet2l &d)
template<>
EIGEN_STRONG_INLINE Packet16c	pcast< Packet2l, Packet16c > (const Packet2l &a, const Packet2l &b, const Packet2l &c, const Packet2l &d, const Packet2l &e, const Packet2l &f, const Packet2l &g, const Packet2l &h)
template<>
EIGEN_STRONG_INLINE Packet16uc	pcast< Packet2l, Packet16uc > (const Packet2l &a, const Packet2l &b, const Packet2l &c, const Packet2l &d, const Packet2l &e, const Packet2l &f, const Packet2l &g, const Packet2l &h)
template<>
EIGEN_STRONG_INLINE Packet4f	pcast< Packet2ul, Packet4f > (const Packet2ul &a, const Packet2ul &b)
template<>
EIGEN_STRONG_INLINE Packet4ui	pcast< Packet2ul, Packet4ui > (const Packet2ul &a, const Packet2ul &b)
template<>
EIGEN_STRONG_INLINE Packet4i	pcast< Packet2ul, Packet4i > (const Packet2ul &a, const Packet2ul &b)
template<>
EIGEN_STRONG_INLINE Packet8us	pcast< Packet2ul, Packet8us > (const Packet2ul &a, const Packet2ul &b, const Packet2ul &c, const Packet2ul &d)
template<>
EIGEN_STRONG_INLINE Packet8s	pcast< Packet2ul, Packet8s > (const Packet2ul &a, const Packet2ul &b, const Packet2ul &c, const Packet2ul &d)
template<>
EIGEN_STRONG_INLINE Packet16uc	pcast< Packet2ul, Packet16uc > (const Packet2ul &a, const Packet2ul &b, const Packet2ul &c, const Packet2ul &d, const Packet2ul &e, const Packet2ul &f, const Packet2ul &g, const Packet2ul &h)
template<>
EIGEN_STRONG_INLINE Packet16c	pcast< Packet2ul, Packet16c > (const Packet2ul &a, const Packet2ul &b, const Packet2ul &c, const Packet2ul &d, const Packet2ul &e, const Packet2ul &f, const Packet2ul &g, const Packet2ul &h)
template<>
EIGEN_STRONG_INLINE Packet4f	pcast< Packet2d, Packet4f > (const Packet2d &a, const Packet2d &b)
template<>
EIGEN_STRONG_INLINE Packet2l	pcast< Packet2d, Packet2l > (const Packet2d &a)
template<>
EIGEN_STRONG_INLINE Packet2ul	pcast< Packet2d, Packet2ul > (const Packet2d &a)
template<>
EIGEN_STRONG_INLINE Packet4i	pcast< Packet2d, Packet4i > (const Packet2d &a, const Packet2d &b)
template<>
EIGEN_STRONG_INLINE Packet4ui	pcast< Packet2d, Packet4ui > (const Packet2d &a, const Packet2d &b)
template<>
EIGEN_STRONG_INLINE Packet8s	pcast< Packet2d, Packet8s > (const Packet2d &a, const Packet2d &b, const Packet2d &c, const Packet2d &d)
template<>
EIGEN_STRONG_INLINE Packet8us	pcast< Packet2d, Packet8us > (const Packet2d &a, const Packet2d &b, const Packet2d &c, const Packet2d &d)
template<>
EIGEN_STRONG_INLINE Packet16c	pcast< Packet2d, Packet16c > (const Packet2d &a, const Packet2d &b, const Packet2d &c, const Packet2d &d, const Packet2d &e, const Packet2d &f, const Packet2d &g, const Packet2d &h)
template<>
EIGEN_STRONG_INLINE Packet16uc	pcast< Packet2d, Packet16uc > (const Packet2d &a, const Packet2d &b, const Packet2d &c, const Packet2d &d, const Packet2d &e, const Packet2d &f, const Packet2d &g, const Packet2d &h)
template<>
EIGEN_STRONG_INLINE Packet2d	pcast< Packet4f, Packet2d > (const Packet4f &a)
template<>
EIGEN_STRONG_INLINE Packet2d	pcast< Packet16c, Packet2d > (const Packet16c &a)
template<>
EIGEN_STRONG_INLINE Packet2d	pcast< Packet16uc, Packet2d > (const Packet16uc &a)
template<>
EIGEN_STRONG_INLINE Packet2d	pcast< Packet8s, Packet2d > (const Packet8s &a)
template<>
EIGEN_STRONG_INLINE Packet2d	pcast< Packet8us, Packet2d > (const Packet8us &a)
template<>
EIGEN_STRONG_INLINE Packet2d	pcast< Packet4i, Packet2d > (const Packet4i &a)
template<>
EIGEN_STRONG_INLINE Packet2d	pcast< Packet4ui, Packet2d > (const Packet4ui &a)
template<>
EIGEN_STRONG_INLINE Packet2d	pcast< Packet2l, Packet2d > (const Packet2l &a)
template<>
EIGEN_STRONG_INLINE Packet2d	pcast< Packet2ul, Packet2d > (const Packet2ul &a)
std::ostream &	operator<< (std::ostream &os, const Packet2cf &value)
std::ostream &	operator<< (std::ostream &os, const PacketBlock< Packet2cf, 2 > &value)
template<>
EIGEN_STRONG_INLINE Packet2cf	pblend (const Selector< 2 > &ifPacket, const Packet2cf &thenPacket, const Packet2cf &elsePacket)
std::ostream &	operator<< (std::ostream &os, const Packet1cd &value)
template<>
EIGEN_DEVICE_FUNC Packet1cd	pgather< std::complex< double >, Packet1cd > (const std::complex< double > *from, Index stride __attribute__((unused)))
template<>
EIGEN_DEVICE_FUNC void	pscatter< std::complex< double >, Packet1cd > (std::complex< double > *to, const Packet1cd &from, Index stride __attribute__((unused)))
std::ostream &	operator<< (std::ostream &os, const PacketBlock< Packet1cd, 2 > &value)
template<>
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet4f	plog< Packet4f > (const Packet4f &_x)
template<>
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet4f	pexp< Packet4f > (const Packet4f &_x)
template<>
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet4f	ptanh< Packet4f > (const Packet4f &_x)
template<bool sine>
Packet4f	psincos_inner_msa_float (const Packet4f &_x)
template<>
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet4f	psin< Packet4f > (const Packet4f &x)
template<>
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet4f	pcos< Packet4f > (const Packet4f &x)
template<>
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet2d	pexp< Packet2d > (const Packet2d &_x)
template<>
EIGEN_STRONG_INLINE Packet4f	pload1< Packet4f > (const float *from)
template<>
EIGEN_STRONG_INLINE Packet4i	pload1< Packet4i > (const int32_t *from)
std::ostream &	operator<< (std::ostream &os, const PacketBlock< Packet4f, 4 > &value)
std::ostream &	operator<< (std::ostream &os, const PacketBlock< Packet4i, 4 > &value)
std::ostream &	operator<< (std::ostream &os, const Packet2d &value)
std::ostream &	operator<< (std::ostream &os, const Packet2l &value)
std::ostream &	operator<< (std::ostream &os, const Packet2ul &value)
std::ostream &	operator<< (std::ostream &os, const PacketBlock< Packet2d, 2 > &value)
template<>
EIGEN_STRONG_INLINE Packet2d	pfloor< Packet2d > (const Packet2d &a)
template<>
EIGEN_STRONG_INLINE Packet2d	pceil< Packet2d > (const Packet2d &a)
template<>
EIGEN_STRONG_INLINE Packet2d	pround< Packet2d > (const Packet2d &a)
template<>
EIGEN_STRONG_INLINE Packet2d	pblend (const Selector< 2 > &ifPacket, const Packet2d &thenPacket, const Packet2d &elsePacket)
uint32x2_t	p2ui_CONJ_XOR ()
template<>
EIGEN_STRONG_INLINE Packet1cf	pcast< float, Packet1cf > (const float &a)
template<>
EIGEN_STRONG_INLINE Packet2cf	pcast< Packet2f, Packet2cf > (const Packet2f &a)
template<>
EIGEN_STRONG_INLINE Packet1cf	pzero (const Packet1cf &)
template<>
EIGEN_STRONG_INLINE Packet1cf	pset1< Packet1cf > (const std::complex< float > &from)
template<>
EIGEN_STRONG_INLINE Packet1cf	padd< Packet1cf > (const Packet1cf &a, const Packet1cf &b)
template<>
EIGEN_STRONG_INLINE Packet1cf	psub< Packet1cf > (const Packet1cf &a, const Packet1cf &b)
template<>
EIGEN_STRONG_INLINE Packet1cf	pnegate (const Packet1cf &a)
template<>
EIGEN_STRONG_INLINE Packet1cf	pconj (const Packet1cf &a)
template<>
EIGEN_STRONG_INLINE Packet1cf	pmul< Packet1cf > (const Packet1cf &a, const Packet1cf &b)
template<>
EIGEN_STRONG_INLINE Packet1cf	pcmp_eq (const Packet1cf &a, const Packet1cf &b)
template<>
EIGEN_STRONG_INLINE Packet1cf	pand< Packet1cf > (const Packet1cf &a, const Packet1cf &b)
template<>
EIGEN_STRONG_INLINE Packet1cf	por< Packet1cf > (const Packet1cf &a, const Packet1cf &b)
template<>
EIGEN_STRONG_INLINE Packet1cf	pxor< Packet1cf > (const Packet1cf &a, const Packet1cf &b)
template<>
EIGEN_STRONG_INLINE Packet1cf	pandnot< Packet1cf > (const Packet1cf &a, const Packet1cf &b)
template<>
EIGEN_STRONG_INLINE Packet1cf	pload< Packet1cf > (const std::complex< float > *from)
template<>
EIGEN_STRONG_INLINE Packet1cf	ploadu< Packet1cf > (const std::complex< float > *from)
template<>
EIGEN_STRONG_INLINE Packet1cf	ploaddup< Packet1cf > (const std::complex< float > *from)
template<>
EIGEN_STRONG_INLINE void	pstore< std::complex< float > > (std::complex< float > *to, const Packet1cf &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< std::complex< float > > (std::complex< float > *to, const Packet1cf &from)
template<>
EIGEN_DEVICE_FUNC Packet1cf	pgather< std::complex< float >, Packet1cf > (const std::complex< float > *from, Index stride)
template<>
EIGEN_DEVICE_FUNC void	pscatter< std::complex< float >, Packet1cf > (std::complex< float > *to, const Packet1cf &from, Index stride)
template<>
EIGEN_STRONG_INLINE std::complex< float >	pfirst< Packet1cf > (const Packet1cf &a)
template<>
EIGEN_STRONG_INLINE Packet1cf	preverse (const Packet1cf &a)
template<>
EIGEN_STRONG_INLINE Packet1cf	pcplxflip< Packet1cf > (const Packet1cf &a)
template<>
EIGEN_STRONG_INLINE std::complex< float >	predux< Packet1cf > (const Packet1cf &a)
template<>
EIGEN_STRONG_INLINE std::complex< float >	predux_mul< Packet1cf > (const Packet1cf &a)
template<>
EIGEN_STRONG_INLINE Packet1cf	pdiv< Packet1cf > (const Packet1cf &a, const Packet1cf &b)
EIGEN_DEVICE_FUNC void	ptranspose (PacketBlock< Packet1cf, 1 > &)
template<>
EIGEN_STRONG_INLINE Packet1cf	psqrt< Packet1cf > (const Packet1cf &a)
template<>
EIGEN_STRONG_INLINE Packet1cf	plog< Packet1cf > (const Packet1cf &a)
template<>
EIGEN_STRONG_INLINE Packet1cf	pexp< Packet1cf > (const Packet1cf &a)
template<>
EIGEN_STRONG_INLINE Packet4bf	pfrexp (const Packet4bf &a, Packet4bf &exponent)
template<>
EIGEN_STRONG_INLINE Packet4bf	pldexp (const Packet4bf &a, const Packet4bf &exponent)
EIGEN_ALWAYS_INLINE Packet2f	make_packet2f (float a, float b)
static EIGEN_STRONG_INLINE int	eigen_neon_shuffle_mask (int p, int q, int r, int s)
template<>
EIGEN_STRONG_INLINE Packet2f	pzero (const Packet2f &)
template<>
EIGEN_STRONG_INLINE Packet2f	pset1< Packet2f > (const float &from)
template<>
EIGEN_STRONG_INLINE Packet4c	pset1< Packet4c > (const int8_t &from)
template<>
EIGEN_STRONG_INLINE Packet8c	pset1< Packet8c > (const int8_t &from)
template<>
EIGEN_STRONG_INLINE Packet4uc	pset1< Packet4uc > (const uint8_t &from)
template<>
EIGEN_STRONG_INLINE Packet8uc	pset1< Packet8uc > (const uint8_t &from)
template<>
EIGEN_STRONG_INLINE Packet4s	pset1< Packet4s > (const int16_t &from)
template<>
EIGEN_STRONG_INLINE Packet4us	pset1< Packet4us > (const uint16_t &from)
template<>
EIGEN_STRONG_INLINE Packet2i	pset1< Packet2i > (const int32_t &from)
template<>
EIGEN_STRONG_INLINE Packet2ui	pset1< Packet2ui > (const uint32_t &from)
template<>
EIGEN_STRONG_INLINE Packet2f	pset1frombits< Packet2f > (uint32_t from)
template<>
EIGEN_STRONG_INLINE Packet2f	plset< Packet2f > (const float &a)
template<>
EIGEN_STRONG_INLINE Packet4c	plset< Packet4c > (const int8_t &a)
template<>
EIGEN_STRONG_INLINE Packet8c	plset< Packet8c > (const int8_t &a)
template<>
EIGEN_STRONG_INLINE Packet4uc	plset< Packet4uc > (const uint8_t &a)
template<>
EIGEN_STRONG_INLINE Packet8uc	plset< Packet8uc > (const uint8_t &a)
template<>
EIGEN_STRONG_INLINE Packet4s	plset< Packet4s > (const int16_t &a)
template<>
EIGEN_STRONG_INLINE Packet4us	plset< Packet4us > (const uint16_t &a)
template<>
EIGEN_STRONG_INLINE Packet2i	plset< Packet2i > (const int32_t &a)
template<>
EIGEN_STRONG_INLINE Packet2ui	plset< Packet2ui > (const uint32_t &a)
template<>
EIGEN_STRONG_INLINE Packet2f	padd< Packet2f > (const Packet2f &a, const Packet2f &b)
template<>
EIGEN_STRONG_INLINE Packet4c	padd< Packet4c > (const Packet4c &a, const Packet4c &b)
template<>
EIGEN_STRONG_INLINE Packet8c	padd< Packet8c > (const Packet8c &a, const Packet8c &b)
template<>
EIGEN_STRONG_INLINE Packet4uc	padd< Packet4uc > (const Packet4uc &a, const Packet4uc &b)
template<>
EIGEN_STRONG_INLINE Packet8uc	padd< Packet8uc > (const Packet8uc &a, const Packet8uc &b)
template<>
EIGEN_STRONG_INLINE Packet4s	padd< Packet4s > (const Packet4s &a, const Packet4s &b)
template<>
EIGEN_STRONG_INLINE Packet4us	padd< Packet4us > (const Packet4us &a, const Packet4us &b)
template<>
EIGEN_STRONG_INLINE Packet2i	padd< Packet2i > (const Packet2i &a, const Packet2i &b)
template<>
EIGEN_STRONG_INLINE Packet2ui	padd< Packet2ui > (const Packet2ui &a, const Packet2ui &b)
template<>
EIGEN_STRONG_INLINE Packet2f	psub< Packet2f > (const Packet2f &a, const Packet2f &b)
template<>
EIGEN_STRONG_INLINE Packet4c	psub< Packet4c > (const Packet4c &a, const Packet4c &b)
template<>
EIGEN_STRONG_INLINE Packet8c	psub< Packet8c > (const Packet8c &a, const Packet8c &b)
template<>
EIGEN_STRONG_INLINE Packet4uc	psub< Packet4uc > (const Packet4uc &a, const Packet4uc &b)
template<>
EIGEN_STRONG_INLINE Packet8uc	psub< Packet8uc > (const Packet8uc &a, const Packet8uc &b)
template<>
EIGEN_STRONG_INLINE Packet4s	psub< Packet4s > (const Packet4s &a, const Packet4s &b)
template<>
EIGEN_STRONG_INLINE Packet4us	psub< Packet4us > (const Packet4us &a, const Packet4us &b)
template<>
EIGEN_STRONG_INLINE Packet2i	psub< Packet2i > (const Packet2i &a, const Packet2i &b)
template<>
EIGEN_STRONG_INLINE Packet2ui	psub< Packet2ui > (const Packet2ui &a, const Packet2ui &b)
template<>
EIGEN_STRONG_INLINE Packet2f	pxor< Packet2f > (const Packet2f &a, const Packet2f &b)
template<>
EIGEN_STRONG_INLINE Packet2f	paddsub< Packet2f > (const Packet2f &a, const Packet2f &b)
template<>
EIGEN_STRONG_INLINE Packet2f	pnegate (const Packet2f &a)
template<>
EIGEN_STRONG_INLINE Packet4c	pnegate (const Packet4c &a)
template<>
EIGEN_STRONG_INLINE Packet8c	pnegate (const Packet8c &a)
template<>
EIGEN_STRONG_INLINE Packet4s	pnegate (const Packet4s &a)
template<>
EIGEN_STRONG_INLINE Packet2i	pnegate (const Packet2i &a)
template<>
EIGEN_STRONG_INLINE Packet2f	pconj (const Packet2f &a)
template<>
EIGEN_STRONG_INLINE Packet4c	pconj (const Packet4c &a)
template<>
EIGEN_STRONG_INLINE Packet8c	pconj (const Packet8c &a)
template<>
EIGEN_STRONG_INLINE Packet4uc	pconj (const Packet4uc &a)
template<>
EIGEN_STRONG_INLINE Packet8uc	pconj (const Packet8uc &a)
template<>
EIGEN_STRONG_INLINE Packet4s	pconj (const Packet4s &a)
template<>
EIGEN_STRONG_INLINE Packet4us	pconj (const Packet4us &a)
template<>
EIGEN_STRONG_INLINE Packet2i	pconj (const Packet2i &a)
template<>
EIGEN_STRONG_INLINE Packet2ui	pconj (const Packet2ui &a)
template<>
EIGEN_STRONG_INLINE Packet2f	pmul< Packet2f > (const Packet2f &a, const Packet2f &b)
template<>
EIGEN_STRONG_INLINE Packet4c	pmul< Packet4c > (const Packet4c &a, const Packet4c &b)
template<>
EIGEN_STRONG_INLINE Packet8c	pmul< Packet8c > (const Packet8c &a, const Packet8c &b)
template<>
EIGEN_STRONG_INLINE Packet4uc	pmul< Packet4uc > (const Packet4uc &a, const Packet4uc &b)
template<>
EIGEN_STRONG_INLINE Packet8uc	pmul< Packet8uc > (const Packet8uc &a, const Packet8uc &b)
template<>
EIGEN_STRONG_INLINE Packet4s	pmul< Packet4s > (const Packet4s &a, const Packet4s &b)
template<>
EIGEN_STRONG_INLINE Packet4us	pmul< Packet4us > (const Packet4us &a, const Packet4us &b)
template<>
EIGEN_STRONG_INLINE Packet2i	pmul< Packet2i > (const Packet2i &a, const Packet2i &b)
template<>
EIGEN_STRONG_INLINE Packet2ui	pmul< Packet2ui > (const Packet2ui &a, const Packet2ui &b)
template<>
EIGEN_STRONG_INLINE Packet4c	pdiv< Packet4c > (const Packet4c &, const Packet4c &)
template<>
EIGEN_STRONG_INLINE Packet8c	pdiv< Packet8c > (const Packet8c &, const Packet8c &)
template<>
EIGEN_STRONG_INLINE Packet16c	pdiv< Packet16c > (const Packet16c &, const Packet16c &)
template<>
EIGEN_STRONG_INLINE Packet4uc	pdiv< Packet4uc > (const Packet4uc &, const Packet4uc &)
template<>
EIGEN_STRONG_INLINE Packet8uc	pdiv< Packet8uc > (const Packet8uc &, const Packet8uc &)
template<>
EIGEN_STRONG_INLINE Packet4s	pdiv< Packet4s > (const Packet4s &, const Packet4s &)
template<>
EIGEN_STRONG_INLINE Packet4us	pdiv< Packet4us > (const Packet4us &, const Packet4us &)
template<>
EIGEN_STRONG_INLINE Packet2i	pdiv< Packet2i > (const Packet2i &, const Packet2i &)
template<>
EIGEN_STRONG_INLINE Packet2ui	pdiv< Packet2ui > (const Packet2ui &, const Packet2ui &)
template<>
EIGEN_STRONG_INLINE Packet2f	pmadd (const Packet2f &a, const Packet2f &b, const Packet2f &c)
template<>
EIGEN_STRONG_INLINE Packet4c	pmadd (const Packet4c &a, const Packet4c &b, const Packet4c &c)
template<>
EIGEN_STRONG_INLINE Packet8c	pmadd (const Packet8c &a, const Packet8c &b, const Packet8c &c)
template<>
EIGEN_STRONG_INLINE Packet4uc	pmadd (const Packet4uc &a, const Packet4uc &b, const Packet4uc &c)
template<>
EIGEN_STRONG_INLINE Packet8uc	pmadd (const Packet8uc &a, const Packet8uc &b, const Packet8uc &c)
template<>
EIGEN_STRONG_INLINE Packet4s	pmadd (const Packet4s &a, const Packet4s &b, const Packet4s &c)
template<>
EIGEN_STRONG_INLINE Packet4us	pmadd (const Packet4us &a, const Packet4us &b, const Packet4us &c)
template<>
EIGEN_STRONG_INLINE Packet2i	pmadd (const Packet2i &a, const Packet2i &b, const Packet2i &c)
template<>
EIGEN_STRONG_INLINE Packet2ui	pmadd (const Packet2ui &a, const Packet2ui &b, const Packet2ui &c)
template<>
EIGEN_STRONG_INLINE Packet2f	pabsdiff< Packet2f > (const Packet2f &a, const Packet2f &b)
template<>
EIGEN_STRONG_INLINE Packet4c	pabsdiff< Packet4c > (const Packet4c &a, const Packet4c &b)
template<>
EIGEN_STRONG_INLINE Packet8c	pabsdiff< Packet8c > (const Packet8c &a, const Packet8c &b)
template<>
EIGEN_STRONG_INLINE Packet4uc	pabsdiff< Packet4uc > (const Packet4uc &a, const Packet4uc &b)
template<>
EIGEN_STRONG_INLINE Packet8uc	pabsdiff< Packet8uc > (const Packet8uc &a, const Packet8uc &b)
template<>
EIGEN_STRONG_INLINE Packet4s	pabsdiff< Packet4s > (const Packet4s &a, const Packet4s &b)
template<>
EIGEN_STRONG_INLINE Packet4us	pabsdiff< Packet4us > (const Packet4us &a, const Packet4us &b)
template<>
EIGEN_STRONG_INLINE Packet2i	pabsdiff< Packet2i > (const Packet2i &a, const Packet2i &b)
template<>
EIGEN_STRONG_INLINE Packet2ui	pabsdiff< Packet2ui > (const Packet2ui &a, const Packet2ui &b)
template<>
EIGEN_STRONG_INLINE Packet2f	pmin< Packet2f > (const Packet2f &a, const Packet2f &b)
template<>
EIGEN_STRONG_INLINE Packet2f	pmin< PropagateNaN, Packet2f > (const Packet2f &a, const Packet2f &b)
template<>
EIGEN_STRONG_INLINE Packet4c	pmin< Packet4c > (const Packet4c &a, const Packet4c &b)
template<>
EIGEN_STRONG_INLINE Packet8c	pmin< Packet8c > (const Packet8c &a, const Packet8c &b)
template<>
EIGEN_STRONG_INLINE Packet4uc	pmin< Packet4uc > (const Packet4uc &a, const Packet4uc &b)
template<>
EIGEN_STRONG_INLINE Packet8uc	pmin< Packet8uc > (const Packet8uc &a, const Packet8uc &b)
template<>
EIGEN_STRONG_INLINE Packet4s	pmin< Packet4s > (const Packet4s &a, const Packet4s &b)
template<>
EIGEN_STRONG_INLINE Packet4us	pmin< Packet4us > (const Packet4us &a, const Packet4us &b)
template<>
EIGEN_STRONG_INLINE Packet2i	pmin< Packet2i > (const Packet2i &a, const Packet2i &b)
template<>
EIGEN_STRONG_INLINE Packet2ui	pmin< Packet2ui > (const Packet2ui &a, const Packet2ui &b)
template<>
EIGEN_STRONG_INLINE Packet2f	pmax< Packet2f > (const Packet2f &a, const Packet2f &b)
template<>
EIGEN_STRONG_INLINE Packet2f	pmax< PropagateNaN, Packet2f > (const Packet2f &a, const Packet2f &b)
template<>
EIGEN_STRONG_INLINE Packet4c	pmax< Packet4c > (const Packet4c &a, const Packet4c &b)
template<>
EIGEN_STRONG_INLINE Packet8c	pmax< Packet8c > (const Packet8c &a, const Packet8c &b)
template<>
EIGEN_STRONG_INLINE Packet4uc	pmax< Packet4uc > (const Packet4uc &a, const Packet4uc &b)
template<>
EIGEN_STRONG_INLINE Packet8uc	pmax< Packet8uc > (const Packet8uc &a, const Packet8uc &b)
template<>
EIGEN_STRONG_INLINE Packet4s	pmax< Packet4s > (const Packet4s &a, const Packet4s &b)
template<>
EIGEN_STRONG_INLINE Packet4us	pmax< Packet4us > (const Packet4us &a, const Packet4us &b)
template<>
EIGEN_STRONG_INLINE Packet2i	pmax< Packet2i > (const Packet2i &a, const Packet2i &b)
template<>
EIGEN_STRONG_INLINE Packet2ui	pmax< Packet2ui > (const Packet2ui &a, const Packet2ui &b)
template<>
EIGEN_STRONG_INLINE Packet2f	pcmp_le< Packet2f > (const Packet2f &a, const Packet2f &b)
template<>
EIGEN_STRONG_INLINE Packet4c	pcmp_le< Packet4c > (const Packet4c &a, const Packet4c &b)
template<>
EIGEN_STRONG_INLINE Packet8c	pcmp_le< Packet8c > (const Packet8c &a, const Packet8c &b)
template<>
EIGEN_STRONG_INLINE Packet4uc	pcmp_le< Packet4uc > (const Packet4uc &a, const Packet4uc &b)
template<>
EIGEN_STRONG_INLINE Packet8uc	pcmp_le< Packet8uc > (const Packet8uc &a, const Packet8uc &b)
template<>
EIGEN_STRONG_INLINE Packet4s	pcmp_le< Packet4s > (const Packet4s &a, const Packet4s &b)
template<>
EIGEN_STRONG_INLINE Packet4us	pcmp_le< Packet4us > (const Packet4us &a, const Packet4us &b)
template<>
EIGEN_STRONG_INLINE Packet2i	pcmp_le< Packet2i > (const Packet2i &a, const Packet2i &b)
template<>
EIGEN_STRONG_INLINE Packet2ui	pcmp_le< Packet2ui > (const Packet2ui &a, const Packet2ui &b)
template<>
EIGEN_STRONG_INLINE Packet2f	pcmp_lt< Packet2f > (const Packet2f &a, const Packet2f &b)
template<>
EIGEN_STRONG_INLINE Packet4c	pcmp_lt< Packet4c > (const Packet4c &a, const Packet4c &b)
template<>
EIGEN_STRONG_INLINE Packet8c	pcmp_lt< Packet8c > (const Packet8c &a, const Packet8c &b)
template<>
EIGEN_STRONG_INLINE Packet4uc	pcmp_lt< Packet4uc > (const Packet4uc &a, const Packet4uc &b)
template<>
EIGEN_STRONG_INLINE Packet8uc	pcmp_lt< Packet8uc > (const Packet8uc &a, const Packet8uc &b)
template<>
EIGEN_STRONG_INLINE Packet4s	pcmp_lt< Packet4s > (const Packet4s &a, const Packet4s &b)
template<>
EIGEN_STRONG_INLINE Packet4us	pcmp_lt< Packet4us > (const Packet4us &a, const Packet4us &b)
template<>
EIGEN_STRONG_INLINE Packet2i	pcmp_lt< Packet2i > (const Packet2i &a, const Packet2i &b)
template<>
EIGEN_STRONG_INLINE Packet2ui	pcmp_lt< Packet2ui > (const Packet2ui &a, const Packet2ui &b)
template<>
EIGEN_STRONG_INLINE Packet2f	pcmp_eq< Packet2f > (const Packet2f &a, const Packet2f &b)
template<>
EIGEN_STRONG_INLINE Packet4c	pcmp_eq< Packet4c > (const Packet4c &a, const Packet4c &b)
template<>
EIGEN_STRONG_INLINE Packet8c	pcmp_eq< Packet8c > (const Packet8c &a, const Packet8c &b)
template<>
EIGEN_STRONG_INLINE Packet4uc	pcmp_eq< Packet4uc > (const Packet4uc &a, const Packet4uc &b)
template<>
EIGEN_STRONG_INLINE Packet8uc	pcmp_eq< Packet8uc > (const Packet8uc &a, const Packet8uc &b)
template<>
EIGEN_STRONG_INLINE Packet4s	pcmp_eq< Packet4s > (const Packet4s &a, const Packet4s &b)
template<>
EIGEN_STRONG_INLINE Packet4us	pcmp_eq< Packet4us > (const Packet4us &a, const Packet4us &b)
template<>
EIGEN_STRONG_INLINE Packet2i	pcmp_eq< Packet2i > (const Packet2i &a, const Packet2i &b)
template<>
EIGEN_STRONG_INLINE Packet2ui	pcmp_eq< Packet2ui > (const Packet2ui &a, const Packet2ui &b)
template<>
EIGEN_STRONG_INLINE Packet2f	pcmp_lt_or_nan< Packet2f > (const Packet2f &a, const Packet2f &b)
template<>
EIGEN_STRONG_INLINE Packet2f	pand< Packet2f > (const Packet2f &a, const Packet2f &b)
template<>
EIGEN_STRONG_INLINE Packet4c	pand< Packet4c > (const Packet4c &a, const Packet4c &b)
template<>
EIGEN_STRONG_INLINE Packet8c	pand< Packet8c > (const Packet8c &a, const Packet8c &b)
template<>
EIGEN_STRONG_INLINE Packet4uc	pand< Packet4uc > (const Packet4uc &a, const Packet4uc &b)
template<>
EIGEN_STRONG_INLINE Packet8uc	pand< Packet8uc > (const Packet8uc &a, const Packet8uc &b)
template<>
EIGEN_STRONG_INLINE Packet4s	pand< Packet4s > (const Packet4s &a, const Packet4s &b)
template<>
EIGEN_STRONG_INLINE Packet4us	pand< Packet4us > (const Packet4us &a, const Packet4us &b)
template<>
EIGEN_STRONG_INLINE Packet2i	pand< Packet2i > (const Packet2i &a, const Packet2i &b)
template<>
EIGEN_STRONG_INLINE Packet2ui	pand< Packet2ui > (const Packet2ui &a, const Packet2ui &b)
template<>
EIGEN_STRONG_INLINE Packet2f	por< Packet2f > (const Packet2f &a, const Packet2f &b)
template<>
EIGEN_STRONG_INLINE Packet4c	por< Packet4c > (const Packet4c &a, const Packet4c &b)
template<>
EIGEN_STRONG_INLINE Packet8c	por< Packet8c > (const Packet8c &a, const Packet8c &b)
template<>
EIGEN_STRONG_INLINE Packet4uc	por< Packet4uc > (const Packet4uc &a, const Packet4uc &b)
template<>
EIGEN_STRONG_INLINE Packet8uc	por< Packet8uc > (const Packet8uc &a, const Packet8uc &b)
template<>
EIGEN_STRONG_INLINE Packet4s	por< Packet4s > (const Packet4s &a, const Packet4s &b)
template<>
EIGEN_STRONG_INLINE Packet4us	por< Packet4us > (const Packet4us &a, const Packet4us &b)
template<>
EIGEN_STRONG_INLINE Packet2i	por< Packet2i > (const Packet2i &a, const Packet2i &b)
template<>
EIGEN_STRONG_INLINE Packet2ui	por< Packet2ui > (const Packet2ui &a, const Packet2ui &b)
template<>
EIGEN_STRONG_INLINE Packet4c	pxor< Packet4c > (const Packet4c &a, const Packet4c &b)
template<>
EIGEN_STRONG_INLINE Packet8c	pxor< Packet8c > (const Packet8c &a, const Packet8c &b)
template<>
EIGEN_STRONG_INLINE Packet4uc	pxor< Packet4uc > (const Packet4uc &a, const Packet4uc &b)
template<>
EIGEN_STRONG_INLINE Packet8uc	pxor< Packet8uc > (const Packet8uc &a, const Packet8uc &b)
template<>
EIGEN_STRONG_INLINE Packet4s	pxor< Packet4s > (const Packet4s &a, const Packet4s &b)
template<>
EIGEN_STRONG_INLINE Packet4us	pxor< Packet4us > (const Packet4us &a, const Packet4us &b)
template<>
EIGEN_STRONG_INLINE Packet2i	pxor< Packet2i > (const Packet2i &a, const Packet2i &b)
template<>
EIGEN_STRONG_INLINE Packet2ui	pxor< Packet2ui > (const Packet2ui &a, const Packet2ui &b)
template<>
EIGEN_STRONG_INLINE Packet2f	pandnot< Packet2f > (const Packet2f &a, const Packet2f &b)
template<>
EIGEN_STRONG_INLINE Packet4c	pandnot< Packet4c > (const Packet4c &a, const Packet4c &b)
template<>
EIGEN_STRONG_INLINE Packet8c	pandnot< Packet8c > (const Packet8c &a, const Packet8c &b)
template<>
EIGEN_STRONG_INLINE Packet4uc	pandnot< Packet4uc > (const Packet4uc &a, const Packet4uc &b)
template<>
EIGEN_STRONG_INLINE Packet8uc	pandnot< Packet8uc > (const Packet8uc &a, const Packet8uc &b)
template<>
EIGEN_STRONG_INLINE Packet4s	pandnot< Packet4s > (const Packet4s &a, const Packet4s &b)
template<>
EIGEN_STRONG_INLINE Packet4us	pandnot< Packet4us > (const Packet4us &a, const Packet4us &b)
template<>
EIGEN_STRONG_INLINE Packet2i	pandnot< Packet2i > (const Packet2i &a, const Packet2i &b)
template<>
EIGEN_STRONG_INLINE Packet2ui	pandnot< Packet2ui > (const Packet2ui &a, const Packet2ui &b)
template<int N>
EIGEN_STRONG_INLINE Packet4c	parithmetic_shift_right (Packet4c &a)
template<int N>
EIGEN_STRONG_INLINE Packet8c	parithmetic_shift_right (Packet8c a)
template<int N>
EIGEN_STRONG_INLINE Packet16c	parithmetic_shift_right (Packet16c a)
template<int N>
EIGEN_STRONG_INLINE Packet4uc	parithmetic_shift_right (Packet4uc &a)
template<int N>
EIGEN_STRONG_INLINE Packet8uc	parithmetic_shift_right (Packet8uc a)
template<int N>
EIGEN_STRONG_INLINE Packet16uc	parithmetic_shift_right (Packet16uc a)
template<int N>
EIGEN_STRONG_INLINE Packet4s	parithmetic_shift_right (Packet4s a)
template<int N>
EIGEN_STRONG_INLINE Packet8s	parithmetic_shift_right (Packet8s a)
template<int N>
EIGEN_STRONG_INLINE Packet4us	parithmetic_shift_right (Packet4us a)
template<int N>
EIGEN_STRONG_INLINE Packet8us	parithmetic_shift_right (Packet8us a)
template<int N>
EIGEN_STRONG_INLINE Packet2i	parithmetic_shift_right (Packet2i a)
template<int N>
EIGEN_STRONG_INLINE Packet4i	parithmetic_shift_right (Packet4i a)
template<int N>
EIGEN_STRONG_INLINE Packet2ui	parithmetic_shift_right (Packet2ui a)
template<int N>
EIGEN_STRONG_INLINE Packet4ui	parithmetic_shift_right (Packet4ui a)
template<int N>
EIGEN_STRONG_INLINE Packet2l	parithmetic_shift_right (Packet2l a)
template<int N>
EIGEN_STRONG_INLINE Packet2ul	parithmetic_shift_right (Packet2ul a)
template<int N>
EIGEN_STRONG_INLINE Packet4c	plogical_shift_right (Packet4c &a)
template<int N>
EIGEN_STRONG_INLINE Packet8c	plogical_shift_right (Packet8c a)
template<int N>
EIGEN_STRONG_INLINE Packet16c	plogical_shift_right (Packet16c a)
template<int N>
EIGEN_STRONG_INLINE Packet4uc	plogical_shift_right (Packet4uc &a)
template<int N>
EIGEN_STRONG_INLINE Packet8uc	plogical_shift_right (Packet8uc a)
template<int N>
EIGEN_STRONG_INLINE Packet16uc	plogical_shift_right (Packet16uc a)
template<int N>
EIGEN_STRONG_INLINE Packet4s	plogical_shift_right (Packet4s a)
template<int N>
EIGEN_STRONG_INLINE Packet8s	plogical_shift_right (Packet8s a)
template<int N>
EIGEN_STRONG_INLINE Packet4us	plogical_shift_right (Packet4us a)
template<int N>
EIGEN_STRONG_INLINE Packet8us	plogical_shift_right (Packet8us a)
template<int N>
EIGEN_STRONG_INLINE Packet2i	plogical_shift_right (Packet2i a)
template<int N>
EIGEN_STRONG_INLINE Packet4i	plogical_shift_right (Packet4i a)
template<int N>
EIGEN_STRONG_INLINE Packet2ui	plogical_shift_right (Packet2ui a)
template<int N>
EIGEN_STRONG_INLINE Packet4ui	plogical_shift_right (Packet4ui a)
template<int N>
EIGEN_STRONG_INLINE Packet2l	plogical_shift_right (Packet2l a)
template<int N>
EIGEN_STRONG_INLINE Packet2ul	plogical_shift_right (Packet2ul a)
template<int N>
EIGEN_STRONG_INLINE Packet4c	plogical_shift_left (Packet4c &a)
template<int N>
EIGEN_STRONG_INLINE Packet8c	plogical_shift_left (Packet8c a)
template<int N>
EIGEN_STRONG_INLINE Packet16c	plogical_shift_left (Packet16c a)
template<int N>
EIGEN_STRONG_INLINE Packet4uc	plogical_shift_left (Packet4uc &a)
template<int N>
EIGEN_STRONG_INLINE Packet8uc	plogical_shift_left (Packet8uc a)
template<int N>
EIGEN_STRONG_INLINE Packet16uc	plogical_shift_left (Packet16uc a)
template<int N>
EIGEN_STRONG_INLINE Packet4s	plogical_shift_left (Packet4s a)
template<int N>
EIGEN_STRONG_INLINE Packet8s	plogical_shift_left (Packet8s a)
template<int N>
EIGEN_STRONG_INLINE Packet4us	plogical_shift_left (Packet4us a)
template<int N>
EIGEN_STRONG_INLINE Packet8us	plogical_shift_left (Packet8us a)
template<int N>
EIGEN_STRONG_INLINE Packet2i	plogical_shift_left (Packet2i a)
template<int N>
EIGEN_STRONG_INLINE Packet4i	plogical_shift_left (Packet4i a)
template<int N>
EIGEN_STRONG_INLINE Packet2ui	plogical_shift_left (Packet2ui a)
template<int N>
EIGEN_STRONG_INLINE Packet4ui	plogical_shift_left (Packet4ui a)
template<int N>
EIGEN_STRONG_INLINE Packet2l	plogical_shift_left (Packet2l a)
template<int N>
EIGEN_STRONG_INLINE Packet2ul	plogical_shift_left (Packet2ul a)
template<>
EIGEN_STRONG_INLINE Packet2f	pload< Packet2f > (const float *from)
template<>
EIGEN_STRONG_INLINE Packet4c	pload< Packet4c > (const int8_t *from)
template<>
EIGEN_STRONG_INLINE Packet8c	pload< Packet8c > (const int8_t *from)
template<>
EIGEN_STRONG_INLINE Packet4uc	pload< Packet4uc > (const uint8_t *from)
template<>
EIGEN_STRONG_INLINE Packet8uc	pload< Packet8uc > (const uint8_t *from)
template<>
EIGEN_STRONG_INLINE Packet4s	pload< Packet4s > (const int16_t *from)
template<>
EIGEN_STRONG_INLINE Packet4us	pload< Packet4us > (const uint16_t *from)
template<>
EIGEN_STRONG_INLINE Packet2i	pload< Packet2i > (const int32_t *from)
template<>
EIGEN_STRONG_INLINE Packet2ui	pload< Packet2ui > (const uint32_t *from)
template<>
EIGEN_STRONG_INLINE Packet2f	ploadu< Packet2f > (const float *from)
template<>
EIGEN_STRONG_INLINE Packet4c	ploadu< Packet4c > (const int8_t *from)
template<>
EIGEN_STRONG_INLINE Packet8c	ploadu< Packet8c > (const int8_t *from)
template<>
EIGEN_STRONG_INLINE Packet4uc	ploadu< Packet4uc > (const uint8_t *from)
template<>
EIGEN_STRONG_INLINE Packet8uc	ploadu< Packet8uc > (const uint8_t *from)
template<>
EIGEN_STRONG_INLINE Packet4s	ploadu< Packet4s > (const int16_t *from)
template<>
EIGEN_STRONG_INLINE Packet4us	ploadu< Packet4us > (const uint16_t *from)
template<>
EIGEN_STRONG_INLINE Packet2i	ploadu< Packet2i > (const int32_t *from)
template<>
EIGEN_STRONG_INLINE Packet2ui	ploadu< Packet2ui > (const uint32_t *from)
template<>
EIGEN_STRONG_INLINE Packet2f	ploaddup< Packet2f > (const float *from)
template<>
EIGEN_STRONG_INLINE Packet4c	ploaddup< Packet4c > (const int8_t *from)
template<>
EIGEN_STRONG_INLINE Packet8c	ploaddup< Packet8c > (const int8_t *from)
template<>
EIGEN_STRONG_INLINE Packet4uc	ploaddup< Packet4uc > (const uint8_t *from)
template<>
EIGEN_STRONG_INLINE Packet8uc	ploaddup< Packet8uc > (const uint8_t *from)
template<>
EIGEN_STRONG_INLINE Packet4s	ploaddup< Packet4s > (const int16_t *from)
template<>
EIGEN_STRONG_INLINE Packet4us	ploaddup< Packet4us > (const uint16_t *from)
template<>
EIGEN_STRONG_INLINE Packet2i	ploaddup< Packet2i > (const int32_t *from)
template<>
EIGEN_STRONG_INLINE Packet2ui	ploaddup< Packet2ui > (const uint32_t *from)
template<>
EIGEN_STRONG_INLINE Packet4c	ploadquad< Packet4c > (const int8_t *from)
template<>
EIGEN_STRONG_INLINE Packet8c	ploadquad< Packet8c > (const int8_t *from)
template<>
EIGEN_STRONG_INLINE Packet4uc	ploadquad< Packet4uc > (const uint8_t *from)
template<>
EIGEN_STRONG_INLINE Packet8uc	ploadquad< Packet8uc > (const uint8_t *from)
template<>
EIGEN_STRONG_INLINE void	pstore< float > (float *to, const Packet2f &from)
template<>
EIGEN_STRONG_INLINE void	pstore< int8_t > (int8_t *to, const Packet4c &from)
template<>
EIGEN_STRONG_INLINE void	pstore< int8_t > (int8_t *to, const Packet8c &from)
template<>
EIGEN_STRONG_INLINE void	pstore< uint8_t > (uint8_t *to, const Packet4uc &from)
template<>
EIGEN_STRONG_INLINE void	pstore< uint8_t > (uint8_t *to, const Packet8uc &from)
template<>
EIGEN_STRONG_INLINE void	pstore< int16_t > (int16_t *to, const Packet4s &from)
template<>
EIGEN_STRONG_INLINE void	pstore< uint16_t > (uint16_t *to, const Packet4us &from)
template<>
EIGEN_STRONG_INLINE void	pstore< int32_t > (int32_t *to, const Packet2i &from)
template<>
EIGEN_STRONG_INLINE void	pstore< uint32_t > (uint32_t *to, const Packet2ui &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< float > (float *to, const Packet2f &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< int8_t > (int8_t *to, const Packet4c &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< int8_t > (int8_t *to, const Packet8c &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< uint8_t > (uint8_t *to, const Packet4uc &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< uint8_t > (uint8_t *to, const Packet8uc &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< int16_t > (int16_t *to, const Packet4s &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< uint16_t > (uint16_t *to, const Packet4us &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< int32_t > (int32_t *to, const Packet2i &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< uint32_t > (uint32_t *to, const Packet2ui &from)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2f	pgather< float, Packet2f > (const float *from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4c	pgather< int8_t, Packet4c > (const int8_t *from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8c	pgather< int8_t, Packet8c > (const int8_t *from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4uc	pgather< uint8_t, Packet4uc > (const uint8_t *from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8uc	pgather< uint8_t, Packet8uc > (const uint8_t *from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4s	pgather< int16_t, Packet4s > (const int16_t *from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4us	pgather< uint16_t, Packet4us > (const uint16_t *from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2i	pgather< int32_t, Packet2i > (const int32_t *from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ui	pgather< uint32_t, Packet2ui > (const uint32_t *from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	pscatter< float, Packet2f > (float *to, const Packet2f &from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	pscatter< int8_t, Packet4c > (int8_t *to, const Packet4c &from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	pscatter< int8_t, Packet8c > (int8_t *to, const Packet8c &from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	pscatter< uint8_t, Packet4uc > (uint8_t *to, const Packet4uc &from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	pscatter< uint8_t, Packet8uc > (uint8_t *to, const Packet8uc &from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	pscatter< int16_t, Packet4s > (int16_t *to, const Packet4s &from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	pscatter< uint16_t, Packet4us > (uint16_t *to, const Packet4us &from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	pscatter< int32_t, Packet2i > (int32_t *to, const Packet2i &from, Index stride)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	pscatter< uint32_t, Packet2ui > (uint32_t *to, const Packet2ui &from, Index stride)
template<>
EIGEN_STRONG_INLINE float	pfirst< Packet2f > (const Packet2f &a)
template<>
EIGEN_STRONG_INLINE int8_t	pfirst< Packet4c > (const Packet4c &a)
template<>
EIGEN_STRONG_INLINE int8_t	pfirst< Packet8c > (const Packet8c &a)
template<>
EIGEN_STRONG_INLINE uint8_t	pfirst< Packet4uc > (const Packet4uc &a)
template<>
EIGEN_STRONG_INLINE uint8_t	pfirst< Packet8uc > (const Packet8uc &a)
template<>
EIGEN_STRONG_INLINE int16_t	pfirst< Packet4s > (const Packet4s &a)
template<>
EIGEN_STRONG_INLINE uint16_t	pfirst< Packet4us > (const Packet4us &a)
template<>
EIGEN_STRONG_INLINE int32_t	pfirst< Packet2i > (const Packet2i &a)
template<>
EIGEN_STRONG_INLINE uint32_t	pfirst< Packet2ui > (const Packet2ui &a)
template<>
EIGEN_STRONG_INLINE Packet2f	preverse (const Packet2f &a)
template<>
EIGEN_STRONG_INLINE Packet4c	preverse (const Packet4c &a)
template<>
EIGEN_STRONG_INLINE Packet8c	preverse (const Packet8c &a)
template<>
EIGEN_STRONG_INLINE Packet4uc	preverse (const Packet4uc &a)
template<>
EIGEN_STRONG_INLINE Packet8uc	preverse (const Packet8uc &a)
template<>
EIGEN_STRONG_INLINE Packet4s	preverse (const Packet4s &a)
template<>
EIGEN_STRONG_INLINE Packet4us	preverse (const Packet4us &a)
template<>
EIGEN_STRONG_INLINE Packet2i	preverse (const Packet2i &a)
template<>
EIGEN_STRONG_INLINE Packet2ui	preverse (const Packet2ui &a)
template<>
EIGEN_STRONG_INLINE Packet2f	pabs (const Packet2f &a)
template<>
EIGEN_STRONG_INLINE Packet4c	pabs< Packet4c > (const Packet4c &a)
template<>
EIGEN_STRONG_INLINE Packet8c	pabs (const Packet8c &a)
template<>
EIGEN_STRONG_INLINE Packet4uc	pabs (const Packet4uc &a)
template<>
EIGEN_STRONG_INLINE Packet8uc	pabs (const Packet8uc &a)
template<>
EIGEN_STRONG_INLINE Packet4s	pabs (const Packet4s &a)
template<>
EIGEN_STRONG_INLINE Packet4us	pabs (const Packet4us &a)
template<>
EIGEN_STRONG_INLINE Packet2i	pabs (const Packet2i &a)
template<>
EIGEN_STRONG_INLINE Packet2ui	pabs (const Packet2ui &a)
template<>
EIGEN_STRONG_INLINE Packet2f	psignbit (const Packet2f &a)
template<>
EIGEN_STRONG_INLINE Packet2f	pfrexp< Packet2f > (const Packet2f &a, Packet2f &exponent)
template<>
EIGEN_STRONG_INLINE Packet2f	pldexp< Packet2f > (const Packet2f &a, const Packet2f &exponent)
template<>
EIGEN_STRONG_INLINE float	predux< Packet2f > (const Packet2f &a)
template<>
EIGEN_STRONG_INLINE int8_t	predux< Packet4c > (const Packet4c &a)
template<>
EIGEN_STRONG_INLINE int8_t	predux< Packet8c > (const Packet8c &a)
template<>
EIGEN_STRONG_INLINE uint8_t	predux< Packet4uc > (const Packet4uc &a)
template<>
EIGEN_STRONG_INLINE uint8_t	predux< Packet8uc > (const Packet8uc &a)
template<>
EIGEN_STRONG_INLINE int16_t	predux< Packet4s > (const Packet4s &a)
template<>
EIGEN_STRONG_INLINE uint16_t	predux< Packet4us > (const Packet4us &a)
template<>
EIGEN_STRONG_INLINE int32_t	predux< Packet2i > (const Packet2i &a)
template<>
EIGEN_STRONG_INLINE uint32_t	predux< Packet2ui > (const Packet2ui &a)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4c	predux_half_dowto4 (const Packet8c &a)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8c	predux_half_dowto4 (const Packet16c &a)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4uc	predux_half_dowto4 (const Packet8uc &a)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8uc	predux_half_dowto4 (const Packet16uc &a)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4s	predux_half_dowto4 (const Packet8s &a)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4us	predux_half_dowto4 (const Packet8us &a)
template<>
EIGEN_STRONG_INLINE float	predux_mul< Packet2f > (const Packet2f &a)
template<>
EIGEN_STRONG_INLINE int8_t	predux_mul< Packet4c > (const Packet4c &a)
template<>
EIGEN_STRONG_INLINE int8_t	predux_mul< Packet8c > (const Packet8c &a)
template<>
EIGEN_STRONG_INLINE uint8_t	predux_mul< Packet4uc > (const Packet4uc &a)
template<>
EIGEN_STRONG_INLINE uint8_t	predux_mul< Packet8uc > (const Packet8uc &a)
template<>
EIGEN_STRONG_INLINE int16_t	predux_mul< Packet4s > (const Packet4s &a)
template<>
EIGEN_STRONG_INLINE uint16_t	predux_mul< Packet4us > (const Packet4us &a)
template<>
EIGEN_STRONG_INLINE int32_t	predux_mul< Packet2i > (const Packet2i &a)
template<>
EIGEN_STRONG_INLINE uint32_t	predux_mul< Packet2ui > (const Packet2ui &a)
template<>
EIGEN_STRONG_INLINE float	predux_min< Packet2f > (const Packet2f &a)
template<>
EIGEN_STRONG_INLINE int8_t	predux_min< Packet4c > (const Packet4c &a)
template<>
EIGEN_STRONG_INLINE int8_t	predux_min< Packet8c > (const Packet8c &a)
template<>
EIGEN_STRONG_INLINE uint8_t	predux_min< Packet4uc > (const Packet4uc &a)
template<>
EIGEN_STRONG_INLINE uint8_t	predux_min< Packet8uc > (const Packet8uc &a)
template<>
EIGEN_STRONG_INLINE int16_t	predux_min< Packet4s > (const Packet4s &a)
template<>
EIGEN_STRONG_INLINE uint16_t	predux_min< Packet4us > (const Packet4us &a)
template<>
EIGEN_STRONG_INLINE int32_t	predux_min< Packet2i > (const Packet2i &a)
template<>
EIGEN_STRONG_INLINE uint32_t	predux_min< Packet2ui > (const Packet2ui &a)
template<>
EIGEN_STRONG_INLINE float	predux_max< Packet2f > (const Packet2f &a)
template<>
EIGEN_STRONG_INLINE int8_t	predux_max< Packet4c > (const Packet4c &a)
template<>
EIGEN_STRONG_INLINE int8_t	predux_max< Packet8c > (const Packet8c &a)
template<>
EIGEN_STRONG_INLINE uint8_t	predux_max< Packet4uc > (const Packet4uc &a)
template<>
EIGEN_STRONG_INLINE uint8_t	predux_max< Packet8uc > (const Packet8uc &a)
template<>
EIGEN_STRONG_INLINE int16_t	predux_max< Packet4s > (const Packet4s &a)
template<>
EIGEN_STRONG_INLINE uint16_t	predux_max< Packet4us > (const Packet4us &a)
template<>
EIGEN_STRONG_INLINE int32_t	predux_max< Packet2i > (const Packet2i &a)
template<>
EIGEN_STRONG_INLINE uint32_t	predux_max< Packet2ui > (const Packet2ui &a)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	ptranspose (PacketBlock< Packet2f, 2 > &kernel)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	ptranspose (PacketBlock< Packet4c, 4 > &kernel)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	ptranspose (PacketBlock< Packet8c, 8 > &kernel)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	ptranspose (PacketBlock< Packet8c, 4 > &kernel)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	ptranspose (PacketBlock< Packet4uc, 4 > &kernel)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	ptranspose (PacketBlock< Packet8uc, 8 > &kernel)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	ptranspose (PacketBlock< Packet8uc, 4 > &kernel)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	ptranspose (PacketBlock< Packet4s, 4 > &kernel)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	ptranspose (PacketBlock< Packet4us, 4 > &kernel)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	ptranspose (PacketBlock< Packet2i, 2 > &kernel)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	ptranspose (PacketBlock< Packet2ui, 2 > &kernel)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2f	pselect (const Packet2f &mask, const Packet2f &a, const Packet2f &b)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8c	pselect (const Packet8c &mask, const Packet8c &a, const Packet8c &b)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8uc	pselect (const Packet8uc &mask, const Packet8uc &a, const Packet8uc &b)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4s	pselect (const Packet4s &mask, const Packet4s &a, const Packet4s &b)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4us	pselect (const Packet4us &mask, const Packet4us &a, const Packet4us &b)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2i	pselect (const Packet2i &mask, const Packet2i &a, const Packet2i &b)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ui	pselect (const Packet2ui &mask, const Packet2ui &a, const Packet2ui &b)
template<>
EIGEN_STRONG_INLINE Packet4uc	psqrt (const Packet4uc &a)
template<>
EIGEN_STRONG_INLINE Packet8uc	psqrt (const Packet8uc &a)
template<>
EIGEN_STRONG_INLINE Packet4us	psqrt (const Packet4us &a)
template<>
EIGEN_STRONG_INLINE Packet2ui	psqrt (const Packet2ui &a)
EIGEN_STRONG_INLINE Packet4f	prsqrt_float_unsafe (const Packet4f &a)
EIGEN_STRONG_INLINE Packet2f	prsqrt_float_unsafe (const Packet2f &a)
template<typename Packet >
Packet	prsqrt_float_common (const Packet &a)
template<>
EIGEN_STRONG_INLINE Packet2f	prsqrt (const Packet2f &a)
template<>
EIGEN_STRONG_INLINE Packet2f	preciprocal< Packet2f > (const Packet2f &a)
template<typename Packet >
EIGEN_STRONG_INLINE Packet	psqrt_float_common (const Packet &a)
template<>
EIGEN_STRONG_INLINE Packet2f	psqrt (const Packet2f &a)
template<typename Packet >
EIGEN_STRONG_INLINE Packet	pdiv_float_common (const Packet &a, const Packet &b)
template<>
EIGEN_STRONG_INLINE Packet2f	pdiv< Packet2f > (const Packet2f &a, const Packet2f &b)
EIGEN_STRONG_INLINE Packet4bf	F32ToBf16 (const Packet4f &p)
EIGEN_STRONG_INLINE Packet4f	Bf16ToF32 (const Packet4bf &p)
EIGEN_STRONG_INLINE Packet4bf	F32MaskToBf16Mask (const Packet4f &p)
template<>
EIGEN_STRONG_INLINE Packet4bf	pset1< Packet4bf > (const bfloat16 &from)
template<>
EIGEN_STRONG_INLINE bfloat16	pfirst< Packet4bf > (const Packet4bf &from)
template<>
EIGEN_STRONG_INLINE Packet4bf	pload< Packet4bf > (const bfloat16 *from)
template<>
EIGEN_STRONG_INLINE Packet4bf	ploadu< Packet4bf > (const bfloat16 *from)
template<>
EIGEN_STRONG_INLINE void	pstore< bfloat16 > (bfloat16 *to, const Packet4bf &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< bfloat16 > (bfloat16 *to, const Packet4bf &from)
template<>
EIGEN_STRONG_INLINE Packet4bf	ploaddup< Packet4bf > (const bfloat16 *from)
template<>
EIGEN_STRONG_INLINE Packet4bf	pabs (const Packet4bf &a)
template<>
EIGEN_STRONG_INLINE Packet4bf	pmin< PropagateNumbers, Packet4bf > (const Packet4bf &a, const Packet4bf &b)
template<>
EIGEN_STRONG_INLINE Packet4bf	pmin< PropagateNaN, Packet4bf > (const Packet4bf &a, const Packet4bf &b)
template<>
EIGEN_STRONG_INLINE Packet4bf	pmin< Packet4bf > (const Packet4bf &a, const Packet4bf &b)
template<>
EIGEN_STRONG_INLINE Packet4bf	pmax< PropagateNumbers, Packet4bf > (const Packet4bf &a, const Packet4bf &b)
template<>
EIGEN_STRONG_INLINE Packet4bf	pmax< PropagateNaN, Packet4bf > (const Packet4bf &a, const Packet4bf &b)
template<>
EIGEN_STRONG_INLINE Packet4bf	pmax< Packet4bf > (const Packet4bf &a, const Packet4bf &b)
template<>
EIGEN_STRONG_INLINE Packet4bf	plset< Packet4bf > (const bfloat16 &a)
template<>
EIGEN_STRONG_INLINE Packet4bf	por (const Packet4bf &a, const Packet4bf &b)
template<>
EIGEN_STRONG_INLINE Packet4bf	pxor (const Packet4bf &a, const Packet4bf &b)
template<>
EIGEN_STRONG_INLINE Packet4bf	pand (const Packet4bf &a, const Packet4bf &b)
template<>
EIGEN_STRONG_INLINE Packet4bf	pandnot (const Packet4bf &a, const Packet4bf &b)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4bf	pselect (const Packet4bf &mask, const Packet4bf &a, const Packet4bf &b)
template<>
EIGEN_STRONG_INLINE Packet4bf	print< Packet4bf > (const Packet4bf &a)
template<>
EIGEN_STRONG_INLINE Packet4bf	pfloor< Packet4bf > (const Packet4bf &a)
template<>
EIGEN_STRONG_INLINE Packet4bf	pceil< Packet4bf > (const Packet4bf &a)
template<>
EIGEN_STRONG_INLINE Packet4bf	pround< Packet4bf > (const Packet4bf &a)
template<>
EIGEN_STRONG_INLINE Packet4bf	ptrunc< Packet4bf > (const Packet4bf &a)
template<>
EIGEN_STRONG_INLINE Packet4bf	pconj (const Packet4bf &a)
template<>
EIGEN_STRONG_INLINE Packet4bf	padd< Packet4bf > (const Packet4bf &a, const Packet4bf &b)
template<>
EIGEN_STRONG_INLINE Packet4bf	psub< Packet4bf > (const Packet4bf &a, const Packet4bf &b)
template<>
EIGEN_STRONG_INLINE Packet4bf	pmul< Packet4bf > (const Packet4bf &a, const Packet4bf &b)
template<>
EIGEN_STRONG_INLINE Packet4bf	pdiv< Packet4bf > (const Packet4bf &a, const Packet4bf &b)
template<>
EIGEN_STRONG_INLINE Packet4bf	pgather< bfloat16, Packet4bf > (const bfloat16 *from, Index stride)
template<>
EIGEN_STRONG_INLINE void	pscatter< bfloat16, Packet4bf > (bfloat16 *to, const Packet4bf &from, Index stride)
template<>
EIGEN_STRONG_INLINE bfloat16	predux< Packet4bf > (const Packet4bf &a)
template<>
EIGEN_STRONG_INLINE bfloat16	predux_max< Packet4bf > (const Packet4bf &a)
template<>
EIGEN_STRONG_INLINE bfloat16	predux_min< Packet4bf > (const Packet4bf &a)
template<>
EIGEN_STRONG_INLINE bfloat16	predux_mul< Packet4bf > (const Packet4bf &a)
template<>
EIGEN_STRONG_INLINE Packet4bf	preverse< Packet4bf > (const Packet4bf &a)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	ptranspose (PacketBlock< Packet4bf, 4 > &kernel)
template<>
EIGEN_STRONG_INLINE Packet4bf	pabsdiff< Packet4bf > (const Packet4bf &a, const Packet4bf &b)
template<>
EIGEN_STRONG_INLINE Packet4bf	pcmp_eq< Packet4bf > (const Packet4bf &a, const Packet4bf &b)
template<>
EIGEN_STRONG_INLINE Packet4bf	pcmp_lt< Packet4bf > (const Packet4bf &a, const Packet4bf &b)
template<>
EIGEN_STRONG_INLINE Packet4bf	pcmp_lt_or_nan< Packet4bf > (const Packet4bf &a, const Packet4bf &b)
template<>
EIGEN_STRONG_INLINE Packet4bf	pcmp_le< Packet4bf > (const Packet4bf &a, const Packet4bf &b)
template<>
EIGEN_STRONG_INLINE Packet4bf	pnegate< Packet4bf > (const Packet4bf &a)
template<>
EIGEN_STRONG_INLINE Packet8c	preinterpret< Packet8c, Packet16c > (const Packet16c &a)
template<>
EIGEN_STRONG_INLINE Packet4c	preinterpret< Packet4c, Packet8c > (const Packet8c &a)
template<>
EIGEN_STRONG_INLINE Packet4c	preinterpret< Packet4c, Packet16c > (const Packet16c &a)
template<>
EIGEN_STRONG_INLINE Packet8uc	preinterpret< Packet8uc, Packet16uc > (const Packet16uc &a)
template<>
EIGEN_STRONG_INLINE Packet4uc	preinterpret< Packet4uc, Packet8uc > (const Packet8uc &a)
template<>
EIGEN_STRONG_INLINE Packet4uc	preinterpret< Packet4uc, Packet16uc > (const Packet16uc &a)
template<>
EIGEN_STRONG_INLINE Packet4s	preinterpret< Packet4s, Packet8s > (const Packet8s &a)
template<>
EIGEN_STRONG_INLINE Packet4us	preinterpret< Packet4us, Packet8us > (const Packet8us &a)
template<>
EIGEN_STRONG_INLINE Packet2i	preinterpret< Packet2i, Packet4i > (const Packet4i &a)
template<>
EIGEN_STRONG_INLINE Packet2ui	preinterpret< Packet2ui, Packet4ui > (const Packet4ui &a)
template<>
EIGEN_STRONG_INLINE Packet2f	preinterpret< Packet2f, Packet4f > (const Packet4f &a)
template<>
EIGEN_STRONG_INLINE Packet2f	preinterpret< Packet2f, Packet2i > (const Packet2i &a)
template<>
EIGEN_STRONG_INLINE Packet2f	preinterpret< Packet2f, Packet2ui > (const Packet2ui &a)
template<>
EIGEN_STRONG_INLINE Packet4c	preinterpret< Packet4c, Packet4uc > (const Packet4uc &a)
template<>
EIGEN_STRONG_INLINE Packet8c	preinterpret< Packet8c, Packet8uc > (const Packet8uc &a)
template<>
EIGEN_STRONG_INLINE Packet4uc	preinterpret< Packet4uc, Packet4c > (const Packet4c &a)
template<>
EIGEN_STRONG_INLINE Packet8uc	preinterpret< Packet8uc, Packet8c > (const Packet8c &a)
template<>
EIGEN_STRONG_INLINE Packet4s	preinterpret< Packet4s, Packet4us > (const Packet4us &a)
template<>
EIGEN_STRONG_INLINE Packet4us	preinterpret< Packet4us, Packet4s > (const Packet4s &a)
template<>
EIGEN_STRONG_INLINE Packet2i	preinterpret< Packet2i, Packet2f > (const Packet2f &a)
template<>
EIGEN_STRONG_INLINE Packet2i	preinterpret< Packet2i, Packet2ui > (const Packet2ui &a)
template<>
EIGEN_STRONG_INLINE Packet2ui	preinterpret< Packet2ui, Packet2f > (const Packet2f &a)
template<>
EIGEN_STRONG_INLINE Packet2ui	preinterpret< Packet2ui, Packet2i > (const Packet2i &a)
template<>
EIGEN_STRONG_INLINE Packet2l	preinterpret< Packet2l, Packet2ul > (const Packet2ul &a)
template<>
EIGEN_STRONG_INLINE Packet2l	pcast< Packet2f, Packet2l > (const Packet2f &a)
template<>
EIGEN_STRONG_INLINE Packet2ul	pcast< Packet2f, Packet2ul > (const Packet2f &a)
template<>
EIGEN_STRONG_INLINE Packet2i	pcast< Packet2f, Packet2i > (const Packet2f &a)
template<>
EIGEN_STRONG_INLINE Packet2ui	pcast< Packet2f, Packet2ui > (const Packet2f &a)
template<>
EIGEN_STRONG_INLINE Packet4s	pcast< Packet4f, Packet4s > (const Packet4f &a)
template<>
EIGEN_STRONG_INLINE Packet4s	pcast< Packet2f, Packet4s > (const Packet2f &a, const Packet2f &b)
template<>
EIGEN_STRONG_INLINE Packet4us	pcast< Packet4f, Packet4us > (const Packet4f &a)
template<>
EIGEN_STRONG_INLINE Packet4us	pcast< Packet2f, Packet4us > (const Packet2f &a, const Packet2f &b)
template<>
EIGEN_STRONG_INLINE Packet8c	pcast< Packet4f, Packet8c > (const Packet4f &a, const Packet4f &b)
template<>
EIGEN_STRONG_INLINE Packet8c	pcast< Packet2f, Packet8c > (const Packet2f &a, const Packet2f &b, const Packet2f &c, const Packet2f &d)
template<>
EIGEN_STRONG_INLINE Packet4c	pcast< Packet4f, Packet4c > (const Packet4f &a)
template<>
EIGEN_STRONG_INLINE Packet8uc	pcast< Packet4f, Packet8uc > (const Packet4f &a, const Packet4f &b)
template<>
EIGEN_STRONG_INLINE Packet8uc	pcast< Packet2f, Packet8uc > (const Packet2f &a, const Packet2f &b, const Packet2f &c, const Packet2f &d)
template<>
EIGEN_STRONG_INLINE Packet4uc	pcast< Packet4f, Packet4uc > (const Packet4f &a)
template<>
EIGEN_STRONG_INLINE Packet4f	pcast< Packet4c, Packet4f > (const Packet4c &a)
template<>
EIGEN_STRONG_INLINE Packet2f	pcast< Packet8c, Packet2f > (const Packet8c &a)
template<>
EIGEN_STRONG_INLINE Packet4i	pcast< Packet8c, Packet4i > (const Packet8c &a)
template<>
EIGEN_STRONG_INLINE Packet4i	pcast< Packet4c, Packet4i > (const Packet4c &a)
template<>
EIGEN_STRONG_INLINE Packet2i	pcast< Packet8c, Packet2i > (const Packet8c &a)
template<>
EIGEN_STRONG_INLINE Packet2ui	pcast< Packet8c, Packet2ui > (const Packet8c &a)
template<>
EIGEN_STRONG_INLINE Packet4ui	pcast< Packet4c, Packet4ui > (const Packet4c &a)
template<>
EIGEN_STRONG_INLINE Packet8s	pcast< Packet8c, Packet8s > (const Packet8c &a)
template<>
EIGEN_STRONG_INLINE Packet4s	pcast< Packet8c, Packet4s > (const Packet8c &a)
template<>
EIGEN_STRONG_INLINE Packet4s	pcast< Packet4c, Packet4s > (const Packet4c &a)
template<>
EIGEN_STRONG_INLINE Packet8us	pcast< Packet8c, Packet8us > (const Packet8c &a)
template<>
EIGEN_STRONG_INLINE Packet4us	pcast< Packet8c, Packet4us > (const Packet8c &a)
template<>
EIGEN_STRONG_INLINE Packet4us	pcast< Packet4c, Packet4us > (const Packet4c &a)
template<>
EIGEN_STRONG_INLINE Packet4f	pcast< Packet4uc, Packet4f > (const Packet4uc &a)
template<>
EIGEN_STRONG_INLINE Packet2f	pcast< Packet8uc, Packet2f > (const Packet8uc &a)
template<>
EIGEN_STRONG_INLINE Packet4ui	pcast< Packet8uc, Packet4ui > (const Packet8uc &a)
template<>
EIGEN_STRONG_INLINE Packet2ui	pcast< Packet8uc, Packet2ui > (const Packet8uc &a)
template<>
EIGEN_STRONG_INLINE Packet4ui	pcast< Packet4uc, Packet4ui > (const Packet4uc &a)
template<>
EIGEN_STRONG_INLINE Packet2i	pcast< Packet8uc, Packet2i > (const Packet8uc &a)
template<>
EIGEN_STRONG_INLINE Packet4i	pcast< Packet4uc, Packet4i > (const Packet4uc &a)
template<>
EIGEN_STRONG_INLINE Packet8us	pcast< Packet8uc, Packet8us > (const Packet8uc &a)
template<>
EIGEN_STRONG_INLINE Packet4us	pcast< Packet4uc, Packet4us > (const Packet4uc &a)
template<>
EIGEN_STRONG_INLINE Packet8s	pcast< Packet8uc, Packet8s > (const Packet8uc &a)
template<>
EIGEN_STRONG_INLINE Packet4s	pcast< Packet4uc, Packet4s > (const Packet4uc &a)
template<>
EIGEN_STRONG_INLINE Packet4f	pcast< Packet4s, Packet4f > (const Packet4s &a)
template<>
EIGEN_STRONG_INLINE Packet2f	pcast< Packet4s, Packet2f > (const Packet4s &a)
template<>
EIGEN_STRONG_INLINE Packet4i	pcast< Packet4s, Packet4i > (const Packet4s &a)
template<>
EIGEN_STRONG_INLINE Packet2i	pcast< Packet4s, Packet2i > (const Packet4s &a)
template<>
EIGEN_STRONG_INLINE Packet4ui	pcast< Packet4s, Packet4ui > (const Packet4s &a)
template<>
EIGEN_STRONG_INLINE Packet2ui	pcast< Packet4s, Packet2ui > (const Packet4s &a)
template<>
EIGEN_STRONG_INLINE Packet8c	pcast< Packet8s, Packet8c > (const Packet8s &a)
template<>
EIGEN_STRONG_INLINE Packet8c	pcast< Packet4s, Packet8c > (const Packet4s &a, const Packet4s &b)
template<>
EIGEN_STRONG_INLINE Packet4c	pcast< Packet4s, Packet4c > (const Packet4s &a)
template<>
EIGEN_STRONG_INLINE Packet8uc	pcast< Packet8s, Packet8uc > (const Packet8s &a)
template<>
EIGEN_STRONG_INLINE Packet8uc	pcast< Packet4s, Packet8uc > (const Packet4s &a, const Packet4s &b)
template<>
EIGEN_STRONG_INLINE Packet4uc	pcast< Packet4s, Packet4uc > (const Packet4s &a)
template<>
EIGEN_STRONG_INLINE Packet4f	pcast< Packet4us, Packet4f > (const Packet4us &a)
template<>
EIGEN_STRONG_INLINE Packet2f	pcast< Packet4us, Packet2f > (const Packet4us &a)
template<>
EIGEN_STRONG_INLINE Packet4ui	pcast< Packet4us, Packet4ui > (const Packet4us &a)
template<>
EIGEN_STRONG_INLINE Packet2ui	pcast< Packet4us, Packet2ui > (const Packet4us &a)
template<>
EIGEN_STRONG_INLINE Packet4i	pcast< Packet4us, Packet4i > (const Packet4us &a)
template<>
EIGEN_STRONG_INLINE Packet2i	pcast< Packet4us, Packet2i > (const Packet4us &a)
template<>
EIGEN_STRONG_INLINE Packet8uc	pcast< Packet8us, Packet8uc > (const Packet8us &a)
template<>
EIGEN_STRONG_INLINE Packet8uc	pcast< Packet4us, Packet8uc > (const Packet4us &a, const Packet4us &b)
template<>
EIGEN_STRONG_INLINE Packet4uc	pcast< Packet4us, Packet4uc > (const Packet4us &a)
template<>
EIGEN_STRONG_INLINE Packet8c	pcast< Packet8us, Packet8c > (const Packet8us &a)
template<>
EIGEN_STRONG_INLINE Packet8c	pcast< Packet4us, Packet8c > (const Packet4us &a, const Packet4us &b)
template<>
EIGEN_STRONG_INLINE Packet4c	pcast< Packet4us, Packet4c > (const Packet4us &a)
template<>
EIGEN_STRONG_INLINE Packet2f	pcast< Packet2i, Packet2f > (const Packet2i &a)
template<>
EIGEN_STRONG_INLINE Packet2l	pcast< Packet2i, Packet2l > (const Packet2i &a)
template<>
EIGEN_STRONG_INLINE Packet2ul	pcast< Packet2i, Packet2ul > (const Packet2i &a)
template<>
EIGEN_STRONG_INLINE Packet4s	pcast< Packet4i, Packet4s > (const Packet4i &a)
template<>
EIGEN_STRONG_INLINE Packet4s	pcast< Packet2i, Packet4s > (const Packet2i &a, const Packet2i &b)
template<>
EIGEN_STRONG_INLINE Packet4us	pcast< Packet4i, Packet4us > (const Packet4i &a)
template<>
EIGEN_STRONG_INLINE Packet4us	pcast< Packet2i, Packet4us > (const Packet2i &a, const Packet2i &b)
template<>
EIGEN_STRONG_INLINE Packet8c	pcast< Packet4i, Packet8c > (const Packet4i &a, const Packet4i &b)
template<>
EIGEN_STRONG_INLINE Packet8c	pcast< Packet2i, Packet8c > (const Packet2i &a, const Packet2i &b, const Packet2i &c, const Packet2i &d)
template<>
EIGEN_STRONG_INLINE Packet4c	pcast< Packet4i, Packet4c > (const Packet4i &a)
template<>
EIGEN_STRONG_INLINE Packet8uc	pcast< Packet4i, Packet8uc > (const Packet4i &a, const Packet4i &b)
template<>
EIGEN_STRONG_INLINE Packet8uc	pcast< Packet2i, Packet8uc > (const Packet2i &a, const Packet2i &b, const Packet2i &c, const Packet2i &d)
template<>
EIGEN_STRONG_INLINE Packet4uc	pcast< Packet4i, Packet4uc > (const Packet4i &a)
template<>
EIGEN_STRONG_INLINE Packet2f	pcast< Packet2ui, Packet2f > (const Packet2ui &a)
template<>
EIGEN_STRONG_INLINE Packet2ul	pcast< Packet2ui, Packet2ul > (const Packet2ui &a)
template<>
EIGEN_STRONG_INLINE Packet2l	pcast< Packet2ui, Packet2l > (const Packet2ui &a)
template<>
EIGEN_STRONG_INLINE Packet4us	pcast< Packet2ui, Packet4us > (const Packet2ui &a, const Packet2ui &b)
template<>
EIGEN_STRONG_INLINE Packet4us	pcast< Packet4ui, Packet4us > (const Packet4ui &a)
template<>
EIGEN_STRONG_INLINE Packet4s	pcast< Packet2ui, Packet4s > (const Packet2ui &a, const Packet2ui &b)
template<>
EIGEN_STRONG_INLINE Packet4s	pcast< Packet4ui, Packet4s > (const Packet4ui &a)
template<>
EIGEN_STRONG_INLINE Packet8uc	pcast< Packet4ui, Packet8uc > (const Packet4ui &a, const Packet4ui &b)
template<>
EIGEN_STRONG_INLINE Packet8uc	pcast< Packet2ui, Packet8uc > (const Packet2ui &a, const Packet2ui &b, const Packet2ui &c, const Packet2ui &d)
template<>
EIGEN_STRONG_INLINE Packet4uc	pcast< Packet4ui, Packet4uc > (const Packet4ui &a)
template<>
EIGEN_STRONG_INLINE Packet8c	pcast< Packet4ui, Packet8c > (const Packet4ui &a, const Packet4ui &b)
template<>
EIGEN_STRONG_INLINE Packet8c	pcast< Packet2ui, Packet8c > (const Packet2ui &a, const Packet2ui &b, const Packet2ui &c, const Packet2ui &d)
template<>
EIGEN_STRONG_INLINE Packet4c	pcast< Packet4ui, Packet4c > (const Packet4ui &a)
template<>
EIGEN_STRONG_INLINE Packet2f	pcast< Packet2l, Packet2f > (const Packet2l &a)
template<>
EIGEN_STRONG_INLINE Packet2i	pcast< Packet2l, Packet2i > (const Packet2l &a)
template<>
EIGEN_STRONG_INLINE Packet2ui	pcast< Packet2l, Packet2ui > (const Packet2l &a)
template<>
EIGEN_STRONG_INLINE Packet4s	pcast< Packet2l, Packet4s > (const Packet2l &a, const Packet2l &b)
template<>
EIGEN_STRONG_INLINE Packet4us	pcast< Packet2l, Packet4us > (const Packet2l &a, const Packet2l &b)
template<>
EIGEN_STRONG_INLINE Packet8c	pcast< Packet2l, Packet8c > (const Packet2l &a, const Packet2l &b, const Packet2l &c, const Packet2l &d)
template<>
EIGEN_STRONG_INLINE Packet4c	pcast< Packet2l, Packet4c > (const Packet2l &a, const Packet2l &b)
template<>
EIGEN_STRONG_INLINE Packet8uc	pcast< Packet2l, Packet8uc > (const Packet2l &a, const Packet2l &b, const Packet2l &c, const Packet2l &d)
template<>
EIGEN_STRONG_INLINE Packet4uc	pcast< Packet2l, Packet4uc > (const Packet2l &a, const Packet2l &b)
template<>
EIGEN_STRONG_INLINE Packet2f	pcast< Packet2ul, Packet2f > (const Packet2ul &a)
template<>
EIGEN_STRONG_INLINE Packet2ui	pcast< Packet2ul, Packet2ui > (const Packet2ul &a)
template<>
EIGEN_STRONG_INLINE Packet2i	pcast< Packet2ul, Packet2i > (const Packet2ul &a)
template<>
EIGEN_STRONG_INLINE Packet4us	pcast< Packet2ul, Packet4us > (const Packet2ul &a, const Packet2ul &b)
template<>
EIGEN_STRONG_INLINE Packet4s	pcast< Packet2ul, Packet4s > (const Packet2ul &a, const Packet2ul &b)
template<>
EIGEN_STRONG_INLINE Packet8uc	pcast< Packet2ul, Packet8uc > (const Packet2ul &a, const Packet2ul &b, const Packet2ul &c, const Packet2ul &d)
template<>
EIGEN_STRONG_INLINE Packet4uc	pcast< Packet2ul, Packet4uc > (const Packet2ul &a, const Packet2ul &b)
template<>
EIGEN_STRONG_INLINE Packet8c	pcast< Packet2ul, Packet8c > (const Packet2ul &a, const Packet2ul &b, const Packet2ul &c, const Packet2ul &d)
template<>
EIGEN_STRONG_INLINE Packet4c	pcast< Packet2ul, Packet4c > (const Packet2ul &a, const Packet2ul &b)
template<>
EIGEN_STRONG_INLINE Packet2cf	pmul (const Packet2cf &a, const Packet2cf &b)
template<>
EIGEN_STRONG_INLINE Packet1cd	pmul (const Packet1cd &a, const Packet1cd &b)
template<>
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet2d	psqrt< Packet2d > (const Packet2d &x)
template<>
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet16b	psqrt< Packet16b > (const Packet16b &x)
EIGEN_ALWAYS_INLINE int64_t	_mm_extract_epi64_0 (const __m128i &a)
EIGEN_ALWAYS_INLINE int64_t	_mm_extract_epi64_1 (const __m128i &a)
template<>
EIGEN_STRONG_INLINE Packet16b	pset1< Packet16b > (const bool &from)
template<>
EIGEN_STRONG_INLINE Packet4f	peven_mask (const Packet4f &)
template<>
EIGEN_STRONG_INLINE Packet2l	peven_mask (const Packet2l &)
template<>
EIGEN_STRONG_INLINE Packet4i	peven_mask (const Packet4i &)
template<>
EIGEN_STRONG_INLINE Packet4ui	peven_mask (const Packet4ui &)
template<>
EIGEN_STRONG_INLINE Packet2d	peven_mask (const Packet2d &)
template<>
EIGEN_STRONG_INLINE Packet2l	pzero (const Packet2l &)
template<>
EIGEN_STRONG_INLINE Packet4i	pzero (const Packet4i &)
template<>
EIGEN_STRONG_INLINE Packet4ui	pzero (const Packet4ui &)
template<>
EIGEN_STRONG_INLINE Packet16b	padd< Packet16b > (const Packet16b &a, const Packet16b &b)
template<typename Packet >
EIGEN_STRONG_INLINE Packet	padds (const Packet &a, const Packet &b)
template<>
EIGEN_STRONG_INLINE Packet4f	padds< Packet4f > (const Packet4f &a, const Packet4f &b)
template<>
EIGEN_STRONG_INLINE Packet2d	padds< Packet2d > (const Packet2d &a, const Packet2d &b)
template<>
EIGEN_STRONG_INLINE Packet16b	psub< Packet16b > (const Packet16b &a, const Packet16b &b)
template<>
EIGEN_STRONG_INLINE Packet16b	pmul< Packet16b > (const Packet16b &a, const Packet16b &b)
template<>
EIGEN_STRONG_INLINE Packet2l	ptrue< Packet2l > (const Packet2l &a)
template<>
EIGEN_STRONG_INLINE Packet4i	ptrue< Packet4i > (const Packet4i &a)
template<>
EIGEN_STRONG_INLINE Packet16b	ptrue< Packet16b > (const Packet16b &)
template<>
EIGEN_STRONG_INLINE Packet4f	ptrue< Packet4f > (const Packet4f &a)
template<>
EIGEN_STRONG_INLINE Packet2d	ptrue< Packet2d > (const Packet2d &a)
template<>
EIGEN_STRONG_INLINE Packet16b	pand< Packet16b > (const Packet16b &a, const Packet16b &b)
template<>
EIGEN_STRONG_INLINE Packet16b	por< Packet16b > (const Packet16b &a, const Packet16b &b)
template<>
EIGEN_STRONG_INLINE Packet16b	pxor< Packet16b > (const Packet16b &a, const Packet16b &b)
template<>
EIGEN_STRONG_INLINE Packet4f	pcmp_lt (const Packet4f &a, const Packet4f &b)
template<>
EIGEN_STRONG_INLINE Packet2d	pcmp_le (const Packet2d &a, const Packet2d &b)
template<>
EIGEN_STRONG_INLINE Packet2d	pcmp_lt (const Packet2d &a, const Packet2d &b)
template<>
EIGEN_STRONG_INLINE Packet2d	pcmp_lt_or_nan (const Packet2d &a, const Packet2d &b)
template<>
EIGEN_STRONG_INLINE Packet2d	pcmp_eq (const Packet2d &a, const Packet2d &b)
template<>
EIGEN_STRONG_INLINE Packet4i	pcmp_le (const Packet4i &a, const Packet4i &b)
template<>
EIGEN_STRONG_INLINE Packet2l	pcmp_lt (const Packet2l &a, const Packet2l &b)
template<>
EIGEN_STRONG_INLINE Packet2l	pcmp_eq (const Packet2l &a, const Packet2l &b)
template<>
EIGEN_STRONG_INLINE Packet2l	pcmp_le (const Packet2l &a, const Packet2l &b)
template<>
EIGEN_STRONG_INLINE Packet16b	pcmp_eq (const Packet16b &a, const Packet16b &b)
template<>
EIGEN_STRONG_INLINE Packet4ui	pcmp_eq (const Packet4ui &a, const Packet4ui &b)
template<>
EIGEN_STRONG_INLINE Packet4ui	pcmp_lt (const Packet4ui &a, const Packet4ui &b)
template<>
EIGEN_STRONG_INLINE Packet4ui	pcmp_le (const Packet4ui &a, const Packet4ui &b)
template<typename Packet , typename Op >
EIGEN_STRONG_INLINE Packet	pminmax_propagate_numbers (const Packet &a, const Packet &b, Op op)
template<typename Packet , typename Op >
EIGEN_STRONG_INLINE Packet	pminmax_propagate_nan (const Packet &a, const Packet &b, Op op)
template<>
EIGEN_STRONG_INLINE Packet4f	pmin< PropagateNumbers, Packet4f > (const Packet4f &a, const Packet4f &b)
template<>
EIGEN_STRONG_INLINE Packet2d	pmin< PropagateNumbers, Packet2d > (const Packet2d &a, const Packet2d &b)
template<>
EIGEN_STRONG_INLINE Packet4f	pmax< PropagateNumbers, Packet4f > (const Packet4f &a, const Packet4f &b)
template<>
EIGEN_STRONG_INLINE Packet2d	pmax< PropagateNumbers, Packet2d > (const Packet2d &a, const Packet2d &b)
template<>
EIGEN_STRONG_INLINE Packet4i	psignbit (const Packet4i &a)
template<>
EIGEN_STRONG_INLINE Packet4ui	psignbit (const Packet4ui &a)
template<>
EIGEN_STRONG_INLINE Packet2l	psignbit (const Packet2l &a)
template<>
EIGEN_STRONG_INLINE Packet16b	pload< Packet16b > (const bool *from)
template<>
EIGEN_STRONG_INLINE Packet16b	ploadu< Packet16b > (const bool *from)
template<typename Packet >
EIGEN_STRONG_INLINE Packet	ploadl (const typename unpacket_traits< Packet >::type *from)
template<>
EIGEN_STRONG_INLINE Packet4f	ploadl< Packet4f > (const float *from)
template<>
EIGEN_STRONG_INLINE Packet2d	ploadl< Packet2d > (const double *from)
template<typename Packet >
EIGEN_STRONG_INLINE Packet	ploads (const typename unpacket_traits< Packet >::type *from)
template<>
EIGEN_STRONG_INLINE Packet4f	ploads< Packet4f > (const float *from)
template<>
EIGEN_STRONG_INLINE Packet2d	ploads< Packet2d > (const double *from)
template<>
EIGEN_STRONG_INLINE Packet16b	ploaddup< Packet16b > (const bool *from)
template<>
EIGEN_STRONG_INLINE Packet16b	ploadquad< Packet16b > (const bool *from)
template<>
EIGEN_STRONG_INLINE void	pstore< bool > (bool *to, const Packet16b &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< bool > (bool *to, const Packet16b &from)
template<typename Scalar , typename Packet >
EIGEN_STRONG_INLINE void	pstorel (Scalar *to, const Packet &from)
template<>
EIGEN_STRONG_INLINE void	pstorel (float *to, const Packet4f &from)
template<>
EIGEN_STRONG_INLINE void	pstorel (double *to, const Packet2d &from)
template<typename Scalar , typename Packet >
EIGEN_STRONG_INLINE void	pstores (Scalar *to, const Packet &from)
template<>
EIGEN_STRONG_INLINE void	pstores (float *to, const Packet4f &from)
template<>
EIGEN_STRONG_INLINE void	pstores (double *to, const Packet2d &from)
template<>
EIGEN_STRONG_INLINE Packet16b	preverse (const Packet16b &a)
template<>
EIGEN_STRONG_INLINE bool	pfirst< Packet16b > (const Packet16b &a)
template<>
EIGEN_STRONG_INLINE Packet16b	pgather< bool, Packet16b > (const bool *from, Index stride)
template<>
EIGEN_STRONG_INLINE void	pscatter< bool, Packet16b > (bool *to, const Packet16b &from, Index stride)
template<>
EIGEN_STRONG_INLINE void	pstore1< Packet4f > (float *to, const float &a)
template<>
EIGEN_STRONG_INLINE void	pstore1< Packet2d > (double *to, const double &a)
template<>
EIGEN_STRONG_INLINE Packet2d	pfrexp_generic_get_biased_exponent (const Packet2d &a)
template<>
EIGEN_STRONG_INLINE Packet2d	pldexp_fast< Packet2d > (const Packet2d &a, const Packet2d &exponent)
template<>
EIGEN_STRONG_INLINE void	pbroadcast4< Packet2d > (const double *a, Packet2d &a0, Packet2d &a1, Packet2d &a2, Packet2d &a3)
EIGEN_STRONG_INLINE void	punpackp (Packet4f *vecs)
template<>
EIGEN_STRONG_INLINE bool	predux< Packet16b > (const Packet16b &a)
template<>
EIGEN_STRONG_INLINE bool	predux_mul< Packet16b > (const Packet16b &a)
template<>
EIGEN_STRONG_INLINE bool	predux_any (const Packet2d &x)
template<>
EIGEN_STRONG_INLINE bool	predux_any (const Packet2l &x)
template<>
EIGEN_STRONG_INLINE bool	predux_any (const Packet4i &x)
template<>
EIGEN_STRONG_INLINE bool	predux_any (const Packet4ui &x)
EIGEN_STRONG_INLINE void	ptranspose (PacketBlock< Packet16b, 4 > &kernel)
EIGEN_STRONG_INLINE void	ptranspose (PacketBlock< Packet16b, 16 > &kernel)
EIGEN_STRONG_INLINE __m128i	sse_blend_mask (const Selector< 2 > &ifPacket)
EIGEN_STRONG_INLINE __m128i	sse_blend_mask (const Selector< 4 > &ifPacket)
template<>
EIGEN_STRONG_INLINE Packet2l	pblend (const Selector< 2 > &ifPacket, const Packet2l &thenPacket, const Packet2l &elsePacket)
template<>
EIGEN_STRONG_INLINE Packet4ui	pblend (const Selector< 4 > &ifPacket, const Packet4ui &thenPacket, const Packet4ui &elsePacket)
template<>
EIGEN_STRONG_INLINE Packet16b	pcast< Packet4f, Packet16b > (const Packet4f &a, const Packet4f &b, const Packet4f &c, const Packet4f &d)
template<>
EIGEN_STRONG_INLINE Packet4f	pcast< Packet16b, Packet4f > (const Packet16b &a)
template<>
EIGEN_STRONG_INLINE Packet2d	preinterpret< Packet2d, Packet4f > (const Packet4f &a)
template<>
EIGEN_STRONG_INLINE Packet4f	preinterpret< Packet4f, Packet2d > (const Packet2d &a)
template<>
EIGEN_STRONG_INLINE PacketXf	pexp< PacketXf > (const PacketXf &x)
template<>
EIGEN_STRONG_INLINE PacketXf	plog< PacketXf > (const PacketXf &x)
template<>
EIGEN_STRONG_INLINE PacketXf	psin< PacketXf > (const PacketXf &x)
template<>
EIGEN_STRONG_INLINE PacketXf	pcos< PacketXf > (const PacketXf &x)
template<>
EIGEN_STRONG_INLINE PacketXf	ptanh< PacketXf > (const PacketXf &x)
template<>
EIGEN_STRONG_INLINE void	prefetch< numext::int32_t > (const numext::int32_t *addr)
template<>
EIGEN_STRONG_INLINE PacketXi	pset1< PacketXi > (const numext::int32_t &from)
template<>
EIGEN_STRONG_INLINE PacketXi	plset< PacketXi > (const numext::int32_t &a)
template<>
EIGEN_STRONG_INLINE PacketXi	padd< PacketXi > (const PacketXi &a, const PacketXi &b)
template<>
EIGEN_STRONG_INLINE PacketXi	psub< PacketXi > (const PacketXi &a, const PacketXi &b)
template<>
EIGEN_STRONG_INLINE PacketXi	pnegate (const PacketXi &a)
template<>
EIGEN_STRONG_INLINE PacketXi	pconj (const PacketXi &a)
template<>
EIGEN_STRONG_INLINE PacketXi	pmul< PacketXi > (const PacketXi &a, const PacketXi &b)
template<>
EIGEN_STRONG_INLINE PacketXi	pdiv< PacketXi > (const PacketXi &a, const PacketXi &b)
template<>
EIGEN_STRONG_INLINE PacketXi	pmadd (const PacketXi &a, const PacketXi &b, const PacketXi &c)
template<>
EIGEN_STRONG_INLINE PacketXi	pmin< PacketXi > (const PacketXi &a, const PacketXi &b)
template<>
EIGEN_STRONG_INLINE PacketXi	pmax< PacketXi > (const PacketXi &a, const PacketXi &b)
template<>
EIGEN_STRONG_INLINE PacketXi	pcmp_le< PacketXi > (const PacketXi &a, const PacketXi &b)
template<>
EIGEN_STRONG_INLINE PacketXi	pcmp_lt< PacketXi > (const PacketXi &a, const PacketXi &b)
template<>
EIGEN_STRONG_INLINE PacketXi	pcmp_eq< PacketXi > (const PacketXi &a, const PacketXi &b)
template<>
EIGEN_STRONG_INLINE PacketXi	ptrue< PacketXi > (const PacketXi &)
template<>
EIGEN_STRONG_INLINE PacketXi	pzero< PacketXi > (const PacketXi &)
template<>
EIGEN_STRONG_INLINE PacketXi	pand< PacketXi > (const PacketXi &a, const PacketXi &b)
template<>
EIGEN_STRONG_INLINE PacketXi	por< PacketXi > (const PacketXi &a, const PacketXi &b)
template<>
EIGEN_STRONG_INLINE PacketXi	pxor< PacketXi > (const PacketXi &a, const PacketXi &b)
template<>
EIGEN_STRONG_INLINE PacketXi	pandnot< PacketXi > (const PacketXi &a, const PacketXi &b)
template<int N>
EIGEN_STRONG_INLINE PacketXi	parithmetic_shift_right (PacketXi a)
template<int N>
EIGEN_STRONG_INLINE PacketXi	plogical_shift_right (PacketXi a)
template<int N>
EIGEN_STRONG_INLINE PacketXi	plogical_shift_left (PacketXi a)
template<>
EIGEN_STRONG_INLINE PacketXi	pload< PacketXi > (const numext::int32_t *from)
template<>
EIGEN_STRONG_INLINE PacketXi	ploadu< PacketXi > (const numext::int32_t *from)
template<>
EIGEN_STRONG_INLINE PacketXi	ploaddup< PacketXi > (const numext::int32_t *from)
template<>
EIGEN_STRONG_INLINE PacketXi	ploadquad< PacketXi > (const numext::int32_t *from)
template<>
EIGEN_STRONG_INLINE void	pstore< numext::int32_t > (numext::int32_t *to, const PacketXi &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< numext::int32_t > (numext::int32_t *to, const PacketXi &from)
template<>
EIGEN_DEVICE_FUNC PacketXi	pgather< numext::int32_t, PacketXi > (const numext::int32_t *from, Index stride)
template<>
EIGEN_DEVICE_FUNC void	pscatter< numext::int32_t, PacketXi > (numext::int32_t *to, const PacketXi &from, Index stride)
template<>
EIGEN_STRONG_INLINE numext::int32_t	pfirst< PacketXi > (const PacketXi &a)
template<>
EIGEN_STRONG_INLINE PacketXi	preverse (const PacketXi &a)
template<>
EIGEN_STRONG_INLINE PacketXi	pabs (const PacketXi &a)
template<>
EIGEN_STRONG_INLINE numext::int32_t	predux< PacketXi > (const PacketXi &a)
template<>
EIGEN_STRONG_INLINE numext::int32_t	predux_mul< PacketXi > (const PacketXi &a)
template<>
EIGEN_STRONG_INLINE numext::int32_t	predux_min< PacketXi > (const PacketXi &a)
template<>
EIGEN_STRONG_INLINE numext::int32_t	predux_max< PacketXi > (const PacketXi &a)
template<int N>
EIGEN_DEVICE_FUNC void	ptranspose (PacketBlock< PacketXi, N > &kernel)
template<>
EIGEN_STRONG_INLINE PacketXf	pset1< PacketXf > (const float &from)
template<>
EIGEN_STRONG_INLINE PacketXf	pset1frombits< PacketXf > (numext::uint32_t from)
template<>
EIGEN_STRONG_INLINE PacketXf	plset< PacketXf > (const float &a)
template<>
EIGEN_STRONG_INLINE PacketXf	padd< PacketXf > (const PacketXf &a, const PacketXf &b)
template<>
EIGEN_STRONG_INLINE PacketXf	psub< PacketXf > (const PacketXf &a, const PacketXf &b)
template<>
EIGEN_STRONG_INLINE PacketXf	pnegate (const PacketXf &a)
template<>
EIGEN_STRONG_INLINE PacketXf	pconj (const PacketXf &a)
template<>
EIGEN_STRONG_INLINE PacketXf	pmul< PacketXf > (const PacketXf &a, const PacketXf &b)
template<>
EIGEN_STRONG_INLINE PacketXf	pdiv< PacketXf > (const PacketXf &a, const PacketXf &b)
template<>
EIGEN_STRONG_INLINE PacketXf	pmadd (const PacketXf &a, const PacketXf &b, const PacketXf &c)
template<>
EIGEN_STRONG_INLINE PacketXf	pmin< PacketXf > (const PacketXf &a, const PacketXf &b)
template<>
EIGEN_STRONG_INLINE PacketXf	pmin< PropagateNaN, PacketXf > (const PacketXf &a, const PacketXf &b)
template<>
EIGEN_STRONG_INLINE PacketXf	pmin< PropagateNumbers, PacketXf > (const PacketXf &a, const PacketXf &b)
template<>
EIGEN_STRONG_INLINE PacketXf	pmax< PacketXf > (const PacketXf &a, const PacketXf &b)
template<>
EIGEN_STRONG_INLINE PacketXf	pmax< PropagateNaN, PacketXf > (const PacketXf &a, const PacketXf &b)
template<>
EIGEN_STRONG_INLINE PacketXf	pmax< PropagateNumbers, PacketXf > (const PacketXf &a, const PacketXf &b)
template<>
EIGEN_STRONG_INLINE PacketXf	pcmp_le< PacketXf > (const PacketXf &a, const PacketXf &b)
template<>
EIGEN_STRONG_INLINE PacketXf	pcmp_lt< PacketXf > (const PacketXf &a, const PacketXf &b)
template<>
EIGEN_STRONG_INLINE PacketXf	pcmp_eq< PacketXf > (const PacketXf &a, const PacketXf &b)
template<>
EIGEN_STRONG_INLINE PacketXf	pcmp_lt_or_nan< PacketXf > (const PacketXf &a, const PacketXf &b)
template<>
EIGEN_STRONG_INLINE PacketXf	pfloor< PacketXf > (const PacketXf &a)
template<>
EIGEN_STRONG_INLINE PacketXf	ptrue< PacketXf > (const PacketXf &)
template<>
EIGEN_STRONG_INLINE PacketXf	pand< PacketXf > (const PacketXf &a, const PacketXf &b)
template<>
EIGEN_STRONG_INLINE PacketXf	por< PacketXf > (const PacketXf &a, const PacketXf &b)
template<>
EIGEN_STRONG_INLINE PacketXf	pxor< PacketXf > (const PacketXf &a, const PacketXf &b)
template<>
EIGEN_STRONG_INLINE PacketXf	pandnot< PacketXf > (const PacketXf &a, const PacketXf &b)
template<>
EIGEN_STRONG_INLINE PacketXf	pload< PacketXf > (const float *from)
template<>
EIGEN_STRONG_INLINE PacketXf	ploadu< PacketXf > (const float *from)
template<>
EIGEN_STRONG_INLINE PacketXf	ploaddup< PacketXf > (const float *from)
template<>
EIGEN_STRONG_INLINE PacketXf	ploadquad< PacketXf > (const float *from)
template<>
EIGEN_STRONG_INLINE void	pstore< float > (float *to, const PacketXf &from)
template<>
EIGEN_STRONG_INLINE void	pstoreu< float > (float *to, const PacketXf &from)
template<>
EIGEN_DEVICE_FUNC PacketXf	pgather< float, PacketXf > (const float *from, Index stride)
template<>
EIGEN_DEVICE_FUNC void	pscatter< float, PacketXf > (float *to, const PacketXf &from, Index stride)
template<>
EIGEN_STRONG_INLINE float	pfirst< PacketXf > (const PacketXf &a)
template<>
EIGEN_STRONG_INLINE PacketXf	preverse (const PacketXf &a)
template<>
EIGEN_STRONG_INLINE PacketXf	pabs (const PacketXf &a)
template<>
EIGEN_STRONG_INLINE PacketXf	pfrexp< PacketXf > (const PacketXf &a, PacketXf &exponent)
template<>
EIGEN_STRONG_INLINE float	predux< PacketXf > (const PacketXf &a)
template<>
EIGEN_STRONG_INLINE float	predux_mul< PacketXf > (const PacketXf &a)
template<>
EIGEN_STRONG_INLINE float	predux_min< PacketXf > (const PacketXf &a)
template<>
EIGEN_STRONG_INLINE float	predux_max< PacketXf > (const PacketXf &a)
template<int N>
EIGEN_DEVICE_FUNC void	ptranspose (PacketBlock< PacketXf, N > &kernel)
template<>
EIGEN_STRONG_INLINE PacketXf	pldexp< PacketXf > (const PacketXf &a, const PacketXf &exponent)
template<>
EIGEN_STRONG_INLINE PacketXf	psqrt< PacketXf > (const PacketXf &a)
template<>
EIGEN_STRONG_INLINE PacketXf	pcast< PacketXi, PacketXf > (const PacketXi &a)
template<>
EIGEN_STRONG_INLINE PacketXi	pcast< PacketXf, PacketXi > (const PacketXf &a)
template<>
EIGEN_STRONG_INLINE PacketXf	preinterpret< PacketXf, PacketXi > (const PacketXi &a)
template<>
EIGEN_STRONG_INLINE PacketXi	preinterpret< PacketXi, PacketXf > (const PacketXf &a)
template<>
EIGEN_DEVICE_FUNC Packet1cd	pgather< std::complex< double >, Packet1cd > (const std::complex< double > *from, Index stride EIGEN_UNUSED)
template<>
EIGEN_DEVICE_FUNC void	pscatter< std::complex< double >, Packet1cd > (std::complex< double > *to, const Packet1cd &from, Index stride EIGEN_UNUSED)
template<>
EIGEN_STRONG_INLINE Packet1cd	plog< Packet1cd > (const Packet1cd &a, const Packet1cd &b)
template<>
EIGEN_STRONG_INLINE Packet2cf	plog< Packet2cf > (const Packet2cf &a, const Packet2cf &b)
template<>
EIGEN_STRONG_INLINE Packet2cf	pexp< Packet2cf > (const Packet2cf &a, const Packet2cf &b)
static	EIGEN_DECLARE_CONST_Packet4f (1, 1.0f)
static	EIGEN_DECLARE_CONST_Packet4f (half, 0.5f)
static	EIGEN_DECLARE_CONST_Packet4i (0x7f, 0x7f)
static	EIGEN_DECLARE_CONST_Packet4i (23, 23)
static	EIGEN_DECLARE_CONST_Packet4f_FROM_INT (inv_mant_mask, ~0x7f800000)
static	EIGEN_DECLARE_CONST_Packet4f_FROM_INT (min_norm_pos, 0x00800000)
static	EIGEN_DECLARE_CONST_Packet4f_FROM_INT (minus_inf, 0xff800000)
static	EIGEN_DECLARE_CONST_Packet4f_FROM_INT (minus_nan, 0xffffffff)
static	EIGEN_DECLARE_CONST_Packet4f (cephes_SQRTHF, 0.707106781186547524f)
static	EIGEN_DECLARE_CONST_Packet4f (cephes_log_p0, 7.0376836292E-2f)
static	EIGEN_DECLARE_CONST_Packet4f (cephes_log_p1, -1.1514610310E-1f)
static	EIGEN_DECLARE_CONST_Packet4f (cephes_log_p2, 1.1676998740E-1f)
static	EIGEN_DECLARE_CONST_Packet4f (cephes_log_p3, -1.2420140846E-1f)
static	EIGEN_DECLARE_CONST_Packet4f (cephes_log_p4,+1.4249322787E-1f)
static	EIGEN_DECLARE_CONST_Packet4f (cephes_log_p5, -1.6668057665E-1f)
static	EIGEN_DECLARE_CONST_Packet4f (cephes_log_p6,+2.0000714765E-1f)
static	EIGEN_DECLARE_CONST_Packet4f (cephes_log_p7, -2.4999993993E-1f)
static	EIGEN_DECLARE_CONST_Packet4f (cephes_log_p8,+3.3333331174E-1f)
static	EIGEN_DECLARE_CONST_Packet4f (cephes_log_q1, -2.12194440e-4f)
static	EIGEN_DECLARE_CONST_Packet4f (cephes_log_q2, 0.693359375f)
static	EIGEN_DECLARE_CONST_Packet4f (exp_hi, 88.3762626647950f)
static	EIGEN_DECLARE_CONST_Packet4f (exp_lo, -88.3762626647949f)
static	EIGEN_DECLARE_CONST_Packet4f (cephes_LOG2EF, 1.44269504088896341f)
static	EIGEN_DECLARE_CONST_Packet4f (cephes_exp_C1, 0.693359375f)
static	EIGEN_DECLARE_CONST_Packet4f (cephes_exp_C2, -2.12194440e-4f)
static	EIGEN_DECLARE_CONST_Packet4f (cephes_exp_p0, 1.9875691500E-4f)
static	EIGEN_DECLARE_CONST_Packet4f (cephes_exp_p1, 1.3981999507E-3f)
static	EIGEN_DECLARE_CONST_Packet4f (cephes_exp_p2, 8.3334519073E-3f)
static	EIGEN_DECLARE_CONST_Packet4f (cephes_exp_p3, 4.1665795894E-2f)
static	EIGEN_DECLARE_CONST_Packet4f (cephes_exp_p4, 1.6666665459E-1f)
static	EIGEN_DECLARE_CONST_Packet4f (cephes_exp_p5, 5.0000001201E-1f)
static	EIGEN_DECLARE_CONST_Packet2d (1, 1.0)
static	EIGEN_DECLARE_CONST_Packet2d (2, 2.0)
static	EIGEN_DECLARE_CONST_Packet2d (half, 0.5)
static	EIGEN_DECLARE_CONST_Packet2d (exp_hi, 709.437)
static	EIGEN_DECLARE_CONST_Packet2d (exp_lo, -709.436139303)
static	EIGEN_DECLARE_CONST_Packet2d (cephes_LOG2EF, 1.4426950408889634073599)
static	EIGEN_DECLARE_CONST_Packet2d (cephes_exp_p0, 1.26177193074810590878e-4)
static	EIGEN_DECLARE_CONST_Packet2d (cephes_exp_p1, 3.02994407707441961300e-2)
static	EIGEN_DECLARE_CONST_Packet2d (cephes_exp_p2, 9.99999999999999999910e-1)
static	EIGEN_DECLARE_CONST_Packet2d (cephes_exp_q0, 3.00198505138664455042e-6)
static	EIGEN_DECLARE_CONST_Packet2d (cephes_exp_q1, 2.52448340349684104192e-3)
static	EIGEN_DECLARE_CONST_Packet2d (cephes_exp_q2, 2.27265548208155028766e-1)
static	EIGEN_DECLARE_CONST_Packet2d (cephes_exp_q3, 2.00000000000000000009e0)
static	EIGEN_DECLARE_CONST_Packet2d (cephes_exp_C1, 0.693145751953125)
static	EIGEN_DECLARE_CONST_Packet2d (cephes_exp_C2, 1.42860682030941723212e-6)
template<>
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet2d	prsqrt< Packet2d > (const Packet2d &x)
template<>
EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet4f	prsqrt< Packet4f > (const Packet4f &x)
static	EIGEN_DECLARE_CONST_FAST_Packet4i (ZERO, 0)
static	EIGEN_DECLARE_CONST_FAST_Packet4i (ONE, 1)
static	EIGEN_DECLARE_CONST_FAST_Packet2d (ZERO, 0)
static	EIGEN_DECLARE_CONST_FAST_Packet2l (ZERO, 0)
static	EIGEN_DECLARE_CONST_FAST_Packet2l (ONE, 1)
static	EIGEN_DECLARE_CONST_FAST_Packet4f (ZERO, 0)
static	EIGEN_DECLARE_CONST_FAST_Packet4i (MINUS1, -1)
template<>
EIGEN_STRONG_INLINE Packet4i	pabs< Packet4i > (const Packet4i &a)
template<>
EIGEN_STRONG_INLINE Packet2d	pabs< Packet2d > (const Packet2d &a)
template<int element>
EIGEN_STRONG_INLINE Packet4f	vec_splat_packet4f (const Packet4f &from)
template<>
EIGEN_STRONG_INLINE Packet4f	pabs< Packet4f > (const Packet4f &a)
template<typename DstXprType , typename SrcXprType , typename Functor >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	resize_if_allowed (DstXprType &dst, const SrcXprType &src, const Functor &)
template<typename DstXprType , typename SrcXprType , typename T1 , typename T2 >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	resize_if_allowed (DstXprType &dst, const SrcXprType &src, const internal::assign_op< T1, T2 > &)
template<typename DstXprType , typename SrcXprType , typename Functor >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void	call_dense_assignment_loop (DstXprType &dst, const SrcXprType &src, const Functor &func)
template<typename DstXprType , typename SrcXprType >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	call_dense_assignment_loop (DstXprType &dst, const SrcXprType &src)
template<typename Dst , typename Src >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	call_assignment (Dst &dst, const Src &src)
template<typename Dst , typename Src >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	call_assignment (const Dst &dst, const Src &src)
template<typename Dst , typename Src , typename Func >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void	call_assignment (Dst &dst, const Src &src, const Func &func, std::enable_if_t< evaluator_assume_aliasing< Src >::value, void * >=0)
template<typename Dst , typename Src , typename Func >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	call_assignment (Dst &dst, const Src &src, const Func &func, std::enable_if_t<!evaluator_assume_aliasing< Src >::value, void * >=0)
template<typename Dst , template< typename > class StorageBase, typename Src , typename Func >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void	call_assignment (NoAlias< Dst, StorageBase > &dst, const Src &src, const Func &func)
template<typename Dst , typename Src , typename Func >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void	call_assignment_no_alias (Dst &dst, const Src &src, const Func &func)
template<typename Dst , typename Src , typename Func >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	call_restricted_packet_assignment_no_alias (Dst &dst, const Src &src, const Func &func)
template<typename Dst , typename Src >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void	call_assignment_no_alias (Dst &dst, const Src &src)
template<typename Dst , typename Src , typename Func >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void	call_assignment_no_alias_no_transpose (Dst &dst, const Src &src, const Func &func)
template<typename Dst , typename Src >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void	call_assignment_no_alias_no_transpose (Dst &dst, const Src &src)
template<typename Dst , typename Src >
EIGEN_DEVICE_FUNC void	check_for_aliasing (const Dst &dst, const Src &src)
template<typename Decomposition >
Decomposition::RealScalar	rcond_invmatrix_L1_norm_estimate (const Decomposition &dec)
template<typename Decomposition >
Decomposition::RealScalar	rcond_estimate_helper (typename Decomposition::RealScalar matrix_norm, const Decomposition &dec)
	Reciprocal condition number estimator. More...
template<int Alignment, typename Derived >
static Index	first_aligned (const DenseBase< Derived > &m)
template<typename Derived >
static Index	first_default_aligned (const DenseBase< Derived > &m)
template<typename T , int Size, int Options, int Alignment>
EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE void	swap_plain_array (plain_array< T, Size, Options, Alignment > &a, plain_array< T, Size, Options, Alignment > &b, Index a_size, Index b_size)
template<typename Dst , typename Src , typename Func , typename Device >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void	call_assignment_no_alias (DeviceWrapper< Dst, Device > dst, const Src &src, const Func &func)
template<typename DstXprType , typename SrcXprType , typename Functor , typename Device >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void	call_dense_assignment_loop (DstXprType &dst, const SrcXprType &src, const Functor &func, Device &device)
template<typename T >
constexpr int	exponent_digits ()
template<typename Target , typename Packet >
EIGEN_DEVICE_FUNC Target	preinterpret (const Packet &a)
template<typename SrcPacket , typename TgtPacket >
EIGEN_DEVICE_FUNC TgtPacket	pcast (const SrcPacket &a)
template<typename SrcPacket , typename TgtPacket >
EIGEN_DEVICE_FUNC TgtPacket	pcast (const SrcPacket &a, const SrcPacket &b)
template<typename SrcPacket , typename TgtPacket >
EIGEN_DEVICE_FUNC TgtPacket	pcast (const SrcPacket &a, const SrcPacket &b, const SrcPacket &c, const SrcPacket &d)
template<typename SrcPacket , typename TgtPacket >
EIGEN_DEVICE_FUNC TgtPacket	pcast (const SrcPacket &a, const SrcPacket &b, const SrcPacket &c, const SrcPacket &d, const SrcPacket &e, const SrcPacket &f, const SrcPacket &g, const SrcPacket &h)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	padd (const Packet &a, const Packet &b)
template<>
EIGEN_DEVICE_FUNC bool	padd (const bool &a, const bool &b)
template<typename Packet >
EIGEN_DEVICE_FUNC std::enable_if_t< unpacket_traits< Packet >::masked_fpops_available, Packet >	padd (const Packet &a, const Packet &b, typename unpacket_traits< Packet >::mask_t umask)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	psub (const Packet &a, const Packet &b)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	pnegate (const Packet &a)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	pconj (const Packet &a)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	pmul (const Packet &a, const Packet &b)
template<>
EIGEN_DEVICE_FUNC bool	pmul (const bool &a, const bool &b)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	pdiv (const Packet &a, const Packet &b)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	ptrue (const Packet &a)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	pzero (const Packet &a)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	pcmp_le (const Packet &a, const Packet &b)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	pcmp_lt (const Packet &a, const Packet &b)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	pcmp_eq (const Packet &a, const Packet &b)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	pcmp_lt_or_nan (const Packet &a, const Packet &b)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	pand (const Packet &a, const Packet &b)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	por (const Packet &a, const Packet &b)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	pxor (const Packet &a, const Packet &b)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	pnot (const Packet &a)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	pandnot (const Packet &a, const Packet &b)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	pselect (const Packet &mask, const Packet &a, const Packet &b)
template<>
EIGEN_DEVICE_FUNC bool	pselect< bool > (const bool &cond, const bool &a, const bool &b)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	pmin (const Packet &a, const Packet &b)
template<int NaNPropagation, typename Packet >
EIGEN_DEVICE_FUNC Packet	pmin (const Packet &a, const Packet &b)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	pmax (const Packet &a, const Packet &b)
template<int NaNPropagation, typename Packet >
EIGEN_DEVICE_FUNC Packet	pmax (const Packet &a, const Packet &b)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	pabs (const Packet &a)
template<>
EIGEN_DEVICE_FUNC unsigned int	pabs (const unsigned int &a)
template<>
EIGEN_DEVICE_FUNC unsigned long	pabs (const unsigned long &a)
template<>
EIGEN_DEVICE_FUNC unsigned long long	pabs (const unsigned long long &a)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	paddsub (const Packet &a, const Packet &b)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	parg (const Packet &a)
template<int N, typename T >
EIGEN_DEVICE_FUNC T	parithmetic_shift_right (const T &a)
template<int N, typename T >
EIGEN_DEVICE_FUNC T	plogical_shift_right (const T &a)
template<int N, typename T >
EIGEN_DEVICE_FUNC T	plogical_shift_left (const T &a)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	pfrexp (const Packet &a, Packet &exponent)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	pldexp (const Packet &a, const Packet &exponent)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	pabsdiff (const Packet &a, const Packet &b)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	pload (const typename unpacket_traits< Packet >::type *from)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	pload_partial (const typename unpacket_traits< Packet >::type *from, const Index n, const Index offset=0)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	ploadu (const typename unpacket_traits< Packet >::type *from)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	ploadu_partial (const typename unpacket_traits< Packet >::type *from, const Index n, const Index offset=0)
template<typename Packet >
EIGEN_DEVICE_FUNC std::enable_if_t< unpacket_traits< Packet >::masked_load_available, Packet >	ploadu (const typename unpacket_traits< Packet >::type *from, typename unpacket_traits< Packet >::mask_t umask)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	pset1 (const typename unpacket_traits< Packet >::type &a)
template<typename Packet , typename BitsType >
EIGEN_DEVICE_FUNC Packet	pset1frombits (BitsType a)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	pload1 (const typename unpacket_traits< Packet >::type *a)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet	ploaddup (const typename unpacket_traits< Packet >::type *from)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	ploadquad (const typename unpacket_traits< Packet >::type *from)
template<typename Packet >
EIGEN_DEVICE_FUNC void	pbroadcast4 (const typename unpacket_traits< Packet >::type *a, Packet &a0, Packet &a1, Packet &a2, Packet &a3)
template<typename Packet >
EIGEN_DEVICE_FUNC void	pbroadcast2 (const typename unpacket_traits< Packet >::type *a, Packet &a0, Packet &a1)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet	plset (const typename unpacket_traits< Packet >::type &a)
	Returns a packet with coefficients (a,a+1,...,a+packet_size-1). More...
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	peven_mask (const Packet &)
template<typename Scalar , typename Packet >
EIGEN_DEVICE_FUNC void	pstore (Scalar *to, const Packet &from)
template<typename Scalar , typename Packet >
EIGEN_DEVICE_FUNC void	pstore_partial (Scalar *to, const Packet &from, const Index n, const Index offset=0)
template<typename Scalar , typename Packet >
EIGEN_DEVICE_FUNC void	pstoreu (Scalar *to, const Packet &from)
template<typename Scalar , typename Packet >
EIGEN_DEVICE_FUNC void	pstoreu_partial (Scalar *to, const Packet &from, const Index n, const Index offset=0)
template<typename Scalar , typename Packet >
EIGEN_DEVICE_FUNC std::enable_if_t< unpacket_traits< Packet >::masked_store_available, void >	pstoreu (Scalar *to, const Packet &from, typename unpacket_traits< Packet >::mask_t umask)
template<typename Scalar , typename Packet >
EIGEN_DEVICE_FUNC Packet	pgather (const Scalar *from, Index)
template<typename Scalar , typename Packet >
EIGEN_DEVICE_FUNC Packet	pgather_partial (const Scalar *from, Index stride, const Index n)
template<typename Scalar , typename Packet >
EIGEN_DEVICE_FUNC void	pscatter (Scalar *to, const Packet &from, Index)
template<typename Scalar , typename Packet >
EIGEN_DEVICE_FUNC void	pscatter_partial (Scalar *to, const Packet &from, Index stride, const Index n)
template<typename Scalar >
EIGEN_DEVICE_FUNC void	prefetch (const Scalar *addr)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	preverse (const Packet &a)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	pcplxflip (const Packet &a)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	pisnan (const Packet &a)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	pisinf (const Packet &a)
template<typename Packet >
EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	psin (const Packet &a)
template<typename Packet >
EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	pcos (const Packet &a)
template<typename Packet >
EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	ptan (const Packet &a)
template<typename Packet >
EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	pasin (const Packet &a)
template<typename Packet >
EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	pacos (const Packet &a)
template<typename Packet >
EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	psinh (const Packet &a)
template<typename Packet >
EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	pcosh (const Packet &a)
template<typename Packet >
EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	patan (const Packet &a)
template<typename Packet >
EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	ptanh (const Packet &a)
template<typename Packet >
EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	patanh (const Packet &a)
template<typename Packet >
EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	pexp (const Packet &a)
template<typename Packet >
EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	pexp2 (const Packet &a)
template<typename Packet >
EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	pexpm1 (const Packet &a)
template<typename Packet >
EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	plog (const Packet &a)
template<typename Packet >
EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	plog1p (const Packet &a)
template<typename Packet >
EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	plog10 (const Packet &a)
template<typename Packet >
EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	plog2 (const Packet &a)
template<typename Packet >
EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	psqrt (const Packet &a)
template<typename Packet >
EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	pcbrt (const Packet &a)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet	pround (const Packet &a)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet	pfloor (const Packet &a)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet	print (const Packet &a)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet	pceil (const Packet &a)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet	ptrunc (const Packet &a)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	psign (const Packet &a)
template<>
EIGEN_DEVICE_FUNC bool	psign (const bool &a)
template<typename Packet >
EIGEN_DEVICE_FUNC unpacket_traits< Packet >::type	pfirst (const Packet &a)
template<typename Packet >
EIGEN_DEVICE_FUNC std::conditional_t<(unpacket_traits< Packet >::size % 8)==0, typename unpacket_traits< Packet >::half, Packet >	predux_half_dowto4 (const Packet &a)
template<typename Packet , typename Op >
EIGEN_DEVICE_FUNC unpacket_traits< Packet >::type	predux_helper (const Packet &a, Op op)
template<typename Packet >
EIGEN_DEVICE_FUNC unpacket_traits< Packet >::type	predux (const Packet &a)
template<typename Packet >
EIGEN_DEVICE_FUNC unpacket_traits< Packet >::type	predux_mul (const Packet &a)
template<typename Packet >
EIGEN_DEVICE_FUNC unpacket_traits< Packet >::type	predux_min (const Packet &a)
template<int NaNPropagation, typename Packet >
EIGEN_DEVICE_FUNC unpacket_traits< Packet >::type	predux_min (const Packet &a)
template<typename Packet >
EIGEN_DEVICE_FUNC unpacket_traits< Packet >::type	predux_max (const Packet &a)
template<int NaNPropagation, typename Packet >
EIGEN_DEVICE_FUNC unpacket_traits< Packet >::type	predux_max (const Packet &a)
template<typename Packet >
EIGEN_DEVICE_FUNC bool	predux_any (const Packet &a)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	pmadd (const Packet &a, const Packet &b, const Packet &c)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	pmsub (const Packet &a, const Packet &b, const Packet &c)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	pnmadd (const Packet &a, const Packet &b, const Packet &c)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	pnmsub (const Packet &a, const Packet &b, const Packet &c)
template<typename Packet >
void	pstore1 (typename unpacket_traits< Packet >::type *to, const typename unpacket_traits< Packet >::type &a)
template<typename Packet , int Alignment>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet	ploadt (const typename unpacket_traits< Packet >::type *from)
template<typename Packet , int Alignment>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet	ploadt_partial (const typename unpacket_traits< Packet >::type *from, const Index n, const Index offset=0)
template<typename Scalar , typename Packet , int Alignment>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void	pstoret (Scalar *to, const Packet &from)
template<typename Scalar , typename Packet , int Alignment>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void	pstoret_partial (Scalar *to, const Packet &from, const Index n, const Index offset=0)
template<typename Packet , int LoadMode>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet	ploadt_ro (const typename unpacket_traits< Packet >::type *from)
template<>
std::complex< float >	pmul (const std::complex< float > &a, const std::complex< float > &b)
template<>
std::complex< double >	pmul (const std::complex< double > &a, const std::complex< double > &b)
template<typename Packet >
EIGEN_DEVICE_FUNC void	ptranspose (PacketBlock< Packet, 1 > &)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	pblend (const Selector< unpacket_traits< Packet >::size > &ifPacket, const Packet &thenPacket, const Packet &elsePacket)
template<typename Packet >
EIGEN_DEVICE_FUNC Packet	preciprocal (const Packet &a)
template<typename Packet >
EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	prsqrt (const Packet &a)
template<typename Packet >
EIGEN_DEVICE_FUNC constexpr EIGEN_ALWAYS_INLINE Packet	psignbit (const Packet &a)
template<typename Packet , std::enable_if_t< is_scalar< Packet >::value, int > = 0>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet	patan2 (const Packet &y, const Packet &x)
template<typename Packet , std::enable_if_t< is_scalar< Packet >::value, int > = 0>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet	pcarg (const Packet &a)
template<typename Derived >
std::ostream &	print_matrix (std::ostream &s, const Derived &_m, const IOFormat &fmt)
template<typename T >
EIGEN_DEVICE_FUNC std::complex< T >	complex_sqrt (const std::complex< T > &a_x)
template<typename T >
EIGEN_DEVICE_FUNC std::complex< T >	complex_rsqrt (const std::complex< T > &a_x)
template<typename OldType , typename NewType >
EIGEN_DEVICE_FUNC NewType	cast (const OldType &x)
template<typename T >
EIGEN_DEVICE_FUNC std::complex< T >	complex_log (const std::complex< T > &z)
template<typename BitsType >
EIGEN_DEVICE_FUNC int	clz (BitsType bits)
template<typename BitsType >
EIGEN_DEVICE_FUNC int	ctz (BitsType bits)
template<typename BitsType >
int	log2_ceil (const BitsType &x)
template<typename BitsType >
int	log2_floor (const BitsType &x)
template<typename T >
EIGEN_DEVICE_FUNC std::enable_if_t<!(std::numeric_limits< T >::has_infinity||std::numeric_limits< T >::has_quiet_NaN||std::numeric_limits< T >::has_signaling_NaN), bool >	isfinite_impl (const T &)
template<typename T >
EIGEN_DEVICE_FUNC std::enable_if_t<(std::numeric_limits< T >::has_infinity||std::numeric_limits< T >::has_quiet_NaN||std::numeric_limits< T >::has_signaling_NaN) &&(!NumTraits< T >::IsComplex), bool >	isfinite_impl (const T &x)
template<typename T >
EIGEN_DEVICE_FUNC std::enable_if_t<!std::numeric_limits< T >::has_infinity, bool >	isinf_impl (const T &)
template<typename T >
EIGEN_DEVICE_FUNC std::enable_if_t<(std::numeric_limits< T >::has_infinity &&!NumTraits< T >::IsComplex), bool >	isinf_impl (const T &x)
template<typename T >
EIGEN_DEVICE_FUNC std::enable_if_t<!(std::numeric_limits< T >::has_quiet_NaN||std::numeric_limits< T >::has_signaling_NaN), bool >	isnan_impl (const T &)
template<typename T >
EIGEN_DEVICE_FUNC std::enable_if_t<(std::numeric_limits< T >::has_quiet_NaN||std::numeric_limits< T >::has_signaling_NaN) &&(!NumTraits< T >::IsComplex), bool >	isnan_impl (const T &x)
template<typename T >
EIGEN_DEVICE_FUNC bool	isfinite_impl (const std::complex< T > &x)
template<typename T >
EIGEN_DEVICE_FUNC bool	isnan_impl (const std::complex< T > &x)
template<typename T >
EIGEN_DEVICE_FUNC bool	isinf_impl (const std::complex< T > &x)
template<typename Scalar , typename OtherScalar >
EIGEN_DEVICE_FUNC bool	isMuchSmallerThan (const Scalar &x, const OtherScalar &y, const typename NumTraits< Scalar >::Real &precision=NumTraits< Scalar >::dummy_precision())
template<typename Scalar >
EIGEN_DEVICE_FUNC bool	isApprox (const Scalar &x, const Scalar &y, const typename NumTraits< Scalar >::Real &precision=NumTraits< Scalar >::dummy_precision())
template<typename Scalar >
EIGEN_DEVICE_FUNC bool	isApproxOrLessThan (const Scalar &x, const Scalar &y, const typename NumTraits< Scalar >::Real &precision=NumTraits< Scalar >::dummy_precision())
template<typename RealScalar >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE RealScalar	positive_real_hypot (const RealScalar &x, const RealScalar &y)
template<typename PacketType , typename Func >
EIGEN_DEVICE_FUNC PacketType	packetwise_redux_empty_value (const Func &)
template<typename PacketType , typename Scalar >
EIGEN_DEVICE_FUNC PacketType	packetwise_redux_empty_value (const scalar_product_op< Scalar, Scalar > &)
	EIGEN_CATCH_ASSIGN_XPR_OP_PRODUCT (assign_op, scalar_sum_op, add_assign_op)
	EIGEN_CATCH_ASSIGN_XPR_OP_PRODUCT (add_assign_op, scalar_sum_op, add_assign_op)
	EIGEN_CATCH_ASSIGN_XPR_OP_PRODUCT (sub_assign_op, scalar_sum_op, sub_assign_op)
	EIGEN_CATCH_ASSIGN_XPR_OP_PRODUCT (assign_op, scalar_difference_op, sub_assign_op)
	EIGEN_CATCH_ASSIGN_XPR_OP_PRODUCT (add_assign_op, scalar_difference_op, sub_assign_op)
	EIGEN_CATCH_ASSIGN_XPR_OP_PRODUCT (sub_assign_op, scalar_difference_op, add_assign_op)
template<typename Dst , typename Lhs , typename Rhs , typename Func >
void EIGEN_DEVICE_FUNC	outer_product_selector_run (Dst &dst, const Lhs &lhs, const Rhs &rhs, const Func &func, const false_type &)
template<typename Dst , typename Lhs , typename Rhs , typename Func >
void EIGEN_DEVICE_FUNC	outer_product_selector_run (Dst &dst, const Lhs &lhs, const Rhs &rhs, const Func &func, const true_type &)
std::ptrdiff_t	manage_caching_sizes_helper (std::ptrdiff_t a, std::ptrdiff_t b)
void	manage_caching_sizes (Action action, std::ptrdiff_t *l1, std::ptrdiff_t *l2, std::ptrdiff_t *l3)
template<typename LhsScalar , typename RhsScalar , int KcFactor, typename Index >
void	evaluateProductBlockingSizesHeuristic (Index &k, Index &m, Index &n, Index num_threads=1)
template<typename Index >
bool	useSpecificBlockingSizes (Index &k, Index &m, Index &n)
template<typename LhsScalar , typename RhsScalar , int KcFactor, typename Index >
void	computeProductBlockingSizes (Index &k, Index &m, Index &n, Index num_threads=1)
	Computes the blocking parameters for a m x k times k x n matrix product. More...
template<typename LhsScalar , typename RhsScalar , typename Index >
void	computeProductBlockingSizes (Index &k, Index &m, Index &n, Index num_threads=1)
template<typename Packet >
DoublePacket< Packet >	padd (const DoublePacket< Packet > &a, const DoublePacket< Packet > &b)
template<typename Packet >
const DoublePacket< Packet > &	predux_half_dowto4 (const DoublePacket< Packet > &a, std::enable_if_t< unpacket_traits< Packet >::size<=8 > *=0)
template<typename Packet >
DoublePacket< typename unpacket_traits< Packet >::half >	predux_half_dowto4 (const DoublePacket< Packet > &a, std::enable_if_t< unpacket_traits< Packet >::size==16 > *=0)
template<typename Scalar , typename RealPacket >
void	loadQuadToDoublePacket (const Scalar *b, DoublePacket< RealPacket > &dest, std::enable_if_t< unpacket_traits< RealPacket >::size<=8 > *=0)
template<typename Scalar , typename RealPacket >
void	loadQuadToDoublePacket (const Scalar *b, DoublePacket< RealPacket > &dest, std::enable_if_t< unpacket_traits< RealPacket >::size==16 > *=0)
void	manage_multi_threading (Action action, int *v)
template<bool Condition, typename Functor , typename Index >
EIGEN_STRONG_INLINE void	parallelize_gemm (const Functor &func, Index rows, Index cols, Index, bool)
template<typename Scalar >
	EIGEN_MATHFUNC_RETVAL (random, Scalar) random(const Scalar &x
template<typename Scalar >
	EIGEN_MATHFUNC_RETVAL (random, Scalar) random()
template<typename BitsType >
EIGEN_DEVICE_FUNC BitsType	getRandomBits (int numRandomBits)
template<typename ExpressionType , typename Scalar >
void	stable_norm_kernel (const ExpressionType &bl, Scalar &ssq, Scalar &scale, Scalar &invScale)
template<typename VectorType , typename RealScalar >
void	stable_norm_impl_inner_step (const VectorType &vec, RealScalar &ssq, RealScalar &scale, RealScalar &invScale)
template<typename VectorType >
VectorType::RealScalar	stable_norm_impl (const VectorType &vec, std::enable_if_t< VectorType::IsVectorAtCompileTime > *=0)
template<typename MatrixType >
MatrixType::RealScalar	stable_norm_impl (const MatrixType &mat, std::enable_if_t<!MatrixType::IsVectorAtCompileTime > *=0)
template<typename Derived >
NumTraits< typename traits< Derived >::Scalar >::Real	blueNorm_impl (const EigenBase< Derived > &_vec)
template<typename MatrixType , Index Alignment>
void	BlockedInPlaceTranspose (MatrixType &m)
template<int Mode, bool SetOpposite, typename DstXprType , typename SrcXprType , typename Functor >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	call_triangular_assignment_loop (DstXprType &dst, const SrcXprType &src, const Functor &func)
template<int Mode, bool SetOpposite, typename DstXprType , typename SrcXprType >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	call_triangular_assignment_loop (DstXprType &dst, const SrcXprType &src)
template<typename T >
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE const T::Scalar *	extract_data (const T &m)
template<typename ResScalar , typename Lhs , typename Rhs >
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE ResScalar	combine_scalar_factors (const ResScalar &alpha, const Lhs &lhs, const Rhs &rhs)
template<typename ResScalar , typename Lhs , typename Rhs >
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE ResScalar	combine_scalar_factors (const Lhs &lhs, const Rhs &rhs)
template<std::size_t I_, class T , std::size_t N>
constexpr T &	array_get (std::array< T, N > &a)
template<std::size_t I_, class T , std::size_t N>
constexpr T &&	array_get (std::array< T, N > &&a)
template<std::size_t I_, class T , std::size_t N>
constexpr T const &	array_get (std::array< T, N > const &a)
template<int SizeAtCompileTime, typename Indices >
IvcType< Indices, SizeAtCompileTime >	CreateIndexSequence (size_t size, const Indices &indices)
template<typename T >
EIGEN_DEVICE_FUNC Index	get_runtime_value (const T &x)
template<typename T >
EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE void	ignore_unused_variable (const T &)
EIGEN_DEVICE_FUNC bool	all ()
template<typename T , typename... Ts>
EIGEN_DEVICE_FUNC bool	all (T t, Ts... ts)
EIGEN_DEVICE_FUNC void	check_that_malloc_is_allowed ()
EIGEN_DEVICE_FUNC void	throw_std_bad_alloc ()
EIGEN_DEVICE_FUNC void *	handmade_aligned_malloc (std::size_t size, std::size_t alignment=EIGEN_DEFAULT_ALIGN_BYTES)
EIGEN_DEVICE_FUNC void	handmade_aligned_free (void *ptr)
EIGEN_DEVICE_FUNC void *	handmade_aligned_realloc (void *ptr, std::size_t new_size, std::size_t old_size, std::size_t alignment=EIGEN_DEFAULT_ALIGN_BYTES)
	Reallocates aligned memory. Since we know that our handmade version is based on std::malloc we can use std::realloc to implement efficient reallocation. More...
EIGEN_DEVICE_FUNC void *	aligned_malloc (std::size_t size)
EIGEN_DEVICE_FUNC void	aligned_free (void *ptr)
EIGEN_DEVICE_FUNC void *	aligned_realloc (void *ptr, std::size_t new_size, std::size_t old_size)
	Reallocates an aligned block of memory. More...
template<bool Align>
EIGEN_DEVICE_FUNC void *	conditional_aligned_malloc (std::size_t size)
template<>
EIGEN_DEVICE_FUNC void *	conditional_aligned_malloc< false > (std::size_t size)
template<bool Align>
EIGEN_DEVICE_FUNC void	conditional_aligned_free (void *ptr)
template<>
EIGEN_DEVICE_FUNC void	conditional_aligned_free< false > (void *ptr)
template<bool Align>
EIGEN_DEVICE_FUNC void *	conditional_aligned_realloc (void *ptr, std::size_t new_size, std::size_t old_size)
template<>
EIGEN_DEVICE_FUNC void *	conditional_aligned_realloc< false > (void *ptr, std::size_t new_size, std::size_t old_size)
template<typename T >
EIGEN_DEVICE_FUNC void	destruct_elements_of_array (T *ptr, std::size_t size)
template<typename T >
EIGEN_DEVICE_FUNC T *	default_construct_elements_of_array (T *ptr, std::size_t size)
template<typename T >
EIGEN_DEVICE_FUNC T *	copy_construct_elements_of_array (T *ptr, const T *src, std::size_t size)
template<typename T >
EIGEN_DEVICE_FUNC T *	move_construct_elements_of_array (T *ptr, T *src, std::size_t size)
template<typename T >
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void	check_size_for_overflow (std::size_t size)
template<typename T >
EIGEN_DEVICE_FUNC T *	aligned_new (std::size_t size)
template<typename T , bool Align>
EIGEN_DEVICE_FUNC T *	conditional_aligned_new (std::size_t size)
template<typename T >
EIGEN_DEVICE_FUNC void	aligned_delete (T *ptr, std::size_t size)
template<typename T , bool Align>
EIGEN_DEVICE_FUNC void	conditional_aligned_delete (T *ptr, std::size_t size)
template<typename T , bool Align>
EIGEN_DEVICE_FUNC T *	conditional_aligned_realloc_new (T *pts, std::size_t new_size, std::size_t old_size)
template<typename T , bool Align>
EIGEN_DEVICE_FUNC T *	conditional_aligned_new_auto (std::size_t size)
template<typename T , bool Align>
EIGEN_DEVICE_FUNC T *	conditional_aligned_realloc_new_auto (T *pts, std::size_t new_size, std::size_t old_size)
template<typename T , bool Align>
EIGEN_DEVICE_FUNC void	conditional_aligned_delete_auto (T *ptr, std::size_t size)
template<int Alignment, typename Scalar , typename Index >
EIGEN_DEVICE_FUNC Index	first_aligned (const Scalar *array, Index size)
template<typename Scalar , typename Index >
EIGEN_DEVICE_FUNC Index	first_default_aligned (const Scalar *array, Index size)
template<typename Index >
Index	first_multiple (Index size, Index base)
template<typename T >
EIGEN_DEVICE_FUNC void	smart_copy (const T *start, const T *end, T *target)
template<typename T >
void	smart_memmove (const T *start, const T *end, T *target)
template<typename T >
EIGEN_DEVICE_FUNC T *	smart_move (T *start, T *end, T *target)
template<typename T >
void	swap (scoped_array< T > &a, scoped_array< T > &b)
void	queryCacheSizes (int &l1, int &l2, int &l3)
int	queryL1CacheSize ()
int	queryTopLevelCacheSize ()
template<class T , class... Args>
EIGEN_DEVICE_FUNC T *	construct_at (T *p, Args &&... args)
template<class T >
EIGEN_DEVICE_FUNC void	destroy_at (T *p)
template<typename T >
EIGEN_CONSTEXPR auto	index_list_size (const T &x)
template<typename T , std::ptrdiff_t N>
EIGEN_CONSTEXPR std::ptrdiff_t	index_list_size (const T(&)[N])
template<typename T >
const T *	return_ptr ()
template<typename Scalar >
EIGEN_STRONG_INLINE bool	is_identically_zero (const Scalar &s)
template<typename A , typename B >
constexpr void	plain_enum_asserts (A, B)
template<typename A , typename B >
constexpr int	plain_enum_min (A a, B b)
template<typename A , typename B >
constexpr int	plain_enum_max (A a, B b)
template<typename A , typename B >
constexpr int	min_size_prefer_dynamic (A a, B b)
template<typename A , typename B >
constexpr int	min_size_prefer_fixed (A a, B b)
template<typename A , typename B >
constexpr int	max_size_prefer_dynamic (A a, B b)
template<typename A , typename B >
constexpr bool	enum_eq_not_dynamic (A a, B b)
template<typename A , typename B >
constexpr bool	enum_lt_not_dynamic (A a, B b)
template<typename A , typename B >
constexpr bool	enum_le_not_dynamic (A a, B b)
template<typename A , typename B >
constexpr bool	enum_gt_not_dynamic (A a, B b)
template<typename A , typename B >
constexpr bool	enum_ge_not_dynamic (A a, B b)
constexpr bool	logical_xor (bool a, bool b)
constexpr bool	check_implication (bool a, bool b)
constexpr bool	is_constant_evaluated ()
template<std::size_t n, typename T , T a, T... as>
constexpr T	array_get (const numeric_list< T, a, as... > &)
template<typename Array , int... n>
constexpr EIGEN_STRONG_INLINE Array	h_array_reverse (Array arr, numeric_list< int, n... >)
template<typename T , std::size_t N>
constexpr EIGEN_STRONG_INLINE array< T, N >	array_reverse (array< T, N > arr)
template<typename Reducer , typename T , std::size_t N>
constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE auto	array_reduce (const array< T, N > &arr, T identity) -> decltype(h_array_reduce< Reducer, T, N >::run(arr, identity))
template<typename T , std::size_t N>
constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE auto	array_sum (const array< T, N > &arr) -> decltype(array_reduce< sum_op, T, N >(arr, static_cast< T >(0)))
template<typename T , std::size_t N>
constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE auto	array_prod (const array< T, N > &arr) -> decltype(array_reduce< product_op, T, N >(arr, static_cast< T >(1)))
template<typename t >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE t	array_prod (const std::vector< t > &a)
template<typename Op , typename A , typename B , std::size_t N, int... n>
constexpr EIGEN_STRONG_INLINE array< decltype(Op::run(A(), B())), N >	h_array_zip (array< A, N > a, array< B, N > b, numeric_list< int, n... >)
template<typename Op , typename A , typename B , std::size_t N>
constexpr EIGEN_STRONG_INLINE array< decltype(Op::run(A(), B())), N >	array_zip (array< A, N > a, array< B, N > b)
template<typename Reducer , typename Op , typename A , typename B , std::size_t N, int... n>
constexpr EIGEN_STRONG_INLINE auto	h_array_zip_and_reduce (array< A, N > a, array< B, N > b, numeric_list< int, n... >) -> decltype(reduce< Reducer, typename id_numeric< int, n, decltype(Op::run(A(), B()))>::type... >::run(Op::run(array_get< n >(a), array_get< n >(b))...))
template<typename Reducer , typename Op , typename A , typename B , std::size_t N>
constexpr EIGEN_STRONG_INLINE auto	array_zip_and_reduce (array< A, N > a, array< B, N > b) -> decltype(h_array_zip_and_reduce< Reducer, Op, A, B, N >(a, b, typename gen_numeric_list< int, N >::type()))
template<typename Op , typename A , std::size_t N, int... n>
constexpr EIGEN_STRONG_INLINE array< decltype(Op::run(A())), N >	h_array_apply (array< A, N > a, numeric_list< int, n... >)
template<typename Op , typename A , std::size_t N>
constexpr EIGEN_STRONG_INLINE array< decltype(Op::run(A())), N >	array_apply (array< A, N > a)
template<typename Reducer , typename Op , typename A , std::size_t N, int... n>
constexpr EIGEN_STRONG_INLINE auto	h_array_apply_and_reduce (array< A, N > arr, numeric_list< int, n... >) -> decltype(reduce< Reducer, typename id_numeric< int, n, decltype(Op::run(A()))>::type... >::run(Op::run(array_get< n >(arr))...))
template<typename Reducer , typename Op , typename A , std::size_t N>
constexpr EIGEN_STRONG_INLINE auto	array_apply_and_reduce (array< A, N > a) -> decltype(h_array_apply_and_reduce< Reducer, Op, A, N >(a, typename gen_numeric_list< int, N >::type()))
template<int n, typename t >
constexpr array< t, n >	repeat (t v)
template<class InstType , typename ArrType , std::size_t N, bool Reverse = false>
InstType	instantiate_by_c_array (ArrType *arr)
template<typename SizeType >
Index	get_runtime_reshape_size (SizeType size, Index, Index)
Index	get_runtime_reshape_size (AutoSize_t, Index other, Index total)
constexpr int	get_compiletime_reshape_order (int flags, int order)
template<typename IndexType >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE make_unsigned< IndexType >::type	returnUnsignedIndexValue (const IndexType &idx)
template<typename IndexDest , typename IndexSrc >
EIGEN_DEVICE_FUNC IndexDest	convert_index (const IndexSrc &idx)
constexpr int	compute_default_alignment_helper (int ArrayBytes, int AlignmentBytes)
constexpr unsigned	compute_matrix_flags (int Options)
constexpr int	size_at_compile_time (int rows, int cols)
template<typename T >
EIGEN_DEVICE_FUNC T *	const_cast_ptr (const T *ptr)
template<typename T1 , typename T2 >
EIGEN_DEVICE_FUNC bool	is_same_dense (const T1 &mat1, const T2 &mat2, std::enable_if_t< possibly_same_dense< T1, T2 >::value > *=0)
template<typename T1 , typename T2 >
EIGEN_DEVICE_FUNC bool	is_same_dense (const T1 &, const T2 &, std::enable_if_t<!possibly_same_dense< T1, T2 >::value > *=0)
	EIGEN_MEMBER_FUNCTOR (norm,(Size+5) *NumTraits< Scalar >::MulCost+(Size - 1) *NumTraits< Scalar >::AddCost)
	EIGEN_MEMBER_FUNCTOR (stableNorm,(Size+5) *NumTraits< Scalar >::MulCost+(Size - 1) *NumTraits< Scalar >::AddCost)
	EIGEN_MEMBER_FUNCTOR (blueNorm,(Size+5) *NumTraits< Scalar >::MulCost+(Size - 1) *NumTraits< Scalar >::AddCost)
	EIGEN_MEMBER_FUNCTOR (hypotNorm,(Size - 1) *functor_traits< scalar_hypot_op< Scalar > >::Cost)
	EIGEN_MEMBER_FUNCTOR (all,(Size - 1) *NumTraits< Scalar >::AddCost)
	EIGEN_MEMBER_FUNCTOR (any,(Size - 1) *NumTraits< Scalar >::AddCost)
	EIGEN_MEMBER_FUNCTOR (count,(Size - 1) *NumTraits< Scalar >::AddCost)
	EIGEN_MAKE_PARTIAL_REDUX_FUNCTOR (sum,(Size - 1) *NumTraits< Scalar >::AddCost, 1, internal::scalar_sum_op)
	EIGEN_MAKE_PARTIAL_REDUX_FUNCTOR (minCoeff,(Size - 1) *NumTraits< Scalar >::AddCost, 1, internal::scalar_min_op)
	EIGEN_MAKE_PARTIAL_REDUX_FUNCTOR (maxCoeff,(Size - 1) *NumTraits< Scalar >::AddCost, 1, internal::scalar_max_op)
	EIGEN_MAKE_PARTIAL_REDUX_FUNCTOR (prod,(Size - 1) *NumTraits< Scalar >::MulCost, 1, internal::scalar_product_op)
template<typename MatrixType , typename DiagType , typename SubDiagType >
EIGEN_DEVICE_FUNC ComputationInfo	computeFromTridiagonal_impl (DiagType &diag, SubDiagType &subdiag, const Index maxIterations, bool computeEigenvectors, MatrixType &eivec)
	Compute the eigendecomposition from a tridiagonal matrix. More...
template<int StorageOrder, typename RealScalar , typename Scalar , typename Index >
static EIGEN_DEVICE_FUNC void	tridiagonal_qr_step (RealScalar *diag, RealScalar *subdiag, Index start, Index end, Scalar *matrixQ, Index n)
template<typename MatrixType , typename CoeffVectorType >
EIGEN_DEVICE_FUNC void	tridiagonalization_inplace (MatrixType &matA, CoeffVectorType &hCoeffs)
template<typename MatrixType , typename DiagonalType , typename SubDiagonalType , typename CoeffVectorType , typename WorkSpaceType >
EIGEN_DEVICE_FUNC void	tridiagonalization_inplace (MatrixType &mat, DiagonalType &diag, SubDiagonalType &subdiag, CoeffVectorType &hcoeffs, WorkSpaceType &workspace, bool extractQ)
	Performs a full tridiagonalization in place. More...
template<typename Scalar , int Dim>
static EIGEN_DEVICE_FUNC Matrix< Scalar, 2, 2 >	toRotationMatrix (const Scalar &s)
template<typename Scalar , int Dim, typename OtherDerived >
static EIGEN_DEVICE_FUNC Matrix< Scalar, Dim, Dim >	toRotationMatrix (const RotationBase< OtherDerived, Dim > &r)
template<typename Scalar , int Dim, typename OtherDerived >
static EIGEN_DEVICE_FUNC const MatrixBase< OtherDerived > &	toRotationMatrix (const MatrixBase< OtherDerived > &mat)
template<typename TriangularFactorType , typename VectorsType , typename CoeffsType >
void	make_block_householder_triangular_factor (TriangularFactorType &triFactor, const VectorsType &vectors, const CoeffsType &hCoeffs)
template<typename MatrixType , typename VectorsType , typename CoeffsType >
void	apply_block_householder_on_the_left (MatrixType &mat, const VectorsType &vectors, const CoeffsType &hCoeffs, bool forward)
template<typename MatrixType , typename Rhs , typename Dest , typename Preconditioner >
bool	bicgstab (const MatrixType &mat, const Rhs &rhs, Dest &x, const Preconditioner &precond, Index &iters, typename Dest::RealScalar &tol_error)
template<typename MatrixType , typename Rhs , typename Dest , typename Preconditioner >
EIGEN_DONT_INLINE void	conjugate_gradient (const MatrixType &mat, const Rhs &rhs, Dest &x, const Preconditioner &precond, Index &iters, typename Dest::RealScalar &tol_error)
template<typename VectorV , typename VectorI >
Index	QuickSplit (VectorV &row, VectorI &ind, Index ncut)
template<typename MatrixType , typename Rhs , typename Dest , typename Preconditioner >
EIGEN_DONT_INLINE void	least_square_conjugate_gradient (const MatrixType &mat, const Rhs &rhs, Dest &x, const Preconditioner &precond, Index &iters, typename Dest::RealScalar &tol_error)
template<typename VectorX , typename VectorY , typename OtherScalar >
EIGEN_DEVICE_FUNC void	apply_rotation_in_the_plane (DenseBase< VectorX > &xpr_x, DenseBase< VectorY > &xpr_y, const JacobiRotation< OtherScalar > &j)
template<typename Derived >
EIGEN_DEVICE_FUNC const Derived::Scalar	bruteforce_det3_helper (const MatrixBase< Derived > &matrix, int a, int b, int c)
template<typename MatrixType , typename ResultType >
EIGEN_DEVICE_FUNC void	compute_inverse_size2_helper (const MatrixType &matrix, const typename ResultType::Scalar &invdet, ResultType &result)
template<typename MatrixType , int i, int j>
EIGEN_DEVICE_FUNC MatrixType::Scalar	cofactor_3x3 (const MatrixType &m)
template<typename MatrixType , typename ResultType >
EIGEN_DEVICE_FUNC void	compute_inverse_size3_helper (const MatrixType &matrix, const typename ResultType::Scalar &invdet, const Matrix< typename ResultType::Scalar, 3, 1 > &cofactors_col0, ResultType &result)
template<typename Derived >
EIGEN_DEVICE_FUNC const Derived::Scalar	general_det3_helper (const MatrixBase< Derived > &matrix, int i1, int i2, int i3, int j1, int j2, int j3)
template<typename MatrixType , int i, int j>
EIGEN_DEVICE_FUNC MatrixType::Scalar	cofactor_4x4 (const MatrixType &matrix)
template<typename MatrixType , typename TranspositionType >
void	partial_lu_inplace (MatrixType &lu, TranspositionType &row_transpositions, typename TranspositionType::StorageIndex &nb_transpositions)
template<typename MatrixType , typename RealScalar , typename Index >
void	real_2x2_jacobi_svd (const MatrixType &matrix, Index p, Index q, JacobiRotation< RealScalar > *j_left, JacobiRotation< RealScalar > *j_right)
template<typename T >
T	amd_flip (const T &i)
template<typename T >
T	amd_unflip (const T &i)
template<typename T0 , typename T1 >
bool	amd_marked (const T0 *w, const T1 &j)
template<typename T0 , typename T1 >
void	amd_mark (const T0 *w, const T1 &j)
template<typename StorageIndex >
static StorageIndex	cs_wclear (StorageIndex mark, StorageIndex lemax, StorageIndex *w, StorageIndex n)
template<typename StorageIndex >
StorageIndex	cs_tdfs (StorageIndex j, StorageIndex k, StorageIndex *head, const StorageIndex *next, StorageIndex *post, StorageIndex *stack)
template<typename Scalar , typename StorageIndex >
void	minimum_degree_ordering (SparseMatrix< Scalar, ColMajor, StorageIndex > &C, PermutationMatrix< Dynamic, Dynamic, StorageIndex > &perm)
template<typename MatrixType >
void	ordering_helper_at_plus_a (const MatrixType &A, MatrixType &symmat)
void	eigen_pastix (pastix_data_t **pastix_data, int pastix_comm, int n, int *ptr, int *idx, float *vals, int *perm, int *invp, float *x, int nbrhs, int *iparm, double *dparm)
void	eigen_pastix (pastix_data_t **pastix_data, int pastix_comm, int n, int *ptr, int *idx, double *vals, int *perm, int *invp, double *x, int nbrhs, int *iparm, double *dparm)
void	eigen_pastix (pastix_data_t **pastix_data, int pastix_comm, int n, int *ptr, int *idx, std::complex< float > *vals, int *perm, int *invp, std::complex< float > *x, int nbrhs, int *iparm, double *dparm)
void	eigen_pastix (pastix_data_t **pastix_data, int pastix_comm, int n, int *ptr, int *idx, std::complex< double > *vals, int *perm, int *invp, std::complex< double > *x, int nbrhs, int *iparm, double *dparm)
template<typename MatrixType >
void	c_to_fortran_numbering (MatrixType &mat)
template<typename MatrixType >
void	fortran_to_c_numbering (MatrixType &mat)
template<typename MatrixQR , typename HCoeffs >
void	householder_qr_inplace_unblocked (MatrixQR &mat, HCoeffs &hCoeffs, typename MatrixQR::Scalar *tempData=0)
template<typename MatrixQR , typename HCoeffs , typename VectorQR >
void	householder_qr_inplace_update (MatrixQR &mat, HCoeffs &hCoeffs, const VectorQR &newColumn, typename MatrixQR::Index k, typename MatrixQR::Scalar *tempData)
template<typename Lhs , typename Rhs , typename ResultType >
static void	conservative_sparse_sparse_product_impl (const Lhs &lhs, const Rhs &rhs, ResultType &res, bool sortedInsertion=false)
template<typename Lhs , typename Rhs , typename ResultType >
static void	sparse_sparse_to_dense_product_impl (const Lhs &lhs, const Rhs &rhs, ResultType &res)
template<typename DstXprType , typename SrcXprType >
void	assign_sparse_to_sparse (DstXprType &dst, const SrcXprType &src)
	EIGEN_CATCH_ASSIGN_DENSE_OP_SPARSE (assign_op, scalar_sum_op, add_assign_op)
	EIGEN_CATCH_ASSIGN_DENSE_OP_SPARSE (add_assign_op, scalar_sum_op, add_assign_op)
	EIGEN_CATCH_ASSIGN_DENSE_OP_SPARSE (sub_assign_op, scalar_sum_op, sub_assign_op)
	EIGEN_CATCH_ASSIGN_DENSE_OP_SPARSE (assign_op, scalar_difference_op, sub_assign_op)
	EIGEN_CATCH_ASSIGN_DENSE_OP_SPARSE (add_assign_op, scalar_difference_op, sub_assign_op)
	EIGEN_CATCH_ASSIGN_DENSE_OP_SPARSE (sub_assign_op, scalar_difference_op, add_assign_op)
template<typename Index , typename IndexVector >
Index	etree_find (Index i, IndexVector &pp)
template<typename MatrixType , typename IndexVector >
int	coletree (const MatrixType &mat, IndexVector &parent, IndexVector &firstRowElt, typename MatrixType::StorageIndex *perm=0)
template<typename IndexVector >
void	nr_etdfs (typename IndexVector::Scalar n, IndexVector &parent, IndexVector &first_kid, IndexVector &next_kid, IndexVector &post, typename IndexVector::Scalar postnum)
template<typename IndexVector >
void	treePostorder (typename IndexVector::Scalar n, IndexVector &parent, IndexVector &post)
	Post order a tree. More...
template<typename SparseLhsType , typename DenseRhsType , typename DenseResType , typename AlphaType >
void	sparse_time_dense_product (const SparseLhsType &lhs, const DenseRhsType &rhs, DenseResType &res, const AlphaType &alpha)
template<typename InputIterator , typename SparseMatrixType , typename DupFunctor >
void	set_from_triplets (const InputIterator &begin, const InputIterator &end, SparseMatrixType &mat, DupFunctor dup_func)
template<typename InputIterator , typename SparseMatrixType , typename DupFunctor >
void	set_from_triplets_sorted (const InputIterator &begin, const InputIterator &end, SparseMatrixType &mat, DupFunctor dup_func)
template<typename InputIterator , typename SparseMatrixType , typename DupFunctor >
void	insert_from_triplets (const InputIterator &begin, const InputIterator &end, SparseMatrixType &mat, DupFunctor dup_func)
template<typename InputIterator , typename SparseMatrixType , typename DupFunctor >
void	insert_from_triplets_sorted (const InputIterator &begin, const InputIterator &end, SparseMatrixType &mat, DupFunctor dup_func)
template<int SrcMode, int DstMode, bool NonHermitian, typename MatrixType , int DestOrder>
void	permute_symm_to_symm (const MatrixType &mat, SparseMatrix< typename MatrixType::Scalar, DestOrder, typename MatrixType::StorageIndex > &_dest, const typename MatrixType::StorageIndex *perm=0)
template<int Mode, bool NonHermitian, typename MatrixType , int DestOrder>
void	permute_symm_to_fullsymm (const MatrixType &mat, SparseMatrix< typename MatrixType::Scalar, DestOrder, typename MatrixType::StorageIndex > &_dest, const typename MatrixType::StorageIndex *perm=0)
template<int Mode, typename SparseLhsType , typename DenseRhsType , typename DenseResType , typename AlphaType >
void	sparse_selfadjoint_time_dense_product (const SparseLhsType &lhs, const DenseRhsType &rhs, DenseResType &res, const AlphaType &alpha)
template<int SrcMode_, int DstMode_, bool NonHermitian, typename MatrixType , int DstOrder>
void	permute_symm_to_symm (const MatrixType &mat, SparseMatrix< typename MatrixType::Scalar, DstOrder, typename MatrixType::StorageIndex > &_dest, const typename MatrixType::StorageIndex *perm)
template<typename Decomposition , typename Rhs , typename Dest >
std::enable_if_t< Rhs::ColsAtCompileTime !=1 &&Dest::ColsAtCompileTime !=1 >	solve_sparse_through_dense_panels (const Decomposition &dec, const Rhs &rhs, Dest &dest)
template<typename Decomposition , typename Rhs , typename Dest >
std::enable_if_t< Rhs::ColsAtCompileTime==1||Dest::ColsAtCompileTime==1 >	solve_sparse_through_dense_panels (const Decomposition &dec, const Rhs &rhs, Dest &dest)
template<typename Lhs , typename Rhs , typename ResultType >
static void	sparse_sparse_product_with_pruning_impl (const Lhs &lhs, const Rhs &rhs, ResultType &res, const typename ResultType::RealScalar &tolerance)
Index	LUnumTempV (Index &m, Index &w, Index &t, Index &b)
template<typename Scalar >
Index	LUTempSpace (Index &m, Index &w)
template<typename MatrixType >
SluMatrix	asSluMatrix (MatrixType &mat)
template<typename Scalar , int Flags, typename Index >
Map< SparseMatrix< Scalar, Flags, Index > >	map_superlu (SluMatrix &sluMat)
constexpr int	get_qr_preconditioner (int options)
constexpr int	get_computation_options (int options)
constexpr bool	should_svd_compute_thin_u (int options)
constexpr bool	should_svd_compute_full_u (int options)
constexpr bool	should_svd_compute_thin_v (int options)
constexpr bool	should_svd_compute_full_v (int options)
template<typename MatrixType , int Options>
void	check_svd_options_assertions (unsigned int computationOptions, Index rows, Index cols)
template<typename MatrixType >
void	upperbidiagonalization_inplace_unblocked (MatrixType &mat, typename MatrixType::RealScalar *diagonal, typename MatrixType::RealScalar *upper_diagonal, typename MatrixType::Scalar *tempData=0)
template<typename MatrixType >
void	upperbidiagonalization_blocked_helper (MatrixType &A, typename MatrixType::RealScalar *diagonal, typename MatrixType::RealScalar *upper_diagonal, Index bs, Ref< Matrix< typename MatrixType::Scalar, Dynamic, Dynamic, traits< MatrixType >::Flags &RowMajorBit > > X, Ref< Matrix< typename MatrixType::Scalar, Dynamic, Dynamic, traits< MatrixType >::Flags &RowMajorBit > > Y)
template<typename MatrixType , typename BidiagType >
void	upperbidiagonalization_inplace_blocked (MatrixType &A, BidiagType &bidiagonal, Index maxBlockSize=32, typename MatrixType::Scalar *=0)
template<>
EIGEN_STRONG_INLINE double	cast (const AnnoyingScalar &x)
template<typename Dst , template< typename > class StorageBase, typename Src , typename Func >
EIGEN_DEVICE_FUNC void	call_assignment (const NoAlias< Dst, StorageBase > &dst, const Src &src, const Func &func)
template<typename Dst , template< typename > class StorageBase, typename Src , typename Func >
EIGEN_DEVICE_FUNC void	call_restricted_packet_assignment (const NoAlias< Dst, StorageBase > &dst, const Src &src, const Func &func)
template<typename Kernel , typename... Args, size_t... Indices, size_t... OutputIndices>
EIGEN_DEVICE_FUNC void	run_serialized (std::index_sequence< Indices... >, std::index_sequence< OutputIndices... >, Kernel kernel, uint8_t *buffer, size_t capacity)
template<typename Kernel , typename... Args>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void	run_serialized (Kernel kernel, uint8_t *buffer, size_t capacity)
template<int Layout, typename IndexType , int NumDims>
EIGEN_ALWAYS_INLINE DSizes< IndexType, NumDims >	strides (const DSizes< IndexType, NumDims > &dimensions)
template<int Layout, typename IndexType , size_t NumDims>
EIGEN_ALWAYS_INLINE DSizes< IndexType, NumDims >	strides (const Eigen::array< IndexType, NumDims > &dimensions)
template<int Layout, std::ptrdiff_t... Indices>
EIGEN_STRONG_INLINE DSizes< std::ptrdiff_t, sizeof...(Indices)>	strides (const Sizes< Indices... > &sizes)
template<DenseIndex n, typename Index , std::size_t Rank>
const Index	array_get (DimensionList< Index, Rank > &)
template<DenseIndex n, typename Index , std::size_t Rank>
const Index	array_get (const DimensionList< Index, Rank > &)
template<typename std::ptrdiff_t... Indices>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::ptrdiff_t	array_prod (const Sizes< Indices... > &)
template<std::ptrdiff_t n, typename std::ptrdiff_t... Indices>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::ptrdiff_t	array_get (const Sizes< Indices... > &)
template<std::ptrdiff_t n>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::ptrdiff_t	array_get (const Sizes<> &)
template<typename T >
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T	loadConstant (const T *address)
template<typename T >
EIGEN_DEVICE_FUNC void	update_value (T &val, Index new_val)
template<Index n>
EIGEN_DEVICE_FUNC void	update_value (type2index< n > &val, Index new_val)
template<typename T >
EIGEN_DEVICE_FUNC void	update_value (T &val, IndexPair< Index > new_val)
template<Index f, Index s>
EIGEN_DEVICE_FUNC void	update_value (type2indexpair< f, s > &val, IndexPair< Index > new_val)
template<int N, typename T , typename... O>
constexpr EIGEN_DEVICE_FUNC IndexTupleExtractor< N, T, O... >::ValType &	array_get (IndexTuple< T, O... > &tuple)
template<int N, typename T , typename... O>
constexpr EIGEN_DEVICE_FUNC const IndexTupleExtractor< N, T, O... >::ValType &	array_get (const IndexTuple< T, O... > &tuple)
template<typename FirstType , typename... OtherTypes>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index	array_prod (const IndexList< FirstType, OtherTypes... > &sizes)
template<Index N, typename FirstType , typename... OtherTypes>
constexpr EIGEN_DEVICE_FUNC Index	array_get (IndexList< FirstType, OtherTypes... > &a)
template<Index N, typename FirstType , typename... OtherTypes>
constexpr EIGEN_DEVICE_FUNC Index	array_get (const IndexList< FirstType, OtherTypes... > &a)
template<typename T >
static EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bool	index_known_statically (Index i)
template<typename T >
static EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bool	all_indices_known_statically ()
template<typename T >
static EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bool	indices_statically_known_to_increase ()
template<typename T >
static EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bool	index_statically_eq (Index i, Index value)
template<typename T >
static EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bool	index_statically_ne (Index i, Index value)
template<typename T >
static EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bool	index_statically_gt (Index i, Index value)
template<typename T >
static EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bool	index_statically_lt (Index i, Index value)
template<typename T >
static EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bool	index_pair_first_statically_eq (Index i, Index value)
template<typename T >
static EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bool	index_pair_second_statically_eq (Index i, Index value)
template<typename Derived , int N>
void	initialize_tensor (TensorEvaluator< Derived, DefaultDevice > &tensor, const typename Initializer< Derived, traits< Derived >::NumDimensions >::InitList &vals)
template<typename T >
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE std::enable_if_t< sizeof(T)==4, int >	count_leading_zeros (const T val)
template<typename T >
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE std::enable_if_t< sizeof(T)==8, int >	count_leading_zeros (const T val)
template<typename T >
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE uint32_t	muluh (const uint32_t a, const T b)
template<typename T >
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE uint64_t	muluh (const uint64_t a, const T b)
template<typename T , bool div_gt_one>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T	operator/ (const T &numerator, const TensorIntDivisor< T, div_gt_one > &divisor)
template<typename IndexType , typename Index , Index First, Index... Is>
EIGEN_CONSTEXPR EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE array< Index, 1+sizeof...(Is)>	customIndices2Array (IndexType &idx, numeric_list< Index, First, Is... >)
template<typename IndexType , typename Index >
EIGEN_CONSTEXPR EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE array< Index, 0 >	customIndices2Array (IndexType &, numeric_list< Index >)
template<typename Index , std::size_t NumIndices, typename IndexType >
EIGEN_CONSTEXPR EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE array< Index, NumIndices >	customIndices2Array (IndexType &idx)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE uint64_t	get_random_seed ()
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE unsigned	PCG_XSH_RS_generator (uint64_t *state, uint64_t stream)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE uint64_t	PCG_XSH_RS_state (uint64_t seed)
template<typename T >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T	RandomToTypeUniform (uint64_t *state, uint64_t stream)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half	RandomToTypeUniform< Eigen::half > (uint64_t *state, uint64_t stream)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::bfloat16	RandomToTypeUniform< Eigen::bfloat16 > (uint64_t *state, uint64_t stream)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float	RandomToTypeUniform< float > (uint64_t *state, uint64_t stream)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double	RandomToTypeUniform< double > (uint64_t *state, uint64_t stream)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex< float >	RandomToTypeUniform< std::complex< float > > (uint64_t *state, uint64_t stream)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex< double >	RandomToTypeUniform< std::complex< double > > (uint64_t *state, uint64_t stream)
template<typename T >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T	RandomToTypeNormal (uint64_t *state, uint64_t stream)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex< float >	RandomToTypeNormal< std::complex< float > > (uint64_t *state, uint64_t stream)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex< double >	RandomToTypeNormal< std::complex< double > > (uint64_t *state, uint64_t stream)
template<typename T >
EIGEN_DEVICE_FUNC Index	LeafSize ()
template<>
EIGEN_DEVICE_FUNC Index	LeafSize< half > ()
template<>
EIGEN_DEVICE_FUNC Index	LeafSize< bfloat16 > ()
template<typename Self >
EIGEN_STRONG_INLINE void	ReduceScalar (Self &self, Index offset, typename Self::CoeffReturnType *data)
template<typename Self >
EIGEN_STRONG_INLINE void	ReducePacket (Self &self, Index offset, typename Self::CoeffReturnType *data)
template<typename HL , typename LL , typename HR , typename LR >
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool	operator== (const TensorUInt128< HL, LL > &lhs, const TensorUInt128< HR, LR > &rhs)
template<typename HL , typename LL , typename HR , typename LR >
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool	operator!= (const TensorUInt128< HL, LL > &lhs, const TensorUInt128< HR, LR > &rhs)
template<typename HL , typename LL , typename HR , typename LR >
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool	operator>= (const TensorUInt128< HL, LL > &lhs, const TensorUInt128< HR, LR > &rhs)
template<typename HL , typename LL , typename HR , typename LR >
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool	operator< (const TensorUInt128< HL, LL > &lhs, const TensorUInt128< HR, LR > &rhs)
template<typename HL , typename LL , typename HR , typename LR >
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE TensorUInt128< uint64_t, uint64_t >	operator+ (const TensorUInt128< HL, LL > &lhs, const TensorUInt128< HR, LR > &rhs)
template<typename HL , typename LL , typename HR , typename LR >
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE TensorUInt128< uint64_t, uint64_t >	operator- (const TensorUInt128< HL, LL > &lhs, const TensorUInt128< HR, LR > &rhs)
template<typename HL , typename LL , typename HR , typename LR >
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorUInt128< uint64_t, uint64_t >	operator* (const TensorUInt128< HL, LL > &lhs, const TensorUInt128< HR, LR > &rhs)
template<typename HL , typename LL , typename HR , typename LR >
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorUInt128< uint64_t, uint64_t >	operator/ (const TensorUInt128< HL, LL > &lhs, const TensorUInt128< HR, LR > &rhs)
template<typename Index , std::size_t N, int... ii, int... jj>
constexpr static std::array< Index, N >	tensor_static_symgroup_index_permute (std::array< Index, N > idx, internal::numeric_list< int, ii... >, internal::numeric_list< int, jj... >)
template<typename Index , int... ii>
static std::vector< Index >	tensor_static_symgroup_index_permute (std::vector< Index > idx, internal::numeric_list< int, ii... >)
template<std::size_t I_, class T >
constexpr T &	array_get (std::vector< T > &a)
template<std::size_t I_, class T >
constexpr T &&	array_get (std::vector< T > &&a)
template<std::size_t I_, class T >
constexpr T const &	array_get (std::vector< T > const &a)
template<typename DerivativeType , typename OtherDerivativeType >
maybe_coherent_pad_helper< DerivativeType, OtherDerivativeType >::type	MaybeCoherentPad (const DerivativeType &x, const OtherDerivativeType &y)
template<typename Op , typename LhsDerivativeType , typename RhsDerivativeType >
auto	MakeCoherentCwiseBinaryOp (const LhsDerivativeType &x, const RhsDerivativeType &y, Op op=Op())
template<typename BVH , typename Intersector >
bool	intersect_helper (const BVH &tree, Intersector &intersector, typename BVH::Index root)
template<typename BVH , typename Minimizer >
Minimizer::Scalar	minimize_helper (const BVH &tree, Minimizer &minimizer, typename BVH::Index root, typename Minimizer::Scalar minimum)
template<typename T >
T *	fftw_cast (const T *p)
fftw_complex *	fftw_cast (const std::complex< double > *p)
fftwf_complex *	fftw_cast (const std::complex< float > *p)
fftwl_complex *	fftw_cast (const std::complex< long double > *p)
template<typename MatrixType , typename Rhs , typename Dest , typename Preconditioner >
bool	bicgstabl (const MatrixType &mat, const Rhs &rhs, Dest &x, const Preconditioner &precond, Index &iters, typename Dest::RealScalar &tol_error, Index L)
template<typename VectorType , typename IndexType >
void	sortWithPermutation (VectorType &vec, IndexType &perm, typename IndexType::Scalar &ncut)
	Computes a permutation vector to have a sorted sequence. More...
template<typename MatrixType , typename Rhs , typename Dest , typename Preconditioner >
bool	gmres (const MatrixType &mat, const Rhs &rhs, Dest &x, const Preconditioner &precond, Index &iters, const Index &restart, typename Dest::RealScalar &tol_error)
template<typename Vector , typename RealScalar >
Vector::Scalar	omega (const Vector &t, const Vector &s, RealScalar angle)
template<typename MatrixType , typename Rhs , typename Dest , typename Preconditioner >
bool	idrs (const MatrixType &A, const Rhs &b, Dest &x, const Preconditioner &precond, Index &iter, typename Dest::RealScalar &relres, Index S, bool smoothing, typename Dest::RealScalar angle, bool replacement)
template<typename MatrixType , typename Rhs , typename Dest , typename Preconditioner >
bool	idrstabl (const MatrixType &mat, const Rhs &rhs, Dest &x, const Preconditioner &precond, Index &iters, typename Dest::RealScalar &tol_error, Index L, Index S)
template<typename MatrixType , typename Rhs , typename Dest , typename Preconditioner >
EIGEN_DONT_INLINE void	minres (const MatrixType &mat, const Rhs &rhs, Dest &x, const Preconditioner &precond, Index &iters, typename Dest::RealScalar &tol_error)
template<typename QRSolver , typename VectorType >
void	lmpar2 (const QRSolver &qr, const VectorType &diag, const VectorType &qtb, typename VectorType::Scalar m_delta, typename VectorType::Scalar &par, VectorType &x)
template<typename Scalar >
void	covar (Matrix< Scalar, Dynamic, Dynamic > &r, const VectorXi &ipvt, Scalar tol=std::sqrt(NumTraits< Scalar >::epsilon()))
template<typename Scalar , int Rows, int Cols, typename PermIndex >
void	lmqrsolv (Matrix< Scalar, Rows, Cols > &s, const PermutationMatrix< Dynamic, Dynamic, PermIndex > &iPerm, const Matrix< Scalar, Dynamic, 1 > &diag, const Matrix< Scalar, Dynamic, 1 > &qtb, Matrix< Scalar, Dynamic, 1 > &x, Matrix< Scalar, Dynamic, 1 > &sdiag)
template<typename Scalar , int Options_, typename Index >
void	lmqrsolv (SparseMatrix< Scalar, Options_, Index > &s, const PermutationMatrix< Dynamic, Dynamic > &iPerm, const Matrix< Scalar, Dynamic, 1 > &diag, const Matrix< Scalar, Dynamic, 1 > &qtb, Matrix< Scalar, Dynamic, 1 > &x, Matrix< Scalar, Dynamic, 1 > &sdiag)
template<typename MatA , typename MatU , typename MatV >
void	matrix_exp_pade3 (const MatA &A, MatU &U, MatV &V)
	Compute the (3,3)-Padé approximant to the exponential. More...
template<typename MatA , typename MatU , typename MatV >
void	matrix_exp_pade5 (const MatA &A, MatU &U, MatV &V)
	Compute the (5,5)-Padé approximant to the exponential. More...
template<typename MatA , typename MatU , typename MatV >
void	matrix_exp_pade7 (const MatA &A, MatU &U, MatV &V)
	Compute the (7,7)-Padé approximant to the exponential. More...
template<typename MatA , typename MatU , typename MatV >
void	matrix_exp_pade9 (const MatA &A, MatU &U, MatV &V)
	Compute the (9,9)-Padé approximant to the exponential. More...
template<typename MatA , typename MatU , typename MatV >
void	matrix_exp_pade13 (const MatA &A, MatU &U, MatV &V)
	Compute the (13,13)-Padé approximant to the exponential. More...
template<typename ArgType , typename ResultType >
void	matrix_exp_compute (const ArgType &arg, ResultType &result, true_type)
template<typename ArgType , typename ResultType >
void	matrix_exp_compute (const ArgType &arg, ResultType &result, false_type)
template<typename MatrixType >
NumTraits< typename MatrixType::Scalar >::Real	matrix_function_compute_mu (const MatrixType &A)
template<typename Index , typename ListOfClusters >
ListOfClusters::iterator	matrix_function_find_cluster (Index key, ListOfClusters &clusters)
	Find cluster in clusters containing some value. More...
template<typename EivalsType , typename Cluster >
void	matrix_function_partition_eigenvalues (const EivalsType &eivals, std::list< Cluster > &clusters)
	Partition eigenvalues in clusters of ei'vals close to each other. More...
template<typename ListOfClusters , typename Index >
void	matrix_function_compute_cluster_size (const ListOfClusters &clusters, Matrix< Index, Dynamic, 1 > &clusterSize)
	Compute size of each cluster given a partitioning. More...
template<typename VectorType >
void	matrix_function_compute_block_start (const VectorType &clusterSize, VectorType &blockStart)
	Compute start of each block using clusterSize. More...
template<typename EivalsType , typename ListOfClusters , typename VectorType >
void	matrix_function_compute_map (const EivalsType &eivals, const ListOfClusters &clusters, VectorType &eivalToCluster)
	Compute mapping of eigenvalue indices to cluster indices. More...
template<typename DynVectorType , typename VectorType >
void	matrix_function_compute_permutation (const DynVectorType &blockStart, const DynVectorType &eivalToCluster, VectorType &permutation)
	Compute permutation which groups ei'vals in same cluster together. More...
template<typename VectorType , typename MatrixType >
void	matrix_function_permute_schur (VectorType &permutation, MatrixType &U, MatrixType &T)
	Permute Schur decomposition in U and T according to permutation. More...
template<typename MatrixType , typename AtomicType , typename VectorType >
void	matrix_function_compute_block_atomic (const MatrixType &T, AtomicType &atomic, const VectorType &blockStart, const VectorType &clusterSize, MatrixType &fT)
	Compute block diagonal part of matrix function. More...
template<typename MatrixType >
MatrixType	matrix_function_solve_triangular_sylvester (const MatrixType &A, const MatrixType &B, const MatrixType &C)
	Solve a triangular Sylvester equation AX + XB = C. More...
template<typename MatrixType , typename VectorType >
void	matrix_function_compute_above_diagonal (const MatrixType &T, const VectorType &blockStart, const VectorType &clusterSize, MatrixType &fT)
	Compute part of matrix function above block diagonal. More...
template<typename MatrixType >
void	matrix_log_compute_2x2 (const MatrixType &A, MatrixType &result)
	Compute logarithm of 2x2 triangular matrix. More...
int	matrix_log_get_pade_degree (float normTminusI)
int	matrix_log_get_pade_degree (double normTminusI)
int	matrix_log_get_pade_degree (long double normTminusI)
template<typename MatrixType >
void	matrix_log_compute_pade (MatrixType &result, const MatrixType &T, int degree)
template<typename MatrixType >
void	matrix_log_compute_big (const MatrixType &A, MatrixType &result)
	Compute logarithm of triangular matrices with size > 2. More...
template<typename MatrixType , typename ResultType >
void	matrix_sqrt_quasi_triangular_2x2_diagonal_block (const MatrixType &T, Index i, ResultType &sqrtT)
template<typename MatrixType , typename ResultType >
void	matrix_sqrt_quasi_triangular_1x1_off_diagonal_block (const MatrixType &T, Index i, Index j, ResultType &sqrtT)
template<typename MatrixType , typename ResultType >
void	matrix_sqrt_quasi_triangular_1x2_off_diagonal_block (const MatrixType &T, Index i, Index j, ResultType &sqrtT)
template<typename MatrixType , typename ResultType >
void	matrix_sqrt_quasi_triangular_2x1_off_diagonal_block (const MatrixType &T, Index i, Index j, ResultType &sqrtT)
template<typename MatrixType >
void	matrix_sqrt_quasi_triangular_solve_auxiliary_equation (MatrixType &X, const MatrixType &A, const MatrixType &B, const MatrixType &C)
template<typename MatrixType , typename ResultType >
void	matrix_sqrt_quasi_triangular_2x2_off_diagonal_block (const MatrixType &T, Index i, Index j, ResultType &sqrtT)
template<typename MatrixType , typename ResultType >
void	matrix_sqrt_quasi_triangular_diagonal (const MatrixType &T, ResultType &sqrtT)
template<typename MatrixType , typename ResultType >
void	matrix_sqrt_quasi_triangular_off_diagonal (const MatrixType &T, ResultType &sqrtT)
template<typename Scalar >
Scalar	stem_function_exp (Scalar x, int)
	The exponential function (and its derivatives). More...
template<typename Scalar >
Scalar	stem_function_cos (Scalar x, int n)
	Cosine (and its derivatives). More...
template<typename Scalar >
Scalar	stem_function_sin (Scalar x, int n)
	Sine (and its derivatives). More...
template<typename Scalar >
Scalar	stem_function_cosh (Scalar x, int n)
	Hyperbolic cosine (and its derivatives). More...
template<typename Scalar >
Scalar	stem_function_sinh (Scalar x, int n)
	Hyperbolic sine (and its derivatives). More...
template<typename Scalar >
void	chkder (const Matrix< Scalar, Dynamic, 1 > &x, const Matrix< Scalar, Dynamic, 1 > &fvec, const Matrix< Scalar, Dynamic, Dynamic > &fjac, Matrix< Scalar, Dynamic, 1 > &xp, const Matrix< Scalar, Dynamic, 1 > &fvecp, int mode, Matrix< Scalar, Dynamic, 1 > &err)
template<typename Scalar >
void	dogleg (const Matrix< Scalar, Dynamic, Dynamic > &qrfac, const Matrix< Scalar, Dynamic, 1 > &diag, const Matrix< Scalar, Dynamic, 1 > &qtb, Scalar delta, Matrix< Scalar, Dynamic, 1 > &x)
template<typename FunctorType , typename Scalar >
DenseIndex	fdjac1 (const FunctorType &Functor, Matrix< Scalar, Dynamic, 1 > &x, Matrix< Scalar, Dynamic, 1 > &fvec, Matrix< Scalar, Dynamic, Dynamic > &fjac, DenseIndex ml, DenseIndex mu, Scalar epsfcn)
template<typename Scalar >
void	lmpar (Matrix< Scalar, Dynamic, Dynamic > &r, const VectorXi &ipvt, const Matrix< Scalar, Dynamic, 1 > &diag, const Matrix< Scalar, Dynamic, 1 > &qtb, Scalar delta, Scalar &par, Matrix< Scalar, Dynamic, 1 > &x)
template<typename Scalar >
void	lmpar2 (const ColPivHouseholderQR< Matrix< Scalar, Dynamic, Dynamic > > &qr, const Matrix< Scalar, Dynamic, 1 > &diag, const Matrix< Scalar, Dynamic, 1 > &qtb, Scalar delta, Scalar &par, Matrix< Scalar, Dynamic, 1 > &x)
template<typename Scalar >
void	qrsolv (Matrix< Scalar, Dynamic, Dynamic > &s, const VectorXi &ipvt, const Matrix< Scalar, Dynamic, 1 > &diag, const Matrix< Scalar, Dynamic, 1 > &qtb, Matrix< Scalar, Dynamic, 1 > &x, Matrix< Scalar, Dynamic, 1 > &sdiag)
template<typename Scalar >
void	r1mpyq (DenseIndex m, DenseIndex n, Scalar *a, const std::vector< JacobiRotation< Scalar > > &v_givens, const std::vector< JacobiRotation< Scalar > > &w_givens)
template<typename Scalar >
void	r1updt (Matrix< Scalar, Dynamic, Dynamic > &s, const Matrix< Scalar, Dynamic, 1 > &u, std::vector< JacobiRotation< Scalar > > &v_givens, std::vector< JacobiRotation< Scalar > > &w_givens, Matrix< Scalar, Dynamic, 1 > &v, Matrix< Scalar, Dynamic, 1 > &w, bool *sing)
template<typename Scalar >
void	rwupdt (Matrix< Scalar, Dynamic, Dynamic > &r, const Matrix< Scalar, Dynamic, 1 > &w, Matrix< Scalar, Dynamic, 1 > &b, Scalar alpha)
template<typename Scalar , typename StorageIndex >
void	GetMarketLine (const char *line, StorageIndex &i, StorageIndex &j, Scalar &value)
template<>
void	GetMarketLine (const char *line, int &i, int &j, float &value)
template<>
void	GetMarketLine (const char *line, int &i, int &j, double &value)
template<>
void	GetMarketLine (const char *line, int &i, int &j, std::complex< float > &value)
template<>
void	GetMarketLine (const char *line, int &i, int &j, std::complex< double > &value)
template<typename Scalar , typename StorageIndex >
void	GetMarketLine (const char *line, StorageIndex &i, StorageIndex &j, std::complex< Scalar > &value)
template<typename RealScalar >
void	GetDenseElt (const std::string &line, RealScalar &val)
template<typename RealScalar >
void	GetDenseElt (const std::string &line, std::complex< RealScalar > &val)
template<typename Scalar >
void	putMarketHeader (std::string &header, int sym)
template<typename Scalar , typename StorageIndex >
void	PutMatrixElt (Scalar value, StorageIndex row, StorageIndex col, std::ofstream &out)
template<typename Scalar , typename StorageIndex >
void	PutMatrixElt (std::complex< Scalar > value, StorageIndex row, StorageIndex col, std::ofstream &out)
template<typename Scalar >
void	putDenseElt (Scalar value, std::ofstream &out)
template<typename Scalar >
void	putDenseElt (std::complex< Scalar > value, std::ofstream &out)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	pbessel_i0 (const Packet &x)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	pbessel_i0e (const Packet &x)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	pbessel_i1 (const Packet &x)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	pbessel_i1e (const Packet &x)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	pbessel_j0 (const Packet &x)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	pbessel_j1 (const Packet &x)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	pbessel_y0 (const Packet &x)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	pbessel_y1 (const Packet &x)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	pbessel_k0 (const Packet &x)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	pbessel_k0e (const Packet &x)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	pbessel_k1 (const Packet &x)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	pbessel_k1e (const Packet &x)
template<typename T >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T	erf_over_x_double_small (const T &x2)
template<typename T >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T	erfc_double_large (const T &x, const T &x2)
template<typename T >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T	flipsign (const T &should_flipsign, const T &x)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double	flipsign< double > (const double &should_flipsign, const double &x)
template<>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float	flipsign< float > (const float &should_flipsign, const float &x)
template<typename T , typename ScalarType >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T	generic_ndtri_gt_exp_neg_two (const T &b)
template<typename T , typename ScalarType >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T	generic_ndtri_lt_exp_neg_two (const T &b, const T &should_flipsign)
template<typename T , typename ScalarType >
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T	generic_ndtri (const T &a)
template<typename Scalar >
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar	main_igamma_term (Scalar a, Scalar x)
template<typename Scalar , IgammaComputationMode mode>
EIGEN_DEVICE_FUNC int	igamma_num_iterations ()
template<typename Packet >
EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	plgamma (const Packet &a)
template<typename Packet >
EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	pdigamma (const Packet &a)
template<typename Packet >
EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	pzeta (const Packet &x, const Packet &q)
template<typename Packet >
EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	ppolygamma (const Packet &n, const Packet &x)
template<typename Packet >
EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	perf (const Packet &a)
template<typename Packet >
EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	perfc (const Packet &a)
template<typename Packet >
EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet	pndtri (const Packet &a)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet	pigamma (const Packet &a, const Packet &x)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet	pigamma_der_a (const Packet &a, const Packet &x)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet	pgamma_sample_der_alpha (const Packet &alpha, const Packet &sample)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet	pigammac (const Packet &a, const Packet &x)
template<typename Packet >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet	pbetainc (const Packet &a, const Packet &b, const Packet &x)

Variables
static Packet4ui	p4ui_CONJ_XOR
static const Packet16uc	p16uc_GETREAL32 = {0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27}
static const Packet16uc	p16uc_GETIMAG32 = {4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31}
static const Packet16uc	p16uc_GETREAL32b = {0, 1, 2, 3, 16, 17, 18, 19, 8, 9, 10, 11, 24, 25, 26, 27}
static const Packet16uc	p16uc_GETIMAG32b = {4, 5, 6, 7, 20, 21, 22, 23, 12, 13, 14, 15, 28, 29, 30, 31}
static const Packet4i	mask4 [4] = {{0, 0, 0, 0}, {-1, 0, 0, 0}, {-1, -1, 0, 0}, {-1, -1, -1, 0}}
static Packet16uc	p16uc_MERGE16_32_1 = {0, 1, 16, 17, 2, 3, 18, 19, 0, 1, 16, 17, 2, 3, 18, 19}
static Packet16uc	p16uc_MERGE16_32_2 = {4, 5, 20, 21, 6, 7, 22, 23, 4, 5, 20, 21, 6, 7, 22, 23}
static Packet16uc	p16uc_MERGE16_32_3 = {8, 9, 24, 25, 10, 11, 26, 27, 8, 9, 24, 25, 10, 11, 26, 27}
static Packet16uc	p16uc_MERGE16_32_4 = {12, 13, 28, 29, 14, 15, 30, 31, 12, 13, 28, 29, 14, 15, 30, 31}
static Packet16uc	p16uc_MERGE16_32_5 = {0, 1, 16, 17, 16, 17, 16, 17, 0, 1, 16, 17, 16, 17, 16, 17}
static Packet16uc	p16uc_MERGE16_32_6 = {2, 3, 18, 19, 18, 19, 18, 19, 2, 3, 18, 19, 18, 19, 18, 19}
static Packet16uc	p16uc_MERGE16_32_7 = {4, 5, 20, 21, 20, 21, 20, 21, 4, 5, 20, 21, 20, 21, 20, 21}
static Packet16uc	p16uc_MERGE16_32_8 = {6, 7, 22, 23, 22, 23, 22, 23, 6, 7, 22, 23, 22, 23, 22, 23}
static Packet16uc	p16uc_ELEMENT_VEC3
static Packet4f	p4f_MZERO
static Packet4f	p4f_ONE = vec_ctf(p4i_ONE, 0)
static Packet4f	p4f_COUNTDOWN = {0.0, 1.0, 2.0, 3.0}
static Packet4i	p4i_COUNTDOWN = {0, 1, 2, 3}
static Packet8s	p8s_COUNTDOWN = {0, 1, 2, 3, 4, 5, 6, 7}
static Packet8us	p8us_COUNTDOWN = {0, 1, 2, 3, 4, 5, 6, 7}
static Packet16c	p16c_COUNTDOWN = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}
static Packet16uc	p16uc_COUNTDOWN = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}
static Packet16uc	p16uc_REVERSE32 = {12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3}
static Packet16uc	p16uc_REVERSE16 = {14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1}
static Packet16uc	p16uc_REVERSE8 = {15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0}
static const Packet16uc	p16uc_DUPLICATE16_EVEN = {0, 1, 0, 1, 4, 5, 4, 5, 8, 9, 8, 9, 12, 13, 12, 13}
static const Packet16uc	p16uc_DUPLICATE16_ODD = {2, 3, 2, 3, 6, 7, 6, 7, 10, 11, 10, 11, 14, 15, 14, 15}
static Packet16uc	p16uc_QUADRUPLICATE16_HI = {0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3}
static Packet16uc	p16uc_QUADRUPLICATE16 = {0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3}
static Packet16uc	p16uc_MERGEE16 = {0, 1, 16, 17, 4, 5, 20, 21, 8, 9, 24, 25, 12, 13, 28, 29}
static Packet16uc	p16uc_MERGEO16 = {2, 3, 18, 19, 6, 7, 22, 23, 10, 11, 26, 27, 14, 15, 30, 31}
static Packet16uc	p16uc_MERGEL16 = {2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31}
static Packet16uc	p16uc_FORWARD = p16uc_REVERSE32
static Packet16uc	p16uc_PSET32_WODD
static Packet16uc	p16uc_PSET32_WEVEN
static Packet16uc	p16uc_HALF64_0_16
static Packet16uc	p16uc_PSET64_HI
static Packet16uc	p16uc_PSET64_LO
static Packet16uc	p16uc_TRANSPOSE64_HI
static Packet16uc	p16uc_TRANSPOSE64_LO
static Packet16uc	p16uc_COMPLEX32_REV
static Packet4ui	p4ui_CONJ_XOR
static Packet2ul	p2ul_CONJ_XOR1
static Packet2ul	p2ul_CONJ_XOR2
static Packet2d	p2d_ONE = {1.0, 1.0}
static Packet2d	p2d_ZERO_
static Packet4f	p4f_MZERO = {0x80000000, 0x80000000, 0x80000000, 0x80000000}
static Packet4i	p4i_COUNTDOWN = {0, 1, 2, 3}
static Packet4f	p4f_COUNTDOWN = {0.0, 1.0, 2.0, 3.0}
static Packet2d	p2d_COUNTDOWN
static Packet16uc	p16uc_PSET64_HI = {0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7}
static Packet16uc	p16uc_DUPLICATE32_HI = {0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7, 4, 5, 6, 7}
static Packet16uc	p16uc_FORWARD = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}
static Packet16uc	p16uc_REVERSE32 = {12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3}
static Packet16uc	p16uc_REVERSE64 = {8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7}
static Packet16uc	p16uc_PSET32_WODD
static Packet16uc	p16uc_PSET32_WEVEN
static Packet16uc	p16uc_PSET64_LO
static Packet16uc	p16uc_TRANSPOSE64_HI = {0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23}
static Packet16uc	p16uc_TRANSPOSE64_LO = {8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31}
static Packet16uc	p16uc_COMPLEX32_REV
static Packet16uc	p16uc_COMPLEX32_REV2
const std::ptrdiff_t	defaultL1CacheSize = EIGEN_SET_DEFAULT_L1_CACHE_SIZE(16 * 1024)
const std::ptrdiff_t	defaultL2CacheSize = EIGEN_SET_DEFAULT_L2_CACHE_SIZE(512 * 1024)
const std::ptrdiff_t	defaultL3CacheSize = EIGEN_SET_DEFAULT_L3_CACHE_SIZE(512 * 1024)
const Scalar &	y
template<typename A >
constexpr bool	is_int_or_enum_v = std::is_enum<A>::value || std::is_integral<A>::value
template<typename... Ts>
constexpr EIGEN_DEVICE_FUNCdecltype(reduce< product_op, Ts... >::run((*((Ts *) 0))...)) EIGEN_STRONG_INLIN	arg_prod )(Ts... ts)
template<typename... Ts>
constexpr decltype(reduce< sum_op, Ts... >::run((*((Ts *) 0))...)) EIGEN_STRONG_INLIN	arg_sum )(Ts... ts)
static const float	matrix_function_separation = 0.1f
	Maximum distance allowed between eigenvalues to be considered "close". More...

Detailed Description

Namespace containing low-level routines from the Eigen library.

Typedef Documentation

__attribute__

typedef svfloat32_t PacketXf Eigen::internal::__attribute__((arm_sve_vector_bits(EIGEN_ARM64_SVE_VL)))

add_const_on_value_type_t

template<typename T >

using Eigen::internal::add_const_on_value_type_t = typedef typename add_const_on_value_type<T>::type

cmp_return_t

template<typename LhsScalar , typename RhsScalar , bool UseTypedComparators>

using Eigen::internal::cmp_return_t = typedef typename typed_cmp_helper<LhsScalar, RhsScalar, UseTypedComparators>::type

conj_expr_if

template<bool Cond, typename T >

using Eigen::internal::conj_expr_if = typedef std::conditional<!Cond, const T&, CwiseUnaryOp<scalar_conjugate_op<typename traits<T>::Scalar>, T> >

extract_output_indices

template<typename... Types>

using Eigen::internal::extract_output_indices = typedef typename extract_output_indices_helper<sizeof...(Types), 0, std::index_sequence<>, Types...>::type

IvcType

template<typename Indices , int SizeAtCompileTime>

using Eigen::internal::IvcType = typedef typename internal::IndexedViewHelperIndicesWrapper<Indices, SizeAtCompileTime>::type

Packet16b

typedef eigen_packet_wrapper<__m128i, 1> Eigen::internal::Packet16b

Packet16bf

typedef eigen_packet_wrapper<__m256i, 2> Eigen::internal::Packet16bf

Packet16c

typedef int8x16_t Eigen::internal::Packet16c

Packet16f

typedef __m512 Eigen::internal::Packet16f

Packet16h

typedef eigen_packet_wrapper< __m256i, 1 > Eigen::internal::Packet16h

Packet16i

typedef __m512i Eigen::internal::Packet16i

Packet16uc

typedef __vector unsigned char Eigen::internal::Packet16uc

Packet2d

typedef __vector double Eigen::internal::Packet2d

Packet2f

typedef float32x2_t Eigen::internal::Packet2f

Packet2i

typedef int32x2_t Eigen::internal::Packet2i

Packet2l

typedef __vector long long Eigen::internal::Packet2l

Packet2ui

typedef uint32x2_t Eigen::internal::Packet2ui

Packet2ul

typedef __vector unsigned long long Eigen::internal::Packet2ul

Packet32h

typedef __m512h Eigen::internal::Packet32h

Packet4bf

typedef eigen_packet_wrapper<uint16x4_t, 19> Eigen::internal::Packet4bf

Packet4bi

typedef __vector __bool int Eigen::internal::Packet4bi

Packet4c

typedef eigen_packet_wrapper<int32_t, 2> Eigen::internal::Packet4c

Packet4d

typedef __m256d Eigen::internal::Packet4d

Packet4f

typedef __vector float Eigen::internal::Packet4f

Packet4i

typedef __vector int Eigen::internal::Packet4i

Packet4s

typedef int16x4_t Eigen::internal::Packet4s

Packet4uc

typedef eigen_packet_wrapper<uint32_t, 5> Eigen::internal::Packet4uc

Packet4ui

typedef __vector unsigned int Eigen::internal::Packet4ui

Packet4us

typedef uint16x4_t Eigen::internal::Packet4us

Packet8bf

typedef eigen_packet_wrapper< __m128i, 3 > Eigen::internal::Packet8bf

Packet8bi

typedef __vector __bool short Eigen::internal::Packet8bi

Packet8c

typedef int8x8_t Eigen::internal::Packet8c

Packet8d

typedef __m512d Eigen::internal::Packet8d

Packet8f

typedef __m256 Eigen::internal::Packet8f

Packet8h

typedef eigen_packet_wrapper< __m128i, 2 > Eigen::internal::Packet8h

Packet8i

typedef __vector short int Eigen::internal::Packet8i

Packet8l

typedef eigen_packet_wrapper<__m512i, 1> Eigen::internal::Packet8l

Packet8s

typedef int16x8_t Eigen::internal::Packet8s

Packet8uc

typedef uint8x8_t Eigen::internal::Packet8uc

Packet8ui

typedef eigen_packet_wrapper<__m256i, 4> Eigen::internal::Packet8ui

Packet8us

typedef uint16x8_t Eigen::internal::Packet8us

reduce_all

template<bool... values>

using Eigen::internal::reduce_all = typedef std::is_same<std::integer_sequence<bool, values..., true>, std::integer_sequence<bool, true, values...> >

reduce_any

template<bool... values>

using Eigen::internal::reduce_any = typedef std::integral_constant<bool, !std::is_same<std::integer_sequence<bool, values..., false>, std::integer_sequence<bool, false, values...> >::value>

remove_all_t

template<typename T >

using Eigen::internal::remove_all_t = typedef typename remove_all<T>::type

SsePrefetchPtrType

typedef const char* Eigen::internal::SsePrefetchPtrType

vecFullDouble

typedef Packet8d Eigen::internal::vecFullDouble

vecFullFloat

typedef Packet16f Eigen::internal::vecFullFloat

vecHalfDouble

typedef Packet4d Eigen::internal::vecHalfDouble

vecHalfFloat

typedef Packet8f Eigen::internal::vecHalfFloat

void_t

template<typename... >

using Eigen::internal::void_t = typedef void

Implementation of std::void_t for SFINAE.

Pre C++17: Custom implementation.

Post C++17: Uses std::void_t

WithStorageOrder

template<class Source , int Order>

using Eigen::internal::WithStorageOrder = typedef SparseMatrix<typename Source::Scalar, Order, typename Source::StorageIndex>

Enumeration Type Documentation

anonymous enum

anonymous enum

Enumerator
meta_floor_log2_terminate
meta_floor_log2_move_up
meta_floor_log2_move_down
meta_floor_log2_bogus

{ meta_floor_log2_terminate, meta_floor_log2_move_up, meta_floor_log2_move_down, meta_floor_log2_bogus };

anonymous enum

anonymous enum

Enumerator
SDP_AsScalarProduct
SDP_AsCwiseProduct

{ SDP_AsScalarProduct, SDP_AsCwiseProduct };

anonymous enum

anonymous enum

Enumerator
SVA_RuntimeSwitch
SVA_Inner
SVA_Outer

{ SVA_RuntimeSwitch, SVA_Inner, SVA_Outer };

anonymous enum

anonymous enum

Enumerator
LUNoMarker

{ LUNoMarker = 3 };

anonymous enum

anonymous enum

Enumerator
emptyIdxLU

{ emptyIdxLU = -1 };

anonymous enum

anonymous enum

Enumerator
PreconditionIfMoreColsThanRows
PreconditionIfMoreRowsThanCols

{ PreconditionIfMoreColsThanRows, PreconditionIfMoreRowsThanCols };

anonymous enum

anonymous enum

Enumerator
ShardByRow
ShardByCol

{ ShardByRow = 0, ShardByCol = 1 };

anonymous enum

anonymous enum

Enumerator
Rhs
Lhs

{ Rhs = 0, Lhs = 1 };

ComparisonName

enum Eigen::internal::ComparisonName : unsigned int

Constants for comparison functors

Enumerator
cmp_EQ
cmp_LT
cmp_LE
cmp_UNORD
cmp_NEQ
cmp_GT
cmp_GE

                    : unsigned int {
  cmp_EQ = 0,
  cmp_LT = 1,
  cmp_LE = 2,
  cmp_UNORD = 3,
  cmp_NEQ = 4,
  cmp_GT = 5,
  cmp_GE = 6
};

GEBPPacketSizeType

enum Eigen::internal::GEBPPacketSizeType

Enumerator
GEBPPacketFull
GEBPPacketHalf
GEBPPacketQuarter

{ GEBPPacketFull = 0, GEBPPacketHalf, GEBPPacketQuarter };

GEMVPacketSizeType

enum Eigen::internal::GEMVPacketSizeType

Enumerator
GEMVPacketFull
GEMVPacketHalf
GEMVPacketQuarter

{ GEMVPacketFull = 0, GEMVPacketHalf, GEMVPacketQuarter };

IgammaComputationMode

enum Eigen::internal::IgammaComputationMode

Enumerator
VALUE
DERIVATIVE
SAMPLE_DERIVATIVE

{ VALUE, DERIVATIVE, SAMPLE_DERIVATIVE };

MemType

enum Eigen::internal::MemType

Enumerator
LUSUP
UCOL
LSUB
USUB
LLVL
ULVL

{ LUSUP, UCOL, LSUB, USUB, LLVL, ULVL };

OptionsMasks

enum Eigen::internal::OptionsMasks

Enumerator
QRPreconditionerBits
ComputationOptionsBits

                  {
  QRPreconditionerBits = NoQRPreconditioner | HouseholderQRPreconditioner | ColPivHouseholderQRPreconditioner |
                         FullPivHouseholderQRPreconditioner,
  ComputationOptionsBits = ComputeThinU | ComputeFullU | ComputeThinV | ComputeFullV
};

PermPermProduct_t

enum Eigen::internal::PermPermProduct_t

Enumerator
PermPermProduct

{ PermPermProduct };

SignMatrix

enum Eigen::internal::SignMatrix

Enumerator
PositiveSemiDef
NegativeSemiDef
ZeroSign
Indefinite

{ PositiveSemiDef, NegativeSemiDef, ZeroSign, Indefinite };

TensorBlockKind

enum Eigen::internal::TensorBlockKind

Enumerator
kExpr
kView
kMaterializedInScratch
kMaterializedInOutput

                     {
  // Tensor block that is a lazy expression that must be assigned to a
  // destination using TensorBlockAssign.
  kExpr,
 
  // Tensor block that is a view into a memory buffer owned by an underlying
  // Tensor expression (e.g. it can be a view into a Tensor buffer).
  kView,
 
  // Tensor block that was materialized in a scratch memory buffer, allocated
  // with TensorBlockScratchAllocator. This block must be copied to a
  // destination, similar to a block of `kExpr` type.
  kMaterializedInScratch,
 
  // Tensor block that was materialized directly into the final output memory
  // buffer. For example if the left side of an assignment is a Tensor, we can
  // directly materialize the block in the destination memory.
  //
  // If strides in the output buffer do not match tensor block strides, the
  // Tensor expression will be invalid, and should not be used by
  // TensorBlockAssign or for constructing another block expression.
  kMaterializedInOutput
};

TensorBlockShapeType

enum Eigen::internal::TensorBlockShapeType

strong

Enumerator
kUniformAllDims
kSkewedInnerDims

{ kUniformAllDims, kSkewedInnerDims };

TiledEvaluation

enum Eigen::internal::TiledEvaluation

Enumerator
Off
On

                     {
  Off = 0,  // tiled evaluation is not supported
  On = 1,   // still work in progress (see TensorBlock.h)
};

Function Documentation

__UNPACK_TYPE__()

template<typename Packet >

EIGEN_STRONG_INLINE Eigen::internal::__UNPACK_TYPE__

(

Packet

)

const &

                                                                           {
  EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) x;
  vec_ste(a, 0, &x);
  return x;
}

References a, EIGEN_ALIGN16, and plotDoE::x.

Referenced by bscalec(), bsetzero(), pgather_common(), pload_common(), pload_ignore(), pload_partial_common(), ploadu_common(), ploadu_partial_common(), pscatter_common(), pstore_partial_common(), and pstoreu_partial_common().

_mm_extract_epi64_0()

EIGEN_ALWAYS_INLINE int64_t Eigen::internal::_mm_extract_epi64_0

(

const __m128i &

a

)

                                                                  {
  return numext::bit_cast<int64_t>(_mm_cvtsd_f64(_mm_castsi128_pd(a)));
}

References a.

Referenced by pfirst< Packet8l >().

_mm_extract_epi64_1()

EIGEN_ALWAYS_INLINE int64_t Eigen::internal::_mm_extract_epi64_1

(

const __m128i &

a

)

                                                                  {
  return numext::bit_cast<int64_t>(_mm_cvtsd_f64(_mm_shuffle_pd(_mm_castsi128_pd(a), _mm_castsi128_pd(a), 0x1)));
}

References a.

absolute_split()

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::absolute_split	(	const Packet &	x,
		Packet &	n,
		Packet &	r
	)

                                                                                                 {
  n = pround(x);
  r = psub(x, n);
}

References n, pround(), psub(), UniformPSDSelfTest::r, and plotDoE::x.

Referenced by generic_pow_impl().

addResults()

template<Index num_acc>

EIGEN_ALWAYS_INLINE void Eigen::internal::addResults

(

Packet4f(&)

acc[num_acc][4]

)

                                                                 {
  for (Index i = 0, j = 0; j < num_acc; i++, j += 2) {
    for (Index x = 0, y = 0; x < 2; x++, y += 2) {
      for (Index w = 0, z = 0; w < 2; w++, z += 2) {
        acc[i][y + w] = acc[j + x][z + 0] + acc[j + x][z + 1];
      }
    }
  }
}

References i, j, w, plotDoE::x, and y.

aligned_delete()

template<typename T >

EIGEN_DEVICE_FUNC void Eigen::internal::aligned_delete ( T *  ptr, std::size_t  size  )

inline

Deletes objects constructed with aligned_new The size parameters tells on how many objects to call the destructor of T.

                                                                     {
  destruct_elements_of_array<T>(ptr, size);
  aligned_free(ptr);
}

References aligned_free(), and size.

Referenced by check_aligned_new(), dontalign(), EIGEN_DECLARE_TEST(), map_class_matrix(), map_class_vector(), map_static_methods(), and Eigen::internal::gemm_blocking_space< StorageOrder, LhsScalar_, RhsScalar_, MaxRows, MaxCols, MaxDepth, KcFactor, false >::~gemm_blocking_space().

aligned_free()

EIGEN_DEVICE_FUNC void Eigen::internal::aligned_free ( void *  ptr )

inline

Frees memory allocated with aligned_malloc.

                                                      {
#if (EIGEN_DEFAULT_ALIGN_BYTES == 0) || EIGEN_MALLOC_ALREADY_ALIGNED
 
  if (ptr != nullptr) {
    check_that_malloc_is_allowed();
    EIGEN_USING_STD(free)
    free(ptr);
  }
 
#else
  handmade_aligned_free(ptr);
#endif
}

References check_that_malloc_is_allowed(), EIGEN_USING_STD, and handmade_aligned_free().

Referenced by aligned_delete(), aligned_new(), aligned_realloc(), check_aligned_malloc(), conditional_aligned_free(), Eigen::aligned_allocator< T >::deallocate(), Eigen::DefaultDevice::deallocate(), TestAllocator::deallocate(), and Eigen::internal::aligned_stack_memory_handler< T >::~aligned_stack_memory_handler().

aligned_malloc()

EIGEN_DEVICE_FUNC void* Eigen::internal::aligned_malloc ( std::size_t  size )

inline

Allocates size bytes. The returned pointer is guaranteed to have 16 or 32 bytes alignment depending on the requirements. On allocation error, the returned pointer is null, and std::bad_alloc is thrown.

                                                              {
  if (size == 0) return nullptr;
 
  void* result;
#if (EIGEN_DEFAULT_ALIGN_BYTES == 0) || EIGEN_MALLOC_ALREADY_ALIGNED
 
  check_that_malloc_is_allowed();
  EIGEN_USING_STD(malloc)
  result = malloc(size);
 
#if EIGEN_DEFAULT_ALIGN_BYTES == 16
  eigen_assert((size < 16 || (std::size_t(result) % 16) == 0) &&
               "System's malloc returned an unaligned pointer. Compile with EIGEN_MALLOC_ALREADY_ALIGNED=0 to fallback "
               "to handmade aligned memory allocator.");
#endif
#else
  result = handmade_aligned_malloc(size);
#endif
 
  if (!result && size) throw_std_bad_alloc();
 
  return result;
}

References check_that_malloc_is_allowed(), eigen_assert, EIGEN_USING_STD, handmade_aligned_malloc(), size, and throw_std_bad_alloc().

Referenced by aligned_new(), aligned_realloc(), Eigen::DefaultDevice::allocate(), TestAllocator::allocate(), Eigen::aligned_allocator< T >::allocate(), check_aligned_malloc(), conditional_aligned_malloc(), and Eigen::internal::trmv_selector< Mode, RowMajor >::run().

aligned_new()

template<typename T >

EIGEN_DEVICE_FUNC T* Eigen::internal::aligned_new ( std::size_t  size )

inline

Allocates size objects of type T. The returned pointer is guaranteed to have 16 bytes alignment. On allocation error, the returned pointer is undefined, but a std::bad_alloc is thrown. The default constructor of T is called.

                                                        {
  check_size_for_overflow<T>(size);
  T* result = static_cast<T*>(aligned_malloc(sizeof(T) * size));
  EIGEN_TRY { return default_construct_elements_of_array(result, size); }
  EIGEN_CATCH(...) {
    aligned_free(result);
    EIGEN_THROW;
  }
  return result;
}

References aligned_free(), aligned_malloc(), default_construct_elements_of_array(), EIGEN_CATCH, EIGEN_THROW, EIGEN_TRY, and size.

aligned_realloc()

EIGEN_DEVICE_FUNC void* Eigen::internal::aligned_realloc ( void *  ptr, std::size_t  new_size, std::size_t  old_size  )

inline

Reallocates an aligned block of memory.

Exceptions

std::bad_alloc

on allocation failure

                                                                                                  {
  if (ptr == nullptr) return aligned_malloc(new_size);
  if (old_size == new_size) return ptr;
  if (new_size == 0) {
    aligned_free(ptr);
    return nullptr;
  }
 
  void* result;
#if (EIGEN_DEFAULT_ALIGN_BYTES == 0) || EIGEN_MALLOC_ALREADY_ALIGNED
  EIGEN_UNUSED_VARIABLE(old_size)
 
  check_that_malloc_is_allowed();
  EIGEN_USING_STD(realloc)
  result = realloc(ptr, new_size);
#else
  result = handmade_aligned_realloc(ptr, new_size, old_size);
#endif
 
  if (!result && new_size) throw_std_bad_alloc();
 
  return result;
}

References aligned_free(), aligned_malloc(), check_that_malloc_is_allowed(), EIGEN_UNUSED_VARIABLE, EIGEN_USING_STD, handmade_aligned_realloc(), and throw_std_bad_alloc().

Referenced by conditional_aligned_realloc().

all() [1/2]

EIGEN_DEVICE_FUNC bool Eigen::internal::all ( )

inline

{

Referenced by Eigen::TensorMap< PlainObjectType, Options_, MakePointer_ >::operator()().

all() [2/2]

template<typename T , typename... Ts>

EIGEN_DEVICE_FUNC bool Eigen::internal::all	(	T	t,
		Ts...	ts
	)

                                    { return true; }
 
template <typename T, typename... Ts>

all_indices_known_statically()

template<typename T >

static EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bool Eigen::internal::all_indices_known_statically ( )

static

                                                                             {
  return all_indices_known_statically_impl<T>::run();
}

References Eigen::internal::all_indices_known_statically_impl< T >::run().

amd_flip()

template<typename T >

T Eigen::internal::amd_flip ( const T &  i )

inline

                              {
  return -i - 2;
}

References i.

Referenced by amd_mark(), amd_unflip(), and minimum_degree_ordering().

amd_mark()

template<typename T0 , typename T1 >

void Eigen::internal::amd_mark ( const T0 *  w, const T1 &  j  )

inline

                                               {
  return w[j] = amd_flip(w[j]);
}

References amd_flip(), j, and w.

amd_marked()

template<typename T0 , typename T1 >

bool Eigen::internal::amd_marked ( const T0 *  w, const T1 &  j  )

inline

                                                 {
  return w[j] < 0;
}

References j, and w.

amd_unflip()

template<typename T >

T Eigen::internal::amd_unflip ( const T &  i )

inline

                                {
  return i < 0 ? amd_flip(i) : i;
}

References amd_flip(), and i.

apply_block_householder_on_the_left()

template<typename MatrixType , typename VectorsType , typename CoeffsType >

void Eigen::internal::apply_block_householder_on_the_left	(	MatrixType &	mat,
		const VectorsType &	vectors,
		const CoeffsType &	hCoeffs,
		bool	forward
	)

if forward then perform mat = H0 * H1 * H2 * mat otherwise perform mat = H2 * H1 * H0 * mat

                                                       {
  enum { TFactorSize = VectorsType::ColsAtCompileTime };
  Index nbVecs = vectors.cols();
  Matrix<typename MatrixType::Scalar, TFactorSize, TFactorSize, RowMajor> T(nbVecs, nbVecs);
 
  if (forward)
    make_block_householder_triangular_factor(T, vectors, hCoeffs);
  else
    make_block_householder_triangular_factor(T, vectors, hCoeffs.conjugate());
  const TriangularView<const VectorsType, UnitLower> V(vectors);
 
  // A -= V T V^* A
  Matrix<typename MatrixType::Scalar, VectorsType::ColsAtCompileTime, MatrixType::ColsAtCompileTime,
         (VectorsType::MaxColsAtCompileTime == 1 && MatrixType::MaxColsAtCompileTime != 1) ? RowMajor : ColMajor,
         VectorsType::MaxColsAtCompileTime, MatrixType::MaxColsAtCompileTime>
      tmp = V.adjoint() * mat;
  // FIXME add .noalias() once the triangular product can work inplace
  if (forward)
    tmp = T.template triangularView<Upper>() * tmp;
  else
    tmp = T.template triangularView<Upper>().adjoint() * tmp;
  mat.noalias() -= V * tmp;
}

References Eigen::ColMajor, make_block_householder_triangular_factor(), Eigen::RowMajor, tmp, and V.

Referenced by Eigen::HouseholderSequence< VectorsType, CoeffsType, Side >::applyThisOnTheLeft(), and Eigen::internal::householder_qr_inplace_blocked< MatrixQR, HCoeffs, MatrixQRScalar, InnerStrideIsOne >::run().

apply_rotation_in_the_plane()

template<typename VectorX , typename VectorY , typename OtherScalar >

EIGEN_DEVICE_FUNC void Eigen::internal::apply_rotation_in_the_plane	(	DenseBase< VectorX > &	xpr_x,
		DenseBase< VectorY > &	xpr_y,
		const JacobiRotation< OtherScalar > &	j
	)

\jacobi_module Applies the clock wise 2D rotation j to the set of 2D vectors of coordinates x and y: \( \left ( \begin{array}{cc} x \\ y \end{array} \right ) = J \left ( \begin{array}{cc} x \\ y \end{array} \right ) \)

See also

MatrixBase::applyOnTheLeft(), MatrixBase::applyOnTheRight()

                                                                                                {
  typedef typename VectorX::Scalar Scalar;
  constexpr bool Vectorizable = (int(evaluator<VectorX>::Flags) & int(evaluator<VectorY>::Flags) & PacketAccessBit) &&
                                (int(packet_traits<Scalar>::size) == int(packet_traits<OtherScalar>::size));
 
  eigen_assert(xpr_x.size() == xpr_y.size());
  Index size = xpr_x.size();
  Index incrx = xpr_x.derived().innerStride();
  Index incry = xpr_y.derived().innerStride();
 
  Scalar* EIGEN_RESTRICT x = &xpr_x.derived().coeffRef(0);
  Scalar* EIGEN_RESTRICT y = &xpr_y.derived().coeffRef(0);
 
  OtherScalar c = j.c();
  OtherScalar s = j.s();
  if (numext::is_exactly_one(c) && numext::is_exactly_zero(s)) return;
 
  constexpr int Alignment = (std::min)(int(evaluator<VectorX>::Alignment), int(evaluator<VectorY>::Alignment));
  apply_rotation_in_the_plane_selector<Scalar, OtherScalar, VectorX::SizeAtCompileTime, Alignment, Vectorizable>::run(
      x, incrx, y, incry, size, c, s);
}

References calibrate::c, eigen_assert, EIGEN_RESTRICT, int(), Eigen::numext::is_exactly_one(), Eigen::numext::is_exactly_zero(), j, min, Eigen::PacketAccessBit, Eigen::internal::apply_rotation_in_the_plane_selector< Scalar, OtherScalar, SizeAtCompileTime, MinAlignment, Vectorizable >::run(), s, size, plotDoE::x, and y.

Referenced by Eigen::MatrixBase< Derived >::applyOnTheLeft(), Eigen::MatrixBase< Derived >::applyOnTheRight(), llt_rank_update_lower(), and rot().

array_apply()

template<typename Op , typename A , std::size_t N>

constexpr EIGEN_STRONG_INLINE array<decltype(Op::run(A())), N> Eigen::internal::array_apply ( array< A, N >  a )

constexpr

                                                                                        {
  return h_array_apply<Op>(a, typename gen_numeric_list<int, N>::type());
}

References a.

array_apply_and_reduce()

template<typename Reducer , typename Op , typename A , std::size_t N>

constexpr EIGEN_STRONG_INLINE auto Eigen::internal::array_apply_and_reduce ( array< A, N >  a ) -> decltype(h_array_apply_and_reduce<Reducer, Op, A, N>(a, typename gen_numeric_list<int, N>::type()))

constexpr

                                                                                                         {
  return h_array_apply_and_reduce<Reducer, Op, A, N>(a, typename gen_numeric_list<int, N>::type());
}

References a.

Referenced by Eigen::TensorFixedSize< Scalar_, Dimensions_, Options_, IndexType >::checkIndexRange(), and Eigen::Tensor< Scalar_, NumIndices_, Options_, IndexType_ >::checkIndexRange().

array_get() [1/15]

template<DenseIndex n, typename Index , std::size_t Rank>

const Index Eigen::internal::array_get

(

const DimensionList< Index, Rank > &

)

                                                         {
  return n;
}

References n.

array_get() [2/15]

template<Index N, typename FirstType , typename... OtherTypes>

constexpr EIGEN_DEVICE_FUNC Index Eigen::internal::array_get ( const IndexList< FirstType, OtherTypes... > &  a )

constexpr

                                                                                          {
  return IndexTupleExtractor<N, FirstType, OtherTypes...>::get_val(a);
}

References a.

array_get() [3/15]

template<int N, typename T , typename... O>

constexpr EIGEN_DEVICE_FUNC const IndexTupleExtractor<N, T, O...>::ValType& Eigen::internal::array_get ( const IndexTuple< T, O... > &  tuple )

constexpr

                                      {
  return IndexTupleExtractor<N, T, O...>::get_val(tuple);
}

array_get() [4/15]

template<std::size_t n, typename T , T a, T... as>

constexpr T Eigen::internal::array_get ( const numeric_list< T, a, as... > &  )

constexpr

                                                        {
  return get<(int)n, numeric_list<T, a, as...>>::value;
}

References int(), n, and Eigen::value.

array_get() [5/15]

template<std::ptrdiff_t n, typename std::ptrdiff_t... Indices>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::ptrdiff_t Eigen::internal::array_get

(

const Sizes< Indices... > &

)

                                                                                       {
  return get<n, internal::numeric_list<std::ptrdiff_t, Indices...> >::value;
}

References n, and Eigen::value.

array_get() [6/15]

template<std::ptrdiff_t n>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::ptrdiff_t Eigen::internal::array_get

(

const Sizes<> &

)

                                                                             {
  eigen_assert(false && "should never be called");
  return -1;
}

References eigen_assert.

array_get() [7/15]

template<DenseIndex n, typename Index , std::size_t Rank>

const Index Eigen::internal::array_get

(

DimensionList< Index, Rank > &

)

                                                   {
  return n;
}

References n.

array_get() [8/15]

template<Index N, typename FirstType , typename... OtherTypes>

constexpr EIGEN_DEVICE_FUNC Index Eigen::internal::array_get ( IndexList< FirstType, OtherTypes... > &  a )

constexpr

                                                                                    {
  return IndexTupleExtractor<N, FirstType, OtherTypes...>::get_val(a);
}

References a.

array_get() [9/15]

template<int N, typename T , typename... O>

constexpr EIGEN_DEVICE_FUNC IndexTupleExtractor<N, T, O...>::ValType& Eigen::internal::array_get ( IndexTuple< T, O... > &  tuple )

constexpr

                                                                                                                   {
  return IndexTupleExtractor<N, T, O...>::get_val(tuple);
}

array_get() [10/15]

template<std::size_t I_, class T , std::size_t N>

constexpr T&& Eigen::internal::array_get ( std::array< T, N > &&  a )

inlineconstexpr

                                                   {
  return (T&&)STD_GET_ARR_HACK;
}

References STD_GET_ARR_HACK.

array_get() [11/15]

template<std::size_t I_, class T , std::size_t N>

constexpr T& Eigen::internal::array_get ( std::array< T, N > &  a )

inlineconstexpr

                                                 {
  return (T&)STD_GET_ARR_HACK;
}

References STD_GET_ARR_HACK.

Referenced by h_array_reverse(), Eigen::internal::tensor_index_linearization_helper< Index, NumIndices, 0, RowMajor >::run(), Eigen::internal::tensor_index_linearization_helper< Index, NumIndices, n, RowMajor >::run(), Eigen::internal::fixed_size_tensor_index_linearization_helper< Index, NumIndices, n, RowMajor >::run(), Eigen::internal::tensor_vsize_index_linearization_helper< Index, NumIndices, 0, RowMajor >::run(), Eigen::internal::tensor_vsize_index_linearization_helper< Index, NumIndices, n, RowMajor >::run(), Eigen::internal::fixed_size_tensor_index_extraction_helper< Index, n >::run(), and Eigen::internal::tuple_coeff< Idx, ValueT >::values_up_to_statically_known_to_increase().

array_get() [12/15]

template<std::size_t I_, class T , std::size_t N>

constexpr T const& Eigen::internal::array_get ( std::array< T, N > const &  a )

inlineconstexpr

                                                             {
  return (T const&)STD_GET_ARR_HACK;
}

References STD_GET_ARR_HACK.

array_get() [13/15]

template<std::size_t I_, class T >

constexpr T&& Eigen::internal::array_get ( std::vector< T > &&  a )

inlineconstexpr

                                                 {
  return a[I_];
}

References a.

array_get() [14/15]

template<std::size_t I_, class T >

constexpr T& Eigen::internal::array_get ( std::vector< T > &  a )

inlineconstexpr

                                               {
  return a[I_];
}

References a.

array_get() [15/15]

template<std::size_t I_, class T >

constexpr T const& Eigen::internal::array_get ( std::vector< T > const &  a )

inlineconstexpr

                                                           {
  return a[I_];
}

References a.

array_prod() [1/4]

template<typename T , std::size_t N>

constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE auto Eigen::internal::array_prod ( const array< T, N > &  arr ) -> decltype(array_reduce<product_op, T, N>(arr, static_cast<T>(1)))

constexpr

                                                                        {
  return array_reduce<product_op, T, N>(arr, static_cast<T>(1));
}

Referenced by Eigen::TensorReductionEvaluatorBase< const TensorReductionOp< Op, Dims, ArgType, MakePointer_ >, Device >::costPerCoeff(), Eigen::TensorEvaluator< const TensorSlicingOp< StartIndices, Sizes, ArgType >, Device >::evalSubExprsIfNeeded(), Eigen::TensorEvaluator< const TensorScanOp< Op, ArgType >, Device >::evalSubExprsIfNeeded(), Eigen::TensorEvaluator< const TensorForcedEvalOp< ArgType_ >, Device >::evalSubExprsIfNeeded(), Eigen::TensorReductionEvaluatorBase< const TensorReductionOp< Op, Dims, ArgType, MakePointer_ >, Device >::evalSubExprsIfNeededCommon(), Eigen::internal::TensorBlockMapper< NumDims, Layout, IndexType >::InitializeBlockDimensions(), Eigen::internal::ScanLauncher< Self, Reducer, Eigen::SyclDevice, vectorize >::operator()(), Eigen::internal::ScanLauncher< Self, Reducer, Device, Vectorize >::operator()(), Eigen::TensorEvaluator< const TensorSlicingOp< StartIndices, Sizes, ArgType >, Device >::packet(), Eigen::TensorReductionEvaluatorBase< const TensorReductionOp< Op, Dims, ArgType, MakePointer_ >, Device >::packet(), Eigen::Tensor< Scalar_, NumIndices_, Options_, IndexType_ >::resize(), Eigen::TensorStorage< T, DSizes< IndexType, NumIndices_ >, Options_ >::resize(), Eigen::internal::TensorExecutor< Expression, DefaultDevice, true, TiledEvaluation::Off >::run(), Eigen::internal::TensorExecutor< Expression, Device, Vectorizable, Tiling >::run(), Eigen::internal::FullReducer< Self, Op, Device, Vectorizable >::run(), Eigen::internal::TensorPrinter< Tensor, rank, TensorIOFormatLegacy, std::enable_if_t< rank !=0 > >::run(), Eigen::internal::TensorPrinter< Tensor, rank, Format, EnableIf >::run(), Eigen::TensorEvaluator< const TensorPairReducerOp< ReduceOp, Dims, ArgType >, Device >::TensorEvaluator(), Eigen::TensorEvaluator< const TensorReshapingOp< NewDimensions, ArgType >, Device >::TensorEvaluator(), Eigen::TensorReductionEvaluatorBase< const TensorReductionOp< Op, Dims, ArgType, MakePointer_ >, Device >::TensorReductionEvaluatorBase(), Eigen::TensorStorage< T, DSizes< IndexType, NumIndices_ >, Options_ >::TensorStorage(), test_0d(), test_array_reverse_and_reduce(), test_first_dim_reductions_max_with_offset_sycl(), test_full_reductions_max_with_offset_sycl(), test_full_reductions_mean_with_offset_sycl(), test_full_reductions_min_with_offset_sycl(), test_full_reductions_sum_with_offset_sycl(), test_last_dim_reductions_max_with_offset_sycl(), test_rank_zero(), Eigen::DSizes< DenseIndex, NumDims >::TotalSize(), and Eigen::TensorStorage< T, DSizes< IndexType, NumIndices_ >, Options_ >::~TensorStorage().

array_prod() [2/4]

template<typename FirstType , typename... OtherTypes>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index Eigen::internal::array_prod

(

const IndexList< FirstType, OtherTypes... > &

sizes

)

                                                                                                         {
  Index result = 1;
  EIGEN_UNROLL_LOOP
  for (size_t i = 0; i < array_size<IndexList<FirstType, OtherTypes...>>::value; ++i) {
    result *= sizes[i];
  }
  return result;
}

References EIGEN_UNROLL_LOOP, i, sizes, and Eigen::value.

array_prod() [3/4]

template<typename std::ptrdiff_t... Indices>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::ptrdiff_t Eigen::internal::array_prod

(

const Sizes< Indices... > &

)

                                                                                        {
  return Sizes<Indices...>::total_size;
}

array_prod() [4/4]

template<typename t >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE t Eigen::internal::array_prod

(

const std::vector< t > &

a

)

                                                                          {
  eigen_assert(a.size() > 0);
  t prod = 1;
  for (size_t i = 0; i < a.size(); ++i) {
    prod *= a[i];
  }
  return prod;
}

References a, eigen_assert, i, Eigen::prod(), and plotPSD::t.

array_reduce()

template<typename Reducer , typename T , std::size_t N>

constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE auto Eigen::internal::array_reduce ( const array< T, N > &  arr, T  identity  ) -> decltype(h_array_reduce<Reducer, T, N>::run(arr, identity))

constexpr

                                                                 {
  return h_array_reduce<Reducer, T, N>::run(arr, identity);
}

References Eigen::internal::h_array_reduce< Reducer, T, N, n >::run().

array_reverse()

template<typename T , std::size_t N>

constexpr EIGEN_STRONG_INLINE array<T, N> Eigen::internal::array_reverse ( array< T, N >  arr )

constexpr

                                                                         {
  return h_array_reverse(arr, typename gen_numeric_list<int, N>::type());
}

References h_array_reverse().

Referenced by test_array_reverse_and_reduce().

array_sum()

template<typename T , std::size_t N>

constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE auto Eigen::internal::array_sum ( const array< T, N > &  arr ) -> decltype(array_reduce<sum_op, T, N>(arr, static_cast<T>(0)))

constexpr

                                                                    {
  return array_reduce<sum_op, T, N>(arr, static_cast<T>(0));
}

Referenced by test_array_reverse_and_reduce().

array_zip()

template<typename Op , typename A , typename B , std::size_t N>

constexpr EIGEN_STRONG_INLINE array<decltype(Op::run(A(), B())), N> Eigen::internal::array_zip ( array< A, N >  a, array< B, N >  b  )

constexpr

                                                                                                          {
  return h_array_zip<Op>(a, b, typename gen_numeric_list<int, N>::type());
}

References a, and b.

array_zip_and_reduce()

template<typename Reducer , typename Op , typename A , typename B , std::size_t N>

constexpr EIGEN_STRONG_INLINE auto Eigen::internal::array_zip_and_reduce ( array< A, N >  a, array< B, N >  b  ) -> decltype(h_array_zip_and_reduce<Reducer, Op, A, B, N>(a, b, typename gen_numeric_list<int, N>::type()))

constexpr

                                                                                                             {
  return h_array_zip_and_reduce<Reducer, Op, A, B, N>(a, b, typename gen_numeric_list<int, N>::type());
}

References a, and b.

Referenced by Eigen::TensorFixedSize< Scalar_, Dimensions_, Options_, IndexType >::checkIndexRange(), and Eigen::Tensor< Scalar_, NumIndices_, Options_, IndexType_ >::checkIndexRange().

assign_sparse_to_sparse()

template<typename DstXprType , typename SrcXprType >

void Eigen::internal::assign_sparse_to_sparse	(	DstXprType &	dst,
		const SrcXprType &	src
	)

                                                                     {
  typedef typename DstXprType::Scalar Scalar;
  typedef internal::evaluator<DstXprType> DstEvaluatorType;
  typedef internal::evaluator<SrcXprType> SrcEvaluatorType;
 
  SrcEvaluatorType srcEvaluator(src);
 
  constexpr bool transpose = (DstEvaluatorType::Flags & RowMajorBit) != (SrcEvaluatorType::Flags & RowMajorBit);
  const Index outerEvaluationSize = (SrcEvaluatorType::Flags & RowMajorBit) ? src.rows() : src.cols();
 
  Index reserveSize = 0;
  for (Index j = 0; j < outerEvaluationSize; ++j)
    for (typename SrcEvaluatorType::InnerIterator it(srcEvaluator, j); it; ++it) reserveSize++;
 
  if ((!transpose) && src.isRValue()) {
    // eval without temporary
    dst.resize(src.rows(), src.cols());
    dst.setZero();
    dst.reserve(reserveSize);
    for (Index j = 0; j < outerEvaluationSize; ++j) {
      dst.startVec(j);
      for (typename SrcEvaluatorType::InnerIterator it(srcEvaluator, j); it; ++it) {
        Scalar v = it.value();
        dst.insertBackByOuterInner(j, it.index()) = v;
      }
    }
    dst.finalize();
  } else {
    // eval through a temporary
    eigen_assert((((internal::traits<DstXprType>::SupportedAccessPatterns & OuterRandomAccessPattern) ==
                   OuterRandomAccessPattern) ||
                  (!((DstEvaluatorType::Flags & RowMajorBit) != (SrcEvaluatorType::Flags & RowMajorBit)))) &&
                 "the transpose operation is supposed to be handled in SparseMatrix::operator=");
 
    enum { Flip = (DstEvaluatorType::Flags & RowMajorBit) != (SrcEvaluatorType::Flags & RowMajorBit) };
 
    DstXprType temp(src.rows(), src.cols());
 
    temp.reserve(reserveSize);
    for (Index j = 0; j < outerEvaluationSize; ++j) {
      temp.startVec(j);
      for (typename SrcEvaluatorType::InnerIterator it(srcEvaluator, j); it; ++it) {
        Scalar v = it.value();
        temp.insertBackByOuterInner(Flip ? it.index() : j, Flip ? j : it.index()) = v;
      }
    }
    temp.finalize();
 
    dst = temp.markAsRValue();
  }
}

References eigen_assert, j, Eigen::OuterRandomAccessPattern, Eigen::RowMajorBit, anonymous_namespace{skew_symmetric_matrix3.cpp}::transpose(), and v.

Referenced by Eigen::internal::Assignment< DstXprType, SrcXprType, Functor, Sparse2Sparse >::run().

asSluMatrix()

template<typename MatrixType >

SluMatrix Eigen::internal::asSluMatrix

(

MatrixType &

mat

)

                                       {
  return SluMatrix::Map(mat);
}

References Eigen::SluMatrix::Map().

Referenced by Eigen::SuperLUBase< MatrixType_, Derived >::initFactorization().

avx512_blend_mask()

template<size_t N>

EIGEN_STRONG_INLINE int Eigen::internal::avx512_blend_mask

(

const Selector< N > &

ifPacket

)

                                                                       {
  alignas(__m128i) uint8_t aux[sizeof(__m128i)];
  for (size_t i = 0; i < N; i++) aux[i] = static_cast<uint8_t>(ifPacket.select[i]);
  __m128i paux = _mm_sub_epi8(_mm_setzero_si128(), _mm_load_si128(reinterpret_cast<const __m128i*>(aux)));
  return _mm_movemask_epi8(paux);
}

References i, and Eigen::internal::Selector< N >::select.

Referenced by pblend().

avx_blend_mask() [1/2]

EIGEN_STRONG_INLINE __m256i Eigen::internal::avx_blend_mask

(

const Selector< 4 > &

ifPacket

)

                                                                        {
  return _mm256_set_epi64x(0 - ifPacket.select[3], 0 - ifPacket.select[2], 0 - ifPacket.select[1],
                           0 - ifPacket.select[0]);
}

References Eigen::internal::Selector< N >::select.

Referenced by pblend().

avx_blend_mask() [2/2]

EIGEN_STRONG_INLINE __m256i Eigen::internal::avx_blend_mask

(

const Selector< 8 > &

ifPacket

)

                                                                        {
  return _mm256_set_epi32(0 - ifPacket.select[7], 0 - ifPacket.select[6], 0 - ifPacket.select[5],
                          0 - ifPacket.select[4], 0 - ifPacket.select[3], 0 - ifPacket.select[2],
                          0 - ifPacket.select[1], 0 - ifPacket.select[0]);
}

References Eigen::internal::Selector< N >::select.

band()

template<typename Packet , int N>

EIGEN_ALWAYS_INLINE void Eigen::internal::band	(	PacketBlock< Packet, N > &	acc,
		const Packet &	pMask
	)

                                                                                {
  for (int M = 0; M < N; M++) {
    acc.packet[M] = pand<Packet>(acc.packet[M], pMask);
  }
}

References N, and Eigen::internal::PacketBlock< Packet, N >::packet.

bcouple()

template<typename Packet , typename Packetc , int N, bool full>

EIGEN_ALWAYS_INLINE void Eigen::internal::bcouple	(	PacketBlock< Packet, N > &	taccReal,
		PacketBlock< Packet, N > &	taccImag,
		PacketBlock< Packetc, N *2 > &	tRes,
		PacketBlock< Packetc, N > &	acc1,
		PacketBlock< Packetc, N > &	acc2
	)

                                                                {
  bcouple_common<Packet, Packetc, N, full>(taccReal, taccImag, acc1, acc2);
 
  for (int M = 0; M < N; M++) {
    acc1.packet[M] = padd<Packetc>(tRes.packet[M], acc1.packet[M]);
  }
 
  if (full) {
    for (int M = 0; M < N; M++) {
      acc2.packet[M] = padd<Packetc>(tRes.packet[M + N], acc2.packet[M]);
    }
  }
}

References N, and Eigen::internal::PacketBlock< Packet, N >::packet.

bcouple_common()

template<typename Packet , typename Packetc , int N, bool full>

EIGEN_ALWAYS_INLINE void Eigen::internal::bcouple_common	(	PacketBlock< Packet, N > &	taccReal,
		PacketBlock< Packet, N > &	taccImag,
		PacketBlock< Packetc, N > &	acc1,
		PacketBlock< Packetc, N > &	acc2
	)

                                                                                                      {
  for (int M = 0; M < N; M++) {
    acc1.packet[M].v = vec_mergeh(taccReal.packet[M], taccImag.packet[M]);
  }
 
  if (full) {
    for (int M = 0; M < N; M++) {
      acc2.packet[M].v = vec_mergel(taccReal.packet[M], taccImag.packet[M]);
    }
  }
}

References N, and Eigen::internal::PacketBlock< Packet, N >::packet.

Bf16PackHigh()

template<bool lohi>

EIGEN_ALWAYS_INLINE Packet8bf Eigen::internal::Bf16PackHigh	(	Packet4f	hi,
		Packet4f	lo
	)

                                                                     {
  if (lohi) {
    return vec_perm(reinterpret_cast<Packet8us>(hi), reinterpret_cast<Packet8us>(lo), p16uc_MERGEL16);
  } else {
    return vec_perm(reinterpret_cast<Packet8us>(hi), reinterpret_cast<Packet8us>(lo), p16uc_MERGEO16);
  }
}

References p16uc_MERGEL16, and p16uc_MERGEO16.

Bf16PackLow()

template<bool lohi>

EIGEN_ALWAYS_INLINE Packet8bf Eigen::internal::Bf16PackLow	(	Packet4f	hi,
		Packet4f	lo
	)

                                                                    {
  if (lohi) {
    return vec_pack(reinterpret_cast<Packet4ui>(hi), reinterpret_cast<Packet4ui>(lo));
  } else {
    return vec_perm(reinterpret_cast<Packet8us>(hi), reinterpret_cast<Packet8us>(lo), p16uc_MERGEE16);
  }
}

References p16uc_MERGEE16.

Bf16ToF32() [1/3]

EIGEN_STRONG_INLINE Packet16f Eigen::internal::Bf16ToF32

(

const Packet16bf &

a

)

                                                             {
  return _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(a), 16));
}

References a.

Bf16ToF32() [2/3]

EIGEN_STRONG_INLINE Packet4f Eigen::internal::Bf16ToF32

(

const Packet4bf &

p

)

                                                           {
  return Packet4f(vreinterpretq_f32_u32(vshlq_n_u32(vmovl_u16(p), 16)));
}

References p.

Bf16ToF32() [3/3]

EIGEN_STRONG_INLINE Packet8f Eigen::internal::Bf16ToF32

(

const Packet8bf &

a

)

                                                           {
#ifdef EIGEN_VECTORIZE_AVX2
  __m256i extend = _mm256_cvtepu16_epi32(a);
  return _mm256_castsi256_ps(_mm256_slli_epi32(extend, 16));
#else
  __m128i lo = _mm_cvtepu16_epi32(a);
  __m128i hi = _mm_cvtepu16_epi32(_mm_srli_si128(a, 8));
  __m128i lo_shift = _mm_slli_epi32(lo, 16);
  __m128i hi_shift = _mm_slli_epi32(hi, 16);
  return _mm256_castsi256_ps(_mm256_insertf128_si256(_mm256_castsi128_si256(lo_shift), hi_shift, 1));
#endif
}

References a.

Referenced by pabs(), pabsdiff< Packet4bf >(), padd< Packet16bf >(), padd< Packet4bf >(), pcast< Packet16bf, Packet16f >(), pcast< Packet8bf, Packet8f >(), pceil< Packet16bf >(), pceil< Packet4bf >(), pcmp_eq(), pcmp_eq< Packet4bf >(), pcmp_le(), pcmp_le< Packet4bf >(), pcmp_lt(), pcmp_lt< Packet4bf >(), pcmp_lt_or_nan(), pcmp_lt_or_nan< Packet4bf >(), pdiv< Packet16bf >(), pdiv< Packet4bf >(), pfloor< Packet16bf >(), pfloor< Packet4bf >(), pfrexp(), pldexp(), pmax< Packet16bf >(), pmax< Packet4bf >(), pmax< PropagateNaN, Packet4bf >(), pmax< PropagateNumbers, Packet4bf >(), pmin< Packet16bf >(), pmin< Packet4bf >(), pmin< PropagateNaN, Packet4bf >(), pmin< PropagateNumbers, Packet4bf >(), pmul< Packet16bf >(), pmul< Packet4bf >(), predux< Packet16bf >(), predux< Packet4bf >(), predux_max< Packet16bf >(), predux_max< Packet4bf >(), predux_min< Packet16bf >(), predux_min< Packet4bf >(), predux_mul< Packet16bf >(), predux_mul< Packet4bf >(), print< Packet16bf >(), print< Packet4bf >(), print< Packet8bf >(), pround< Packet16bf >(), pround< Packet4bf >(), psub< Packet16bf >(), psub< Packet4bf >(), ptrunc< Packet16bf >(), and ptrunc< Packet4bf >().

Bf16ToF32Even()

EIGEN_STRONG_INLINE Packet4f Eigen::internal::Bf16ToF32Even

(

const Packet8bf &

bf

)

                                                                {
  return plogical_shift_left<16>(reinterpret_cast<Packet4f>(bf.m_val));
}

References Eigen::internal::eigen_packet_wrapper< T, unique_id >::m_val.

Referenced by pcast< Packet8bf, Packet8us >(), pfrexp< Packet8bf >(), pmadd(), predux< Packet8bf >(), predux_max< Packet8bf >(), predux_min< Packet8bf >(), and predux_mul< Packet8bf >().

Bf16ToF32Odd()

EIGEN_STRONG_INLINE Packet4f Eigen::internal::Bf16ToF32Odd

(

const Packet8bf &

bf

)

                                                               {
  const EIGEN_DECLARE_CONST_FAST_Packet4ui(high_mask, 0xFFFF0000);
  return pand<Packet4f>(reinterpret_cast<Packet4f>(bf.m_val), reinterpret_cast<Packet4f>(p4ui_high_mask));
}

References EIGEN_DECLARE_CONST_FAST_Packet4ui(), Eigen::internal::eigen_packet_wrapper< T, unique_id >::m_val, and pand< Packet4f >().

Referenced by pcast< Packet8bf, Packet8us >(), pfrexp< Packet8bf >(), pmadd(), predux< Packet8bf >(), predux_max< Packet8bf >(), predux_min< Packet8bf >(), and predux_mul< Packet8bf >().

bicgstab()

template<typename MatrixType , typename Rhs , typename Dest , typename Preconditioner >

bool Eigen::internal::bicgstab	(	const MatrixType &	mat,
		const Rhs &	rhs,
		Dest &	x,
		const Preconditioner &	precond,
		Index &	iters,
		typename Dest::RealScalar &	tol_error
	)

Low-level bi conjugate gradient stabilized algorithm

Parameters

mat	The matrix A
rhs	The right hand side vector b
x	On input and initial solution, on output the computed solution.
precond	A preconditioner being able to efficiently solve for an approximation of Ax=b (regardless of b)
iters	On input the max number of iteration, on output the number of performed iterations.
tol_error	On input the tolerance error, on output an estimation of the relative error.

Returns

false in the case of numerical issue, for example a break down of BiCGSTAB.

                                                  {
  using std::abs;
  using std::sqrt;
  typedef typename Dest::RealScalar RealScalar;
  typedef typename Dest::Scalar Scalar;
  typedef Matrix<Scalar, Dynamic, 1> VectorType;
  RealScalar tol = tol_error;
  Index maxIters = iters;
 
  Index n = mat.cols();
  VectorType r = rhs - mat * x;
  VectorType r0 = r;
 
  RealScalar r0_sqnorm = r0.squaredNorm();
  RealScalar rhs_sqnorm = rhs.squaredNorm();
  if (rhs_sqnorm == 0) {
    x.setZero();
    return true;
  }
  Scalar rho(1);
  Scalar alpha(1);
  Scalar w(1);
 
  VectorType v = VectorType::Zero(n), p = VectorType::Zero(n);
  VectorType y(n), z(n);
  VectorType kt(n), ks(n);
 
  VectorType s(n), t(n);
 
  RealScalar tol2 = tol * tol * rhs_sqnorm;
  RealScalar eps2 = NumTraits<Scalar>::epsilon() * NumTraits<Scalar>::epsilon();
  Index i = 0;
  Index restarts = 0;
 
  while (r.squaredNorm() > tol2 && i < maxIters) {
    Scalar rho_old = rho;
 
    rho = r0.dot(r);
    if (abs(rho) < eps2 * r0_sqnorm) {
      // The new residual vector became too orthogonal to the arbitrarily chosen direction r0
      // Let's restart with a new r0:
      r = rhs - mat * x;
      r0 = r;
      rho = r0_sqnorm = r.squaredNorm();
      if (restarts++ == 0) i = 0;
    }
    Scalar beta = (rho / rho_old) * (alpha / w);
    p = r + beta * (p - w * v);
 
    y = precond.solve(p);
 
    v.noalias() = mat * y;
 
    alpha = rho / r0.dot(v);
    s = r - alpha * v;
 
    z = precond.solve(s);
    t.noalias() = mat * z;
 
    RealScalar tmp = t.squaredNorm();
    if (tmp > RealScalar(0))
      w = t.dot(s) / tmp;
    else
      w = Scalar(0);
    x += alpha * y + w * z;
    r = s - w * t;
    ++i;
  }
  tol_error = sqrt(r.squaredNorm() / rhs_sqnorm);
  iters = i;
  return true;
}

References abs(), alpha, beta, Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::cols(), oomph::SarahBL::epsilon, i, n, p, UniformPSDSelfTest::r, s, sqrt(), plotPSD::t, tmp, v, w, plotDoE::x, y, and oomph::PseudoSolidHelper::Zero.

Referenced by Eigen::BiCGSTAB< MatrixType_, Preconditioner_ >::_solve_vector_with_guess_impl().

bicgstabl()

template<typename MatrixType , typename Rhs , typename Dest , typename Preconditioner >

bool Eigen::internal::bicgstabl	(	const MatrixType &	mat,
		const Rhs &	rhs,
		Dest &	x,
		const Preconditioner &	precond,
		Index &	iters,
		typename Dest::RealScalar &	tol_error,
		Index	L
	)

Low-level bi conjugate gradient stabilized algorithm with L additional residual minimization steps

Parameters

mat	The matrix A
rhs	The right hand side vector b
x	On input and initial solution, on output the computed solution.
precond	A preconditioner being able to efficiently solve for an approximation of Ax=b (regardless of b)
iters	On input the max number of iteration, on output the number of performed iterations.
tol_error	On input the tolerance error, on output an estimation of the relative error.
L	On input Number of additional GMRES steps to take. If L is too large (~20) instabilities occur.

Returns

false in the case of numerical issue, for example a break down of BiCGSTABL.

                                                            {
  using numext::abs;
  using numext::sqrt;
  typedef typename Dest::RealScalar RealScalar;
  typedef typename Dest::Scalar Scalar;
  const Index N = rhs.size();
  L = L < x.rows() ? L : x.rows();
 
  Index k = 0;
 
  const RealScalar tol = tol_error;
  const Index maxIters = iters;
 
  typedef Matrix<Scalar, Dynamic, 1> VectorType;
  typedef Matrix<Scalar, Dynamic, Dynamic, ColMajor> DenseMatrixType;
 
  DenseMatrixType rHat(N, L + 1);
  DenseMatrixType uHat(N, L + 1);
 
  // We start with an initial guess x_0 and let us set r_0 as (residual
  // calculated from x_0)
  VectorType x0 = x;
  rHat.col(0) = rhs - mat * x0;  // r_0
 
  x.setZero();  // This will contain the updates to the solution.
  // rShadow is arbitrary, but must never be orthogonal to any residual.
  VectorType rShadow = VectorType::Random(N);
 
  VectorType x_prime = x;
 
  // Redundant: x is already set to 0
  // x.setZero();
  VectorType b_prime = rHat.col(0);
 
  // Other vectors and scalars initialization
  Scalar rho0 = 1.0;
  Scalar alpha = 0.0;
  Scalar omega = 1.0;
 
  uHat.col(0).setZero();
 
  bool bicg_convergence = false;
 
  const RealScalar normb = rhs.stableNorm();
  if (internal::isApprox(normb, RealScalar(0))) {
    x.setZero();
    iters = 0;
    return true;
  }
  RealScalar normr = rHat.col(0).stableNorm();
  RealScalar Mx = normr;
  RealScalar Mr = normr;
 
  // Keep track of the solution with the lowest residual
  RealScalar normr_min = normr;
  VectorType x_min = x_prime + x;
 
  // Criterion for when to apply the group-wise update, conform ref 3.
  const RealScalar delta = 0.01;
 
  bool compute_res = false;
  bool update_app = false;
 
  while (normr > tol * normb && k < maxIters) {
    rho0 *= -omega;
 
    for (Index j = 0; j < L; ++j) {
      const Scalar rho1 = rShadow.dot(rHat.col(j));
 
      if (!(numext::isfinite)(rho1) || rho0 == RealScalar(0.0)) {
        // We cannot continue computing, return the best solution found.
        x += x_prime;
 
        // Check if x is better than the best stored solution thus far.
        normr = (rhs - mat * (precond.solve(x) + x0)).stableNorm();
 
        if (normr > normr_min || !(numext::isfinite)(normr)) {
          // x_min is a better solution than x, return x_min
          x = x_min;
          normr = normr_min;
        }
        tol_error = normr / normb;
        iters = k;
        // x contains the updates to x0, add those back to obtain the solution
        x = precond.solve(x);
        x += x0;
        return (normr < tol * normb);
      }
 
      const Scalar beta = alpha * (rho1 / rho0);
      rho0 = rho1;
      // Update search directions
      uHat.leftCols(j + 1) = rHat.leftCols(j + 1) - beta * uHat.leftCols(j + 1);
      uHat.col(j + 1) = mat * precond.solve(uHat.col(j));
      const Scalar sigma = rShadow.dot(uHat.col(j + 1));
      alpha = rho1 / sigma;
      // Update residuals
      rHat.leftCols(j + 1) -= alpha * uHat.middleCols(1, j + 1);
      rHat.col(j + 1) = mat * precond.solve(rHat.col(j));
      // Complete BiCG iteration by updating x
      x += alpha * uHat.col(0);
      normr = rHat.col(0).stableNorm();
      // Check for early exit
      if (normr < tol * normb) {
        /*
          Convergence was achieved during BiCG step.
          Without this check BiCGStab(L) fails for trivial matrices, such as
          when the preconditioner already is the inverse, or the input matrix is
          identity.
        */
        bicg_convergence = true;
        break;
      } else if (normr < normr_min) {
        // We found an x with lower residual, keep this one.
        x_min = x + x_prime;
        normr_min = normr;
      }
    }
    if (!bicg_convergence) {
      /*
        The polynomial/minimize residual step.
 
        QR Householder method for argmin is more stable than (modified)
        Gram-Schmidt, in the sense that there is less loss of orthogonality. It
        is more accurate than solving the normal equations, since the normal
        equations scale with condition number squared.
      */
      const VectorType gamma = rHat.rightCols(L).householderQr().solve(rHat.col(0));
      x += rHat.leftCols(L) * gamma;
      rHat.col(0) -= rHat.rightCols(L) * gamma;
      uHat.col(0) -= uHat.rightCols(L) * gamma;
      normr = rHat.col(0).stableNorm();
      omega = gamma(L - 1);
    }
    if (normr < normr_min) {
      // We found an x with lower residual, keep this one.
      x_min = x + x_prime;
      normr_min = normr;
    }
 
    k++;
 
    /*
      Reliable update part
 
      The recursively computed residual can deviate from the actual residual
      after several iterations. However, computing the residual from the
      definition costs extra MVs and should not be done at each iteration. The
      reliable update strategy computes the true residual from the definition:
      r=b-A*x at strategic intervals. Furthermore a "group wise update" strategy
      is used to combine updates, which improves accuracy.
    */
 
    // Maximum norm of residuals since last update of x.
    Mx = numext::maxi(Mx, normr);
    // Maximum norm of residuals since last computation of the true residual.
    Mr = numext::maxi(Mr, normr);
 
    if (normr < delta * normb && normb <= Mx) {
      update_app = true;
    }
 
    if (update_app || (normr < delta * Mr && normb <= Mr)) {
      compute_res = true;
    }
 
    if (bicg_convergence) {
      update_app = true;
      compute_res = true;
      bicg_convergence = false;
    }
 
    if (compute_res) {
      // Explicitly compute residual from the definition
 
      // This is equivalent to the shifted version of rhs - mat *
      // (precond.solve(x)+x0)
      rHat.col(0) = b_prime - mat * precond.solve(x);
      normr = rHat.col(0).stableNorm();
      Mr = normr;
 
      if (update_app) {
        // After the group wise update, the original problem is translated to a
        // shifted one.
        x_prime += x;
        x.setZero();
        b_prime = rHat.col(0);
        Mx = normr;
      }
    }
    if (normr < normr_min) {
      // We found an x with lower residual, keep this one.
      x_min = x + x_prime;
      normr_min = normr;
    }
 
    compute_res = false;
    update_app = false;
  }
 
  // Convert internal variable to the true solution vector x
  x += x_prime;
 
  normr = (rhs - mat * (precond.solve(x) + x0)).stableNorm();
  if (normr > normr_min || !(numext::isfinite)(normr)) {
    // x_min is a better solution than x, return x_min
    x = x_min;
    normr = normr_min;
  }
  tol_error = normr / normb;
  iters = k;
 
  // x contains the updates to x0, add those back to obtain the solution
  x = precond.solve(x);
  x += x0;
  return true;
}

References Eigen::numext::abs(), alpha, beta, MultiOpt::delta, mathsFunc::gamma(), isApprox(), Eigen::numext::isfinite(), j, k, L, Eigen::numext::maxi(), Problem_Parameter::Mr, N, omega(), calibrate::sigma, Eigen::numext::sqrt(), plotDoE::x, and Global::x0.

Referenced by Eigen::BiCGSTABL< MatrixType_, Preconditioner_ >::_solve_vector_with_guess_impl().

bload()

template<typename DataMapper , typename Packet , const Index accCols, int StorageOrder, bool Complex, int N, bool full>

EIGEN_ALWAYS_INLINE void Eigen::internal::bload	(	PacketBlock< Packet, N *(Complex ? 2 :1)> &	acc,
		const DataMapper &	res,
		Index	row,
		Index	col
	)

                                                                  : 1)>& acc, const DataMapper& res, Index row,
                               Index col) {
  if (StorageOrder == RowMajor) {
    for (int M = 0; M < N; M++) {
      acc.packet[M] = res.template loadPacket<Packet>(row + M, col);
    }
    if (Complex) {
      for (int M = 0; M < N; M++) {
        acc.packet[M + N] = res.template loadPacket<Packet>(row + M, col + accCols);
      }
    }
  } else {
    for (int M = 0; M < N; M++) {
      acc.packet[M] = res.template loadPacket<Packet>(row, col + M);
    }
    if (Complex && full) {
      for (int M = 0; M < N; M++) {
        acc.packet[M + N] = res.template loadPacket<Packet>(row + accCols, col + M);
      }
    }
  }
}

References col(), N, res, row(), and Eigen::RowMajor.

BlockedInPlaceTranspose()

template<typename MatrixType , Index Alignment>

void Eigen::internal::BlockedInPlaceTranspose

(

MatrixType &

m

)

                                            {
  typedef typename MatrixType::Scalar Scalar;
  typedef typename internal::packet_traits<typename MatrixType::Scalar>::type Packet;
  const Index PacketSize = internal::packet_traits<Scalar>::size;
  eigen_assert(m.rows() == m.cols());
  int row_start = 0;
  for (; row_start + PacketSize <= m.rows(); row_start += PacketSize) {
    for (int col_start = row_start; col_start + PacketSize <= m.cols(); col_start += PacketSize) {
      PacketBlock<Packet> A;
      if (row_start == col_start) {
        for (Index i = 0; i < PacketSize; ++i)
          A.packet[i] = m.template packetByOuterInner<Alignment>(row_start + i, col_start);
        internal::ptranspose(A);
        for (Index i = 0; i < PacketSize; ++i)
          m.template writePacket<Alignment>(m.rowIndexByOuterInner(row_start + i, col_start),
                                            m.colIndexByOuterInner(row_start + i, col_start), A.packet[i]);
      } else {
        PacketBlock<Packet> B;
        for (Index i = 0; i < PacketSize; ++i) {
          A.packet[i] = m.template packetByOuterInner<Alignment>(row_start + i, col_start);
          B.packet[i] = m.template packetByOuterInner<Alignment>(col_start + i, row_start);
        }
        internal::ptranspose(A);
        internal::ptranspose(B);
        for (Index i = 0; i < PacketSize; ++i) {
          m.template writePacket<Alignment>(m.rowIndexByOuterInner(row_start + i, col_start),
                                            m.colIndexByOuterInner(row_start + i, col_start), B.packet[i]);
          m.template writePacket<Alignment>(m.rowIndexByOuterInner(col_start + i, row_start),
                                            m.colIndexByOuterInner(col_start + i, row_start), A.packet[i]);
        }
      }
    }
  }
  for (Index row = row_start; row < m.rows(); ++row) {
    m.matrix().row(row).head(row).swap(m.matrix().col(row).head(row).transpose());
  }
}

References eigen_assert, i, m, Eigen::PlainObjectBase< Derived >::packet(), ptranspose(), and row().

blueNorm_impl()

template<typename Derived >

NumTraits<typename traits<Derived>::Scalar>::Real Eigen::internal::blueNorm_impl ( const EigenBase< Derived > &  _vec )

inline

                                                                                                            {
  typedef typename Derived::RealScalar RealScalar;
  using std::abs;
  using std::pow;
  using std::sqrt;
 
  // This program calculates the machine-dependent constants
  // bl, b2, slm, s2m, relerr overfl
  // from the "basic" machine-dependent numbers
  // nbig, ibeta, it, iemin, iemax, rbig.
  // The following define the basic machine-dependent constants.
  // For portability, the PORT subprograms "ilmaeh" and "rlmach"
  // are used. For any specific computer, each of the assignment
  // statements can be replaced
  static const int ibeta = std::numeric_limits<RealScalar>::radix;  // base for floating-point numbers
  static const int it = NumTraits<RealScalar>::digits();            // number of base-beta digits in mantissa
  static const int iemin = NumTraits<RealScalar>::min_exponent();   // minimum exponent
  static const int iemax = NumTraits<RealScalar>::max_exponent();   // maximum exponent
  static const RealScalar rbig = NumTraits<RealScalar>::highest();  // largest floating-point number
  static const RealScalar b1 =
      RealScalar(pow(RealScalar(ibeta), RealScalar(-((1 - iemin) / 2))));  // lower boundary of midrange
  static const RealScalar b2 =
      RealScalar(pow(RealScalar(ibeta), RealScalar((iemax + 1 - it) / 2)));  // upper boundary of midrange
  static const RealScalar s1m =
      RealScalar(pow(RealScalar(ibeta), RealScalar((2 - iemin) / 2)));  // scaling factor for lower range
  static const RealScalar s2m =
      RealScalar(pow(RealScalar(ibeta), RealScalar(-((iemax + it) / 2))));  // scaling factor for upper range
  static const RealScalar eps = RealScalar(pow(double(ibeta), 1 - it));
  static const RealScalar relerr = sqrt(eps);  // tolerance for neglecting asml
 
  const Derived& vec(_vec.derived());
  Index n = vec.size();
  RealScalar ab2 = b2 / RealScalar(n);
  RealScalar asml = RealScalar(0);
  RealScalar amed = RealScalar(0);
  RealScalar abig = RealScalar(0);
 
  for (Index j = 0; j < vec.outerSize(); ++j) {
    for (typename Derived::InnerIterator iter(vec, j); iter; ++iter) {
      RealScalar ax = abs(iter.value());
      if (ax > ab2)
        abig += numext::abs2(ax * s2m);
      else if (ax < b1)
        asml += numext::abs2(ax * s1m);
      else
        amed += numext::abs2(ax);
    }
  }
  if (amed != amed) return amed;  // we got a NaN
  if (abig > RealScalar(0)) {
    abig = sqrt(abig);
    if (abig > rbig)  // overflow, or *this contains INF values
      return abig;    // return INF
    if (amed > RealScalar(0)) {
      abig = abig / s2m;
      amed = sqrt(amed);
    } else
      return abig / s2m;
  } else if (asml > RealScalar(0)) {
    if (amed > RealScalar(0)) {
      abig = sqrt(amed);
      amed = sqrt(asml) / s1m;
    } else
      return sqrt(asml) / s1m;
  } else
    return sqrt(amed);
  asml = numext::mini(abig, amed);
  abig = numext::maxi(abig, amed);
  if (asml <= abig * relerr)
    return abig;
  else
    return abig * sqrt(RealScalar(1) + numext::abs2(asml / abig));
}

References abs(), Eigen::numext::abs2(), plotDoE::ax, Eigen::EigenBase< Derived >::derived(), CRBond_Bessel::eps, j, Eigen::numext::maxi(), Eigen::numext::mini(), n, Eigen::bfloat16_impl::pow(), relerr(), and sqrt().

Referenced by Eigen::MatrixBase< Derived >::blueNorm(), and Eigen::SparseMatrixBase< Derived >::blueNorm().

bmask()

template<typename Packet >

EIGEN_ALWAYS_INLINE Packet Eigen::internal::bmask

(

const Index

remaining_rows

)

                                                             {
#if USE_P10_AND_PVIPR2_0
#ifdef _BIG_ENDIAN
  return Packet(vec_reve(vec_genwm((1 << remaining_rows) - 1)));
#else
  return Packet(vec_genwm((1 << remaining_rows) - 1));
#endif
#else
  return Packet(mask4[remaining_rows]);
#endif
}

References mask4.

bmask< Packet2d >()

template<>

EIGEN_ALWAYS_INLINE Packet2d Eigen::internal::bmask< Packet2d >

(

const Index

remaining_rows

)

                                                                         {
#if USE_P10_AND_PVIPR2_0
  Packet2d mask2 = Packet2d(vec_gendm(remaining_rows));
#ifdef _BIG_ENDIAN
  return preverse(mask2);
#else
  return mask2;
#endif
#else
  Packet2l ret = {-remaining_rows, 0};
  return Packet2d(ret);
#endif
}

References preverse(), and ret.

bruteforce_det3_helper()

template<typename Derived >

EIGEN_DEVICE_FUNC const Derived::Scalar Eigen::internal::bruteforce_det3_helper ( const MatrixBase< Derived > &  matrix, int  a, int  b, int  c  )

inline

                                                                                             {
  return matrix.coeff(0, a) * (matrix.coeff(1, b) * matrix.coeff(2, c) - matrix.coeff(1, c) * matrix.coeff(2, b));
}

References a, b, calibrate::c, and matrix().

Referenced by Eigen::internal::determinant_impl< Derived, 3 >::run().

bscale() [1/2]

template<typename Packet , int N>

EIGEN_ALWAYS_INLINE void Eigen::internal::bscale	(	PacketBlock< Packet, N > &	acc,
		PacketBlock< Packet, N > &	accZ,
		const Packet &	pAlpha
	)

                                                                                                                 {
  for (int M = 0; M < N; M++) {
    acc.packet[M] = pmadd<Packet>(pAlpha, accZ.packet[M], acc.packet[M]);
  }
}

References N, and Eigen::internal::PacketBlock< Packet, N >::packet.

bscale() [2/2]

template<typename Packet , int N, bool mask>

EIGEN_ALWAYS_INLINE void Eigen::internal::bscale	(	PacketBlock< Packet, N > &	acc,
		PacketBlock< Packet, N > &	accZ,
		const Packet &	pAlpha,
		const Packet &	pMask
	)

                                                     {
  if (mask) {
    band<Packet, N>(accZ, pMask);
  } else {
    EIGEN_UNUSED_VARIABLE(pMask);
  }
 
  bscale<Packet, N>(acc, accZ, pAlpha);
}

References EIGEN_UNUSED_VARIABLE.

bscalec()

template<typename Packet , int N, bool mask>

EIGEN_ALWAYS_INLINE void Eigen::internal::bscalec	(	PacketBlock< Packet, N > &	aReal,
		PacketBlock< Packet, N > &	aImag,
		const Packet &	bReal,
		const Packet &	bImag,
		PacketBlock< Packet, N > &	cReal,
		PacketBlock< Packet, N > &	cImag,
		const Packet &	pMask
	)

                                                      {
  if (mask && (sizeof(__UNPACK_TYPE__(Packet)) == sizeof(float))) {
    band<Packet, N>(aReal, pMask);
    band<Packet, N>(aImag, pMask);
  } else {
    EIGEN_UNUSED_VARIABLE(pMask);
  }
 
  bscalec_common<Packet, N>(cReal, aReal, bReal);
 
  bscalec_common<Packet, N>(cImag, aImag, bReal);
 
  pger_common<Packet, true, N>(&cReal, bImag, aImag.packet);
 
  pger_common<Packet, false, N>(&cImag, bImag, aReal.packet);
}

References __UNPACK_TYPE__(), EIGEN_UNUSED_VARIABLE, and Eigen::internal::PacketBlock< Packet, N >::packet.

bscalec_common()

template<typename Packet , int N>

EIGEN_ALWAYS_INLINE void Eigen::internal::bscalec_common	(	PacketBlock< Packet, N > &	acc,
		PacketBlock< Packet, N > &	accZ,
		const Packet &	pAlpha
	)

                                                              {
  for (int M = 0; M < N; M++) {
    acc.packet[M] = vec_mul(accZ.packet[M], pAlpha);
  }
}

References N, and Eigen::internal::PacketBlock< Packet, N >::packet.

bsetzero()

template<typename Packet , int N>

EIGEN_ALWAYS_INLINE void Eigen::internal::bsetzero

(

PacketBlock< Packet, N > &

acc

)

                                                               {
  for (int M = 0; M < N; M++) {
    acc.packet[M] = pset1<Packet>((__UNPACK_TYPE__(Packet))0);
  }
}

References __UNPACK_TYPE__(), N, and Eigen::internal::PacketBlock< Packet, N >::packet.

bsetzeroMMA()

EIGEN_ALWAYS_INLINE void Eigen::internal::bsetzeroMMA

(

__vector_quad *

acc

)

{ __builtin_mma_xxsetaccz(acc); }

bstore()

template<typename DataMapper , typename Packet , int N>

EIGEN_ALWAYS_INLINE void Eigen::internal::bstore	(	PacketBlock< Packet, N > &	acc,
		const DataMapper &	res,
		Index	row
	)

                                                                                               {
  for (int M = 0; M < N; M++) {
    res.template storePacket<Packet>(row, M, acc.packet[M]);
  }
}

References N, Eigen::internal::PacketBlock< Packet, N >::packet, res, and row().

c_to_fortran_numbering()

template<typename MatrixType >

void Eigen::internal::c_to_fortran_numbering

(

MatrixType &

mat

)

                                             {
  if (!(mat.outerIndexPtr()[0])) {
    int i;
    for (i = 0; i <= mat.rows(); ++i) ++mat.outerIndexPtr()[i];
    for (i = 0; i < mat.nonZeros(); ++i) ++mat.innerIndexPtr()[i];
  }
}

References i, Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::innerIndexPtr(), Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::nonZeros(), Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::outerIndexPtr(), and Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::rows().

Referenced by Eigen::PastixLU< MatrixType_, IsStrSym >::grabMatrix(), Eigen::PastixLLT< MatrixType_, UpLo_ >::grabMatrix(), and Eigen::PastixLDLT< MatrixType_, UpLo_ >::grabMatrix().

calcColLoops()

template<Index size>

EIGEN_ALWAYS_INLINE void Eigen::internal::calcColLoops	(	const bfloat16 *&	indexA,
		Index &	row,
		Index	depth,
		Index	cols,
		Index	rows,
		const Packet4f	pAlpha,
		const bfloat16 *	indexB,
		Index	strideB,
		Index	offsetA,
		Index	offsetB,
		Index	bigSuffix,
		float *	result
	)

                                                                                     {
  if ((size == 16) || (rows & size)) {
    indexA += size * offsetA;
    colLoops<size>(depth, cols, rows, pAlpha, indexA, indexB, strideB, offsetB, result + row);
    row += size;
    indexA += bigSuffix * size / 16;
  }
}

References cols, row(), rows, and size.

calcVecColLoops()

template<typename LhsMapper , typename RhsMapper , bool linear>

EIGEN_ALWAYS_INLINE void Eigen::internal::calcVecColLoops	(	Index	cend,
		Index	rows,
		LhsMapper &	lhs,
		RhsMapper &	rhs,
		const Packet4f	pAlpha,
		float *	result
	)

                                                        {
  Index row = 0;
  if (rows >= (MAX_BFLOAT16_VEC_ACC * 4)) {
    colVecColLoopBody<MAX_BFLOAT16_VEC_ACC, LhsMapper, RhsMapper, false, linear>(row, cend, rows, lhs, rhs, pAlpha,
                                                                                 result);
    result += row;
  }
  if (rows & 3) {
    colVecColLoopBodyExtra<LhsMapper, RhsMapper, true, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
  } else {
    colVecColLoopBodyExtra<LhsMapper, RhsMapper, false, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
  }
}

References MAX_BFLOAT16_VEC_ACC, row(), and rows.

calcVecLoops()

template<typename LhsMapper , typename RhsMapper >

EIGEN_ALWAYS_INLINE void Eigen::internal::calcVecLoops	(	Index	cols,
		Index	rows,
		LhsMapper &	lhs,
		RhsMapper &	rhs,
		const Packet4f	pAlpha,
		float *	result
	)

                                                     {
  Index row = 0;
  if (rows >= MAX_BFLOAT16_VEC_ACC) {
    colVecLoopBody<MAX_BFLOAT16_VEC_ACC, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
    result += row;
  }
  colVecLoopBodyExtra<LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
}

References cols, MAX_BFLOAT16_VEC_ACC, row(), and rows.

calcVSXColLoops()

template<Index size>

EIGEN_ALWAYS_INLINE void Eigen::internal::calcVSXColLoops	(	const bfloat16 *&	indexA,
		const float *	indexA2,
		Index &	row,
		Index	depth,
		Index	cols,
		Index	rows,
		const Packet4f	pAlpha,
		const float *	indexB,
		Index	strideA,
		Index	strideB,
		Index	offsetA,
		Index	offsetB,
		Index	bigSuffix,
		float *	result
	)

                                                        {
  if ((size == 16) || (rows & size)) {
    indexA += size * offsetA;
    colVSXLoops<size>(depth, cols, rows, pAlpha, indexA, indexA2, indexB, strideA, strideB, offsetB, result + row);
    row += size;
    indexA += bigSuffix * size / 16;
  }
}

References cols, row(), rows, and size.

call_assignment() [1/6]

template<typename Dst , typename Src >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::call_assignment	(	const Dst &	dst,
		const Src &	src
	)

                                                                                           {
  call_assignment(dst, src, internal::assign_op<typename Dst::Scalar, typename Src::Scalar>());
}

References call_assignment().

call_assignment() [2/6]

template<typename Dst , template< typename > class StorageBase, typename Src , typename Func >

EIGEN_DEVICE_FUNC void Eigen::internal::call_assignment	(	const NoAlias< Dst, StorageBase > &	dst,
		const Src &	src,
		const Func &	func
	)

                                                                                                               {
  call_assignment_no_alias(dst.expression(), src, func);
}

References call_assignment_no_alias(), and Eigen::NoAlias< ExpressionType, StorageBase >::expression().

Referenced by Eigen::add_assign_using_evaluator(), Eigen::copy_using_evaluator(), Eigen::divide_assign_using_evaluator(), Eigen::multiply_assign_using_evaluator(), Eigen::subtract_assign_using_evaluator(), and Eigen::swap_using_evaluator().

call_assignment() [3/6]

template<typename Dst , typename Src >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::call_assignment	(	Dst &	dst,
		const Src &	src
	)

                                                                                     {
  call_assignment(dst, src, internal::assign_op<typename Dst::Scalar, typename Src::Scalar>());
}

Referenced by Eigen::PlainObjectBase< Derived >::_set(), call_assignment(), Eigen::ArrayBase< Derived >::operator*=(), Eigen::DenseBase< Derived >::operator*=(), Eigen::ArrayBase< Derived >::operator+=(), Eigen::DenseBase< Derived >::operator+=(), Eigen::DeviceWrapper< Derived, Device >::operator+=(), Eigen::SparseMatrixBase< Derived >::operator+=(), Eigen::MatrixBase< Derived >::operator+=(), Eigen::ArrayBase< Derived >::operator-=(), Eigen::DenseBase< Derived >::operator-=(), Eigen::DeviceWrapper< Derived, Device >::operator-=(), Eigen::SparseMatrixBase< Derived >::operator-=(), Eigen::MatrixBase< Derived >::operator-=(), Eigen::ArrayBase< Derived >::operator/=(), Eigen::DenseBase< Derived >::operator/=(), Eigen::ArrayBase< Derived >::operator=(), Eigen::DenseBase< Derived >::operator=(), Eigen::DeviceWrapper< Derived, Device >::operator=(), Eigen::MatrixBase< Derived >::operator=(), Eigen::DenseBase< Derived >::swap(), and Eigen::TriangularViewImpl< MatrixType_, Mode_, Dense >::swap().

call_assignment() [4/6]

template<typename Dst , typename Src , typename Func >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void Eigen::internal::call_assignment	(	Dst &	dst,
		const Src &	src,
		const Func &	func,
		std::enable_if_t< evaluator_assume_aliasing< Src >::value, void * >	= 0
	)

                                                                                                                {
  typename plain_matrix_type<Src>::type tmp(src);
  call_assignment_no_alias(dst, tmp, func);
}

References call_assignment_no_alias(), and tmp.

call_assignment() [5/6]

template<typename Dst , typename Src , typename Func >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::call_assignment	(	Dst &	dst,
		const Src &	src,
		const Func &	func,
		std::enable_if_t<!evaluator_assume_aliasing< Src >::value, void * >	= 0
	)

                                                                                                                 {
  call_assignment_no_alias(dst, src, func);
}

References call_assignment_no_alias().

call_assignment() [6/6]

template<typename Dst , template< typename > class StorageBase, typename Src , typename Func >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void Eigen::internal::call_assignment	(	NoAlias< Dst, StorageBase > &	dst,
		const Src &	src,
		const Func &	func
	)

                                                                                                             {
  call_assignment_no_alias(dst.expression(), src, func);
}

References call_assignment_no_alias(), and Eigen::NoAlias< ExpressionType, StorageBase >::expression().

call_assignment_no_alias() [1/3]

template<typename Dst , typename Src , typename Func , typename Device >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void Eigen::internal::call_assignment_no_alias	(	DeviceWrapper< Dst, Device >	dst,
		const Src &	src,
		const Func &	func
	)

                                                                                                                      {
  enum {
    NeedToTranspose = ((int(Dst::RowsAtCompileTime) == 1 && int(Src::ColsAtCompileTime) == 1) ||
                       (int(Dst::ColsAtCompileTime) == 1 && int(Src::RowsAtCompileTime) == 1)) &&
                      int(Dst::SizeAtCompileTime) != 1
  };
 
  using ActualDstTypeCleaned = std::conditional_t<NeedToTranspose, Transpose<Dst>, Dst>;
  using ActualDstType = std::conditional_t<NeedToTranspose, Transpose<Dst>, Dst&>;
  ActualDstType actualDst(dst.derived());
 
  // TODO check whether this is the right place to perform these checks:
  EIGEN_STATIC_ASSERT_LVALUE(Dst)
  EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(ActualDstTypeCleaned, Src)
  EIGEN_CHECK_BINARY_COMPATIBILIY(Func, typename ActualDstTypeCleaned::Scalar, typename Src::Scalar);
 
  // this provides a mechanism for specializing simple assignments, matrix products, etc
  AssignmentWithDevice<ActualDstTypeCleaned, Src, Func, Device>::run(actualDst, src, func, dst.device());
}

References Eigen::DeviceWrapper< Derived, Device >::derived(), EIGEN_STATIC_ASSERT_LVALUE, and int().

call_assignment_no_alias() [2/3]

template<typename Dst , typename Src >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void Eigen::internal::call_assignment_no_alias	(	Dst &	dst,
		const Src &	src
	)

                                                                                                              {
  call_assignment_no_alias(dst, src, internal::assign_op<typename Dst::Scalar, typename Src::Scalar>());
}

References call_assignment_no_alias().

call_assignment_no_alias() [3/3]

template<typename Dst , typename Src , typename Func >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void Eigen::internal::call_assignment_no_alias	(	Dst &	dst,
		const Src &	src,
		const Func &	func
	)

                                                                                                      {
  enum {
    NeedToTranspose = ((int(Dst::RowsAtCompileTime) == 1 && int(Src::ColsAtCompileTime) == 1) ||
                       (int(Dst::ColsAtCompileTime) == 1 && int(Src::RowsAtCompileTime) == 1)) &&
                      int(Dst::SizeAtCompileTime) != 1
  };
 
  typedef std::conditional_t<NeedToTranspose, Transpose<Dst>, Dst> ActualDstTypeCleaned;
  typedef std::conditional_t<NeedToTranspose, Transpose<Dst>, Dst&> ActualDstType;
  ActualDstType actualDst(dst);
 
  // TODO check whether this is the right place to perform these checks:
  EIGEN_STATIC_ASSERT_LVALUE(Dst)
  EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(ActualDstTypeCleaned, Src)
  EIGEN_CHECK_BINARY_COMPATIBILIY(Func, typename ActualDstTypeCleaned::Scalar, typename Src::Scalar);
 
  Assignment<ActualDstTypeCleaned, Src, Func>::run(actualDst, src, func);
}

References EIGEN_CHECK_BINARY_COMPATIBILIY, EIGEN_STATIC_ASSERT_LVALUE, EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE, int(), and run().

Referenced by Eigen::PlainObjectBase< Derived >::_set_noalias(), Eigen::internal::generic_product_impl< Lhs, Rhs, DenseShape, DenseShape, CoeffBasedProductMode >::addTo(), Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::assignDiagonal(), call_assignment(), call_assignment_no_alias(), Eigen::Ref< const TPlainObjectType, Options, StrideType >::construct(), Eigen::internal::generic_product_impl< Lhs, Rhs, DenseShape, DenseShape, CoeffBasedProductMode >::evalTo(), Eigen::DenseBase< Derived >::lazyAssign(), Eigen::TriangularViewImpl< MatrixType_, Mode_, Dense >::operator+=(), Eigen::SparseMatrixBase< Derived >::operator+=(), Eigen::NoAlias< ExpressionType, StorageBase >::operator+=(), Eigen::TriangularViewImpl< MatrixType_, Mode_, Dense >::operator-=(), Eigen::SparseMatrixBase< Derived >::operator-=(), Eigen::NoAlias< ExpressionType, StorageBase >::operator-=(), Eigen::SparseMatrixBase< Derived >::operator=(), Eigen::NoAlias< ExpressionType, StorageBase >::operator=(), Eigen::internal::Assignment< DstXprType, CwiseBinaryOp< internal::scalar_product_op< ScalarBis, Scalar >, const CwiseNullaryOp< internal::scalar_constant_op< ScalarBis >, Plain >, const Product< Lhs, Rhs, DefaultProduct > >, AssignFunc, Dense2Dense >::run(), Eigen::internal::assignment_from_xpr_op_product< DstXprType, OtherXpr, ProductType, Func1, Func2 >::run(), Eigen::internal::assignment_from_dense_op_sparse< DstXprType, Func1, Func2 >::run(), Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::SparseMatrix(), Eigen::internal::generic_product_impl< Lhs, Rhs, DenseShape, DenseShape, CoeffBasedProductMode >::subTo(), and Eigen::internal::unary_evaluator< Inverse< ArgType > >::unary_evaluator().

call_assignment_no_alias_no_transpose() [1/2]

template<typename Dst , typename Src >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void Eigen::internal::call_assignment_no_alias_no_transpose	(	Dst &	dst,
		const Src &	src
	)

                                                                                                                 {
  call_assignment_no_alias_no_transpose(dst, src, internal::assign_op<typename Dst::Scalar, typename Src::Scalar>());
}

References call_assignment_no_alias_no_transpose().

call_assignment_no_alias_no_transpose() [2/2]

template<typename Dst , typename Src , typename Func >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void Eigen::internal::call_assignment_no_alias_no_transpose	(	Dst &	dst,
		const Src &	src,
		const Func &	func
	)

                                                                                                                   {
  // TODO check whether this is the right place to perform these checks:
  EIGEN_STATIC_ASSERT_LVALUE(Dst)
  EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(Dst, Src)
  EIGEN_CHECK_BINARY_COMPATIBILIY(Func, typename Dst::Scalar, typename Src::Scalar);
 
  Assignment<Dst, Src, Func>::run(dst, src, func);
}

References EIGEN_CHECK_BINARY_COMPATIBILIY, EIGEN_STATIC_ASSERT_LVALUE, EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE, and run().

Referenced by call_assignment_no_alias_no_transpose(), Eigen::SparseSelfAdjointView< MatrixType, Mode_ >::operator=(), and Eigen::internal::Assignment< DstXprType, SrcXprType, Functor, SparseSelfAdjoint2Sparse >::run().

call_dense_assignment_loop() [1/3]

template<typename DstXprType , typename SrcXprType >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::call_dense_assignment_loop	(	DstXprType &	dst,
		const SrcXprType &	src
	)

                                                                                                              {
  call_dense_assignment_loop(dst, src, internal::assign_op<typename DstXprType::Scalar, typename SrcXprType::Scalar>());
}

References call_dense_assignment_loop().

call_dense_assignment_loop() [2/3]

template<typename DstXprType , typename SrcXprType , typename Functor >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void Eigen::internal::call_dense_assignment_loop	(	DstXprType &	dst,
		const SrcXprType &	src,
		const Functor &	func
	)

                                                                                                           {
  typedef evaluator<DstXprType> DstEvaluatorType;
  typedef evaluator<SrcXprType> SrcEvaluatorType;
 
  SrcEvaluatorType srcEvaluator(src);
 
  // NOTE To properly handle A = (A*A.transpose())/s with A rectangular,
  // we need to resize the destination after the source evaluator has been created.
  resize_if_allowed(dst, src, func);
 
  DstEvaluatorType dstEvaluator(dst);
 
  typedef generic_dense_assignment_kernel<DstEvaluatorType, SrcEvaluatorType, Functor> Kernel;
  Kernel kernel(dstEvaluator, srcEvaluator, func, dst.const_cast_derived());
 
  dense_assignment_loop<Kernel>::run(kernel);
}

References resize_if_allowed(), and run().

Referenced by call_dense_assignment_loop(), Eigen::internal::Assignment< DstXprType, SrcXprType, Functor, Dense2Dense, Weak >::run(), Eigen::internal::AssignmentWithDevice< DstXprType, SrcXprType, Functor, Device, Dense2Dense, Weak >::run(), Eigen::internal::eigen_fill_impl< Xpr, false >::run(), and Eigen::internal::eigen_zero_impl< Xpr, false >::run().

call_dense_assignment_loop() [3/3]

template<typename DstXprType , typename SrcXprType , typename Functor , typename Device >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR void Eigen::internal::call_dense_assignment_loop	(	DstXprType &	dst,
		const SrcXprType &	src,
		const Functor &	func,
		Device &	device
	)

                                                                                                      {
  using DstEvaluatorType = evaluator<DstXprType>;
  using SrcEvaluatorType = evaluator<SrcXprType>;
 
  SrcEvaluatorType srcEvaluator(src);
 
  // NOTE To properly handle A = (A*A.transpose())/s with A rectangular,
  // we need to resize the destination after the source evaluator has been created.
  resize_if_allowed(dst, src, func);
 
  DstEvaluatorType dstEvaluator(dst);
 
  using Kernel = generic_dense_assignment_kernel<DstEvaluatorType, SrcEvaluatorType, Functor>;
 
  Kernel kernel(dstEvaluator, srcEvaluator, func, dst.const_cast_derived());
 
  dense_assignment_loop_with_device<Kernel, Device>::run(kernel, device);
}

References resize_if_allowed(), and Eigen::internal::dense_assignment_loop_with_device< Kernel, Device, Traversal, Unrolling >::run().

call_restricted_packet_assignment()

template<typename Dst , template< typename > class StorageBase, typename Src , typename Func >

EIGEN_DEVICE_FUNC void Eigen::internal::call_restricted_packet_assignment	(	const NoAlias< Dst, StorageBase > &	dst,
		const Src &	src,
		const Func &	func
	)

                                                                           {
  call_restricted_packet_assignment_no_alias(dst.expression(), src, func);
}

References call_restricted_packet_assignment_no_alias(), and Eigen::NoAlias< ExpressionType, StorageBase >::expression().

Referenced by EIGEN_DECLARE_TEST().

call_restricted_packet_assignment_no_alias()

template<typename Dst , typename Src , typename Func >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::call_restricted_packet_assignment_no_alias	(	Dst &	dst,
		const Src &	src,
		const Func &	func
	)

                                                                                                        {
  typedef evaluator<Dst> DstEvaluatorType;
  typedef evaluator<Src> SrcEvaluatorType;
  typedef restricted_packet_dense_assignment_kernel<DstEvaluatorType, SrcEvaluatorType, Func> Kernel;
 
  EIGEN_STATIC_ASSERT_LVALUE(Dst)
  EIGEN_CHECK_BINARY_COMPATIBILIY(Func, typename Dst::Scalar, typename Src::Scalar);
 
  SrcEvaluatorType srcEvaluator(src);
  resize_if_allowed(dst, src, func);
 
  DstEvaluatorType dstEvaluator(dst);
  Kernel kernel(dstEvaluator, srcEvaluator, func, dst.const_cast_derived());
 
  dense_assignment_loop<Kernel>::run(kernel);
}

References EIGEN_CHECK_BINARY_COMPATIBILIY, EIGEN_STATIC_ASSERT_LVALUE, resize_if_allowed(), and run().

Referenced by call_restricted_packet_assignment(), and Eigen::internal::generic_product_impl< Lhs, Rhs, DenseShape, DenseShape, CoeffBasedProductMode >::eval_dynamic_impl().

call_triangular_assignment_loop() [1/2]

template<int Mode, bool SetOpposite, typename DstXprType , typename SrcXprType >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::call_triangular_assignment_loop	(	DstXprType &	dst,
		const SrcXprType &	src
	)

                                                                                                                   {
  call_triangular_assignment_loop<Mode, SetOpposite>(
      dst, src, internal::assign_op<typename DstXprType::Scalar, typename SrcXprType::Scalar>());
}

call_triangular_assignment_loop() [2/2]

template<int Mode, bool SetOpposite, typename DstXprType , typename SrcXprType , typename Functor >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::call_triangular_assignment_loop	(	DstXprType &	dst,
		const SrcXprType &	src,
		const Functor &	func
	)

                                                                                                {
  typedef evaluator<DstXprType> DstEvaluatorType;
  typedef evaluator<SrcXprType> SrcEvaluatorType;
 
  SrcEvaluatorType srcEvaluator(src);
 
  Index dstRows = src.rows();
  Index dstCols = src.cols();
  if ((dst.rows() != dstRows) || (dst.cols() != dstCols)) dst.resize(dstRows, dstCols);
  DstEvaluatorType dstEvaluator(dst);
 
  typedef triangular_dense_assignment_kernel<Mode&(Lower | Upper), Mode&(UnitDiag | ZeroDiag | SelfAdjoint),
                                             SetOpposite, DstEvaluatorType, SrcEvaluatorType, Functor>
      Kernel;
  Kernel kernel(dstEvaluator, srcEvaluator, func, dst.const_cast_derived());
 
  enum {
    unroll = DstXprType::SizeAtCompileTime != Dynamic && SrcEvaluatorType::CoeffReadCost < HugeCost &&
             DstXprType::SizeAtCompileTime *
                     (int(DstEvaluatorType::CoeffReadCost) + int(SrcEvaluatorType::CoeffReadCost)) / 2 <=
                 EIGEN_UNROLLING_LIMIT
  };
 
  triangular_assignment_loop<Kernel, Mode, unroll ? int(DstXprType::SizeAtCompileTime) : Dynamic, SetOpposite>::run(
      kernel);
}

References Eigen::Dynamic, EIGEN_UNROLLING_LIMIT, Eigen::HugeCost, int(), Eigen::Lower, Eigen::run(), Eigen::SelfAdjoint, Eigen::UnitDiag, Eigen::Upper, and Eigen::ZeroDiag.

Referenced by Eigen::TriangularBase< Derived >::evalToLazy().

cast() [1/2]

template<>

EIGEN_STRONG_INLINE float Eigen::internal::cast

(

const AnnoyingScalar &

x

)

                                                         {
  return double(*x.v);
}

References plotDoE::x.

cast() [2/2]

template<typename OldType , typename NewType >

EIGEN_DEVICE_FUNC NewType Eigen::internal::cast ( const OldType &  x )

inline

                                                        {
  return cast_impl<OldType, NewType>::run(x);
}

References Eigen::internal::cast_impl< OldType, NewType, EnableIf >::run(), and plotDoE::x.

Referenced by mixingtypes(), Eigen::HouseholderSequence< VectorsType, CoeffsType, Side >::operator*(), Eigen::operator*(), polynomialsolver(), and test_cast_helper< SrcPacket, TgtPacket, SrcCoeffRatio, TgtCoeffRatio, true >::run().

cat256()

EIGEN_STRONG_INLINE Packet16f Eigen::internal::cat256	(	Packet8f	a,
		Packet8f	b
	)

                                                             {
  return _mm512_castsi512_ps(
      _mm512_inserti64x4(_mm512_castsi256_si512(_mm256_castps_si256(a)), _mm256_castps_si256(b), 1));
}

References a, and b.

Referenced by pcast< Packet8d, Packet16f >().

cat256i()

EIGEN_STRONG_INLINE Packet16i Eigen::internal::cat256i	(	Packet8i	a,
		Packet8i	b
	)

                                                              {
  return _mm512_inserti64x4(_mm512_castsi256_si512(a), b, 1);
}

References a, and b.

Referenced by pcast< Packet8d, Packet16i >().

check_for_aliasing()

template<typename Dst , typename Src >

EIGEN_DEVICE_FUNC void Eigen::internal::check_for_aliasing ( const Dst &  dst, const Src &  src  )

inline

                                                                                 {
  if ((!Dst::IsVectorAtCompileTime) && dst.rows() > 1 && dst.cols() > 1)
    internal::checkTransposeAliasing_impl<Dst, Src>::run(dst, src);
}

References Eigen::internal::checkTransposeAliasing_impl< Derived, OtherDerived, MightHaveTransposeAliasing >::run().

Referenced by Eigen::internal::Assignment< DstXprType, SrcXprType, Functor, Dense2Dense, Weak >::run(), and Eigen::internal::AssignmentWithDevice< DstXprType, SrcXprType, Functor, Device, Dense2Dense, Weak >::run().

check_implication()

constexpr bool Eigen::internal::check_implication ( bool  a, bool  b  )

inlineconstexpr

Calculate logical IMPLIES at compile time

{ return !a || b; }

References a, and b.

Referenced by Eigen::ConjugateGradient< MatrixType_, UpLo_, Preconditioner_ >::_solve_vector_with_guess_impl(), Eigen::MINRES< MatrixType_, UpLo_, Preconditioner_ >::_solve_vector_with_guess_impl(), Eigen::VectorwiseOp< ExpressionType, Direction >::extendedTo(), Eigen::VectorwiseOp< ExpressionType, Direction >::extendedToOpposite(), Eigen::internal::mapbase_evaluator< Derived, PlainObjectType >::mapbase_evaluator(), Eigen::PlainObjectBase< Derived >::resize(), and Eigen::Transform< Scalar_, Dim_, Mode_, Options_ >::Transform().

check_size_for_overflow()

template<typename T >

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void Eigen::internal::check_size_for_overflow

(

std::size_t

size

)

                                                                                   {
  constexpr std::size_t max_elements = (std::numeric_limits<std::ptrdiff_t>::max)() / sizeof(T);
  if (size > max_elements) throw_std_bad_alloc();
}

References max, size, and throw_std_bad_alloc().

check_svd_options_assertions()

template<typename MatrixType , int Options>

void Eigen::internal::check_svd_options_assertions	(	unsigned int	computationOptions,
		Index	rows,
		Index	cols
	)

                                                                                           {
  EIGEN_STATIC_ASSERT((Options & ComputationOptionsBits) == 0,
                      "SVDBase: Cannot request U or V using both static and runtime options, even if they match. "
                      "Requesting unitaries at runtime is DEPRECATED: "
                      "Prefer requesting unitaries statically, using the Options template parameter.");
  eigen_assert(
      !(should_svd_compute_thin_u(computationOptions) && cols < rows && MatrixType::RowsAtCompileTime != Dynamic) &&
      !(should_svd_compute_thin_v(computationOptions) && rows < cols && MatrixType::ColsAtCompileTime != Dynamic) &&
      "SVDBase: If thin U is requested at runtime, your matrix must have more rows than columns or a dynamic number of "
      "rows."
      "Similarly, if thin V is requested at runtime, you matrix must have more columns than rows or a dynamic number "
      "of columns.");
  (void)computationOptions;
  (void)rows;
  (void)cols;
}

References cols, ComputationOptionsBits, Eigen::Dynamic, eigen_assert, EIGEN_STATIC_ASSERT, rows, should_svd_compute_thin_u(), and should_svd_compute_thin_v().

check_that_malloc_is_allowed()

EIGEN_DEVICE_FUNC void Eigen::internal::check_that_malloc_is_allowed ( )

inline

{}

Referenced by aligned_free(), aligned_malloc(), aligned_realloc(), conditional_aligned_free< false >(), conditional_aligned_malloc< false >(), conditional_aligned_realloc< false >(), handmade_aligned_free(), handmade_aligned_malloc(), and handmade_aligned_realloc().

chkder()

template<typename Scalar >

void Eigen::internal::chkder	(	const Matrix< Scalar, Dynamic, 1 > &	x,
		const Matrix< Scalar, Dynamic, 1 > &	fvec,
		const Matrix< Scalar, Dynamic, Dynamic > &	fjac,
		Matrix< Scalar, Dynamic, 1 > &	xp,
		const Matrix< Scalar, Dynamic, 1 > &	fvecp,
		int	mode,
		Matrix< Scalar, Dynamic, 1 > &	err
	)

                                                                                                {
  using std::abs;
  using std::log;
  using std::sqrt;
 
  typedef DenseIndex Index;
 
  const Scalar eps = sqrt(NumTraits<Scalar>::epsilon());
  const Scalar epsf = chkder_factor * NumTraits<Scalar>::epsilon();
  const Scalar epslog = chkder_log10e * log(eps);
  Scalar temp;
 
  const Index m = fvec.size(), n = x.size();
 
  if (mode != 2) {
    /* mode = 1. */
    xp.resize(n);
    for (Index j = 0; j < n; ++j) {
      temp = eps * abs(x[j]);
      if (temp == 0.) temp = eps;
      xp[j] = x[j] + temp;
    }
  } else {
    /* mode = 2. */
    err.setZero(m);
    for (Index j = 0; j < n; ++j) {
      temp = abs(x[j]);
      if (temp == 0.) temp = 1.;
      err += temp * fjac.col(j);
    }
    for (Index i = 0; i < m; ++i) {
      temp = 1.;
      if (fvec[i] != 0. && fvecp[i] != 0. && abs(fvecp[i] - fvec[i]) >= epsf * abs(fvec[i]))
        temp = eps * abs((fvecp[i] - fvec[i]) / eps - err[i]) / (abs(fvec[i]) + abs(fvecp[i]));
      err[i] = 1.;
      if (temp > NumTraits<Scalar>::epsilon() && temp < eps) err[i] = (chkder_log10e * log(temp) - epslog) / epslog;
      if (temp >= eps) err[i] = 0.;
    }
  }
}

References abs(), chkder_factor, chkder_log10e, CRBond_Bessel::eps, oomph::SarahBL::epsilon, i, j, Eigen::bfloat16_impl::log(), m, n, Eigen::PlainObjectBase< Derived >::resize(), Eigen::PlainObjectBase< Derived >::setZero(), sqrt(), and plotDoE::x.

Referenced by testChkder().

clz()

template<typename BitsType >

EIGEN_DEVICE_FUNC int Eigen::internal::clz ( BitsType  bits )

inline

                                                {
  return count_bits_impl<BitsType>::clz(bits);
}

References Eigen::test::bits(), and Eigen::internal::count_bits_impl< BitsType, EnableIf >::clz().

Referenced by count_leading_zeros(), test_clz_ctz(), and test_clz_ctz_random().

cm_factorize_p()

template<typename StorageIndex_ >

int Eigen::internal::cm_factorize_p ( cholmod_sparse *  A, double  beta[2], StorageIndex_ *  fset, std::size_t  fsize, cholmod_factor *  L, cholmod_common &  Common  )

inline

                                                  {
  return cholmod_factorize_p(A, beta, fset, fsize, L, &Common);
}

References beta, and L.

cm_factorize_p< SuiteSparse_long >()

template<>

int Eigen::internal::cm_factorize_p< SuiteSparse_long > ( cholmod_sparse *  A, double  beta[2], SuiteSparse_long *  fset, std::size_t  fsize, cholmod_factor *  L, cholmod_common &  Common  )

inline

                                                                                                        {
  return cholmod_l_factorize_p(A, beta, fset, fsize, L, &Common);
}

References beta, and L.

cm_solve()

template<typename StorageIndex_ >

cholmod_dense* Eigen::internal::cm_solve ( int  sys, cholmod_factor &  L, cholmod_dense &  B, cholmod_common &  Common  )

inline

                                                                                                     {
  return cholmod_solve(sys, &L, &B, &Common);
}

References L.

cm_solve< SuiteSparse_long >()

template<>

cholmod_dense* Eigen::internal::cm_solve< SuiteSparse_long > ( int  sys, cholmod_factor &  L, cholmod_dense &  B, cholmod_common &  Common  )

inline

                                                                                                                       {
  return cholmod_l_solve(sys, &L, &B, &Common);
}

References L.

cm_spsolve()

template<typename StorageIndex_ >

cholmod_sparse* Eigen::internal::cm_spsolve ( int  sys, cholmod_factor &  L, cholmod_sparse &  B, cholmod_common &  Common  )

inline

                                                                                                         {
  return cholmod_spsolve(sys, &L, &B, &Common);
}

References L.

cm_spsolve< SuiteSparse_long >()

template<>

cholmod_sparse* Eigen::internal::cm_spsolve< SuiteSparse_long > ( int  sys, cholmod_factor &  L, cholmod_sparse &  B, cholmod_common &  Common  )

inline

                                                                            {
  return cholmod_l_spsolve(sys, &L, &B, &Common);
}

References L.

cofactor_3x3()

template<typename MatrixType , int i, int j>

EIGEN_DEVICE_FUNC MatrixType::Scalar Eigen::internal::cofactor_3x3 ( const MatrixType &  m )

inline

                                                                                     {
  enum { i1 = (i + 1) % 3, i2 = (i + 2) % 3, j1 = (j + 1) % 3, j2 = (j + 2) % 3 };
  return m.coeff(i1, j1) * m.coeff(i2, j2) - m.coeff(i1, j2) * m.coeff(i2, j1);
}

References i, j, and m.

cofactor_4x4()

template<typename MatrixType , int i, int j>

EIGEN_DEVICE_FUNC MatrixType::Scalar Eigen::internal::cofactor_4x4 ( const MatrixType &  matrix )

inline

                                                                                          {
  enum { i1 = (i + 1) % 4, i2 = (i + 2) % 4, i3 = (i + 3) % 4, j1 = (j + 1) % 4, j2 = (j + 2) % 4, j3 = (j + 3) % 4 };
  return general_det3_helper(matrix, i1, i2, i3, j1, j2, j3) + general_det3_helper(matrix, i2, i3, i1, j1, j2, j3) +
         general_det3_helper(matrix, i3, i1, i2, j1, j2, j3);
}

References general_det3_helper(), i, j, and matrix().

coletree()

template<typename MatrixType , typename IndexVector >

int Eigen::internal::coletree	(	const MatrixType &	mat,
		IndexVector &	parent,
		IndexVector &	firstRowElt,
		typename MatrixType::StorageIndex *	perm = 0
	)

Compute the column elimination tree of a sparse matrix

Parameters

mat	The matrix in column-major format.
parent	The elimination tree
firstRowElt	The column index of the first element in each row
perm	The permutation to apply to the column of mat

                                                        {
  typedef typename MatrixType::StorageIndex StorageIndex;
  StorageIndex nc = convert_index<StorageIndex>(mat.cols());  // Number of columns
  StorageIndex m = convert_index<StorageIndex>(mat.rows());
  StorageIndex diagSize = (std::min)(nc, m);
  IndexVector root(nc);  // root of subtree of etree
  root.setZero();
  IndexVector pp(nc);  // disjoint sets
  pp.setZero();        // Initialize disjoint sets
  parent.resize(mat.cols());
  // Compute first nonzero column in each row
  firstRowElt.resize(m);
  firstRowElt.setConstant(nc);
  firstRowElt.segment(0, diagSize).setLinSpaced(diagSize, 0, diagSize - 1);
  bool found_diag;
  for (StorageIndex col = 0; col < nc; col++) {
    StorageIndex pcol = col;
    if (perm) pcol = perm[col];
    for (typename MatrixType::InnerIterator it(mat, pcol); it; ++it) {
      Index row = it.row();
      firstRowElt(row) = (std::min)(firstRowElt(row), col);
    }
  }
  /* Compute etree by Liu's algorithm for symmetric matrices,
          except use (firstRowElt[r],c) in place of an edge (r,c) of A.
    Thus each row clique in A'*A is replaced by a star
    centered at its first vertex, which has the same fill. */
  StorageIndex rset, cset, rroot;
  for (StorageIndex col = 0; col < nc; col++) {
    found_diag = col >= m;
    pp(col) = col;
    cset = col;
    root(cset) = col;
    parent(col) = nc;
    /* The diagonal element is treated here even if it does not exist in the matrix
     * hence the loop is executed once more */
    StorageIndex pcol = col;
    if (perm) pcol = perm[col];
    for (typename MatrixType::InnerIterator it(mat, pcol); it || !found_diag;
         ++it) {  //  A sequence of interleaved find and union is performed
      Index i = col;
      if (it) i = it.index();
      if (i == col) found_diag = true;
 
      StorageIndex row = firstRowElt(i);
      if (row >= col) continue;
      rset = internal::etree_find(row, pp);  // Find the name of the set containing row
      rroot = root(rset);
      if (rroot != col) {
        parent(rroot) = col;
        pp(cset) = rset;
        cset = rset;
        root(cset) = col;
      }
    }
  }
  return 0;
}

References col(), Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::cols(), etree_find(), i, m, min, row(), and Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::rows().

Referenced by Eigen::SparseQR< MatrixType_, OrderingType_ >::analyzePattern(), Eigen::SparseLU< MatrixType_, OrderingType_ >::analyzePattern(), and Eigen::SparseQR< MatrixType_, OrderingType_ >::factorize().

colLoopBody()

template<const Index num_acc, const Index num_packets, bool rhsExtraCols, bool lhsExtraRows>

void Eigen::internal::colLoopBody	(	Index &	col,
		Index	depth,
		Index	cols,
		Index	rows,
		const Packet4f	pAlpha,
		const bfloat16 *	indexA,
		const bfloat16 *	indexB,
		Index	strideB,
		Index	offsetB,
		float *	result
	)

                                                                                      {
  constexpr Index step = (num_acc * 4);  // each accumulator has 4 elements
  const Index extra_cols = (rhsExtraCols) ? (cols & 3) : 0;
  const Index extra_rows = (lhsExtraRows) ? (rows & 3) : 0;
  constexpr bool multiIters = !rhsExtraCols && (num_acc == MAX_BFLOAT16_ACC);
  constexpr bool normIters = multiIters && ((num_acc % (num_packets / 4)) == 0);
 
  do {
    colLoopBodyIter<num_acc, num_packets, rhsExtraCols, lhsExtraRows, normIters>(
        depth, rows, pAlpha, indexA, indexB, strideB, offsetB, result, extra_cols, extra_rows);
 
    indexB += strideB * num_acc;
    result += rows * step;
  } while (multiIters && (step <= cols - (col += step)));
}

References col(), cols, MAX_BFLOAT16_ACC, and rows.

colLoopBodyExtra()

template<const Index num_packets, bool rhsExtraCols, bool lhsExtraRows>

void Eigen::internal::colLoopBodyExtra	(	Index	col,
		Index	depth,
		Index	cols,
		Index	rows,
		const Packet4f	pAlpha,
		const bfloat16 *	indexA,
		const bfloat16 *	blockB,
		Index	strideB,
		Index	offsetB,
		float *	result
	)

                                                                                           {
  switch ((cols - col) >> 2) {
    case 7:
      colLoopBodyExtraN<7, num_packets, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB,
                                                                    strideB, offsetB, result);
      break;
    case 6:
      colLoopBodyExtraN<6, num_packets, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB,
                                                                    strideB, offsetB, result);
      break;
    case 5:
      colLoopBodyExtraN<5, num_packets, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB,
                                                                    strideB, offsetB, result);
      break;
    case 4:
      colLoopBodyExtraN<4, num_packets, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB,
                                                                    strideB, offsetB, result);
      break;
    case 3:
      colLoopBodyExtraN<3, num_packets, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB,
                                                                    strideB, offsetB, result);
      break;
    case 2:
      colLoopBodyExtraN<2, num_packets, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB,
                                                                    strideB, offsetB, result);
      break;
    case 1:
      colLoopBodyExtraN<1, num_packets, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB,
                                                                    strideB, offsetB, result);
      break;
    default:
      if (rhsExtraCols) {
        colLoopBody<1, num_packets, true, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, strideB,
                                                        offsetB, result);
      }
      break;
  }
}

References col(), cols, and rows.

colLoopBodyExtraN()

template<const Index num_acc, const Index num_packets, bool rhsExtraCols, bool lhsExtraRows>

EIGEN_ALWAYS_INLINE void Eigen::internal::colLoopBodyExtraN	(	Index	col,
		Index	depth,
		Index	cols,
		Index	rows,
		const Packet4f	pAlpha,
		const bfloat16 *	indexA,
		const bfloat16 *	blockB,
		Index	strideB,
		Index	offsetB,
		float *	result
	)

                                                          {
  if (MAX_BFLOAT16_ACC > num_acc) {
    colLoopBody<num_acc + (rhsExtraCols ? 1 : 0), num_packets, rhsExtraCols, lhsExtraRows>(
        col, depth, cols, rows, pAlpha, indexA, blockB, strideB, offsetB, result);
  }
}

References col(), cols, MAX_BFLOAT16_ACC, and rows.

colLoopBodyIter()

template<const Index num_acc, const Index num_packets, bool rhsExtraCols, bool lhsExtraRows, bool multiIter = false>

EIGEN_ALWAYS_INLINE void Eigen::internal::colLoopBodyIter	(	Index	depth,
		Index	rows,
		const Packet4f	pAlpha,
		const bfloat16 *	indexA,
		const bfloat16 *	indexB,
		Index	strideB,
		Index	offsetB,
		float *	result,
		const Index	extra_cols,
		const Index	extra_rows
	)

                                                                                         {
  constexpr Index num_lhs = multiIter ? (num_packets / 4) : 1;
  constexpr Index num_rhs = (num_acc + num_lhs - 1) / num_lhs;
 
  for (Index offset_row = 0; offset_row < num_packets; offset_row += 4, indexA += (multiIter ? 0 : 8),
             indexB += (multiIter ? (num_rhs * strideB) : 0), result += (multiIter ? (4 * rows * num_rhs) : 4)) {
    Packet4f acc[num_acc][4];
    __vector_quad quad_acc[num_acc];
 
    zeroAccumulators<num_acc>(quad_acc);
 
    Index k;
    for (k = 0; k + 2 <= depth; k += 2) {
      KLoop<num_acc, num_packets, false, rhsExtraCols, lhsExtraRows, num_rhs, num_lhs>(
          indexA, indexB, quad_acc, strideB, k, offsetB, extra_cols, extra_rows);
    }
    if (depth & 1) {
      KLoop<num_acc, num_packets, true, rhsExtraCols, lhsExtraRows, num_rhs, num_lhs>(
          indexA - (multiIter ? 0 : offset_row), indexB, quad_acc, strideB, k, offsetB, extra_cols, extra_rows);
    }
 
    disassembleAccumulators<num_acc>(quad_acc, acc);
 
    outputResults<num_acc, rhsExtraCols, lhsExtraRows, num_rhs, num_lhs>(acc, rows, pAlpha, result, extra_cols,
                                                                         extra_rows);
  }
}

References k, and rows.

colLoops()

template<const Index num_packets, bool lhsExtraRows = false>

EIGEN_ALWAYS_INLINE void Eigen::internal::colLoops	(	Index	depth,
		Index	cols,
		Index	rows,
		const Packet4f	pAlpha,
		const bfloat16 *	indexA,
		const bfloat16 *	blockB,
		Index	strideB,
		Index	offsetB,
		float *	result
	)

                                                                                                       {
  Index col = 0;
  if (cols >= (MAX_BFLOAT16_ACC * 4)) {
    colLoopBody<MAX_BFLOAT16_ACC, num_packets, false, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB,
                                                                    strideB, 0, result);
    blockB += (strideB >> 2) * col;
    result += rows * col;
  }
  if (cols & 3) {
    colLoopBodyExtra<num_packets, true, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, strideB, offsetB,
                                                      result);
  } else {
    colLoopBodyExtra<num_packets, false, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, strideB, 0,
                                                       result);
  }
}

References col(), cols, MAX_BFLOAT16_ACC, and rows.

colVecColLoopBody()

template<const Index num_acc, typename LhsMapper , typename RhsMapper , bool extraRows, bool linear>

void Eigen::internal::colVecColLoopBody	(	Index &	row,
		Index	cend,
		Index	rows,
		LhsMapper &	lhs,
		RhsMapper &	rhs,
		const Packet4f	pAlpha,
		float *	result
	)

                                      {
  constexpr Index step = (num_acc * 4);
  const Index extra_rows = (extraRows) ? (rows & 3) : 0;
  constexpr bool multiIters = !extraRows && (num_acc == MAX_BFLOAT16_VEC_ACC);
 
  do {
    Packet4f acc[num_acc][4];
    __vector_quad quad_acc[num_acc];
 
    zeroAccumulators<num_acc>(quad_acc);
 
    using LhsSubMapper = typename LhsMapper::SubMapper;
 
    LhsSubMapper lhs2 = lhs.getSubMapper(row, 0);
    for (Index j = 0; j + 2 <= cend; j += 2) {
      vecColLoop<num_acc, LhsSubMapper, RhsMapper, false, linear>(j, lhs2, rhs, quad_acc);
    }
    if (cend & 1) {
      vecColLoop<num_acc, LhsSubMapper, RhsMapper, true, linear>(cend - 1, lhs2, rhs, quad_acc);
    }
 
    disassembleAccumulators<num_acc>(quad_acc, acc);
 
    outputVecColResults<num_acc, extraRows>(acc, result, pAlpha, extra_rows);
 
    result += step;
  } while (multiIters && (step <= rows - (row += step)));
}

References j, MAX_BFLOAT16_VEC_ACC, row(), and rows.

colVecColLoopBodyExtra()

template<typename LhsMapper , typename RhsMapper , bool extraRows, bool linear>

EIGEN_ALWAYS_INLINE void Eigen::internal::colVecColLoopBodyExtra	(	Index &	row,
		Index	cend,
		Index	rows,
		LhsMapper &	lhs,
		RhsMapper &	rhs,
		const Packet4f	pAlpha,
		float *	result
	)

                                                                                      {
  switch ((rows - row) >> 2) {
    case 7:
      colVecColLoopBodyExtraN<7, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
      break;
    case 6:
      colVecColLoopBodyExtraN<6, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
      break;
    case 5:
      colVecColLoopBodyExtraN<5, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
      break;
    case 4:
      colVecColLoopBodyExtraN<4, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
      break;
    case 3:
      colVecColLoopBodyExtraN<3, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
      break;
    case 2:
      colVecColLoopBodyExtraN<2, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
      break;
    case 1:
      colVecColLoopBodyExtraN<1, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
      break;
    default:
      if (extraRows) {
        colVecColLoopBody<1, LhsMapper, RhsMapper, true, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
      }
      break;
  }
}

References row(), and rows.

colVecColLoopBodyExtraN()

template<const Index num_acc, typename LhsMapper , typename RhsMapper , bool extraRows, bool linear>

EIGEN_ALWAYS_INLINE void Eigen::internal::colVecColLoopBodyExtraN	(	Index &	row,
		Index	cend,
		Index	rows,
		LhsMapper &	lhs,
		RhsMapper &	rhs,
		const Packet4f	pAlpha,
		float *	result
	)

                                                                                       {
  if (MAX_BFLOAT16_VEC_ACC > num_acc) {
    colVecColLoopBody<num_acc + (extraRows ? 1 : 0), LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs,
                                                                                              pAlpha, result);
  }
}

References MAX_BFLOAT16_VEC_ACC, row(), and rows.

colVecLoopBody()

template<const Index num_acc, typename LhsMapper , typename RhsMapper >

void Eigen::internal::colVecLoopBody	(	Index &	row,
		Index	cols,
		Index	rows,
		LhsMapper &	lhs,
		RhsMapper &	rhs,
		const Packet4f	pAlpha,
		float *	result
	)

                                   {
  constexpr bool multiIters = (num_acc == MAX_BFLOAT16_VEC_ACC);
  const Index extra_cols = (cols & 7);
 
  do {
    Packet4f acc[num_acc][4];
    __vector_quad quad_acc[num_acc];
 
    zeroAccumulators<num_acc>(quad_acc);
 
    const LhsMapper lhs2 = lhs.getSubMapper(row, 0);
    vecLoop<num_acc, LhsMapper, RhsMapper>(cols, lhs2, rhs, quad_acc, extra_cols);
 
    disassembleAccumulators<num_acc>(quad_acc, acc);
 
    preduxVecResults<num_acc>(acc);
 
    outputVecResults<num_acc>(acc, result, pAlpha);
 
    result += num_acc;
  } while (multiIters && (num_acc <= rows - (row += num_acc)));
}

References cols, MAX_BFLOAT16_VEC_ACC, row(), and rows.

colVecLoopBodyExtra()

template<typename LhsMapper , typename RhsMapper >

EIGEN_ALWAYS_INLINE void Eigen::internal::colVecLoopBodyExtra	(	Index &	row,
		Index	cols,
		Index	rows,
		LhsMapper &	lhs,
		RhsMapper &	rhs,
		const Packet4f	pAlpha,
		float *	result
	)

                                                                                   {
  switch (rows - row) {
    case 7:
      colVecLoopBodyExtraN<7, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
      break;
    case 6:
      colVecLoopBodyExtraN<6, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
      break;
    case 5:
      colVecLoopBodyExtraN<5, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
      break;
    case 4:
      colVecLoopBodyExtraN<4, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
      break;
    case 3:
      colVecLoopBodyExtraN<3, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
      break;
    case 2:
      colVecLoopBodyExtraN<2, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
      break;
    case 1:
      colVecLoopBodyExtraN<1, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
      break;
  }
}

References cols, row(), and rows.

colVecLoopBodyExtraN()

template<const Index num_acc, typename LhsMapper , typename RhsMapper >

EIGEN_ALWAYS_INLINE void Eigen::internal::colVecLoopBodyExtraN	(	Index &	row,
		Index	cols,
		Index	rows,
		LhsMapper &	lhs,
		RhsMapper &	rhs,
		const Packet4f	pAlpha,
		float *	result
	)

                                                                                    {
  if (MAX_BFLOAT16_VEC_ACC > num_acc) {
    colVecLoopBody<num_acc, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
  }
}

References cols, MAX_BFLOAT16_VEC_ACC, row(), and rows.

colVSXLoopBody()

template<const Index num_acc, bool rhsExtraCols, bool lhsExtraRows>

void Eigen::internal::colVSXLoopBody	(	Index &	col,
		Index	depth,
		Index	cols,
		Index	rows,
		const Packet4f	pAlpha,
		const float *	indexA,
		const float *	indexB,
		Index	strideB,
		Index	offsetB,
		float *	result
	)

                                                                                      {
  constexpr Index step = (num_acc * 4);  // each accumulator has 4 elements
  const Index extra_cols = (rhsExtraCols) ? (cols & 3) : 0;
  const Index extra_rows = (lhsExtraRows) ? (rows & 3) : 0;
  constexpr bool multiIters = !rhsExtraCols && (num_acc == MAX_BFLOAT16_ACC_VSX);
 
  do {
    colVSXLoopBodyIter<num_acc * 2, rhsExtraCols, lhsExtraRows>(depth, rows, pAlpha, indexA, indexB, strideB, offsetB,
                                                                result, extra_cols, extra_rows);
 
    indexB += strideB * (num_acc * 2);
    result += rows * step;
  } while (multiIters && (step <= cols - (col += step)));
}

References col(), cols, MAX_BFLOAT16_ACC_VSX, and rows.

colVSXLoopBodyExtra()

template<bool rhsExtraCols, bool lhsExtraRows>

void Eigen::internal::colVSXLoopBodyExtra	(	Index	col,
		Index	depth,
		Index	cols,
		Index	rows,
		const Packet4f	pAlpha,
		const float *	indexA,
		const float *	blockB,
		Index	strideB,
		Index	offsetB,
		float *	result
	)

                                                                                           {
  switch ((cols - col) >> 2) {
    case 3:
      colVSXLoopBodyExtraN<3, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, strideB,
                                                          offsetB, result);
      break;
    case 2:
      colVSXLoopBodyExtraN<2, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, strideB,
                                                          offsetB, result);
      break;
    case 1:
      colVSXLoopBodyExtraN<1, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, strideB,
                                                          offsetB, result);
      break;
    default:
      if (rhsExtraCols) {
        colVSXLoopBody<1, true, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, strideB, offsetB, result);
      }
      break;
  }
}

References col(), cols, and rows.

colVSXLoopBodyExtraN()

template<const Index num_acc, bool rhsExtraCols, bool lhsExtraRows>

EIGEN_ALWAYS_INLINE void Eigen::internal::colVSXLoopBodyExtraN	(	Index	col,
		Index	depth,
		Index	cols,
		Index	rows,
		const Packet4f	pAlpha,
		const float *	indexA,
		const float *	blockB,
		Index	strideB,
		Index	offsetB,
		float *	result
	)

                                                             {
  if (MAX_BFLOAT16_ACC_VSX > num_acc) {
    colVSXLoopBody<num_acc + (rhsExtraCols ? 1 : 0), rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA,
                                                                                 blockB, strideB, offsetB, result);
  }
}

References col(), cols, MAX_BFLOAT16_ACC_VSX, and rows.

colVSXLoopBodyIter()

template<const Index num_acc, bool rhsExtraCols, bool lhsExtraRows>

EIGEN_ALWAYS_INLINE void Eigen::internal::colVSXLoopBodyIter	(	Index	depth,
		Index	rows,
		const Packet4f	pAlpha,
		const float *	indexA,
		const float *	indexB,
		Index	strideB,
		Index	offsetB,
		float *	result,
		const Index	extra_cols,
		const Index	extra_rows
	)

                                                                                            {
  constexpr Index num_rhs = num_acc;
 
  Packet4f acc[num_acc][4];
 
  zeroAccumulators<num_acc>(acc);
 
  Index k;
  for (k = 0; k + 2 <= depth; k += 2) {
    KLoop<num_acc, false, rhsExtraCols, num_rhs>(indexA, indexB, acc, strideB, k, offsetB, extra_cols);
  }
  if (depth & 1) {
    KLoop<num_acc, true, rhsExtraCols, num_rhs>(indexA, indexB, acc, strideB, k, offsetB, extra_cols);
  }
 
  outputResultsVSX<num_acc, rhsExtraCols, lhsExtraRows, num_rhs>(acc, rows, pAlpha, result, extra_cols, extra_rows);
}

References k, and rows.

colVSXLoops()

template<Index size, bool lhsExtraRows = false>

EIGEN_ALWAYS_INLINE void Eigen::internal::colVSXLoops	(	Index	depth,
		Index	cols,
		Index	rows,
		const Packet4f	pAlpha,
		const bfloat16 *	indexA,
		const float *	indexA2,
		const float *	blockB2,
		Index	strideA,
		Index	strideB,
		Index	offsetB,
		float *	result2
	)

                                                                    {
  Index delta_rows = 2 * (lhsExtraRows ? (rows & 3) : size);
  for (Index row = 0; row < size; row += 4) {
    convertArrayPointerBF16toF32Dup<lhsExtraRows>(const_cast<float*>(indexA2), strideA, delta_rows, indexA, row,
                                                  rows & 3);
 
    const float* blockB = blockB2;
    float* result = result2 + row;
 
    Index col = 0;
    if (cols >= (MAX_BFLOAT16_ACC_VSX * 4)) {
      colVSXLoopBody<MAX_BFLOAT16_ACC_VSX, false, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA2, blockB,
                                                                strideB, 0, result);
      blockB += (strideB >> 1) * col;
      result += rows * col;
    }
    if (cols & 3) {
      colVSXLoopBodyExtra<true, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA2, blockB, strideB, offsetB,
                                              result);
    } else {
      colVSXLoopBodyExtra<false, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA2, blockB, strideB, 0, result);
    }
  }
}

References col(), cols, MAX_BFLOAT16_ACC_VSX, row(), rows, and size.

combine2Packet16h()

EIGEN_STRONG_INLINE Packet32h Eigen::internal::combine2Packet16h	(	const Packet16h &	a,
		const Packet16h &	b
	)

                                                                                        {
  __m512d result = _mm512_undefined_pd();
  result = _mm512_insertf64x4(result, _mm256_castsi256_pd(a), 0);
  result = _mm512_insertf64x4(result, _mm256_castsi256_pd(b), 1);
  return _mm512_castpd_ph(result);
}

References a, and b.

Referenced by pcos< Packet32h >(), pexp< Packet32h >(), pexpm1< Packet32h >(), pfrexp< Packet32h >(), pldexp< Packet32h >(), plog1p< Packet32h >(), plog2< Packet32h >(), plog< Packet32h >(), psin< Packet32h >(), and ptanh< Packet32h >().

combine_scalar_factors() [1/2]

template<typename ResScalar , typename Lhs , typename Rhs >

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE ResScalar Eigen::internal::combine_scalar_factors	(	const Lhs &	lhs,
		const Rhs &	rhs
	)

                                                                                                       {
  return combine_scalar_factors_impl<ResScalar, Lhs, Rhs>::run(lhs, rhs);
}

References Eigen::internal::combine_scalar_factors_impl< ResScalar, Lhs, Rhs >::run().

combine_scalar_factors() [2/2]

template<typename ResScalar , typename Lhs , typename Rhs >

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE ResScalar Eigen::internal::combine_scalar_factors	(	const ResScalar &	alpha,
		const Lhs &	lhs,
		const Rhs &	rhs
	)

                                                                                       {
  return combine_scalar_factors_impl<ResScalar, Lhs, Rhs>::run(alpha, lhs, rhs);
}

References alpha, and Eigen::internal::combine_scalar_factors_impl< ResScalar, Lhs, Rhs >::run().

Referenced by Eigen::internal::gemv_dense_selector< OnTheRight, ColMajor, true >::run(), Eigen::internal::gemv_dense_selector< OnTheRight, RowMajor, true >::run(), and Eigen::internal::generic_product_impl< Lhs, Rhs, DenseShape, DenseShape, GemmProduct >::scaleAndAddTo().

complex_log()

template<typename T >

EIGEN_DEVICE_FUNC std::complex< T > Eigen::internal::complex_log

(

const std::complex< T > &

z

)

                                                                    {
  // Computes complex log.
  T a = numext::abs(z);
  EIGEN_USING_STD(atan2);
  T b = atan2(z.imag(), z.real());
  return std::complex<T>(numext::log(a), b);
}

References a, Eigen::numext::abs(), Eigen::atan2(), b, EIGEN_USING_STD, and Eigen::numext::log().

Referenced by Eigen::internal::log_impl< std::complex< Scalar > >::run().

complex_rsqrt()

template<typename T >

EIGEN_DEVICE_FUNC std::complex< T > Eigen::internal::complex_rsqrt

(

const std::complex< T > &

a_x

)

                                                                      {
  // Computes the principal reciprocal sqrt of the input.
  //
  // For a complex reciprocal square root of the number z = x + i*y. We want to
  // find real numbers u and v such that
  //    (u + i*v)^2 = 1 / (x + i*y)  <=>
  //    u^2 - v^2 + i*2*u*v = x/|z|^2 - i*v/|z|^2.
  // By equating the real and imaginary parts we get:
  //    u^2 - v^2 = x/|z|^2
  //    2*u*v = y/|z|^2.
  //
  // For x >= 0, this has the numerically stable solution
  //    u = sqrt(0.5 * (x + |z|)) / |z|
  //    v = -y / (2 * u * |z|)
  // and for x < 0,
  //    v = -sign(y) * sqrt(0.5 * (-x + |z|)) / |z|
  //    u = -y / (2 * v * |z|)
  //
  // Letting w = sqrt(0.5 * (|x| + |z|)),
  //   if x == 0: u = w / |z|, v = -sign(y) * w / |z|
  //   if x > 0:  u = w / |z|, v = -y / (2 * w * |z|)
  //   if x < 0:  u = |y| / (2 * w * |z|), v = -sign(y) * w / |z|
 
  const T x = numext::real(z);
  const T y = numext::imag(z);
  const T zero = T(0);
 
  const T abs_z = numext::hypot(x, y);
  const T w = numext::sqrt(T(0.5) * (numext::abs(x) + abs_z));
  const T woz = w / abs_z;
  // Corner cases consistent with 1/sqrt(z) on gcc/clang.
  return numext::is_exactly_zero(abs_z) ? std::complex<T>(NumTraits<T>::infinity(), NumTraits<T>::quiet_NaN())
         : ((numext::isinf)(x) || (numext::isinf)(y)) ? std::complex<T>(zero, zero)
         : numext::is_exactly_zero(x)                 ? std::complex<T>(woz, y < zero ? woz : -woz)
         : x > zero                                   ? std::complex<T>(woz, -y / (2 * w * abs_z))
                    : std::complex<T>(numext::abs(y) / (2 * w * abs_z), y < zero ? woz : -woz);
}

References Eigen::numext::abs(), imag(), Eigen::numext::is_exactly_zero(), Eigen::numext::isinf(), Eigen::numext::sqrt(), w, plotDoE::x, y, and zero().

complex_sqrt()

template<typename T >

EIGEN_DEVICE_FUNC std::complex< T > Eigen::internal::complex_sqrt

(

const std::complex< T > &

a_x

)

                                                                     {
  // Computes the principal sqrt of the input.
  //
  // For a complex square root of the number x + i*y. We want to find real
  // numbers u and v such that
  //    (u + i*v)^2 = x + i*y  <=>
  //    u^2 - v^2 + i*2*u*v = x + i*v.
  // By equating the real and imaginary parts we get:
  //    u^2 - v^2 = x
  //    2*u*v = y.
  //
  // For x >= 0, this has the numerically stable solution
  //    u = sqrt(0.5 * (x + sqrt(x^2 + y^2)))
  //    v = y / (2 * u)
  // and for x < 0,
  //    v = sign(y) * sqrt(0.5 * (-x + sqrt(x^2 + y^2)))
  //    u = y / (2 * v)
  //
  // Letting w = sqrt(0.5 * (|x| + |z|)),
  //   if x == 0: u = w, v = sign(y) * w
  //   if x > 0:  u = w, v = y / (2 * w)
  //   if x < 0:  u = |y| / (2 * w), v = sign(y) * w
 
  const T x = numext::real(z);
  const T y = numext::imag(z);
  const T zero = T(0);
  const T w = numext::sqrt(T(0.5) * (numext::abs(x) + numext::hypot(x, y)));
 
  return (numext::isinf)(y)           ? std::complex<T>(NumTraits<T>::infinity(), y)
         : numext::is_exactly_zero(x) ? std::complex<T>(w, y < zero ? -w : w)
         : x > zero                   ? std::complex<T>(w, y / (2 * w))
                                      : std::complex<T>(numext::abs(y) / (2 * w), y < zero ? -w : w);
}

References Eigen::numext::abs(), imag(), Eigen::numext::is_exactly_zero(), Eigen::numext::isinf(), Eigen::numext::sqrt(), w, plotDoE::x, y, and zero().

compute_default_alignment_helper()

constexpr int Eigen::internal::compute_default_alignment_helper ( int  ArrayBytes, int  AlignmentBytes  )

inlineconstexpr

                                                                                          {
  EIGEN_UNUSED_VARIABLE(ArrayBytes);
  EIGEN_UNUSED_VARIABLE(AlignmentBytes);
  return 0;
}

References EIGEN_UNUSED_VARIABLE.

compute_inverse_size2_helper()

template<typename MatrixType , typename ResultType >

EIGEN_DEVICE_FUNC void Eigen::internal::compute_inverse_size2_helper ( const MatrixType &  matrix, const typename ResultType::Scalar &  invdet, ResultType &  result  )

inline

                                                                               {
  typename ResultType::Scalar temp = matrix.coeff(0, 0);
  result.coeffRef(0, 0) = matrix.coeff(1, 1) * invdet;
  result.coeffRef(1, 0) = -matrix.coeff(1, 0) * invdet;
  result.coeffRef(0, 1) = -matrix.coeff(0, 1) * invdet;
  result.coeffRef(1, 1) = temp * invdet;
}

References matrix().

Referenced by Eigen::internal::compute_inverse_and_det_with_check< MatrixType, ResultType, 2 >::run(), and Eigen::internal::compute_inverse< MatrixType, ResultType, 2 >::run().

compute_inverse_size3_helper()

template<typename MatrixType , typename ResultType >

EIGEN_DEVICE_FUNC void Eigen::internal::compute_inverse_size3_helper ( const MatrixType &  matrix, const typename ResultType::Scalar &  invdet, const Matrix< typename ResultType::Scalar, 3, 1 > &  cofactors_col0, ResultType &  result  )

inline

                                                                                       {
  // Compute cofactors in a way that avoids aliasing issues.
  typedef typename ResultType::Scalar Scalar;
  const Scalar c01 = cofactor_3x3<MatrixType, 0, 1>(matrix) * invdet;
  const Scalar c11 = cofactor_3x3<MatrixType, 1, 1>(matrix) * invdet;
  const Scalar c02 = cofactor_3x3<MatrixType, 0, 2>(matrix) * invdet;
  result.coeffRef(1, 2) = cofactor_3x3<MatrixType, 2, 1>(matrix) * invdet;
  result.coeffRef(2, 1) = cofactor_3x3<MatrixType, 1, 2>(matrix) * invdet;
  result.coeffRef(2, 2) = cofactor_3x3<MatrixType, 2, 2>(matrix) * invdet;
  result.coeffRef(1, 0) = c01;
  result.coeffRef(1, 1) = c11;
  result.coeffRef(2, 0) = c02;
  result.row(0) = cofactors_col0 * invdet;
}

References matrix().

Referenced by Eigen::internal::compute_inverse_and_det_with_check< MatrixType, ResultType, 3 >::run(), and Eigen::internal::compute_inverse< MatrixType, ResultType, 3 >::run().

compute_matrix_flags()

constexpr unsigned Eigen::internal::compute_matrix_flags ( int  Options )

inlineconstexpr

                                                            {
  unsigned row_major_bit = Options & RowMajor ? RowMajorBit : 0;
  // FIXME currently we still have to handle DirectAccessBit at the expression level to handle DenseCoeffsBase<>
  // and then propagate this information to the evaluator's flags.
  // However, I (Gael) think that DirectAccessBit should only matter at the evaluation stage.
  return DirectAccessBit | LvalueBit | NestByRefBit | row_major_bit;
}

References Eigen::DirectAccessBit, Eigen::LvalueBit, Eigen::NestByRefBit, Eigen::RowMajor, and Eigen::RowMajorBit.

computeFromTridiagonal_impl()

template<typename MatrixType , typename DiagType , typename SubDiagType >

EIGEN_DEVICE_FUNC ComputationInfo Eigen::internal::computeFromTridiagonal_impl	(	DiagType &	diag,
		SubDiagType &	subdiag,
		const Index	maxIterations,
		bool	computeEigenvectors,
		MatrixType &	eivec
	)

Compute the eigendecomposition from a tridiagonal matrix.

Parameters

[in,out]	diag	: On input, the diagonal of the matrix, on output the eigenvalues
[in,out]	subdiag	: The subdiagonal part of the matrix (entries are modified during the decomposition)
[in]	maxIterations	: the maximum number of iterations
[in]	computeEigenvectors	: whether the eigenvectors have to be computed or not
[out]	eivec	: The matrix to store the eigenvectors if computeEigenvectors==true. Must be allocated on input.

Returns

Success or NoConvergence

                                                                                 {
  ComputationInfo info;
  typedef typename MatrixType::Scalar Scalar;
 
  Index n = diag.size();
  Index end = n - 1;
  Index start = 0;
  Index iter = 0;  // total number of iterations
 
  typedef typename DiagType::RealScalar RealScalar;
  const RealScalar considerAsZero = (std::numeric_limits<RealScalar>::min)();
  const RealScalar precision_inv = RealScalar(1) / NumTraits<RealScalar>::epsilon();
  while (end > 0) {
    for (Index i = start; i < end; ++i) {
      if (numext::abs(subdiag[i]) < considerAsZero) {
        subdiag[i] = RealScalar(0);
      } else {
        // abs(subdiag[i]) <= epsilon * sqrt(abs(diag[i]) + abs(diag[i+1]))
        // Scaled to prevent underflows.
        const RealScalar scaled_subdiag = precision_inv * subdiag[i];
        if (scaled_subdiag * scaled_subdiag <= (numext::abs(diag[i]) + numext::abs(diag[i + 1]))) {
          subdiag[i] = RealScalar(0);
        }
      }
    }
 
    // find the largest unreduced block at the end of the matrix.
    while (end > 0 && numext::is_exactly_zero(subdiag[end - 1])) {
      end--;
    }
    if (end <= 0) break;
 
    // if we spent too many iterations, we give up
    iter++;
    if (iter > maxIterations * n) break;
 
    start = end - 1;
    while (start > 0 && !numext::is_exactly_zero(subdiag[start - 1])) start--;
 
    internal::tridiagonal_qr_step<MatrixType::Flags & RowMajorBit ? RowMajor : ColMajor>(
        diag.data(), subdiag.data(), start, end, computeEigenvectors ? eivec.data() : (Scalar*)0, n);
  }
  if (iter <= maxIterations * n)
    info = Success;
  else
    info = NoConvergence;
 
  // Sort eigenvalues and corresponding vectors.
  // TODO make the sort optional ?
  // TODO use a better sort algorithm !!
  if (info == Success) {
    for (Index i = 0; i < n - 1; ++i) {
      Index k;
      diag.segment(i, n - i).minCoeff(&k);
      if (k > 0) {
        numext::swap(diag[i], diag[k + i]);
        if (computeEigenvectors) eivec.col(i).swap(eivec.col(k + i));
      }
    }
  }
  return info;
}

References Eigen::numext::abs(), diag, Eigen::placeholders::end, oomph::SarahBL::epsilon, i, Eigen::SelfAdjointEigenSolver< MatrixType_ >::info(), Eigen::numext::is_exactly_zero(), k, min, n, Eigen::NoConvergence, oomph::CumulativeTimings::start(), Eigen::Success, and Eigen::numext::swap().

Referenced by Eigen::SelfAdjointEigenSolver< MatrixType_ >::compute(), and Eigen::SelfAdjointEigenSolver< MatrixType_ >::computeFromTridiagonal().

computeProductBlockingSizes() [1/2]

template<typename LhsScalar , typename RhsScalar , int KcFactor, typename Index >

void Eigen::internal::computeProductBlockingSizes	(	Index &	k,
		Index &	m,
		Index &	n,
		Index	num_threads = 1
	)

Computes the blocking parameters for a m x k times k x n matrix product.

Parameters

[in,out]	k	Input: the third dimension of the product. Output: the blocking size along the same dimension.
[in,out]	m	Input: the number of rows of the left hand side. Output: the blocking size along the same dimension.
[in,out]	n	Input: the number of columns of the right hand side. Output: the blocking size along the same dimension.

Given a m x k times k x n matrix product of scalar types LhsScalar and RhsScalar, this function computes the blocking size parameters along the respective dimensions for matrix products and related algorithms.

The blocking size parameters may be evaluated:

• either by a heuristic based on cache sizes;
• or using fixed prescribed values (for testing purposes).

See also

setCpuCacheSizes

                                                                                      {
  if (!useSpecificBlockingSizes(k, m, n)) {
    evaluateProductBlockingSizesHeuristic<LhsScalar, RhsScalar, KcFactor, Index>(k, m, n, num_threads);
  }
}

References k, m, n, and useSpecificBlockingSizes().

computeProductBlockingSizes() [2/2]

template<typename LhsScalar , typename RhsScalar , typename Index >

void Eigen::internal::computeProductBlockingSizes ( Index &  k, Index &  m, Index &  n, Index  num_threads = 1  )

inline

                                                                                             {
  computeProductBlockingSizes<LhsScalar, RhsScalar, 1, Index>(k, m, n, num_threads);
}

References k, m, and n.

conditional_aligned_delete()

template<typename T , bool Align>

EIGEN_DEVICE_FUNC void Eigen::internal::conditional_aligned_delete ( T *  ptr, std::size_t  size  )

inline

Deletes objects constructed with conditional_aligned_new The size parameters tells on how many objects to call the destructor of T.

                                                                                 {
  destruct_elements_of_array<T>(ptr, size);
  conditional_aligned_free<Align>(ptr);
}

References size.

conditional_aligned_delete_auto()

template<typename T , bool Align>

EIGEN_DEVICE_FUNC void Eigen::internal::conditional_aligned_delete_auto ( T *  ptr, std::size_t  size  )

inline

                                                                                      {
  if (NumTraits<T>::RequireInitialization) destruct_elements_of_array<T>(ptr, size);
  conditional_aligned_free<Align>(ptr);
}

References size.

conditional_aligned_free()

template<bool Align>

EIGEN_DEVICE_FUNC void Eigen::internal::conditional_aligned_free ( void *  ptr )

inline

Frees memory allocated with conditional_aligned_malloc

                                                                  {
  aligned_free(ptr);
}

References aligned_free().

conditional_aligned_free< false >()

template<>

EIGEN_DEVICE_FUNC void Eigen::internal::conditional_aligned_free< false > ( void *  ptr )

inline

                                                                         {
  if (ptr != nullptr) {
    check_that_malloc_is_allowed();
    EIGEN_USING_STD(free)
    free(ptr);
  }
}

References check_that_malloc_is_allowed(), and EIGEN_USING_STD.

Referenced by conditional_aligned_realloc< false >().

conditional_aligned_malloc()

template<bool Align>

EIGEN_DEVICE_FUNC void* Eigen::internal::conditional_aligned_malloc ( std::size_t  size )

inline

Allocates size bytes. If Align is true, then the returned ptr is 16-byte-aligned. On allocation error, the returned pointer is null, and a std::bad_alloc is thrown.

                                                                          {
  return aligned_malloc(size);
}

References aligned_malloc(), and size.

conditional_aligned_malloc< false >()

template<>

EIGEN_DEVICE_FUNC void* Eigen::internal::conditional_aligned_malloc< false > ( std::size_t  size )

inline

                                                                                 {
  if (size == 0) return nullptr;
 
  check_that_malloc_is_allowed();
  EIGEN_USING_STD(malloc)
  void* result = malloc(size);
 
  if (!result && size) throw_std_bad_alloc();
  return result;
}

References check_that_malloc_is_allowed(), EIGEN_USING_STD, size, and throw_std_bad_alloc().

Referenced by conditional_aligned_realloc< false >().

conditional_aligned_new()

template<typename T , bool Align>

EIGEN_DEVICE_FUNC T* Eigen::internal::conditional_aligned_new ( std::size_t  size )

inline

                                                                    {
  check_size_for_overflow<T>(size);
  T* result = static_cast<T*>(conditional_aligned_malloc<Align>(sizeof(T) * size));
  EIGEN_TRY { return default_construct_elements_of_array(result, size); }
  EIGEN_CATCH(...) {
    conditional_aligned_free<Align>(result);
    EIGEN_THROW;
  }
  return result;
}

References default_construct_elements_of_array(), EIGEN_CATCH, EIGEN_THROW, EIGEN_TRY, and size.

conditional_aligned_new_auto()

template<typename T , bool Align>

EIGEN_DEVICE_FUNC T* Eigen::internal::conditional_aligned_new_auto ( std::size_t  size )

inline

                                                                         {
  if (size == 0) return nullptr;  // short-cut. Also fixes Bug 884
  check_size_for_overflow<T>(size);
  T* result = static_cast<T*>(conditional_aligned_malloc<Align>(sizeof(T) * size));
  if (NumTraits<T>::RequireInitialization) {
    EIGEN_TRY { default_construct_elements_of_array(result, size); }
    EIGEN_CATCH(...) {
      conditional_aligned_free<Align>(result);
      EIGEN_THROW;
    }
  }
  return result;
}

References default_construct_elements_of_array(), EIGEN_CATCH, EIGEN_THROW, EIGEN_TRY, and size.

conditional_aligned_realloc()

template<bool Align>

EIGEN_DEVICE_FUNC void* Eigen::internal::conditional_aligned_realloc ( void *  ptr, std::size_t  new_size, std::size_t  old_size  )

inline

                                                                                                              {
  return aligned_realloc(ptr, new_size, old_size);
}

References aligned_realloc().

conditional_aligned_realloc< false >()

template<>

EIGEN_DEVICE_FUNC void* Eigen::internal::conditional_aligned_realloc< false > ( void *  ptr, std::size_t  new_size, std::size_t  old_size  )

inline

                                                                                      {
  if (ptr == nullptr) return conditional_aligned_malloc<false>(new_size);
  if (old_size == new_size) return ptr;
  if (new_size == 0) {
    conditional_aligned_free<false>(ptr);
    return nullptr;
  }
 
  check_that_malloc_is_allowed();
  EIGEN_USING_STD(realloc)
  return realloc(ptr, new_size);
}

References check_that_malloc_is_allowed(), conditional_aligned_free< false >(), conditional_aligned_malloc< false >(), and EIGEN_USING_STD.

conditional_aligned_realloc_new()

template<typename T , bool Align>

EIGEN_DEVICE_FUNC T* Eigen::internal::conditional_aligned_realloc_new ( T *  pts, std::size_t  new_size, std::size_t  old_size  )

inline

                                                                                                            {
  check_size_for_overflow<T>(new_size);
  check_size_for_overflow<T>(old_size);
 
  // If elements need to be explicitly initialized, we cannot simply realloc
  // (or memcpy) the memory block - each element needs to be reconstructed.
  // Otherwise, objects that contain internal pointers like mpfr or
  // AnnoyingScalar can be pointing to the wrong thing.
  T* result = static_cast<T*>(conditional_aligned_malloc<Align>(sizeof(T) * new_size));
  EIGEN_TRY {
    // Move-construct initial elements.
    std::size_t copy_size = (std::min)(old_size, new_size);
    move_construct_elements_of_array(result, pts, copy_size);
 
    // Default-construct remaining elements.
    if (new_size > old_size) {
      default_construct_elements_of_array(result + copy_size, new_size - old_size);
    }
 
    // Delete old elements.
    conditional_aligned_delete<T, Align>(pts, old_size);
  }
  EIGEN_CATCH(...) {
    conditional_aligned_free<Align>(result);
    EIGEN_THROW;
  }
 
  return result;
}

References default_construct_elements_of_array(), EIGEN_CATCH, EIGEN_THROW, EIGEN_TRY, min, and move_construct_elements_of_array().

conditional_aligned_realloc_new_auto()

template<typename T , bool Align>

EIGEN_DEVICE_FUNC T* Eigen::internal::conditional_aligned_realloc_new_auto ( T *  pts, std::size_t  new_size, std::size_t  old_size  )

inline

                                                                                                                 {
  if (NumTraits<T>::RequireInitialization) {
    return conditional_aligned_realloc_new<T, Align>(pts, new_size, old_size);
  }
 
  check_size_for_overflow<T>(new_size);
  check_size_for_overflow<T>(old_size);
  return static_cast<T*>(
      conditional_aligned_realloc<Align>(static_cast<void*>(pts), sizeof(T) * new_size, sizeof(T) * old_size));
}

conjugate_gradient()

template<typename MatrixType , typename Rhs , typename Dest , typename Preconditioner >

EIGEN_DONT_INLINE void Eigen::internal::conjugate_gradient	(	const MatrixType &	mat,
		const Rhs &	rhs,
		Dest &	x,
		const Preconditioner &	precond,
		Index &	iters,
		typename Dest::RealScalar &	tol_error
	)

Low-level conjugate gradient algorithm

Parameters

mat	The matrix A
rhs	The right hand side vector b
x	On input and initial solution, on output the computed solution.
precond	A preconditioner being able to efficiently solve for an approximation of Ax=b (regardless of b)
iters	On input the max number of iteration, on output the number of performed iterations.
tol_error	On input the tolerance error, on output an estimation of the relative error.

                                                                                            {
  typedef typename Dest::RealScalar RealScalar;
  typedef typename Dest::Scalar Scalar;
  typedef Matrix<Scalar, Dynamic, 1> VectorType;
 
  RealScalar tol = tol_error;
  Index maxIters = iters;
 
  Index n = mat.cols();
 
  VectorType residual = rhs - mat * x;  // initial residual
 
  RealScalar rhsNorm2 = rhs.squaredNorm();
  if (rhsNorm2 == 0) {
    x.setZero();
    iters = 0;
    tol_error = 0;
    return;
  }
  const RealScalar considerAsZero = (std::numeric_limits<RealScalar>::min)();
  RealScalar threshold = numext::maxi(RealScalar(tol * tol * rhsNorm2), considerAsZero);
  RealScalar residualNorm2 = residual.squaredNorm();
  if (residualNorm2 < threshold) {
    iters = 0;
    tol_error = numext::sqrt(residualNorm2 / rhsNorm2);
    return;
  }
 
  VectorType p(n);
  p = precond.solve(residual);  // initial search direction
 
  VectorType z(n), tmp(n);
  RealScalar absNew = numext::real(residual.dot(p));  // the square of the absolute value of r scaled by invM
  Index i = 0;
  while (i < maxIters) {
    tmp.noalias() = mat * p;  // the bottleneck of the algorithm
 
    Scalar alpha = absNew / p.dot(tmp);  // the amount we travel on dir
    x += alpha * p;                      // update solution
    residual -= alpha * tmp;             // update residual
 
    residualNorm2 = residual.squaredNorm();
    if (residualNorm2 < threshold) break;
 
    z = precond.solve(residual);  // approximately solve for "A z = residual"
 
    RealScalar absOld = absNew;
    absNew = numext::real(residual.dot(z));  // update the absolute value of r
    RealScalar beta = absNew / absOld;       // calculate the Gram-Schmidt value used to create the new search direction
    p = z + beta * p;                        // update search direction
    i++;
  }
  tol_error = numext::sqrt(residualNorm2 / rhsNorm2);
  iters = i;
}

References alpha, beta, Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::cols(), i, Eigen::numext::maxi(), min, n, p, Eigen::numext::sqrt(), tmp, and plotDoE::x.

Referenced by Eigen::ConjugateGradient< MatrixType_, UpLo_, Preconditioner_ >::_solve_vector_with_guess_impl().

conservative_sparse_sparse_product_impl()

template<typename Lhs , typename Rhs , typename ResultType >

static void Eigen::internal::conservative_sparse_sparse_product_impl ( const Lhs &  lhs, const Rhs &  rhs, ResultType &  res, bool  sortedInsertion = false  )

static

                                                                                  {
  typedef typename remove_all_t<Lhs>::Scalar LhsScalar;
  typedef typename remove_all_t<Rhs>::Scalar RhsScalar;
  typedef typename remove_all_t<ResultType>::Scalar ResScalar;
 
  // make sure to call innerSize/outerSize since we fake the storage order.
  Index rows = lhs.innerSize();
  Index cols = rhs.outerSize();
  eigen_assert(lhs.outerSize() == rhs.innerSize());
 
  ei_declare_aligned_stack_constructed_variable(bool, mask, rows, 0);
  ei_declare_aligned_stack_constructed_variable(ResScalar, values, rows, 0);
  ei_declare_aligned_stack_constructed_variable(Index, indices, rows, 0);
 
  std::memset(mask, 0, sizeof(bool) * rows);
 
  evaluator<Lhs> lhsEval(lhs);
  evaluator<Rhs> rhsEval(rhs);
 
  // estimate the number of non zero entries
  // given a rhs column containing Y non zeros, we assume that the respective Y columns
  // of the lhs differs in average of one non zeros, thus the number of non zeros for
  // the product of a rhs column with the lhs is X+Y where X is the average number of non zero
  // per column of the lhs.
  // Therefore, we have nnz(lhs*rhs) = nnz(lhs) + nnz(rhs)
  Index estimated_nnz_prod = lhsEval.nonZerosEstimate() + rhsEval.nonZerosEstimate();
 
  res.setZero();
  res.reserve(Index(estimated_nnz_prod));
  // we compute each column of the result, one after the other
  for (Index j = 0; j < cols; ++j) {
    res.startVec(j);
    Index nnz = 0;
    for (typename evaluator<Rhs>::InnerIterator rhsIt(rhsEval, j); rhsIt; ++rhsIt) {
      RhsScalar y = rhsIt.value();
      Index k = rhsIt.index();
      for (typename evaluator<Lhs>::InnerIterator lhsIt(lhsEval, k); lhsIt; ++lhsIt) {
        Index i = lhsIt.index();
        LhsScalar x = lhsIt.value();
        if (!mask[i]) {
          mask[i] = true;
          values[i] = x * y;
          indices[nnz] = i;
          ++nnz;
        } else
          values[i] += x * y;
      }
    }
    if (!sortedInsertion) {
      // unordered insertion
      for (Index k = 0; k < nnz; ++k) {
        Index i = indices[k];
        res.insertBackByOuterInnerUnordered(j, i) = values[i];
        mask[i] = false;
      }
    } else {
      // alternative ordered insertion code:
      const Index t200 = rows / 11;  // 11 == (log2(200)*1.39)
      const Index t = (rows * 100) / 139;
 
      // FIXME reserve nnz non zeros
      // FIXME implement faster sorting algorithms for very small nnz
      // if the result is sparse enough => use a quick sort
      // otherwise => loop through the entire vector
      // In order to avoid to perform an expensive log2 when the
      // result is clearly very sparse we use a linear bound up to 200.
      if ((nnz < 200 && nnz < t200) || nnz * numext::log2(int(nnz)) < t) {
        if (nnz > 1) std::sort(indices, indices + nnz);
        for (Index k = 0; k < nnz; ++k) {
          Index i = indices[k];
          res.insertBackByOuterInner(j, i) = values[i];
          mask[i] = false;
        }
      } else {
        // dense path
        for (Index i = 0; i < rows; ++i) {
          if (mask[i]) {
            mask[i] = false;
            res.insertBackByOuterInner(j, i) = values[i];
          }
        }
      }
    }
  }
  res.finalize();
}

References cols, ei_declare_aligned_stack_constructed_variable, eigen_assert, i, j, k, Eigen::numext::log2(), res, rows, plotPSD::t, plotDoE::x, and y.

const_cast_ptr()

template<typename T >

EIGEN_DEVICE_FUNC T* Eigen::internal::const_cast_ptr ( const T *  ptr )

inline

                                                         {
  return const_cast<T*>(ptr);
}

construct_at()

template<class T , class... Args>

EIGEN_DEVICE_FUNC T* Eigen::internal::construct_at	(	T *	p,
		Args &&...	args
	)

This wraps C++20's std::construct_at, using placement new instead if it is not available.

                                                        {
  return ::new (const_cast<void*>(static_cast<const volatile void*>(p))) T(std::forward<Args>(args)...);
}

References compute_granudrum_aor::args, and p.

Referenced by Eigen::internal::qr_preconditioner_impl< MatrixType, Options, FullPivHouseholderQRPreconditioner, PreconditionIfMoreRowsThanCols, true >::allocate(), Eigen::internal::qr_preconditioner_impl< MatrixType, Options, FullPivHouseholderQRPreconditioner, PreconditionIfMoreColsThanRows, true >::allocate(), Eigen::internal::qr_preconditioner_impl< MatrixType, Options, ColPivHouseholderQRPreconditioner, PreconditionIfMoreRowsThanCols, true >::allocate(), Eigen::internal::qr_preconditioner_impl< MatrixType, Options, ColPivHouseholderQRPreconditioner, PreconditionIfMoreColsThanRows, true >::allocate(), Eigen::internal::qr_preconditioner_impl< MatrixType, Options, HouseholderQRPreconditioner, PreconditionIfMoreRowsThanCols, true >::allocate(), Eigen::internal::qr_preconditioner_impl< MatrixType, Options, HouseholderQRPreconditioner, PreconditionIfMoreColsThanRows, true >::allocate(), Eigen::Ref< const SparseVector< MatScalar, MatOptions, MatIndex >, Options, StrideType >::construct(), Eigen::Ref< const SparseMatrix< MatScalar, MatOptions, MatIndex >, Options, StrideType >::construct(), Eigen::RefBase< Derived >::construct(), Eigen::KLU< MatrixType_ >::grab(), Eigen::UmfPackLU< MatrixType_ >::grab(), Eigen::internal::generic_matrix_wrapper< MatrixType, false >::grab(), and Eigen::internal::unary_evaluator< Block< ArgType, BlockRows, BlockCols, InnerPanel >, IteratorBased >::OuterVectorInnerIterator::operator++().

convert_index()

template<typename IndexDest , typename IndexSrc >

EIGEN_DEVICE_FUNC IndexDest Eigen::internal::convert_index ( const IndexSrc &  idx )

inline

                                                                      {
  return convert_index_impl<IndexDest, IndexSrc>::run(idx);
}

References Eigen::internal::convert_index_impl< IndexDest, IndexSrc, IndexDestIsInteger, IndexDestIsSigned, IndexSrcIsInteger, IndexSrcIsSigned >::run().

Referenced by Eigen::internal::AmbiVector< Scalar_, StorageIndex_ >::coeffRef(), and Eigen::IncompleteLUT< Scalar_, StorageIndex_ >::factorize().

convertArrayBF16toF32()

template<typename DataMapper >

EIGEN_ALWAYS_INLINE void Eigen::internal::convertArrayBF16toF32	(	float *	result,
		Index	cols,
		Index	rows,
		const DataMapper &	src
	)

                                                                                                             {
  typedef typename DataMapper::LinearMapper LinearMapper;
  for (Index j = 0; j < cols; j++, result += rows) {
    const LinearMapper src2 = src.getLinearMapper(0, j);
    Index i = 0;
    convertBF16toF32<32, LinearMapper>(i, result, rows, src2);
    convertBF16toF32<16, LinearMapper>(i, result, rows, src2);
    convertBF16toF32<8, LinearMapper>(i, result, rows, src2);
    convertBF16toF32<4, LinearMapper>(i, result, rows, src2);
    convertBF16toF32<1, LinearMapper>(i, result, rows, src2);
  }
}

References cols, i, j, and rows.

convertArrayF32toBF16()

template<typename DataMapper >

EIGEN_ALWAYS_INLINE void Eigen::internal::convertArrayF32toBF16	(	float *	result,
		Index	cols,
		Index	rows,
		const DataMapper &	res
	)

                                                                                                             {
  Index col;
  for (col = 0; col + 4 <= cols; col += 4) {
    convertArrayF32toBF16Col<DataMapper, 4>(result, col, rows, res);
  }
  // extra cols
  switch (cols - col) {
    case 1:
      convertArrayF32toBF16Col<DataMapper, 1>(result, col, rows, res);
      break;
    case 2:
      convertArrayF32toBF16Col<DataMapper, 2>(result, col, rows, res);
      break;
    case 3:
      convertArrayF32toBF16Col<DataMapper, 3>(result, col, rows, res);
      break;
  }
}

References col(), cols, res, and rows.

convertArrayF32toBF16Col()

template<typename DataMapper , const Index size>

EIGEN_ALWAYS_INLINE void Eigen::internal::convertArrayF32toBF16Col	(	float *	result,
		Index	col,
		Index	rows,
		const DataMapper &	res
	)

                                                                                                               {
  const DataMapper res2 = res.getSubMapper(0, col);
  Index row;
  float* result2 = result + col * rows;
  for (row = 0; row + 8 <= rows; row += 8, result2 += 8) {
    // get and save block
    PacketBlock<Packet8bf, size> block;
    BFLOAT16_UNROLL
    for (Index j = 0; j < size; j++) {
      block.packet[j] = convertF32toBF16(result2 + j * rows);
    }
    res2.template storePacketBlock<Packet8bf, size>(row, 0, block);
  }
  // extra rows
  if (row < rows) {
    BFLOAT16_UNROLL
    for (Index j = 0; j < size; j++) {
      Packet8bf fp16 = convertF32toBF16(result2 + j * rows);
      res2.template storePacketPartial<Packet8bf>(row, j, fp16, rows & 7);
    }
  }
}

References BFLOAT16_UNROLL, block(), col(), convertF32toBF16(), j, res, row(), rows, and size.

convertArrayF32toBF16ColVSX()

template<typename DataMapper , const Index size>

EIGEN_ALWAYS_INLINE void Eigen::internal::convertArrayF32toBF16ColVSX	(	float *	result,
		Index	col,
		Index	rows,
		const DataMapper &	res
	)

                                                                                                                  {
  const DataMapper res2 = res.getSubMapper(0, col);
  Index row;
  float* result2 = result + col * rows;
  for (row = 0; row + 8 <= rows; row += 8, result2 += 8) {
    // get and save block
    PacketBlock<Packet8bf, size> block;
    for (Index j = 0; j < size; j++) {
      block.packet[j] = convertF32toBF16VSX(result2 + j * rows);
    }
    res2.template storePacketBlock<Packet8bf, size>(row, 0, block);
  }
  // extra rows
  if (row < rows) {
    for (Index j = 0; j < size; j++) {
      Packet8bf fp16 = convertF32toBF16VSX(result2 + j * rows);
      res2.template storePacketPartial<Packet8bf>(row, j, fp16, rows & 7);
    }
  }
}

References block(), col(), convertF32toBF16VSX(), j, res, row(), rows, and size.

convertArrayF32toBF16VSX()

template<typename DataMapper >

EIGEN_ALWAYS_INLINE void Eigen::internal::convertArrayF32toBF16VSX	(	float *	result,
		Index	cols,
		Index	rows,
		const DataMapper &	res
	)

                                                                                                                {
  Index col;
  for (col = 0; col + 4 <= cols; col += 4) {
    convertArrayF32toBF16ColVSX<DataMapper, 4>(result, col, rows, res);
  }
  // extra cols
  switch (cols - col) {
    case 1:
      convertArrayF32toBF16ColVSX<DataMapper, 1>(result, col, rows, res);
      break;
    case 2:
      convertArrayF32toBF16ColVSX<DataMapper, 2>(result, col, rows, res);
      break;
    case 3:
      convertArrayF32toBF16ColVSX<DataMapper, 3>(result, col, rows, res);
      break;
  }
}

References col(), cols, res, and rows.

convertArrayPointerBF16toF32()

template<bool non_unit_stride>

EIGEN_ALWAYS_INLINE void Eigen::internal::convertArrayPointerBF16toF32	(	float *	result,
		Index	cols,
		Index	rows,
		bfloat16 *	src,
		Index	resInc
	)

                                                                    {
  for (Index col = 0; col < cols; col++, src += (rows * resInc), result += rows) {
    Index i = 0;
    bfloat16* src2 = src;
    convertPointerBF16toF32<32, non_unit_stride>(i, result, rows, src2, resInc);
    convertPointerBF16toF32<16, non_unit_stride>(i, result, rows, src2, resInc);
    convertPointerBF16toF32<8, non_unit_stride>(i, result, rows, src2, resInc);
    convertPointerBF16toF32<4, non_unit_stride>(i, result, rows, src2, resInc);
    convertPointerBF16toF32<1, non_unit_stride>(i, result, rows, src2, resInc);
  }
}

References col(), cols, i, and rows.

Referenced by gemmbfloat16(), gemv_bfloat16_col(), gemv_bfloat16_row(), gemvMMA_bfloat16_col(), and gemvMMA_bfloat16_row().

convertArrayPointerBF16toF32Dup()

template<bool lhsExtraRows>

EIGEN_ALWAYS_INLINE void Eigen::internal::convertArrayPointerBF16toF32Dup	(	float *	result,
		Index	cols,
		Index	rows,
		const bfloat16 *	src,
		Index	delta,
		Index	extra_rows
	)

                                                                                        {
  Index col = 0;
  src += delta * 2;
  for (; col + 4 * 2 <= cols; col += 4 * 2, result += 4 * 4 * 4, src += 4 * rows) {
    convertArrayPointerBF16toF32DupOne<lhsExtraRows, false, 4>(result, rows, src, extra_rows);
  }
  for (; col + 2 <= cols; col += 2, result += 4 * 4, src += rows) {
    convertArrayPointerBF16toF32DupOne<lhsExtraRows, false, 1>(result, rows, src, extra_rows);
  }
  if (cols & 1) {
    convertArrayPointerBF16toF32DupOne<lhsExtraRows, true, 1>(result, rows, src - delta, extra_rows);
  }
}

References col(), cols, MultiOpt::delta, and rows.

convertArrayPointerBF16toF32DupOne()

template<bool lhsExtraRows, bool odd, Index size>

EIGEN_ALWAYS_INLINE void Eigen::internal::convertArrayPointerBF16toF32DupOne	(	float *	result,
		Index	rows,
		const bfloat16 *	src,
		Index	extra_rows
	)

                                                                              {
  Packet4f dup[4 * 4];
  Packet8bf data[4];
 
  for (Index i = 0; i < size; i++) {
    data[i] = ploadu<Packet8bf>(src + rows * i);
  }
 
  for (Index i = 0, j = 0; i < size; i++, j += 4) {
    dup[j + 0] = oneConvertBF16Perm(data[i].m_val, odd ? p16uc_MERGE16_32_5 : p16uc_MERGE16_32_1);
    dup[j + 1] = oneConvertBF16Perm(data[i].m_val, odd ? p16uc_MERGE16_32_6 : p16uc_MERGE16_32_2);
    dup[j + 2] = oneConvertBF16Perm(data[i].m_val, odd ? p16uc_MERGE16_32_7 : p16uc_MERGE16_32_3);
    dup[j + 3] = oneConvertBF16Perm(data[i].m_val, odd ? p16uc_MERGE16_32_8 : p16uc_MERGE16_32_4);
  }
 
  for (Index j = 0; j < 4 * size; j += 4) {
    if (lhsExtraRows) {
      Packet4f z = pset1<Packet4f>(float(0));
      Index i = 0;
      do {
        pstoreu(result + (j + i) * 4, dup[j + i]);
      } while (++i < extra_rows);
      do {
        pstoreu(result + (j + i) * 4, z);
      } while (++i < 4);
    } else {
      for (Index i = 0; i < 4; i++) {
        pstoreu(result + (j + i) * 4, dup[j + i]);
      }
    }
  }
}

References data, i, j, oneConvertBF16Perm(), p16uc_MERGE16_32_1, p16uc_MERGE16_32_2, p16uc_MERGE16_32_3, p16uc_MERGE16_32_4, p16uc_MERGE16_32_5, p16uc_MERGE16_32_6, p16uc_MERGE16_32_7, p16uc_MERGE16_32_8, ploadu< Packet8bf >(), pset1< Packet4f >(), pstoreu(), rows, and size.

convertArrayPointerF32toBF16()

template<bool non_unit_stride = false>

EIGEN_ALWAYS_INLINE void Eigen::internal::convertArrayPointerF32toBF16	(	float *	result,
		Index	rows,
		bfloat16 *	dst,
		Index	resInc = 1
	)

                                                                                                                  {
  Index i = 0;
  convertPointerF32toBF16<32, non_unit_stride>(i, result, rows, dst, resInc);
  convertPointerF32toBF16<16, non_unit_stride>(i, result, rows, dst, resInc);
  convertPointerF32toBF16<8, non_unit_stride>(i, result, rows, dst, resInc);
  convertPointerF32toBF16<1, non_unit_stride>(i, result, rows, dst, resInc);
}

References i, and rows.

Referenced by gemvMMA_bfloat16_col(), and gemvMMA_bfloat16_row().

convertBF16toF32()

template<const Index size, typename DataMapper >

EIGEN_ALWAYS_INLINE void Eigen::internal::convertBF16toF32	(	Index &	i,
		float *	result,
		Index	rows,
		const DataMapper &	src
	)

                                                                                                      {
  constexpr Index extra = ((size < 4) ? 4 : size);
  while (i + size <= rows) {
    PacketBlock<Packet8bf, (size + 7) / 8> r32;
    r32.packet[0] = src.template loadPacket<Packet8bf>(i + 0);
    if (size >= 16) {
      r32.packet[1] = src.template loadPacket<Packet8bf>(i + 8);
    }
    if (size >= 32) {
      r32.packet[2] = src.template loadPacket<Packet8bf>(i + 16);
      r32.packet[3] = src.template loadPacket<Packet8bf>(i + 24);
    }
    storeConvertBlockBF16<size>(result + i, r32, rows & 3);
    i += extra;
    if (size != 32) break;
  }
}

References i, rows, and size.

convertF32toBF16()

EIGEN_ALWAYS_INLINE Packet8bf Eigen::internal::convertF32toBF16

(

const float *

res

)

                                                                 {
  Packet16uc fp16[2];
  __vector_pair fp16_vp = *reinterpret_cast<__vector_pair*>(const_cast<float*>(res));
  __builtin_vsx_disassemble_pair(reinterpret_cast<void*>(fp16), &fp16_vp);
  fp16[0] = __builtin_vsx_xvcvspbf16(fp16[0]);
  fp16[1] = __builtin_vsx_xvcvspbf16(fp16[1]);
  return vec_pack(reinterpret_cast<Packet4ui>(fp16[0]), reinterpret_cast<Packet4ui>(fp16[1]));
}

References res.

Referenced by convertArrayF32toBF16Col(), and convertPointerF32toBF16().

convertF32toBF16VSX()

EIGEN_ALWAYS_INLINE Packet8bf Eigen::internal::convertF32toBF16VSX

(

const float *

res

)

                                                                    {
  return F32ToBf16Both(ploadu<Packet4f>(res + 0), ploadu<Packet4f>(res + 4));
}

References F32ToBf16Both(), ploadu< Packet4f >(), and res.

Referenced by convertArrayF32toBF16ColVSX(), and convertPointerF32toBF16VSX().

convertPointerBF16toF32()

template<const Index size, bool non_unit_stride>

EIGEN_ALWAYS_INLINE void Eigen::internal::convertPointerBF16toF32	(	Index &	i,
		float *	result,
		Index	rows,
		bfloat16 *&	src,
		Index	resInc
	)

                                                                                                                    {
  constexpr Index extra = ((size < 4) ? 4 : size);
  while (i + size <= rows) {
    PacketBlock<Packet8bf, (size + 7) / 8> r32;
    r32.packet[0] = loadBF16fromResult<non_unit_stride, 0>(src, resInc);
    if (size >= 16) {
      r32.packet[1] = loadBF16fromResult<non_unit_stride, 8>(src, resInc);
    }
    if (size >= 32) {
      r32.packet[2] = loadBF16fromResult<non_unit_stride, 16>(src, resInc);
      r32.packet[3] = loadBF16fromResult<non_unit_stride, 24>(src, resInc);
    }
    storeConvertBlockBF16<size>(result + i, r32, rows & 3);
    i += extra;
    src += extra * resInc;
    if (size != 32) break;
  }
}

References i, rows, and size.

convertPointerF32toBF16()

template<const Index size, bool non_unit_stride = false>

EIGEN_ALWAYS_INLINE void Eigen::internal::convertPointerF32toBF16	(	Index &	i,
		float *	result,
		Index	rows,
		bfloat16 *&	dst,
		Index	resInc = 1
	)

                                                                   {
  constexpr Index extra = ((size < 8) ? 8 : size);
  while (i + size <= rows) {
    PacketBlock<Packet8bf, (size + 7) / 8> r32;
    r32.packet[0] = convertF32toBF16(result + i + 0);
    if (size >= 16) {
      r32.packet[1] = convertF32toBF16(result + i + 8);
    }
    if (size >= 32) {
      r32.packet[2] = convertF32toBF16(result + i + 16);
      r32.packet[3] = convertF32toBF16(result + i + 24);
    }
    storeBF16fromResult<size, non_unit_stride, 0>(dst, r32.packet[0], resInc, rows & 7);
    if (size >= 16) {
      storeBF16fromResult<size, non_unit_stride, 8>(dst, r32.packet[1], resInc);
    }
    if (size >= 32) {
      storeBF16fromResult<size, non_unit_stride, 16>(dst, r32.packet[2], resInc);
      storeBF16fromResult<size, non_unit_stride, 24>(dst, r32.packet[3], resInc);
    }
    i += extra;
    dst += extra * resInc;
    if (size != 32) break;
  }
}

References convertF32toBF16(), i, rows, and size.

copy_construct_elements_of_array()

template<typename T >

EIGEN_DEVICE_FUNC T* Eigen::internal::copy_construct_elements_of_array ( T *  ptr, const T *  src, std::size_t  size  )

inline

Copy-constructs the elements of an array. The size parameter tells on how many objects to copy.

                                                                                                   {
  std::size_t i = 0;
  EIGEN_TRY {
    for (i = 0; i < size; ++i) ::new (ptr + i) T(*(src + i));
  }
  EIGEN_CATCH(...) {
    destruct_elements_of_array(ptr, i);
    EIGEN_THROW;
  }
  return ptr;
}

References destruct_elements_of_array(), EIGEN_CATCH, EIGEN_THROW, EIGEN_TRY, i, and size.

copyBToRowMajor()

template<typename Scalar , bool toTemp = true, bool remM = false>

EIGEN_ALWAYS_INLINE void Eigen::internal::copyBToRowMajor	(	Scalar *	B_arr,
		int64_t	LDB,
		int64_t	K,
		Scalar *	B_temp,
		int64_t	LDB_,
		int64_t	remM_ = 0
	)

This routine is used to copy B to/from a temporary array (row-major) for cases where B is column-major.

toTemp: true => copy to temporary array, false => copy from temporary array remM: true = need to handle remainder values for M (M < EIGEN_AVX_MAX_NUM_ROW)

                                                            {
  EIGEN_UNUSED_VARIABLE(remM_);
  using urolls = unrolls::transB<Scalar>;
  using vecHalf = typename std::conditional<std::is_same<Scalar, float>::value, vecHalfFloat, vecFullDouble>::type;
  PacketBlock<vecHalf, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> ymm;
  constexpr int64_t U3 = urolls::PacketSize * 3;
  constexpr int64_t U2 = urolls::PacketSize * 2;
  constexpr int64_t U1 = urolls::PacketSize * 1;
  int64_t K_ = K / U3 * U3;
  int64_t k = 0;
 
  for (; k < K_; k += U3) {
    urolls::template transB_kernel<U3, toTemp, remM>(B_arr + k * LDB, LDB, B_temp, LDB_, ymm, remM_);
    B_temp += U3;
  }
  if (K - k >= U2) {
    urolls::template transB_kernel<U2, toTemp, remM>(B_arr + k * LDB, LDB, B_temp, LDB_, ymm, remM_);
    B_temp += U2;
    k += U2;
  }
  if (K - k >= U1) {
    urolls::template transB_kernel<U1, toTemp, remM>(B_arr + k * LDB, LDB, B_temp, LDB_, ymm, remM_);
    B_temp += U1;
    k += U1;
  }
  EIGEN_IF_CONSTEXPR(U1 > 8) {
    // Note: without "if constexpr" this section of code will also be
    // parsed by the compiler so there is an additional check in {load/store}BBlock
    // to make sure the counter is not non-negative.
    if (K - k >= 8) {
      urolls::template transB_kernel<8, toTemp, remM>(B_arr + k * LDB, LDB, B_temp, LDB_, ymm, remM_);
      B_temp += 8;
      k += 8;
    }
  }
  EIGEN_IF_CONSTEXPR(U1 > 4) {
    // Note: without "if constexpr" this section of code will also be
    // parsed by the compiler so there is an additional check in {load/store}BBlock
    // to make sure the counter is not non-negative.
    if (K - k >= 4) {
      urolls::template transB_kernel<4, toTemp, remM>(B_arr + k * LDB, LDB, B_temp, LDB_, ymm, remM_);
      B_temp += 4;
      k += 4;
    }
  }
  if (K - k >= 2) {
    urolls::template transB_kernel<2, toTemp, remM>(B_arr + k * LDB, LDB, B_temp, LDB_, ymm, remM_);
    B_temp += 2;
    k += 2;
  }
  if (K - k >= 1) {
    urolls::template transB_kernel<1, toTemp, remM>(B_arr + k * LDB, LDB, B_temp, LDB_, ymm, remM_);
    B_temp += 1;
    k += 1;
  }
}

References EIGEN_IF_CONSTEXPR, EIGEN_UNUSED_VARIABLE, k, PlanarWave::K, oomph::OcTreeNames::LDB, compute_granudrum_aor::type, and Eigen::value.

count_leading_zeros() [1/2]

template<typename T >

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE std::enable_if_t<sizeof(T) == 4, int> Eigen::internal::count_leading_zeros

(

const T

val

)

                                                                                                           {
#ifdef EIGEN_GPU_COMPILE_PHASE
  return __clz(val);
#elif defined(SYCL_DEVICE_ONLY)
  return cl::sycl::clz(val);
#elif EIGEN_COMP_MSVC
  unsigned long index;
  _BitScanReverse(&index, val);
  return 31 - index;
#else
  EIGEN_STATIC_ASSERT(sizeof(unsigned long long) == 8, YOU_MADE_A_PROGRAMMING_MISTAKE);
  return __builtin_clz(static_cast<uint32_t>(val));
#endif
}

References clz(), EIGEN_STATIC_ASSERT, and calibrate::val.

Referenced by Eigen::internal::TensorIntDivisor< T, div_gt_one >::TensorIntDivisor().

count_leading_zeros() [2/2]

template<typename T >

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE std::enable_if_t<sizeof(T) == 8, int> Eigen::internal::count_leading_zeros

(

const T

val

)

                                                                                                           {
#ifdef EIGEN_GPU_COMPILE_PHASE
  return __clzll(val);
#elif defined(SYCL_DEVICE_ONLY)
  return static_cast<int>(cl::sycl::clz(val));
#elif EIGEN_COMP_MSVC && EIGEN_ARCH_x86_64
  unsigned long index;
  _BitScanReverse64(&index, val);
  return 63 - index;
#elif EIGEN_COMP_MSVC
  // MSVC's _BitScanReverse64 is not available for 32bits builds.
  unsigned int lo = (unsigned int)(val & 0xffffffff);
  unsigned int hi = (unsigned int)((val >> 32) & 0xffffffff);
  int n;
  if (hi == 0)
    n = 32 + count_leading_zeros<unsigned int>(lo);
  else
    n = count_leading_zeros<unsigned int>(hi);
  return n;
#else
  EIGEN_STATIC_ASSERT(sizeof(unsigned long long) == 8, YOU_MADE_A_PROGRAMMING_MISTAKE);
  return __builtin_clzll(static_cast<uint64_t>(val));
#endif
}

References clz(), EIGEN_STATIC_ASSERT, int(), n, and calibrate::val.

covar()

template<typename Scalar >

void Eigen::internal::covar	(	Matrix< Scalar, Dynamic, Dynamic > &	r,
		const VectorXi &	ipvt,
		Scalar	tol = std::sqrt(NumTraits<Scalar>::epsilon())
	)

                                                             {
  using std::abs;
  /* Local variables */
  Index i, j, k, l, ii, jj;
  bool sing;
  Scalar temp;
 
  /* Function Body */
  const Index n = r.cols();
  const Scalar tolr = tol * abs(r(0, 0));
  Matrix<Scalar, Dynamic, 1> wa(n);
  eigen_assert(ipvt.size() == n);
 
  /* form the inverse of r in the full upper triangle of r. */
  l = -1;
  for (k = 0; k < n; ++k)
    if (abs(r(k, k)) > tolr) {
      r(k, k) = 1. / r(k, k);
      for (j = 0; j <= k - 1; ++j) {
        temp = r(k, k) * r(j, k);
        r(j, k) = 0.;
        r.col(k).head(j + 1) -= r.col(j).head(j + 1) * temp;
      }
      l = k;
    }
 
  /* form the full upper triangle of the inverse of (r transpose)*r */
  /* in the full upper triangle of r. */
  for (k = 0; k <= l; ++k) {
    for (j = 0; j <= k - 1; ++j) r.col(j).head(j + 1) += r.col(k).head(j + 1) * r(j, k);
    r.col(k).head(k + 1) *= r(k, k);
  }
 
  /* form the full lower triangle of the covariance matrix */
  /* in the strict lower triangle of r and in wa. */
  for (j = 0; j < n; ++j) {
    jj = ipvt[j];
    sing = j > l;
    for (i = 0; i <= j; ++i) {
      if (sing) r(i, j) = 0.;
      ii = ipvt[i];
      if (ii > jj) r(ii, jj) = r(i, j);
      if (ii < jj) r(jj, ii) = r(i, j);
    }
    wa[jj] = r(j, j);
  }
 
  /* symmetrize the covariance matrix in r. */
  r.topLeftCorner(n, n).template triangularView<StrictlyUpper>() = r.topLeftCorner(n, n).transpose();
  r.diagonal() = wa;
}

References abs(), eigen_assert, i, j, k, n, and UniformPSDSelfTest::r.

Referenced by testLmder(), and testLmdif().

CreateIndexSequence()

template<int SizeAtCompileTime, typename Indices >

IvcType<Indices, SizeAtCompileTime> Eigen::internal::CreateIndexSequence ( size_t  size, const Indices &  indices  )

inline

                                                                                                    {
  return internal::IndexedViewHelperIndicesWrapper<Indices, SizeAtCompileTime>::CreateIndexSequence(indices, size);
}

References Eigen::internal::IndexedViewHelperIndicesWrapper< Indices, NestedSizeAtCompileTime, EnableIf >::CreateIndexSequence(), and size.

cs_tdfs()

template<typename StorageIndex >

StorageIndex Eigen::internal::cs_tdfs	(	StorageIndex	j,
		StorageIndex	k,
		StorageIndex *	head,
		const StorageIndex *	next,
		StorageIndex *	post,
		StorageIndex *	stack
	)

                                          {
  StorageIndex i, p, top = 0;
  if (!head || !next || !post || !stack) return (-1); /* check inputs */
  stack[0] = j;                                       /* place j on the stack */
  while (top >= 0)                                    /* while (stack is not empty) */
  {
    p = stack[top]; /* p = top of stack */
    i = head[p];    /* i = youngest child of p */
    if (i == -1) {
      top--;         /* p has no unordered children left */
      post[k++] = p; /* node p is the kth postordered node */
    } else {
      head[p] = next[i]; /* remove i from children of p */
      stack[++top] = i;  /* start dfs on child node i */
    }
  }
  return k;
}

References i, j, k, and p.

cs_wclear()

template<typename StorageIndex >

static StorageIndex Eigen::internal::cs_wclear ( StorageIndex  mark, StorageIndex  lemax, StorageIndex *  w, StorageIndex  n  )

static

                                                                                                      {
  StorageIndex k;
  if (mark < 2 || (mark + lemax < 0)) {
    for (k = 0; k < n; k++)
      if (w[k] != 0) w[k] = 1;
    mark = 2;
  }
  return (mark); /* at this point, w[0..n-1] < mark holds */
}

References k, n, and w.

ctz()

template<typename BitsType >

EIGEN_DEVICE_FUNC int Eigen::internal::ctz ( BitsType  bits )

inline

                                                {
  return count_bits_impl<BitsType>::ctz(bits);
}

References Eigen::test::bits(), and Eigen::internal::count_bits_impl< BitsType, EnableIf >::ctz().

Referenced by test_clz_ctz(), and test_clz_ctz_random().

customIndices2Array() [1/3]

template<typename IndexType , typename Index >

EIGEN_CONSTEXPR EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE array<Index, 0> Eigen::internal::customIndices2Array	(	IndexType &	,
		numeric_list< Index >
	)

                                                                                                               {
  return array<Index, 0>();
}

customIndices2Array() [2/3]

template<typename Index , std::size_t NumIndices, typename IndexType >

EIGEN_CONSTEXPR EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE array<Index, NumIndices> Eigen::internal::customIndices2Array

(

IndexType &

idx

)

Make an array (for index/dimensions) out of a custom index

                                                                                                                   {
  return customIndices2Array(idx, typename gen_numeric_list<Index, NumIndices>::type{});
}

References customIndices2Array().

customIndices2Array() [3/3]

template<typename IndexType , typename Index , Index First, Index... Is>

EIGEN_CONSTEXPR EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE array<Index, 1 + sizeof...(Is)> Eigen::internal::customIndices2Array	(	IndexType &	idx,
		numeric_list< Index, First, Is... >
	)

                                                       {
  return {static_cast<Index>(idx[First]), static_cast<Index>(idx[Is])...};
}

Referenced by customIndices2Array().

default_construct_elements_of_array()

template<typename T >

EIGEN_DEVICE_FUNC T* Eigen::internal::default_construct_elements_of_array ( T *  ptr, std::size_t  size  )

inline

Constructs the elements of an array. The size parameter tells on how many objects to call the constructor of T.

                                                                                        {
  std::size_t i = 0;
  EIGEN_TRY {
    for (i = 0; i < size; ++i) ::new (ptr + i) T;
  }
  EIGEN_CATCH(...) {
    destruct_elements_of_array(ptr, i);
    EIGEN_THROW;
  }
  return ptr;
}

References destruct_elements_of_array(), EIGEN_CATCH, EIGEN_THROW, EIGEN_TRY, i, and size.

Referenced by aligned_new(), Eigen::internal::aligned_stack_memory_handler< T >::aligned_stack_memory_handler(), conditional_aligned_new(), conditional_aligned_new_auto(), and conditional_aligned_realloc_new().

destroy_at()

template<class T >

EIGEN_DEVICE_FUNC void Eigen::internal::destroy_at

(

T *

p

)

This wraps C++17's std::destroy_at. If it's not available it calls the destructor. The wrapper is not a full replacement for C++20's std::destroy_at as it cannot be applied to std::array.

                                        {
  p->~T();
}

References p.

Referenced by Eigen::internal::qr_preconditioner_impl< MatrixType, Options, FullPivHouseholderQRPreconditioner, PreconditionIfMoreRowsThanCols, true >::allocate(), Eigen::internal::qr_preconditioner_impl< MatrixType, Options, FullPivHouseholderQRPreconditioner, PreconditionIfMoreColsThanRows, true >::allocate(), Eigen::internal::qr_preconditioner_impl< MatrixType, Options, ColPivHouseholderQRPreconditioner, PreconditionIfMoreRowsThanCols, true >::allocate(), Eigen::internal::qr_preconditioner_impl< MatrixType, Options, ColPivHouseholderQRPreconditioner, PreconditionIfMoreColsThanRows, true >::allocate(), Eigen::internal::qr_preconditioner_impl< MatrixType, Options, HouseholderQRPreconditioner, PreconditionIfMoreRowsThanCols, true >::allocate(), Eigen::internal::qr_preconditioner_impl< MatrixType, Options, HouseholderQRPreconditioner, PreconditionIfMoreColsThanRows, true >::allocate(), Eigen::KLU< MatrixType_ >::grab(), Eigen::UmfPackLU< MatrixType_ >::grab(), Eigen::internal::generic_matrix_wrapper< MatrixType, false >::grab(), Eigen::internal::unary_evaluator< Block< ArgType, BlockRows, BlockCols, InnerPanel >, IteratorBased >::OuterVectorInnerIterator::operator++(), Eigen::Ref< const SparseMatrix< MatScalar, MatOptions, MatIndex >, Options, StrideType >::~Ref(), and Eigen::Ref< const SparseVector< MatScalar, MatOptions, MatIndex >, Options, StrideType >::~Ref().

destruct_elements_of_array()

template<typename T >

EIGEN_DEVICE_FUNC void Eigen::internal::destruct_elements_of_array ( T *  ptr, std::size_t  size  )

inline

Destructs the elements of an array. The size parameters tells on how many objects to call the destructor of T.

                                                                                 {
  // always destruct an array starting from the end.
  if (ptr)
    while (size) ptr[--size].~T();
}

References size.

Referenced by copy_construct_elements_of_array(), default_construct_elements_of_array(), and move_construct_elements_of_array().

disassembleAccumulators()

template<Index num_acc>

EIGEN_ALWAYS_INLINE void Eigen::internal::disassembleAccumulators	(	__vector_quad(&)	quad_acc[num_acc],
		Packet4f(&)	acc[num_acc][4]
	)

                                                                                                                  {
  BFLOAT16_UNROLL
  for (Index k = 0; k < num_acc; k++) __builtin_mma_disassemble_acc((void*)acc[k], &(quad_acc[k]));
}

References BFLOAT16_UNROLL, and k.

dogleg()

template<typename Scalar >

void Eigen::internal::dogleg	(	const Matrix< Scalar, Dynamic, Dynamic > &	qrfac,
		const Matrix< Scalar, Dynamic, 1 > &	diag,
		const Matrix< Scalar, Dynamic, 1 > &	qtb,
		Scalar	delta,
		Matrix< Scalar, Dynamic, 1 > &	x
	)

                                                                                                {
  using std::abs;
  using std::sqrt;
 
  typedef DenseIndex Index;
 
  /* Local variables */
  Index i, j;
  Scalar sum, temp, alpha, bnorm;
  Scalar gnorm, qnorm;
  Scalar sgnorm;
 
  /* Function Body */
  const Scalar epsmch = NumTraits<Scalar>::epsilon();
  const Index n = qrfac.cols();
  eigen_assert(n == qtb.size());
  eigen_assert(n == x.size());
  eigen_assert(n == diag.size());
  Matrix<Scalar, Dynamic, 1> wa1(n), wa2(n);
 
  /* first, calculate the gauss-newton direction. */
  for (j = n - 1; j >= 0; --j) {
    temp = qrfac(j, j);
    if (temp == 0.) {
      temp = epsmch * qrfac.col(j).head(j + 1).maxCoeff();
      if (temp == 0.) temp = epsmch;
    }
    if (j == n - 1)
      x[j] = qtb[j] / temp;
    else
      x[j] = (qtb[j] - qrfac.row(j).tail(n - j - 1).dot(x.tail(n - j - 1))) / temp;
  }
 
  /* test whether the gauss-newton direction is acceptable. */
  qnorm = diag.cwiseProduct(x).stableNorm();
  if (qnorm <= delta) return;
 
  // TODO : this path is not tested by Eigen unit tests
 
  /* the gauss-newton direction is not acceptable. */
  /* next, calculate the scaled gradient direction. */
 
  wa1.fill(0.);
  for (j = 0; j < n; ++j) {
    wa1.tail(n - j) += qrfac.row(j).tail(n - j) * qtb[j];
    wa1[j] /= diag[j];
  }
 
  /* calculate the norm of the scaled gradient and test for */
  /* the special case in which the scaled gradient is zero. */
  gnorm = wa1.stableNorm();
  sgnorm = 0.;
  alpha = delta / qnorm;
  if (gnorm == 0.) goto algo_end;
 
  /* calculate the point along the scaled gradient */
  /* at which the quadratic is minimized. */
  wa1.array() /= (diag * gnorm).array();
  // TODO : once unit tests cover this part,:
  // wa2 = qrfac.template triangularView<Upper>() * wa1;
  for (j = 0; j < n; ++j) {
    sum = 0.;
    for (i = j; i < n; ++i) {
      sum += qrfac(j, i) * wa1[i];
    }
    wa2[j] = sum;
  }
  temp = wa2.stableNorm();
  sgnorm = gnorm / temp / temp;
 
  /* test whether the scaled gradient direction is acceptable. */
  alpha = 0.;
  if (sgnorm >= delta) goto algo_end;
 
  /* the scaled gradient direction is not acceptable. */
  /* finally, calculate the point along the dogleg */
  /* at which the quadratic is minimized. */
  bnorm = qtb.stableNorm();
  temp = bnorm / gnorm * (bnorm / qnorm) * (sgnorm / delta);
  temp = temp - delta / qnorm * numext::abs2(sgnorm / delta) +
         sqrt(numext::abs2(temp - delta / qnorm) +
              (1. - numext::abs2(delta / qnorm)) * (1. - numext::abs2(sgnorm / delta)));
  alpha = delta / qnorm * (1. - numext::abs2(sgnorm / delta)) / temp;
algo_end:
 
  /* form appropriate convex combination of the gauss-newton */
  /* direction and the scaled gradient direction. */
  temp = (1. - alpha) * (std::min)(sgnorm, delta);
  x = temp * wa1 + alpha * x;
}

References abs(), Eigen::numext::abs2(), alpha, Eigen::PlainObjectBase< Derived >::cols(), MultiOpt::delta, diag, eigen_assert, oomph::SarahBL::epsilon, i, j, min, n, sqrt(), and plotDoE::x.

doubleword_div_fp()

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::doubleword_div_fp	(	const Packet &	x_hi,
		const Packet &	x_lo,
		const Packet &	y,
		Packet &	z_hi,
		Packet &	z_lo
	)

                                                                                         {
  const Packet t_hi = pdiv(x_hi, y);
  Packet pi_hi, pi_lo;
  twoprod(t_hi, y, pi_hi, pi_lo);
  const Packet delta_hi = psub(x_hi, pi_hi);
  const Packet delta_t = psub(delta_hi, pi_lo);
  const Packet delta = padd(delta_t, x_lo);
  const Packet t_lo = pdiv(delta, y);
  fast_twosum(t_hi, t_lo, z_hi, z_lo);
}

References MultiOpt::delta, fast_twosum(), padd(), pdiv(), psub(), twoprod(), and y.

Referenced by Eigen::internal::accurate_log2< double >::operator()().

EIGEN_CATCH_ASSIGN_DENSE_OP_SPARSE() [1/6]

Eigen::internal::EIGEN_CATCH_ASSIGN_DENSE_OP_SPARSE	(	add_assign_op	,
		scalar_difference_op	,
		sub_assign_op
	)

EIGEN_CATCH_ASSIGN_DENSE_OP_SPARSE() [2/6]

Eigen::internal::EIGEN_CATCH_ASSIGN_DENSE_OP_SPARSE	(	add_assign_op	,
		scalar_sum_op	,
		add_assign_op
	)

EIGEN_CATCH_ASSIGN_DENSE_OP_SPARSE() [3/6]

Eigen::internal::EIGEN_CATCH_ASSIGN_DENSE_OP_SPARSE	(	assign_op	,
		scalar_difference_op	,
		sub_assign_op
	)

EIGEN_CATCH_ASSIGN_DENSE_OP_SPARSE() [4/6]

Eigen::internal::EIGEN_CATCH_ASSIGN_DENSE_OP_SPARSE	(	assign_op	,
		scalar_sum_op	,
		add_assign_op
	)

EIGEN_CATCH_ASSIGN_DENSE_OP_SPARSE() [5/6]

Eigen::internal::EIGEN_CATCH_ASSIGN_DENSE_OP_SPARSE	(	sub_assign_op	,
		scalar_difference_op	,
		add_assign_op
	)

EIGEN_CATCH_ASSIGN_DENSE_OP_SPARSE() [6/6]

Eigen::internal::EIGEN_CATCH_ASSIGN_DENSE_OP_SPARSE	(	sub_assign_op	,
		scalar_sum_op	,
		sub_assign_op
	)

EIGEN_CATCH_ASSIGN_XPR_OP_PRODUCT() [1/6]

Eigen::internal::EIGEN_CATCH_ASSIGN_XPR_OP_PRODUCT	(	add_assign_op	,
		scalar_difference_op	,
		sub_assign_op
	)

EIGEN_CATCH_ASSIGN_XPR_OP_PRODUCT() [2/6]

Eigen::internal::EIGEN_CATCH_ASSIGN_XPR_OP_PRODUCT	(	add_assign_op	,
		scalar_sum_op	,
		add_assign_op
	)

EIGEN_CATCH_ASSIGN_XPR_OP_PRODUCT() [3/6]

Eigen::internal::EIGEN_CATCH_ASSIGN_XPR_OP_PRODUCT	(	assign_op	,
		scalar_difference_op	,
		sub_assign_op
	)

EIGEN_CATCH_ASSIGN_XPR_OP_PRODUCT() [4/6]

Eigen::internal::EIGEN_CATCH_ASSIGN_XPR_OP_PRODUCT	(	assign_op	,
		scalar_sum_op	,
		add_assign_op
	)

EIGEN_CATCH_ASSIGN_XPR_OP_PRODUCT() [5/6]

Eigen::internal::EIGEN_CATCH_ASSIGN_XPR_OP_PRODUCT	(	sub_assign_op	,
		scalar_difference_op	,
		add_assign_op
	)

EIGEN_CATCH_ASSIGN_XPR_OP_PRODUCT() [6/6]

Eigen::internal::EIGEN_CATCH_ASSIGN_XPR_OP_PRODUCT	(	sub_assign_op	,
		scalar_sum_op	,
		sub_assign_op
	)

EIGEN_DECLARE_CONST_FAST_Packet2d()

static Eigen::internal::EIGEN_DECLARE_CONST_FAST_Packet2d ( ZERO  , 0    )

static

EIGEN_DECLARE_CONST_FAST_Packet2l() [1/2]

static Eigen::internal::EIGEN_DECLARE_CONST_FAST_Packet2l ( ONE  , 1    )

static

EIGEN_DECLARE_CONST_FAST_Packet2l() [2/2]

static Eigen::internal::EIGEN_DECLARE_CONST_FAST_Packet2l ( ZERO  , 0    )

static

EIGEN_DECLARE_CONST_FAST_Packet4f() [1/2]

static Eigen::internal::EIGEN_DECLARE_CONST_FAST_Packet4f ( ZERO  , 0    )

static

EIGEN_DECLARE_CONST_FAST_Packet4f() [2/2]

static Eigen::internal::EIGEN_DECLARE_CONST_FAST_Packet4f ( ZERO  , 0    )

static

EIGEN_DECLARE_CONST_FAST_Packet4i() [1/7]

static Eigen::internal::EIGEN_DECLARE_CONST_FAST_Packet4i ( MINUS1  , -  1  )

static

EIGEN_DECLARE_CONST_FAST_Packet4i() [2/7]

static Eigen::internal::EIGEN_DECLARE_CONST_FAST_Packet4i ( MINUS1  , -  1  )

static

EIGEN_DECLARE_CONST_FAST_Packet4i() [3/7]

static Eigen::internal::EIGEN_DECLARE_CONST_FAST_Packet4i ( MINUS16  , -  16  )

static

EIGEN_DECLARE_CONST_FAST_Packet4i() [4/7]

static Eigen::internal::EIGEN_DECLARE_CONST_FAST_Packet4i ( ONE  , 1    )

static

EIGEN_DECLARE_CONST_FAST_Packet4i() [5/7]

static Eigen::internal::EIGEN_DECLARE_CONST_FAST_Packet4i ( ONE  , 1    )

static

EIGEN_DECLARE_CONST_FAST_Packet4i() [6/7]

static Eigen::internal::EIGEN_DECLARE_CONST_FAST_Packet4i ( ZERO  , 0    )

static

EIGEN_DECLARE_CONST_FAST_Packet4i() [7/7]

static Eigen::internal::EIGEN_DECLARE_CONST_FAST_Packet4i ( ZERO  , 0    )

static

EIGEN_DECLARE_CONST_FAST_Packet4ui() [1/2]

static Eigen::internal::EIGEN_DECLARE_CONST_FAST_Packet4ui ( PREV0DOT5  , 0x3EFFFFFFu    )

static

EIGEN_DECLARE_CONST_FAST_Packet4ui() [2/2]

static Eigen::internal::EIGEN_DECLARE_CONST_FAST_Packet4ui ( SIGN  , 0x80000000u    )

static

Referenced by Bf16ToF32Odd(), F32ToBf16(), pcast< Packet8bf, Packet8us >(), pcast< Packet8us, Packet8bf >(), plogical_shift_left(), and plogical_shift_right().

EIGEN_DECLARE_CONST_FAST_Packet8us()

static Eigen::internal::EIGEN_DECLARE_CONST_FAST_Packet8us ( ONE  , 1    )

static

Referenced by F32ToBf16Two(), pabs(), plogical_shift_left(), plogical_shift_right(), and pnegate< Packet8bf >().

EIGEN_DECLARE_CONST_Packet2d() [1/15]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet2d ( 1  , 1.  0  )

static

Referenced by pexp< Packet2d >().

EIGEN_DECLARE_CONST_Packet2d() [2/15]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet2d ( 2  , 2.  0  )

static

EIGEN_DECLARE_CONST_Packet2d() [3/15]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet2d ( cephes_exp_C1  , 0.  693145751953125  )

static

EIGEN_DECLARE_CONST_Packet2d() [4/15]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet2d ( cephes_exp_C2  , 1.42860682030941723212e-  6  )

static

EIGEN_DECLARE_CONST_Packet2d() [5/15]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet2d ( cephes_exp_p0  , 1.26177193074810590878e-  4  )

static

EIGEN_DECLARE_CONST_Packet2d() [6/15]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet2d ( cephes_exp_p1  , 3.02994407707441961300e-  2  )

static

EIGEN_DECLARE_CONST_Packet2d() [7/15]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet2d ( cephes_exp_p2  , 9.99999999999999999910e-  1  )

static

EIGEN_DECLARE_CONST_Packet2d() [8/15]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet2d ( cephes_exp_q0  , 3.00198505138664455042e-  6  )

static

EIGEN_DECLARE_CONST_Packet2d() [9/15]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet2d ( cephes_exp_q1  , 2.52448340349684104192e-  3  )

static

EIGEN_DECLARE_CONST_Packet2d() [10/15]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet2d ( cephes_exp_q2  , 2.27265548208155028766e-  1  )

static

EIGEN_DECLARE_CONST_Packet2d() [11/15]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet2d ( cephes_exp_q3  , 2.  00000000000000000009e0  )

static

EIGEN_DECLARE_CONST_Packet2d() [12/15]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet2d ( cephes_LOG2EF  , 1.  4426950408889634073599  )

static

EIGEN_DECLARE_CONST_Packet2d() [13/15]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet2d ( exp_hi  , 709.  437  )

static

EIGEN_DECLARE_CONST_Packet2d() [14/15]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet2d ( exp_lo  , -709.  436139303  )

static

EIGEN_DECLARE_CONST_Packet2d() [15/15]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet2d ( half  , 0.  5  )

static

EIGEN_DECLARE_CONST_Packet4f() [1/25]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet4f ( 1  , 1.  0f  )

static

Referenced by pexp< Packet4f >(), plog< Packet4f >(), psincos_inner_msa_float(), and ptanh< Packet4f >().

EIGEN_DECLARE_CONST_Packet4f() [2/25]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet4f ( cephes_exp_C1  , 0.  693359375f  )

static

EIGEN_DECLARE_CONST_Packet4f() [3/25]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet4f ( cephes_exp_C2  , -2.12194440e-  4f  )

static

EIGEN_DECLARE_CONST_Packet4f() [4/25]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet4f ( cephes_exp_p0  , 1.9875691500E-  4f  )

static

EIGEN_DECLARE_CONST_Packet4f() [5/25]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet4f ( cephes_exp_p1  , 1.3981999507E-  3f  )

static

EIGEN_DECLARE_CONST_Packet4f() [6/25]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet4f ( cephes_exp_p2  , 8.3334519073E-  3f  )

static

EIGEN_DECLARE_CONST_Packet4f() [7/25]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet4f ( cephes_exp_p3  , 4.1665795894E-  2f  )

static

EIGEN_DECLARE_CONST_Packet4f() [8/25]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet4f ( cephes_exp_p4  , 1.6666665459E-  1f  )

static

EIGEN_DECLARE_CONST_Packet4f() [9/25]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet4f ( cephes_exp_p5  , 5.0000001201E-  1f  )

static

EIGEN_DECLARE_CONST_Packet4f() [10/25]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet4f ( cephes_LOG2EF  , 1.  44269504088896341f  )

static

EIGEN_DECLARE_CONST_Packet4f() [11/25]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet4f ( cephes_log_p0  , 7.0376836292E-  2f  )

static

EIGEN_DECLARE_CONST_Packet4f() [12/25]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet4f ( cephes_log_p1  , -1.1514610310E-  1f  )

static

EIGEN_DECLARE_CONST_Packet4f() [13/25]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet4f ( cephes_log_p2  , 1.1676998740E-  1f  )

static

EIGEN_DECLARE_CONST_Packet4f() [14/25]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet4f ( cephes_log_p3  , -1.2420140846E-  1f  )

static

EIGEN_DECLARE_CONST_Packet4f() [15/25]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet4f ( cephes_log_p4  , +1.4249322787E-  1f  )

static

EIGEN_DECLARE_CONST_Packet4f() [16/25]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet4f ( cephes_log_p5  , -1.6668057665E-  1f  )

static

EIGEN_DECLARE_CONST_Packet4f() [17/25]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet4f ( cephes_log_p6  , +2.0000714765E-  1f  )

static

EIGEN_DECLARE_CONST_Packet4f() [18/25]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet4f ( cephes_log_p7  , -2.4999993993E-  1f  )

static

EIGEN_DECLARE_CONST_Packet4f() [19/25]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet4f ( cephes_log_p8  , +3.3333331174E-  1f  )

static

EIGEN_DECLARE_CONST_Packet4f() [20/25]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet4f ( cephes_log_q1  , -2.12194440e-  4f  )

static

EIGEN_DECLARE_CONST_Packet4f() [21/25]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet4f ( cephes_log_q2  , 0.  693359375f  )

static

EIGEN_DECLARE_CONST_Packet4f() [22/25]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet4f ( cephes_SQRTHF  , 0.  707106781186547524f  )

static

EIGEN_DECLARE_CONST_Packet4f() [23/25]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet4f ( exp_hi  , 88.  3762626647950f  )

static

EIGEN_DECLARE_CONST_Packet4f() [24/25]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet4f ( exp_lo  , -88.  3762626647949f  )

static

EIGEN_DECLARE_CONST_Packet4f() [25/25]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet4f ( half  , 0.  5f  )

static

EIGEN_DECLARE_CONST_Packet4f_FROM_INT() [1/4]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet4f_FROM_INT ( inv_mant_mask  , ~  0x7f800000  )

static

EIGEN_DECLARE_CONST_Packet4f_FROM_INT() [2/4]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet4f_FROM_INT ( min_norm_pos  , 0x00800000    )

static

EIGEN_DECLARE_CONST_Packet4f_FROM_INT() [3/4]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet4f_FROM_INT ( minus_inf  , 0xff800000    )

static

EIGEN_DECLARE_CONST_Packet4f_FROM_INT() [4/4]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet4f_FROM_INT ( minus_nan  , 0xffffffff    )

static

EIGEN_DECLARE_CONST_Packet4i() [1/2]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet4i ( 0x7f  , 0x7f    )

static

EIGEN_DECLARE_CONST_Packet4i() [2/2]

static Eigen::internal::EIGEN_DECLARE_CONST_Packet4i ( 23  , 23    )

static

eigen_lsx_shuffle_mask()

static EIGEN_STRONG_INLINE int Eigen::internal::eigen_lsx_shuffle_mask ( int  p, int  q, int  r, int  s  )

static

                                                                                  {
  return ((s) << 6 | (r) << 4 | (q) << 2 | (p));
}

References p, Eigen::numext::q, UniformPSDSelfTest::r, and s.

Referenced by vec4f_movehl(), vec4f_movelh(), vec4f_swizzle1(), vec4f_swizzle2(), vec4f_unpackhi(), and vec4f_unpacklo().

EIGEN_MAKE_PARTIAL_REDUX_FUNCTOR() [1/4]

Eigen::internal::EIGEN_MAKE_PARTIAL_REDUX_FUNCTOR	(	maxCoeff	,
		(Size - 1) *NumTraits< Scalar >::AddCost	,
		1	,
		internal::scalar_max_op
	)

EIGEN_MAKE_PARTIAL_REDUX_FUNCTOR() [2/4]

Eigen::internal::EIGEN_MAKE_PARTIAL_REDUX_FUNCTOR	(	minCoeff	,
		(Size - 1) *NumTraits< Scalar >::AddCost	,
		1	,
		internal::scalar_min_op
	)

EIGEN_MAKE_PARTIAL_REDUX_FUNCTOR() [3/4]

Eigen::internal::EIGEN_MAKE_PARTIAL_REDUX_FUNCTOR	(	prod	,
		(Size - 1) *NumTraits< Scalar >::MulCost	,
		1	,
		internal::scalar_product_op
	)

EIGEN_MAKE_PARTIAL_REDUX_FUNCTOR() [4/4]

Eigen::internal::EIGEN_MAKE_PARTIAL_REDUX_FUNCTOR	(	sum	,
		(Size - 1) *NumTraits< Scalar >::AddCost	,
		1	,
		internal::scalar_sum_op
	)

EIGEN_MATHFUNC_RETVAL() [1/2]

template<typename Scalar >

Eigen::internal::EIGEN_MATHFUNC_RETVAL ( random  , Scalar    )

inline

                                                      {
  return EIGEN_MATHFUNC_IMPL(random, Scalar)::run();
}

References EIGEN_MATHFUNC_IMPL, and Eigen::run().

EIGEN_MATHFUNC_RETVAL() [2/2]

template<typename Scalar >

Eigen::internal::EIGEN_MATHFUNC_RETVAL ( random  , Scalar    ) const &

inline

References EIGEN_MATHFUNC_IMPL, Eigen::run(), plotDoE::x, and y.

EIGEN_MEMBER_FUNCTOR() [1/7]

Eigen::internal::EIGEN_MEMBER_FUNCTOR	(	all	,
		(Size - 1) *NumTraits< Scalar >::AddCost
	)

EIGEN_MEMBER_FUNCTOR() [2/7]

Eigen::internal::EIGEN_MEMBER_FUNCTOR	(	any	,
		(Size - 1) *NumTraits< Scalar >::AddCost
	)

EIGEN_MEMBER_FUNCTOR() [3/7]

Eigen::internal::EIGEN_MEMBER_FUNCTOR	(	blueNorm	,
		(Size+5) *NumTraits< Scalar >::MulCost+(Size - 1) *NumTraits< Scalar >::AddCost
	)

EIGEN_MEMBER_FUNCTOR() [4/7]

Eigen::internal::EIGEN_MEMBER_FUNCTOR	(	count	,
		(Size - 1) *NumTraits< Scalar >::AddCost
	)

EIGEN_MEMBER_FUNCTOR() [5/7]

Eigen::internal::EIGEN_MEMBER_FUNCTOR	(	hypotNorm	,
		(Size - 1) *functor_traits< scalar_hypot_op< Scalar > >::Cost
	)

EIGEN_MEMBER_FUNCTOR() [6/7]

Eigen::internal::EIGEN_MEMBER_FUNCTOR	(	norm	,
		(Size+5) *NumTraits< Scalar >::MulCost+(Size - 1) *NumTraits< Scalar >::AddCost
	)

EIGEN_MEMBER_FUNCTOR() [7/7]

Eigen::internal::EIGEN_MEMBER_FUNCTOR	(	stableNorm	,
		(Size+5) *NumTraits< Scalar >::MulCost+(Size - 1) *NumTraits< Scalar >::AddCost
	)

eigen_neon_shuffle_mask()

static EIGEN_STRONG_INLINE int Eigen::internal::eigen_neon_shuffle_mask ( int  p, int  q, int  r, int  s  )

static

                                                                                   {
  return ((s) << 6 | (r) << 4 | (q) << 2 | (p));
}

References p, Eigen::numext::q, UniformPSDSelfTest::r, and s.

eigen_pastix() [1/4]

void Eigen::internal::eigen_pastix ( pastix_data_t **  pastix_data, int  pastix_comm, int  n, int *  ptr, int *  idx, double *  vals, int *  perm, int *  invp, double *  x, int  nbrhs, int *  iparm, double *  dparm  )

inline

                                                                                                {
  if (n == 0) {
    ptr = NULL;
    idx = NULL;
    vals = NULL;
  }
  if (nbrhs == 0) {
    x = NULL;
    nbrhs = 1;
  }
  d_pastix(pastix_data, pastix_comm, n, ptr, idx, vals, perm, invp, x, nbrhs, iparm, dparm);
}

References n, and plotDoE::x.

eigen_pastix() [2/4]

void Eigen::internal::eigen_pastix ( pastix_data_t **  pastix_data, int  pastix_comm, int  n, int *  ptr, int *  idx, float *  vals, int *  perm, int *  invp, float *  x, int  nbrhs, int *  iparm, double *  dparm  )

inline

                                                                                               {
  if (n == 0) {
    ptr = NULL;
    idx = NULL;
    vals = NULL;
  }
  if (nbrhs == 0) {
    x = NULL;
    nbrhs = 1;
  }
  s_pastix(pastix_data, pastix_comm, n, ptr, idx, vals, perm, invp, x, nbrhs, iparm, dparm);
}

References n, and plotDoE::x.

Referenced by Eigen::PastixBase< Derived >::_solve_impl(), Eigen::PastixBase< Derived >::analyzePattern(), Eigen::PastixBase< Derived >::clean(), Eigen::PastixBase< Derived >::factorize(), and Eigen::PastixBase< Derived >::init().

eigen_pastix() [3/4]

void Eigen::internal::eigen_pastix ( pastix_data_t **  pastix_data, int  pastix_comm, int  n, int *  ptr, int *  idx, std::complex< double > *  vals, int *  perm, int *  invp, std::complex< double > *  x, int  nbrhs, int *  iparm, double *  dparm  )

inline

                                                    {
  if (n == 0) {
    ptr = NULL;
    idx = NULL;
    vals = NULL;
  }
  if (nbrhs == 0) {
    x = NULL;
    nbrhs = 1;
  }
  z_pastix(pastix_data, pastix_comm, n, ptr, idx, reinterpret_cast<PASTIX_DCOMPLEX *>(vals), perm, invp,
           reinterpret_cast<PASTIX_DCOMPLEX *>(x), nbrhs, iparm, dparm);
}

References n, PASTIX_DCOMPLEX, and plotDoE::x.

eigen_pastix() [4/4]

void Eigen::internal::eigen_pastix ( pastix_data_t **  pastix_data, int  pastix_comm, int  n, int *  ptr, int *  idx, std::complex< float > *  vals, int *  perm, int *  invp, std::complex< float > *  x, int  nbrhs, int *  iparm, double *  dparm  )

inline

                                        {
  if (n == 0) {
    ptr = NULL;
    idx = NULL;
    vals = NULL;
  }
  if (nbrhs == 0) {
    x = NULL;
    nbrhs = 1;
  }
  c_pastix(pastix_data, pastix_comm, n, ptr, idx, reinterpret_cast<PASTIX_COMPLEX *>(vals), perm, invp,
           reinterpret_cast<PASTIX_COMPLEX *>(x), nbrhs, iparm, dparm);
}

References n, PASTIX_COMPLEX, and plotDoE::x.

enum_eq_not_dynamic()

template<typename A , typename B >

constexpr bool Eigen::internal::enum_eq_not_dynamic ( A  a, B  b  )

inlineconstexpr

                                                    {
  plain_enum_asserts(a, b);
  if ((int)a == Dynamic || (int)b == Dynamic) return false;
  return (int)a == (int)b;
}

References a, b, Eigen::Dynamic, int(), and plain_enum_asserts().

enum_ge_not_dynamic()

template<typename A , typename B >

constexpr bool Eigen::internal::enum_ge_not_dynamic ( A  a, B  b  )

inlineconstexpr

                                                    {
  plain_enum_asserts(a, b);
  if ((int)a == Dynamic || (int)b == Dynamic) return false;
  return (int)a >= (int)b;
}

References a, b, Eigen::Dynamic, int(), and plain_enum_asserts().

enum_gt_not_dynamic()

template<typename A , typename B >

constexpr bool Eigen::internal::enum_gt_not_dynamic ( A  a, B  b  )

inlineconstexpr

                                                    {
  plain_enum_asserts(a, b);
  if ((int)a == Dynamic || (int)b == Dynamic) return false;
  return (int)a > (int)b;
}

References a, b, Eigen::Dynamic, int(), and plain_enum_asserts().

enum_le_not_dynamic()

template<typename A , typename B >

constexpr bool Eigen::internal::enum_le_not_dynamic ( A  a, B  b  )

inlineconstexpr

                                                    {
  plain_enum_asserts(a, b);
  if ((int)a == Dynamic || (int)b == Dynamic) return false;
  return (int)a <= (int)b;
}

References a, b, Eigen::Dynamic, int(), and plain_enum_asserts().

enum_lt_not_dynamic()

template<typename A , typename B >

constexpr bool Eigen::internal::enum_lt_not_dynamic ( A  a, B  b  )

inlineconstexpr

                                                    {
  plain_enum_asserts(a, b);
  if ((int)a == Dynamic || (int)b == Dynamic) return false;
  return (int)a < (int)b;
}

References a, b, Eigen::Dynamic, int(), and plain_enum_asserts().

erf_over_x_double_small()

template<typename T >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T Eigen::internal::erf_over_x_double_small

(

const T &

x2

)

                                                                             {
  // erf(x)/x =  S(x^2) / T(x^2), x^2 <= 1.
  //
  // Coefficients for S and T generated with Rminimax command:
  //  ./ratapprox --function="erf(x)" --dom='[-1,1]' --type=[9,10]
  //  --num="odd" --numF="[D]" --den="even" --denF="[D]" --log --dispCoeff="dec"
  constexpr double alpha[] = {1.9493725660006057018823477644531294572516344487667083740234375e-04,
                              1.8272566210022942682217328425053892715368419885635375976562500e-03,
                              4.5303363351690106863856044583371840417385101318359375000000000e-02,
                              1.4215015503619179981775744181504705920815467834472656250000000e-01,
                              1.1283791670955125585606992899556644260883331298828125000000000e+00};
  constexpr double beta[] = {2.0294484101083099089526257108317963684385176748037338256835938e-05,
                             6.8117805899186819641732970609382391558028757572174072265625000e-04,
                             1.0582026056098614921752165685120417037978768348693847656250000e-02,
                             9.3252603143757495374188692949246615171432495117187500000000000e-02,
                             4.5931062818368939559832142549566924571990966796875000000000000e-01,
                             1.0};
  const T num_small = ppolevl<T, 4>::run(x2, alpha);
  const T denom_small = ppolevl<T, 5>::run(x2, beta);
  return pdiv(num_small, denom_small);
}

References alpha, beta, pdiv(), Eigen::internal::ppolevl< Packet, N >::run(), and Global_parameters::x2().

Referenced by Eigen::internal::generic_fast_erf< Scalar >::run(), and Eigen::internal::generic_fast_erfc< Scalar >::run().

erfc_double_large()

template<typename T >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T Eigen::internal::erfc_double_large	(	const T &	x,
		const T &	x2
	)

                                                                                   {
  constexpr double gamma[] = {1.5252844933226974316088642158462107545346952974796295166015625e-04,
                              1.0909912393738931124520519233556115068495273590087890625000000e-02,
                              1.0628604636755033252537572252549580298364162445068359375000000e-01,
                              3.3492472973137982217295416376146022230386734008789062500000000e-01,
                              4.5065776215933289750026347064704168587923049926757812500000000e-01,
                              2.9433039130294824659017649537418037652969360351562500000000000e-01,
                              9.8792676360600226170838311645638896152377128601074218750000000e-02,
                              1.7095935395503719655962981960328761488199234008789062500000000e-02,
                              1.4249109729504577659398023570247460156679153442382812500000000e-03,
                              4.4567378313647954771875570045835956989321857690811157226562500e-05};
  constexpr double delta[] = {2.041985103115789845773520028160419315099716186523437500000000e-03,
                              5.316030659946043707142493417450168635696172714233398437500000e-02,
                              3.426242193784684864077405563875799998641014099121093750000000e-01,
                              8.565637124308049799026321124983951449394226074218750000000000e-01,
                              1.000000000000000000000000000000000000000000000000000000000000e+00,
                              5.968805280570776972126623149961233139038085937500000000000000e-01,
                              1.890922854723317836356244470152887515723705291748046875000000e-01,
                              3.152505418656005586885981983868987299501895904541015625000000e-02,
                              2.565085751861882583380047861965067568235099315643310546875000e-03,
                              7.899362131678837697403017248376499992446042597293853759765625e-05};
  // Compute exp(-x^2).
  const T x2_lo = twoprod_low(x, x, x2);
  // Here we use that
  //   exp(-x^2) = exp(-(x2+x2_lo)^2) ~= exp(-x2)*exp(-x2_lo) ~= exp(-x2)*(1-x2_lo)
  // since x2_lo < kClamp *eps << 1 in the region we care about. This trick reduces the max error
  // from 258 ulps to below 7 ulps.
  const T exp2_hi = pexp(pnegate(x2));
  const T z = pnmadd(exp2_hi, x2_lo, exp2_hi);
  // Compute r = P / Q.
  const T q2 = preciprocal(x2);
  const T num_large = ppolevl<T, 9>::run(q2, gamma);
  const T denom_large = pmul(x, ppolevl<T, 9>::run(q2, delta));
  const T r = pdiv(num_large, denom_large);
  const T maybe_two = pand(pcmp_lt(x, pset1<T>(0.0)), pset1<T>(2.0));
  return pmadd(z, r, maybe_two);
}

References MultiOpt::delta, mathsFunc::gamma(), pand(), pcmp_lt(), pdiv(), pexp(), pmadd(), pmul(), pnegate(), pnmadd(), preciprocal(), UniformPSDSelfTest::r, Eigen::internal::ppolevl< Packet, N >::run(), twoprod_low(), Eigen::numext::x, and Global_parameters::x2().

Referenced by Eigen::internal::generic_fast_erf< Scalar >::run(), and Eigen::internal::generic_fast_erfc< Scalar >::run().

etree_find()

template<typename Index , typename IndexVector >

Index Eigen::internal::etree_find	(	Index	i,
		IndexVector &	pp
	)

Find the root of the tree/set containing the vertex i : Use Path halving

                                           {
  Index p = pp(i);   // Parent
  Index gp = pp(p);  // Grand parent
  while (gp != p) {
    pp(i) = gp;  // Parent pointer on find path is changed to former grand parent
    i = gp;
    p = pp(i);
    gp = pp(p);
  }
  return p;
}

References i, and p.

Referenced by coletree().

evaluateProductBlockingSizesHeuristic()

template<typename LhsScalar , typename RhsScalar , int KcFactor, typename Index >

void Eigen::internal::evaluateProductBlockingSizesHeuristic	(	Index &	k,
		Index &	m,
		Index &	n,
		Index	num_threads = 1
	)

                                                                                                {
  typedef gebp_traits<LhsScalar, RhsScalar> Traits;
 
  // Explanations:
  // Let's recall that the product algorithms form mc x kc vertical panels A' on the lhs and
  // kc x nc blocks B' on the rhs. B' has to fit into L2/L3 cache. Moreover, A' is processed
  // per mr x kc horizontal small panels where mr is the blocking size along the m dimension
  // at the register level. This small horizontal panel has to stay within L1 cache.
  std::ptrdiff_t l1, l2, l3;
  manage_caching_sizes(GetAction, &l1, &l2, &l3);
#ifdef EIGEN_VECTORIZE_AVX512
  // We need to find a rationale for that, but without this adjustment,
  // performance with AVX512 is pretty bad, like -20% slower.
  // One reason is that with increasing packet-size, the blocking size k
  // has to become pretty small if we want that 1 lhs panel fit within L1.
  // For instance, with the 3pX4 kernel and double, the size of the lhs+rhs panels are:
  //   k*(3*64 + 4*8) Bytes, with l1=32kBytes, and k%8=0, we have k=144.
  // This is quite small for a good reuse of the accumulation registers.
  l1 *= 4;
#endif
 
  if (num_threads > 1) {
    typedef typename Traits::ResScalar ResScalar;
    enum {
      kdiv = KcFactor * (Traits::mr * sizeof(LhsScalar) + Traits::nr * sizeof(RhsScalar)),
      ksub = Traits::mr * (Traits::nr * sizeof(ResScalar)),
      kr = 8,
      mr = Traits::mr,
      nr = Traits::nr
    };
    // Increasing k gives us more time to prefetch the content of the "C"
    // registers. However once the latency is hidden there is no point in
    // increasing the value of k, so we'll cap it at 320 (value determined
    // experimentally).
    // To avoid that k vanishes, we make k_cache at least as big as kr
    const Index k_cache = numext::maxi<Index>(kr, (numext::mini<Index>)((l1 - ksub) / kdiv, 320));
    if (k_cache < k) {
      k = k_cache - (k_cache % kr);
      eigen_internal_assert(k > 0);
    }
 
    const Index n_cache = (l2 - l1) / (nr * sizeof(RhsScalar) * k);
    const Index n_per_thread = numext::div_ceil(n, num_threads);
    if (n_cache <= n_per_thread) {
      // Don't exceed the capacity of the l2 cache.
      eigen_internal_assert(n_cache >= static_cast<Index>(nr));
      n = n_cache - (n_cache % nr);
      eigen_internal_assert(n > 0);
    } else {
      n = (numext::mini<Index>)(n, (n_per_thread + nr - 1) - ((n_per_thread + nr - 1) % nr));
    }
 
    if (l3 > l2) {
      // l3 is shared between all cores, so we'll give each thread its own chunk of l3.
      const Index m_cache = (l3 - l2) / (sizeof(LhsScalar) * k * num_threads);
      const Index m_per_thread = numext::div_ceil(m, num_threads);
      if (m_cache < m_per_thread && m_cache >= static_cast<Index>(mr)) {
        m = m_cache - (m_cache % mr);
        eigen_internal_assert(m > 0);
      } else {
        m = (numext::mini<Index>)(m, (m_per_thread + mr - 1) - ((m_per_thread + mr - 1) % mr));
      }
    }
  } else {
    // In unit tests we do not want to use extra large matrices,
    // so we reduce the cache size to check the blocking strategy is not flawed
#ifdef EIGEN_DEBUG_SMALL_PRODUCT_BLOCKS
    l1 = 9 * 1024;
    l2 = 32 * 1024;
    l3 = 512 * 1024;
#endif
 
    // Early return for small problems because the computation below are time consuming for small problems.
    // Perhaps it would make more sense to consider k*n*m??
    // Note that for very tiny problem, this function should be bypassed anyway
    // because we use the coefficient-based implementation for them.
    if ((numext::maxi)(k, (numext::maxi)(m, n)) < 48) return;
 
    typedef typename Traits::ResScalar ResScalar;
    enum {
      k_peeling = 8,
      k_div = KcFactor * (Traits::mr * sizeof(LhsScalar) + Traits::nr * sizeof(RhsScalar)),
      k_sub = Traits::mr * (Traits::nr * sizeof(ResScalar))
    };
 
    // ---- 1st level of blocking on L1, yields kc ----
 
    // Blocking on the third dimension (i.e., k) is chosen so that an horizontal panel
    // of size mr x kc of the lhs plus a vertical panel of kc x nr of the rhs both fits within L1 cache.
    // We also include a register-level block of the result (mx x nr).
    // (In an ideal world only the lhs panel would stay in L1)
    // Moreover, kc has to be a multiple of 8 to be compatible with loop peeling, leading to a maximum blocking size of:
    const Index max_kc = numext::maxi<Index>(((l1 - k_sub) / k_div) & (~(k_peeling - 1)), 1);
    const Index old_k = k;
    if (k > max_kc) {
      // We are really blocking on the third dimension:
      // -> reduce blocking size to make sure the last block is as large as possible
      //    while keeping the same number of sweeps over the result.
      k = (k % max_kc) == 0 ? max_kc
                            : max_kc - k_peeling * ((max_kc - 1 - (k % max_kc)) / (k_peeling * (k / max_kc + 1)));
 
      eigen_internal_assert(((old_k / k) == (old_k / max_kc)) && "the number of sweeps has to remain the same");
    }
 
// ---- 2nd level of blocking on max(L2,L3), yields nc ----
 
// TODO find a reliable way to get the actual amount of cache per core to use for 2nd level blocking, that is:
//      actual_l2 = max(l2, l3/nb_core_sharing_l3)
// The number below is quite conservative: it is better to underestimate the cache size rather than overestimating it)
// For instance, it corresponds to 6MB of L3 shared among 4 cores.
#ifdef EIGEN_DEBUG_SMALL_PRODUCT_BLOCKS
    const Index actual_l2 = l3;
#else
    const Index actual_l2 = 1572864;  // == 1.5 MB
#endif
 
    // Here, nc is chosen such that a block of kc x nc of the rhs fit within half of L2.
    // The second half is implicitly reserved to access the result and lhs coefficients.
    // When k<max_kc, then nc can arbitrarily growth. In practice, it seems to be fruitful
    // to limit this growth: we bound nc to growth by a factor x1.5.
    // However, if the entire lhs block fit within L1, then we are not going to block on the rows at all,
    // and it becomes fruitful to keep the packed rhs blocks in L1 if there is enough remaining space.
    Index max_nc;
    const Index lhs_bytes = m * k * sizeof(LhsScalar);
    const Index remaining_l1 = l1 - k_sub - lhs_bytes;
    if (remaining_l1 >= Index(Traits::nr * sizeof(RhsScalar)) * k) {
      // L1 blocking
      max_nc = remaining_l1 / (k * sizeof(RhsScalar));
    } else {
      // L2 blocking
      max_nc = (3 * actual_l2) / (2 * 2 * max_kc * sizeof(RhsScalar));
    }
    // WARNING Below, we assume that Traits::nr is a power of two.
    Index nc = numext::mini<Index>(actual_l2 / (2 * k * sizeof(RhsScalar)), max_nc) & (~(Traits::nr - 1));
    if (n > nc) {
      // We are really blocking over the columns:
      // -> reduce blocking size to make sure the last block is as large as possible
      //    while keeping the same number of sweeps over the packed lhs.
      //    Here we allow one more sweep if this gives us a perfect match, thus the commented "-1"
      n = (n % nc) == 0 ? nc : (nc - Traits::nr * ((nc /*-1*/ - (n % nc)) / (Traits::nr * (n / nc + 1))));
    } else if (old_k == k) {
      // So far, no blocking at all, i.e., kc==k, and nc==n.
      // In this case, let's perform a blocking over the rows such that the packed lhs data is kept in cache L1/L2
      // TODO: part of this blocking strategy is now implemented within the kernel itself, so the L1-based heuristic
      // here should be obsolete.
      Index problem_size = k * n * sizeof(LhsScalar);
      Index actual_lm = actual_l2;
      Index max_mc = m;
      if (problem_size <= 1024) {
        // problem is small enough to keep in L1
        // Let's choose m such that lhs's block fit in 1/3 of L1
        actual_lm = l1;
      } else if (l3 != 0 && problem_size <= 32768) {
        // we have both L2 and L3, and problem is small enough to be kept in L2
        // Let's choose m such that lhs's block fit in 1/3 of L2
        actual_lm = l2;
        max_mc = (numext::mini<Index>)(576, max_mc);
      }
      Index mc = (numext::mini<Index>)(actual_lm / (3 * k * sizeof(LhsScalar)), max_mc);
      if (mc > Traits::mr)
        mc -= mc % Traits::mr;
      else if (mc == 0)
        return;
      m = (m % mc) == 0 ? mc : (mc - Traits::mr * ((mc /*-1*/ - (m % mc)) / (Traits::mr * (m / mc + 1))));
    }
  }
}

References Eigen::numext::div_ceil(), eigen_internal_assert, Eigen::GetAction, k, m, manage_caching_sizes(), Eigen::numext::maxi(), and n.

exponent_digits()

template<typename T >

constexpr int Eigen::internal::exponent_digits ( )

constexpr

                                {
  return CHAR_BIT * sizeof(T) - NumTraits<T>::digits() - NumTraits<T>::IsSigned;
}

extract128()

template<int I_>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::extract128

(

Packet8d

x

)

                                                    {
  return _mm_castsi128_pd(_mm512_extracti32x4_epi32(_mm512_castpd_si512(x), I_));
}

References plotDoE::x.

extract256()

template<int I_>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::extract256

(

Packet16f

x

)

                                                     {
  return _mm256_castsi256_ps(_mm512_extracti64x4_epi64(_mm512_castps_si512(x), I_));
}

References plotDoE::x.

extract2Packet16h()

EIGEN_STRONG_INLINE void Eigen::internal::extract2Packet16h	(	const Packet32h &	x,
		Packet16h &	a,
		Packet16h &	b
	)

                                                                                           {
  a = _mm256_castpd_si256(_mm512_extractf64x4_pd(_mm512_castph_pd(x), 0));
  b = _mm256_castpd_si256(_mm512_extractf64x4_pd(_mm512_castph_pd(x), 1));
}

References a, b, and plotDoE::x.

Referenced by pcos< Packet32h >(), pexp< Packet32h >(), pexpm1< Packet32h >(), pfrexp< Packet32h >(), pldexp< Packet32h >(), plog1p< Packet32h >(), plog2< Packet32h >(), plog< Packet32h >(), psin< Packet32h >(), and ptanh< Packet32h >().

extract_data()

template<typename T >

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE const T::Scalar* Eigen::internal::extract_data

(

const T &

m

)

                                                                                       {
  return extract_data_selector<T>::run(m);
}

References m, and Eigen::internal::extract_data_selector< T, HasUsableDirectAccess >::run().

Referenced by Eigen::TriangularViewImpl< MatrixType, Mode, Sparse >::_solve_impl(), Eigen::internal::checkTransposeAliasing_impl< Derived, OtherDerived, MightHaveTransposeAliasing >::run(), Eigen::internal::compute_inverse_and_det_with_check< MatrixType, ResultType, 4 >::run(), Eigen::internal::check_transpose_aliasing_run_time_selector< Scalar, DestIsTransposed, CwiseBinaryOp< BinOp, DerivedA, DerivedB > >::run(), Eigen::internal::check_transpose_aliasing_run_time_selector< Scalar, DestIsTransposed, OtherDerived >::run(), and Eigen::internal::Assignment< DstXprType, Inverse< XprType >, internal::assign_op< typename DstXprType::Scalar, typename XprType::Scalar >, Dense2Dense >::run().

F32MaskToBf16Mask()

EIGEN_STRONG_INLINE Packet4bf Eigen::internal::F32MaskToBf16Mask

(

const Packet4f &

p

)

{ return vmovn_u32(vreinterpretq_u32_f32(p)); }

References p.

Referenced by pcmp_eq< Packet4bf >(), pcmp_le< Packet4bf >(), pcmp_lt< Packet4bf >(), and pcmp_lt_or_nan< Packet4bf >().

F32ToBf16() [1/5]

EIGEN_STRONG_INLINE Packet16bf Eigen::internal::F32ToBf16

(

const Packet16f &

a

)

                                                             {
  Packet16bf r;
 
#if defined(EIGEN_VECTORIZE_AVX512BF16) && EIGEN_GNUC_STRICT_AT_LEAST(10, 1, 0)
  // Since GCC 10.1 supports avx512bf16 and C style explicit cast
  // (C++ static_cast is not supported yet), do conversion via intrinsic
  // and register path for performance.
  r = (__m256i)(_mm512_cvtneps_pbh(a));
 
#else
  __m512i t;
  __m512i input = _mm512_castps_si512(a);
  __m512i nan = _mm512_set1_epi32(0x7fc0);
 
  // uint32_t lsb = (input >> 16) & 1;
  t = _mm512_and_si512(_mm512_srli_epi32(input, 16), _mm512_set1_epi32(1));
  // uint32_t rounding_bias = 0x7fff + lsb;
  t = _mm512_add_epi32(t, _mm512_set1_epi32(0x7fff));
  // input += rounding_bias;
  t = _mm512_add_epi32(t, input);
  // input = input >> 16;
  t = _mm512_srli_epi32(t, 16);
 
  // Check NaN before converting back to bf16
  __mmask16 mask = _mm512_cmp_ps_mask(a, a, _CMP_ORD_Q);
 
  t = _mm512_mask_blend_epi32(mask, nan, t);
  // output.value = static_cast<uint16_t>(input);
  r = _mm512_cvtepi32_epi16(t);
#endif  // EIGEN_VECTORIZE_AVX512BF16
 
  return r;
}

References a, UniformPSDSelfTest::r, and plotPSD::t.

F32ToBf16() [2/5]

EIGEN_STRONG_INLINE Packet4bf Eigen::internal::F32ToBf16

(

const Packet4f &

p

)

                                                           {
  // See the scalar implementation in BFloat16.h for a comprehensible explanation
  // of this fast rounding algorithm
  Packet4ui input = Packet4ui(vreinterpretq_u32_f32(p));
 
  // lsb = (input >> 16) & 1
  Packet4ui lsb = vandq_u32(vshrq_n_u32(input, 16), vdupq_n_u32(1));
 
  // rounding_bias = 0x7fff + lsb
  Packet4ui rounding_bias = vaddq_u32(lsb, vdupq_n_u32(0x7fff));
 
  // input += rounding_bias
  input = vaddq_u32(input, rounding_bias);
 
  // input = input >> 16
  input = vshrq_n_u32(input, 16);
 
  // Replace float-nans by bfloat16-nans, that is 0x7fc0
  const Packet4ui bf16_nan = vdupq_n_u32(0x7fc0);
  const Packet4ui mask = vceqq_f32(p, p);
  input = vbslq_u32(mask, input, bf16_nan);
 
  // output = static_cast<uint16_t>(input)
  return vmovn_u32(input);
}

References p.

F32ToBf16() [3/5]

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::F32ToBf16

(

const Packet8f &

a

)

                                                           {
  __m256i input = _mm256_castps_si256(a);
 
#ifdef EIGEN_VECTORIZE_AVX2
  // uint32_t lsb = (input >> 16);
  __m256i t = _mm256_srli_epi32(input, 16);
  // uint32_t lsb = lsb & 1;
  t = _mm256_and_si256(t, _mm256_set1_epi32(1));
  // uint32_t rounding_bias = 0x7fff + lsb;
  t = _mm256_add_epi32(t, _mm256_set1_epi32(0x7fff));
  // input += rounding_bias;
  t = _mm256_add_epi32(t, input);
  // input = input >> 16;
  t = _mm256_srli_epi32(t, 16);
  // Check NaN before converting back to bf16
  __m256 mask = _mm256_cmp_ps(a, a, _CMP_ORD_Q);
  __m256i nan = _mm256_set1_epi32(0x7fc0);
  t = _mm256_blendv_epi8(nan, t, _mm256_castps_si256(mask));
  // output = numext::bit_cast<uint16_t>(input);
  return _mm_packus_epi32(_mm256_extractf128_si256(t, 0), _mm256_extractf128_si256(t, 1));
#else
  // uint32_t lsb = (input >> 16);
  __m128i lo = _mm_srli_epi32(_mm256_extractf128_si256(input, 0), 16);
  __m128i hi = _mm_srli_epi32(_mm256_extractf128_si256(input, 1), 16);
  // uint32_t lsb = lsb & 1;
  lo = _mm_and_si128(lo, _mm_set1_epi32(1));
  hi = _mm_and_si128(hi, _mm_set1_epi32(1));
  // uint32_t rounding_bias = 0x7fff + lsb;
  lo = _mm_add_epi32(lo, _mm_set1_epi32(0x7fff));
  hi = _mm_add_epi32(hi, _mm_set1_epi32(0x7fff));
  // input += rounding_bias;
  lo = _mm_add_epi32(lo, _mm256_extractf128_si256(input, 0));
  hi = _mm_add_epi32(hi, _mm256_extractf128_si256(input, 1));
  // input = input >> 16;
  lo = _mm_srli_epi32(lo, 16);
  hi = _mm_srli_epi32(hi, 16);
  // Check NaN before converting back to bf16
  __m256 mask = _mm256_cmp_ps(a, a, _CMP_ORD_Q);
  __m128i nan = _mm_set1_epi32(0x7fc0);
  lo = _mm_blendv_epi8(nan, lo, _mm_castps_si128(_mm256_castps256_ps128(mask)));
  hi = _mm_blendv_epi8(nan, hi, _mm_castps_si128(_mm256_extractf128_ps(mask, 1)));
  // output = numext::bit_cast<uint16_t>(input);
  return _mm_packus_epi32(lo, hi);
#endif
}

References a, and plotPSD::t.

F32ToBf16() [4/5]

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::F32ToBf16	(	Packet4f	even,
		Packet4f	odd
	)

Convert and pack two float Packets into one bfloat16 Packet - odd & even order

                                                                     {
#ifdef _ARCH_PWR10
  return pmerge(reinterpret_cast<Packet4ui>(F32ToBf16(even).m_val), reinterpret_cast<Packet4ui>(F32ToBf16(odd).m_val));
#else
  return F32ToBf16Two<false>(even, odd);
#endif
}

References F32ToBf16(), and pmerge().

F32ToBf16() [5/5]

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::F32ToBf16

(

Packet4f

p4f

)

                                                      {
#ifdef _ARCH_PWR10
  return reinterpret_cast<Packet8us>(__builtin_vsx_xvcvspbf16(reinterpret_cast<Packet16uc>(p4f)));
#else
  Packet4ui input = reinterpret_cast<Packet4ui>(p4f);
  Packet4ui lsb = plogical_shift_right<16>(input);
  lsb = pand<Packet4ui>(lsb, reinterpret_cast<Packet4ui>(p4i_ONE));
 
  EIGEN_DECLARE_CONST_FAST_Packet4ui(BIAS, 0x7FFFu);
  Packet4ui rounding_bias = padd<Packet4ui>(lsb, p4ui_BIAS);
  input = padd<Packet4ui>(input, rounding_bias);
 
  const EIGEN_DECLARE_CONST_FAST_Packet4ui(nan, 0x7FC00000);
#ifdef _ARCH_PWR9
  Packet4bi nan_selector = vec_test_data_class(p4f, __VEC_CLASS_FP_NAN);
  input = vec_sel(input, p4ui_nan, nan_selector);
 
#ifdef SUPPORT_BF16_SUBNORMALS
  Packet4bi subnormal_selector = vec_test_data_class(p4f, __VEC_CLASS_FP_SUBNORMAL);
  input = vec_sel(input, reinterpret_cast<Packet4ui>(p4f), subnormal_selector);
#endif
#else
#ifdef SUPPORT_BF16_SUBNORMALS
  // Test NaN and Subnormal
  const EIGEN_DECLARE_CONST_FAST_Packet4ui(exp_mask, 0x7F800000);
  Packet4ui exp = pand<Packet4ui>(p4ui_exp_mask, reinterpret_cast<Packet4ui>(p4f));
 
  const EIGEN_DECLARE_CONST_FAST_Packet4ui(mantissa_mask, 0x7FFFFF);
  Packet4ui mantissa = pand<Packet4ui>(p4ui_mantissa_mask, reinterpret_cast<Packet4ui>(p4f));
 
  Packet4bi is_max_exp = vec_cmpeq(exp, p4ui_exp_mask);
  Packet4bi is_mant_zero = vec_cmpeq(mantissa, reinterpret_cast<Packet4ui>(p4i_ZERO));
 
  Packet4ui nan_selector =
      pandnot<Packet4ui>(reinterpret_cast<Packet4ui>(is_max_exp), reinterpret_cast<Packet4ui>(is_mant_zero));
 
  Packet4bi is_zero_exp = vec_cmpeq(exp, reinterpret_cast<Packet4ui>(p4i_ZERO));
 
  Packet4ui subnormal_selector =
      pandnot<Packet4ui>(reinterpret_cast<Packet4ui>(is_zero_exp), reinterpret_cast<Packet4ui>(is_mant_zero));
 
  input = vec_sel(input, p4ui_nan, nan_selector);
  input = vec_sel(input, reinterpret_cast<Packet4ui>(p4f), subnormal_selector);
#else
  // Test only NaN
  Packet4bi nan_selector = vec_cmpeq(p4f, p4f);
 
  input = vec_sel(p4ui_nan, input, nan_selector);
#endif
#endif
 
  input = plogical_shift_right<16>(input);
  return reinterpret_cast<Packet8us>(input);
#endif
}

References __VEC_CLASS_FP_NAN, EIGEN_DECLARE_CONST_FAST_Packet4ui(), Eigen::bfloat16_impl::exp(), padd< Packet4ui >(), pand< Packet4ui >(), and pandnot< Packet4ui >().

Referenced by F32ToBf16(), F32ToBf16Both(), pabs(), pabsdiff< Packet4bf >(), padd< Packet16bf >(), padd< Packet4bf >(), pcast< Packet16f, Packet16bf >(), pcast< Packet8f, Packet8bf >(), pcast< Packet8us, Packet8bf >(), pceil< Packet16bf >(), pceil< Packet4bf >(), pdiv< Packet16bf >(), pdiv< Packet4bf >(), pfloor< Packet16bf >(), pfloor< Packet4bf >(), pfrexp(), pfrexp< Packet8bf >(), pldexp(), plset< Packet16bf >(), plset< Packet4bf >(), pmadd(), pmax< Packet16bf >(), pmax< Packet4bf >(), pmax< PropagateNaN, Packet4bf >(), pmax< PropagateNumbers, Packet4bf >(), pmin< Packet16bf >(), pmin< Packet4bf >(), pmin< PropagateNaN, Packet4bf >(), pmin< PropagateNumbers, Packet4bf >(), pmul< Packet16bf >(), pmul< Packet4bf >(), print< Packet16bf >(), print< Packet4bf >(), print< Packet8bf >(), pround< Packet16bf >(), pround< Packet4bf >(), psub< Packet16bf >(), psub< Packet4bf >(), ptrunc< Packet16bf >(), and ptrunc< Packet4bf >().

F32ToBf16Bool()

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::F32ToBf16Bool	(	Packet4f	even,
		Packet4f	odd
	)

                                                                         {
  return pmerge(reinterpret_cast<Packet4ui>(even), reinterpret_cast<Packet4ui>(odd));
}

References pmerge().

F32ToBf16Both()

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::F32ToBf16Both	(	Packet4f	lo,
		Packet4f	hi
	)

Convert and pack two float Packets into one bfloat16 Packet - low & high order

                                                                      {
#ifdef _ARCH_PWR10
  Packet8bf fp16_0 = F32ToBf16(lo);
  Packet8bf fp16_1 = F32ToBf16(hi);
  return vec_pack(reinterpret_cast<Packet4ui>(fp16_0.m_val), reinterpret_cast<Packet4ui>(fp16_1.m_val));
#else
  return F32ToBf16Two(lo, hi);
#endif
}

References F32ToBf16(), F32ToBf16Two(), and Eigen::internal::eigen_packet_wrapper< T, unique_id >::m_val.

Referenced by convertF32toBF16VSX(), and pcast< Packet4f, Packet8bf >().

F32ToBf16Two()

template<bool lohi = true>

EIGEN_ALWAYS_INLINE Packet8bf Eigen::internal::F32ToBf16Two	(	Packet4f	lo,
		Packet4f	hi
	)

Convert and pack two float Packets into one bfloat16 Packet

Parameters

lohi	to expect either a low & high OR odd & even order

                                                                     {
  Packet8us p4f = Bf16PackHigh<lohi>(lo, hi);
  Packet8us p4f2 = Bf16PackLow<lohi>(lo, hi);
 
  Packet8us lsb = pand<Packet8us>(p4f, p8us_ONE);
  EIGEN_DECLARE_CONST_FAST_Packet8us(BIAS, 0x7FFFu);
  lsb = padd<Packet8us>(lsb, p8us_BIAS);
  lsb = padd<Packet8us>(lsb, p4f2);
 
  Packet8bi rounding_bias = vec_cmplt(lsb, p4f2);
  Packet8us input = psub<Packet8us>(p4f, reinterpret_cast<Packet8us>(rounding_bias));
 
#ifdef _ARCH_PWR9
  Packet4bi nan_selector_lo = vec_test_data_class(lo, __VEC_CLASS_FP_NAN);
  Packet4bi nan_selector_hi = vec_test_data_class(hi, __VEC_CLASS_FP_NAN);
  Packet8us nan_selector =
      Bf16PackLow<lohi>(reinterpret_cast<Packet4f>(nan_selector_lo), reinterpret_cast<Packet4f>(nan_selector_hi));
 
  input = vec_sel(input, p8us_BIAS, nan_selector);
 
#ifdef SUPPORT_BF16_SUBNORMALS
  Packet4bi subnormal_selector_lo = vec_test_data_class(lo, __VEC_CLASS_FP_SUBNORMAL);
  Packet4bi subnormal_selector_hi = vec_test_data_class(hi, __VEC_CLASS_FP_SUBNORMAL);
  Packet8us subnormal_selector = Bf16PackLow<lohi>(reinterpret_cast<Packet4f>(subnormal_selector_lo),
                                                   reinterpret_cast<Packet4f>(subnormal_selector_hi));
 
  input = vec_sel(input, reinterpret_cast<Packet8us>(p4f), subnormal_selector);
#endif
#else
#ifdef SUPPORT_BF16_SUBNORMALS
  // Test NaN and Subnormal
  const EIGEN_DECLARE_CONST_FAST_Packet8us(exp_mask, 0x7F80);
  Packet8us exp = pand<Packet8us>(p8us_exp_mask, p4f);
 
  const EIGEN_DECLARE_CONST_FAST_Packet8us(mantissa_mask, 0x7Fu);
  Packet8us mantissa = pand<Packet8us>(p8us_mantissa_mask, p4f);
 
  Packet8bi is_max_exp = vec_cmpeq(exp, p8us_exp_mask);
  Packet8bi is_mant_zero = vec_cmpeq(mantissa, reinterpret_cast<Packet8us>(p4i_ZERO));
 
  Packet8us nan_selector =
      pandnot<Packet8us>(reinterpret_cast<Packet8us>(is_max_exp), reinterpret_cast<Packet8us>(is_mant_zero));
 
  Packet8bi is_zero_exp = vec_cmpeq(exp, reinterpret_cast<Packet8us>(p4i_ZERO));
 
  Packet8us subnormal_selector =
      pandnot<Packet8us>(reinterpret_cast<Packet8us>(is_zero_exp), reinterpret_cast<Packet8us>(is_mant_zero));
 
  // Using BIAS as NaN (since any or all of the last 7 bits can be set)
  input = vec_sel(input, p8us_BIAS, nan_selector);
  input = vec_sel(input, reinterpret_cast<Packet8us>(p4f), subnormal_selector);
#else
  // Test only NaN
  Packet4bi nan_selector_lo = vec_cmpeq(lo, lo);
  Packet4bi nan_selector_hi = vec_cmpeq(hi, hi);
  Packet8us nan_selector =
      Bf16PackLow<lohi>(reinterpret_cast<Packet4f>(nan_selector_lo), reinterpret_cast<Packet4f>(nan_selector_hi));
 
  input = vec_sel(p8us_BIAS, input, nan_selector);
#endif
#endif
 
  return input;
}

References __VEC_CLASS_FP_NAN, EIGEN_DECLARE_CONST_FAST_Packet8us(), Eigen::bfloat16_impl::exp(), padd< Packet8us >(), pand< Packet8us >(), pandnot< Packet8us >(), and psub< Packet8us >().

Referenced by F32ToBf16Both().

fast_twosum() [1/3]

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::fast_twosum	(	const Packet &	x,
		const Packet &	y,
		Packet &	s_hi,
		Packet &	s_lo
	)

                                                                                                                     {
  s_hi = padd(x, y);
  const Packet t = psub(s_hi, x);
  s_lo = psub(y, t);
}

References padd(), psub(), plotPSD::t, plotDoE::x, and y.

Referenced by doubleword_div_fp(), fast_twosum(), generic_pow_impl(), Eigen::internal::accurate_log2< double >::operator()(), Eigen::internal::accurate_log2< float >::operator()(), twoprod(), and twosum().

fast_twosum() [2/3]

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::fast_twosum	(	const Packet &	x,
		const Packet &	y_hi,
		const Packet &	y_lo,
		Packet &	s_hi,
		Packet &	s_lo
	)

                                                                                   {
  Packet r_hi, r_lo;
  fast_twosum(x, y_hi, r_hi, r_lo);
  const Packet s = padd(y_lo, r_lo);
  fast_twosum(r_hi, s, s_hi, s_lo);
}

References fast_twosum(), padd(), s, and plotDoE::x.

fast_twosum() [3/3]

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::fast_twosum	(	const Packet &	x_hi,
		const Packet &	x_lo,
		const Packet &	y_hi,
		const Packet &	y_lo,
		Packet &	s_hi,
		Packet &	s_lo
	)

                                                                                                       {
  Packet r_hi, r_lo;
  fast_twosum(x_hi, y_hi, r_hi, r_lo);
  const Packet s = padd(padd(y_lo, r_lo), x_lo);
  fast_twosum(r_hi, s, s_hi, s_lo);
}

References fast_twosum(), padd(), and s.

fdjac1()

template<typename FunctorType , typename Scalar >

DenseIndex Eigen::internal::fdjac1	(	const FunctorType &	Functor,
		Matrix< Scalar, Dynamic, 1 > &	x,
		Matrix< Scalar, Dynamic, 1 > &	fvec,
		Matrix< Scalar, Dynamic, Dynamic > &	fjac,
		DenseIndex	ml,
		DenseIndex	mu,
		Scalar	epsfcn
	)

                                                                                                       {
  using std::abs;
  using std::sqrt;
 
  typedef DenseIndex Index;
 
  /* Local variables */
  Scalar h;
  Index j, k;
  Scalar eps, temp;
  Index msum;
  int iflag;
  Index start, length;
 
  /* Function Body */
  const Scalar epsmch = NumTraits<Scalar>::epsilon();
  const Index n = x.size();
  eigen_assert(fvec.size() == n);
  Matrix<Scalar, Dynamic, 1> wa1(n);
  Matrix<Scalar, Dynamic, 1> wa2(n);
 
  eps = sqrt((std::max)(epsfcn, epsmch));
  msum = ml + mu + 1;
  if (msum >= n) {
    /* computation of dense approximate jacobian. */
    for (j = 0; j < n; ++j) {
      temp = x[j];
      h = eps * abs(temp);
      if (h == 0.) h = eps;
      x[j] = temp + h;
      iflag = Functor(x, wa1);
      if (iflag < 0) return iflag;
      x[j] = temp;
      fjac.col(j) = (wa1 - fvec) / h;
    }
 
  } else {
    /* computation of banded approximate jacobian. */
    for (k = 0; k < msum; ++k) {
      for (j = k; (msum < 0) ? (j > n) : (j < n); j += msum) {
        wa2[j] = x[j];
        h = eps * abs(wa2[j]);
        if (h == 0.) h = eps;
        x[j] = wa2[j] + h;
      }
      iflag = Functor(x, wa1);
      if (iflag < 0) return iflag;
      for (j = k; (msum < 0) ? (j > n) : (j < n); j += msum) {
        x[j] = wa2[j];
        h = eps * abs(wa2[j]);
        if (h == 0.) h = eps;
        fjac.col(j).setZero();
        start = std::max<Index>(0, j - mu);
        length = (std::min)(n - 1, j + ml) - start + 1;
        fjac.col(j).segment(start, length) = (wa1.segment(start, length) - fvec.segment(start, length)) / h;
      }
    }
  }
  return 0;
}

References abs(), eigen_assert, CRBond_Bessel::eps, oomph::SarahBL::epsilon, j, k, max, min, Global_Parameters::mu, n, Eigen::PlainObjectBase< Derived >::setZero(), sqrt(), oomph::CumulativeTimings::start(), and plotDoE::x.

Referenced by Eigen::HybridNonLinearSolver< FunctorType, Scalar >::solveNumericalDiffOneStep().

fftw_cast() [1/4]

fftw_complex* Eigen::internal::fftw_cast ( const std::complex< double > *  p )

inline

                                                            {
  return const_cast<fftw_complex *>(reinterpret_cast<const fftw_complex *>(p));
}

References p.

fftw_cast() [2/4]

fftwf_complex* Eigen::internal::fftw_cast ( const std::complex< float > *  p )

inline

                                                            {
  return const_cast<fftwf_complex *>(reinterpret_cast<const fftwf_complex *>(p));
}

References p.

fftw_cast() [3/4]

fftwl_complex* Eigen::internal::fftw_cast ( const std::complex< long double > *  p )

inline

                                                                  {
  return const_cast<fftwl_complex *>(reinterpret_cast<const fftwl_complex *>(p));
}

References p.

fftw_cast() [4/4]

template<typename T >

T* Eigen::internal::fftw_cast ( const T *  p )

inline

                                {
  return const_cast<T *>(p);
}

References p.

Referenced by Eigen::internal::fftw_impl< Scalar_ >::fwd(), Eigen::internal::fftw_impl< Scalar_ >::fwd2(), Eigen::internal::fftw_impl< Scalar_ >::inv(), and Eigen::internal::fftw_impl< Scalar_ >::inv2().

first_aligned() [1/2]

template<int Alignment, typename Derived >

static Index Eigen::internal::first_aligned ( const DenseBase< Derived > &  m )

inlinestatic

Returns

the index of the first element of the array stored by m that is properly aligned with respect to Alignment for vectorization.

Template Parameters

Alignment

requested alignment in Bytes.

There is also the variant first_aligned(const Scalar*, Integer) defined in Memory.h. See it for more documentation.

                                                               {
  enum { ReturnZero = (int(evaluator<Derived>::Alignment) >= Alignment) || !(Derived::Flags & DirectAccessBit) };
  return first_aligned_impl<Alignment, Derived, ReturnZero>::run(m.derived());
}

References Eigen::DirectAccessBit, int(), m, and Eigen::internal::first_aligned_impl< Alignment, Derived, JustReturnZero >::run().

Referenced by first_default_aligned().

first_aligned() [2/2]

template<int Alignment, typename Scalar , typename Index >

EIGEN_DEVICE_FUNC Index Eigen::internal::first_aligned ( const Scalar *  array, Index  size  )

inline

Returns the index of the first element of the array that is well aligned with respect to the requested Alignment.

Template Parameters

Alignment

requested alignment in Bytes.

Parameters

array	the address of the start of the array
size	the size of the array

Note

If no element of the array is well aligned or the requested alignment is not a multiple of a scalar, the size of the array is returned. For example with SSE, the requested alignment is typically 16-bytes. If packet size for the given scalar type is 1, then everything is considered well-aligned.

Otherwise, if the Alignment is larger that the scalar size, we rely on the assumptions that sizeof(Scalar) is a power of 2. On the other hand, we do not assume that the array address is a multiple of sizeof(Scalar), as that fails for example with Scalar=double on certain 32-bit platforms, see bug #79.

There is also the variant first_aligned(const MatrixBase&) defined in DenseCoeffsBase.h.

See also

first_default_aligned()

                                                                              {
  const Index ScalarSize = sizeof(Scalar);
  const Index AlignmentSize = Alignment / ScalarSize;
  const Index AlignmentMask = AlignmentSize - 1;
 
  if (AlignmentSize <= 1) {
    // Either the requested alignment if smaller than a scalar, or it exactly match a 1 scalar
    // so that all elements of the array have the same alignment.
    return 0;
  } else if ((std::uintptr_t(array) & (sizeof(Scalar) - 1)) || (Alignment % ScalarSize) != 0) {
    // The array is not aligned to the size of a single scalar, or the requested alignment is not a multiple of the
    // scalar size. Consequently, no element of the array is well aligned.
    return size;
  } else {
    Index first = (AlignmentSize - (Index((std::uintptr_t(array) / sizeof(Scalar))) & AlignmentMask)) & AlignmentMask;
    return (first < size) ? first : size;
  }
}

References size.

first_default_aligned() [1/2]

template<typename Derived >

static Index Eigen::internal::first_default_aligned ( const DenseBase< Derived > &  m )

inlinestatic

                                                                       {
  typedef typename Derived::Scalar Scalar;
  typedef typename packet_traits<Scalar>::type DefaultPacketType;
  return internal::first_aligned<int(unpacket_traits<DefaultPacketType>::alignment), Derived>(m);
}

References first_aligned(), int(), and m.

Referenced by Eigen::internal::blas_data_mapper< Scalar, Index, StorageOrder, AlignmentType, 1 >::firstAligned(), Eigen::internal::SparseLUImpl< Scalar, StorageIndex >::panel_bmod(), Eigen::internal::redux_impl< Func, Evaluator, LinearVectorizedTraversal, NoUnrolling >::run(), Eigen::internal::LU_kernel_bmod< SegSizeAtCompileTime >::run(), Eigen::internal::selfadjoint_matrix_vector_product< Scalar, Index, StorageOrder, UpLo, ConjugateLhs, ConjugateRhs, Version >::run(), Eigen::internal::apply_rotation_in_the_plane_selector< Scalar, OtherScalar, SizeAtCompileTime, MinAlignment, true >::run(), test_first_aligned_helper(), and test_none_aligned_helper().

first_default_aligned() [2/2]

template<typename Scalar , typename Index >

EIGEN_DEVICE_FUNC Index Eigen::internal::first_default_aligned ( const Scalar *  array, Index  size  )

inline

Returns the index of the first element of the array that is well aligned with respect the largest packet requirement.

See also

first_aligned(Scalar*,Index) and first_default_aligned(DenseBase<Derived>)

                                                                                      {
  typedef typename packet_traits<Scalar>::type DefaultPacketType;
  return first_aligned<unpacket_traits<DefaultPacketType>::alignment>(array, size);
}

References size.

first_multiple()

template<typename Index >

Index Eigen::internal::first_multiple ( Index  size, Index  base  )

inline

Returns the smallest integer multiple of base and greater or equal to size

                                                    {
  return ((size + base - 1) / base) * base;
}

References size.

Referenced by Eigen::TensorEvaluator< const TensorBroadcastingOp< Broadcast, ArgType >, Device >::BroadcastBlockAlongBcastDim(), and Eigen::internal::LU_kernel_bmod< SegSizeAtCompileTime >::run().

flipsign()

template<typename T >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T Eigen::internal::flipsign	(	const T &	should_flipsign,
		const T &	x
	)

                                                                                       {
  typedef typename unpacket_traits<T>::type Scalar;
  const T sign_mask = pset1<T>(Scalar(-0.0));
  T sign_bit = pand<T>(should_flipsign, sign_mask);
  return pxor<T>(sign_bit, x);
}

References Eigen::numext::x.

Referenced by generic_ndtri_lt_exp_neg_two().

flipsign< double >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double Eigen::internal::flipsign< double >	(	const double &	should_flipsign,
		const double &	x
	)

                                                                                                              {
  return should_flipsign == 0 ? x : -x;
}

References Eigen::numext::x.

flipsign< float >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float Eigen::internal::flipsign< float >	(	const float &	should_flipsign,
		const float &	x
	)

                                                                                                          {
  return should_flipsign == 0 ? x : -x;
}

References Eigen::numext::x.

float2half() [1/2]

EIGEN_STRONG_INLINE Packet16h Eigen::internal::float2half

(

const Packet16f &

a

)

                                                             {
  return _mm512_cvtps_ph(a, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
}

References a.

float2half() [2/2]

EIGEN_STRONG_INLINE Packet8h Eigen::internal::float2half

(

const Packet8f &

a

)

                                                           {
#ifdef EIGEN_HAS_FP16_C
  return _mm256_cvtps_ph(a, _MM_FROUND_TO_NEAREST_INT);
#else
  __m128i lo = float2half(_mm256_extractf128_ps(a, 0));
  __m128i hi = float2half(_mm256_extractf128_ps(a, 1));
  return _mm_packus_epi32(lo, hi);
#endif
}

References a.

Referenced by padd< Packet16h >(), padd< Packet8h >(), pcast< Packet16f, Packet16h >(), pcast< Packet8f, Packet8h >(), pceil< Packet16h >(), pceil< Packet8h >(), pdiv< Packet16h >(), pdiv< Packet8h >(), pfloor< Packet16h >(), pfloor< Packet8h >(), pfrexp(), pldexp(), plset< Packet16h >(), plset< Packet8h >(), pmax< Packet16h >(), pmax< Packet8h >(), pmin< Packet16h >(), pmin< Packet8h >(), pmul< Packet16h >(), pmul< Packet8h >(), print< Packet16h >(), print< Packet8h >(), pround< Packet16h >(), pround< Packet8h >(), psub< Packet16h >(), psub< Packet8h >(), ptrunc< Packet16h >(), and ptrunc< Packet8h >().

fortran_to_c_numbering()

template<typename MatrixType >

void Eigen::internal::fortran_to_c_numbering

(

MatrixType &

mat

)

                                             {
  // Check the Numbering
  if (mat.outerIndexPtr()[0] == 1) {  // Convert to C-style numbering
    int i;
    for (i = 0; i <= mat.rows(); ++i) --mat.outerIndexPtr()[i];
    for (i = 0; i < mat.nonZeros(); ++i) --mat.innerIndexPtr()[i];
  }
}

References i, Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::innerIndexPtr(), Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::nonZeros(), Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::outerIndexPtr(), and Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::rows().

gemm()

template<typename Scalar , typename Packet , typename RhsPacket , typename DataMapper , const Index accRows, const Index accCols>

EIGEN_STRONG_INLINE void Eigen::internal::gemm	(	const DataMapper &	res,
		const Scalar *	blockA,
		const Scalar *	blockB,
		Index	rows,
		Index	depth,
		Index	cols,
		Scalar	alpha,
		Index	strideA,
		Index	strideB,
		Index	offsetA,
		Index	offsetB
	)

                                             {
  const Index remaining_rows = rows % accCols;
 
  if (strideA == -1) strideA = depth;
  if (strideB == -1) strideB = depth;
 
  const Packet pAlpha = pset1<Packet>(alpha);
  const Packet pMask = bmask<Packet>(remaining_rows);
 
  Index col = 0;
  for (; col + accRows <= cols; col += accRows) {
    gemm_cols<Scalar, Packet, DataMapper, accRows, accCols>(res, blockA, blockB, depth, strideA, offsetA, strideB,
                                                            offsetB, col, rows, remaining_rows, pAlpha, pMask);
  }
 
  if (col != cols) {
    gemm_extra_cols<Scalar, Packet, DataMapper, accCols>(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB,
                                                         col, rows, cols, remaining_rows, pAlpha, pMask);
  }
}

References alpha, col(), cols, res, and rows.

gemm_cols()

template<typename Scalar , typename Packet , typename DataMapper , const Index accRows, const Index accCols>

EIGEN_ALWAYS_INLINE void Eigen::internal::gemm_cols	(	const DataMapper &	res,
		const Scalar *	blockA,
		const Scalar *	blockB,
		Index	depth,
		Index	strideA,
		Index	offsetA,
		Index	strideB,
		Index	offsetB,
		Index	col,
		Index	rows,
		Index	remaining_rows,
		const Packet &	pAlpha,
		const Packet &	pMask
	)

                                                                                                    {
  const DataMapper res3 = res.getSubMapper(0, col);
 
  const Scalar* rhs_base = blockB + col * strideB + MICRO_NEW_ROWS * offsetB;
  const Scalar* lhs_base = blockA + accCols * offsetA;
  Index row = 0;
 
#define MAX_UNROLL 7
  while (row + MAX_UNROLL * accCols <= rows) {
    MICRO_UNROLL_ITER2(MAX_UNROLL, 0);
  }
  switch ((rows - row) / accCols) {
#if MAX_UNROLL > 7
    case 7:
      MICRO_UNROLL_ITER(MICRO_UNROLL_ITER2, 7)
      break;
#endif
#if MAX_UNROLL > 6
    case 6:
      MICRO_UNROLL_ITER(MICRO_UNROLL_ITER2, 6)
      break;
#endif
#if MAX_UNROLL > 5
    case 5:
      MICRO_UNROLL_ITER(MICRO_UNROLL_ITER2, 5)
      break;
#endif
#if MAX_UNROLL > 4
    case 4:
      MICRO_UNROLL_ITER(MICRO_UNROLL_ITER2, 4)
      break;
#endif
#if MAX_UNROLL > 3
    case 3:
      MICRO_UNROLL_ITER(MICRO_UNROLL_ITER2, 3)
      break;
#endif
#if MAX_UNROLL > 2
    case 2:
      MICRO_UNROLL_ITER(MICRO_UNROLL_ITER2, 2)
      break;
#endif
#if MAX_UNROLL > 1
    case 1:
      MICRO_UNROLL_ITER(MICRO_UNROLL_ITER2, 1)
      break;
#endif
    default:
      break;
  }
#undef MAX_UNROLL
 
  if (remaining_rows > 0) {
    gemm_extra_row<Scalar, Packet, DataMapper, accRows, accCols>(res3, blockA, rhs_base, depth, strideA, offsetA,
                                                                 strideB, row, rows, remaining_rows, pAlpha, pMask);
  }
}

References col(), MAX_UNROLL, MICRO_NEW_ROWS, MICRO_UNROLL_ITER, MICRO_UNROLL_ITER2, res, row(), and rows.

gemm_complex()

template<typename LhsScalar , typename RhsScalar , typename Scalarc , typename Scalar , typename Packet , typename Packetc , typename RhsPacket , typename DataMapper , const Index accRows, const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>

EIGEN_STRONG_INLINE void Eigen::internal::gemm_complex	(	const DataMapper &	res,
		const LhsScalar *	blockAc,
		const RhsScalar *	blockBc,
		Index	rows,
		Index	depth,
		Index	cols,
		Scalarc	alpha,
		Index	strideA,
		Index	strideB,
		Index	offsetA,
		Index	offsetB
	)

                                                                    {
  const Index remaining_rows = rows % accCols;
 
  if (strideA == -1) strideA = depth;
  if (strideB == -1) strideB = depth;
 
  const Packet pAlphaReal = pset1<Packet>(alpha.real());
  const Packet pAlphaImag = pset1<Packet>(alpha.imag());
  const Packet pMask = bmask<Packet>(remaining_rows);
 
  const Scalar* blockA = (Scalar*)blockAc;
  const Scalar* blockB = (Scalar*)blockBc;
 
  Index col = 0;
  for (; col + accRows <= cols; col += accRows) {
    gemm_complex_cols<Scalar, Packet, Packetc, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal,
                      RhsIsReal>(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows,
                                 remaining_rows, pAlphaReal, pAlphaImag, pMask);
  }
 
  if (col != cols) {
    gemm_complex_extra_cols<Scalar, Packet, Packetc, DataMapper, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal,
                            RhsIsReal>(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, cols,
                                       remaining_rows, pAlphaReal, pAlphaImag, pMask);
  }
}

References alpha, col(), cols, gemm_complex_cols(), gemm_complex_extra_cols(), res, and rows.

Referenced by Eigen::internal::gebp_kernel< double, std::complex< double >, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >::operator()(), Eigen::internal::gebp_kernel< float, std::complex< float >, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >::operator()(), Eigen::internal::gebp_kernel< std::complex< double >, double, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >::operator()(), Eigen::internal::gebp_kernel< std::complex< double >, std::complex< double >, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >::operator()(), Eigen::internal::gebp_kernel< std::complex< float >, float, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >::operator()(), and Eigen::internal::gebp_kernel< std::complex< float >, std::complex< float >, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >::operator()().

gemm_complex_cols()

template<typename Scalar , typename Packet , typename Packetc , typename DataMapper , const Index accRows, const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>

EIGEN_ALWAYS_INLINE void Eigen::internal::gemm_complex_cols	(	const DataMapper &	res,
		const Scalar *	blockA,
		const Scalar *	blockB,
		Index	depth,
		Index	strideA,
		Index	offsetA,
		Index	strideB,
		Index	offsetB,
		Index	col,
		Index	rows,
		Index	remaining_rows,
		const Packet &	pAlphaReal,
		const Packet &	pAlphaImag,
		const Packet &	pMask
	)

                                                                                          {
  const DataMapper res3 = res.getSubMapper(0, col);
 
  const Scalar* rhs_base = blockB + advanceCols * col * strideB + MICRO_NEW_ROWS * offsetB;
  const Scalar* lhs_base = blockA + accCols * offsetA;
  Index row = 0;
 
#define MAX_COMPLEX_UNROLL 4
  while (row + MAX_COMPLEX_UNROLL * accCols <= rows) {
    MICRO_COMPLEX_UNROLL_ITER2(MAX_COMPLEX_UNROLL, 0);
  }
  switch ((rows - row) / accCols) {
#if MAX_COMPLEX_UNROLL > 4
    case 4:
      MICRO_COMPLEX_UNROLL_ITER(MICRO_COMPLEX_UNROLL_ITER2, 4)
      break;
#endif
#if MAX_COMPLEX_UNROLL > 3
    case 3:
      MICRO_COMPLEX_UNROLL_ITER(MICRO_COMPLEX_UNROLL_ITER2, 3)
      break;
#endif
#if MAX_COMPLEX_UNROLL > 2
    case 2:
      MICRO_COMPLEX_UNROLL_ITER(MICRO_COMPLEX_UNROLL_ITER2, 2)
      break;
#endif
#if MAX_COMPLEX_UNROLL > 1
    case 1:
      MICRO_COMPLEX_UNROLL_ITER(MICRO_COMPLEX_UNROLL_ITER2, 1)
      break;
#endif
    default:
      break;
  }
#undef MAX_COMPLEX_UNROLL
 
  if (remaining_rows > 0) {
    gemm_complex_extra_row<Scalar, Packet, Packetc, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal,
                           RhsIsReal>(res3, blockA, rhs_base, depth, strideA, offsetA, strideB, row, rows,
                                      remaining_rows, pAlphaReal, pAlphaImag, pMask);
  }
}

References advanceCols, col(), gemm_complex_extra_row(), MAX_COMPLEX_UNROLL, MICRO_COMPLEX_UNROLL_ITER, MICRO_COMPLEX_UNROLL_ITER2, MICRO_NEW_ROWS, res, row(), and rows.

Referenced by gemm_complex().

gemm_complex_extra_cols()

template<typename Scalar , typename Packet , typename Packetc , typename DataMapper , const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>

EIGEN_ALWAYS_INLINE void Eigen::internal::gemm_complex_extra_cols	(	const DataMapper &	res,
		const Scalar *	blockA,
		const Scalar *	blockB,
		Index	depth,
		Index	strideA,
		Index	offsetA,
		Index	strideB,
		Index	offsetB,
		Index	col,
		Index	rows,
		Index	cols,
		Index	remaining_rows,
		const Packet &	pAlphaReal,
		const Packet &	pAlphaImag,
		const Packet &	pMask
	)

                                                                      {
  MICRO_EXTRA(MICRO_COMPLEX_EXTRA_COLS, cols - col, true)
}

References col(), cols, MICRO_COMPLEX_EXTRA_COLS, and MICRO_EXTRA.

Referenced by gemm_complex(), and gemm_complexMMA().

gemm_complex_extra_row()

template<typename Scalar , typename Packet , typename Packetc , typename DataMapper , const Index accRows, const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>

EIGEN_ALWAYS_INLINE void Eigen::internal::gemm_complex_extra_row	(	const DataMapper &	res,
		const Scalar *	lhs_base,
		const Scalar *	rhs_base,
		Index	depth,
		Index	strideA,
		Index	offsetA,
		Index	strideB,
		Index	row,
		Index	rows,
		Index	remaining_rows,
		const Packet &	pAlphaReal,
		const Packet &	pAlphaImag,
		const Packet &	pMask
	)

                                                                                               {
  MICRO_EXTRA(MICRO_COMPLEX_EXTRA_ROWS, remaining_rows, false)
}

References MICRO_COMPLEX_EXTRA_ROWS, and MICRO_EXTRA.

Referenced by gemm_complex_cols().

gemm_complex_unrolled_iteration()

template<int unroll_factor, typename Scalar , typename Packet , typename Packetc , typename DataMapper , const Index accRows, const Index accCols, const Index accCols2, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>

EIGEN_ALWAYS_INLINE void Eigen::internal::gemm_complex_unrolled_iteration	(	const DataMapper &	res,
		const Scalar *	lhs_base,
		const Scalar *	rhs_base,
		Index	depth,
		Index	strideA,
		Index	offsetA,
		Index	strideB,
		Index &	row,
		const Packet &	pAlphaReal,
		const Packet &	pAlphaImag,
		const Packet &	pMask
	)

                                                                              {
  const Scalar *rhs_ptr_real0 = rhs_base, *rhs_ptr_real1 = NULL, *rhs_ptr_real2 = NULL;
  const Scalar *rhs_ptr_imag0 = NULL, *rhs_ptr_imag1 = NULL, *rhs_ptr_imag2 = NULL;
  const Index imag_delta = accCols * strideA;
  const Index imag_delta2 = accCols2 * strideA;
  const Scalar *lhs_ptr_real0 = NULL, *lhs_ptr_real1 = NULL;
  const Scalar *lhs_ptr_real2 = NULL, *lhs_ptr_real3 = NULL;
  PacketBlock<Packet, accRows> accReal0, accImag0, accReal1, accImag1;
  PacketBlock<Packet, accRows> accReal2, accImag2, accReal3, accImag3;
  PacketBlock<Packet, accRows> taccReal, taccImag;
  PacketBlock<Packetc, accRows> acc0, acc1;
  PacketBlock<Packetc, accRows * 2> tRes;
 
  MICRO_COMPLEX_SRC2_PTR
  MICRO_COMPLEX_SRC_PTR
  MICRO_COMPLEX_DST_PTR
 
  Index k = 0;
  for (; k + PEEL_COMPLEX <= depth; k += PEEL_COMPLEX) {
    MICRO_COMPLEX_PREFETCHN(accRows)
    MICRO_COMPLEX_PREFETCH
    MICRO_COMPLEX_ONE_PEEL4
  }
  for (; k < depth; k++) {
    MICRO_COMPLEX_ONE4
  }
  MICRO_COMPLEX_STORE
 
  MICRO_COMPLEX_UPDATE
}

References k, MICRO_COMPLEX_DST_PTR, MICRO_COMPLEX_ONE4, MICRO_COMPLEX_ONE_PEEL4, MICRO_COMPLEX_PREFETCH, MICRO_COMPLEX_PREFETCHN, MICRO_COMPLEX_SRC2_PTR, MICRO_COMPLEX_SRC_PTR, MICRO_COMPLEX_STORE, MICRO_COMPLEX_UPDATE, and PEEL_COMPLEX.

gemm_complex_unrolled_MMA_iteration()

template<int unroll_factor, typename Scalar , typename Packet , typename Packetc , typename RhsPacket , typename DataMapper , const Index accRows, const Index accCols, const Index accCols2, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal, const Index accItr>

EIGEN_ALWAYS_INLINE void Eigen::internal::gemm_complex_unrolled_MMA_iteration	(	const DataMapper &	res0,
		const DataMapper &	res1,
		const DataMapper &	res2,
		const DataMapper &	res3,
		const Scalar *	lhs_base,
		const Scalar *	rhs_base,
		Index	depth,
		Index	strideA,
		Index	offsetA,
		Index	strideB,
		Index &	row,
		const Packet &	pAlphaReal,
		const Packet &	pAlphaImag,
		const Packet &	pMask
	)

                                                                                                            {
  const Scalar *rhs_ptr_real0 = rhs_base, *rhs_ptr_real1 = NULL, *rhs_ptr_real2 = NULL, *rhs_ptr_real3 = NULL;
  const Scalar *rhs_ptr_imag0 = NULL, *rhs_ptr_imag1 = NULL, *rhs_ptr_imag2 = NULL, *rhs_ptr_imag3 = NULL;
  const Index imag_delta = accCols * strideA;
  const Index imag_delta2 = accCols2 * strideA;
 
  if (!RhsIsReal) {
    rhs_ptr_imag0 = rhs_base + accRows * strideB;
  } else {
    EIGEN_UNUSED_VARIABLE(rhs_ptr_imag0);
  }
  if (accItr > 1) {
    if (!RhsIsReal) {
      rhs_ptr_real1 = rhs_base + (2 * accRows * strideB);
      rhs_ptr_imag1 = rhs_base + (3 * accRows * strideB);
    } else {
      rhs_ptr_real1 = rhs_base + accRows * strideB;
      EIGEN_UNUSED_VARIABLE(rhs_ptr_imag1);
    }
  } else {
    EIGEN_UNUSED_VARIABLE(rhs_ptr_real1);
    EIGEN_UNUSED_VARIABLE(rhs_ptr_imag1);
    EIGEN_UNUSED_VARIABLE(res1);
  }
  if (accItr > 2) {
    if (!RhsIsReal) {
      rhs_ptr_real2 = rhs_base + (4 * accRows * strideB);
      rhs_ptr_imag2 = rhs_base + (5 * accRows * strideB);
      rhs_ptr_real3 = rhs_base + (6 * accRows * strideB);
      rhs_ptr_imag3 = rhs_base + (7 * accRows * strideB);
    } else {
      rhs_ptr_real2 = rhs_base + (2 * accRows * strideB);
      rhs_ptr_real3 = rhs_base + (3 * accRows * strideB);
      EIGEN_UNUSED_VARIABLE(rhs_ptr_imag2);
      EIGEN_UNUSED_VARIABLE(rhs_ptr_imag3);
    }
  } else {
    EIGEN_UNUSED_VARIABLE(rhs_ptr_real2);
    EIGEN_UNUSED_VARIABLE(rhs_ptr_real3);
    EIGEN_UNUSED_VARIABLE(rhs_ptr_imag2);
    EIGEN_UNUSED_VARIABLE(rhs_ptr_imag3);
    EIGEN_UNUSED_VARIABLE(res2);
    EIGEN_UNUSED_VARIABLE(res3);
  }
  const Scalar *lhs_ptr_real0 = NULL, *lhs_ptr_real1 = NULL;
  const Scalar *lhs_ptr_real2 = NULL, *lhs_ptr_real3 = NULL;
  __vector_quad accReal0, accImag0, accReal1, accImag1, accReal2, accImag2, accReal3, accImag3;
 
  MICRO_COMPLEX_MMA_SRC_PTR
  MICRO_COMPLEX_MMA_DST_PTR
 
  Index k = 0, depth2 = depth - PEEL_COMPLEX_MMA;
  for (; k <= depth2; k += PEEL_COMPLEX_MMA) {
    EIGEN_POWER_PREFETCH(rhs_ptr_real);
    if (!RhsIsReal) {
      EIGEN_POWER_PREFETCH(rhs_ptr_imag);
    }
    MICRO_COMPLEX_MMA_PREFETCH
    MICRO_COMPLEX_MMA_ONE_PEEL
  }
  for (; k < depth; k++) {
    MICRO_COMPLEX_MMA_ONE
  }
  MICRO_COMPLEX_MMA_STORE
 
  MICRO_COMPLEX_UPDATE
}

References EIGEN_POWER_PREFETCH, EIGEN_UNUSED_VARIABLE, k, MICRO_COMPLEX_MMA_DST_PTR, MICRO_COMPLEX_MMA_ONE, MICRO_COMPLEX_MMA_ONE_PEEL, MICRO_COMPLEX_MMA_PREFETCH, MICRO_COMPLEX_MMA_SRC_PTR, MICRO_COMPLEX_MMA_STORE, MICRO_COMPLEX_UPDATE, and PEEL_COMPLEX_MMA.

gemm_complexMMA()

template<typename LhsScalar , typename RhsScalar , typename Scalarc , typename Scalar , typename Packet , typename Packetc , typename RhsPacket , typename DataMapper , const Index accRows, const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>

void Eigen::internal::gemm_complexMMA	(	const DataMapper &	res,
		const LhsScalar *	blockAc,
		const RhsScalar *	blockBc,
		Index	rows,
		Index	depth,
		Index	cols,
		Scalarc	alpha,
		Index	strideA,
		Index	strideB,
		Index	offsetA,
		Index	offsetB
	)

                                                                                                            {
  const Index remaining_rows = rows % accCols;
 
  if (strideA == -1) strideA = depth;
  if (strideB == -1) strideB = depth;
 
  const Packet pAlphaReal = pset1<Packet>(alpha.real());
  const Packet pAlphaImag = pset1<Packet>(alpha.imag());
  const Packet pMask = bmask<Packet>(remaining_rows);
 
  const Scalar* blockA = (Scalar*)blockAc;
  const Scalar* blockB = (Scalar*)blockBc;
 
  typedef typename std::conditional_t<(sizeof(Scalar) == sizeof(float)), RhsPacket, __vector_pair> RhsPacket2;
 
  Index col = 0;
#ifdef GEMM_MULTIPLE_COLS
  MICRO_COMPLEX_MMA_COLS(4);
  MICRO_COMPLEX_MMA_COLS(2);
#endif
  MICRO_COMPLEX_MMA_COLS(1);
 
  if (col != cols) {
    gemm_complex_extra_cols<Scalar, Packet, Packetc, DataMapper, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal,
                            RhsIsReal>(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, cols,
                                       remaining_rows, pAlphaReal, pAlphaImag, pMask);
  }
}

References alpha, col(), cols, gemm_complex_extra_cols(), MICRO_COMPLEX_MMA_COLS, res, and rows.

Referenced by Eigen::internal::gebp_kernel< double, std::complex< double >, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >::operator()(), Eigen::internal::gebp_kernel< float, std::complex< float >, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >::operator()(), Eigen::internal::gebp_kernel< std::complex< double >, double, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >::operator()(), Eigen::internal::gebp_kernel< std::complex< double >, std::complex< double >, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >::operator()(), Eigen::internal::gebp_kernel< std::complex< float >, float, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >::operator()(), and Eigen::internal::gebp_kernel< std::complex< float >, std::complex< float >, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >::operator()().

gemm_extra_cols()

template<typename Scalar , typename Packet , typename DataMapper , const Index accCols>

EIGEN_ALWAYS_INLINE void Eigen::internal::gemm_extra_cols	(	const DataMapper &	res,
		const Scalar *	blockA,
		const Scalar *	blockB,
		Index	depth,
		Index	strideA,
		Index	offsetA,
		Index	strideB,
		Index	offsetB,
		Index	col,
		Index	rows,
		Index	cols,
		Index	remaining_rows,
		const Packet &	pAlpha,
		const Packet &	pMask
	)

                                                              {
  MICRO_EXTRA(MICRO_EXTRA_COLS, cols - col, true)
}

References col(), cols, MICRO_EXTRA, and MICRO_EXTRA_COLS.

gemm_extra_row()

template<typename Scalar , typename Packet , typename DataMapper , const Index accRows, const Index accCols>

EIGEN_ALWAYS_INLINE void Eigen::internal::gemm_extra_row	(	const DataMapper &	res,
		const Scalar *	lhs_base,
		const Scalar *	rhs_base,
		Index	depth,
		Index	strideA,
		Index	offsetA,
		Index	strideB,
		Index	row,
		Index	rows,
		Index	remaining_rows,
		const Packet &	pAlpha,
		const Packet &	pMask
	)

                                                                                                         {
  MICRO_EXTRA(MICRO_EXTRA_ROWS, remaining_rows, false)
}

References MICRO_EXTRA, and MICRO_EXTRA_ROWS.

gemm_kern_avx512()

template<typename Scalar , int max_a_unroll, int max_b_unroll, bool is_alpha1, bool is_beta0, bool is_unit_inc>

EIGEN_DONT_INLINE void Eigen::internal::gemm_kern_avx512	(	Index	m,
		Index	n,
		Index	k,
		Scalar *	alpha,
		const Scalar *	a,
		const Scalar *	b,
		Scalar *	c,
		Index	ldc,
		Index	inc = 1,
		Index	a_stride = -1,
		Index	b_stride = -1,
		Index	a_off = 0,
		Index	b_off = 0
	)

                                                                          {
  if (a_stride == -1) a_stride = k;
  if (b_stride == -1) b_stride = k;
 
  gemm_class<Scalar, is_unit_inc> g(m, n, k, ldc, inc, alpha, a, b, c, is_alpha1, is_beta0, a_stride, b_stride, a_off,
                                    b_off);
  g.template compute_kern<max_a_unroll, max_b_unroll>();
}

References a, alpha, b, calibrate::c, k, ldc, m, and n.

gemm_unrolled_complex_row_iteration()

template<typename Scalar , typename Packet , typename Packetc , typename DataMapper , const Index accRows, const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal, const Index remaining_rows>

EIGEN_ALWAYS_INLINE void Eigen::internal::gemm_unrolled_complex_row_iteration	(	const DataMapper &	res,
		const Scalar *	lhs_base,
		const Scalar *	rhs_base,
		Index	depth,
		Index	strideA,
		Index	offsetA,
		Index	strideB,
		Index	row,
		Index	rows,
		const Packet &	pAlphaReal,
		const Packet &	pAlphaImag,
		const Packet &	pMask
	)

                                                                                  {
  const Scalar *rhs_ptr_real0 = rhs_base, *rhs_ptr_real1 = NULL, *rhs_ptr_real2 = NULL;
  const Scalar *rhs_ptr_imag0 = NULL, *rhs_ptr_imag1 = NULL, *rhs_ptr_imag2 = NULL;
  const Scalar* lhs_ptr_real = lhs_base + advanceRows * row * strideA + remaining_rows * offsetA;
  const Scalar* lhs_ptr_imag = NULL;
  if (!LhsIsReal)
    lhs_ptr_imag = lhs_ptr_real + remaining_rows * strideA;
  else
    EIGEN_UNUSED_VARIABLE(lhs_ptr_imag);
  PacketBlock<Packet, accRows> accReal0, accImag0, accReal1, accImag1, accReal2, accImag2, accReal3, accImag3;
  PacketBlock<Packet, accRows> taccReal, taccImag;
  PacketBlock<Packetc, accRows> acc0, acc1;
  PacketBlock<Packetc, accRows * 2> tRes;
 
  MICRO_COMPLEX_SRC2_PTR
 
  bsetzero<Packet, accRows>(accReal0);
  bsetzero<Packet, accRows>(accImag0);
 
  Index remaining_depth = depth & -quad_traits<Scalar>::rows;
  Index k = 0;
  if (remaining_depth >= PEEL_COMPLEX_ROW) {
    MICRO_COMPLEX_ZERO_PEEL_ROW
    do {
      MICRO_COMPLEX_PREFETCHN(accRows)
      EIGEN_POWER_PREFETCH(lhs_ptr_real);
      if (!LhsIsReal) {
        EIGEN_POWER_PREFETCH(lhs_ptr_imag);
      }
      MICRO_COMPLEX_WORK_PEEL_ROW
    } while ((k += PEEL_COMPLEX_ROW) + PEEL_COMPLEX_ROW <= remaining_depth);
    MICRO_COMPLEX_ADD_PEEL_ROW
  }
  for (; k < depth; k++) {
    MICRO_COMPLEX_EXTRA_ROW<Scalar, Packet, accRows, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal, remaining_rows>(
        lhs_ptr_real, lhs_ptr_imag, rhs_ptr_real0, rhs_ptr_real1, rhs_ptr_real2, rhs_ptr_imag0, rhs_ptr_imag1,
        rhs_ptr_imag2, accReal0, accImag0);
  }
 
  constexpr bool full = (remaining_rows > accColsC);
  bload<DataMapper, Packetc, accColsC, ColMajor, true, accRows, full>(tRes, res, row, 0);
  if ((accRows == 1) || (rows >= accCols)) {
    bscalec<Packet, accRows, true>(accReal0, accImag0, pAlphaReal, pAlphaImag, taccReal, taccImag, pMask);
    bcouple<Packet, Packetc, accRows, full>(taccReal, taccImag, tRes, acc0, acc1);
    bstore<DataMapper, Packetc, accRows>(acc0, res, row + 0);
    if (full) {
      bstore<DataMapper, Packetc, accRows>(acc1, res, row + accColsC);
    }
  } else {
    bscalec<Packet, accRows, false>(accReal0, accImag0, pAlphaReal, pAlphaImag, taccReal, taccImag, pMask);
    bcouple<Packet, Packetc, accRows, full>(taccReal, taccImag, tRes, acc0, acc1);
 
    if ((sizeof(Scalar) == sizeof(float)) && (remaining_rows == 1)) {
      for (Index j = 0; j < accRows; j++) {
        res(row + 0, j) = pfirst<Packetc>(acc0.packet[j]);
      }
    } else {
      bstore<DataMapper, Packetc, accRows>(acc0, res, row + 0);
      if (full) {
        for (Index j = 0; j < accRows; j++) {
          res(row + accColsC, j) = pfirst<Packetc>(acc1.packet[j]);
        }
      }
    }
  }
}

References accColsC, advanceRows, EIGEN_POWER_PREFETCH, EIGEN_UNUSED_VARIABLE, j, k, MICRO_COMPLEX_ADD_PEEL_ROW, MICRO_COMPLEX_PREFETCHN, MICRO_COMPLEX_SRC2_PTR, MICRO_COMPLEX_WORK_PEEL_ROW, MICRO_COMPLEX_ZERO_PEEL_ROW, Eigen::internal::PacketBlock< Packet, N >::packet, PEEL_COMPLEX_ROW, res, row(), and rows.

gemm_unrolled_iteration()

template<int unroll_factor, typename Scalar , typename Packet , typename DataMapper , const Index accRows, const Index accCols, const Index accCols2>

EIGEN_ALWAYS_INLINE void Eigen::internal::gemm_unrolled_iteration	(	const DataMapper &	res,
		const Scalar *	lhs_base,
		const Scalar *	rhs_base,
		Index	depth,
		Index	strideA,
		Index	offsetA,
		Index	strideB,
		Index &	row,
		const Packet &	pAlpha,
		const Packet &	pMask
	)

  {
  const Scalar *rhs_ptr0 = rhs_base, *rhs_ptr1 = NULL, *rhs_ptr2 = NULL;
  const Scalar *lhs_ptr0 = NULL, *lhs_ptr1 = NULL, *lhs_ptr2 = NULL, *lhs_ptr3 = NULL, *lhs_ptr4 = NULL,
               *lhs_ptr5 = NULL, *lhs_ptr6 = NULL, *lhs_ptr7 = NULL;
  PacketBlock<Packet, accRows> accZero0, accZero1, accZero2, accZero3, accZero4, accZero5, accZero6, accZero7;
  PacketBlock<Packet, accRows> acc;
 
  MICRO_SRC2_PTR
  MICRO_SRC_PTR
  MICRO_DST_PTR
 
  Index k = 0;
  for (; k + PEEL <= depth; k += PEEL) {
    MICRO_PREFETCHN(accRows)
    MICRO_PREFETCH
    MICRO_ONE_PEEL4
  }
  for (; k < depth; k++) {
    MICRO_ONE4
  }
  MICRO_STORE
 
  MICRO_UPDATE
}

References k, MICRO_DST_PTR, MICRO_ONE4, MICRO_ONE_PEEL4, MICRO_PREFETCH, MICRO_PREFETCHN, MICRO_SRC2_PTR, MICRO_SRC_PTR, MICRO_STORE, MICRO_UPDATE, and PEEL.

gemm_unrolled_MMA_iteration()

template<int unroll_factor, typename Scalar , typename Packet , typename RhsPacket , typename DataMapper , const Index accRows, const Index accCols, bool full, const Index accItr>

EIGEN_ALWAYS_INLINE void Eigen::internal::gemm_unrolled_MMA_iteration	(	const DataMapper &	res0,
		const DataMapper &	res1,
		const DataMapper &	res2,
		const DataMapper &	res3,
		const Scalar *	lhs_base,
		const Scalar *	rhs_base,
		Index	depth,
		Index	strideA,
		Index	strideB,
		Index	offsetA,
		Index &	row,
		const Packet &	pAlpha,
		Index	accCols2
	)

                                                                                           {
  const Scalar *rhs_ptr0 = rhs_base, *rhs_ptr1 = NULL, *rhs_ptr2 = NULL, *rhs_ptr3 = NULL;
  const Scalar *lhs_ptr0 = NULL, *lhs_ptr1 = NULL, *lhs_ptr2 = NULL, *lhs_ptr3 = NULL, *lhs_ptr4 = NULL,
               *lhs_ptr5 = NULL, *lhs_ptr6 = NULL, *lhs_ptr7 = NULL;
  __vector_quad accZero0, accZero1, accZero2, accZero3, accZero4, accZero5, accZero6, accZero7;
 
  if (accItr > 1) {
    rhs_ptr1 = rhs_base + (accRows * strideB);
  } else {
    EIGEN_UNUSED_VARIABLE(strideB);
    EIGEN_UNUSED_VARIABLE(rhs_ptr1);
    EIGEN_UNUSED_VARIABLE(res1);
  }
  if (accItr > 2) {
    rhs_ptr2 = rhs_base + (2 * accRows * strideB);
    rhs_ptr3 = rhs_base + (3 * accRows * strideB);
  } else {
    EIGEN_UNUSED_VARIABLE(rhs_ptr2);
    EIGEN_UNUSED_VARIABLE(rhs_ptr3);
    EIGEN_UNUSED_VARIABLE(res2);
    EIGEN_UNUSED_VARIABLE(res3);
  }
 
  MICRO_MMA_SRC_PTR
  MICRO_MMA_DST_PTR
 
  Index k = 0, depth2 = depth - PEEL_MMA;
  for (; k <= depth2; k += PEEL_MMA) {
    EIGEN_POWER_PREFETCH(rhs_ptr);
    MICRO_MMA_PREFETCH
    MICRO_MMA_ONE_PEEL
  }
  for (; k < depth; k++) {
    MICRO_MMA_ONE
  }
  MICRO_MMA_STORE
 
  MICRO_UPDATE
}

References EIGEN_POWER_PREFETCH, EIGEN_UNUSED_VARIABLE, k, MICRO_MMA_DST_PTR, MICRO_MMA_ONE, MICRO_MMA_ONE_PEEL, MICRO_MMA_PREFETCH, MICRO_MMA_SRC_PTR, MICRO_MMA_STORE, MICRO_UPDATE, and PEEL_MMA.

gemm_unrolled_row_iteration()

template<typename Scalar , typename Packet , typename DataMapper , const Index accRows, const Index accCols, const Index remaining_rows>

EIGEN_ALWAYS_INLINE void Eigen::internal::gemm_unrolled_row_iteration	(	const DataMapper &	res,
		const Scalar *	lhs_base,
		const Scalar *	rhs_base,
		Index	depth,
		Index	strideA,
		Index	offsetA,
		Index	strideB,
		Index	row,
		Index	rows,
		const Packet &	pAlpha,
		const Packet &	pMask
	)

                                                                          {
  const Scalar *rhs_ptr0 = rhs_base, *rhs_ptr1 = NULL, *rhs_ptr2 = NULL;
  const Scalar* lhs_ptr = lhs_base + row * strideA + remaining_rows * offsetA;
  PacketBlock<Packet, accRows> accZero0, accZero1, accZero2, accZero3, accZero4, accZero5, accZero6, accZero7, acc;
 
  MICRO_SRC2_PTR
  bsetzero<Packet, accRows>(accZero0);
 
  Index remaining_depth = depth & -quad_traits<Scalar>::rows;
  Index k = 0;
  if (remaining_depth >= PEEL_ROW) {
    MICRO_ZERO_PEEL_ROW
    do {
      MICRO_PREFETCHN(accRows)
      EIGEN_POWER_PREFETCH(lhs_ptr);
      MICRO_WORK_PEEL_ROW
    } while ((k += PEEL_ROW) + PEEL_ROW <= remaining_depth);
    MICRO_ADD_PEEL_ROW
  }
  for (; k < depth; k++) {
    MICRO_EXTRA_ROW<Scalar, Packet, accRows, remaining_rows>(lhs_ptr, rhs_ptr0, rhs_ptr1, rhs_ptr2, accZero0);
  }
 
#ifdef USE_PARTIAL_PACKETS
  EIGEN_UNUSED_VARIABLE(rows);
  EIGEN_UNUSED_VARIABLE(pMask);
  bload_partial<DataMapper, Packet, 0, false, accRows>(acc, res, row, remaining_rows);
  bscale<Packet, accRows>(acc, accZero0, pAlpha);
  bstore_partial<DataMapper, Packet, accRows>(acc, res, row, remaining_rows);
#else
  bload<DataMapper, Packet, 0, ColMajor, false, accRows>(acc, res, row, 0);
  if ((accRows == 1) || (rows >= accCols)) {
    bscale<Packet, accRows, true>(acc, accZero0, pAlpha, pMask);
    bstore<DataMapper, Packet, accRows>(acc, res, row);
  } else {
    bscale<Packet, accRows, false>(acc, accZero0, pAlpha, pMask);
    for (Index j = 0; j < accRows; j++) {
      for (Index i = 0; i < remaining_rows; i++) {
        res(row + i, j) = acc.packet[j][i];
      }
    }
  }
#endif
}

References EIGEN_POWER_PREFETCH, EIGEN_UNUSED_VARIABLE, i, j, k, MICRO_ADD_PEEL_ROW, MICRO_PREFETCHN, MICRO_SRC2_PTR, MICRO_WORK_PEEL_ROW, MICRO_ZERO_PEEL_ROW, Eigen::internal::PacketBlock< Packet, N >::packet, PEEL_ROW, res, row(), and rows.

gemmbfloat16()

template<typename DataMapper >

void Eigen::internal::gemmbfloat16	(	const DataMapper &	res,
		const bfloat16 *	indexA,
		const bfloat16 *	indexB,
		Index	rows,
		Index	depth,
		Index	cols,
		bfloat16	alpha,
		Index	strideA,
		Index	strideB,
		Index	offsetA,
		Index	offsetB
	)

                                                                                                          {
  float falpha = Eigen::bfloat16_impl::bfloat16_to_float(alpha);
  const Packet4f pAlpha = pset1<Packet4f>(falpha);
 
  if (strideA == -1) strideA = depth;
  if (strideB == -1) strideB = depth;
 
  ei_declare_aligned_stack_constructed_variable(float, result, cols* rows, 0);
  ei_declare_aligned_stack_constructed_variable(float, indexB2, strideB* cols, 0);
  ei_declare_aligned_stack_constructed_variable(float, indexA2, ((strideA + 1) & -2) * 4 * 2, 0);
 
  convertArrayBF16toF32<DataMapper>(result, cols, rows, res);
  convertArrayPointerBF16toF32(indexB2, cols, strideB, const_cast<bfloat16*>(indexB));
 
  Index bigSuffix = 2 * 8 * (strideA - offsetA);
  float* indexBF32 = indexB2 + 4 * offsetB;
  offsetB *= 3;
  strideB *= 2;
 
  Index row = 0;
  // LHS (8x16) block
  while (row + 16 <= rows) {
    calcVSXColLoops<16>(indexA, indexA2, row, depth, cols, rows, pAlpha, indexBF32, strideA, strideB, offsetA, offsetB,
                        bigSuffix, result);
  }
  // LHS (8x8) block
  calcVSXColLoops<8>(indexA, indexA2, row, depth, cols, rows, pAlpha, indexBF32, strideA, strideB, offsetA, offsetB,
                     bigSuffix, result);
  // LHS (8x4) block
  calcVSXColLoops<4>(indexA, indexA2, row, depth, cols, rows, pAlpha, indexBF32, strideA, strideB, offsetA, offsetB,
                     bigSuffix, result);
  // extra rows
  if (rows & 3) {
    // This index is the beginning of remaining block.
    colVSXLoops<4, true>(depth, cols, rows, pAlpha, indexA, indexA2, indexBF32, strideA, strideB, offsetB,
                         result + row);
  }
 
  // Convert back to bfloat16
  convertArrayF32toBF16VSX<DataMapper>(result, cols, rows, res);
}

References alpha, Eigen::bfloat16_impl::bfloat16_to_float(), cols, convertArrayPointerBF16toF32(), ei_declare_aligned_stack_constructed_variable, pset1< Packet4f >(), res, row(), and rows.

gemmKernel()

template<typename Scalar , bool isARowMajor, bool isCRowMajor, bool isAdd, bool handleKRem>

void Eigen::internal::gemmKernel	(	Scalar *	A_arr,
		Scalar *	B_arr,
		Scalar *	C_arr,
		int64_t	M,
		int64_t	N,
		int64_t	K,
		int64_t	LDA,
		int64_t	LDB,
		int64_t	LDC
	)

GEMM like operation for trsm panel updates. Computes: C -= A*B K must be multiple of 4.

Unrolls used are {1,2,4,8}x{U1,U2,U3}; For good performance we want K to be large with M/N relatively small, but also large enough to use the {8,U3} unroll block.

isARowMajor: is A_arr row-major? isCRowMajor: is C_arr row-major? (B_arr is assumed to be row-major). isAdd: C += A*B or C -= A*B (used by trsm) handleKRem: Handle arbitrary K? This is not needed for trsm.

                             {
  using urolls = unrolls::gemm<Scalar, isAdd>;
  constexpr int64_t U3 = urolls::PacketSize * 3;
  constexpr int64_t U2 = urolls::PacketSize * 2;
  constexpr int64_t U1 = urolls::PacketSize * 1;
  using vec = typename std::conditional<std::is_same<Scalar, float>::value, vecFullFloat, vecFullDouble>::type;
  int64_t N_ = (N / U3) * U3;
  int64_t M_ = (M / EIGEN_AVX_MAX_NUM_ROW) * EIGEN_AVX_MAX_NUM_ROW;
  int64_t K_ = (K / EIGEN_AVX_MAX_K_UNROL) * EIGEN_AVX_MAX_K_UNROL;
  int64_t j = 0;
  for (; j < N_; j += U3) {
    constexpr int64_t EIGEN_AVX_MAX_B_LOAD = EIGEN_AVX_B_LOAD_SETS * 3;
    int64_t i = 0;
    for (; i < M_; i += EIGEN_AVX_MAX_NUM_ROW) {
      Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)], *B_t = &B_arr[0 * LDB + j];
      PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm;
      urolls::template setzero<3, EIGEN_AVX_MAX_NUM_ROW>(zmm);
      for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) {
        urolls::template microKernel<isARowMajor, 3, EIGEN_AVX_MAX_NUM_ROW, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_B_LOAD,
                                     EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB, LDA, zmm);
        B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
        EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
        else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
      }
      EIGEN_IF_CONSTEXPR(handleKRem) {
        for (int64_t k = K_; k < K; k++) {
          urolls::template microKernel<isARowMajor, 3, EIGEN_AVX_MAX_NUM_ROW, 1, EIGEN_AVX_B_LOAD_SETS * 3,
                                       EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB, LDA, zmm);
          B_t += LDB;
          EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
          else A_t += LDA;
        }
      }
      EIGEN_IF_CONSTEXPR(isCRowMajor) {
        urolls::template updateC<3, EIGEN_AVX_MAX_NUM_ROW>(&C_arr[i * LDC + j], LDC, zmm);
        urolls::template storeC<3, EIGEN_AVX_MAX_NUM_ROW>(&C_arr[i * LDC + j], LDC, zmm);
      }
      else {
        transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U3, false, false>(zmm, &C_arr[i + j * LDC], LDC);
      }
    }
    if (M - i >= 4) {  // Note: this block assumes EIGEN_AVX_MAX_NUM_ROW = 8. Should be removed otherwise
      Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)];
      Scalar *B_t = &B_arr[0 * LDB + j];
      PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm;
      urolls::template setzero<3, 4>(zmm);
      for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) {
        urolls::template microKernel<isARowMajor, 3, 4, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_B_LOAD_SETS * 3,
                                     EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB, LDA, zmm);
        B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
        EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
        else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
      }
      EIGEN_IF_CONSTEXPR(handleKRem) {
        for (int64_t k = K_; k < K; k++) {
          urolls::template microKernel<isARowMajor, 3, 4, 1, EIGEN_AVX_B_LOAD_SETS * 3, EIGEN_AVX_MAX_A_BCAST>(
              B_t, A_t, LDB, LDA, zmm);
          B_t += LDB;
          EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
          else A_t += LDA;
        }
      }
      EIGEN_IF_CONSTEXPR(isCRowMajor) {
        urolls::template updateC<3, 4>(&C_arr[i * LDC + j], LDC, zmm);
        urolls::template storeC<3, 4>(&C_arr[i * LDC + j], LDC, zmm);
      }
      else {
        transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U3, true, false>(zmm, &C_arr[i + j * LDC], LDC, 4);
      }
      i += 4;
    }
    if (M - i >= 2) {
      Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)];
      Scalar *B_t = &B_arr[0 * LDB + j];
      PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm;
      urolls::template setzero<3, 2>(zmm);
      for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) {
        urolls::template microKernel<isARowMajor, 3, 2, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_B_LOAD_SETS * 3,
                                     EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB, LDA, zmm);
        B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
        EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
        else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
      }
      EIGEN_IF_CONSTEXPR(handleKRem) {
        for (int64_t k = K_; k < K; k++) {
          urolls::template microKernel<isARowMajor, 3, 2, 1, EIGEN_AVX_B_LOAD_SETS * 3, EIGEN_AVX_MAX_A_BCAST>(
              B_t, A_t, LDB, LDA, zmm);
          B_t += LDB;
          EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
          else A_t += LDA;
        }
      }
      EIGEN_IF_CONSTEXPR(isCRowMajor) {
        urolls::template updateC<3, 2>(&C_arr[i * LDC + j], LDC, zmm);
        urolls::template storeC<3, 2>(&C_arr[i * LDC + j], LDC, zmm);
      }
      else {
        transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U3, true, false>(zmm, &C_arr[i + j * LDC], LDC, 2);
      }
      i += 2;
    }
    if (M - i > 0) {
      Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)];
      Scalar *B_t = &B_arr[0 * LDB + j];
      PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm;
      urolls::template setzero<3, 1>(zmm);
      {
        for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) {
          urolls::template microKernel<isARowMajor, 3, 1, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_B_LOAD_SETS * 3, 1>(
              B_t, A_t, LDB, LDA, zmm);
          B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
          EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
          else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
        }
        EIGEN_IF_CONSTEXPR(handleKRem) {
          for (int64_t k = K_; k < K; k++) {
            urolls::template microKernel<isARowMajor, 3, 1, 1, EIGEN_AVX_B_LOAD_SETS * 3, 1>(B_t, A_t, LDB, LDA, zmm);
            B_t += LDB;
            EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
            else A_t += LDA;
          }
        }
        EIGEN_IF_CONSTEXPR(isCRowMajor) {
          urolls::template updateC<3, 1>(&C_arr[i * LDC + j], LDC, zmm);
          urolls::template storeC<3, 1>(&C_arr[i * LDC + j], LDC, zmm);
        }
        else {
          transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U3, true, false>(zmm, &C_arr[i + j * LDC], LDC, 1);
        }
      }
    }
  }
  if (N - j >= U2) {
    constexpr int64_t EIGEN_AVX_MAX_B_LOAD = EIGEN_AVX_B_LOAD_SETS * 2;
    int64_t i = 0;
    for (; i < M_; i += EIGEN_AVX_MAX_NUM_ROW) {
      Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)], *B_t = &B_arr[0 * LDB + j];
      EIGEN_IF_CONSTEXPR(isCRowMajor) B_t = &B_arr[0 * LDB + j];
      PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm;
      urolls::template setzero<2, EIGEN_AVX_MAX_NUM_ROW>(zmm);
      for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) {
        urolls::template microKernel<isARowMajor, 2, EIGEN_AVX_MAX_NUM_ROW, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_B_LOAD,
                                     EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB, LDA, zmm);
        B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
        EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
        else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
      }
      EIGEN_IF_CONSTEXPR(handleKRem) {
        for (int64_t k = K_; k < K; k++) {
          urolls::template microKernel<isARowMajor, 2, EIGEN_AVX_MAX_NUM_ROW, 1, EIGEN_AVX_MAX_B_LOAD,
                                       EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB, LDA, zmm);
          B_t += LDB;
          EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
          else A_t += LDA;
        }
      }
      EIGEN_IF_CONSTEXPR(isCRowMajor) {
        urolls::template updateC<2, EIGEN_AVX_MAX_NUM_ROW>(&C_arr[i * LDC + j], LDC, zmm);
        urolls::template storeC<2, EIGEN_AVX_MAX_NUM_ROW>(&C_arr[i * LDC + j], LDC, zmm);
      }
      else {
        transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U2, false, false>(zmm, &C_arr[i + j * LDC], LDC);
      }
    }
    if (M - i >= 4) {  // Note: this block assumes EIGEN_AVX_MAX_NUM_ROW = 8. Should be removed otherwise
      Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)];
      Scalar *B_t = &B_arr[0 * LDB + j];
      PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm;
      urolls::template setzero<2, 4>(zmm);
      for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) {
        urolls::template microKernel<isARowMajor, 2, 4, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_B_LOAD,
                                     EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB, LDA, zmm);
        B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
        EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
        else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
      }
      EIGEN_IF_CONSTEXPR(handleKRem) {
        for (int64_t k = K_; k < K; k++) {
          urolls::template microKernel<isARowMajor, 2, 4, 1, EIGEN_AVX_MAX_B_LOAD, EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB,
                                                                                                          LDA, zmm);
          B_t += LDB;
          EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
          else A_t += LDA;
        }
      }
      EIGEN_IF_CONSTEXPR(isCRowMajor) {
        urolls::template updateC<2, 4>(&C_arr[i * LDC + j], LDC, zmm);
        urolls::template storeC<2, 4>(&C_arr[i * LDC + j], LDC, zmm);
      }
      else {
        transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U2, true, false>(zmm, &C_arr[i + j * LDC], LDC, 4);
      }
      i += 4;
    }
    if (M - i >= 2) {
      Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)];
      Scalar *B_t = &B_arr[0 * LDB + j];
      PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm;
      urolls::template setzero<2, 2>(zmm);
      for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) {
        urolls::template microKernel<isARowMajor, 2, 2, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_B_LOAD,
                                     EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB, LDA, zmm);
        B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
        EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
        else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
      }
      EIGEN_IF_CONSTEXPR(handleKRem) {
        for (int64_t k = K_; k < K; k++) {
          urolls::template microKernel<isARowMajor, 2, 2, 1, EIGEN_AVX_MAX_B_LOAD, EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB,
                                                                                                          LDA, zmm);
          B_t += LDB;
          EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
          else A_t += LDA;
        }
      }
      EIGEN_IF_CONSTEXPR(isCRowMajor) {
        urolls::template updateC<2, 2>(&C_arr[i * LDC + j], LDC, zmm);
        urolls::template storeC<2, 2>(&C_arr[i * LDC + j], LDC, zmm);
      }
      else {
        transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U2, true, false>(zmm, &C_arr[i + j * LDC], LDC, 2);
      }
      i += 2;
    }
    if (M - i > 0) {
      Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)];
      Scalar *B_t = &B_arr[0 * LDB + j];
      PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm;
      urolls::template setzero<2, 1>(zmm);
      for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) {
        urolls::template microKernel<isARowMajor, 2, 1, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_B_LOAD, 1>(B_t, A_t, LDB,
                                                                                                        LDA, zmm);
        B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
        EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
        else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
      }
      EIGEN_IF_CONSTEXPR(handleKRem) {
        for (int64_t k = K_; k < K; k++) {
          urolls::template microKernel<isARowMajor, 2, 1, 1, EIGEN_AVX_MAX_B_LOAD, 1>(B_t, A_t, LDB, LDA, zmm);
          B_t += LDB;
          EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
          else A_t += LDA;
        }
      }
      EIGEN_IF_CONSTEXPR(isCRowMajor) {
        urolls::template updateC<2, 1>(&C_arr[i * LDC + j], LDC, zmm);
        urolls::template storeC<2, 1>(&C_arr[i * LDC + j], LDC, zmm);
      }
      else {
        transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U2, true, false>(zmm, &C_arr[i + j * LDC], LDC, 1);
      }
    }
    j += U2;
  }
  if (N - j >= U1) {
    constexpr int64_t EIGEN_AVX_MAX_B_LOAD = EIGEN_AVX_B_LOAD_SETS * 1;
    int64_t i = 0;
    for (; i < M_; i += EIGEN_AVX_MAX_NUM_ROW) {
      Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)], *B_t = &B_arr[0 * LDB + j];
      PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm;
      urolls::template setzero<1, EIGEN_AVX_MAX_NUM_ROW>(zmm);
      for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) {
        urolls::template microKernel<isARowMajor, 1, EIGEN_AVX_MAX_NUM_ROW, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_B_LOAD,
                                     EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB, LDA, zmm);
        B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
        EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
        else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
      }
      EIGEN_IF_CONSTEXPR(handleKRem) {
        for (int64_t k = K_; k < K; k++) {
          urolls::template microKernel<isARowMajor, 1, EIGEN_AVX_MAX_NUM_ROW, 1, EIGEN_AVX_B_LOAD_SETS * 1,
                                       EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB, LDA, zmm);
          B_t += LDB;
          EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
          else A_t += LDA;
        }
      }
      EIGEN_IF_CONSTEXPR(isCRowMajor) {
        urolls::template updateC<1, EIGEN_AVX_MAX_NUM_ROW>(&C_arr[i * LDC + j], LDC, zmm);
        urolls::template storeC<1, EIGEN_AVX_MAX_NUM_ROW>(&C_arr[i * LDC + j], LDC, zmm);
      }
      else {
        transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U1, false, false>(zmm, &C_arr[i + j * LDC], LDC);
      }
    }
    if (M - i >= 4) {  // Note: this block assumes EIGEN_AVX_MAX_NUM_ROW = 8. Should be removed otherwise
      Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)];
      Scalar *B_t = &B_arr[0 * LDB + j];
      PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm;
      urolls::template setzero<1, 4>(zmm);
      for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) {
        urolls::template microKernel<isARowMajor, 1, 4, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_B_LOAD,
                                     EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB, LDA, zmm);
        B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
        EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
        else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
      }
      EIGEN_IF_CONSTEXPR(handleKRem) {
        for (int64_t k = K_; k < K; k++) {
          urolls::template microKernel<isARowMajor, 1, 4, 1, EIGEN_AVX_MAX_B_LOAD, EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB,
                                                                                                          LDA, zmm);
          B_t += LDB;
          EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
          else A_t += LDA;
        }
      }
      EIGEN_IF_CONSTEXPR(isCRowMajor) {
        urolls::template updateC<1, 4>(&C_arr[i * LDC + j], LDC, zmm);
        urolls::template storeC<1, 4>(&C_arr[i * LDC + j], LDC, zmm);
      }
      else {
        transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U1, true, false>(zmm, &C_arr[i + j * LDC], LDC, 4);
      }
      i += 4;
    }
    if (M - i >= 2) {
      Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)];
      Scalar *B_t = &B_arr[0 * LDB + j];
      PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm;
      urolls::template setzero<1, 2>(zmm);
      for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) {
        urolls::template microKernel<isARowMajor, 1, 2, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_B_LOAD,
                                     EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB, LDA, zmm);
        B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
        EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
        else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
      }
      EIGEN_IF_CONSTEXPR(handleKRem) {
        for (int64_t k = K_; k < K; k++) {
          urolls::template microKernel<isARowMajor, 1, 2, 1, EIGEN_AVX_MAX_B_LOAD, EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB,
                                                                                                          LDA, zmm);
          B_t += LDB;
          EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
          else A_t += LDA;
        }
      }
      EIGEN_IF_CONSTEXPR(isCRowMajor) {
        urolls::template updateC<1, 2>(&C_arr[i * LDC + j], LDC, zmm);
        urolls::template storeC<1, 2>(&C_arr[i * LDC + j], LDC, zmm);
      }
      else {
        transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U1, true, false>(zmm, &C_arr[i + j * LDC], LDC, 2);
      }
      i += 2;
    }
    if (M - i > 0) {
      Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)];
      Scalar *B_t = &B_arr[0 * LDB + j];
      PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm;
      urolls::template setzero<1, 1>(zmm);
      {
        for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) {
          urolls::template microKernel<isARowMajor, 1, 1, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_B_LOAD, 1>(B_t, A_t, LDB,
                                                                                                          LDA, zmm);
          B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
          EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
          else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
        }
        EIGEN_IF_CONSTEXPR(handleKRem) {
          for (int64_t k = K_; k < K; k++) {
            urolls::template microKernel<isARowMajor, 1, 1, 1, EIGEN_AVX_B_LOAD_SETS * 1, 1>(B_t, A_t, LDB, LDA, zmm);
            B_t += LDB;
            EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
            else A_t += LDA;
          }
        }
        EIGEN_IF_CONSTEXPR(isCRowMajor) {
          urolls::template updateC<1, 1>(&C_arr[i * LDC + j], LDC, zmm);
          urolls::template storeC<1, 1>(&C_arr[i * LDC + j], LDC, zmm);
        }
        else {
          transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U1, true, false>(zmm, &C_arr[i + j * LDC], LDC, 1);
        }
      }
    }
    j += U1;
  }
  if (N - j > 0) {
    constexpr int64_t EIGEN_AVX_MAX_B_LOAD = EIGEN_AVX_B_LOAD_SETS * 1;
    int64_t i = 0;
    for (; i < M_; i += EIGEN_AVX_MAX_NUM_ROW) {
      Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)];
      Scalar *B_t = &B_arr[0 * LDB + j];
      PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm;
      urolls::template setzero<1, EIGEN_AVX_MAX_NUM_ROW>(zmm);
      for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) {
        urolls::template microKernel<isARowMajor, 1, EIGEN_AVX_MAX_NUM_ROW, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_B_LOAD,
                                     EIGEN_AVX_MAX_A_BCAST, true>(B_t, A_t, LDB, LDA, zmm, N - j);
        B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
        EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
        else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
      }
      EIGEN_IF_CONSTEXPR(handleKRem) {
        for (int64_t k = K_; k < K; k++) {
          urolls::template microKernel<isARowMajor, 1, EIGEN_AVX_MAX_NUM_ROW, 1, EIGEN_AVX_MAX_B_LOAD,
                                       EIGEN_AVX_MAX_A_BCAST, true>(B_t, A_t, LDB, LDA, zmm, N - j);
          B_t += LDB;
          EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
          else A_t += LDA;
        }
      }
      EIGEN_IF_CONSTEXPR(isCRowMajor) {
        urolls::template updateC<1, EIGEN_AVX_MAX_NUM_ROW, true>(&C_arr[i * LDC + j], LDC, zmm, N - j);
        urolls::template storeC<1, EIGEN_AVX_MAX_NUM_ROW, true>(&C_arr[i * LDC + j], LDC, zmm, N - j);
      }
      else {
        transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U1, false, true>(zmm, &C_arr[i + j * LDC], LDC, 0, N - j);
      }
    }
    if (M - i >= 4) {  // Note: this block assumes EIGEN_AVX_MAX_NUM_ROW = 8. Should be removed otherwise
      Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)];
      Scalar *B_t = &B_arr[0 * LDB + j];
      PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm;
      urolls::template setzero<1, 4>(zmm);
      for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) {
        urolls::template microKernel<isARowMajor, 1, 4, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_B_LOAD,
                                     EIGEN_AVX_MAX_A_BCAST, true>(B_t, A_t, LDB, LDA, zmm, N - j);
        B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
        EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
        else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
      }
      EIGEN_IF_CONSTEXPR(handleKRem) {
        for (int64_t k = K_; k < K; k++) {
          urolls::template microKernel<isARowMajor, 1, 4, 1, EIGEN_AVX_MAX_B_LOAD, EIGEN_AVX_MAX_A_BCAST, true>(
              B_t, A_t, LDB, LDA, zmm, N - j);
          B_t += LDB;
          EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
          else A_t += LDA;
        }
      }
      EIGEN_IF_CONSTEXPR(isCRowMajor) {
        urolls::template updateC<1, 4, true>(&C_arr[i * LDC + j], LDC, zmm, N - j);
        urolls::template storeC<1, 4, true>(&C_arr[i * LDC + j], LDC, zmm, N - j);
      }
      else {
        transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U1, true, true>(zmm, &C_arr[i + j * LDC], LDC, 4, N - j);
      }
      i += 4;
    }
    if (M - i >= 2) {
      Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)];
      Scalar *B_t = &B_arr[0 * LDB + j];
      PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm;
      urolls::template setzero<1, 2>(zmm);
      for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) {
        urolls::template microKernel<isARowMajor, 1, 2, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_B_LOAD,
                                     EIGEN_AVX_MAX_A_BCAST, true>(B_t, A_t, LDB, LDA, zmm, N - j);
        B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
        EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
        else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
      }
      EIGEN_IF_CONSTEXPR(handleKRem) {
        for (int64_t k = K_; k < K; k++) {
          urolls::template microKernel<isARowMajor, 1, 2, 1, EIGEN_AVX_MAX_B_LOAD, EIGEN_AVX_MAX_A_BCAST, true>(
              B_t, A_t, LDB, LDA, zmm, N - j);
          B_t += LDB;
          EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
          else A_t += LDA;
        }
      }
      EIGEN_IF_CONSTEXPR(isCRowMajor) {
        urolls::template updateC<1, 2, true>(&C_arr[i * LDC + j], LDC, zmm, N - j);
        urolls::template storeC<1, 2, true>(&C_arr[i * LDC + j], LDC, zmm, N - j);
      }
      else {
        transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U1, true, true>(zmm, &C_arr[i + j * LDC], LDC, 2, N - j);
      }
      i += 2;
    }
    if (M - i > 0) {
      Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)];
      Scalar *B_t = &B_arr[0 * LDB + j];
      PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm;
      urolls::template setzero<1, 1>(zmm);
      for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) {
        urolls::template microKernel<isARowMajor, 1, 1, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_B_LOAD, 1, true>(
            B_t, A_t, LDB, LDA, zmm, N - j);
        B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
        EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
        else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
      }
      EIGEN_IF_CONSTEXPR(handleKRem) {
        for (int64_t k = K_; k < K; k++) {
          urolls::template microKernel<isARowMajor, 1, 1, 1, EIGEN_AVX_MAX_B_LOAD, 1, true>(B_t, A_t, LDB, LDA, zmm,
                                                                                            N - j);
          B_t += LDB;
          EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
          else A_t += LDA;
        }
      }
      EIGEN_IF_CONSTEXPR(isCRowMajor) {
        urolls::template updateC<1, 1, true>(&C_arr[i * LDC + j], LDC, zmm, N - j);
        urolls::template storeC<1, 1, true>(&C_arr[i * LDC + j], LDC, zmm, N - j);
      }
      else {
        transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U1, true, true>(zmm, &C_arr[i + j * LDC], LDC, 1, N - j);
      }
    }
  }
}

References EIGEN_AVX_B_LOAD_SETS, EIGEN_AVX_MAX_A_BCAST, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_NUM_ROW, EIGEN_IF_CONSTEXPR, i, j, k, PlanarWave::K, oomph::OcTreeNames::LDB, N, compute_granudrum_aor::type, and Eigen::value.

gemmMMA()

template<typename Scalar , typename Packet , typename RhsPacket , typename DataMapper , const Index accRows, const Index accCols>

void Eigen::internal::gemmMMA	(	const DataMapper &	res,
		const Scalar *	blockA,
		const Scalar *	blockB,
		Index	rows,
		Index	depth,
		Index	cols,
		Scalar	alpha,
		Index	strideA,
		Index	strideB,
		Index	offsetA,
		Index	offsetB
	)

                                                                                       {
  const Index remaining_rows = rows % accCols;
 
  if (strideA == -1) strideA = depth;
  if (strideB == -1) strideB = depth;
 
  const Packet pAlpha = pset1<Packet>(alpha);
  const Packet pMask = bmask<Packet>(remaining_rows);
 
  typedef typename std::conditional_t<(sizeof(Scalar) == sizeof(float)), RhsPacket, __vector_pair> RhsPacket2;
 
  Index col = 0;
#ifdef GEMM_MULTIPLE_COLS
  MICRO_MMA_COLS(4);
  MICRO_MMA_COLS(2);
#endif
  MICRO_MMA_COLS(1);
 
  if (col != cols) {
    gemm_extra_cols<Scalar, Packet, DataMapper, accCols>(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB,
                                                         col, rows, cols, remaining_rows, pAlpha, pMask);
  }
}

References alpha, col(), cols, MICRO_MMA_COLS, res, and rows.

gemmMMA_cols()

template<typename Scalar , typename Packet , typename RhsPacket , typename DataMapper , const Index accRows, const Index accCols, const Index accItr>

EIGEN_ALWAYS_INLINE void Eigen::internal::gemmMMA_cols	(	const DataMapper &	res,
		const Scalar *	blockA,
		const Scalar *	blockB,
		Index	depth,
		Index	strideA,
		Index	offsetA,
		Index	strideB,
		Index	offsetB,
		Index	col,
		Index	rows,
		Index	remaining_rows,
		const Packet &	pAlpha,
		const Packet &	pMask
	)

                                                                                                       {
  const DataMapper res30 = res.getSubMapper(0, col);
  const DataMapper res31 = (accItr > 1) ? res30.getSubMapper(0, accRows * 1) : res30;
  const DataMapper res32 = (accItr > 2) ? res30.getSubMapper(0, accRows * 2) : res30;
  const DataMapper res33 = (accItr > 2) ? res30.getSubMapper(0, accRows * 3) : res30;
 
  const Scalar* rhs_base = blockB + col * strideB + accRows * offsetB;
  const Scalar* lhs_base = blockA + accCols * offsetA;
  Index row = 0;
 
#define MAX_MMA_UNROLL 7
 
#if MAX_MMA_UNROLL < 2
  if (1) {
#elif MAX_MMA_UNROLL < 4
  if (accItr <= 2) {
#else
  if (accItr == 1) {
#endif
    MICRO_MMA_ROWS(MAX_MMA_UNROLL);
  } else if (accItr == 2) {
    MICRO_MMA_ROWS(4);
  } else {
    MICRO_MMA_ROWS(2);
  }
  switch ((rows - row) / accCols) {
#if MAX_MMA_UNROLL > 7
    case 7:
      if (accItr == 1) {
        MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 7)
      }
      break;
#endif
#if MAX_MMA_UNROLL > 6
    case 6:
      if (accItr == 1) {
        MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 6)
      }
      break;
#endif
#if MAX_MMA_UNROLL > 5
    case 5:
      if (accItr == 1) {
        MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 5)
      }
      break;
#endif
#if MAX_MMA_UNROLL > 4
    case 4:
      if (accItr == 1) {
        MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 4)
      }
      break;
#endif
#if MAX_MMA_UNROLL > 3
    case 3:
      if (accItr <= 2) {
        MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 3)
      }
      break;
#endif
#if MAX_MMA_UNROLL > 2
    case 2:
      if (accItr <= 2) {
        MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 2)
      }
      break;
#endif
#if MAX_MMA_UNROLL > 1
    case 1:
      MICRO_UNROLL_ITER(MICRO_MMA_UNROLL_ITER2, 1)
      break;
#endif
    default:
      break;
  }
#undef MAX_MMA_UNROLL
 
  if (remaining_rows > 0) {
    MICRO_MMA_UNROLL_ITER(MICRO_MMA_EXTRA_ROWS1, 0)
  }
}

References col(), MAX_MMA_UNROLL, MICRO_MMA_EXTRA_ROWS1, MICRO_MMA_ROWS, MICRO_MMA_UNROLL_ITER, MICRO_MMA_UNROLL_ITER2, MICRO_UNROLL_ITER, res, row(), and rows.

gemmMMA_complex_cols()

template<typename Scalar , typename Packet , typename Packetc , typename RhsPacket , typename DataMapper , const Index accRows, const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal, const Index accItr>

EIGEN_ALWAYS_INLINE void Eigen::internal::gemmMMA_complex_cols	(	const DataMapper &	res,
		const Scalar *	blockA,
		const Scalar *	blockB,
		Index	depth,
		Index	strideA,
		Index	offsetA,
		Index	strideB,
		Index	offsetB,
		Index	col,
		Index	rows,
		Index	remaining_rows,
		const Packet &	pAlphaReal,
		const Packet &	pAlphaImag,
		const Packet &	pMask
	)

                                                                                             {
  const DataMapper res30 = res.getSubMapper(0, col);
  const DataMapper res31 = (accItr > 1) ? res30.getSubMapper(0, accRows * 1) : res30;
  const DataMapper res32 = (accItr > 2) ? res30.getSubMapper(0, accRows * 2) : res30;
  const DataMapper res33 = (accItr > 2) ? res30.getSubMapper(0, accRows * 3) : res30;
 
  const Scalar* rhs_base = blockB + advanceCols * col * strideB + accRows * offsetB;
  const Scalar* lhs_base = blockA + accCols * offsetA;
  Index row = 0;
 
#define MAX_COMPLEX_MMA_UNROLL 4
 
#if MAX_COMPLEX_MMA_UNROLL < 2
  if (1) {
#elif MAX_COMPLEX_MMA_UNROLL < 4
  if (accItr <= 2) {
#else
  if (accItr == 1) {
#endif
    MICRO_COMPLEX_MMA_ROWS(MAX_COMPLEX_MMA_UNROLL);
  } else if (accItr == 2) {
    MICRO_COMPLEX_MMA_ROWS(2);
  } else {
    MICRO_COMPLEX_MMA_ROWS(1);
  }
  switch ((rows - row) / accCols) {
#if MAX_COMPLEX_MMA_UNROLL > 3
    case 3:
      if (accItr == 1) {
        MICRO_COMPLEX_UNROLL_ITER(MICRO_COMPLEX_MMA_UNROLL_ITER2, 3)
      }
      break;
#endif
#if MAX_COMPLEX_MMA_UNROLL > 2
    case 2:
      if (accItr == 1) {
        MICRO_COMPLEX_UNROLL_ITER(MICRO_COMPLEX_MMA_UNROLL_ITER2, 2)
      }
      break;
#endif
#if MAX_COMPLEX_MMA_UNROLL > 1
    case 1:
      if (accItr <= 2) {
        MICRO_COMPLEX_UNROLL_ITER(MICRO_COMPLEX_MMA_UNROLL_ITER2, 1)
      }
      break;
#endif
    default:
      break;
  }
#undef MAX_COMPLEX_MMA_UNROLL
 
  if (remaining_rows > 0) {
    MICRO_MMA_UNROLL_ITER(MICRO_COMPLEX_MMA_EXTRA_ROWS1, 0)
  }
}

References advanceCols, col(), MAX_COMPLEX_MMA_UNROLL, MICRO_COMPLEX_MMA_EXTRA_ROWS1, MICRO_COMPLEX_MMA_ROWS, MICRO_COMPLEX_MMA_UNROLL_ITER2, MICRO_COMPLEX_UNROLL_ITER, MICRO_MMA_UNROLL_ITER, res, row(), and rows.

gemmMMAbfloat16()

template<typename DataMapper >

void Eigen::internal::gemmMMAbfloat16	(	const DataMapper &	res,
		const bfloat16 *	indexA,
		const bfloat16 *	indexB,
		Index	rows,
		Index	depth,
		Index	cols,
		bfloat16	alpha,
		Index	strideA,
		Index	strideB,
		Index	offsetA,
		Index	offsetB
	)

                                                                                                             {
  float falpha = Eigen::bfloat16_impl::bfloat16_to_float(alpha);
  const Packet4f pAlpha = pset1<Packet4f>(falpha);
  ei_declare_aligned_stack_constructed_variable(float, result, cols* rows, 0);
 
  convertArrayBF16toF32<DataMapper>(result, cols, rows, res);
 
  if (strideA == -1) strideA = depth;
  if (strideB == -1) strideB = depth;
  // Packing is done in blocks.
  // There's 4 possible sizes of blocks
  // Blocks of 8 columns with 16 elements (8x16)
  // Blocks of 8 columns with 8 elements (8x8). This happens when there's 16 > rows >= 8
  // Blocks of 8 columns with 4 elements (8x4). This happens when there's 8 > rows >= 4
  // Blocks of 8 columns with < 4 elements. This happens when there's less than 4 remaining rows
 
  // Loop for LHS standard block (8x16)
  Index bigSuffix = (2 * 8) * (strideA - offsetA);
  indexB += 4 * offsetB;
  strideB *= 4;
  offsetB *= 3;
 
  Index row = 0;
  while (row + 16 <= rows) {
    calcColLoops<16>(indexA, row, depth, cols, rows, pAlpha, indexB, strideB, offsetA, offsetB, bigSuffix, result);
  }
  // LHS (8x8) block
  calcColLoops<8>(indexA, row, depth, cols, rows, pAlpha, indexB, strideB, offsetA, offsetB, bigSuffix, result);
  // LHS (8x4) block
  calcColLoops<4>(indexA, row, depth, cols, rows, pAlpha, indexB, strideB, offsetA, offsetB, bigSuffix, result);
  // extra rows
  if (rows & 3) {
    // This index is the beginning of remaining block.
    colLoops<4, true>(depth, cols, rows, pAlpha, indexA, indexB, strideB, offsetB, result + row);
  }
 
  // Convert back to bfloat16
  convertArrayF32toBF16<DataMapper>(result, cols, rows, res);
}

References alpha, Eigen::bfloat16_impl::bfloat16_to_float(), cols, ei_declare_aligned_stack_constructed_variable, pset1< Packet4f >(), res, row(), and rows.

gemvMMA_bfloat16_col()

template<typename LhsMapper , typename RhsMapper >

void Eigen::internal::gemvMMA_bfloat16_col	(	Index	rows,
		Index	cols,
		const LhsMapper &	alhs,
		const RhsMapper &	rhs,
		bfloat16 *	res,
		Index	resIncr,
		bfloat16	alpha
	)

                                                         {
  EIGEN_UNUSED_VARIABLE(resIncr);
  eigen_internal_assert(resIncr == 1);
 
  // The following copy tells the compiler that lhs's attributes are not modified outside this function
  // This helps GCC to generate proper code.
  LhsMapper lhs(alhs);
  RhsMapper rhs2(rhs);
 
  const Index lhsStride = lhs.stride();
 
  // TODO: improve the following heuristic:
  const Index block_cols = cols < 128 ? cols : (lhsStride * sizeof(bfloat16) < 16000 ? 16 : 8);
  float falpha = Eigen::bfloat16_impl::bfloat16_to_float(alpha);
  Packet4f pAlpha = pset1<Packet4f>(falpha);
 
  ei_declare_aligned_stack_constructed_variable(float, result, rows, 0);
 
  convertArrayPointerBF16toF32(result, 1, rows, res);
 
  for (Index j2 = 0; j2 < cols; j2 += block_cols) {
    Index jend = numext::mini(j2 + block_cols, cols);
 
    using LhsSubMapper = typename LhsMapper::SubMapper;
 
    LhsSubMapper lhs2 = lhs.getSubMapper(0, j2);
    UseMMAStride<RhsMapper, LhsSubMapper>::run(j2, jend, rows, lhs2, rhs2, pAlpha, result);
  }
 
  convertArrayPointerF32toBF16(result, rows, res);
}

References alpha, Eigen::bfloat16_impl::bfloat16_to_float(), cols, convertArrayPointerBF16toF32(), convertArrayPointerF32toBF16(), ei_declare_aligned_stack_constructed_variable, eigen_internal_assert, EIGEN_UNUSED_VARIABLE, Eigen::numext::mini(), pset1< Packet4f >(), res, rows, and Eigen::internal::UseMMAStride< RhsMapper, LhsMapper, typename >::run().

gemvMMA_bfloat16_row()

template<typename LhsMapper , typename RhsMapper >

EIGEN_STRONG_INLINE void Eigen::internal::gemvMMA_bfloat16_row	(	Index	rows,
		Index	cols,
		const LhsMapper &	alhs,
		const RhsMapper &	rhs,
		bfloat16 *	res,
		Index	resIncr,
		bfloat16	alpha
	)

                                                                                            {
  typedef typename RhsMapper::LinearMapper LinearMapper;
 
  // The following copy tells the compiler that lhs's attributes are not modified outside this function
  // This helps GCC to generate proper code.
  LhsMapper lhs(alhs);
  LinearMapper rhs2 = rhs.getLinearMapper(0, 0);
 
  eigen_internal_assert(rhs.stride() == 1);
 
  float falpha = Eigen::bfloat16_impl::bfloat16_to_float(alpha);
  const Packet4f pAlpha = pset1<Packet4f>(falpha);
 
  ei_declare_aligned_stack_constructed_variable(float, result, rows, 0);
  if (resIncr == 1) {
    convertArrayPointerBF16toF32(result, 1, rows, res);
  } else {
    convertArrayPointerBF16toF32<true>(result, 1, rows, res, resIncr);
  }
  calcVecLoops<LhsMapper, LinearMapper>(cols, rows, lhs, rhs2, pAlpha, result);
  if (resIncr == 1) {
    convertArrayPointerF32toBF16(result, rows, res);
  } else {
    convertArrayPointerF32toBF16<true>(result, rows, res, resIncr);
  }
}

References alpha, Eigen::bfloat16_impl::bfloat16_to_float(), cols, convertArrayPointerBF16toF32(), convertArrayPointerF32toBF16(), ei_declare_aligned_stack_constructed_variable, eigen_internal_assert, pset1< Packet4f >(), res, and rows.

general_det3_helper()

template<typename Derived >

EIGEN_DEVICE_FUNC const Derived::Scalar Eigen::internal::general_det3_helper ( const MatrixBase< Derived > &  matrix, int  i1, int  i2, int  i3, int  j1, int  j2, int  j3  )

inline

                                                                                                                    {
  return matrix.coeff(i1, j1) *
         (matrix.coeff(i2, j2) * matrix.coeff(i3, j3) - matrix.coeff(i2, j3) * matrix.coeff(i3, j2));
}

References matrix().

Referenced by cofactor_4x4().

generic_atan()

template<typename Packet >

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::generic_atan

(

const Packet &

x

)

Returns

atan(x)

                                                                                            {
  typedef typename unpacket_traits<Packet>::type Scalar;
 
  constexpr Scalar kPiOverTwo = static_cast<Scalar>(EIGEN_PI / 2);
 
  const Packet cst_signmask = pset1<Packet>(-Scalar(0));
  const Packet cst_one = pset1<Packet>(Scalar(1));
  const Packet cst_pi_over_two = pset1<Packet>(kPiOverTwo);
 
  //   "Large": For |x| > 1, use atan(1/x) = sign(x)*pi/2 - atan(x).
  //   "Small": For |x| <= 1, approximate atan(x) directly by a polynomial
  //            calculated using Rminimax.
 
  const Packet abs_x = pabs(x_in);
  const Packet x_signmask = pand(x_in, cst_signmask);
  const Packet large_mask = pcmp_lt(cst_one, abs_x);
  const Packet x = pselect(large_mask, preciprocal(abs_x), abs_x);
  const Packet p = patan_reduced<Scalar>::run(x);
  // Apply transformations according to the range reduction masks.
  Packet result = pselect(large_mask, psub(cst_pi_over_two, p), p);
  // Return correct sign
  return pxor(result, x_signmask);
}

References EIGEN_PI, p, pabs(), pand(), pcmp_lt(), preciprocal(), pselect(), psub(), pxor(), Eigen::internal::patan_reduced< Scalar >::run(), and plotDoE::x.

generic_ceil()

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet Eigen::internal::generic_ceil

(

const Packet &

a

)

                                                                           {
  using Scalar = typename unpacket_traits<Packet>::type;
  const Packet cst_1 = pset1<Packet>(Scalar(1));
  const Packet sign_mask = pset1<Packet>(static_cast<Scalar>(-0.0));
  Packet rint_a = generic_rint(a);
  // if rint(a) < a, then rint(a) == floor(a)
  Packet mask = pcmp_lt(rint_a, a);
  Packet offset = pand(cst_1, mask);
  Packet result = padd(rint_a, offset);
  // Signed zero must remain signed (e.g. ceil(-0.02) == -0).
  result = por(result, pand(sign_mask, a));
  return result;
}

References a, generic_rint(), padd(), pand(), pcmp_lt(), and por().

Referenced by Eigen::internal::nearest_integer_packetop_impl< Packet, false, false >::run_ceil().

generic_exp2()

template<typename Packet >

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::generic_exp2

(

const Packet &

x

)

Returns

exp2(x)

                                                                                          {
  typedef typename unpacket_traits<Packet>::type Scalar;
  constexpr int max_exponent = std::numeric_limits<Scalar>::max_exponent;
  constexpr int digits = std::numeric_limits<Scalar>::digits;
  constexpr Scalar max_cap = Scalar(max_exponent + 1);
  constexpr Scalar min_cap = -Scalar(max_exponent + digits - 1);
  Packet x = pmax(pmin(_x, pset1<Packet>(max_cap)), pset1<Packet>(min_cap));
  Packet p_hi, p_lo;
  twoprod(pset1<Packet>(Scalar(EIGEN_LN2)), x, p_hi, p_lo);
  Packet exp2_hi = pexp(p_hi);
  Packet exp2_lo = padd(pset1<Packet>(Scalar(1)), p_lo);
  return pmul(exp2_hi, exp2_lo);
}

References EIGEN_LN2, padd(), pexp(), pmax(), pmin(), pmul(), twoprod(), and plotDoE::x.

Referenced by generic_pow_impl(), and pexp2< Packet8bf >().

generic_expm1()

template<typename Packet >

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::generic_expm1

(

const Packet &

x

)

Returns

exp(x)-1 computed using W. Kahan's formula. See: http://www.plunk.org/~hatch/rightway.php

Returns

exp(x)-1

                                                                                          {
  typedef typename unpacket_traits<Packet>::type ScalarType;
  const Packet one = pset1<Packet>(ScalarType(1));
  const Packet neg_one = pset1<Packet>(ScalarType(-1));
  Packet u = pexp(x);
  Packet one_mask = pcmp_eq(u, one);
  Packet u_minus_one = psub(u, one);
  Packet neg_one_mask = pcmp_eq(u_minus_one, neg_one);
  Packet logu = plog(u);
  // The following comparison is to catch the case where
  // exp(x) = +inf. It is written in this way to avoid having
  // to form the constant +inf, which depends on the packet
  // type.
  Packet pos_inf_mask = pcmp_eq(logu, u);
  Packet expm1 = pmul(u_minus_one, pdiv(x, logu));
  expm1 = pselect(pos_inf_mask, u, expm1);
  return pselect(one_mask, x, pselect(neg_one_mask, neg_one, expm1));
}

References Eigen::bfloat16_impl::expm1(), pcmp_eq(), pdiv(), pexp(), plog(), pmul(), pselect(), psub(), and plotDoE::x.

generic_floor()

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet Eigen::internal::generic_floor

(

const Packet &

a

)

                                                                            {
  using Scalar = typename unpacket_traits<Packet>::type;
  const Packet cst_1 = pset1<Packet>(Scalar(1));
  Packet rint_a = generic_rint(a);
  // if a < rint(a), then rint(a) == ceil(a)
  Packet mask = pcmp_lt(a, rint_a);
  Packet offset = pand(cst_1, mask);
  Packet result = psub(rint_a, offset);
  return result;
}

References a, generic_rint(), pand(), pcmp_lt(), and psub().

Referenced by generic_round(), generic_trunc(), and Eigen::internal::nearest_integer_packetop_impl< Packet, false, false >::run_floor().

generic_log1p()

template<typename Packet >

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::generic_log1p

(

const Packet &

x

)

Returns

log(1 + x) computed using W. Kahan's formula. See: http://www.plunk.org/~hatch/rightway.php

Returns

log(1 + x)

                                                                                          {
  typedef typename unpacket_traits<Packet>::type ScalarType;
  const Packet one = pset1<Packet>(ScalarType(1));
  Packet xp1 = padd(x, one);
  Packet small_mask = pcmp_eq(xp1, one);
  Packet log1 = plog(xp1);
  Packet inf_mask = pcmp_eq(xp1, log1);
  Packet log_large = pmul(x, pdiv(log1, psub(xp1, one)));
  return pselect(por(small_mask, inf_mask), x, log_large);
}

References padd(), pcmp_eq(), pdiv(), plog(), pmul(), por(), pselect(), psub(), and plotDoE::x.

generic_ndtri()

template<typename T , typename ScalarType >

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T Eigen::internal::generic_ndtri

(

const T &

a

)

                                                                  {
  const T maxnum = pset1<T>(NumTraits<ScalarType>::infinity());
  const T neg_maxnum = pset1<T>(-NumTraits<ScalarType>::infinity());
 
  const T zero = pset1<T>(ScalarType(0));
  const T one = pset1<T>(ScalarType(1));
  // exp(-2)
  const T exp_neg_two = pset1<T>(ScalarType(0.13533528323661269189));
  T b, ndtri, should_flipsign;
 
  should_flipsign = pcmp_le(a, psub(one, exp_neg_two));
  b = pselect(should_flipsign, a, psub(one, a));
 
  ndtri = pselect(pcmp_lt(exp_neg_two, b), generic_ndtri_gt_exp_neg_two<T, ScalarType>(b),
                  generic_ndtri_lt_exp_neg_two<T, ScalarType>(b, should_flipsign));
 
  return pselect(pcmp_eq(a, zero), neg_maxnum, pselect(pcmp_eq(one, a), maxnum, ndtri));
}

References a, Eigen::numext::b, pcmp_eq(), pcmp_le(), pcmp_lt(), pselect(), psub(), and zero().

Referenced by pndtri().

generic_ndtri_gt_exp_neg_two()

template<typename T , typename ScalarType >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T Eigen::internal::generic_ndtri_gt_exp_neg_two

(

const T &

b

)

                                                                                 {
  const ScalarType p0[] = {ScalarType(-5.99633501014107895267e1), ScalarType(9.80010754185999661536e1),
                           ScalarType(-5.66762857469070293439e1), ScalarType(1.39312609387279679503e1),
                           ScalarType(-1.23916583867381258016e0)};
  const ScalarType q0[] = {ScalarType(1.0),
                           ScalarType(1.95448858338141759834e0),
                           ScalarType(4.67627912898881538453e0),
                           ScalarType(8.63602421390890590575e1),
                           ScalarType(-2.25462687854119370527e2),
                           ScalarType(2.00260212380060660359e2),
                           ScalarType(-8.20372256168333339912e1),
                           ScalarType(1.59056225126211695515e1),
                           ScalarType(-1.18331621121330003142e0)};
  const T sqrt2pi = pset1<T>(ScalarType(2.50662827463100050242e0));
  const T half = pset1<T>(ScalarType(0.5));
  T c, c2, ndtri_gt_exp_neg_two;
 
  c = psub(b, half);
  c2 = pmul(c, c);
  ndtri_gt_exp_neg_two =
      pmadd(c, pmul(c2, pdiv(internal::ppolevl<T, 4>::run(c2, p0), internal::ppolevl<T, 8>::run(c2, q0))), c);
  return pmul(ndtri_gt_exp_neg_two, sqrt2pi);
}

References Eigen::numext::b, calibrate::c, p0, pdiv(), pmadd(), pmul(), and psub().

generic_ndtri_lt_exp_neg_two()

template<typename T , typename ScalarType >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T Eigen::internal::generic_ndtri_lt_exp_neg_two	(	const T &	b,
		const T &	should_flipsign
	)

                                                                                                           {
  /* Approximation for interval z = sqrt(-2 log a ) between 2 and 8
   * i.e., a between exp(-2) = .135 and exp(-32) = 1.27e-14.
   */
  const ScalarType p1[] = {ScalarType(4.05544892305962419923e0),   ScalarType(3.15251094599893866154e1),
                           ScalarType(5.71628192246421288162e1),   ScalarType(4.40805073893200834700e1),
                           ScalarType(1.46849561928858024014e1),   ScalarType(2.18663306850790267539e0),
                           ScalarType(-1.40256079171354495875e-1), ScalarType(-3.50424626827848203418e-2),
                           ScalarType(-8.57456785154685413611e-4)};
  const ScalarType q1[] = {ScalarType(1.0),
                           ScalarType(1.57799883256466749731e1),
                           ScalarType(4.53907635128879210584e1),
                           ScalarType(4.13172038254672030440e1),
                           ScalarType(1.50425385692907503408e1),
                           ScalarType(2.50464946208309415979e0),
                           ScalarType(-1.42182922854787788574e-1),
                           ScalarType(-3.80806407691578277194e-2),
                           ScalarType(-9.33259480895457427372e-4)};
  /* Approximation for interval z = sqrt(-2 log a ) between 8 and 64
   * i.e., a between exp(-32) = 1.27e-14 and exp(-2048) = 3.67e-890.
   */
  const ScalarType p2[] = {ScalarType(3.23774891776946035970e0),  ScalarType(6.91522889068984211695e0),
                           ScalarType(3.93881025292474443415e0),  ScalarType(1.33303460815807542389e0),
                           ScalarType(2.01485389549179081538e-1), ScalarType(1.23716634817820021358e-2),
                           ScalarType(3.01581553508235416007e-4), ScalarType(2.65806974686737550832e-6),
                           ScalarType(6.23974539184983293730e-9)};
  const ScalarType q2[] = {ScalarType(1.0),
                           ScalarType(6.02427039364742014255e0),
                           ScalarType(3.67983563856160859403e0),
                           ScalarType(1.37702099489081330271e0),
                           ScalarType(2.16236993594496635890e-1),
                           ScalarType(1.34204006088543189037e-2),
                           ScalarType(3.28014464682127739104e-4),
                           ScalarType(2.89247864745380683936e-6),
                           ScalarType(6.79019408009981274425e-9)};
  const T eight = pset1<T>(ScalarType(8.0));
  const T neg_two = pset1<T>(ScalarType(-2));
  T x, x0, x1, z;
 
  x = psqrt(pmul(neg_two, plog(b)));
  x0 = psub(x, pdiv(plog(x), x));
  z = preciprocal(x);
  x1 =
      pmul(z, pselect(pcmp_lt(x, eight), pdiv(internal::ppolevl<T, 8>::run(z, p1), internal::ppolevl<T, 8>::run(z, q1)),
                      pdiv(internal::ppolevl<T, 8>::run(z, p2), internal::ppolevl<T, 8>::run(z, q2))));
  return flipsign(should_flipsign, psub(x0, x1));
}

References Eigen::numext::b, flipsign(), p1, pcmp_lt(), pdiv(), plog(), pmul(), preciprocal(), pselect(), psqrt(), psub(), Eigen::numext::x, Global::x0, and Global_parameters::x1().

generic_pow()

template<typename Packet >

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::generic_pow	(	const Packet &	x,
		const Packet &	y
	)

                                                                                                         {
  typedef typename unpacket_traits<Packet>::type Scalar;
 
  const Packet cst_inf = pset1<Packet>(NumTraits<Scalar>::infinity());
  const Packet cst_zero = pset1<Packet>(Scalar(0));
  const Packet cst_one = pset1<Packet>(Scalar(1));
  const Packet cst_nan = pset1<Packet>(NumTraits<Scalar>::quiet_NaN());
 
  const Packet x_abs = pabs(x);
  Packet pow = generic_pow_impl(x_abs, y);
 
  // In the following we enforce the special case handling prescribed in
  // https://en.cppreference.com/w/cpp/numeric/math/pow.
 
  // Predicates for sign and magnitude of x.
  const Packet x_is_negative = pcmp_lt(x, cst_zero);
  const Packet x_is_zero = pcmp_eq(x, cst_zero);
  const Packet x_is_one = pcmp_eq(x, cst_one);
  const Packet x_has_signbit = psignbit(x);
  const Packet x_abs_gt_one = pcmp_lt(cst_one, x_abs);
  const Packet x_abs_is_inf = pcmp_eq(x_abs, cst_inf);
 
  // Predicates for sign and magnitude of y.
  const Packet y_abs = pabs(y);
  const Packet y_abs_is_inf = pcmp_eq(y_abs, cst_inf);
  const Packet y_is_negative = pcmp_lt(y, cst_zero);
  const Packet y_is_zero = pcmp_eq(y, cst_zero);
  const Packet y_is_one = pcmp_eq(y, cst_one);
  // Predicates for whether y is integer and odd/even.
  const Packet y_is_int = pandnot(pcmp_eq(pfloor(y), y), y_abs_is_inf);
  const Packet y_div_2 = pmul(y, pset1<Packet>(Scalar(0.5)));
  const Packet y_is_even = pcmp_eq(pround(y_div_2), y_div_2);
  const Packet y_is_odd_int = pandnot(y_is_int, y_is_even);
  // Smallest exponent for which (1 + epsilon) overflows to infinity.
  EIGEN_CONSTEXPR Scalar huge_exponent =
      (NumTraits<Scalar>::max_exponent() * Scalar(EIGEN_LN2)) / NumTraits<Scalar>::epsilon();
  const Packet y_abs_is_huge = pcmp_le(pset1<Packet>(huge_exponent), y_abs);
 
  // *  pow(base, exp) returns NaN if base is finite and negative
  //    and exp is finite and non-integer.
  pow = pselect(pandnot(x_is_negative, y_is_int), cst_nan, pow);
 
  // * pow(±0, exp), where exp is negative, finite, and is an even integer or
  // a non-integer, returns +∞
  // * pow(±0, exp), where exp is positive non-integer or a positive even
  // integer, returns +0
  // * pow(+0, exp), where exp is a negative odd integer, returns +∞
  // * pow(-0, exp), where exp is a negative odd integer, returns -∞
  // * pow(+0, exp), where exp is a positive odd integer, returns +0
  // * pow(-0, exp), where exp is a positive odd integer, returns -0
  // Sign is flipped by the rule below.
  pow = pselect(x_is_zero, pselect(y_is_negative, cst_inf, cst_zero), pow);
 
  // pow(base, exp) returns -pow(abs(base), exp) if base has the sign bit set,
  // and exp is an odd integer exponent.
  pow = pselect(pand(x_has_signbit, y_is_odd_int), pnegate(pow), pow);
 
  // * pow(base, -∞) returns +∞ for any |base|<1
  // * pow(base, -∞) returns +0 for any |base|>1
  // * pow(base, +∞) returns +0 for any |base|<1
  // * pow(base, +∞) returns +∞ for any |base|>1
  // * pow(±0, -∞) returns +∞
  // * pow(-1, +-∞) = 1
  Packet inf_y_val = pselect(por(pand(y_is_negative, x_is_zero), pxor(y_is_negative, x_abs_gt_one)), cst_inf, cst_zero);
  inf_y_val = pselect(pcmp_eq(x, pset1<Packet>(Scalar(-1.0))), cst_one, inf_y_val);
  pow = pselect(y_abs_is_huge, inf_y_val, pow);
 
  // * pow(+∞, exp) returns +0 for any negative exp
  // * pow(+∞, exp) returns +∞ for any positive exp
  // * pow(-∞, exp) returns -0 if exp is a negative odd integer.
  // * pow(-∞, exp) returns +0 if exp is a negative non-integer or negative
  //     even integer.
  // * pow(-∞, exp) returns -∞ if exp is a positive odd integer.
  // * pow(-∞, exp) returns +∞ if exp is a positive non-integer or positive
  //     even integer.
  auto x_pos_inf_value = pselect(y_is_negative, cst_zero, cst_inf);
  auto x_neg_inf_value = pselect(y_is_odd_int, pnegate(x_pos_inf_value), x_pos_inf_value);
  pow = pselect(x_abs_is_inf, pselect(x_is_negative, x_neg_inf_value, x_pos_inf_value), pow);
 
  // All cases of NaN inputs return NaN, except the two below.
  pow = pselect(por(pisnan(x), pisnan(y)), cst_nan, pow);
 
  // * pow(base, 1) returns base.
  // * pow(base, +/-0) returns 1, regardless of base, even NaN.
  // * pow(+1, exp) returns 1, regardless of exponent, even NaN.
  pow = pselect(y_is_one, x, pselect(por(x_is_one, y_is_zero), cst_one, pow));
 
  return pow;
}

References EIGEN_CONSTEXPR, EIGEN_LN2, generic_pow_impl(), pabs(), pand(), pandnot(), pcmp_eq(), pcmp_le(), pcmp_lt(), pfloor(), pisnan(), pmul(), pnegate(), por(), Eigen::bfloat16_impl::pow(), pround(), pselect(), psignbit(), pxor(), plotDoE::x, and y.

Referenced by Eigen::internal::scalar_pow_op< Scalar, Exponent >::packetOp(), and Eigen::internal::unary_pow_impl< Packet, ScalarExponent, false, false, ExponentIsSigned >::run().

generic_pow_impl()

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet Eigen::internal::generic_pow_impl	(	const Packet &	x,
		const Packet &	y
	)

                                                                                                {
  typedef typename unpacket_traits<Packet>::type Scalar;
  // Split x into exponent e_x and mantissa m_x.
  Packet e_x;
  Packet m_x = pfrexp(x, e_x);
 
  // Adjust m_x to lie in [1/sqrt(2):sqrt(2)] to minimize absolute error in log2(m_x).
  EIGEN_CONSTEXPR Scalar sqrt_half = Scalar(0.70710678118654752440);
  const Packet m_x_scale_mask = pcmp_lt(m_x, pset1<Packet>(sqrt_half));
  m_x = pselect(m_x_scale_mask, pmul(pset1<Packet>(Scalar(2)), m_x), m_x);
  e_x = pselect(m_x_scale_mask, psub(e_x, pset1<Packet>(Scalar(1))), e_x);
 
  // Compute log2(m_x) with 6 extra bits of accuracy.
  Packet rx_hi, rx_lo;
  accurate_log2<Scalar>()(m_x, rx_hi, rx_lo);
 
  // Compute the two terms {y * e_x, y * r_x} in f = y * log2(x) with doubled
  // precision using double word arithmetic.
  Packet f1_hi, f1_lo, f2_hi, f2_lo;
  twoprod(e_x, y, f1_hi, f1_lo);
  twoprod(rx_hi, rx_lo, y, f2_hi, f2_lo);
  // Sum the two terms in f using double word arithmetic. We know
  // that |e_x| > |log2(m_x)|, except for the case where e_x==0.
  // This means that we can use fast_twosum(f1,f2).
  // In the case e_x == 0, e_x * y = f1 = 0, so we don't lose any
  // accuracy by violating the assumption of fast_twosum, because
  // it's a no-op.
  Packet f_hi, f_lo;
  fast_twosum(f1_hi, f1_lo, f2_hi, f2_lo, f_hi, f_lo);
 
  // Split f into integer and fractional parts.
  Packet n_z, r_z;
  absolute_split(f_hi, n_z, r_z);
  r_z = padd(r_z, f_lo);
  Packet n_r;
  absolute_split(r_z, n_r, r_z);
  n_z = padd(n_z, n_r);
 
  // We now have an accurate split of f = n_z + r_z and can compute
  //   x^y = 2**{n_z + r_z) = exp2(r_z) * 2**{n_z}.
  // Multiplication by the second factor can be done exactly using pldexp(), since
  // it is an integer power of 2.
  const Packet e_r = generic_exp2(r_z);
 
  // Since we know that e_r is in [1/sqrt(2); sqrt(2)], we can use the fast version
  // of pldexp to multiply by 2**{n_z} when |n_z| is sufficiently small.
  constexpr Scalar kPldExpThresh = std::numeric_limits<Scalar>::max_exponent - 2;
  const Packet pldexp_fast_unsafe = pcmp_lt(pset1<Packet>(kPldExpThresh), pabs(n_z));
  if (predux_any(pldexp_fast_unsafe)) {
    return pldexp(e_r, n_z);
  }
  return pldexp_fast(e_r, n_z);
}

References absolute_split(), EIGEN_CONSTEXPR, fast_twosum(), generic_exp2(), pabs(), padd(), pcmp_lt(), pfrexp(), pldexp(), pldexp_fast(), pmul(), predux_any(), pselect(), psub(), twoprod(), plotDoE::x, and y.

Referenced by Eigen::internal::unary_pow::gen_pow(), and generic_pow().

generic_rint()

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet Eigen::internal::generic_rint

(

const Packet &

a

)

                                                                           {
  using Scalar = typename unpacket_traits<Packet>::type;
  using IntType = typename numext::get_integer_by_size<sizeof(Scalar)>::signed_type;
  // Adds and subtracts signum(a) * 2^kMantissaBits to force rounding.
  const IntType kLimit = IntType(1) << (NumTraits<Scalar>::digits() - 1);
  const Packet cst_limit = pset1<Packet>(static_cast<Scalar>(kLimit));
  Packet abs_a = pabs(a);
  Packet sign_a = pandnot(a, abs_a);
  Packet rint_a = padd(abs_a, cst_limit);
  // Don't compile-away addition and subtraction.
  EIGEN_OPTIMIZATION_BARRIER(rint_a);
  rint_a = psub(rint_a, cst_limit);
  rint_a = por(rint_a, sign_a);
  // If greater than limit (or NaN), simply return a.
  Packet mask = pcmp_lt(abs_a, cst_limit);
  Packet result = pselect(mask, rint_a, a);
  return result;
}

References a, EIGEN_OPTIMIZATION_BARRIER, pabs(), padd(), pandnot(), pcmp_lt(), por(), pselect(), and psub().

Referenced by generic_ceil(), generic_floor(), and Eigen::internal::nearest_integer_packetop_impl< Packet, false, false >::run_rint().

generic_round()

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet Eigen::internal::generic_round

(

const Packet &

a

)

                                                                            {
  using Scalar = typename unpacket_traits<Packet>::type;
  const Packet cst_half = pset1<Packet>(Scalar(0.5));
  const Packet cst_1 = pset1<Packet>(Scalar(1));
  Packet abs_a = pabs(a);
  Packet sign_a = pandnot(a, abs_a);
  Packet floor_abs_a = generic_floor(abs_a);
  Packet diff = psub(abs_a, floor_abs_a);
  Packet mask = pcmp_le(cst_half, diff);
  Packet offset = pand(cst_1, mask);
  Packet result = padd(floor_abs_a, offset);
  result = por(result, sign_a);
  return result;
}

References a, generic_floor(), pabs(), padd(), pand(), pandnot(), pcmp_le(), por(), and psub().

Referenced by Eigen::internal::nearest_integer_packetop_impl< Packet, false, false >::run_round().

generic_trunc()

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet Eigen::internal::generic_trunc

(

const Packet &

a

)

                                                                            {
  Packet abs_a = pabs(a);
  Packet sign_a = pandnot(a, abs_a);
  Packet floor_abs_a = generic_floor(abs_a);
  Packet result = por(floor_abs_a, sign_a);
  return result;
}

References a, generic_floor(), pabs(), pandnot(), and por().

Referenced by Eigen::internal::nearest_integer_packetop_impl< Packet, false, false >::run_trunc().

get_compiletime_reshape_order()

constexpr int Eigen::internal::get_compiletime_reshape_order ( int  flags, int  order  )

inlineconstexpr

                                                                         {
  return order == AutoOrder ? flags & RowMajorBit : order;
}

References Eigen::AutoOrder, and Eigen::RowMajorBit.

get_computation_options()

constexpr int Eigen::internal::get_computation_options ( int  options )

constexpr

{ return options & ComputationOptionsBits; }

References ComputationOptionsBits.

Referenced by Eigen::BDCSVD< MatrixType_, Options_ >::allocate(), Eigen::BDCSVD< MatrixType_, Options_ >::BDCSVD(), and Eigen::JacobiSVD< MatrixType_, Options_ >::JacobiSVD().

get_qr_preconditioner()

constexpr int Eigen::internal::get_qr_preconditioner ( int  options )

constexpr

{ return options & QRPreconditionerBits; }

References QRPreconditionerBits.

get_random_seed()

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE uint64_t Eigen::internal::get_random_seed

(

)

                                                                 {
#if defined(EIGEN_GPU_COMPILE_PHASE)
  // We don't support 3d kernels since we currently only use 1 and
  // 2d kernels.
  gpu_assert(threadIdx.z == 0);
  return blockIdx.x * blockDim.x + threadIdx.x + gridDim.x * blockDim.x * (blockIdx.y * blockDim.y + threadIdx.y);
#else
  // Rely on Eigen's random implementation.
  return random<uint64_t>();
#endif
}

References blockDim, blockIdx, and threadIdx.

Referenced by PCG_XSH_RS_state().

get_runtime_reshape_size() [1/2]

Index Eigen::internal::get_runtime_reshape_size ( AutoSize_t  , Index  other, Index  total  )

inline

{ return total / other; }

get_runtime_reshape_size() [2/2]

template<typename SizeType >

Index Eigen::internal::get_runtime_reshape_size	(	SizeType	size,
		Index	,
		Index
	)

                                                                {
  return internal::get_runtime_value(size);
}

References get_runtime_value(), and size.

get_runtime_value()

template<typename T >

EIGEN_DEVICE_FUNC Index Eigen::internal::get_runtime_value

(

const T &

x

)

                                                      {
  return x;
}

References plotDoE::x.

Referenced by get_runtime_reshape_size().

getAdjointVal()

template<typename Scalar , int StorageOrder>

EIGEN_ALWAYS_INLINE std::complex<Scalar> Eigen::internal::getAdjointVal	(	Index	i,
		Index	j,
		const_blas_data_mapper< std::complex< Scalar >, Index, StorageOrder > &	dt
	)

Symm packing is related to packing of symmetric adjoint blocks, as expected the packing leaves the diagonal real, whatever is below it is copied from the respective upper diagonal element and conjugated. There's no PanelMode available for symm packing.

Packing in general is supposed to leave the lhs block and the rhs block easy to be read by gemm using its respective rank-update instructions. The float32/64 versions are different because at this moment the size of the accumulator is fixed at 512-bits so you can't have a 4x4 accumulator of 64-bit elements.

As mentioned earlier MatrixProduct breaks complex numbers into a real vector and a complex vector so packing has to take that into account, at the moment, we run pack the real part and then the imaginary part, this is the main reason why packing for complex is broken down into several different parts, also the reason why we endup having a float32/64 and complex float32/64 version.

                                                                                           {
  std::complex<Scalar> v;
  if (i < j) {
    v.real(dt(j, i).real());
    v.imag(-dt(j, i).imag());
  } else if (i > j) {
    v.real(dt(i, j).real());
    v.imag(dt(i, j).imag());
  } else {
    v.real(dt(i, j).real());
    v.imag((Scalar)0.0);
  }
  return v;
}

References i, Eigen::imag(), j, Eigen::real(), and v.

GetDenseElt() [1/2]

template<typename RealScalar >

void Eigen::internal::GetDenseElt ( const std::string &  line, RealScalar &  val  )

inline

                                                                {
  std::istringstream newline(line);
  newline >> val;
}

References calibrate::line, and calibrate::val.

Referenced by Eigen::loadMarketDense().

GetDenseElt() [2/2]

template<typename RealScalar >

void Eigen::internal::GetDenseElt ( const std::string &  line, std::complex< RealScalar > &  val  )

inline

                                                                            {
  RealScalar valR, valI;
  std::istringstream newline(line);
  newline >> valR >> valI;
  val = std::complex<RealScalar>(valR, valI);
}

References calibrate::line, and calibrate::val.

GetMarketLine() [1/6]

template<>

void Eigen::internal::GetMarketLine ( const char *  line, int &  i, int &  j, double &  value  )

inline

                                                                           {
  std::sscanf(line, "%d %d %lg", &i, &j, &value);
}

References i, j, calibrate::line, and Eigen::value.

GetMarketLine() [2/6]

template<>

void Eigen::internal::GetMarketLine ( const char *  line, int &  i, int &  j, float &  value  )

inline

                                                                          {
  std::sscanf(line, "%d %d %g", &i, &j, &value);
}

References i, j, calibrate::line, and Eigen::value.

GetMarketLine() [3/6]

template<>

void Eigen::internal::GetMarketLine ( const char *  line, int &  i, int &  j, std::complex< double > &  value  )

inline

                                                                                       {
  std::sscanf(line, "%d %d %lg %lg", &i, &j, &numext::real_ref(value), &numext::imag_ref(value));
}

References i, Eigen::numext::imag_ref(), j, calibrate::line, Eigen::numext::real_ref(), and Eigen::value.

GetMarketLine() [4/6]

template<>

void Eigen::internal::GetMarketLine ( const char *  line, int &  i, int &  j, std::complex< float > &  value  )

inline

                                                                                      {
  std::sscanf(line, "%d %d %g %g", &i, &j, &numext::real_ref(value), &numext::imag_ref(value));
}

References i, Eigen::numext::imag_ref(), j, calibrate::line, Eigen::numext::real_ref(), and Eigen::value.

GetMarketLine() [5/6]

template<typename Scalar , typename StorageIndex >

void Eigen::internal::GetMarketLine ( const char *  line, StorageIndex &  i, StorageIndex &  j, Scalar &  value  )

inline

                                                                                             {
  std::stringstream sline(line);
  sline >> i >> j >> value;
}

References i, j, calibrate::line, and Eigen::value.

Referenced by Eigen::loadMarket().

GetMarketLine() [6/6]

template<typename Scalar , typename StorageIndex >

void Eigen::internal::GetMarketLine ( const char *  line, StorageIndex &  i, StorageIndex &  j, std::complex< Scalar > &  value  )

inline

                                                                                                         {
  std::stringstream sline(line);
  Scalar valR, valI;
  sline >> i >> j >> valR >> valI;
  value = std::complex<Scalar>(valR, valI);
}

References i, j, calibrate::line, and Eigen::value.

getRandomBits()

template<typename BitsType >

EIGEN_DEVICE_FUNC BitsType Eigen::internal::getRandomBits ( int  numRandomBits )

inline

                                                                   {
  return random_bits_impl<BitsType>::run(numRandomBits);
}

References Eigen::internal::random_bits_impl< Scalar >::run().

gmres()

template<typename MatrixType , typename Rhs , typename Dest , typename Preconditioner >

bool Eigen::internal::gmres	(	const MatrixType &	mat,
		const Rhs &	rhs,
		Dest &	x,
		const Preconditioner &	precond,
		Index &	iters,
		const Index &	restart,
		typename Dest::RealScalar &	tol_error
	)

Generalized Minimal Residual Algorithm based on the Arnoldi algorithm implemented with Householder reflections.

Parameters:

Parameters

mat	matrix of linear system of equations
rhs	right hand side vector of linear system of equations
x	on input: initial guess, on output: solution
precond	preconditioner used
iters	on input: maximum number of iterations to perform on output: number of iterations performed
restart	number of iterations for a restart
tol_error	on input: relative residual tolerance on output: residuum achieved

See also

IterativeMethods::bicgstab()

For references, please see:

Saad, Y. and Schultz, M. H. GMRES: A Generalized Minimal Residual Algorithm for Solving Nonsymmetric Linear Systems. SIAM J.Sci.Stat.Comp. 7, 1986, pp. 856 - 869.

Saad, Y. Iterative Methods for Sparse Linear Systems. Society for Industrial and Applied Mathematics, Philadelphia, 2003.

Walker, H. F. Implementations of the GMRES method. Comput.Phys.Comm. 53, 1989, pp. 311 - 320.

Walker, H. F. Implementation of the GMRES Method using Householder Transformations. SIAM J.Sci.Stat.Comp. 9, 1988, pp. 152 - 163.

                                                                     {
  using std::abs;
  using std::sqrt;
 
  typedef typename Dest::RealScalar RealScalar;
  typedef typename Dest::Scalar Scalar;
  typedef Matrix<Scalar, Dynamic, 1> VectorType;
  typedef Matrix<Scalar, Dynamic, Dynamic, ColMajor> FMatrixType;
 
  const RealScalar considerAsZero = (std::numeric_limits<RealScalar>::min)();
 
  if (rhs.norm() <= considerAsZero) {
    x.setZero();
    tol_error = 0;
    return true;
  }
 
  RealScalar tol = tol_error;
  const Index maxIters = iters;
  iters = 0;
 
  const Index m = mat.rows();
 
  // residual and preconditioned residual
  VectorType p0 = rhs - mat * x;
  VectorType r0 = precond.solve(p0);
 
  const RealScalar r0Norm = r0.norm();
 
  // is initial guess already good enough?
  if (r0Norm == 0) {
    tol_error = 0;
    return true;
  }
 
  // storage for Hessenberg matrix and Householder data
  FMatrixType H = FMatrixType::Zero(m, restart + 1);
  VectorType w = VectorType::Zero(restart + 1);
  VectorType tau = VectorType::Zero(restart + 1);
 
  // storage for Jacobi rotations
  std::vector<JacobiRotation<Scalar> > G(restart);
 
  // storage for temporaries
  VectorType t(m), v(m), workspace(m), x_new(m);
 
  // generate first Householder vector
  Ref<VectorType> H0_tail = H.col(0).tail(m - 1);
  RealScalar beta;
  r0.makeHouseholder(H0_tail, tau.coeffRef(0), beta);
  w(0) = Scalar(beta);
 
  for (Index k = 1; k <= restart; ++k) {
    ++iters;
 
    v = VectorType::Unit(m, k - 1);
 
    // apply Householder reflections H_{1} ... H_{k-1} to v
    // TODO: use a HouseholderSequence
    for (Index i = k - 1; i >= 0; --i) {
      v.tail(m - i).applyHouseholderOnTheLeft(H.col(i).tail(m - i - 1), tau.coeffRef(i), workspace.data());
    }
 
    // apply matrix M to v:  v = mat * v;
    t.noalias() = mat * v;
    v = precond.solve(t);
 
    // apply Householder reflections H_{k-1} ... H_{1} to v
    // TODO: use a HouseholderSequence
    for (Index i = 0; i < k; ++i) {
      v.tail(m - i).applyHouseholderOnTheLeft(H.col(i).tail(m - i - 1), tau.coeffRef(i), workspace.data());
    }
 
    if (v.tail(m - k).norm() != 0.0) {
      if (k <= restart) {
        // generate new Householder vector
        Ref<VectorType> Hk_tail = H.col(k).tail(m - k - 1);
        v.tail(m - k).makeHouseholder(Hk_tail, tau.coeffRef(k), beta);
 
        // apply Householder reflection H_{k} to v
        v.tail(m - k).applyHouseholderOnTheLeft(Hk_tail, tau.coeffRef(k), workspace.data());
      }
    }
 
    if (k > 1) {
      for (Index i = 0; i < k - 1; ++i) {
        // apply old Givens rotations to v
        v.applyOnTheLeft(i, i + 1, G[i].adjoint());
      }
    }
 
    if (k < m && v(k) != (Scalar)0) {
      // determine next Givens rotation
      G[k - 1].makeGivens(v(k - 1), v(k));
 
      // apply Givens rotation to v and w
      v.applyOnTheLeft(k - 1, k, G[k - 1].adjoint());
      w.applyOnTheLeft(k - 1, k, G[k - 1].adjoint());
    }
 
    // insert coefficients into upper matrix triangle
    H.col(k - 1).head(k) = v.head(k);
 
    tol_error = abs(w(k)) / r0Norm;
    bool stop = (k == m || tol_error < tol || iters == maxIters);
 
    if (stop || k == restart) {
      // solve upper triangular system
      Ref<VectorType> y = w.head(k);
      H.topLeftCorner(k, k).template triangularView<Upper>().solveInPlace(y);
 
      // use Horner-like scheme to calculate solution vector
      x_new.setZero();
      for (Index i = k - 1; i >= 0; --i) {
        x_new(i) += y(i);
        // apply Householder reflection H_{i} to x_new
        x_new.tail(m - i).applyHouseholderOnTheLeft(H.col(i).tail(m - i - 1), tau.coeffRef(i), workspace.data());
      }
 
      x += x_new;
 
      if (stop) {
        return true;
      } else {
        k = 0;
 
        // reset data for restart
        p0.noalias() = rhs - mat * x;
        r0 = precond.solve(p0);
 
        // clear Hessenberg matrix and Householder data
        H.setZero();
        w.setZero();
        tau.setZero();
 
        // generate first Householder vector
        r0.makeHouseholder(H0_tail, tau.coeffRef(0), beta);
        w(0) = Scalar(beta);
      }
    }
  }
 
  return false;
}

References abs(), adjoint(), beta, G, H, i, k, m, min, p0, Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::rows(), sqrt(), plotPSD::t, v, w, plotDoE::x, y, and oomph::PseudoSolidHelper::Zero.

Referenced by Eigen::GMRES< MatrixType_, Preconditioner_ >::_solve_vector_with_guess_impl(), and main().

h_array_apply()

template<typename Op , typename A , std::size_t N, int... n>

constexpr EIGEN_STRONG_INLINE array<decltype(Op::run(A())), N> Eigen::internal::h_array_apply ( array< A, N >  a, numeric_list< int, n... >    )

constexpr

                                                                                                                   {
  return array<decltype(Op::run(A())), N>{{Op::run(array_get<n>(a))...}};
}

References a, N, and run().

h_array_apply_and_reduce()

template<typename Reducer , typename Op , typename A , std::size_t N, int... n>

constexpr EIGEN_STRONG_INLINE auto Eigen::internal::h_array_apply_and_reduce ( array< A, N >  arr, numeric_list< int, n... >    ) -> decltype(reduce<Reducer, typename id_numeric<int, n, decltype(Op::run(A()))>::type...>::run( Op::run(array_get<n>(arr))...))

constexpr

                                      {
  return reduce<Reducer, typename id_numeric<int, n, decltype(Op::run(A()))>::type...>::run(
      Op::run(array_get<n>(arr))...);
}

References int(), n, run(), Eigen::run(), and Eigen::internal::skip< n, a >::type.

h_array_reverse()

template<typename Array , int... n>

constexpr EIGEN_STRONG_INLINE Array Eigen::internal::h_array_reverse ( Array  arr, numeric_list< int, n... >    )

constexpr

                                                                                        {
  return {{array_get<sizeof...(n) - n - 1>(arr)...}};
}

References array_get(), and n.

Referenced by array_reverse().

h_array_zip()

template<typename Op , typename A , typename B , std::size_t N, int... n>

constexpr EIGEN_STRONG_INLINE array<decltype(Op::run(A(), B())), N> Eigen::internal::h_array_zip ( array< A, N >  a, array< B, N >  b, numeric_list< int, n... >    )

constexpr

                                                                                                         {
  return array<decltype(Op::run(A(), B())), N>{{Op::run(array_get<n>(a), array_get<n>(b))...}};
}

References a, b, N, and run().

h_array_zip_and_reduce()

template<typename Reducer , typename Op , typename A , typename B , std::size_t N, int... n>

constexpr EIGEN_STRONG_INLINE auto Eigen::internal::h_array_zip_and_reduce ( array< A, N >  a, array< B, N >  b, numeric_list< int, n... >    ) -> decltype(reduce<Reducer, typename id_numeric<int, n, decltype(Op::run(A(), B()))>::type...>::run( Op::run(array_get<n>(a), array_get<n>(b))...))

constexpr

                                                     {
  return reduce<Reducer, typename id_numeric<int, n, decltype(Op::run(A(), B()))>::type...>::run(
      Op::run(array_get<n>(a), array_get<n>(b))...);
}

References a, b, int(), n, run(), Eigen::run(), and Eigen::internal::skip< n, a >::type.

half2float() [1/2]

EIGEN_STRONG_INLINE Packet16f Eigen::internal::half2float

(

const Packet16h &

a

)

{ return _mm512_cvtph_ps(a); }

References a.

half2float() [2/2]

EIGEN_STRONG_INLINE Packet8f Eigen::internal::half2float

(

const Packet8h &

a

)

                                                           {
#ifdef EIGEN_HAS_FP16_C
  return _mm256_cvtph_ps(a);
#else
  Eigen::internal::Packet8f pp = _mm256_castsi256_ps(
      _mm256_insertf128_si256(_mm256_castsi128_si256(half2floatsse(a)), half2floatsse(_mm_srli_si128(a, 8)), 1));
  return pp;
#endif
}

References a.

Referenced by padd< Packet16h >(), padd< Packet8h >(), pcast< Packet16h, Packet16f >(), pcast< Packet8h, Packet8f >(), pceil< Packet16h >(), pceil< Packet8h >(), pcmp_eq(), pcmp_le(), pcmp_lt(), pcmp_lt_or_nan(), pdiv< Packet16h >(), pdiv< Packet8h >(), pfloor< Packet16h >(), pfloor< Packet8h >(), pfrexp(), pldexp(), pmax< Packet16h >(), pmax< Packet8h >(), pmin< Packet16h >(), pmin< Packet8h >(), pmul< Packet16h >(), pmul< Packet8h >(), predux< Packet16h >(), predux< Packet8h >(), predux_max< Packet16h >(), predux_max< Packet8h >(), predux_min< Packet16h >(), predux_min< Packet8h >(), predux_mul< Packet16h >(), predux_mul< Packet8h >(), print< Packet16h >(), print< Packet8h >(), pround< Packet16h >(), pround< Packet8h >(), psub< Packet16h >(), psub< Packet8h >(), ptrunc< Packet16h >(), and ptrunc< Packet8h >().

handmade_aligned_free()

EIGEN_DEVICE_FUNC void Eigen::internal::handmade_aligned_free ( void *  ptr )

inline

Frees memory allocated with handmade_aligned_malloc

                                                               {
  if (ptr != nullptr) {
    uint8_t offset = static_cast<uint8_t>(*(static_cast<uint8_t*>(ptr) - 1));
    void* original = static_cast<void*>(static_cast<uint8_t*>(ptr) - offset);
 
    check_that_malloc_is_allowed();
    EIGEN_USING_STD(free)
    free(original);
  }
}

References check_that_malloc_is_allowed(), and EIGEN_USING_STD.

Referenced by aligned_free(), check_handmade_aligned_malloc(), Eigen::MaxSizeVector< T >::MaxSizeVector(), triSolve(), and Eigen::MaxSizeVector< T >::~MaxSizeVector().

handmade_aligned_malloc()

EIGEN_DEVICE_FUNC void* Eigen::internal::handmade_aligned_malloc ( std::size_t  size, std::size_t  alignment = EIGEN_DEFAULT_ALIGN_BYTES  )

inline

Like malloc, but the returned pointer is guaranteed to be aligned to alignment. Fast, but wastes alignment additional bytes of memory. Does not throw any exception.

                                                                                                        {
  eigen_assert(alignment >= sizeof(void*) && alignment <= 128 && (alignment & (alignment - 1)) == 0 &&
               "Alignment must be at least sizeof(void*), less than or equal to 128, and a power of 2");
 
  check_that_malloc_is_allowed();
  EIGEN_USING_STD(malloc)
  void* original = malloc(size + alignment);
  if (original == nullptr) return nullptr;
  uint8_t offset = static_cast<uint8_t>(alignment - (reinterpret_cast<std::size_t>(original) & (alignment - 1)));
  void* aligned = static_cast<void*>(static_cast<uint8_t*>(original) + offset);
  *(static_cast<uint8_t*>(aligned) - 1) = offset;
  return aligned;
}

References check_that_malloc_is_allowed(), eigen_assert, EIGEN_USING_STD, and size.

Referenced by aligned_malloc(), check_handmade_aligned_malloc(), handmade_aligned_realloc(), and triSolve().

handmade_aligned_realloc()

EIGEN_DEVICE_FUNC void* Eigen::internal::handmade_aligned_realloc ( void *  ptr, std::size_t  new_size, std::size_t  old_size, std::size_t  alignment = EIGEN_DEFAULT_ALIGN_BYTES  )

inline

Reallocates aligned memory. Since we know that our handmade version is based on std::malloc we can use std::realloc to implement efficient reallocation.

                                                                                                         {
  if (ptr == nullptr) return handmade_aligned_malloc(new_size, alignment);
  uint8_t old_offset = *(static_cast<uint8_t*>(ptr) - 1);
  void* old_original = static_cast<uint8_t*>(ptr) - old_offset;
 
  check_that_malloc_is_allowed();
  EIGEN_USING_STD(realloc)
  void* original = realloc(old_original, new_size + alignment);
  if (original == nullptr) return nullptr;
  if (original == old_original) return ptr;
  uint8_t offset = static_cast<uint8_t>(alignment - (reinterpret_cast<std::size_t>(original) & (alignment - 1)));
  void* aligned = static_cast<void*>(static_cast<uint8_t*>(original) + offset);
  if (offset != old_offset) {
    const void* src = static_cast<const void*>(static_cast<uint8_t*>(original) + old_offset);
    std::size_t count = (std::min)(new_size, old_size);
    std::memmove(aligned, src, count);
  }
  *(static_cast<uint8_t*>(aligned) - 1) = offset;
  return aligned;
}

References check_that_malloc_is_allowed(), EIGEN_USING_STD, handmade_aligned_malloc(), and min.

Referenced by aligned_realloc().

householder_qr_inplace_unblocked()

template<typename MatrixQR , typename HCoeffs >

void Eigen::internal::householder_qr_inplace_unblocked	(	MatrixQR &	mat,
		HCoeffs &	hCoeffs,
		typename MatrixQR::Scalar *	tempData = 0
	)

                                                                                                              {
  typedef typename MatrixQR::Scalar Scalar;
  typedef typename MatrixQR::RealScalar RealScalar;
  Index rows = mat.rows();
  Index cols = mat.cols();
  Index size = (std::min)(rows, cols);
 
  eigen_assert(hCoeffs.size() == size);
 
  typedef Matrix<Scalar, MatrixQR::ColsAtCompileTime, 1> TempType;
  TempType tempVector;
  if (tempData == 0) {
    tempVector.resize(cols);
    tempData = tempVector.data();
  }
 
  for (Index k = 0; k < size; ++k) {
    Index remainingRows = rows - k;
    Index remainingCols = cols - k - 1;
 
    RealScalar beta;
    mat.col(k).tail(remainingRows).makeHouseholderInPlace(hCoeffs.coeffRef(k), beta);
    mat.coeffRef(k, k) = beta;
 
    // apply H to remaining part of m_qr from the left
    mat.bottomRightCorner(remainingRows, remainingCols)
        .applyHouseholderOnTheLeft(mat.col(k).tail(remainingRows - 1), hCoeffs.coeffRef(k), tempData + k + 1);
  }
}

References Eigen::HouseholderQR< MatrixType_ >::cols(), Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::cols(), eigen_assert, Eigen::HouseholderQR< MatrixType_ >::hCoeffs(), min, Eigen::HouseholderQR< MatrixType_ >::rows(), Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::rows(), and Eigen::EigenBase< Derived >::size().

Referenced by householder_update(), and Eigen::internal::householder_qr_inplace_blocked< MatrixQR, HCoeffs, MatrixQRScalar, InnerStrideIsOne >::run().

householder_qr_inplace_update()

template<typename MatrixQR , typename HCoeffs , typename VectorQR >

void Eigen::internal::householder_qr_inplace_update	(	MatrixQR &	mat,
		HCoeffs &	hCoeffs,
		const VectorQR &	newColumn,
		typename MatrixQR::Index	k,
		typename MatrixQR::Scalar *	tempData
	)

Basically a modified copy of Eigen::internal::householder_qr_inplace_unblocked that performs a rank-1 update of the QR matrix in compact storage. This function assumes, that the first k-1 columns of the matrix mat contain the QR decomposition of \(A^N\) up to column k-1. Then the QR decomposition of the k-th column (given by newColumn) is computed by applying the k-1 Householder projectors on it and finally compute the projector \(H_k\) of it. On exit the matrix mat and the vector hCoeffs contain the QR decomposition of the first k columns of \(A^N\). The tempData argument must point to at least mat.cols() scalars.

                                                                                                {
  typedef typename MatrixQR::Index Index;
  typedef typename MatrixQR::RealScalar RealScalar;
  Index rows = mat.rows();
 
  eigen_assert(k < mat.cols());
  eigen_assert(k < rows);
  eigen_assert(hCoeffs.size() == mat.cols());
  eigen_assert(newColumn.size() == rows);
  eigen_assert(tempData);
 
  // Store new column in mat at column k
  mat.col(k) = newColumn;
  // Apply H = H_1...H_{k-1} on newColumn (skip if k=0)
  for (Index i = 0; i < k; ++i) {
    Index remainingRows = rows - i;
    mat.col(k)
        .tail(remainingRows)
        .applyHouseholderOnTheLeft(mat.col(i).tail(remainingRows - 1), hCoeffs.coeffRef(i), tempData + i + 1);
  }
  // Construct Householder projector in-place in column k
  RealScalar beta;
  mat.col(k).tail(rows - k).makeHouseholderInPlace(hCoeffs.coeffRef(k), beta);
  mat.coeffRef(k, k) = beta;
}

References beta, Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::coeffRef(), Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::cols(), eigen_assert, Eigen::HouseholderQR< MatrixType_ >::hCoeffs(), i, k, Eigen::HouseholderQR< MatrixType_ >::rows(), and Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::rows().

Referenced by householder_update().

idrs()

template<typename MatrixType , typename Rhs , typename Dest , typename Preconditioner >

bool Eigen::internal::idrs	(	const MatrixType &	A,
		const Rhs &	b,
		Dest &	x,
		const Preconditioner &	precond,
		Index &	iter,
		typename Dest::RealScalar &	relres,
		Index	S,
		bool	smoothing,
		typename Dest::RealScalar	angle,
		bool	replacement
	)

                            {
  typedef typename Dest::RealScalar RealScalar;
  typedef typename Dest::Scalar Scalar;
  typedef Matrix<Scalar, Dynamic, 1> VectorType;
  typedef Matrix<Scalar, Dynamic, Dynamic, ColMajor> DenseMatrixType;
  const Index N = b.size();
  S = S < x.rows() ? S : x.rows();
  const RealScalar tol = relres;
  const Index maxit = iter;
 
  bool trueres = false;
 
  FullPivLU<DenseMatrixType> lu_solver;
 
  DenseMatrixType P;
  {
    HouseholderQR<DenseMatrixType> qr(DenseMatrixType::Random(N, S));
    P = (qr.householderQ() * DenseMatrixType::Identity(N, S));
  }
 
  const RealScalar normb = b.stableNorm();
 
  if (internal::isApprox(normb, RealScalar(0))) {
    // Solution is the zero vector
    x.setZero();
    iter = 0;
    relres = 0;
    return true;
  }
  // from http://homepage.tudelft.nl/1w5b5/IDRS/manual.pdf
  // A peak in the residual is considered dangerously high if‖ri‖/‖b‖> C(tol/epsilon).
  // With epsilon the relative machine precision. The factor tol/epsilon corresponds
  // to the size of a finite precision number that is so large that the absolute
  // round-off error in this number, when propagated through the process, makes it
  // impossible to achieve the required accuracy. The factor C accounts for the
  // accumulation of round-off errors. This parameter has been set to 10^{-3}.
  // mp is epsilon/C 10^3 * eps is very conservative, so normally no residual
  // replacements will take place. It only happens if things go very wrong. Too many
  // restarts may ruin the convergence.
  const RealScalar mp = RealScalar(1e3) * NumTraits<Scalar>::epsilon();
 
  // Compute initial residual
  const RealScalar tolb = tol * normb;  // Relative tolerance
  VectorType r = b - A * x;
 
  VectorType x_s, r_s;
 
  if (smoothing) {
    x_s = x;
    r_s = r;
  }
 
  RealScalar normr = r.stableNorm();
 
  if (normr <= tolb) {
    // Initial guess is a good enough solution
    iter = 0;
    relres = normr / normb;
    return true;
  }
 
  DenseMatrixType G = DenseMatrixType::Zero(N, S);
  DenseMatrixType U = DenseMatrixType::Zero(N, S);
  DenseMatrixType M = DenseMatrixType::Identity(S, S);
  VectorType t(N), v(N);
  Scalar om = 1.;
 
  // Main iteration loop, guild G-spaces:
  iter = 0;
 
  while (normr > tolb && iter < maxit) {
    // New right hand size for small system:
    VectorType f = (r.adjoint() * P).adjoint();
 
    for (Index k = 0; k < S; ++k) {
      // Solve small system and make v orthogonal to P:
      // c = M(k:s,k:s)\f(k:s);
      lu_solver.compute(M.block(k, k, S - k, S - k));
      VectorType c = lu_solver.solve(f.segment(k, S - k));
      // v = r - G(:,k:s)*c;
      v = r - G.rightCols(S - k) * c;
      // Preconditioning
      v = precond.solve(v);
 
      // Compute new U(:,k) and G(:,k), G(:,k) is in space G_j
      U.col(k) = U.rightCols(S - k) * c + om * v;
      G.col(k) = A * U.col(k);
 
      // Bi-Orthogonalise the new basis vectors:
      for (Index i = 0; i < k - 1; ++i) {
        // alpha =  ( P(:,i)'*G(:,k) )/M(i,i);
        Scalar alpha = P.col(i).dot(G.col(k)) / M(i, i);
        G.col(k) = G.col(k) - alpha * G.col(i);
        U.col(k) = U.col(k) - alpha * U.col(i);
      }
 
      // New column of M = P'*G  (first k-1 entries are zero)
      // M(k:s,k) = (G(:,k)'*P(:,k:s))';
      M.block(k, k, S - k, 1) = (G.col(k).adjoint() * P.rightCols(S - k)).adjoint();
 
      if (internal::isApprox(M(k, k), Scalar(0))) {
        return false;
      }
 
      // Make r orthogonal to q_i, i = 0..k-1
      Scalar beta = f(k) / M(k, k);
      r = r - beta * G.col(k);
      x = x + beta * U.col(k);
      normr = r.stableNorm();
 
      if (replacement && normr > tolb / mp) {
        trueres = true;
      }
 
      // Smoothing:
      if (smoothing) {
        t = r_s - r;
        // gamma is a Scalar, but the conversion is not allowed
        Scalar gamma = t.dot(r_s) / t.stableNorm();
        r_s = r_s - gamma * t;
        x_s = x_s - gamma * (x_s - x);
        normr = r_s.stableNorm();
      }
 
      if (normr < tolb || iter == maxit) {
        break;
      }
 
      // New f = P'*r (first k  components are zero)
      if (k < S - 1) {
        f.segment(k + 1, S - (k + 1)) = f.segment(k + 1, S - (k + 1)) - beta * M.block(k + 1, k, S - (k + 1), 1);
      }
    }  // end for
 
    if (normr < tolb || iter == maxit) {
      break;
    }
 
    // Now we have sufficient vectors in G_j to compute residual in G_j+1
    // Note: r is already perpendicular to P so v = r
    // Preconditioning
    v = r;
    v = precond.solve(v);
 
    // Matrix-vector multiplication:
    t = A * v;
 
    // Computation of a new omega
    om = internal::omega(t, r, angle);
 
    if (om == RealScalar(0.0)) {
      return false;
    }
 
    r = r - om * t;
    x = x + om * v;
    normr = r.stableNorm();
 
    if (replacement && normr > tolb / mp) {
      trueres = true;
    }
 
    // Residual replacement?
    if (trueres && normr < normb) {
      r = b - A * x;
      trueres = false;
    }
 
    // Smoothing:
    if (smoothing) {
      t = r_s - r;
      Scalar gamma = t.dot(r_s) / t.stableNorm();
      r_s = r_s - gamma * t;
      x_s = x_s - gamma * (x_s - x);
      normr = r_s.stableNorm();
    }
 
    iter++;
 
  }  // end while
 
  if (smoothing) {
    x = x_s;
  }
  relres = normr / normb;
  return true;
}

References adjoint(), alpha, Jeffery_Solution::angle(), b, beta, calibrate::c, Eigen::FullPivLU< MatrixType_, PermutationIndex_ >::compute(), oomph::SarahBL::epsilon, f(), G, mathsFunc::gamma(), i, isApprox(), k, N, omega(), Global_Physical_Variables::P, qr(), UniformPSDSelfTest::r, oomph::QuadTreeNames::S, Eigen::SolverBase< Derived >::solve(), plotPSD::t, RachelsAdvectionDiffusion::U, v, plotDoE::x, and oomph::PseudoSolidHelper::Zero.

Referenced by Eigen::IDRS< MatrixType_, Preconditioner_ >::_solve_vector_with_guess_impl().

idrstabl()

template<typename MatrixType , typename Rhs , typename Dest , typename Preconditioner >

bool Eigen::internal::idrstabl	(	const MatrixType &	mat,
		const Rhs &	rhs,
		Dest &	x,
		const Preconditioner &	precond,
		Index &	iters,
		typename Dest::RealScalar &	tol_error,
		Index	L,
		Index	S
	)

                                                                    {
  /*
    Setup and type definitions.
  */
  using numext::abs;
  using numext::sqrt;
  typedef typename Dest::Scalar Scalar;
  typedef typename Dest::RealScalar RealScalar;
  typedef Matrix<Scalar, Dynamic, 1> VectorType;
  typedef Matrix<Scalar, Dynamic, Dynamic, ColMajor> DenseMatrixType;
 
  const Index N = x.rows();
 
  Index k = 0;  // Iteration counter
  const Index maxIters = iters;
 
  const RealScalar rhs_norm = rhs.stableNorm();
  const RealScalar tol = tol_error * rhs_norm;
 
  if (rhs_norm == 0) {
    /*
      If b==0, then the exact solution is x=0.
      rhs_norm is needed for other calculations anyways, this exit is a freebie.
    */
    x.setZero();
    tol_error = 0.0;
    return true;
  }
  // Construct decomposition objects beforehand.
  FullPivLU<DenseMatrixType> lu_solver;
 
  if (S >= N || L >= N) {
    /*
      The matrix is very small, or the choice of L and S is very poor
      in that case solving directly will be best.
    */
    lu_solver.compute(DenseMatrixType(mat));
    x = lu_solver.solve(rhs);
    tol_error = (rhs - mat * x).stableNorm() / rhs_norm;
    return true;
  }
 
  // Define maximum sizes to prevent any reallocation later on.
  DenseMatrixType u(N, L + 1);
  DenseMatrixType r(N, L + 1);
 
  DenseMatrixType V(N * (L + 1), S);
 
  VectorType alpha(S);
  VectorType gamma(L);
  VectorType update(N);
 
  /*
    Main IDRSTABL algorithm
  */
  // Set up the initial residual
  VectorType x0 = x;
  r.col(0) = rhs - mat * x;
  x.setZero();  // The final solution will be x0+x
 
  tol_error = r.col(0).stableNorm();
 
  // FOM = Full orthogonalisation method
  DenseMatrixType h_FOM = DenseMatrixType::Zero(S, S - 1);
 
  // Construct an initial U matrix of size N x S
  DenseMatrixType U(N * (L + 1), S);
  for (Index col_index = 0; col_index < S; ++col_index) {
    // Arnoldi-like process to generate a set of orthogonal vectors spanning
    // {u,A*u,A*A*u,...,A^(S-1)*u}. This construction can be combined with the
    // Full Orthogonalization Method (FOM) from Ref.3 to provide a possible
    // early exit with no additional MV.
    if (col_index != 0) {
      /*
      Modified Gram-Schmidt strategy:
      */
      VectorType w = mat * precond.solve(u.col(0));
      for (Index i = 0; i < col_index; ++i) {
        auto v = U.col(i).head(N);
        h_FOM(i, col_index - 1) = v.dot(w);
        w -= h_FOM(i, col_index - 1) * v;
      }
      u.col(0) = w;
      h_FOM(col_index, col_index - 1) = u.col(0).stableNorm();
 
      if (abs(h_FOM(col_index, col_index - 1)) != RealScalar(0)) {
        /*
        This only happens if u is NOT exactly zero. In case it is exactly zero
        it would imply that that this u has no component in the direction of the
        current residual.
 
        By then setting u to zero it will not contribute any further (as it
        should). Whereas attempting to normalize results in division by zero.
 
        Such cases occur if:
        1. The basis of dimension <S is sufficient to exactly solve the linear
        system. I.e. the current residual is in span{r,Ar,...A^{m-1}r}, where
        (m-1)<=S.
        2. Two vectors vectors generated from r, Ar,... are (numerically)
        parallel.
 
        In case 1, the exact solution to the system can be obtained from the
        "Full Orthogonalization Method" (Algorithm 6.4 in the book of Saad),
        without any additional MV.
 
        Contrary to what one would suspect, the comparison with ==0.0 for
        floating-point types is intended here. Any arbitrary non-zero u is fine
        to continue, however if u contains either NaN or Inf the algorithm will
        break down.
        */
        u.col(0) /= h_FOM(col_index, col_index - 1);
      }
    } else {
      u.col(0) = r.col(0);
      u.col(0).normalize();
    }
 
    U.col(col_index).head(N) = u.col(0);
  }
 
  if (S > 1) {
    // Check for early FOM exit.
    Scalar beta = r.col(0).stableNorm();
    VectorType e1 = VectorType::Zero(S - 1);
    e1(0) = beta;
    lu_solver.compute(h_FOM.topLeftCorner(S - 1, S - 1));
    VectorType y = lu_solver.solve(e1);
    VectorType x2 = x + U.topLeftCorner(N, S - 1) * y;
 
    // Using proposition 6.7 in Saad, one MV can be saved to calculate the
    // residual
    RealScalar FOM_residual = (h_FOM(S - 1, S - 2) * y(S - 2) * U.col(S - 1).head(N)).stableNorm();
 
    if (FOM_residual < tol) {
      // Exit, the FOM algorithm was already accurate enough
      iters = k;
      // Convert back to the unpreconditioned solution
      x = precond.solve(x2);
      // x contains the updates to x0, add those back to obtain the solution
      x += x0;
      tol_error = FOM_residual / rhs_norm;
      return true;
    }
  }
 
  /*
    Select an initial (N x S) matrix R0.
    1. Generate random R0, orthonormalize the result.
    2. This results in R0, however to save memory and compute we only need the
    adjoint of R0. This is given by the matrix R_T.\ Additionally, the matrix
    (mat.adjoint()*R_tilde).adjoint()=R_tilde.adjoint()*mat by the
    anti-distributivity property of the adjoint. This results in AR_T, which is
    constant if R_T does not have to be regenerated and can be precomputed.
    Based on reference 4, this has zero probability in exact arithmetic.
  */
 
  // Original IDRSTABL and Kensuke choose S random vectors:
  const HouseholderQR<DenseMatrixType> qr(DenseMatrixType::Random(N, S));
  DenseMatrixType R_T = (qr.householderQ() * DenseMatrixType::Identity(N, S)).adjoint();
  DenseMatrixType AR_T = DenseMatrixType(R_T * mat);
 
  // Pre-allocate sigma.
  DenseMatrixType sigma(S, S);
 
  bool reset_while = false;  // Should the while loop be reset for some reason?
 
  while (k < maxIters) {
    for (Index j = 1; j <= L; ++j) {
      /*
        The IDR Step
      */
      // Construction of the sigma-matrix, and the decomposition of sigma.
      for (Index i = 0; i < S; ++i) {
        sigma.col(i).noalias() = AR_T * precond.solve(U.block(N * (j - 1), i, N, 1));
      }
 
      lu_solver.compute(sigma);
      // Obtain the update coefficients alpha
      if (j == 1) {
        // alpha=inverse(sigma)*(R_T*r_0);
        alpha.noalias() = lu_solver.solve(R_T * r.col(0));
      } else {
        // alpha=inverse(sigma)*(AR_T*r_{j-2})
        alpha.noalias() = lu_solver.solve(AR_T * precond.solve(r.col(j - 2)));
      }
 
      // Obtain new solution and residual from this update
      update.noalias() = U.topRows(N) * alpha;
      r.col(0) -= mat * precond.solve(update);
      x += update;
 
      for (Index i = 1; i <= j - 2; ++i) {
        // This only affects the case L>2
        r.col(i) -= U.block(N * (i + 1), 0, N, S) * alpha;
      }
      if (j > 1) {
        // r=[r;A*r_{j-2}]
        r.col(j - 1).noalias() = mat * precond.solve(r.col(j - 2));
      }
      tol_error = r.col(0).stableNorm();
 
      if (tol_error < tol) {
        // If at this point the algorithm has converged, exit.
        reset_while = true;
        break;
      }
 
      bool break_normalization = false;
      for (Index q = 1; q <= S; ++q) {
        if (q == 1) {
          // u = r;
          u.leftCols(j + 1) = r.leftCols(j + 1);
        } else {
          // u=[u_1;u_2;...;u_j]
          u.leftCols(j) = u.middleCols(1, j);
        }
 
        // Obtain the update coefficients beta implicitly
        // beta=lu_sigma.solve(AR_T * u.block(N * (j - 1), 0, N, 1)
        u.reshaped().head(u.rows() * j) -= U.topRows(N * j) * lu_solver.solve(AR_T * precond.solve(u.col(j - 1)));
 
        // u=[u;Au_{j-1}]
        u.col(j).noalias() = mat * precond.solve(u.col(j - 1));
 
        // Orthonormalize u_j to the columns of V_j(:,1:q-1)
        if (q > 1) {
          /*
          Modified Gram-Schmidt-like procedure to make u orthogonal to the
          columns of V from Ref. 1.
 
          The vector mu from Ref. 1 is obtained implicitly:
          mu=V.block(N * j, 0, N, q - 1).adjoint() * u.block(N * j, 0, N, 1).
          */
          for (Index i = 0; i <= q - 2; ++i) {
            auto v = V.col(i).segment(N * j, N);
            Scalar h = v.squaredNorm();
            h = v.dot(u.col(j)) / h;
            u.reshaped().head(u.rows() * (j + 1)) -= h * V.block(0, i, N * (j + 1), 1);
          }
        }
        // Normalize u and assign to a column of V
        Scalar normalization_constant = u.col(j).stableNorm();
        //  If u is exactly zero, this will lead to a NaN. Small, non-zero u is
        //  fine.
        if (normalization_constant == RealScalar(0.0)) {
          break_normalization = true;
          break;
        } else {
          u.leftCols(j + 1) /= normalization_constant;
        }
 
        V.block(0, q - 1, N * (j + 1), 1).noalias() = u.reshaped().head(u.rows() * (j + 1));
      }
 
      if (break_normalization == false) {
        U = V;
      }
    }
    if (reset_while) {
      break;
    }
 
    // r=[r;mat*r_{L-1}]
    r.col(L).noalias() = mat * precond.solve(r.col(L - 1));
 
    /*
            The polynomial step
    */
    ColPivHouseholderQR<DenseMatrixType> qr_solver(r.rightCols(L));
    gamma.noalias() = qr_solver.solve(r.col(0));
 
    // Update solution and residual using the "minimized residual coefficients"
    update.noalias() = r.leftCols(L) * gamma;
    x += update;
    r.col(0) -= mat * precond.solve(update);
 
    // Update iteration info
    ++k;
    tol_error = r.col(0).stableNorm();
 
    if (tol_error < tol) {
      // Slightly early exit by moving the criterion before the update of U,
      // after the main while loop the result of that calculation would not be
      // needed.
      break;
    }
 
    /*
    U=U0-sum(gamma_j*U_j)
    Consider the first iteration. Then U only contains U0, so at the start of
    the while-loop U should be U0. Therefore only the first N rows of U have to
    be updated.
    */
    for (Index i = 1; i <= L; ++i) {
      U.topRows(N) -= U.block(N * i, 0, N, S) * gamma(i - 1);
    }
  }
 
  /*
          Exit after the while loop terminated.
  */
  iters = k;
  // Convert back to the unpreconditioned solution
  x = precond.solve(x);
  // x contains the updates to x0, add those back to obtain the solution
  x += x0;
  tol_error = tol_error / rhs_norm;
  return true;
}

References abs(), Eigen::numext::abs(), adjoint(), alpha, beta, Eigen::FullPivLU< MatrixType_, PermutationIndex_ >::compute(), mathsFunc::gamma(), i, j, k, L, N, Eigen::numext::q, qr(), UniformPSDSelfTest::r, oomph::QuadTreeNames::S, calibrate::sigma, Eigen::SolverBase< Derived >::solve(), Eigen::numext::sqrt(), stableNorm(), RachelsAdvectionDiffusion::U, v, V, w, plotDoE::x, Global::x0, Global_parameters::x2(), y, and oomph::PseudoSolidHelper::Zero.

Referenced by Eigen::IDRSTABL< MatrixType_, Preconditioner_ >::_solve_vector_with_guess_impl().

igamma_num_iterations()

template<typename Scalar , IgammaComputationMode mode>

EIGEN_DEVICE_FUNC int Eigen::internal::igamma_num_iterations

(

)

                                              {
  /* Returns the maximum number of internal iterations for igamma computation.
   */
  if (mode == VALUE) {
    return 2000;
  }
 
  if (internal::is_same<Scalar, float>::value) {
    return 200;
  } else if (internal::is_same<Scalar, double>::value) {
    return 500;
  } else {
    return 2000;
  }
}

References VALUE.

ignore_unused_variable()

template<typename T >

EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE void Eigen::internal::ignore_unused_variable ( const T &  )

constexpr

{}

index_known_statically()

template<typename T >

static EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bool Eigen::internal::index_known_statically ( Index  i )

static

                                                                              {
  return index_known_statically_impl<T>::run(i);
}

References i, and Eigen::internal::index_known_statically_impl< T >::run().

index_list_size() [1/2]

template<typename T >

EIGEN_CONSTEXPR auto Eigen::internal::index_list_size

(

const T &

x

)

Analogue of the std::ssize free function. It returns the signed size of the container or view x of type T

It currently supports:

• any types T defining a member T::size() const
• plain C arrays as T[N]

For C++20, this function just forwards to std::ssize, or any ADL discoverable ssize function.

                                                 {
  using R = std::common_type_t<std::ptrdiff_t, std::make_signed_t<decltype(x.size())>>;
  return static_cast<R>(x.size());
}

References R, and plotDoE::x.

Referenced by Eigen::internal::IndexedViewHelper< Indices, EnableIf >::size().

index_list_size() [2/2]

template<typename T , std::ptrdiff_t N>

EIGEN_CONSTEXPR std::ptrdiff_t Eigen::internal::index_list_size

(

const

T(&)[N]

)

                                                             {
  return N;
}

References N.

index_pair_first_statically_eq()

template<typename T >

static EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bool Eigen::internal::index_pair_first_statically_eq ( Index  i, Index  value  )

static

                                                                                                   {
  return index_pair_first_statically_eq_impl<T>::run(i, value);
}

References i, Eigen::internal::index_pair_first_statically_eq_impl< Tx >::run(), and Eigen::value.

index_pair_second_statically_eq()

template<typename T >

static EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bool Eigen::internal::index_pair_second_statically_eq ( Index  i, Index  value  )

static

                                                                                                    {
  return index_pair_second_statically_eq_impl<T>::run(i, value);
}

References i, Eigen::internal::index_pair_second_statically_eq_impl< Tx >::run(), and Eigen::value.

index_statically_eq()

template<typename T >

static EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bool Eigen::internal::index_statically_eq ( Index  i, Index  value  )

static

                                                                                        {
  return index_statically_eq_impl<T>::run(i, value);
}

References i, Eigen::internal::index_statically_eq_impl< Tx >::run(), and Eigen::value.

index_statically_gt()

template<typename T >

static EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bool Eigen::internal::index_statically_gt ( Index  i, Index  value  )

static

                                                                                        {
  return index_statically_gt_impl<T>::run(i, value);
}

References i, Eigen::internal::index_statically_gt_impl< T >::run(), and Eigen::value.

index_statically_lt()

template<typename T >

static EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bool Eigen::internal::index_statically_lt ( Index  i, Index  value  )

static

                                                                                        {
  return index_statically_lt_impl<T>::run(i, value);
}

References i, Eigen::internal::index_statically_lt_impl< T >::run(), and Eigen::value.

index_statically_ne()

template<typename T >

static EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bool Eigen::internal::index_statically_ne ( Index  i, Index  value  )

static

                                                                                        {
  return index_statically_ne_impl<T>::run(i, value);
}

References i, Eigen::internal::index_statically_ne_impl< T >::run(), and Eigen::value.

indices_statically_known_to_increase()

template<typename T >

static EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bool Eigen::internal::indices_statically_known_to_increase ( )

static

                                                                                     {
  return indices_statically_known_to_increase_impl<T>::run();
}

References Eigen::internal::indices_statically_known_to_increase_impl< T >::run().

initialize_tensor()

template<typename Derived , int N>

void Eigen::internal::initialize_tensor	(	TensorEvaluator< Derived, DefaultDevice > &	tensor,
		const typename Initializer< Derived, traits< Derived >::NumDimensions >::InitList &	vals
	)

                                                                                                        {
  Eigen::array<typename traits<Derived>::Index, traits<Derived>::NumDimensions> indices;
  Initializer<Derived, traits<Derived>::NumDimensions>::run(tensor, &indices, vals);
}

References Eigen::run().

insert_from_triplets()

template<typename InputIterator , typename SparseMatrixType , typename DupFunctor >

void Eigen::internal::insert_from_triplets	(	const InputIterator &	begin,
		const InputIterator &	end,
		SparseMatrixType &	mat,
		DupFunctor	dup_func
	)

                                               {
  using Scalar = typename SparseMatrixType::Scalar;
  using SrcXprType =
      CwiseBinaryOp<scalar_disjunction_op<DupFunctor, Scalar>, const SparseMatrixType, const SparseMatrixType>;
 
  // set_from_triplets is necessary to sort the inner indices and remove the duplicate entries
  SparseMatrixType trips(mat.rows(), mat.cols());
  set_from_triplets(begin, end, trips, dup_func);
 
  SrcXprType src = mat.binaryExpr(trips, scalar_disjunction_op<DupFunctor, Scalar>(dup_func));
  // the sparse assignment procedure creates a temporary matrix and swaps the final result
  assign_sparse_to_sparse<SparseMatrixType, SrcXprType>(mat, src);
}

References Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::cols(), Eigen::placeholders::end, Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::rows(), and set_from_triplets().

insert_from_triplets_sorted()

template<typename InputIterator , typename SparseMatrixType , typename DupFunctor >

void Eigen::internal::insert_from_triplets_sorted	(	const InputIterator &	begin,
		const InputIterator &	end,
		SparseMatrixType &	mat,
		DupFunctor	dup_func
	)

                                                      {
  using Scalar = typename SparseMatrixType::Scalar;
  using SrcXprType =
      CwiseBinaryOp<scalar_disjunction_op<DupFunctor, Scalar>, const SparseMatrixType, const SparseMatrixType>;
 
  // TODO: process triplets without making a copy
  SparseMatrixType trips(mat.rows(), mat.cols());
  set_from_triplets_sorted(begin, end, trips, dup_func);
 
  SrcXprType src = mat.binaryExpr(trips, scalar_disjunction_op<DupFunctor, Scalar>(dup_func));
  // the sparse assignment procedure creates a temporary matrix and swaps the final result
  assign_sparse_to_sparse<SparseMatrixType, SrcXprType>(mat, src);
}

References Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::cols(), Eigen::placeholders::end, Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::rows(), and set_from_triplets_sorted().

instantiate_by_c_array()

template<class InstType , typename ArrType , std::size_t N, bool Reverse = false>

InstType Eigen::internal::instantiate_by_c_array

(

ArrType *

arr

)

                                              {
  return h_instantiate_by_c_array<InstType, ArrType, N, Reverse>::run(arr);
}

References run().

intersect_helper()

template<typename BVH , typename Intersector >

bool Eigen::internal::intersect_helper	(	const BVH &	tree,
		Intersector &	intersector,
		typename BVH::Index	root
	)

                                                                                         {
  typedef typename BVH::Index Index;
  typedef typename BVH::VolumeIterator VolIter;
  typedef typename BVH::ObjectIterator ObjIter;
 
  VolIter vBegin = VolIter(), vEnd = VolIter();
  ObjIter oBegin = ObjIter(), oEnd = ObjIter();
 
  std::vector<Index> todo(1, root);
 
  while (!todo.empty()) {
    tree.getChildren(todo.back(), vBegin, vEnd, oBegin, oEnd);
    todo.pop_back();
 
    for (; vBegin != vEnd; ++vBegin)  // go through child volumes
      if (intersector.intersectVolume(tree.getVolume(*vBegin))) todo.push_back(*vBegin);
 
    for (; oBegin != oEnd; ++oBegin)                          // go through child objects
      if (intersector.intersectObject(*oBegin)) return true;  // intersector said to stop query
  }
  return false;
}

Referenced by Eigen::BVIntersect().

is_constant_evaluated()

constexpr bool Eigen::internal::is_constant_evaluated ( )

constexpr

Provide fallback for std::is_constant_evaluated for pre-C++20.

{ return false; }

Referenced by Eigen::internal::block_evaluator< ArgType, BlockRows, BlockCols, InnerPanel, true >::block_evaluator().

is_identically_zero()

template<typename Scalar >

EIGEN_STRONG_INLINE bool Eigen::internal::is_identically_zero

(

const Scalar &

s

)

                                                              {
  return is_identically_zero_impl<Scalar>::run(s);
}

References Eigen::internal::is_identically_zero_impl< Scalar >::run(), and s.

Referenced by Eigen::internal::triangular_solve_vector< LhsScalar, RhsScalar, Index, OnTheLeft, Mode, Conjugate, RowMajor >::run(), and Eigen::internal::triangular_solve_vector< LhsScalar, RhsScalar, Index, OnTheLeft, Mode, Conjugate, ColMajor >::run().

is_same_dense() [1/2]

template<typename T1 , typename T2 >

EIGEN_DEVICE_FUNC bool Eigen::internal::is_same_dense	(	const T1 &	,
		const T2 &	,
		std::enable_if_t<!possibly_same_dense< T1, T2 >::value > *	= 0
	)

                                                                                                                   {
  return false;
}

is_same_dense() [2/2]

template<typename T1 , typename T2 >

EIGEN_DEVICE_FUNC bool Eigen::internal::is_same_dense	(	const T1 &	mat1,
		const T2 &	mat2,
		std::enable_if_t< possibly_same_dense< T1, T2 >::value > *	= 0
	)

                                                                                            {
  return (mat1.data() == mat2.data()) && (mat1.innerStride() == mat2.innerStride()) &&
         (mat1.outerStride() == mat2.outerStride());
}

References mat1().

Referenced by Eigen::TriangularViewImpl< MatrixType_, Mode_, Dense >::_solve_impl(), Eigen::LLT< MatrixType_, UpLo_ >::compute(), Eigen::RealSchur< MatrixType_ >::computeFromHessenberg(), EIGEN_DECLARE_TEST(), Eigen::internal::triangular_solve_retval< Side, TriangularType, Rhs >::evalTo(), Eigen::HouseholderSequence< VectorsType, CoeffsType, Side >::evalTo(), Eigen::internal::permutation_matrix_product< ExpressionType, Side, Transposed, DenseShape >::run(), and Eigen::internal::transposition_matrix_product< ExpressionType, Side, Transposed, ExpressionShape >::run().

isApprox()

template<typename Scalar >

EIGEN_DEVICE_FUNC bool Eigen::internal::isApprox ( const Scalar &  x, const Scalar &  y, const typename NumTraits< Scalar >::Real &  precision = NumTraits<Scalar>::dummy_precision()  )

inline

                                                                                          {
  return scalar_fuzzy_impl<Scalar>::isApprox(x, y, precision);
}

References plotDoE::x, and y.

Referenced by Eigen::test::areApprox(), aux_evalSolver(), Eigen::internal::companion< Scalar_, Deg_ >::balancedR(), basicStuffComplex(), bicgstabl(), binary_op_test(), check_extremity_accuracy(), cholesky(), contraction_batch(), contraction_both_transposed(), contraction_lhs_transposed(), contraction_rhs_transposed(), EIGEN_DECLARE_TEST(), evalSolverSugarFunction(), float_pow_test_impl(), idrs(), Eigen::AngleAxis< Scalar_ >::isApprox(), Eigen::Rotation2D< Scalar_ >::isApprox(), Eigen::UniformScaling< Scalar_ >::isApprox(), Eigen::internal::scalar_fuzzy_default_impl< Scalar, false, false >::isApproxOrLessThan(), Eigen::MatrixBase< Derived >::isIdentity(), Eigen::MatrixBase< Derived >::isUnitary(), packetmath(), packetmath_notcomplex(), quaternion(), cast_test_impl< SrcType, DstType, RowsAtCompileTime, ColsAtCompileTime >::run(), test3dRotation(), test_accelerate_ldlt(), test_accelerate_llt(), test_accelerate_qr(), test_conj_helper(), test_gaussian_sycl(), test_isApprox(), boost::multiprecision::test_isApprox(), Eigen::test_isApprox(), test_larg_expr1D(), test_larg_expr2D(), test_larg_expr3D(), test_matrix_vector(), test_multithread_contraction(), test_no_out_of_bounds(), test_scalar(), test_sycl_contraction(), test_sycl_cumsum(), test_tensor_vector(), testSingular(), testVectorType(), transformations(), trmv(), and unary_op_test().

isApproxOrLessThan()

template<typename Scalar >

EIGEN_DEVICE_FUNC bool Eigen::internal::isApproxOrLessThan ( const Scalar &  x, const Scalar &  y, const typename NumTraits< Scalar >::Real &  precision = NumTraits<Scalar>::dummy_precision()  )

inline

                                                                                          {
  return scalar_fuzzy_impl<Scalar>::isApproxOrLessThan(x, y, precision);
}

References plotDoE::x, and y.

Referenced by Eigen::test_isApproxOrLessThan(), and boost::multiprecision::test_isApproxOrLessThan().

isfinite_impl() [1/3]

template<typename T >

EIGEN_DEVICE_FUNC bool Eigen::internal::isfinite_impl

(

const std::complex< T > &

x

)

                                                             {
  return (numext::isfinite)(numext::real(x)) && (numext::isfinite)(numext::imag(x));
}

References imag(), Eigen::numext::isfinite(), and plotDoE::x.

isfinite_impl() [2/3]

template<typename T >

EIGEN_DEVICE_FUNC std::enable_if_t<!(std::numeric_limits<T>::has_infinity || std::numeric_limits<T>::has_quiet_NaN || std::numeric_limits<T>::has_signaling_NaN), bool> Eigen::internal::isfinite_impl

(

const T &

)

                        {
  return true;
}

isfinite_impl() [3/3]

template<typename T >

EIGEN_DEVICE_FUNC std::enable_if_t<(std::numeric_limits<T>::has_infinity || std::numeric_limits<T>::has_quiet_NaN || std::numeric_limits<T>::has_signaling_NaN) && (!NumTraits<T>::IsComplex), bool> Eigen::internal::isfinite_impl

(

const T &

x

)

                          {
  EIGEN_USING_STD(isfinite);
  return isfinite EIGEN_NOT_A_MACRO(x);
}

isinf_impl() [1/3]

template<typename T >

EIGEN_DEVICE_FUNC bool Eigen::internal::isinf_impl

(

const std::complex< T > &

x

)

                                                          {
  return ((numext::isinf)(numext::real(x)) || (numext::isinf)(numext::imag(x))) && (!(numext::isnan)(x));
}

References imag(), Eigen::numext::isinf(), Eigen::numext::isnan(), and plotDoE::x.

isinf_impl() [2/3]

template<typename T >

EIGEN_DEVICE_FUNC std::enable_if_t<!std::numeric_limits<T>::has_infinity, bool> Eigen::internal::isinf_impl

(

const T &

)

                                                                                               {
  return false;
}

isinf_impl() [3/3]

template<typename T >

EIGEN_DEVICE_FUNC std::enable_if_t<(std::numeric_limits<T>::has_infinity && !NumTraits<T>::IsComplex), bool> Eigen::internal::isinf_impl

(

const T &

x

)

                {
  EIGEN_USING_STD(isinf);
  return isinf EIGEN_NOT_A_MACRO(x);
}

isMuchSmallerThan()

template<typename Scalar , typename OtherScalar >

EIGEN_DEVICE_FUNC bool Eigen::internal::isMuchSmallerThan ( const Scalar &  x, const OtherScalar &  y, const typename NumTraits< Scalar >::Real &  precision = NumTraits<Scalar>::dummy_precision()  )

inline

                                                                                          {
  return scalar_fuzzy_impl<Scalar>::template isMuchSmallerThan<OtherScalar>(x, y, precision);
}

References plotDoE::x, and y.

Referenced by array_generic(), Eigen::PolynomialSolver< Scalar_, Deg_ >::compute(), Eigen::FullPivHouseholderQR< MatrixType_, PermutationIndex_ >::computeInPlace(), Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::EIGEN_STATIC_ASSERT(), Eigen::EigenSolver< MatrixType_ >::eigenvectors(), Eigen::internal::unary_evaluator< SparseView< ArgType >, IteratorBased >::InnerIterator::incrementToNonZero(), Eigen::internal::unary_evaluator< SparseView< ArgType >, IndexBased >::InnerIterator::incrementToNonZero(), Eigen::Hyperplane< Scalar_, AmbientDim_, Options_ >::intersection(), Eigen::test::isApproxAbs(), Eigen::MatrixBase< Derived >::isDiagonal(), Eigen::MatrixBase< Derived >::isIdentity(), Eigen::MatrixBase< Derived >::isUnitary(), Eigen::DenseBase< Derived >::isZero(), main(), Eigen::SparseVector< Scalar_, Options_, StorageIndex_ >::prune(), Eigen::EigenSolver< MatrixType_ >::pseudoEigenvalueMatrix(), Eigen::internal::unitOrthogonal_selector< Derived, 3 >::run(), adjoint_specific< false >::run(), Eigen::ComplexSchur< MatrixType_ >::subdiagonalEntryIsNeglegible(), boost::multiprecision::test_isMuchSmallerThan(), and Eigen::test_isMuchSmallerThan().

isnan_impl() [1/3]

template<typename T >

EIGEN_DEVICE_FUNC bool Eigen::internal::isnan_impl

(

const std::complex< T > &

x

)

                                                          {
  return (numext::isnan)(numext::real(x)) || (numext::isnan)(numext::imag(x));
}

References imag(), Eigen::numext::isnan(), and plotDoE::x.

isnan_impl() [2/3]

template<typename T >

EIGEN_DEVICE_FUNC std::enable_if_t<!(std::numeric_limits<T>::has_quiet_NaN || std::numeric_limits<T>::has_signaling_NaN), bool> Eigen::internal::isnan_impl

(

const T &

)

                         {
  return false;
}

isnan_impl() [3/3]

template<typename T >

EIGEN_DEVICE_FUNC std::enable_if_t< (std::numeric_limits<T>::has_quiet_NaN || std::numeric_limits<T>::has_signaling_NaN) && (!NumTraits<T>::IsComplex), bool> Eigen::internal::isnan_impl

(

const T &

x

)

                       {
  EIGEN_USING_STD(isnan);
  return isnan EIGEN_NOT_A_MACRO(x);
}

KLoop() [1/2]

template<Index num_acc, Index num_packets, bool zero, bool rhsExtraCols, bool lhsExtraRows, Index num_rhs, Index num_lhs>

EIGEN_ALWAYS_INLINE void Eigen::internal::KLoop	(	const bfloat16 *	indexA,
		const bfloat16 *	indexB,
		__vector_quad(&)	quad_acc[num_acc],
		Index	strideB,
		Index	k,
		Index	offsetB,
		Index	extra_cols,
		Index	extra_rows
	)

                                                                                                          {
  Packet8bf lhs[num_lhs], rhs[num_rhs];
 
  BFLOAT16_UNROLL
  for (Index i = 0; i < (num_rhs - (rhsExtraCols ? 1 : 0)); i++) {
    rhs[i] = loadRhsBfloat16<zero>(indexB + k * 4, strideB, i);
  }
  if (rhsExtraCols) {
    rhs[num_rhs - 1] = loadRhsBfloat16<zero>(indexB + k * extra_cols - offsetB, strideB, num_rhs - 1);
  }
 
  indexA += k * (lhsExtraRows ? extra_rows : num_packets);
  if (num_lhs == 1) {
    lhs[0] = loadBfloat16<zero>(indexA);
  } else {
    BFLOAT16_UNROLL
    for (Index j = 0; j < num_lhs; j += 2) {
      Packet8bf lhs1 = ploadu<Packet8bf>(indexA + (j + 0) * (zero ? 4 : 8));
      if (zero) {
        Packet8bf lhs2 = pset1<Packet8bf>(Eigen::bfloat16(0));
        lhs[j + 0] = vec_mergeh(lhs1.m_val, lhs2.m_val);
        lhs[j + 1] = vec_mergel(lhs1.m_val, lhs2.m_val);
      } else {
        lhs[j + 0] = lhs1;
        lhs[j + 1] = ploadu<Packet8bf>(indexA + (j + 1) * 8);
      }
    }
  }
 
  BFLOAT16_UNROLL
  for (Index i = 0, x = 0; i < num_rhs; i++) {
    BFLOAT16_UNROLL
    for (Index j = 0; j < num_lhs; j++, x++) {
      __builtin_mma_xvbf16ger2pp(&(quad_acc[x]), reinterpret_cast<Packet16uc>(rhs[i].m_val),
                                 reinterpret_cast<Packet16uc>(lhs[j].m_val));
    }
  }
}

References BFLOAT16_UNROLL, i, j, k, Eigen::internal::eigen_packet_wrapper< T, unique_id >::m_val, ploadu< Packet8bf >(), pset1< Packet8bf >(), plotDoE::x, and zero().

KLoop() [2/2]

template<Index num_acc, bool zero, bool rhsExtraCols, Index num_rhs>

EIGEN_ALWAYS_INLINE void Eigen::internal::KLoop	(	const float *	indexA,
		const float *	indexB,
		Packet4f(&)	acc[num_acc][4],
		Index	strideB,
		Index	k,
		Index	offsetB,
		Index	extra_cols
	)

                                                                         {
  constexpr Index num_lhs = 4;
  Packet4f lhs[num_lhs], rhs[num_rhs];
 
  constexpr Index real_rhs = (num_rhs - (rhsExtraCols ? 2 : 0));
  for (Index i = 0; i < real_rhs; i += 2) {
    loadTwoRhsFloat32<zero>(indexB + k * 4, strideB, i, rhs[i + 0], rhs[i + 1]);
  }
  if (rhsExtraCols) {
    loadTwoRhsFloat32<zero>(indexB + k * extra_cols - offsetB, strideB, real_rhs, rhs[real_rhs + 0], rhs[real_rhs + 1]);
  }
 
  indexA += 2 * k * 4;
  for (Index j = 0; j < num_lhs; j++) {
    lhs[j] = ploadu<Packet4f>(indexA + j * 4);
  }
 
  for (Index j = 0; j < num_rhs; j++) {
    for (Index i = 0; i < num_lhs; i++) {
      acc[j][i] = pmadd(rhs[j], lhs[i], acc[j][i]);
    }
  }
}

References i, j, k, ploadu< Packet4f >(), and pmadd().

LeafSize()

template<typename T >

EIGEN_DEVICE_FUNC Index Eigen::internal::LeafSize ( )

inline

                                          {
  return 1024;
}

LeafSize< bfloat16 >()

template<>

EIGEN_DEVICE_FUNC Index Eigen::internal::LeafSize< bfloat16 > ( )

inline

                                                    {
  return 128;
}

LeafSize< half >()

template<>

EIGEN_DEVICE_FUNC Index Eigen::internal::LeafSize< half > ( )

inline

                                                {
  return 200;
}

least_square_conjugate_gradient()

template<typename MatrixType , typename Rhs , typename Dest , typename Preconditioner >

EIGEN_DONT_INLINE void Eigen::internal::least_square_conjugate_gradient	(	const MatrixType &	mat,
		const Rhs &	rhs,
		Dest &	x,
		const Preconditioner &	precond,
		Index &	iters,
		typename Dest::RealScalar &	tol_error
	)

Low-level conjugate gradient algorithm for least-square problems

Parameters

mat	The matrix A
rhs	The right hand side vector b
x	On input and initial solution, on output the computed solution.
precond	A preconditioner being able to efficiently solve for an approximation of A'Ax=b (regardless of b)
iters	On input the max number of iteration, on output the number of performed iterations.
tol_error	On input the tolerance error, on output an estimation of the relative error.

                                                                                           {
  using std::abs;
  using std::sqrt;
  typedef typename Dest::RealScalar RealScalar;
  typedef typename Dest::Scalar Scalar;
  typedef Matrix<Scalar, Dynamic, 1> VectorType;
 
  RealScalar tol = tol_error;
  Index maxIters = iters;
 
  Index m = mat.rows(), n = mat.cols();
 
  VectorType residual = rhs - mat * x;
  VectorType normal_residual = mat.adjoint() * residual;
 
  RealScalar rhsNorm2 = (mat.adjoint() * rhs).squaredNorm();
  if (rhsNorm2 == 0) {
    x.setZero();
    iters = 0;
    tol_error = 0;
    return;
  }
  RealScalar threshold = tol * tol * rhsNorm2;
  RealScalar residualNorm2 = normal_residual.squaredNorm();
  if (residualNorm2 < threshold) {
    iters = 0;
    tol_error = sqrt(residualNorm2 / rhsNorm2);
    return;
  }
 
  VectorType p(n);
  p = precond.solve(normal_residual);  // initial search direction
 
  VectorType z(n), tmp(m);
  RealScalar absNew = numext::real(normal_residual.dot(p));  // the square of the absolute value of r scaled by invM
  Index i = 0;
  while (i < maxIters) {
    tmp.noalias() = mat * p;
 
    Scalar alpha = absNew / tmp.squaredNorm();             // the amount we travel on dir
    x += alpha * p;                                        // update solution
    residual -= alpha * tmp;                               // update residual
    normal_residual.noalias() = mat.adjoint() * residual;  // update residual of the normal equation
 
    residualNorm2 = normal_residual.squaredNorm();
    if (residualNorm2 < threshold) break;
 
    z = precond.solve(normal_residual);  // approximately solve for "A'A z = normal_residual"
 
    RealScalar absOld = absNew;
    absNew = numext::real(normal_residual.dot(z));  // update the absolute value of r
    RealScalar beta = absNew / absOld;  // calculate the Gram-Schmidt value used to create the new search direction
    p = z + beta * p;                   // update search direction
    i++;
  }
  tol_error = sqrt(residualNorm2 / rhsNorm2);
  iters = i;
}

References abs(), Eigen::SparseMatrixBase< Derived >::adjoint(), alpha, beta, Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::cols(), i, m, n, p, Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::rows(), sqrt(), tmp, and plotDoE::x.

Referenced by Eigen::LeastSquaresConjugateGradient< MatrixType_, Preconditioner_ >::_solve_vector_with_guess_impl().

llt_rank_update_lower()

template<typename MatrixType , typename VectorType >

static Index Eigen::internal::llt_rank_update_lower ( MatrixType &  mat, const VectorType &  vec, const typename MatrixType::RealScalar &  sigma  )

static

                                                                               {
  using std::sqrt;
  typedef typename MatrixType::Scalar Scalar;
  typedef typename MatrixType::RealScalar RealScalar;
  typedef typename MatrixType::ColXpr ColXpr;
  typedef internal::remove_all_t<ColXpr> ColXprCleaned;
  typedef typename ColXprCleaned::SegmentReturnType ColXprSegment;
  typedef Matrix<Scalar, Dynamic, 1> TempVectorType;
  typedef typename TempVectorType::SegmentReturnType TempVecSegment;
 
  Index n = mat.cols();
  eigen_assert(mat.rows() == n && vec.size() == n);
 
  TempVectorType temp;
 
  if (sigma > 0) {
    // This version is based on Givens rotations.
    // It is faster than the other one below, but only works for updates,
    // i.e., for sigma > 0
    temp = sqrt(sigma) * vec;
 
    for (Index i = 0; i < n; ++i) {
      JacobiRotation<Scalar> g;
      g.makeGivens(mat(i, i), -temp(i), &mat(i, i));
 
      Index rs = n - i - 1;
      if (rs > 0) {
        ColXprSegment x(mat.col(i).tail(rs));
        TempVecSegment y(temp.tail(rs));
        apply_rotation_in_the_plane(x, y, g);
      }
    }
  } else {
    temp = vec;
    RealScalar beta = 1;
    for (Index j = 0; j < n; ++j) {
      RealScalar Ljj = numext::real(mat.coeff(j, j));
      RealScalar dj = numext::abs2(Ljj);
      Scalar wj = temp.coeff(j);
      RealScalar swj2 = sigma * numext::abs2(wj);
      RealScalar gamma = dj * beta + swj2;
 
      RealScalar x = dj + swj2 / beta;
      if (x <= RealScalar(0)) return j;
      RealScalar nLjj = sqrt(x);
      mat.coeffRef(j, j) = nLjj;
      beta += swj2 / dj;
 
      // Update the terms of L
      Index rs = n - j - 1;
      if (rs) {
        temp.tail(rs) -= (wj / Ljj) * mat.col(j).tail(rs);
        if (!numext::is_exactly_zero(gamma))
          mat.col(j).tail(rs) =
              (nLjj / Ljj) * mat.col(j).tail(rs) + (nLjj * sigma * numext::conj(wj) / gamma) * temp.tail(rs);
      }
    }
  }
  return -1;
}

References Eigen::numext::abs2(), apply_rotation_in_the_plane(), beta, Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::coeff(), Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::coeffRef(), Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::cols(), conj(), eigen_assert, mathsFunc::gamma(), i, Eigen::numext::is_exactly_zero(), j, Eigen::JacobiRotation< Scalar >::makeGivens(), n, Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::rows(), calibrate::sigma, sqrt(), plotDoE::x, and y.

Referenced by Eigen::internal::llt_inplace< Scalar, Lower >::rankUpdate(), Eigen::internal::lapacke_helpers::rank_update< Lower >::run(), and Eigen::internal::lapacke_helpers::rank_update< Upper >::run().

lmpar()

template<typename Scalar >

void Eigen::internal::lmpar	(	Matrix< Scalar, Dynamic, Dynamic > &	r,
		const VectorXi &	ipvt,
		const Matrix< Scalar, Dynamic, 1 > &	diag,
		const Matrix< Scalar, Dynamic, 1 > &	qtb,
		Scalar	delta,
		Scalar &	par,
		Matrix< Scalar, Dynamic, 1 > &	x
	)

                                                                                                            {
  using std::abs;
  using std::sqrt;
  typedef DenseIndex Index;
 
  /* Local variables */
  Index i, j, l;
  Scalar fp;
  Scalar parc, parl;
  Index iter;
  Scalar temp, paru;
  Scalar gnorm;
  Scalar dxnorm;
 
  /* Function Body */
  const Scalar dwarf = (std::numeric_limits<Scalar>::min)();
  const Index n = r.cols();
  eigen_assert(n == diag.size());
  eigen_assert(n == qtb.size());
  eigen_assert(n == x.size());
 
  Matrix<Scalar, Dynamic, 1> wa1, wa2;
 
  /* compute and store in x the gauss-newton direction. if the */
  /* jacobian is rank-deficient, obtain a least squares solution. */
  Index nsing = n - 1;
  wa1 = qtb;
  for (j = 0; j < n; ++j) {
    if (r(j, j) == 0. && nsing == n - 1) nsing = j - 1;
    if (nsing < n - 1) wa1[j] = 0.;
  }
  for (j = nsing; j >= 0; --j) {
    wa1[j] /= r(j, j);
    temp = wa1[j];
    for (i = 0; i < j; ++i) wa1[i] -= r(i, j) * temp;
  }
 
  for (j = 0; j < n; ++j) x[ipvt[j]] = wa1[j];
 
  /* initialize the iteration counter. */
  /* evaluate the function at the origin, and test */
  /* for acceptance of the gauss-newton direction. */
  iter = 0;
  wa2 = diag.cwiseProduct(x);
  dxnorm = wa2.blueNorm();
  fp = dxnorm - delta;
  if (fp <= Scalar(0.1) * delta) {
    par = 0;
    return;
  }
 
  /* if the jacobian is not rank deficient, the newton */
  /* step provides a lower bound, parl, for the zero of */
  /* the function. otherwise set this bound to zero. */
  parl = 0.;
  if (nsing >= n - 1) {
    for (j = 0; j < n; ++j) {
      l = ipvt[j];
      wa1[j] = diag[l] * (wa2[l] / dxnorm);
    }
    // it's actually a triangularView.solveInplace(), though in a weird
    // way:
    for (j = 0; j < n; ++j) {
      Scalar sum = 0.;
      for (i = 0; i < j; ++i) sum += r(i, j) * wa1[i];
      wa1[j] = (wa1[j] - sum) / r(j, j);
    }
    temp = wa1.blueNorm();
    parl = fp / delta / temp / temp;
  }
 
  /* calculate an upper bound, paru, for the zero of the function. */
  for (j = 0; j < n; ++j) wa1[j] = r.col(j).head(j + 1).dot(qtb.head(j + 1)) / diag[ipvt[j]];
 
  gnorm = wa1.stableNorm();
  paru = gnorm / delta;
  if (paru == 0.) paru = dwarf / (std::min)(delta, Scalar(0.1));
 
  /* if the input par lies outside of the interval (parl,paru), */
  /* set par to the closer endpoint. */
  par = (std::max)(par, parl);
  par = (std::min)(par, paru);
  if (par == 0.) par = gnorm / dxnorm;
 
  /* beginning of an iteration. */
  while (true) {
    ++iter;
 
    /* evaluate the function at the current value of par. */
    if (par == 0.) par = (std::max)(dwarf, Scalar(.001) * paru); /* Computing MAX */
    wa1 = sqrt(par) * diag;
 
    Matrix<Scalar, Dynamic, 1> sdiag(n);
    qrsolv<Scalar>(r, ipvt, wa1, qtb, x, sdiag);
 
    wa2 = diag.cwiseProduct(x);
    dxnorm = wa2.blueNorm();
    temp = fp;
    fp = dxnorm - delta;
 
    /* if the function is small enough, accept the current value */
    /* of par. also test for the exceptional cases where parl */
    /* is zero or the number of iterations has reached 10. */
    if (abs(fp) <= Scalar(0.1) * delta || (parl == 0. && fp <= temp && temp < 0.) || iter == 10) break;
 
    /* compute the newton correction. */
    for (j = 0; j < n; ++j) {
      l = ipvt[j];
      wa1[j] = diag[l] * (wa2[l] / dxnorm);
    }
    for (j = 0; j < n; ++j) {
      wa1[j] /= sdiag[j];
      temp = wa1[j];
      for (i = j + 1; i < n; ++i) wa1[i] -= r(i, j) * temp;
    }
    temp = wa1.blueNorm();
    parc = fp / delta / temp / temp;
 
    /* depending on the sign of the function, update parl or paru. */
    if (fp > 0.) parl = (std::max)(parl, par);
    if (fp < 0.) paru = (std::min)(paru, par);
 
    /* compute an improved estimate for par. */
    /* Computing MAX */
    par = (std::max)(parl, par + parc);
 
    /* end of an iteration. */
  }
 
  /* termination. */
  if (iter == 0) par = 0.;
  return;
}

References abs(), MultiOpt::delta, diag, eigen_assert, i, j, max, min, n, calibrate::par, UniformPSDSelfTest::r, sqrt(), and plotDoE::x.

lmpar2() [1/2]

template<typename Scalar >

void Eigen::internal::lmpar2	(	const ColPivHouseholderQR< Matrix< Scalar, Dynamic, Dynamic > > &	qr,
		const Matrix< Scalar, Dynamic, 1 > &	diag,
		const Matrix< Scalar, Dynamic, 1 > &	qtb,
		Scalar	delta,
		Scalar &	par,
		Matrix< Scalar, Dynamic, 1 > &	x
	)

{
  using std::abs;
  using std::sqrt;
  typedef DenseIndex Index;
 
  /* Local variables */
  Index j;
  Scalar fp;
  Scalar parc, parl;
  Index iter;
  Scalar temp, paru;
  Scalar gnorm;
  Scalar dxnorm;
 
  /* Function Body */
  const Scalar dwarf = (std::numeric_limits<Scalar>::min)();
  const Index n = qr.matrixQR().cols();
  eigen_assert(n == diag.size());
  eigen_assert(n == qtb.size());
 
  Matrix<Scalar, Dynamic, 1> wa1, wa2;
 
  /* compute and store in x the gauss-newton direction. if the */
  /* jacobian is rank-deficient, obtain a least squares solution. */
 
  //    const Index rank = qr.nonzeroPivots(); // exactly double(0.)
  const Index rank = qr.rank();  // use a threshold
  wa1 = qtb;
  wa1.tail(n - rank).setZero();
  qr.matrixQR().topLeftCorner(rank, rank).template triangularView<Upper>().solveInPlace(wa1.head(rank));
 
  x = qr.colsPermutation() * wa1;
 
  /* initialize the iteration counter. */
  /* evaluate the function at the origin, and test */
  /* for acceptance of the gauss-newton direction. */
  iter = 0;
  wa2 = diag.cwiseProduct(x);
  dxnorm = wa2.blueNorm();
  fp = dxnorm - delta;
  if (fp <= Scalar(0.1) * delta) {
    par = 0;
    return;
  }
 
  /* if the jacobian is not rank deficient, the newton */
  /* step provides a lower bound, parl, for the zero of */
  /* the function. otherwise set this bound to zero. */
  parl = 0.;
  if (rank == n) {
    wa1 = qr.colsPermutation().inverse() * diag.cwiseProduct(wa2) / dxnorm;
    qr.matrixQR().topLeftCorner(n, n).transpose().template triangularView<Lower>().solveInPlace(wa1);
    temp = wa1.blueNorm();
    parl = fp / delta / temp / temp;
  }
 
  /* calculate an upper bound, paru, for the zero of the function. */
  for (j = 0; j < n; ++j)
    wa1[j] = qr.matrixQR().col(j).head(j + 1).dot(qtb.head(j + 1)) / diag[qr.colsPermutation().indices()(j)];
 
  gnorm = wa1.stableNorm();
  paru = gnorm / delta;
  if (paru == 0.) paru = dwarf / (std::min)(delta, Scalar(0.1));
 
  /* if the input par lies outside of the interval (parl,paru), */
  /* set par to the closer endpoint. */
  par = (std::max)(par, parl);
  par = (std::min)(par, paru);
  if (par == 0.) par = gnorm / dxnorm;
 
  /* beginning of an iteration. */
  Matrix<Scalar, Dynamic, Dynamic> s = qr.matrixQR();
  while (true) {
    ++iter;
 
    /* evaluate the function at the current value of par. */
    if (par == 0.) par = (std::max)(dwarf, Scalar(.001) * paru); /* Computing MAX */
    wa1 = sqrt(par) * diag;
 
    Matrix<Scalar, Dynamic, 1> sdiag(n);
    qrsolv<Scalar>(s, qr.colsPermutation().indices(), wa1, qtb, x, sdiag);
 
    wa2 = diag.cwiseProduct(x);
    dxnorm = wa2.blueNorm();
    temp = fp;
    fp = dxnorm - delta;
 
    /* if the function is small enough, accept the current value */
    /* of par. also test for the exceptional cases where parl */
    /* is zero or the number of iterations has reached 10. */
    if (abs(fp) <= Scalar(0.1) * delta || (parl == 0. && fp <= temp && temp < 0.) || iter == 10) break;
 
    /* compute the newton correction. */
    wa1 = qr.colsPermutation().inverse() * diag.cwiseProduct(wa2 / dxnorm);
    // we could almost use this here, but the diagonal is outside qr, in sdiag[]
    // qr.matrixQR().topLeftCorner(n, n).transpose().template triangularView<Lower>().solveInPlace(wa1);
    for (j = 0; j < n; ++j) {
      wa1[j] /= sdiag[j];
      temp = wa1[j];
      for (Index i = j + 1; i < n; ++i) wa1[i] -= s(i, j) * temp;
    }
    temp = wa1.blueNorm();
    parc = fp / delta / temp / temp;
 
    /* depending on the sign of the function, update parl or paru. */
    if (fp > 0.) parl = (std::max)(parl, par);
    if (fp < 0.) paru = (std::min)(paru, par);
 
    /* compute an improved estimate for par. */
    par = (std::max)(parl, par + parc);
  }
  if (iter == 0) par = 0.;
  return;
}

References abs(), MultiOpt::delta, diag, eigen_assert, i, j, max, min, n, calibrate::par, qr(), s, Eigen::PlainObjectBase< Derived >::setZero(), sqrt(), and plotDoE::x.

lmpar2() [2/2]

template<typename QRSolver , typename VectorType >

void Eigen::internal::lmpar2	(	const QRSolver &	qr,
		const VectorType &	diag,
		const VectorType &	qtb,
		typename VectorType::Scalar	m_delta,
		typename VectorType::Scalar &	par,
		VectorType &	x
	)

{
  using std::abs;
  using std::sqrt;
  typedef typename QRSolver::MatrixType MatrixType;
  typedef typename QRSolver::Scalar Scalar;
  //    typedef typename QRSolver::StorageIndex StorageIndex;
 
  /* Local variables */
  Index j;
  Scalar fp;
  Scalar parc, parl;
  Index iter;
  Scalar temp, paru;
  Scalar gnorm;
  Scalar dxnorm;
 
  // Make a copy of the triangular factor.
  // This copy is modified during call the qrsolv
  MatrixType s;
  s = qr.matrixR();
 
  /* Function Body */
  const Scalar dwarf = (std::numeric_limits<Scalar>::min)();
  const Index n = qr.matrixR().cols();
  eigen_assert(n == diag.size());
  eigen_assert(n == qtb.size());
 
  VectorType wa1, wa2;
 
  /* compute and store in x the gauss-newton direction. if the */
  /* jacobian is rank-deficient, obtain a least squares solution. */
 
  //    const Index rank = qr.nonzeroPivots(); // exactly double(0.)
  const Index rank = qr.rank();  // use a threshold
  wa1 = qtb;
  wa1.tail(n - rank).setZero();
  // FIXME There is no solve in place for sparse triangularView
  wa1.head(rank) = s.topLeftCorner(rank, rank).template triangularView<Upper>().solve(qtb.head(rank));
 
  x = qr.colsPermutation() * wa1;
 
  /* initialize the iteration counter. */
  /* evaluate the function at the origin, and test */
  /* for acceptance of the gauss-newton direction. */
  iter = 0;
  wa2 = diag.cwiseProduct(x);
  dxnorm = wa2.blueNorm();
  fp = dxnorm - m_delta;
  if (fp <= Scalar(0.1) * m_delta) {
    par = 0;
    return;
  }
 
  /* if the jacobian is not rank deficient, the newton */
  /* step provides a lower bound, parl, for the zero of */
  /* the function. otherwise set this bound to zero. */
  parl = 0.;
  if (rank == n) {
    wa1 = qr.colsPermutation().inverse() * diag.cwiseProduct(wa2) / dxnorm;
    s.topLeftCorner(n, n).transpose().template triangularView<Lower>().solveInPlace(wa1);
    temp = wa1.blueNorm();
    parl = fp / m_delta / temp / temp;
  }
 
  /* calculate an upper bound, paru, for the zero of the function. */
  for (j = 0; j < n; ++j) wa1[j] = s.col(j).head(j + 1).dot(qtb.head(j + 1)) / diag[qr.colsPermutation().indices()(j)];
 
  gnorm = wa1.stableNorm();
  paru = gnorm / m_delta;
  if (paru == 0.) paru = dwarf / (std::min)(m_delta, Scalar(0.1));
 
  /* if the input par lies outside of the interval (parl,paru), */
  /* set par to the closer endpoint. */
  par = (std::max)(par, parl);
  par = (std::min)(par, paru);
  if (par == 0.) par = gnorm / dxnorm;
 
  /* beginning of an iteration. */
  while (true) {
    ++iter;
 
    /* evaluate the function at the current value of par. */
    if (par == 0.) par = (std::max)(dwarf, Scalar(.001) * paru); /* Computing MAX */
    wa1 = sqrt(par) * diag;
 
    VectorType sdiag(n);
    lmqrsolv(s, qr.colsPermutation(), wa1, qtb, x, sdiag);
 
    wa2 = diag.cwiseProduct(x);
    dxnorm = wa2.blueNorm();
    temp = fp;
    fp = dxnorm - m_delta;
 
    /* if the function is small enough, accept the current value */
    /* of par. also test for the exceptional cases where parl */
    /* is zero or the number of iterations has reached 10. */
    if (abs(fp) <= Scalar(0.1) * m_delta || (parl == 0. && fp <= temp && temp < 0.) || iter == 10) break;
 
    /* compute the newton correction. */
    wa1 = qr.colsPermutation().inverse() * diag.cwiseProduct(wa2 / dxnorm);
    // we could almost use this here, but the diagonal is outside qr, in sdiag[]
    for (j = 0; j < n; ++j) {
      wa1[j] /= sdiag[j];
      temp = wa1[j];
      for (Index i = j + 1; i < n; ++i) wa1[i] -= s.coeff(i, j) * temp;
    }
    temp = wa1.blueNorm();
    parc = fp / m_delta / temp / temp;
 
    /* depending on the sign of the function, update parl or paru. */
    if (fp > 0.) parl = (std::max)(parl, par);
    if (fp < 0.) paru = (std::min)(paru, par);
 
    /* compute an improved estimate for par. */
    par = (std::max)(parl, par + parc);
  }
  if (iter == 0) par = 0.;
  return;
}

References abs(), diag, eigen_assert, i, j, lmqrsolv(), max, min, n, calibrate::par, qr(), s, sqrt(), and plotDoE::x.

Referenced by Eigen::LevenbergMarquardt< FunctorType_ >::minimizeOneStep().

lmqrsolv() [1/2]

template<typename Scalar , int Rows, int Cols, typename PermIndex >

void Eigen::internal::lmqrsolv	(	Matrix< Scalar, Rows, Cols > &	s,
		const PermutationMatrix< Dynamic, Dynamic, PermIndex > &	iPerm,
		const Matrix< Scalar, Dynamic, 1 > &	diag,
		const Matrix< Scalar, Dynamic, 1 > &	qtb,
		Matrix< Scalar, Dynamic, 1 > &	x,
		Matrix< Scalar, Dynamic, 1 > &	sdiag
	)

                                                                                {
  /* Local variables */
  Index i, j, k;
  Scalar temp;
  Index n = s.cols();
  Matrix<Scalar, Dynamic, 1> wa(n);
  JacobiRotation<Scalar> givens;
 
  /* Function Body */
  // the following will only change the lower triangular part of s, including
  // the diagonal, though the diagonal is restored afterward
 
  /*     copy r and (q transpose)*b to preserve input and initialize s. */
  /*     in particular, save the diagonal elements of r in x. */
  x = s.diagonal();
  wa = qtb;
 
  s.topLeftCorner(n, n).template triangularView<StrictlyLower>() = s.topLeftCorner(n, n).transpose();
  /*     eliminate the diagonal matrix d using a givens rotation. */
  for (j = 0; j < n; ++j) {
    /*        prepare the row of d to be eliminated, locating the */
    /*        diagonal element using p from the qr factorization. */
    const PermIndex l = iPerm.indices()(j);
    if (diag[l] == 0.) break;
    sdiag.tail(n - j).setZero();
    sdiag[j] = diag[l];
 
    /*        the transformations to eliminate the row of d */
    /*        modify only a single element of (q transpose)*b */
    /*        beyond the first n, which is initially zero. */
    Scalar qtbpj = 0.;
    for (k = j; k < n; ++k) {
      /*           determine a givens rotation which eliminates the */
      /*           appropriate element in the current row of d. */
      givens.makeGivens(-s(k, k), sdiag[k]);
 
      /*           compute the modified diagonal element of r and */
      /*           the modified element of ((q transpose)*b,0). */
      s(k, k) = givens.c() * s(k, k) + givens.s() * sdiag[k];
      temp = givens.c() * wa[k] + givens.s() * qtbpj;
      qtbpj = -givens.s() * wa[k] + givens.c() * qtbpj;
      wa[k] = temp;
 
      /*           accumulate the transformation in the row of s. */
      for (i = k + 1; i < n; ++i) {
        temp = givens.c() * s(i, k) + givens.s() * sdiag[i];
        sdiag[i] = -givens.s() * s(i, k) + givens.c() * sdiag[i];
        s(i, k) = temp;
      }
    }
  }
 
  /*     solve the triangular system for z. if the system is */
  /*     singular, then obtain a least squares solution. */
  Index nsing;
  for (nsing = 0; nsing < n && sdiag[nsing] != 0; nsing++) {
  }
 
  wa.tail(n - nsing).setZero();
  s.topLeftCorner(nsing, nsing).transpose().template triangularView<Upper>().solveInPlace(wa.head(nsing));
 
  // restore
  sdiag = s.diagonal();
  s.diagonal() = x;
 
  /* permute the components of z back to components of x. */
  x = iPerm * wa;
}

References Eigen::JacobiRotation< Scalar >::c(), diag, i, Eigen::PermutationMatrix< SizeAtCompileTime, MaxSizeAtCompileTime, StorageIndex_ >::indices(), j, k, Eigen::JacobiRotation< Scalar >::makeGivens(), n, s, Eigen::JacobiRotation< Scalar >::s(), Eigen::PlainObjectBase< Derived >::setZero(), and plotDoE::x.

Referenced by lmpar2().

lmqrsolv() [2/2]

template<typename Scalar , int Options_, typename Index >

void Eigen::internal::lmqrsolv	(	SparseMatrix< Scalar, Options_, Index > &	s,
		const PermutationMatrix< Dynamic, Dynamic > &	iPerm,
		const Matrix< Scalar, Dynamic, 1 > &	diag,
		const Matrix< Scalar, Dynamic, 1 > &	qtb,
		Matrix< Scalar, Dynamic, 1 > &	x,
		Matrix< Scalar, Dynamic, 1 > &	sdiag
	)

                                                                                {
  /* Local variables */
  typedef SparseMatrix<Scalar, RowMajor, Index> FactorType;
  Index i, j, k, l;
  Scalar temp;
  Index n = s.cols();
  Matrix<Scalar, Dynamic, 1> wa(n);
  JacobiRotation<Scalar> givens;
 
  /* Function Body */
  // the following will only change the lower triangular part of s, including
  // the diagonal, though the diagonal is restored afterward
 
  /*     copy r and (q transpose)*b to preserve input and initialize R. */
  wa = qtb;
  FactorType R(s);
  // Eliminate the diagonal matrix d using a givens rotation
  for (j = 0; j < n; ++j) {
    // Prepare the row of d to be eliminated, locating the
    // diagonal element using p from the qr factorization
    l = iPerm.indices()(j);
    if (diag(l) == Scalar(0)) break;
    sdiag.tail(n - j).setZero();
    sdiag[j] = diag[l];
    // the transformations to eliminate the row of d
    // modify only a single element of (q transpose)*b
    // beyond the first n, which is initially zero.
 
    Scalar qtbpj = 0;
    // Browse the nonzero elements of row j of the upper triangular s
    for (k = j; k < n; ++k) {
      typename FactorType::InnerIterator itk(R, k);
      for (; itk; ++itk) {
        if (itk.index() < k)
          continue;
        else
          break;
      }
      // At this point, we have the diagonal element R(k,k)
      //  Determine a givens rotation which eliminates
      //  the appropriate element in the current row of d
      givens.makeGivens(-itk.value(), sdiag(k));
 
      // Compute the modified diagonal element of r and
      // the modified element of ((q transpose)*b,0).
      itk.valueRef() = givens.c() * itk.value() + givens.s() * sdiag(k);
      temp = givens.c() * wa(k) + givens.s() * qtbpj;
      qtbpj = -givens.s() * wa(k) + givens.c() * qtbpj;
      wa(k) = temp;
 
      // Accumulate the transformation in the remaining k row/column of R
      for (++itk; itk; ++itk) {
        i = itk.index();
        temp = givens.c() * itk.value() + givens.s() * sdiag(i);
        sdiag(i) = -givens.s() * itk.value() + givens.c() * sdiag(i);
        itk.valueRef() = temp;
      }
    }
  }
 
  // Solve the triangular system for z. If the system is
  // singular, then obtain a least squares solution
  Index nsing;
  for (nsing = 0; nsing < n && sdiag(nsing) != 0; nsing++) {
  }
 
  wa.tail(n - nsing).setZero();
  //     x = wa;
  wa.head(nsing) = R.topLeftCorner(nsing, nsing).template triangularView<Upper>().solve /*InPlace*/ (wa.head(nsing));
 
  sdiag = R.diagonal();
  // Permute the components of z back to components of x
  x = iPerm * wa;
}

References Eigen::JacobiRotation< Scalar >::c(), diag, i, Eigen::PermutationMatrix< SizeAtCompileTime, MaxSizeAtCompileTime, StorageIndex_ >::indices(), j, k, Eigen::JacobiRotation< Scalar >::makeGivens(), n, R, s, Eigen::JacobiRotation< Scalar >::s(), Eigen::PlainObjectBase< Derived >::setZero(), and plotDoE::x.

loadAndMultiplyF32()

EIGEN_ALWAYS_INLINE Packet4f Eigen::internal::loadAndMultiplyF32	(	Packet4f	acc,
		const Packet4f	pAlpha,
		float *	result
	)

                                                                                                    {
  Packet4f result_block = ploadu<Packet4f>(result);
  return pmadd(acc, pAlpha, result_block);
}

References ploadu< Packet4f >(), and pmadd().

Referenced by storeResults().

loadBF16fromResult()

template<bool non_unit_stride, Index delta>

EIGEN_ALWAYS_INLINE Packet8bf Eigen::internal::loadBF16fromResult	(	bfloat16 *	src,
		Index	resInc
	)

                                                                              {
  if (non_unit_stride) {
    return pgather<bfloat16, Packet8bf>(src + delta * resInc, resInc);
  } else {
    return ploadu<Packet8bf>(src + delta);
  }
}

References MultiOpt::delta, pgather< bfloat16, Packet8bf >(), and ploadu< Packet8bf >().

loadBfloat16()

template<bool zero>

EIGEN_ALWAYS_INLINE Packet8bf Eigen::internal::loadBfloat16

(

const bfloat16 *

indexA

)

                                                                   {
  Packet8bf lhs1 = ploadu<Packet8bf>(indexA);
  if (zero) {
    Packet8bf lhs2 = pset1<Packet8bf>(Eigen::bfloat16(0));
    return vec_mergeh(lhs1.m_val, lhs2.m_val);
  } else {
    return lhs1;
  }
}

References Eigen::internal::eigen_packet_wrapper< T, unique_id >::m_val, ploadu< Packet8bf >(), pset1< Packet8bf >(), and zero().

loadColData()

template<typename RhsMapper , bool linear>

EIGEN_ALWAYS_INLINE Packet8bf Eigen::internal::loadColData	(	RhsMapper &	rhs,
		Index	j
	)

loadConstant()

template<typename T >

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T Eigen::internal::loadConstant

(

const T *

address

)

                                                                       {
  return *address;
}

Referenced by Eigen::TensorEvaluator< const Derived, Device >::coeff(), and Eigen::internal::CoeffLoader< Tensor, true, MakePointer_ >::coeff().

loadQuadToDoublePacket() [1/2]

template<typename Scalar , typename RealPacket >

void Eigen::internal::loadQuadToDoublePacket	(	const Scalar *	b,
		DoublePacket< RealPacket > &	dest
	)

                                                                                       {
  dest.first = pset1<RealPacket>(numext::real(*b));
  dest.second = pset1<RealPacket>(numext::imag(*b));
}

References b, Eigen::internal::DoublePacket< Packet >::first, imag(), and Eigen::internal::DoublePacket< Packet >::second.

Referenced by Eigen::internal::gebp_traits< std::complex< RealScalar >, std::complex< RealScalar >, ConjLhs_, ConjRhs_, Arch, PacketSize_ >::loadRhsQuad().

loadQuadToDoublePacket() [2/2]

template<typename Scalar , typename RealPacket >

void Eigen::internal::loadQuadToDoublePacket	(	const Scalar *	b,
		DoublePacket< RealPacket > &	dest,
		std::enable_if_t< unpacket_traits< RealPacket >::size==16 > *	= 0
	)

                                                                                        {
  // yes, that's pretty hackish too :(
  typedef typename NumTraits<Scalar>::Real RealScalar;
  RealScalar r[4] = {numext::real(b[0]), numext::real(b[0]), numext::real(b[1]), numext::real(b[1])};
  RealScalar i[4] = {numext::imag(b[0]), numext::imag(b[0]), numext::imag(b[1]), numext::imag(b[1])};
  dest.first = ploadquad<RealPacket>(r);
  dest.second = ploadquad<RealPacket>(i);
}

References b, Eigen::internal::DoublePacket< Packet >::first, i, imag(), UniformPSDSelfTest::r, and Eigen::internal::DoublePacket< Packet >::second.

loadRhsBfloat16()

template<bool zero>

EIGEN_ALWAYS_INLINE Packet8bf Eigen::internal::loadRhsBfloat16	(	const bfloat16 *	blockB,
		Index	strideB,
		Index	i
	)

                                                                                              {
  return loadBfloat16<zero>(blockB + strideB * i);
}

References i.

loadTwoRhsFloat32()

template<bool zero>

EIGEN_ALWAYS_INLINE void Eigen::internal::loadTwoRhsFloat32	(	const float *	block,
		Index	strideB,
		Index	i,
		Packet4f &	dhs0,
		Packet4f &	dhs1
	)

                                                                                                                       {
  dhs0 = ploadu<Packet4f>(block + strideB * i + 0);
  if (zero) {
    Packet4f dhs2 = pset1<Packet4f>(float(0));
    dhs1 = vec_mergel(dhs0, dhs2);
    dhs0 = vec_mergeh(dhs0, dhs2);
  } else {
    dhs1 = ploadu<Packet4f>(block + strideB * i + 4);
  }
}

References block(), i, ploadu< Packet4f >(), pset1< Packet4f >(), and zero().

loadVecLoop()

template<Index num_acc, typename LhsMapper , bool zero>

EIGEN_ALWAYS_INLINE void Eigen::internal::loadVecLoop	(	Index	k,
		LhsMapper &	lhs,
		Packet8bf(&)	a0[num_acc],
		Packet8bf	b1
	)

                                                                                                      {
  a0[k + 0] = lhs.template loadPacket<Packet8bf>(k * 4, 0);
  if (!zero) {
    b1 = lhs.template loadPacket<Packet8bf>(k * 4, 1);
  }
  if (num_acc > (k + 1)) {
    a0[k + 1] = vec_mergel(a0[k + 0].m_val, b1.m_val);
  }
  a0[k + 0] = vec_mergeh(a0[k + 0].m_val, b1.m_val);
}

References k, Eigen::internal::eigen_packet_wrapper< T, unique_id >::m_val, and zero().

log2_ceil()

template<typename BitsType >

int Eigen::internal::log2_ceil

(

const BitsType &

x

)

                                 {
  return log_2_impl<BitsType>::run_ceil(x);
}

Referenced by Eigen::CoreThreadPoolDevice::calculateLevels(), and Eigen::internal::random_int_impl< Scalar, false, true >::run().

log2_floor()

template<typename BitsType >

int Eigen::internal::log2_floor

(

const BitsType &

x

)

                                  {
  return log_2_impl<BitsType>::run_floor(x);
}

logical_xor()

constexpr bool Eigen::internal::logical_xor ( bool  a, bool  b  )

inlineconstexpr

Calculate logical XOR at compile time

{ return a != b; }

References a, and b.

Referenced by Eigen::internal::selfadjoint_product_impl< Lhs, LhsMode, false, Rhs, RhsMode, false >::run(), Eigen::internal::product_selfadjoint_matrix< Scalar, Index, LhsStorageOrder, LhsSelfAdjoint, ConjugateLhs, RhsStorageOrder, RhsSelfAdjoint, ConjugateRhs, RowMajor, ResInnerStride >::run(), and Eigen::internal::selfadjoint_matrix_vector_product< Scalar, Index, StorageOrder, UpLo, ConjugateLhs, ConjugateRhs, Version >::run().

LUnumTempV()

Index Eigen::internal::LUnumTempV ( Index &  m, Index &  w, Index &  t, Index &  b  )

inline

{ return (std::max)(m, (t + b) * w); }

References b, m, max, plotPSD::t, and w.

Referenced by Eigen::SparseLU< MatrixType_, OrderingType_ >::factorize().

LUTempSpace()

template<typename Scalar >

Index Eigen::internal::LUTempSpace ( Index &  m, Index &  w  )

inline

                                             {
  return (2 * w + 4 + LUNoMarker) * m * sizeof(Index) + (w + 1) * m * sizeof(Scalar);
}

References LUNoMarker, m, and w.

main_igamma_term()

template<typename Scalar >

static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar Eigen::internal::main_igamma_term ( Scalar  a, Scalar  x  )

static

                                                                                         {
  /* Compute  x**a * exp(-x) / gamma(a)  */
  Scalar logax = a * numext::log(x) - x - lgamma_impl<Scalar>::run(a);
  if (logax < -numext::log(NumTraits<Scalar>::highest()) ||
      // Assuming x and a aren't Nan.
      (numext::isnan)(logax)) {
    return Scalar(0);
  }
  return numext::exp(logax);
}

References a, Eigen::numext::exp(), Eigen::numext::isnan(), Eigen::numext::log(), Eigen::internal::lgamma_impl< Scalar >::run(), and Eigen::numext::x.

make_block_householder_triangular_factor()

template<typename TriangularFactorType , typename VectorsType , typename CoeffsType >

void Eigen::internal::make_block_householder_triangular_factor	(	TriangularFactorType &	triFactor,
		const VectorsType &	vectors,
		const CoeffsType &	hCoeffs
	)

                                                                         {
  const Index nbVecs = vectors.cols();
  eigen_assert(triFactor.rows() == nbVecs && triFactor.cols() == nbVecs && vectors.rows() >= nbVecs);
 
  for (Index i = nbVecs - 1; i >= 0; --i) {
    Index rs = vectors.rows() - i - 1;
    Index rt = nbVecs - i - 1;
 
    if (rt > 0) {
      triFactor.row(i).tail(rt).noalias() = -hCoeffs(i) * vectors.col(i).tail(rs).adjoint() *
                                            vectors.bottomRightCorner(rs, rt).template triangularView<UnitLower>();
 
      // FIXME use the following line with .noalias() once the triangular product can work inplace
      // triFactor.row(i).tail(rt) = triFactor.row(i).tail(rt) * triFactor.bottomRightCorner(rt,rt).template
      // triangularView<Upper>();
      for (Index j = nbVecs - 1; j > i; --j) {
        typename TriangularFactorType::Scalar z = triFactor(i, j);
        triFactor(i, j) = z * triFactor(j, j);
        if (nbVecs - j - 1 > 0) triFactor.row(i).tail(nbVecs - j - 1) += z * triFactor.row(j).tail(nbVecs - j - 1);
      }
    }
    triFactor(i, i) = hCoeffs(i);
  }
}

References eigen_assert, i, and j.

Referenced by apply_block_householder_on_the_left().

make_packet2d()

EIGEN_ALWAYS_INLINE Packet2d Eigen::internal::make_packet2d	(	double	a,
		double	b
	)

                                                               {
  double from[2] = {a, b};
  return (Packet2d)__lsx_vld(from, 0);
}

References a, and b.

Referenced by paddsub< Packet2d >(), pnegate(), and shuffle().

make_packet2f()

EIGEN_ALWAYS_INLINE Packet2f Eigen::internal::make_packet2f	(	float	a,
		float	b
	)

{ return Packet2f{a, b}; }

References a, and b.

Referenced by paddsub< Packet2f >().

make_packet4f()

EIGEN_ALWAYS_INLINE Packet4f Eigen::internal::make_packet4f	(	float	a,
		float	b,
		float	c,
		float	d
	)

                                                                               {
  float from[4] = {a, b, c, d};
  return (Packet4f)__lsx_vld(from, 0);
}

References a, b, and calibrate::c.

Referenced by paddsub< Packet4f >(), shuffle1(), shuffle2(), and shuffle2< true >().

MakeCoherentCwiseBinaryOp()

template<typename Op , typename LhsDerivativeType , typename RhsDerivativeType >

auto Eigen::internal::MakeCoherentCwiseBinaryOp	(	const LhsDerivativeType &	x,
		const RhsDerivativeType &	y,
		Op	op = Op()
	)

                                                                                                     {
  const auto& lhs = MaybeCoherentPad(x, y);
  const auto& rhs = MaybeCoherentPad(y, x);
  return CwiseBinaryOp<Op, remove_all_t<decltype(lhs)>, remove_all_t<decltype(rhs)>>(lhs, rhs, op);
}

References MaybeCoherentPad(), op, plotDoE::x, and y.

Referenced by Eigen::AutoDiffScalar< DerivativeType >::operator*(), Eigen::AutoDiffScalar< DerivativeType >::operator+(), Eigen::AutoDiffScalar< DerivativeType >::operator-(), and Eigen::AutoDiffScalar< DerivativeType >::operator/().

manage_caching_sizes()

void Eigen::internal::manage_caching_sizes ( Action  action, std::ptrdiff_t *  l1, std::ptrdiff_t *  l2, std::ptrdiff_t *  l3  )

inline

                                                                                                      {
  static CacheSizes m_cacheSizes;
 
  if (action == SetAction) {
    // set the cpu cache size and cache all block sizes from a global cache size in byte
    eigen_internal_assert(l1 != 0 && l2 != 0);
    m_cacheSizes.m_l1 = *l1;
    m_cacheSizes.m_l2 = *l2;
    m_cacheSizes.m_l3 = *l3;
  } else if (action == GetAction) {
    eigen_internal_assert(l1 != 0 && l2 != 0);
    *l1 = m_cacheSizes.m_l1;
    *l2 = m_cacheSizes.m_l2;
    *l3 = m_cacheSizes.m_l3;
  } else {
    eigen_internal_assert(false);
  }
}

References calibrate::action, eigen_internal_assert, Eigen::GetAction, Eigen::internal::CacheSizes::m_l1, Eigen::internal::CacheSizes::m_l2, Eigen::internal::CacheSizes::m_l3, and Eigen::SetAction.

Referenced by evaluateProductBlockingSizesHeuristic(), Eigen::l1CacheSize(), Eigen::l2CacheSize(), Eigen::l3CacheSize(), Eigen::internal::triangular_solve_matrix< Scalar, Index, OnTheLeft, Mode, Conjugate, TriStorageOrder, ColMajor, OtherInnerStride >::run(), Eigen::internal::triangular_solve_matrix< Scalar, Index, OnTheRight, Mode, Conjugate, TriStorageOrder, ColMajor, OtherInnerStride >::run(), and Eigen::setCpuCacheSizes().

manage_caching_sizes_helper()

std::ptrdiff_t Eigen::internal::manage_caching_sizes_helper ( std::ptrdiff_t  a, std::ptrdiff_t  b  )

inline

Returns

b if a<=0, and returns a otherwise.

{ return a <= 0 ? b : a; }

References a, and b.

Referenced by Eigen::internal::CacheSizes::CacheSizes().

manage_multi_threading()

void Eigen::internal::manage_multi_threading ( Action  action, int *  v  )

inline

                                                          {
  if (action == SetAction) {
    eigen_internal_assert(v != nullptr);
  } else if (action == GetAction) {
    eigen_internal_assert(v != nullptr);
    *v = 1;
  } else {
    eigen_internal_assert(false);
  }
}

References calibrate::action, eigen_internal_assert, Eigen::GetAction, Eigen::SetAction, and v.

Referenced by Eigen::nbThreads(), and Eigen::setNbThreads().

map_superlu()

template<typename Scalar , int Flags, typename Index >

Map<SparseMatrix<Scalar, Flags, Index> > Eigen::internal::map_superlu

(

SluMatrix &

sluMat

)

View a Super LU matrix as an Eigen expression

                                                                        {
  eigen_assert(((Flags & RowMajor) == RowMajor && sluMat.Stype == SLU_NR) ||
               ((Flags & ColMajor) == ColMajor && sluMat.Stype == SLU_NC));
 
  Index outerSize = (Flags & RowMajor) == RowMajor ? sluMat.ncol : sluMat.nrow;
 
  return Map<SparseMatrix<Scalar, Flags, Index> >(sluMat.nrow, sluMat.ncol, sluMat.storage.outerInd[outerSize],
                                                  sluMat.storage.outerInd, sluMat.storage.innerInd,
                                                  reinterpret_cast<Scalar *>(sluMat.storage.values));
}

References Eigen::ColMajor, eigen_assert, Eigen::SluMatrix::innerInd, SuperMatrix::ncol, SuperMatrix::nrow, Eigen::SluMatrix::outerInd, Eigen::RowMajor, SLU_NC, SLU_NR, Eigen::SluMatrix::storage, SuperMatrix::Stype, and Eigen::SluMatrix::values.

matrix_exp_compute() [1/2]

template<typename ArgType , typename ResultType >

void Eigen::internal::matrix_exp_compute	(	const ArgType &	arg,
		ResultType &	result,
		false_type
	)

{
  typedef typename ArgType::PlainObject MatrixType;
  typedef typename traits<MatrixType>::Scalar Scalar;
  typedef typename NumTraits<Scalar>::Real RealScalar;
  typedef typename std::complex<RealScalar> ComplexScalar;
  result = arg.matrixFunction(internal::stem_function_exp<ComplexScalar>);
}

matrix_exp_compute() [2/2]

template<typename ArgType , typename ResultType >

void Eigen::internal::matrix_exp_compute	(	const ArgType &	arg,
		ResultType &	result,
		true_type
	)

{
  typedef typename ArgType::PlainObject MatrixType;
  MatrixType U, V;
  int squarings;
  matrix_exp_computeUV<MatrixType>::run(arg, U, V, squarings);  // Pade approximant is (U+V) / (-U+V)
  MatrixType numer = U + V;
  MatrixType denom = -U + V;
  result = denom.partialPivLu().solve(numer);
  for (int i = 0; i < squarings; i++) result *= result;  // undo scaling by repeated squaring
}

References i, Eigen::internal::matrix_exp_computeUV< MatrixType, RealScalar >::run(), RachelsAdvectionDiffusion::U, and V.

Referenced by Eigen::MatrixExponentialReturnValue< Derived >::evalTo().

matrix_exp_pade13()

template<typename MatA , typename MatU , typename MatV >

void Eigen::internal::matrix_exp_pade13	(	const MatA &	A,
		MatU &	U,
		MatV &	V
	)

Compute the (13,13)-Padé approximant to the exponential.

After exit, \( (V+U)(V-U)^{-1} \) is the Padé approximant of \( \exp(A) \) around \( A = 0 \).

                                                        {
  typedef typename MatA::PlainObject MatrixType;
  typedef typename NumTraits<typename traits<MatrixType>::Scalar>::Real RealScalar;
  const RealScalar b[] = {64764752532480000.L,
                          32382376266240000.L,
                          7771770303897600.L,
                          1187353796428800.L,
                          129060195264000.L,
                          10559470521600.L,
                          670442572800.L,
                          33522128640.L,
                          1323241920.L,
                          40840800.L,
                          960960.L,
                          16380.L,
                          182.L,
                          1.L};
  const MatrixType A2 = A * A;
  const MatrixType A4 = A2 * A2;
  const MatrixType A6 = A4 * A2;
  V = b[13] * A6 + b[11] * A4 + b[9] * A2;  // used for temporary storage
  MatrixType tmp = A6 * V;
  tmp += b[7] * A6 + b[5] * A4 + b[3] * A2 + b[1] * MatrixType::Identity(A.rows(), A.cols());
  U.noalias() = A * tmp;
  tmp = b[12] * A6 + b[10] * A4 + b[8] * A2;
  V.noalias() = A6 * tmp;
  V += b[6] * A6 + b[4] * A4 + b[2] * A2 + b[0] * MatrixType::Identity(A.rows(), A.cols());
}

References b, Eigen::PlainObjectBase< Derived >::cols(), Eigen::PlainObjectBase< Derived >::rows(), tmp, RachelsAdvectionDiffusion::U, and V.

Referenced by Eigen::internal::matrix_exp_computeUV< MatrixType, double >::run(), and Eigen::internal::matrix_exp_computeUV< MatrixType, long double >::run().

matrix_exp_pade3()

template<typename MatA , typename MatU , typename MatV >

void Eigen::internal::matrix_exp_pade3	(	const MatA &	A,
		MatU &	U,
		MatV &	V
	)

Compute the (3,3)-Padé approximant to the exponential.

After exit, \( (V+U)(V-U)^{-1} \) is the Padé approximant of \( \exp(A) \) around \( A = 0 \).

                                                       {
  typedef typename MatA::PlainObject MatrixType;
  typedef typename NumTraits<typename traits<MatA>::Scalar>::Real RealScalar;
  const RealScalar b[] = {120.L, 60.L, 12.L, 1.L};
  const MatrixType A2 = A * A;
  const MatrixType tmp = b[3] * A2 + b[1] * MatrixType::Identity(A.rows(), A.cols());
  U.noalias() = A * tmp;
  V = b[2] * A2 + b[0] * MatrixType::Identity(A.rows(), A.cols());
}

References b, Eigen::PlainObjectBase< Derived >::cols(), Eigen::PlainObjectBase< Derived >::rows(), tmp, RachelsAdvectionDiffusion::U, and V.

Referenced by Eigen::internal::matrix_exp_computeUV< MatrixType, float >::run(), Eigen::internal::matrix_exp_computeUV< MatrixType, double >::run(), and Eigen::internal::matrix_exp_computeUV< MatrixType, long double >::run().

matrix_exp_pade5()

template<typename MatA , typename MatU , typename MatV >

void Eigen::internal::matrix_exp_pade5	(	const MatA &	A,
		MatU &	U,
		MatV &	V
	)

Compute the (5,5)-Padé approximant to the exponential.

After exit, \( (V+U)(V-U)^{-1} \) is the Padé approximant of \( \exp(A) \) around \( A = 0 \).

                                                       {
  typedef typename MatA::PlainObject MatrixType;
  typedef typename NumTraits<typename traits<MatrixType>::Scalar>::Real RealScalar;
  const RealScalar b[] = {30240.L, 15120.L, 3360.L, 420.L, 30.L, 1.L};
  const MatrixType A2 = A * A;
  const MatrixType A4 = A2 * A2;
  const MatrixType tmp = b[5] * A4 + b[3] * A2 + b[1] * MatrixType::Identity(A.rows(), A.cols());
  U.noalias() = A * tmp;
  V = b[4] * A4 + b[2] * A2 + b[0] * MatrixType::Identity(A.rows(), A.cols());
}

References b, Eigen::PlainObjectBase< Derived >::cols(), Eigen::PlainObjectBase< Derived >::rows(), tmp, RachelsAdvectionDiffusion::U, and V.

Referenced by Eigen::internal::matrix_exp_computeUV< MatrixType, float >::run(), Eigen::internal::matrix_exp_computeUV< MatrixType, double >::run(), and Eigen::internal::matrix_exp_computeUV< MatrixType, long double >::run().

matrix_exp_pade7()

template<typename MatA , typename MatU , typename MatV >

void Eigen::internal::matrix_exp_pade7	(	const MatA &	A,
		MatU &	U,
		MatV &	V
	)

Compute the (7,7)-Padé approximant to the exponential.

After exit, \( (V+U)(V-U)^{-1} \) is the Padé approximant of \( \exp(A) \) around \( A = 0 \).

                                                       {
  typedef typename MatA::PlainObject MatrixType;
  typedef typename NumTraits<typename traits<MatrixType>::Scalar>::Real RealScalar;
  const RealScalar b[] = {17297280.L, 8648640.L, 1995840.L, 277200.L, 25200.L, 1512.L, 56.L, 1.L};
  const MatrixType A2 = A * A;
  const MatrixType A4 = A2 * A2;
  const MatrixType A6 = A4 * A2;
  const MatrixType tmp = b[7] * A6 + b[5] * A4 + b[3] * A2 + b[1] * MatrixType::Identity(A.rows(), A.cols());
  U.noalias() = A * tmp;
  V = b[6] * A6 + b[4] * A4 + b[2] * A2 + b[0] * MatrixType::Identity(A.rows(), A.cols());
}

References b, Eigen::PlainObjectBase< Derived >::cols(), Eigen::PlainObjectBase< Derived >::rows(), tmp, RachelsAdvectionDiffusion::U, and V.

Referenced by Eigen::internal::matrix_exp_computeUV< MatrixType, float >::run(), Eigen::internal::matrix_exp_computeUV< MatrixType, double >::run(), and Eigen::internal::matrix_exp_computeUV< MatrixType, long double >::run().

matrix_exp_pade9()

template<typename MatA , typename MatU , typename MatV >

void Eigen::internal::matrix_exp_pade9	(	const MatA &	A,
		MatU &	U,
		MatV &	V
	)

Compute the (9,9)-Padé approximant to the exponential.

After exit, \( (V+U)(V-U)^{-1} \) is the Padé approximant of \( \exp(A) \) around \( A = 0 \).

                                                       {
  typedef typename MatA::PlainObject MatrixType;
  typedef typename NumTraits<typename traits<MatrixType>::Scalar>::Real RealScalar;
  const RealScalar b[] = {17643225600.L, 8821612800.L, 2075673600.L, 302702400.L, 30270240.L,
                          2162160.L,     110880.L,     3960.L,       90.L,        1.L};
  const MatrixType A2 = A * A;
  const MatrixType A4 = A2 * A2;
  const MatrixType A6 = A4 * A2;
  const MatrixType A8 = A6 * A2;
  const MatrixType tmp =
      b[9] * A8 + b[7] * A6 + b[5] * A4 + b[3] * A2 + b[1] * MatrixType::Identity(A.rows(), A.cols());
  U.noalias() = A * tmp;
  V = b[8] * A8 + b[6] * A6 + b[4] * A4 + b[2] * A2 + b[0] * MatrixType::Identity(A.rows(), A.cols());
}

References b, Eigen::PlainObjectBase< Derived >::cols(), Eigen::PlainObjectBase< Derived >::rows(), tmp, RachelsAdvectionDiffusion::U, and V.

Referenced by Eigen::internal::matrix_exp_computeUV< MatrixType, double >::run(), and Eigen::internal::matrix_exp_computeUV< MatrixType, long double >::run().

matrix_function_compute_above_diagonal()

template<typename MatrixType , typename VectorType >

void Eigen::internal::matrix_function_compute_above_diagonal	(	const MatrixType &	T,
		const VectorType &	blockStart,
		const VectorType &	clusterSize,
		MatrixType &	fT
	)

Compute part of matrix function above block diagonal.

This routine completes the computation of fT, denoting a matrix function applied to the triangular matrix T. It assumes that the block diagonal part of fT has already been computed. The part below the diagonal is zero, because T is upper triangular.

                                                                                           {
  typedef internal::traits<MatrixType> Traits;
  typedef typename MatrixType::Scalar Scalar;
  static const int Options = MatrixType::Options;
  typedef Matrix<Scalar, Dynamic, Dynamic, Options, Traits::RowsAtCompileTime, Traits::ColsAtCompileTime> DynMatrixType;
 
  for (Index k = 1; k < clusterSize.rows(); k++) {
    for (Index i = 0; i < clusterSize.rows() - k; i++) {
      // compute (i, i+k) block
      DynMatrixType A = T.block(blockStart(i), blockStart(i), clusterSize(i), clusterSize(i));
      DynMatrixType B = -T.block(blockStart(i + k), blockStart(i + k), clusterSize(i + k), clusterSize(i + k));
      DynMatrixType C = fT.block(blockStart(i), blockStart(i), clusterSize(i), clusterSize(i)) *
                        T.block(blockStart(i), blockStart(i + k), clusterSize(i), clusterSize(i + k));
      C -= T.block(blockStart(i), blockStart(i + k), clusterSize(i), clusterSize(i + k)) *
           fT.block(blockStart(i + k), blockStart(i + k), clusterSize(i + k), clusterSize(i + k));
      for (Index m = i + 1; m < i + k; m++) {
        C += fT.block(blockStart(i), blockStart(m), clusterSize(i), clusterSize(m)) *
             T.block(blockStart(m), blockStart(i + k), clusterSize(m), clusterSize(i + k));
        C -= T.block(blockStart(i), blockStart(m), clusterSize(i), clusterSize(m)) *
             fT.block(blockStart(m), blockStart(i + k), clusterSize(m), clusterSize(i + k));
      }
      fT.block(blockStart(i), blockStart(i + k), clusterSize(i), clusterSize(i + k)) =
          matrix_function_solve_triangular_sylvester(A, B, C);
    }
  }
}

References i, k, m, and matrix_function_solve_triangular_sylvester().

Referenced by Eigen::internal::matrix_function_compute< MatrixType, 1 >::run().

matrix_function_compute_block_atomic()

template<typename MatrixType , typename AtomicType , typename VectorType >

void Eigen::internal::matrix_function_compute_block_atomic	(	const MatrixType &	T,
		AtomicType &	atomic,
		const VectorType &	blockStart,
		const VectorType &	clusterSize,
		MatrixType &	fT
	)

Compute block diagonal part of matrix function.

This routine computes the matrix function applied to the block diagonal part of T (which should be upper triangular), with the blocking given by blockStart and clusterSize. The matrix function of each diagonal block is computed by atomic. The off-diagonal parts of fT are set to zero.

                                                                                         {
  fT.setZero(T.rows(), T.cols());
  for (Index i = 0; i < clusterSize.rows(); ++i) {
    fT.block(blockStart(i), blockStart(i), clusterSize(i), clusterSize(i)) =
        atomic.compute(T.block(blockStart(i), blockStart(i), clusterSize(i), clusterSize(i)));
  }
}

References i.

Referenced by Eigen::internal::matrix_function_compute< MatrixType, 1 >::run().

matrix_function_compute_block_start()

template<typename VectorType >

void Eigen::internal::matrix_function_compute_block_start	(	const VectorType &	clusterSize,
		VectorType &	blockStart
	)

Compute start of each block using clusterSize.

                                                                                                {
  blockStart.resize(clusterSize.rows());
  blockStart(0) = 0;
  for (Index i = 1; i < clusterSize.rows(); i++) {
    blockStart(i) = blockStart(i - 1) + clusterSize(i - 1);
  }
}

References i.

Referenced by Eigen::internal::matrix_function_compute< MatrixType, 1 >::run().

matrix_function_compute_cluster_size()

template<typename ListOfClusters , typename Index >

void Eigen::internal::matrix_function_compute_cluster_size	(	const ListOfClusters &	clusters,
		Matrix< Index, Dynamic, 1 > &	clusterSize
	)

Compute size of each cluster given a partitioning.

                                                                                                                  {
  const Index numClusters = static_cast<Index>(clusters.size());
  clusterSize.setZero(numClusters);
  Index clusterIndex = 0;
  for (typename ListOfClusters::const_iterator cluster = clusters.begin(); cluster != clusters.end(); ++cluster) {
    clusterSize[clusterIndex] = cluster->size();
    ++clusterIndex;
  }
}

References Eigen::PlainObjectBase< Derived >::setZero().

Referenced by Eigen::internal::matrix_function_compute< MatrixType, 1 >::run().

matrix_function_compute_map()

template<typename EivalsType , typename ListOfClusters , typename VectorType >

void Eigen::internal::matrix_function_compute_map	(	const EivalsType &	eivals,
		const ListOfClusters &	clusters,
		VectorType &	eivalToCluster
	)

Compute mapping of eigenvalue indices to cluster indices.

                                                                                                                       {
  eivalToCluster.resize(eivals.rows());
  Index clusterIndex = 0;
  for (typename ListOfClusters::const_iterator cluster = clusters.begin(); cluster != clusters.end(); ++cluster) {
    for (Index i = 0; i < eivals.rows(); ++i) {
      if (std::find(cluster->begin(), cluster->end(), i) != cluster->end()) {
        eivalToCluster[i] = clusterIndex;
      }
    }
    ++clusterIndex;
  }
}

References eivals, and i.

Referenced by Eigen::internal::matrix_function_compute< MatrixType, 1 >::run().

matrix_function_compute_mu()

template<typename MatrixType >

NumTraits<typename MatrixType::Scalar>::Real Eigen::internal::matrix_function_compute_mu

(

const MatrixType &

A

)

                                                                                                  {
  typedef typename plain_col_type<MatrixType>::type VectorType;
  Index rows = A.rows();
  const MatrixType N = MatrixType::Identity(rows, rows) - A;
  VectorType e = VectorType::Ones(rows);
  N.template triangularView<Upper>().solveInPlace(e);
  return e.cwiseAbs().maxCoeff();
}

References e(), N, rows, and Eigen::PlainObjectBase< Derived >::rows().

Referenced by Eigen::internal::MatrixFunctionAtomic< MatrixType >::compute().

matrix_function_compute_permutation()

template<typename DynVectorType , typename VectorType >

void Eigen::internal::matrix_function_compute_permutation	(	const DynVectorType &	blockStart,
		const DynVectorType &	eivalToCluster,
		VectorType &	permutation
	)

Compute permutation which groups ei'vals in same cluster together.

                                                                  {
  DynVectorType indexNextEntry = blockStart;
  permutation.resize(eivalToCluster.rows());
  for (Index i = 0; i < eivalToCluster.rows(); i++) {
    Index cluster = eivalToCluster[i];
    permutation[i] = indexNextEntry[cluster];
    ++indexNextEntry[cluster];
  }
}

References i.

Referenced by Eigen::internal::matrix_function_compute< MatrixType, 1 >::run().

matrix_function_find_cluster()

template<typename Index , typename ListOfClusters >

ListOfClusters::iterator Eigen::internal::matrix_function_find_cluster	(	Index	key,
		ListOfClusters &	clusters
	)

Find cluster in clusters containing some value.

Parameters

[in]

key

Value to find

Returns

Iterator to cluster containing key, or clusters.end() if no cluster in m_clusters contains key.

                                                                                                  {
  typename std::list<Index>::iterator j;
  for (typename ListOfClusters::iterator i = clusters.begin(); i != clusters.end(); ++i) {
    j = std::find(i->begin(), i->end(), key);
    if (j != i->end()) return i;
  }
  return clusters.end();
}

References i, and j.

Referenced by matrix_function_partition_eigenvalues().

matrix_function_partition_eigenvalues()

template<typename EivalsType , typename Cluster >

void Eigen::internal::matrix_function_partition_eigenvalues	(	const EivalsType &	eivals,
		std::list< Cluster > &	clusters
	)

Partition eigenvalues in clusters of ei'vals close to each other.

Parameters

[in]	eivals	Eigenvalues
[out]	clusters	Resulting partition of eigenvalues

The partition satisfies the following two properties:

Any eigenvalue in a certain cluster is at most matrix_function_separation() away from another eigenvalue

in the same cluster.

The distance between two eigenvalues in different clusters is more than matrix_function_separation().

The implementation follows Algorithm 4.1 in the paper of Davies and Higham.

                                                                                                 {
  typedef typename EivalsType::RealScalar RealScalar;
  for (Index i = 0; i < eivals.rows(); ++i) {
    // Find cluster containing i-th ei'val, adding a new cluster if necessary
    typename std::list<Cluster>::iterator qi = matrix_function_find_cluster(i, clusters);
    if (qi == clusters.end()) {
      Cluster l;
      l.push_back(i);
      clusters.push_back(l);
      qi = clusters.end();
      --qi;
    }
 
    // Look for other element to add to the set
    for (Index j = i + 1; j < eivals.rows(); ++j) {
      if (abs(eivals(j) - eivals(i)) <= RealScalar(matrix_function_separation) &&
          std::find(qi->begin(), qi->end(), j) == qi->end()) {
        typename std::list<Cluster>::iterator qj = matrix_function_find_cluster(j, clusters);
        if (qj == clusters.end()) {
          qi->push_back(j);
        } else {
          qi->insert(qi->end(), qj->begin(), qj->end());
          clusters.erase(qj);
        }
      }
    }
  }
}

References abs(), eivals, i, j, matrix_function_find_cluster(), and matrix_function_separation.

Referenced by Eigen::internal::matrix_function_compute< MatrixType, 1 >::run().

matrix_function_permute_schur()

template<typename VectorType , typename MatrixType >

void Eigen::internal::matrix_function_permute_schur	(	VectorType &	permutation,
		MatrixType &	U,
		MatrixType &	T
	)

Permute Schur decomposition in U and T according to permutation.

                                                                                          {
  for (Index i = 0; i < permutation.rows() - 1; i++) {
    Index j;
    for (j = i; j < permutation.rows(); j++) {
      if (permutation(j) == i) break;
    }
    eigen_assert(permutation(j) == i);
    for (Index k = j - 1; k >= i; k--) {
      JacobiRotation<typename MatrixType::Scalar> rotation;
      rotation.makeGivens(T(k, k + 1), T(k + 1, k + 1) - T(k, k));
      T.applyOnTheLeft(k, k + 1, rotation.adjoint());
      T.applyOnTheRight(k, k + 1, rotation);
      U.applyOnTheRight(k, k + 1, rotation);
      std::swap(permutation.coeffRef(k), permutation.coeffRef(k + 1));
    }
  }
}

References Eigen::JacobiRotation< Scalar >::adjoint(), eigen_assert, i, j, k, Eigen::JacobiRotation< Scalar >::makeGivens(), swap(), and RachelsAdvectionDiffusion::U.

Referenced by Eigen::internal::matrix_function_compute< MatrixType, 1 >::run().

matrix_function_solve_triangular_sylvester()

template<typename MatrixType >

MatrixType Eigen::internal::matrix_function_solve_triangular_sylvester	(	const MatrixType &	A,
		const MatrixType &	B,
		const MatrixType &	C
	)

Solve a triangular Sylvester equation AX + XB = C.

Parameters

[in]	A	the matrix A; should be square and upper triangular
[in]	B	the matrix B; should be square and upper triangular
[in]	C	the matrix C; should have correct size.

Returns

the solution X.

If A is m-by-m and B is n-by-n, then both C and X are m-by-n. The (i,j)-th component of the Sylvester equation is

\[ \sum_{k=i}^m A_{ik} X_{kj} + \sum_{k=1}^j X_{ik} B_{kj} = C_{ij}. \]

This can be re-arranged to yield:

\[ X_{ij} = \frac{1}{A_{ii} + B_{jj}} \Bigl( C_{ij} - \sum_{k=i+1}^m A_{ik} X_{kj} - \sum_{k=1}^{j-1} X_{ik} B_{kj} \Bigr). \]

It is assumed that A and B are such that the numerator is never zero (otherwise the Sylvester equation does not have a unique solution). In that case, these equations can be evaluated in the order \( i=m,\ldots,1 \) and \( j=1,\ldots,n \).

                                                                                                                     {
  eigen_assert(A.rows() == A.cols());
  eigen_assert(A.isUpperTriangular());
  eigen_assert(B.rows() == B.cols());
  eigen_assert(B.isUpperTriangular());
  eigen_assert(C.rows() == A.rows());
  eigen_assert(C.cols() == B.rows());
 
  typedef typename MatrixType::Scalar Scalar;
 
  Index m = A.rows();
  Index n = B.rows();
  MatrixType X(m, n);
 
  for (Index i = m - 1; i >= 0; --i) {
    for (Index j = 0; j < n; ++j) {
      // Compute AX = \sum_{k=i+1}^m A_{ik} X_{kj}
      Scalar AX;
      if (i == m - 1) {
        AX = 0;
      } else {
        Matrix<Scalar, 1, 1> AXmatrix = A.row(i).tail(m - 1 - i) * X.col(j).tail(m - 1 - i);
        AX = AXmatrix(0, 0);
      }
 
      // Compute XB = \sum_{k=1}^{j-1} X_{ik} B_{kj}
      Scalar XB;
      if (j == 0) {
        XB = 0;
      } else {
        Matrix<Scalar, 1, 1> XBmatrix = X.row(i).head(j) * B.col(j).head(j);
        XB = XBmatrix(0, 0);
      }
 
      X(i, j) = (C(i, j) - AX - XB) / (A(i, i) + B(j, j));
    }
  }
  return X;
}

References Eigen::PlainObjectBase< Derived >::cols(), eigen_assert, i, j, m, n, Eigen::PlainObjectBase< Derived >::rows(), and X.

Referenced by matrix_function_compute_above_diagonal().

matrix_log_compute_2x2()

template<typename MatrixType >

void Eigen::internal::matrix_log_compute_2x2	(	const MatrixType &	A,
		MatrixType &	result
	)

Compute logarithm of 2x2 triangular matrix.

                                                                     {
  typedef typename MatrixType::Scalar Scalar;
  typedef typename MatrixType::RealScalar RealScalar;
  using std::abs;
  using std::ceil;
  using std::imag;
  using std::log;
 
  Scalar logA00 = log(A(0, 0));
  Scalar logA11 = log(A(1, 1));
 
  result(0, 0) = logA00;
  result(1, 0) = Scalar(0);
  result(1, 1) = logA11;
 
  Scalar y = A(1, 1) - A(0, 0);
  if (y == Scalar(0)) {
    result(0, 1) = A(0, 1) / A(0, 0);
  } else if ((abs(A(0, 0)) < RealScalar(0.5) * abs(A(1, 1))) || (abs(A(0, 0)) > 2 * abs(A(1, 1)))) {
    result(0, 1) = A(0, 1) * (logA11 - logA00) / y;
  } else {
    // computation in previous branch is inaccurate if A(1,1) \approx A(0,0)
    RealScalar unwindingNumber = ceil((imag(logA11 - logA00) - RealScalar(EIGEN_PI)) / RealScalar(2 * EIGEN_PI));
    result(0, 1) = A(0, 1) * (numext::log1p(y / A(0, 0)) + Scalar(0, RealScalar(2 * EIGEN_PI) * unwindingNumber)) / y;
  }
}

References abs(), Eigen::bfloat16_impl::ceil(), EIGEN_PI, imag(), Eigen::imag(), Eigen::bfloat16_impl::log(), Eigen::bfloat16_impl::log1p(), and y.

Referenced by Eigen::internal::MatrixLogarithmAtomic< MatrixType >::compute().

matrix_log_compute_big()

template<typename MatrixType >

void Eigen::internal::matrix_log_compute_big	(	const MatrixType &	A,
		MatrixType &	result
	)

Compute logarithm of triangular matrices with size > 2.

This uses a inverse scale-and-square algorithm.

                                                                     {
  typedef typename MatrixType::Scalar Scalar;
  typedef typename NumTraits<Scalar>::Real RealScalar;
  using std::pow;
 
  int numberOfSquareRoots = 0;
  int numberOfExtraSquareRoots = 0;
  int degree;
  MatrixType T = A, sqrtT;
 
  const int maxPadeDegree = matrix_log_max_pade_degree<Scalar>::value;
  const RealScalar maxNormForPade = RealScalar(maxPadeDegree <= 5 ? 5.3149729967117310e-1L :  // single precision
                                                   maxPadeDegree <= 7 ? 2.6429608311114350e-1L
                                                                      :  // double precision
                                                   maxPadeDegree <= 8 ? 2.32777776523703892094e-1L
                                                                      :  // extended precision
                                                   maxPadeDegree <= 10 ? 1.05026503471351080481093652651105e-1L
                                                                       :                       // double-double
                                                   1.1880960220216759245467951592883642e-1L);  // quadruple precision
 
  while (true) {
    RealScalar normTminusI = (T - MatrixType::Identity(T.rows(), T.rows())).cwiseAbs().colwise().sum().maxCoeff();
    if (normTminusI < maxNormForPade) {
      degree = matrix_log_get_pade_degree(normTminusI);
      int degree2 = matrix_log_get_pade_degree(normTminusI / RealScalar(2));
      if ((degree - degree2 <= 1) || (numberOfExtraSquareRoots == 1)) break;
      ++numberOfExtraSquareRoots;
    }
    matrix_sqrt_triangular(T, sqrtT);
    T = sqrtT.template triangularView<Upper>();
    ++numberOfSquareRoots;
  }
 
  matrix_log_compute_pade(result, T, degree);
  result *= pow(RealScalar(2), RealScalar(numberOfSquareRoots));  // TODO replace by bitshift if possible
}

References constants::degree, matrix_log_compute_pade(), matrix_log_get_pade_degree(), Eigen::matrix_sqrt_triangular(), and Eigen::bfloat16_impl::pow().

Referenced by Eigen::internal::MatrixLogarithmAtomic< MatrixType >::compute().

matrix_log_compute_pade()

template<typename MatrixType >

void Eigen::internal::matrix_log_compute_pade	(	MatrixType &	result,
		const MatrixType &	T,
		int	degree
	)

                                                                                  {
  typedef typename NumTraits<typename MatrixType::Scalar>::Real RealScalar;
  const int minPadeDegree = 3;
  const int maxPadeDegree = 11;
  eigen_assert(degree >= minPadeDegree && degree <= maxPadeDegree);
  // FIXME this creates float-conversion-warnings if these are enabled.
  // Either manually convert each value, or disable the warning locally
  const RealScalar nodes[][maxPadeDegree] = {
      {0.1127016653792583114820734600217600L, 0.5000000000000000000000000000000000L,  // degree 3
       0.8872983346207416885179265399782400L},
      {0.0694318442029737123880267555535953L, 0.3300094782075718675986671204483777L,  // degree 4
       0.6699905217924281324013328795516223L, 0.9305681557970262876119732444464048L},
      {0.0469100770306680036011865608503035L, 0.2307653449471584544818427896498956L,  // degree 5
       0.5000000000000000000000000000000000L, 0.7692346550528415455181572103501044L,
       0.9530899229693319963988134391496965L},
      {0.0337652428984239860938492227530027L, 0.1693953067668677431693002024900473L,  // degree 6
       0.3806904069584015456847491391596440L, 0.6193095930415984543152508608403560L,
       0.8306046932331322568306997975099527L, 0.9662347571015760139061507772469973L},
      {0.0254460438286207377369051579760744L, 0.1292344072003027800680676133596058L,  // degree 7
       0.2970774243113014165466967939615193L, 0.5000000000000000000000000000000000L,
       0.7029225756886985834533032060384807L, 0.8707655927996972199319323866403942L,
       0.9745539561713792622630948420239256L},
      {0.0198550717512318841582195657152635L, 0.1016667612931866302042230317620848L,  // degree 8
       0.2372337950418355070911304754053768L, 0.4082826787521750975302619288199080L,
       0.5917173212478249024697380711800920L, 0.7627662049581644929088695245946232L,
       0.8983332387068133697957769682379152L, 0.9801449282487681158417804342847365L},
      {0.0159198802461869550822118985481636L, 0.0819844463366821028502851059651326L,  // degree 9
       0.1933142836497048013456489803292629L, 0.3378732882980955354807309926783317L,
       0.5000000000000000000000000000000000L, 0.6621267117019044645192690073216683L,
       0.8066857163502951986543510196707371L, 0.9180155536633178971497148940348674L,
       0.9840801197538130449177881014518364L},
      {0.0130467357414141399610179939577740L, 0.0674683166555077446339516557882535L,  // degree 10
       0.1602952158504877968828363174425632L, 0.2833023029353764046003670284171079L,
       0.4255628305091843945575869994351400L, 0.5744371694908156054424130005648600L,
       0.7166976970646235953996329715828921L, 0.8397047841495122031171636825574368L,
       0.9325316833444922553660483442117465L, 0.9869532642585858600389820060422260L},
      {0.0108856709269715035980309994385713L, 0.0564687001159523504624211153480364L,  // degree 11
       0.1349239972129753379532918739844233L, 0.2404519353965940920371371652706952L,
       0.3652284220238275138342340072995692L, 0.5000000000000000000000000000000000L,
       0.6347715779761724861657659927004308L, 0.7595480646034059079628628347293048L,
       0.8650760027870246620467081260155767L, 0.9435312998840476495375788846519636L,
       0.9891143290730284964019690005614287L}};
 
  const RealScalar weights[][maxPadeDegree] = {
      {0.2777777777777777777777777777777778L, 0.4444444444444444444444444444444444L,  // degree 3
       0.2777777777777777777777777777777778L},
      {0.1739274225687269286865319746109997L, 0.3260725774312730713134680253890003L,  // degree 4
       0.3260725774312730713134680253890003L, 0.1739274225687269286865319746109997L},
      {0.1184634425280945437571320203599587L, 0.2393143352496832340206457574178191L,  // degree 5
       0.2844444444444444444444444444444444L, 0.2393143352496832340206457574178191L,
       0.1184634425280945437571320203599587L},
      {0.0856622461895851725201480710863665L, 0.1803807865240693037849167569188581L,  // degree 6
       0.2339569672863455236949351719947755L, 0.2339569672863455236949351719947755L,
       0.1803807865240693037849167569188581L, 0.0856622461895851725201480710863665L},
      {0.0647424830844348466353057163395410L, 0.1398526957446383339507338857118898L,  // degree 7
       0.1909150252525594724751848877444876L, 0.2089795918367346938775510204081633L,
       0.1909150252525594724751848877444876L, 0.1398526957446383339507338857118898L,
       0.0647424830844348466353057163395410L},
      {0.0506142681451881295762656771549811L, 0.1111905172266872352721779972131204L,  // degree 8
       0.1568533229389436436689811009933007L, 0.1813418916891809914825752246385978L,
       0.1813418916891809914825752246385978L, 0.1568533229389436436689811009933007L,
       0.1111905172266872352721779972131204L, 0.0506142681451881295762656771549811L},
      {0.0406371941807872059859460790552618L, 0.0903240803474287020292360156214564L,  // degree 9
       0.1303053482014677311593714347093164L, 0.1561735385200014200343152032922218L,
       0.1651196775006298815822625346434870L, 0.1561735385200014200343152032922218L,
       0.1303053482014677311593714347093164L, 0.0903240803474287020292360156214564L,
       0.0406371941807872059859460790552618L},
      {0.0333356721543440687967844049466659L, 0.0747256745752902965728881698288487L,  // degree 10
       0.1095431812579910219977674671140816L, 0.1346333596549981775456134607847347L,
       0.1477621123573764350869464973256692L, 0.1477621123573764350869464973256692L,
       0.1346333596549981775456134607847347L, 0.1095431812579910219977674671140816L,
       0.0747256745752902965728881698288487L, 0.0333356721543440687967844049466659L},
      {0.0278342835580868332413768602212743L, 0.0627901847324523123173471496119701L,  // degree 11
       0.0931451054638671257130488207158280L, 0.1165968822959952399592618524215876L,
       0.1314022722551233310903444349452546L, 0.1364625433889503153572417641681711L,
       0.1314022722551233310903444349452546L, 0.1165968822959952399592618524215876L,
       0.0931451054638671257130488207158280L, 0.0627901847324523123173471496119701L,
       0.0278342835580868332413768602212743L}};
 
  MatrixType TminusI = T - MatrixType::Identity(T.rows(), T.rows());
  result.setZero(T.rows(), T.rows());
  for (int k = 0; k < degree; ++k) {
    RealScalar weight = weights[degree - minPadeDegree][k];
    RealScalar node = nodes[degree - minPadeDegree][k];
    result +=
        weight *
        (MatrixType::Identity(T.rows(), T.rows()) + node * TminusI).template triangularView<Upper>().solve(TminusI);
  }
}

References constants::degree, eigen_assert, k, and calibrate::weights.

Referenced by matrix_log_compute_big().

matrix_log_get_pade_degree() [1/3]

int Eigen::internal::matrix_log_get_pade_degree ( double  normTminusI )

inline

                                                          {
  const double maxNormForPade[] = {1.6206284795015624e-2 /* degree = 3 */, 5.3873532631381171e-2, 1.1352802267628681e-1,
                                   1.8662860613541288e-1, 2.642960831111435e-1};
  const int minPadeDegree = matrix_log_min_pade_degree<double>::value;
  const int maxPadeDegree = matrix_log_max_pade_degree<double>::value;
  int degree = minPadeDegree;
  for (; degree <= maxPadeDegree; ++degree)
    if (normTminusI <= maxNormForPade[degree - minPadeDegree]) break;
  return degree;
}

References constants::degree.

matrix_log_get_pade_degree() [2/3]

int Eigen::internal::matrix_log_get_pade_degree ( float  normTminusI )

inline

                                                         {
  const float maxNormForPade[] = {2.5111573934555054e-1 /* degree = 3 */, 4.0535837411880493e-1, 5.3149729967117310e-1};
  const int minPadeDegree = matrix_log_min_pade_degree<float>::value;
  const int maxPadeDegree = matrix_log_max_pade_degree<float>::value;
  int degree = minPadeDegree;
  for (; degree <= maxPadeDegree; ++degree)
    if (normTminusI <= maxNormForPade[degree - minPadeDegree]) break;
  return degree;
}

References constants::degree.

Referenced by matrix_log_compute_big().

matrix_log_get_pade_degree() [3/3]

int Eigen::internal::matrix_log_get_pade_degree ( long double  normTminusI )

inline

                                                               {
#if LDBL_MANT_DIG == 53  // double precision
  const long double maxNormForPade[] = {1.6206284795015624e-2L /* degree = 3 */, 5.3873532631381171e-2L,
                                        1.1352802267628681e-1L, 1.8662860613541288e-1L, 2.642960831111435e-1L};
#elif LDBL_MANT_DIG <= 64   // extended precision
  const long double maxNormForPade[] = {5.48256690357782863103e-3L /* degree = 3 */,
                                        2.34559162387971167321e-2L,
                                        5.84603923897347449857e-2L,
                                        1.08486423756725170223e-1L,
                                        1.68385767881294446649e-1L,
                                        2.32777776523703892094e-1L};
#elif LDBL_MANT_DIG <= 106  // double-double
  const long double maxNormForPade[] = {8.58970550342939562202529664318890e-5L /* degree = 3 */,
                                        9.34074328446359654039446552677759e-4L,
                                        4.26117194647672175773064114582860e-3L,
                                        1.21546224740281848743149666560464e-2L,
                                        2.61100544998339436713088248557444e-2L,
                                        4.66170074627052749243018566390567e-2L,
                                        7.32585144444135027565872014932387e-2L,
                                        1.05026503471351080481093652651105e-1L};
#else                       // quadruple precision
  const long double maxNormForPade[] = {4.7419931187193005048501568167858103e-5L /* degree = 3 */,
                                        5.8853168473544560470387769480192666e-4L,
                                        2.9216120366601315391789493628113520e-3L,
                                        8.8415758124319434347116734705174308e-3L,
                                        1.9850836029449446668518049562565291e-2L,
                                        3.6688019729653446926585242192447447e-2L,
                                        5.9290962294020186998954055264528393e-2L,
                                        8.6998436081634343903250580992127677e-2L,
                                        1.1880960220216759245467951592883642e-1L};
#endif
  const int minPadeDegree = matrix_log_min_pade_degree<long double>::value;
  const int maxPadeDegree = matrix_log_max_pade_degree<long double>::value;
  int degree = minPadeDegree;
  for (; degree <= maxPadeDegree; ++degree)
    if (normTminusI <= maxNormForPade[degree - minPadeDegree]) break;
  return degree;
}

References constants::degree.

matrix_sqrt_quasi_triangular_1x1_off_diagonal_block()

template<typename MatrixType , typename ResultType >

void Eigen::internal::matrix_sqrt_quasi_triangular_1x1_off_diagonal_block	(	const MatrixType &	T,
		Index	i,
		Index	j,
		ResultType &	sqrtT
	)

                                                                                                                   {
  typedef typename traits<MatrixType>::Scalar Scalar;
  Scalar tmp = (sqrtT.row(i).segment(i + 1, j - i - 1) * sqrtT.col(j).segment(i + 1, j - i - 1)).value();
  sqrtT.coeffRef(i, j) = (T.coeff(i, j) - tmp) / (sqrtT.coeff(i, i) + sqrtT.coeff(j, j));
}

References i, j, tmp, and Eigen::value.

Referenced by matrix_sqrt_quasi_triangular_off_diagonal().

matrix_sqrt_quasi_triangular_1x2_off_diagonal_block()

template<typename MatrixType , typename ResultType >

void Eigen::internal::matrix_sqrt_quasi_triangular_1x2_off_diagonal_block	(	const MatrixType &	T,
		Index	i,
		Index	j,
		ResultType &	sqrtT
	)

                                                                                                                   {
  typedef typename traits<MatrixType>::Scalar Scalar;
  Matrix<Scalar, 1, 2> rhs = T.template block<1, 2>(i, j);
  if (j - i > 1) rhs -= sqrtT.block(i, i + 1, 1, j - i - 1) * sqrtT.block(i + 1, j, j - i - 1, 2);
  Matrix<Scalar, 2, 2> A = sqrtT.coeff(i, i) * Matrix<Scalar, 2, 2>::Identity();
  A += sqrtT.template block<2, 2>(j, j).transpose();
  sqrtT.template block<1, 2>(i, j).transpose() = A.fullPivLu().solve(rhs.transpose());
}

References Eigen::PlainObjectBase< Derived >::coeff(), i, and j.

Referenced by matrix_sqrt_quasi_triangular_off_diagonal().

matrix_sqrt_quasi_triangular_2x1_off_diagonal_block()

template<typename MatrixType , typename ResultType >

void Eigen::internal::matrix_sqrt_quasi_triangular_2x1_off_diagonal_block	(	const MatrixType &	T,
		Index	i,
		Index	j,
		ResultType &	sqrtT
	)

                                                                                                                   {
  typedef typename traits<MatrixType>::Scalar Scalar;
  Matrix<Scalar, 2, 1> rhs = T.template block<2, 1>(i, j);
  if (j - i > 2) rhs -= sqrtT.block(i, i + 2, 2, j - i - 2) * sqrtT.block(i + 2, j, j - i - 2, 1);
  Matrix<Scalar, 2, 2> A = sqrtT.coeff(j, j) * Matrix<Scalar, 2, 2>::Identity();
  A += sqrtT.template block<2, 2>(i, i);
  sqrtT.template block<2, 1>(i, j) = A.fullPivLu().solve(rhs);
}

References Eigen::PlainObjectBase< Derived >::coeff(), i, and j.

Referenced by matrix_sqrt_quasi_triangular_off_diagonal().

matrix_sqrt_quasi_triangular_2x2_diagonal_block()

template<typename MatrixType , typename ResultType >

void Eigen::internal::matrix_sqrt_quasi_triangular_2x2_diagonal_block	(	const MatrixType &	T,
		Index	i,
		ResultType &	sqrtT
	)

                                                                                                      {
  // TODO: This case (2-by-2 blocks with complex conjugate eigenvalues) is probably hidden somewhere
  //       in EigenSolver. If we expose it, we could call it directly from here.
  typedef typename traits<MatrixType>::Scalar Scalar;
  Matrix<Scalar, 2, 2> block = T.template block<2, 2>(i, i);
  EigenSolver<Matrix<Scalar, 2, 2> > es(block);
  sqrtT.template block<2, 2>(i, i) =
      (es.eigenvectors() * es.eigenvalues().cwiseSqrt().asDiagonal() * es.eigenvectors().inverse()).real();
}

References block(), es, i, and Eigen::real().

Referenced by matrix_sqrt_quasi_triangular_diagonal().

matrix_sqrt_quasi_triangular_2x2_off_diagonal_block()

template<typename MatrixType , typename ResultType >

void Eigen::internal::matrix_sqrt_quasi_triangular_2x2_off_diagonal_block	(	const MatrixType &	T,
		Index	i,
		Index	j,
		ResultType &	sqrtT
	)

                                                                                                                   {
  typedef typename traits<MatrixType>::Scalar Scalar;
  Matrix<Scalar, 2, 2> A = sqrtT.template block<2, 2>(i, i);
  Matrix<Scalar, 2, 2> B = sqrtT.template block<2, 2>(j, j);
  Matrix<Scalar, 2, 2> C = T.template block<2, 2>(i, j);
  if (j - i > 2) C -= sqrtT.block(i, i + 2, 2, j - i - 2) * sqrtT.block(i + 2, j, j - i - 2, 2);
  Matrix<Scalar, 2, 2> X;
  matrix_sqrt_quasi_triangular_solve_auxiliary_equation(X, A, B, C);
  sqrtT.template block<2, 2>(i, j) = X;
}

References i, j, matrix_sqrt_quasi_triangular_solve_auxiliary_equation(), and X.

Referenced by matrix_sqrt_quasi_triangular_off_diagonal().

matrix_sqrt_quasi_triangular_diagonal()

template<typename MatrixType , typename ResultType >

void Eigen::internal::matrix_sqrt_quasi_triangular_diagonal	(	const MatrixType &	T,
		ResultType &	sqrtT
	)

                                                                                   {
  using std::sqrt;
  const Index size = T.rows();
  for (Index i = 0; i < size; i++) {
    if (i == size - 1 || T.coeff(i + 1, i) == 0) {
      eigen_assert(T(i, i) >= 0);
      sqrtT.coeffRef(i, i) = sqrt(T.coeff(i, i));
    } else {
      matrix_sqrt_quasi_triangular_2x2_diagonal_block(T, i, sqrtT);
      ++i;
    }
  }
}

References eigen_assert, i, matrix_sqrt_quasi_triangular_2x2_diagonal_block(), size, and sqrt().

Referenced by Eigen::matrix_sqrt_quasi_triangular().

matrix_sqrt_quasi_triangular_off_diagonal()

template<typename MatrixType , typename ResultType >

void Eigen::internal::matrix_sqrt_quasi_triangular_off_diagonal	(	const MatrixType &	T,
		ResultType &	sqrtT
	)

                                                                                       {
  const Index size = T.rows();
  for (Index j = 1; j < size; j++) {
    if (T.coeff(j, j - 1) != 0)  // if T(j-1:j, j-1:j) is a 2-by-2 block
      continue;
    for (Index i = j - 1; i >= 0; i--) {
      if (i > 0 && T.coeff(i, i - 1) != 0)  // if T(i-1:i, i-1:i) is a 2-by-2 block
        continue;
      bool iBlockIs2x2 = (i < size - 1) && (T.coeff(i + 1, i) != 0);
      bool jBlockIs2x2 = (j < size - 1) && (T.coeff(j + 1, j) != 0);
      if (iBlockIs2x2 && jBlockIs2x2)
        matrix_sqrt_quasi_triangular_2x2_off_diagonal_block(T, i, j, sqrtT);
      else if (iBlockIs2x2 && !jBlockIs2x2)
        matrix_sqrt_quasi_triangular_2x1_off_diagonal_block(T, i, j, sqrtT);
      else if (!iBlockIs2x2 && jBlockIs2x2)
        matrix_sqrt_quasi_triangular_1x2_off_diagonal_block(T, i, j, sqrtT);
      else if (!iBlockIs2x2 && !jBlockIs2x2)
        matrix_sqrt_quasi_triangular_1x1_off_diagonal_block(T, i, j, sqrtT);
    }
  }
}

References i, j, matrix_sqrt_quasi_triangular_1x1_off_diagonal_block(), matrix_sqrt_quasi_triangular_1x2_off_diagonal_block(), matrix_sqrt_quasi_triangular_2x1_off_diagonal_block(), matrix_sqrt_quasi_triangular_2x2_off_diagonal_block(), and size.

Referenced by Eigen::matrix_sqrt_quasi_triangular().

matrix_sqrt_quasi_triangular_solve_auxiliary_equation()

template<typename MatrixType >

void Eigen::internal::matrix_sqrt_quasi_triangular_solve_auxiliary_equation	(	MatrixType &	X,
		const MatrixType &	A,
		const MatrixType &	B,
		const MatrixType &	C
	)

                                                                                {
  typedef typename traits<MatrixType>::Scalar Scalar;
  Matrix<Scalar, 4, 4> coeffMatrix = Matrix<Scalar, 4, 4>::Zero();
  coeffMatrix.coeffRef(0, 0) = A.coeff(0, 0) + B.coeff(0, 0);
  coeffMatrix.coeffRef(1, 1) = A.coeff(0, 0) + B.coeff(1, 1);
  coeffMatrix.coeffRef(2, 2) = A.coeff(1, 1) + B.coeff(0, 0);
  coeffMatrix.coeffRef(3, 3) = A.coeff(1, 1) + B.coeff(1, 1);
  coeffMatrix.coeffRef(0, 1) = B.coeff(1, 0);
  coeffMatrix.coeffRef(0, 2) = A.coeff(0, 1);
  coeffMatrix.coeffRef(1, 0) = B.coeff(0, 1);
  coeffMatrix.coeffRef(1, 3) = A.coeff(0, 1);
  coeffMatrix.coeffRef(2, 0) = A.coeff(1, 0);
  coeffMatrix.coeffRef(2, 3) = B.coeff(1, 0);
  coeffMatrix.coeffRef(3, 1) = A.coeff(1, 0);
  coeffMatrix.coeffRef(3, 2) = B.coeff(0, 1);
 
  Matrix<Scalar, 4, 1> rhs;
  rhs.coeffRef(0) = C.coeff(0, 0);
  rhs.coeffRef(1) = C.coeff(0, 1);
  rhs.coeffRef(2) = C.coeff(1, 0);
  rhs.coeffRef(3) = C.coeff(1, 1);
 
  Matrix<Scalar, 4, 1> result;
  result = coeffMatrix.fullPivLu().solve(rhs);
 
  X.coeffRef(0, 0) = result.coeff(0);
  X.coeffRef(0, 1) = result.coeff(1);
  X.coeffRef(1, 0) = result.coeff(2);
  X.coeffRef(1, 1) = result.coeff(3);
}

References Eigen::PlainObjectBase< Derived >::coeff(), Eigen::Matrix< Scalar_, Rows_, Cols_, Options_, MaxRows_, MaxCols_ >::coeffRef(), X, and oomph::PseudoSolidHelper::Zero.

Referenced by matrix_sqrt_quasi_triangular_2x2_off_diagonal_block().

max_size_prefer_dynamic()

template<typename A , typename B >

constexpr int Eigen::internal::max_size_prefer_dynamic ( A  a, B  b  )

inlineconstexpr

see min_size_prefer_fixed. No need for a separate variant for MaxSizes here.

                                                       {
  plain_enum_asserts(a, b);
  if ((int)a == Dynamic || (int)b == Dynamic) return Dynamic;
  return plain_enum_max(a, b);
}

References a, b, Eigen::Dynamic, plain_enum_asserts(), and plain_enum_max().

Referenced by householder().

MaybeCoherentPad()

template<typename DerivativeType , typename OtherDerivativeType >

maybe_coherent_pad_helper<DerivativeType, OtherDerivativeType>::type Eigen::internal::MaybeCoherentPad	(	const DerivativeType &	x,
		const OtherDerivativeType &	y
	)

                                                           {
  return maybe_coherent_pad_helper<DerivativeType, OtherDerivativeType>::pad(x, y);
}

References Eigen::internal::maybe_coherent_pad_helper< DerivativeType, OtherDerivativeType, EnableIf >::pad(), plotDoE::x, and y.

Referenced by MakeCoherentCwiseBinaryOp().

MICRO_COMPLEX_EXTRA_ROW()

template<typename Scalar , typename Packet , const Index accRows, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal, const Index remaining_rows>

EIGEN_ALWAYS_INLINE void Eigen::internal::MICRO_COMPLEX_EXTRA_ROW	(	const Scalar *&	lhs_ptr_real,
		const Scalar *&	lhs_ptr_imag,
		const Scalar *&	rhs_ptr_real0,
		const Scalar *&	rhs_ptr_real1,
		const Scalar *&	rhs_ptr_real2,
		const Scalar *&	rhs_ptr_imag0,
		const Scalar *&	rhs_ptr_imag1,
		const Scalar *&	rhs_ptr_imag2,
		PacketBlock< Packet, accRows > &	accReal,
		PacketBlock< Packet, accRows > &	accImag
	)

                                                                                        {
  MICRO_COMPLEX_BROADCAST_EXTRA
  pgerc<accRows, Scalar, Packet, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(&accReal, &accImag, lhs_ptr_real,
                                                                                   lhs_ptr_imag, rhsV, rhsVi);
  MICRO_COMPLEX_ADD_COLS(1)
}

References MICRO_COMPLEX_ADD_COLS, and MICRO_COMPLEX_BROADCAST_EXTRA.

MICRO_EXTRA_ROW()

template<typename Scalar , typename Packet , const Index accRows, const Index remaining_rows>

EIGEN_ALWAYS_INLINE void Eigen::internal::MICRO_EXTRA_ROW	(	const Scalar *&	lhs_ptr,
		const Scalar *&	rhs_ptr0,
		const Scalar *&	rhs_ptr1,
		const Scalar *&	rhs_ptr2,
		PacketBlock< Packet, accRows > &	accZero
	)

                                                                                                         {
  MICRO_BROADCAST_EXTRA
  pger<accRows, Scalar, Packet, false>(&accZero, lhs_ptr, rhsV);
  lhs_ptr += remaining_rows;
}

References MICRO_BROADCAST_EXTRA.

min_size_prefer_dynamic()

template<typename A , typename B >

constexpr int Eigen::internal::min_size_prefer_dynamic ( A  a, B  b  )

inlineconstexpr

min_size_prefer_dynamic gives the min between compile-time sizes. 0 has absolute priority, followed by 1, followed by Dynamic, followed by other finite values. The reason for giving Dynamic the priority over finite values is that min(3, Dynamic) should be Dynamic, since that could be anything between 0 and 3.

                                                       {
  plain_enum_asserts(a, b);
  if ((int)a == 0 || (int)b == 0) return 0;
  if ((int)a == 1 || (int)b == 1) return 1;
  if ((int)a == Dynamic || (int)b == Dynamic) return Dynamic;
  return plain_enum_min(a, b);
}

References a, b, Eigen::Dynamic, plain_enum_asserts(), and plain_enum_min().

Referenced by check_random_matrix(), svd_compute_checks(), and Eigen::umeyama().

min_size_prefer_fixed()

template<typename A , typename B >

constexpr int Eigen::internal::min_size_prefer_fixed ( A  a, B  b  )

inlineconstexpr

min_size_prefer_fixed is a variant of min_size_prefer_dynamic comparing MaxSizes. The difference is that finite values now have priority over Dynamic, so that min(3, Dynamic) gives 3. Indeed, whatever the actual value is (between 0 and 3), it is not more than 3.

                                                     {
  plain_enum_asserts(a, b);
  if ((int)a == 0 || (int)b == 0) return 0;
  if ((int)a == 1 || (int)b == 1) return 1;
  if ((int)a == Dynamic && (int)b == Dynamic) return Dynamic;
  if ((int)a == Dynamic) return (int)b;
  if ((int)b == Dynamic) return (int)a;
  return plain_enum_min(a, b);
}

References a, b, Eigen::Dynamic, plain_enum_asserts(), and plain_enum_min().

Referenced by partial_lu_inplace().

minimize_helper()

template<typename BVH , typename Minimizer >

Minimizer::Scalar Eigen::internal::minimize_helper	(	const BVH &	tree,
		Minimizer &	minimizer,
		typename BVH::Index	root,
		typename Minimizer::Scalar	minimum
	)

                                                                             {
  typedef typename Minimizer::Scalar Scalar;
  typedef typename BVH::Index Index;
  typedef std::pair<Scalar, Index> QueueElement;  // first element is priority
  typedef typename BVH::VolumeIterator VolIter;
  typedef typename BVH::ObjectIterator ObjIter;
 
  VolIter vBegin = VolIter(), vEnd = VolIter();
  ObjIter oBegin = ObjIter(), oEnd = ObjIter();
  std::priority_queue<QueueElement, std::vector<QueueElement>, std::greater<QueueElement> >
      todo;  // smallest is at the top
 
  todo.push(std::make_pair(Scalar(), root));
 
  while (!todo.empty()) {
    tree.getChildren(todo.top().second, vBegin, vEnd, oBegin, oEnd);
    todo.pop();
 
    for (; oBegin != oEnd; ++oBegin)  // go through child objects
      minimum = (std::min)(minimum, minimizer.minimumOnObject(*oBegin));
 
    for (; vBegin != vEnd; ++vBegin) {  // go through child volumes
      Scalar val = minimizer.minimumOnVolume(tree.getVolume(*vBegin));
      if (val < minimum) todo.push(std::make_pair(val, *vBegin));
    }
  }
 
  return minimum;
}

References min, and calibrate::val.

Referenced by Eigen::BVMinimize().

minres()

template<typename MatrixType , typename Rhs , typename Dest , typename Preconditioner >

EIGEN_DONT_INLINE void Eigen::internal::minres	(	const MatrixType &	mat,
		const Rhs &	rhs,
		Dest &	x,
		const Preconditioner &	precond,
		Index &	iters,
		typename Dest::RealScalar &	tol_error
	)

Low-level MINRES algorithm

Parameters

mat	The matrix A
rhs	The right hand side vector b
x	On input and initial solution, on output the computed solution.
precond	A right preconditioner being able to efficiently solve for an approximation of Ax=b (regardless of b)
iters	On input the max number of iteration, on output the number of performed iterations.
tol_error	On input the tolerance error, on output an estimation of the relative error.

                                                                                {
  using std::sqrt;
  typedef typename Dest::RealScalar RealScalar;
  typedef typename Dest::Scalar Scalar;
  typedef Matrix<Scalar, Dynamic, 1> VectorType;
 
  // Check for zero rhs
  const RealScalar rhsNorm2(rhs.squaredNorm());
  if (rhsNorm2 == 0) {
    x.setZero();
    iters = 0;
    tol_error = 0;
    return;
  }
 
  // initialize
  const Index maxIters(iters);                                    // initialize maxIters to iters
  const Index N(mat.cols());                                      // the size of the matrix
  const RealScalar threshold2(tol_error * tol_error * rhsNorm2);  // convergence threshold (compared to residualNorm2)
 
  // Initialize preconditioned Lanczos
  VectorType v_old(N);                // will be initialized inside loop
  VectorType v(VectorType::Zero(N));  // initialize v
  VectorType v_new(rhs - mat * x);    // initialize v_new
  RealScalar residualNorm2(v_new.squaredNorm());
  VectorType w(N);                         // will be initialized inside loop
  VectorType w_new(precond.solve(v_new));  // initialize w_new
                                           //            RealScalar beta; // will be initialized inside loop
  RealScalar beta_new2(v_new.dot(w_new));
  eigen_assert(beta_new2 >= 0.0 && "PRECONDITIONER IS NOT POSITIVE DEFINITE");
  RealScalar beta_new(sqrt(beta_new2));
  const RealScalar beta_one(beta_new);
  // Initialize other variables
  RealScalar c(1.0);  // the cosine of the Givens rotation
  RealScalar c_old(1.0);
  RealScalar s(0.0);                      // the sine of the Givens rotation
  RealScalar s_old(0.0);                  // the sine of the Givens rotation
  VectorType p_oold(N);                   // will be initialized in loop
  VectorType p_old(VectorType::Zero(N));  // initialize p_old=0
  VectorType p(p_old);                    // initialize p=0
  RealScalar eta(1.0);
 
  iters = 0;  // reset iters
  while (iters < maxIters) {
    // Preconditioned Lanczos
    /* Note that there are 4 variants on the Lanczos algorithm. These are
     * described in Paige, C. C. (1972). Computational variants of
     * the Lanczos method for the eigenproblem. IMA Journal of Applied
     * Mathematics, 10(3), 373-381. The current implementation corresponds
     * to the case A(2,7) in the paper. It also corresponds to
     * algorithm 6.14 in Y. Saad, Iterative Methods for Sparse Linear
     * Systems, 2003 p.173. For the preconditioned version see
     * A. Greenbaum, Iterative Methods for Solving Linear Systems, SIAM (1987).
     */
    const RealScalar beta(beta_new);
    v_old = v;          // update: at first time step, this makes v_old = 0 so value of beta doesn't matter
    v_new /= beta_new;  // overwrite v_new for next iteration
    w_new /= beta_new;  // overwrite w_new for next iteration
    v = v_new;          // update
    w = w_new;          // update
    v_new.noalias() = mat * w - beta * v_old;  // compute v_new
    const RealScalar alpha = v_new.dot(w);
    v_new -= alpha * v;            // overwrite v_new
    w_new = precond.solve(v_new);  // overwrite w_new
    beta_new2 = v_new.dot(w_new);  // compute beta_new
    eigen_assert(beta_new2 >= 0.0 && "PRECONDITIONER IS NOT POSITIVE DEFINITE");
    beta_new = sqrt(beta_new2);  // compute beta_new
 
    // Givens rotation
    const RealScalar r2 = s * alpha + c * c_old * beta;  // s, s_old, c and c_old are still from previous iteration
    const RealScalar r3 = s_old * beta;                  // s, s_old, c and c_old are still from previous iteration
    const RealScalar r1_hat = c * alpha - c_old * s * beta;
    const RealScalar r1 = sqrt(std::pow(r1_hat, 2) + std::pow(beta_new, 2));
    c_old = c;          // store for next iteration
    s_old = s;          // store for next iteration
    c = r1_hat / r1;    // new cosine
    s = beta_new / r1;  // new sine
 
    // Update solution
    p_oold = p_old;
    p_old = p;
    p.noalias() = (w - r2 * p_old - r3 * p_oold) / r1;  // IS NOALIAS REQUIRED?
    x += beta_one * c * eta * p;
 
    /* Update the squared residual. Note that this is the estimated residual.
    The real residual |Ax-b|^2 may be slightly larger */
    residualNorm2 *= s * s;
 
    if (residualNorm2 < threshold2) {
      break;
    }
 
    eta = -s * eta;  // update eta
    iters++;         // increment iteration number (for output purposes)
  }
 
  /* Compute error. Note that this is the estimated error. The real
   error |Ax-b|/|b| may be slightly larger */
  tol_error = std::sqrt(residualNorm2 / rhsNorm2);
}

References alpha, beta, calibrate::c, Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::cols(), eigen_assert, TestSoln::eta, N, p, Eigen::bfloat16_impl::pow(), s, sqrt(), v, w, plotDoE::x, and oomph::PseudoSolidHelper::Zero.

Referenced by Eigen::MINRES< MatrixType_, UpLo_, Preconditioner_ >::_solve_vector_with_guess_impl(), and main().

move_construct_elements_of_array()

template<typename T >

EIGEN_DEVICE_FUNC T* Eigen::internal::move_construct_elements_of_array ( T *  ptr, T *  src, std::size_t  size  )

inline

Move-constructs the elements of an array. The size parameter tells on how many objects to move.

                                                                                             {
  std::size_t i = 0;
  EIGEN_TRY {
    for (i = 0; i < size; ++i) ::new (ptr + i) T(std::move(*(src + i)));
  }
  EIGEN_CATCH(...) {
    destruct_elements_of_array(ptr, i);
    EIGEN_THROW;
  }
  return ptr;
}

References destruct_elements_of_array(), EIGEN_CATCH, EIGEN_THROW, EIGEN_TRY, i, and size.

Referenced by conditional_aligned_realloc_new().

multVec()

template<Index num_acc>

EIGEN_ALWAYS_INLINE void Eigen::internal::multVec	(	__vector_quad(&)	quad_acc[num_acc],
		Packet8bf(&)	a0[num_acc],
		Packet8bf	b0
	)

                                                                                                             {
  BFLOAT16_UNROLL
  for (Index k = 0; k < num_acc; k++) {
    __builtin_mma_xvbf16ger2pp(&(quad_acc[k]), reinterpret_cast<Packet16uc>(b0.m_val),
                               reinterpret_cast<Packet16uc>(a0[k].m_val));
  }
}

References BFLOAT16_UNROLL, k, and Eigen::internal::eigen_packet_wrapper< T, unique_id >::m_val.

multVecLoop()

template<Index num_acc, typename LhsMapper , typename RhsMapper , bool extra>

EIGEN_ALWAYS_INLINE void Eigen::internal::multVecLoop	(	__vector_quad(&)	quad_acc[num_acc],
		const LhsMapper &	lhs,
		RhsMapper &	rhs,
		Index	j,
		Index	extra_cols
	)

                                                       {
  Packet8bf a0[num_acc], b0;
 
  if (extra) {
    b0 = rhs.template loadPacketPartial<Packet8bf>(j, extra_cols);
  } else {
    b0 = rhs.template loadPacket<Packet8bf>(j);
  }
 
  const LhsMapper lhs2 = lhs.getSubMapper(0, j);
  BFLOAT16_UNROLL
  for (Index k = 0; k < num_acc; k++) {
    if (extra) {
      a0[k] = lhs2.template loadPacketPartial<Packet8bf>(k, 0, extra_cols);
    } else {
      a0[k] = lhs2.template loadPacket<Packet8bf>(k, 0);
    }
  }
 
  multVec<num_acc>(quad_acc, a0, b0);
}

References BFLOAT16_UNROLL, j, and k.

muluh() [1/2]

template<typename T >

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE uint32_t Eigen::internal::muluh	(	const uint32_t	a,
		const T	b
	)

                                                                                  {
#if defined(EIGEN_GPU_COMPILE_PHASE)
  return __umulhi(a, b);
#elif defined(SYCL_DEVICE_ONLY)
  return cl::sycl::mul_hi(a, static_cast<uint32_t>(b));
#else
  return (static_cast<uint64_t>(a) * b) >> 32;
#endif
}

References a, and b.

Referenced by Eigen::internal::TensorIntDivisor< T, div_gt_one >::divide().

muluh() [2/2]

template<typename T >

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE uint64_t Eigen::internal::muluh	(	const uint64_t	a,
		const T	b
	)

                                                                                  {
#if defined(EIGEN_GPU_COMPILE_PHASE)
  return __umul64hi(a, b);
#elif defined(SYCL_DEVICE_ONLY)
  return cl::sycl::mul_hi(a, static_cast<uint64_t>(b));
#elif EIGEN_COMP_MSVC && (EIGEN_ARCH_x86_64 || EIGEN_ARCH_ARM64)
  return __umulh(a, static_cast<uint64_t>(b));
#elif EIGEN_HAS_BUILTIN_INT128
  __uint128_t v = static_cast<__uint128_t>(a) * static_cast<__uint128_t>(b);
  return static_cast<uint64_t>(v >> 64);
#else
  return (TensorUInt128<static_val<0>, uint64_t>(a) * TensorUInt128<static_val<0>, uint64_t>(b)).upper();
#endif
}

References a, b, and v.

nr_etdfs()

template<typename IndexVector >

void Eigen::internal::nr_etdfs	(	typename IndexVector::Scalar	n,
		IndexVector &	parent,
		IndexVector &	first_kid,
		IndexVector &	next_kid,
		IndexVector &	post,
		typename IndexVector::Scalar	postnum
	)

Depth-first search from vertex n. No recursion. This routine was contributed by Cédric Doucet, CEDRAT Group, Meylan, France.

                                                                     {
  typedef typename IndexVector::Scalar StorageIndex;
  StorageIndex current = n, first, next;
  while (postnum != n) {
    // No kid for the current node
    first = first_kid(current);
 
    // no kid for the current node
    if (first == -1) {
      // Numbering this node because it has no kid
      post(current) = postnum++;
 
      // looking for the next kid
      next = next_kid(current);
      while (next == -1) {
        // No more kids : back to the parent node
        current = parent(current);
        // numbering the parent node
        post(current) = postnum++;
 
        // Get the next kid
        next = next_kid(current);
      }
      // stopping criterion
      if (postnum == n + 1) return;
 
      // Updating current node
      current = next;
    } else {
      current = first;
    }
  }
}

References n.

Referenced by treePostorder().

omega()

template<typename Vector , typename RealScalar >

Vector::Scalar Eigen::internal::omega	(	const Vector &	t,
		const Vector &	s,
		RealScalar	angle
	)

Low-level Induced Dimension Reduction algorithm

Parameters

A	The matrix A
b	The right hand side vector b
x	On input and initial solution, on output the computed solution.
precond	A preconditioner being able to efficiently solve for an approximation of Ax=b (regardless of b)
iter	On input the max number of iteration, on output the number of performed iterations.
relres	On input the tolerance error, on output an estimation of the relative error.
S	On input Number of the dimension of the shadow space.
smoothing	switches residual smoothing on.
angle	small omega lead to faster convergence at the expense of numerical stability
replacement	switches on a residual replacement strategy to increase accuracy of residual at the expense of more Mat*vec products

Returns

false in the case of numerical issue, for example a break down of IDRS.

                                                                                {
  using numext::abs;
  typedef typename Vector::Scalar Scalar;
  const RealScalar ns = s.stableNorm();
  const RealScalar nt = t.stableNorm();
  const Scalar ts = t.dot(s);
  const RealScalar rho = abs(ts / (nt * ns));
 
  if (rho < angle) {
    if (ts == Scalar(0)) {
      return Scalar(0);
    }
    // Original relation for om is given by
    // om = om * angle / rho;
    // To alleviate potential (near) division by zero this can be rewritten as
    // om = angle * (ns / nt) * (ts / abs(ts)) = angle * (ns / nt) * sgn(ts)
    return angle * (ns / nt) * (ts / abs(ts));
  }
  return ts / (nt * nt);
}

References abs(), Eigen::numext::abs(), Jeffery_Solution::angle(), s, and plotPSD::t.

Referenced by oomph::Problem::activate_hopf_tracking(), AxisymOscillatingDisk::AxisymOscillatingDisk(), bicgstabl(), oomph::Problem::bifurcation_adapt_helper(), Coil::Coil(), oomph::DampedJacobi< MATRIX >::DampedJacobi(), oomph::SegregatableFSIProblem::enable_under_relaxation(), ExactSoln::get_exact_u(), HorizontalScrew::HorizontalScrew(), idrs(), HGridOptimiser::initialisePolyFunc(), main(), ExactSoln::prescribed_traction(), AxisymOscillatingDisk::residual_for_dispersion(), Screw::Screw(), Coil::set(), TriangulatedScrewSelfTest::setupInitialConditions(), CoupledBeam::setupOomph(), ABCProblem< ELEMENT, TIMESTEPPERT >::solve(), oomph::BlockHopfLinearSolver::solve(), oomph::BlockHopfLinearSolver::solve_for_two_rhs(), and oomph::BiCGStab< MATRIX >::solve_helper().

oneConvertBF16Hi()

EIGEN_ALWAYS_INLINE Packet4f Eigen::internal::oneConvertBF16Hi

(

Packet8us

data

)

                                                              {
  Packet8us z = pset1<Packet8us>(0);
#ifdef _BIG_ENDIAN
  return reinterpret_cast<Packet4f>(vec_mergeh(data, z));
#else
  return reinterpret_cast<Packet4f>(vec_mergeh(z, data));
#endif
}

References data, and pset1< Packet8us >().

Referenced by loadVecLoopVSX(), multVSXVecLoop(), and storeConvertTwoBF16().

oneConvertBF16Lo()

EIGEN_ALWAYS_INLINE Packet4f Eigen::internal::oneConvertBF16Lo

(

Packet8us

data

)

                                                              {
  Packet8us z = pset1<Packet8us>(0);
#ifdef _BIG_ENDIAN
  return reinterpret_cast<Packet4f>(vec_mergel(data, z));
#else
  return reinterpret_cast<Packet4f>(vec_mergel(z, data));
#endif
}

References data, and pset1< Packet8us >().

Referenced by loadVecLoopVSX(), multVSXVecLoop(), and storeConvertTwoBF16().

oneConvertBF16Perm()

EIGEN_ALWAYS_INLINE Packet4f Eigen::internal::oneConvertBF16Perm	(	Packet8us	data,
		Packet16uc	mask
	)

                                                                                 {
  Packet8us z = pset1<Packet8us>(0);
#ifdef _BIG_ENDIAN
  return reinterpret_cast<Packet4f>(vec_perm(data, z, mask));
#else
  return reinterpret_cast<Packet4f>(vec_perm(z, data, mask));
#endif
}

References data, and pset1< Packet8us >().

Referenced by convertArrayPointerBF16toF32DupOne(), and vecColLoopVSX().

operator!=()

template<typename HL , typename LL , typename HR , typename LR >

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool Eigen::internal::operator!=	(	const TensorUInt128< HL, LL > &	lhs,
		const TensorUInt128< HR, LR > &	rhs
	)

                                                                                        {
  return (lhs.high != rhs.high) || (lhs.low != rhs.low);
}

References Eigen::internal::TensorUInt128< HIGH, LOW >::high, and Eigen::internal::TensorUInt128< HIGH, LOW >::low.

operator*()

template<typename HL , typename LL , typename HR , typename LR >

static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorUInt128<uint64_t, uint64_t> Eigen::internal::operator* ( const TensorUInt128< HL, LL > &  lhs, const TensorUInt128< HR, LR > &  rhs  )

static

                                                                        {
  // Split each 128-bit integer into 4 32-bit integers, and then do the
  // multiplications by hand as follow:
  //   lhs      a  b  c  d
  //   rhs      e  f  g  h
  //           -----------
  //           ah bh ch dh
  //           bg cg dg
  //           cf df
  //           de
  // The result is stored in 2 64bit integers, high and low.
 
  const uint64_t LOW = 0x00000000FFFFFFFFLL;
  const uint64_t HIGH = 0xFFFFFFFF00000000LL;
 
  uint64_t d = lhs.low & LOW;
  uint64_t c = (lhs.low & HIGH) >> 32LL;
  uint64_t b = lhs.high & LOW;
  uint64_t a = (lhs.high & HIGH) >> 32LL;
 
  uint64_t h = rhs.low & LOW;
  uint64_t g = (rhs.low & HIGH) >> 32LL;
  uint64_t f = rhs.high & LOW;
  uint64_t e = (rhs.high & HIGH) >> 32LL;
 
  // Compute the low 32 bits of low
  uint64_t acc = d * h;
  uint64_t low = acc & LOW;
  //  Compute the high 32 bits of low. Add a carry every time we wrap around
  acc >>= 32LL;
  uint64_t carry = 0;
  uint64_t acc2 = acc + c * h;
  if (acc2 < acc) {
    carry++;
  }
  acc = acc2 + d * g;
  if (acc < acc2) {
    carry++;
  }
  low |= (acc << 32LL);
 
  // Carry forward the high bits of acc to initiate the computation of the
  // low 32 bits of high
  acc2 = (acc >> 32LL) | (carry << 32LL);
  carry = 0;
 
  acc = acc2 + b * h;
  if (acc < acc2) {
    carry++;
  }
  acc2 = acc + c * g;
  if (acc2 < acc) {
    carry++;
  }
  acc = acc2 + d * f;
  if (acc < acc2) {
    carry++;
  }
  uint64_t high = acc & LOW;
 
  // Start to compute the high 32 bits of high.
  acc2 = (acc >> 32LL) | (carry << 32LL);
 
  acc = acc2 + a * h;
  acc2 = acc + b * g;
  acc = acc2 + c * f;
  acc2 = acc + d * e;
  high |= (acc2 << 32LL);
 
  return TensorUInt128<uint64_t, uint64_t>(high, low);
}

References a, b, calibrate::c, e(), f(), Eigen::internal::TensorUInt128< HIGH, LOW >::high, and Eigen::internal::TensorUInt128< HIGH, LOW >::low.

operator+()

template<typename HL , typename LL , typename HR , typename LR >

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE TensorUInt128<uint64_t, uint64_t> Eigen::internal::operator+	(	const TensorUInt128< HL, LL > &	lhs,
		const TensorUInt128< HR, LR > &	rhs
	)

                                                                                                                    {
  TensorUInt128<uint64_t, uint64_t> result(lhs.high + rhs.high, lhs.low + rhs.low);
  if (result.low < rhs.low) {
    result.high += 1;
  }
  return result;
}

References Eigen::internal::TensorUInt128< HIGH, LOW >::high, and Eigen::internal::TensorUInt128< HIGH, LOW >::low.

operator-()

template<typename HL , typename LL , typename HR , typename LR >

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE TensorUInt128<uint64_t, uint64_t> Eigen::internal::operator-	(	const TensorUInt128< HL, LL > &	lhs,
		const TensorUInt128< HR, LR > &	rhs
	)

                                                                                                                    {
  TensorUInt128<uint64_t, uint64_t> result(lhs.high - rhs.high, lhs.low - rhs.low);
  if (result.low > lhs.low) {
    result.high -= 1;
  }
  return result;
}

References Eigen::internal::TensorUInt128< HIGH, LOW >::high, and Eigen::internal::TensorUInt128< HIGH, LOW >::low.

operator/() [1/2]

template<typename T , bool div_gt_one>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T Eigen::internal::operator/	(	const T &	numerator,
		const TensorIntDivisor< T, div_gt_one > &	divisor
	)

                                                                                                                      {
  return divisor.divide(numerator);
}

References Eigen::internal::TensorIntDivisor< T, div_gt_one >::divide().

operator/() [2/2]

template<typename HL , typename LL , typename HR , typename LR >

static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorUInt128<uint64_t, uint64_t> Eigen::internal::operator/ ( const TensorUInt128< HL, LL > &  lhs, const TensorUInt128< HR, LR > &  rhs  )

static

                                                                        {
  if (rhs == TensorUInt128<static_val<0>, static_val<1>>(1)) {
    return TensorUInt128<uint64_t, uint64_t>(lhs.high, lhs.low);
  } else if (lhs < rhs) {
    return TensorUInt128<uint64_t, uint64_t>(0);
  } else {
    // calculate the biggest power of 2 times rhs that's less than or equal to lhs
    TensorUInt128<uint64_t, uint64_t> power2(1);
    TensorUInt128<uint64_t, uint64_t> d(rhs);
    TensorUInt128<uint64_t, uint64_t> tmp(lhs - d);
    while (lhs >= d) {
      tmp = tmp - d;
      d = d + d;
      power2 = power2 + power2;
    }
 
    tmp = TensorUInt128<uint64_t, uint64_t>(lhs.high, lhs.low);
    TensorUInt128<uint64_t, uint64_t> result(0);
    while (power2 != TensorUInt128<static_val<0>, static_val<0>>(0)) {
      if (tmp >= d) {
        tmp = tmp - d;
        result = result + power2;
      }
      // Shift right
      power2 = TensorUInt128<uint64_t, uint64_t>(power2.high >> 1, (power2.low >> 1) | (power2.high << 63));
      d = TensorUInt128<uint64_t, uint64_t>(d.high >> 1, (d.low >> 1) | (d.high << 63));
    }
 
    return result;
  }
}

References Eigen::internal::TensorUInt128< HIGH, LOW >::high, Eigen::internal::TensorUInt128< HIGH, LOW >::low, and tmp.

operator<()

template<typename HL , typename LL , typename HR , typename LR >

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool Eigen::internal::operator<	(	const TensorUInt128< HL, LL > &	lhs,
		const TensorUInt128< HR, LR > &	rhs
	)

                                                                                       {
  if (lhs.high != rhs.high) {
    return lhs.high < rhs.high;
  }
  return lhs.low < rhs.low;
}

References Eigen::internal::TensorUInt128< HIGH, LOW >::high, and Eigen::internal::TensorUInt128< HIGH, LOW >::low.

operator<<() [1/15]

std::ostream& Eigen::internal::operator<< ( std::ostream &  os, const Packet1cd &  value  )

inline

                                                                      {
  os << "[ (" << value.v[0] << ", " << value.v[1] << "i) ]";
  return os;
}

References Eigen::value.

operator<<() [2/15]

std::ostream& Eigen::internal::operator<< ( std::ostream &  os, const Packet2cf &  value  )

inline

                                                                      {
  os << "[ (" << value.v[0] << ", " << value.v[1]
     << "i),"
        "  ("
     << value.v[2] << ", " << value.v[3] << "i) ]";
  return os;
}

References Eigen::value.

operator<<() [3/15]

std::ostream & Eigen::internal::operator<< ( std::ostream &  os, const Packet2d &  value  )

inline

                                                                     {
  os << "[ " << value[0] << ", " << value[1] << " ]";
  return os;
}

References Eigen::value.

operator<<() [4/15]

std::ostream & Eigen::internal::operator<< ( std::ostream &  os, const Packet2l &  value  )

inline

                                                                     {
  os << "[ " << value[0] << ", " << value[1] << " ]";
  return os;
}

References Eigen::value.

operator<<() [5/15]

std::ostream & Eigen::internal::operator<< ( std::ostream &  os, const Packet2ul &  value  )

inline

                                                                      {
  os << "[ " << value[0] << ", " << value[1] << " ]";
  return os;
}

References Eigen::value.

operator<<() [6/15]

std::ostream& Eigen::internal::operator<< ( std::ostream &  os, const PacketBlock< Packet1cd, 2 > &  value  )

inline

                                                                                      {
  os << "[ " << value.packet[0] << ", " << std::endl << "  " << value.packet[1] << " ]";
  return os;
}

References Eigen::value.

operator<<() [7/15]

std::ostream& Eigen::internal::operator<< ( std::ostream &  os, const PacketBlock< Packet2cf, 2 > &  value  )

inline

                                                                                      {
  os << "[ " << value.packet[0] << ", " << std::endl << "  " << value.packet[1] << " ]";
  return os;
}

References Eigen::value.

operator<<() [8/15]

std::ostream& Eigen::internal::operator<< ( std::ostream &  os, const PacketBlock< Packet2d, 2 > &  value  )

inline

                                                                                     {
  os << "[ " << value.packet[0] << "," << std::endl << "  " << value.packet[1] << " ]";
  return os;
}

References Eigen::value.

operator<<() [9/15]

std::ostream& Eigen::internal::operator<< ( std::ostream &  os, const PacketBlock< Packet4f, 4 > &  value  )

inline

                                                                                     {
  os << "[ " << value.packet[0] << "," << std::endl
     << "  " << value.packet[1] << "," << std::endl
     << "  " << value.packet[2] << "," << std::endl
     << "  " << value.packet[3] << " ]";
  return os;
}

References Eigen::value.

operator<<() [10/15]

std::ostream& Eigen::internal::operator<< ( std::ostream &  os, const PacketBlock< Packet4i, 4 > &  value  )

inline

                                                                                     {
  os << "[ " << value.packet[0] << "," << std::endl
     << "  " << value.packet[1] << "," << std::endl
     << "  " << value.packet[2] << "," << std::endl
     << "  " << value.packet[3] << " ]";
  return os;
}

References Eigen::value.

operator<<() [11/15]

std::ostream& Eigen::internal::operator<< ( std::ostream &  s, const Packet16c &  v  )

inline

                                                                 {
  union {
    Packet16c v;
    signed char n[16];
  } vt;
  vt.v = v;
  for (int i = 0; i < 16; i++) s << vt.n[i] << ", ";
  return s;
}

References i, n, s, and v.

operator<<() [12/15]

std::ostream& Eigen::internal::operator<< ( std::ostream &  s, const Packet16uc &  v  )

inline

                                                                  {
  union {
    Packet16uc v;
    unsigned char n[16];
  } vt;
  vt.v = v;
  for (int i = 0; i < 16; i++) s << vt.n[i] << ", ";
  return s;
}

References i, n, s, and v.

operator<<() [13/15]

std::ostream & Eigen::internal::operator<< ( std::ostream &  s, const Packet4f &  v  )

inline

                                                                {
  union {
    Packet4f v;
    float n[4];
  } vt;
  vt.v = v;
  s << vt.n[0] << ", " << vt.n[1] << ", " << vt.n[2] << ", " << vt.n[3];
  return s;
}

References n, s, and v.

operator<<() [14/15]

std::ostream & Eigen::internal::operator<< ( std::ostream &  s, const Packet4i &  v  )

inline

                                                                {
  union {
    Packet4i v;
    int n[4];
  } vt;
  vt.v = v;
  s << vt.n[0] << ", " << vt.n[1] << ", " << vt.n[2] << ", " << vt.n[3];
  return s;
}

References n, s, and v.

operator<<() [15/15]

std::ostream & Eigen::internal::operator<< ( std::ostream &  s, const Packet4ui &  v  )

inline

                                                                 {
  union {
    Packet4ui v;
    unsigned int n[4];
  } vt;
  vt.v = v;
  s << vt.n[0] << ", " << vt.n[1] << ", " << vt.n[2] << ", " << vt.n[3];
  return s;
}

References n, s, and v.

operator==()

template<typename HL , typename LL , typename HR , typename LR >

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool Eigen::internal::operator==	(	const TensorUInt128< HL, LL > &	lhs,
		const TensorUInt128< HR, LR > &	rhs
	)

                                                                                        {
  return (lhs.high == rhs.high) && (lhs.low == rhs.low);
}

References Eigen::internal::TensorUInt128< HIGH, LOW >::high, and Eigen::internal::TensorUInt128< HIGH, LOW >::low.

operator>=()

template<typename HL , typename LL , typename HR , typename LR >

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool Eigen::internal::operator>=	(	const TensorUInt128< HL, LL > &	lhs,
		const TensorUInt128< HR, LR > &	rhs
	)

                                                                                        {
  if (lhs.high != rhs.high) {
    return lhs.high > rhs.high;
  }
  return lhs.low >= rhs.low;
}

References Eigen::internal::TensorUInt128< HIGH, LOW >::high, and Eigen::internal::TensorUInt128< HIGH, LOW >::low.

outer_product_selector_run() [1/2]

template<typename Dst , typename Lhs , typename Rhs , typename Func >

void EIGEN_DEVICE_FUNC Eigen::internal::outer_product_selector_run	(	Dst &	dst,
		const Lhs &	lhs,
		const Rhs &	rhs,
		const Func &	func,
		const false_type &
	)

                                                                     {
  evaluator<Rhs> rhsEval(rhs);
  ei_declare_local_nested_eval(Lhs, lhs, Rhs::SizeAtCompileTime, actual_lhs);
  // FIXME if cols is large enough, then it might be useful to make sure that lhs is sequentially stored
  // FIXME not very good if rhs is real and lhs complex while alpha is real too
  const Index cols = dst.cols();
  for (Index j = 0; j < cols; ++j) func(dst.col(j), rhsEval.coeff(Index(0), j) * actual_lhs);
}

References cols, ei_declare_local_nested_eval, func(), j, and Lhs.

Referenced by Eigen::internal::generic_product_impl< Lhs, Rhs, DenseShape, DenseShape, OuterProduct >::addTo(), Eigen::internal::generic_product_impl< Lhs, Rhs, DenseShape, DenseShape, OuterProduct >::evalTo(), Eigen::internal::generic_product_impl< Lhs, Rhs, DenseShape, DenseShape, OuterProduct >::scaleAndAddTo(), and Eigen::internal::generic_product_impl< Lhs, Rhs, DenseShape, DenseShape, OuterProduct >::subTo().

outer_product_selector_run() [2/2]

template<typename Dst , typename Lhs , typename Rhs , typename Func >

void EIGEN_DEVICE_FUNC Eigen::internal::outer_product_selector_run	(	Dst &	dst,
		const Lhs &	lhs,
		const Rhs &	rhs,
		const Func &	func,
		const true_type &
	)

                                                                    {
  evaluator<Lhs> lhsEval(lhs);
  ei_declare_local_nested_eval(Rhs, rhs, Lhs::SizeAtCompileTime, actual_rhs);
  // FIXME if rows is large enough, then it might be useful to make sure that rhs is sequentially stored
  // FIXME not very good if lhs is real and rhs complex while alpha is real too
  const Index rows = dst.rows();
  for (Index i = 0; i < rows; ++i) func(dst.row(i), lhsEval.coeff(i, Index(0)) * actual_rhs);
}

References ei_declare_local_nested_eval, func(), i, Rhs, and rows.

outputResults()

template<Index num_acc, bool rhsExtraCols, bool lhsExtraRows, Index num_rhs, Index num_lhs>

EIGEN_ALWAYS_INLINE void Eigen::internal::outputResults	(	Packet4f(&)	acc[num_acc][4],
		Index	rows,
		const Packet4f	pAlpha,
		float *	result,
		const Index	extra_cols,
		Index	extra_rows
	)

                                                                                 {
  BFLOAT16_UNROLL
  for (Index i = 0, k = 0; i < num_rhs - (rhsExtraCols ? 1 : 0); i++, result += 4 * rows) {
    BFLOAT16_UNROLL
    for (Index j = 0; j < num_lhs; j++, k++) {
      storeResults<false, lhsExtraRows>(acc[k], rows, pAlpha, result + j * 4, extra_cols, extra_rows);
    }
  }
  if (rhsExtraCols) {
    storeResults<rhsExtraCols, lhsExtraRows>(acc[num_acc - 1], rows, pAlpha, result, extra_cols, extra_rows);
  }
}

References BFLOAT16_UNROLL, i, j, k, and rows.

outputResultsVSX()

template<Index num_acc, bool rhsExtraCols, bool lhsExtraRows, Index num_rhs>

EIGEN_ALWAYS_INLINE void Eigen::internal::outputResultsVSX	(	Packet4f(&)	acc[num_acc][4],
		Index	rows,
		const Packet4f	pAlpha,
		float *	result,
		const Index	extra_cols,
		Index	extra_rows
	)

                                                                                    {
  tranposeResults<num_acc>(acc);
  addResults<num_acc>(acc);
 
  constexpr Index real_rhs = ((num_rhs / 2) - (rhsExtraCols ? 1 : 0));
  Index k = 0;
  for (Index i = 0; i < real_rhs; i++, result += 4 * rows, k++) {
    storeResults<false, lhsExtraRows>(acc[k], rows, pAlpha, result, extra_cols, extra_rows);
  }
  if (rhsExtraCols) {
    storeResults<rhsExtraCols, lhsExtraRows>(acc[k], rows, pAlpha, result, extra_cols, extra_rows);
  }
}

References i, k, and rows.

outputVecColResults()

template<Index num_acc, bool extraRows, Index size = 4>

EIGEN_ALWAYS_INLINE void Eigen::internal::outputVecColResults	(	Packet4f(&)	acc[num_acc][size],
		float *	result,
		Packet4f	pAlpha,
		Index	extra_rows
	)

outputVecResults()

template<Index num_acc, Index size = 4>

EIGEN_ALWAYS_INLINE void Eigen::internal::outputVecResults	(	Packet4f(&)	acc[num_acc][size],
		float *	result,
		Packet4f	pAlpha
	)

p2ui_CONJ_XOR()

uint32x2_t Eigen::internal::p2ui_CONJ_XOR ( )

inline

                                  {
  static const uint32_t conj_XOR_DATA[] = {0x00000000, 0x80000000};
  return vld1_u32(conj_XOR_DATA);
}

Referenced by pconj(), and pmul< Packet1cf >().

pabs() [1/37]

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::pabs ( const Packet &  a )

inline

Returns

the absolute value of a

                                                      {
  return numext::abs(a);
}

References a, and Eigen::numext::abs().

pabs() [2/37]

template<>

EIGEN_STRONG_INLINE Packet16bf Eigen::internal::pabs

(

const Packet16bf &

a

)

                                                         {
  const __m256i sign_mask = _mm256_set1_epi16(static_cast<numext::uint16_t>(0x8000));
  return _mm256_andnot_si256(sign_mask, a);
}

References a.

pabs() [3/37]

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::pabs

(

const Packet16c &

a

)

                                                       {
  return vec_abs(a);
}

References a.

pabs() [4/37]

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::pabs

(

const Packet16f &

a

)

                                                       {
  // _mm512_abs_ps intrinsic not found, so hack around it
  return _mm512_castsi512_ps(_mm512_and_si512(_mm512_castps_si512(a), _mm512_set1_epi32(0x7fffffff)));
}

References a.

pabs() [5/37]

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::pabs

(

const Packet16h &

a

)

                                                       {
  const __m256i sign_mask = _mm256_set1_epi16(static_cast<numext::uint16_t>(0x8000));
  return _mm256_andnot_si256(sign_mask, a);
}

References a.

pabs() [6/37]

template<>

EIGEN_STRONG_INLINE Packet16i Eigen::internal::pabs

(

const Packet16i &

a

)

                                                       {
  return _mm512_abs_epi32(a);
}

References a.

pabs() [7/37]

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::pabs

(

const Packet16uc &

a

)

                                                         {
  return a;
}

References a.

pabs() [8/37]

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pabs

(

const Packet2d &

a

)

                                                     {
  return (Packet2d)__lsx_vbitclri_d((__m128i)a, 63);
}

References a.

pabs() [9/37]

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::pabs

(

const Packet2f &

a

)

                                                     {
  return vabs_f32(a);
}

References a.

pabs() [10/37]

template<>

EIGEN_STRONG_INLINE Packet2i Eigen::internal::pabs

(

const Packet2i &

a

)

                                                     {
  return vabs_s32(a);
}

References a.

pabs() [11/37]

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::pabs

(

const Packet2l &

a

)

                                                     {
  return __lsx_vabsd_d(a, pzero(a));
}

References a, and pzero().

pabs() [12/37]

template<>

EIGEN_STRONG_INLINE Packet2ui Eigen::internal::pabs

(

const Packet2ui &

a

)

                                                       {
  return a;
}

References a.

pabs() [13/37]

template<>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::pabs

(

const Packet2ul &

a

)

                                                       {
  return a;
}

References a.

pabs() [14/37]

template<>

EIGEN_STRONG_INLINE Packet4bf Eigen::internal::pabs

(

const Packet4bf &

a

)

                                                       {
  return F32ToBf16(pabs<Packet4f>(Bf16ToF32(a)));
}

References a, Bf16ToF32(), F32ToBf16(), and pabs< Packet4f >().

pabs() [15/37]

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::pabs

(

const Packet4d &

a

)

                                                     {
  const Packet4d mask = _mm256_castsi256_pd(_mm256_set1_epi64x(0x7FFFFFFFFFFFFFFF));
  return _mm256_and_pd(a, mask);
}

References a.

pabs() [16/37]

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pabs

(

const Packet4f &

a

)

                                                     {
  return vec_abs(a);
}

References a.

Referenced by generic_atan(), generic_pow(), generic_pow_impl(), generic_rint(), generic_round(), generic_trunc(), Eigen::internal::unary_pow::handle_negative_exponent(), Eigen::internal::unary_pow::handle_nonint_nonint_errors(), pabs< Packet4f >(), pabsdiff< Packet16c >(), pabsdiff< Packet16uc >(), pabsdiff< Packet4f >(), pabsdiff< Packet4i >(), pabsdiff< Packet4ui >(), pabsdiff< Packet8s >(), pabsdiff< Packet8us >(), packetmath_notcomplex(), Eigen::internal::scalar_abs_op< Scalar >::packetOp(), Eigen::internal::scalar_isfinite_op< Scalar, true >::packetOp(), pacos_float(), pasin_float(), patanh_double(), patanh_float(), pblueNorm(), pdiv_complex(), pdiv_float_common(), pexp_complex(), pexp_double(), pexp_float(), pfrexp_generic(), pfrexp_generic_get_biased_exponent(), phypot_complex(), pisinf(), plog_complex(), psincos_double(), psincos_float(), psincos_inner_msa_float(), psqrt_complex(), ptanh< Packet4f >(), Eigen::internal::psign_impl< Packet, std::enable_if_t<!NumTraits< typename unpacket_traits< Packet >::type >::IsComplex &&!NumTraits< typename unpacket_traits< Packet >::type >::IsInteger > >::run(), Eigen::internal::psign_impl< Packet, std::enable_if_t< NumTraits< typename unpacket_traits< Packet >::type >::IsComplex &&unpacket_traits< Packet >::vectorizable > >::run(), Eigen::internal::generic_i0e< T, float >::run(), Eigen::internal::generic_i0e< T, double >::run(), Eigen::internal::generic_i0< T, ScalarType >::run(), Eigen::internal::generic_i1e< T, float >::run(), Eigen::internal::generic_i1e< T, double >::run(), Eigen::internal::generic_i1< T, ScalarType >::run(), Eigen::internal::generic_j0< T, float >::run(), Eigen::internal::generic_j0< T, double >::run(), Eigen::internal::generic_j1< T, float >::run(), Eigen::internal::generic_j1< T, double >::run(), and twosum().

pabs() [17/37]

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pabs

(

const Packet4i &

a

)

                                                     {
  return vec_abs(a);
}

References a.

pabs() [18/37]

template<>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::pabs

(

const Packet4s &

a

)

                                                     {
  return vabs_s16(a);
}

References a.

pabs() [19/37]

template<>

EIGEN_STRONG_INLINE Packet4uc Eigen::internal::pabs

(

const Packet4uc &

a

)

                                                       {
  return a;
}

References a.

pabs() [20/37]

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::pabs

(

const Packet4ui &

a

)

                                                       {
  return a;
}

References a.

pabs() [21/37]

template<>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::pabs

(

const Packet4us &

a

)

                                                       {
  return a;
}

References a.

pabs() [22/37]

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::pabs

(

const Packet8bf &

a

)

                                                       {
  EIGEN_DECLARE_CONST_FAST_Packet8us(abs_mask, 0x7FFF);
  return pand<Packet8us>(p8us_abs_mask, a);
}

References a, EIGEN_DECLARE_CONST_FAST_Packet8us(), and pand< Packet8us >().

pabs() [23/37]

template<>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::pabs

(

const Packet8c &

a

)

                                                     {
  return vabs_s8(a);
}

References a.

pabs() [24/37]

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::pabs

(

const Packet8d &

a

)

                                                     {
  // _mm512_abs_ps intrinsic not found, so hack around it
  return _mm512_castsi512_pd(_mm512_and_si512(_mm512_castpd_si512(a), _mm512_set1_epi64(0x7fffffffffffffff)));
}

References a.

pabs() [25/37]

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::pabs

(

const Packet8f &

a

)

                                                     {
  const Packet8f mask = _mm256_castsi256_ps(_mm256_set1_epi32(0x7FFFFFFF));
  return _mm256_and_ps(a, mask);
}

References a.

pabs() [26/37]

template<>

EIGEN_STRONG_INLINE Packet8h Eigen::internal::pabs

(

const Packet8h &

a

)

                                                     {
  const __m128i sign_mask = _mm_set1_epi16(static_cast<numext::uint16_t>(0x8000));
  return _mm_andnot_si128(sign_mask, a);
}

References a.

pabs() [27/37]

template<>

EIGEN_STRONG_INLINE Packet8i Eigen::internal::pabs

(

const Packet8i &

a

)

                                                     {
#ifdef EIGEN_VECTORIZE_AVX2
  return _mm256_abs_epi32(a);
#else
  __m128i lo = _mm_abs_epi32(_mm256_extractf128_si256(a, 0));
  __m128i hi = _mm_abs_epi32(_mm256_extractf128_si256(a, 1));
  return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1);
#endif
}

References a.

pabs() [28/37]

template<>

EIGEN_STRONG_INLINE Packet8l Eigen::internal::pabs

(

const Packet8l &

a

)

                                                     {
  return _mm512_abs_epi64(a);
}

References a.

pabs() [29/37]

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::pabs

(

const Packet8s &

a

)

                                                     {
  return vec_abs(a);
}

References a.

pabs() [30/37]

template<>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::pabs

(

const Packet8uc &

a

)

                                                       {
  return a;
}

References a.

pabs() [31/37]

template<>

EIGEN_STRONG_INLINE Packet8ui Eigen::internal::pabs

(

const Packet8ui &

a

)

                                                       {
  return a;
}

References a.

pabs() [32/37]

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::pabs

(

const Packet8us &

a

)

                                                       {
  return a;
}

References a.

pabs() [33/37]

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::pabs

(

const PacketXf &

a

)

                                                     {
  return svabs_f32_x(svptrue_b32(), a);
}

References a.

pabs() [34/37]

template<>

EIGEN_STRONG_INLINE PacketXi Eigen::internal::pabs

(

const PacketXi &

a

)

                                                     {
  return svabs_s32_x(svptrue_b32(), a);
}

References a.

pabs() [35/37]

template<>

EIGEN_DEVICE_FUNC unsigned int Eigen::internal::pabs ( const unsigned int &  a )

inline

                                                                  {
  return a;
}

References a.

pabs() [36/37]

template<>

EIGEN_DEVICE_FUNC unsigned long Eigen::internal::pabs ( const unsigned long &  a )

inline

                                                                    {
  return a;
}

References a.

pabs() [37/37]

template<>

EIGEN_DEVICE_FUNC unsigned long long Eigen::internal::pabs ( const unsigned long long &  a )

inline

                                                                              {
  return a;
}

References a.

pabs< Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pabs< Packet2d >

(

const Packet2d &

a

)

                                                               {
  return vec_abs(a);
}

References a.

pabs< Packet32h >()

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::pabs< Packet32h >

(

const Packet32h &

a

)

                                                                  {
  return _mm512_abs_ph(a);
}

References a.

pabs< Packet4c >()

template<>

EIGEN_STRONG_INLINE Packet4c Eigen::internal::pabs< Packet4c >

(

const Packet4c &

a

)

                                                               {
  return vget_lane_s32(vreinterpret_s32_s8(vabs_s8(vreinterpret_s8_s32(vdup_n_s32(a)))), 0);
}

References a.

pabs< Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pabs< Packet4f >

(

const Packet4f &

a

)

                                                               {
  Packet4f res;
  res.v4f[0] = pabs(a.v4f[0]);
  res.v4f[1] = pabs(a.v4f[1]);
  return res;
}

References a, pabs(), and res.

Referenced by pabs().

pabs< Packet4i >()

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pabs< Packet4i >

(

const Packet4i &

a

)

                                                               {
  return vec_abs(a);
}

References a.

pabsdiff()

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::pabsdiff ( const Packet &  a, const Packet &  b  )

inline

Returns

the min of a and b (coeff-wise)

                                                                           {
  return pselect(pcmp_lt(a, b), psub(b, a), psub(a, b));
}

References a, b, pcmp_lt(), pselect(), and psub().

Referenced by packetmath_notcomplex(), and Eigen::internal::scalar_absolute_difference_op< LhsScalar, RhsScalar >::packetOp().

pabsdiff< Packet16c >()

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::pabsdiff< Packet16c >	(	const Packet16c &	a,
		const Packet16c &	b
	)

                                                                                          {
  Packet16c v = psub(a, b);
  return pabs(v);
}

References a, b, pabs(), psub(), and v.

pabsdiff< Packet16uc >()

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::pabsdiff< Packet16uc >	(	const Packet16uc &	a,
		const Packet16uc &	b
	)

                                                                                              {
  Packet16uc v = psub(a, b);
  return pabs(v);
}

References a, b, pabs(), psub(), and v.

pabsdiff< Packet2f >()

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::pabsdiff< Packet2f >	(	const Packet2f &	a,
		const Packet2f &	b
	)

                                                                                      {
  return vabd_f32(a, b);
}

References a, and b.

pabsdiff< Packet2i >()

template<>

EIGEN_STRONG_INLINE Packet2i Eigen::internal::pabsdiff< Packet2i >	(	const Packet2i &	a,
		const Packet2i &	b
	)

                                                                                      {
  return vabd_s32(a, b);
}

References a, and b.

pabsdiff< Packet2ui >()

template<>

EIGEN_STRONG_INLINE Packet2ui Eigen::internal::pabsdiff< Packet2ui >	(	const Packet2ui &	a,
		const Packet2ui &	b
	)

                                                                                          {
  return vabd_u32(a, b);
}

References a, and b.

pabsdiff< Packet4bf >()

template<>

EIGEN_STRONG_INLINE Packet4bf Eigen::internal::pabsdiff< Packet4bf >	(	const Packet4bf &	a,
		const Packet4bf &	b
	)

                                                                                          {
  return F32ToBf16(pabsdiff<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
}

References a, b, Bf16ToF32(), F32ToBf16(), and pabsdiff< Packet4f >().

pabsdiff< Packet4c >()

template<>

EIGEN_STRONG_INLINE Packet4c Eigen::internal::pabsdiff< Packet4c >	(	const Packet4c &	a,
		const Packet4c &	b
	)

                                                                                      {
  return vget_lane_s32(
      vreinterpret_s32_s8(vabd_s8(vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
}

References a, and b.

pabsdiff< Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pabsdiff< Packet4f >	(	const Packet4f &	a,
		const Packet4f &	b
	)

                                                                                      {
  Packet4f v = psub(a, b);
  return pabs(v);
}

References a, b, pabs(), psub(), and v.

Referenced by pabsdiff< Packet4bf >().

pabsdiff< Packet4i >()

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pabsdiff< Packet4i >	(	const Packet4i &	a,
		const Packet4i &	b
	)

                                                                                      {
  Packet4i v = psub(a, b);
  return pabs(v);
}

References a, b, pabs(), psub(), and v.

pabsdiff< Packet4s >()

template<>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::pabsdiff< Packet4s >	(	const Packet4s &	a,
		const Packet4s &	b
	)

                                                                                      {
  return vabd_s16(a, b);
}

References a, and b.

pabsdiff< Packet4uc >()

template<>

EIGEN_STRONG_INLINE Packet4uc Eigen::internal::pabsdiff< Packet4uc >	(	const Packet4uc &	a,
		const Packet4uc &	b
	)

                                                                                          {
  return vget_lane_u32(
      vreinterpret_u32_u8(vabd_u8(vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
}

References a, and b.

pabsdiff< Packet4ui >()

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::pabsdiff< Packet4ui >	(	const Packet4ui &	a,
		const Packet4ui &	b
	)

                                                                                          {
  Packet4ui v = psub(a, b);
  return pabs(v);
}

References a, b, pabs(), psub(), and v.

pabsdiff< Packet4us >()

template<>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::pabsdiff< Packet4us >	(	const Packet4us &	a,
		const Packet4us &	b
	)

                                                                                          {
  return vabd_u16(a, b);
}

References a, and b.

pabsdiff< Packet8c >()

template<>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::pabsdiff< Packet8c >	(	const Packet8c &	a,
		const Packet8c &	b
	)

                                                                                      {
  return vabd_s8(a, b);
}

References a, and b.

pabsdiff< Packet8s >()

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::pabsdiff< Packet8s >	(	const Packet8s &	a,
		const Packet8s &	b
	)

                                                                                      {
  Packet8s v = psub(a, b);
  return pabs(v);
}

References a, b, pabs(), psub(), and v.

pabsdiff< Packet8uc >()

template<>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::pabsdiff< Packet8uc >	(	const Packet8uc &	a,
		const Packet8uc &	b
	)

                                                                                          {
  return vabd_u8(a, b);
}

References a, and b.

pabsdiff< Packet8us >()

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::pabsdiff< Packet8us >	(	const Packet8us &	a,
		const Packet8us &	b
	)

                                                                                          {
  Packet8us v = psub(a, b);
  return pabs(v);
}

References a, b, pabs(), psub(), and v.

Pack16To8()

EIGEN_STRONG_INLINE __m128i Eigen::internal::Pack16To8

(

Packet8f

rf

)

                                                   {
  return _mm_packs_epi32(_mm256_extractf128_si256(_mm256_castps_si256(rf), 0),
                         _mm256_extractf128_si256(_mm256_castps_si256(rf), 1));
}

Referenced by pcmp_eq(), pcmp_le(), pcmp_lt(), and pcmp_lt_or_nan().

Pack32To16()

EIGEN_STRONG_INLINE __m256i Eigen::internal::Pack32To16

(

Packet16f

rf

)

                                                     {
  // Split data into small pieces and handle with AVX instructions
  // to guarantee internal order of vector.
  // Operation:
  //   dst[15:0]    := Saturate16(rf[31:0])
  //   dst[31:16]   := Saturate16(rf[63:32])
  //   ...
  //   dst[255:240] := Saturate16(rf[255:224])
  __m256i lo = _mm256_castps_si256(extract256<0>(rf));
  __m256i hi = _mm256_castps_si256(extract256<1>(rf));
  __m128i result_lo = _mm_packs_epi32(_mm256_extractf128_si256(lo, 0), _mm256_extractf128_si256(lo, 1));
  __m128i result_hi = _mm_packs_epi32(_mm256_extractf128_si256(hi, 0), _mm256_extractf128_si256(hi, 1));
  return _mm256_insertf128_si256(_mm256_castsi128_si256(result_lo), result_hi, 1);
}

Referenced by pcmp_eq(), pcmp_le(), pcmp_lt(), and pcmp_lt_or_nan().

packetwise_redux_empty_value() [1/2]

template<typename PacketType , typename Func >

EIGEN_DEVICE_FUNC PacketType Eigen::internal::packetwise_redux_empty_value

(

const Func &

)

                                                                       {
  const typename unpacket_traits<PacketType>::type zero(0);
  return pset1<PacketType>(zero);
}

References zero().

packetwise_redux_empty_value() [2/2]

template<typename PacketType , typename Scalar >

EIGEN_DEVICE_FUNC PacketType Eigen::internal::packetwise_redux_empty_value

(

const scalar_product_op< Scalar, Scalar > &

)

                                                                                                    {
  return pset1<PacketType>(Scalar(1));
}

pacos()

template<typename Packet >

EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::pacos

(

const Packet &

a

)

Returns

the arc cosine of a (coeff-wise)

                                                                                   {
  EIGEN_USING_STD(acos);
  return acos(a);
}

References a, acos(), and EIGEN_USING_STD.

Referenced by packetmath_real(), and Eigen::internal::scalar_acos_op< Scalar >::packetOp().

pacos_float()

template<typename Packet >

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::pacos_float

(

const Packet &

x

)

Returns

acos(x) for single precision float

                                                                                           {
  typedef typename unpacket_traits<Packet>::type Scalar;
  static_assert(std::is_same<Scalar, float>::value, "Scalar type must be float");
 
  const Packet cst_one = pset1<Packet>(Scalar(1));
  const Packet cst_pi = pset1<Packet>(Scalar(EIGEN_PI));
  const Packet p6 = pset1<Packet>(Scalar(2.36423197202384471893310546875e-3));
  const Packet p5 = pset1<Packet>(Scalar(-1.1368644423782825469970703125e-2));
  const Packet p4 = pset1<Packet>(Scalar(2.717843465507030487060546875e-2));
  const Packet p3 = pset1<Packet>(Scalar(-4.8969544470310211181640625e-2));
  const Packet p2 = pset1<Packet>(Scalar(8.8804088532924652099609375e-2));
  const Packet p1 = pset1<Packet>(Scalar(-0.214591205120086669921875));
  const Packet p0 = pset1<Packet>(Scalar(1.57079637050628662109375));
 
  // For x in [0:1], we approximate acos(x)/sqrt(1-x), which is a smooth
  // function, by a 6'th order polynomial.
  // For x in [-1:0) we use that acos(-x) = pi - acos(x).
  const Packet neg_mask = psignbit(x_in);
  const Packet abs_x = pabs(x_in);
 
  // Evaluate the polynomial using Horner's rule:
  //   P(x) = p0 + x * (p1 +  x * (p2 + ... (p5 + x * p6)) ... ) .
  // We evaluate even and odd terms independently to increase
  // instruction level parallelism.
  Packet x2 = pmul(x_in, x_in);
  Packet p_even = pmadd(p6, x2, p4);
  Packet p_odd = pmadd(p5, x2, p3);
  p_even = pmadd(p_even, x2, p2);
  p_odd = pmadd(p_odd, x2, p1);
  p_even = pmadd(p_even, x2, p0);
  Packet p = pmadd(p_odd, abs_x, p_even);
 
  // The polynomial approximates acos(x)/sqrt(1-x), so
  // multiply by sqrt(1-x) to get acos(x).
  // Conveniently returns NaN for arguments outside [-1:1].
  Packet denom = psqrt(psub(cst_one, abs_x));
  Packet result = pmul(denom, p);
  // Undo mapping for negative arguments.
  return pselect(neg_mask, psub(cst_pi, result), result);
}

References EIGEN_PI, p, p0, p1, pabs(), pmadd(), pmul(), pselect(), psignbit(), psqrt(), psub(), Eigen::value, and Global_parameters::x2().

padd() [1/4]

template<>

EIGEN_DEVICE_FUNC bool Eigen::internal::padd ( const bool &  a, const bool &  b  )

inline

                                                                 {
  return a || b;
}

References a, and b.

padd() [2/4]

template<typename Packet >

DoublePacket<Packet> Eigen::internal::padd	(	const DoublePacket< Packet > &	a,
		const DoublePacket< Packet > &	b
	)

                                                                                        {
  DoublePacket<Packet> res;
  res.first = padd(a.first, b.first);
  res.second = padd(a.second, b.second);
  return res;
}

References a, b, padd(), and res.

padd() [3/4]

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::padd ( const Packet &  a, const Packet &  b  )

inline

Returns

a + b (coeff-wise)

                                                                       {
  return a + b;
}

References a, and b.

Referenced by Eigen::internal::gebp_traits< std::complex< RealScalar >, std::complex< RealScalar >, ConjLhs_, ConjRhs_, Arch, PacketSize_ >::acc(), Eigen::internal::add_assign_op< DstScalar, SrcScalar >::assignPacket(), benchVec(), doubleword_div_fp(), fast_twosum(), generic_ceil(), generic_exp2(), generic_log1p(), generic_pow_impl(), generic_rint(), generic_round(), Eigen::internal::gebp_traits< LhsScalar_, RhsScalar_, ConjLhs_, ConjRhs_, Arch, PacketSize_ >::madd(), Eigen::internal::gebp_traits< std::complex< RealScalar >, RealScalar, ConjLhs_, false, Arch, PacketSize_ >::madd_impl(), Eigen::internal::gebp_traits< RealScalar, std::complex< RealScalar >, false, ConjRhs_, Arch, PacketSize_ >::madd_impl(), Eigen::internal::gebp_kernel< LhsScalar, RhsScalar, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >::operator()(), Eigen::internal::lhs_process_one_packet< nr, LhsProgress, RhsProgress, LhsScalar, RhsScalar, ResScalar, AccPacket, LhsPacket, RhsPacket, ResPacket, GEBPTraits, LinearMapper, DataMapper >::operator()(), Eigen::internal::accurate_log2< double >::operator()(), Eigen::internal::Packet1cd::operator*=(), Eigen::internal::Packet2cf::operator*=(), Eigen::internal::Packet1cd::operator+=(), Eigen::internal::Packet2cf::operator+=(), Eigen::internal::Packet1cd::operator/=(), Eigen::internal::scalar_logistic_op< float >::packetOp(), Eigen::internal::scalar_sum_op< LhsScalar, RhsScalar >::packetOp(), Eigen::internal::scalar_logistic_op_impl< T, EnableIf >::packetOp(), Eigen::internal::linspaced_op_impl< Scalar, false >::packetOp(), padd(), padd< Packet16h >(), padd< Packet8h >(), paddsub(), paddsub< Packet2d >(), paddsub< Packet2f >(), paddsub< Packet4f >(), patanh_double(), patanh_float(), pblueNorm(), pdiv_complex(), pexp< Packet4f >(), pexp_float(), pfrexp_generic(), phypot_complex(), pldexp< Packet4d >(), pldexp< Packet8d >(), pldexp_fast(), pldexp_fast< Packet2d >(), pldexp_fast< Packet4d >(), pldexp_generic(), plog< Packet4f >(), plog_impl_double(), plog_impl_float(), plset< Packet4d >(), plset< Packet8f >(), plset< Packet8i >(), plset< Packet8ui >(), pmadd(), predux< Packet2cd >(), predux< Packet4cd >(), predux< Packet4cf >(), predux< Packet8cf >(), pround(), pround< Packet16f >(), pround< Packet32h >(), pround< Packet4d >(), pround< Packet8d >(), pround< Packet8f >(), psincos_double(), psincos_float(), psincos_inner_msa_float(), psqrt_complex(), Eigen::internal::inner_product_impl< Evaluator, true >::run(), Eigen::internal::compute_inverse_size4< Architecture::Target, float, MatrixType, ResultType >::run(), Eigen::internal::compute_inverse_size4< Architecture::Target, double, MatrixType, ResultType >::run(), Eigen::internal::psign_impl< Packet, std::enable_if_t< NumTraits< typename unpacket_traits< Packet >::type >::IsComplex &&unpacket_traits< Packet >::vectorizable > >::run(), Eigen::internal::quat_product< Architecture::Target, Derived, OtherDerived, float >::run(), Eigen::internal::generic_j0< T, float >::run(), Eigen::internal::generic_j0< T, double >::run(), Eigen::internal::generic_y0< T, float >::run(), Eigen::internal::generic_y0< T, double >::run(), Eigen::internal::generic_j1< T, float >::run(), Eigen::internal::generic_j1< T, double >::run(), Eigen::internal::generic_y1< T, float >::run(), Eigen::internal::generic_y1< T, double >::run(), Eigen::internal::apply_rotation_in_the_plane_selector< Scalar, OtherScalar, SizeAtCompileTime, MinAlignment, true >::run(), trig_reduce_medium_double(), twoprod(), twosum(), Eigen::internal::gemm_class< Scalar, is_unit_inc >::vaddm(), and veltkamp_splitting().

padd() [4/4]

template<typename Packet >

EIGEN_DEVICE_FUNC std::enable_if_t<unpacket_traits<Packet>::masked_fpops_available, Packet> Eigen::internal::padd ( const Packet &  a, const Packet &  b, typename unpacket_traits< Packet >::mask_t  umask  )

inline

Returns

a packet version of *from, (un-aligned masked add) There is no generic implementation. We only have implementations for specialized cases. Generic case should not be called.

padd< Packet16b >()

template<>

EIGEN_STRONG_INLINE Packet16b Eigen::internal::padd< Packet16b >	(	const Packet16b &	a,
		const Packet16b &	b
	)

                                                                                      {
  return _mm_or_si128(a, b);
}

References a, and b.

padd< Packet16bf >()

template<>

EIGEN_STRONG_INLINE Packet16bf Eigen::internal::padd< Packet16bf >	(	const Packet16bf &	a,
		const Packet16bf &	b
	)

                                                                                          {
  return F32ToBf16(padd<Packet16f>(Bf16ToF32(a), Bf16ToF32(b)));
}

References a, b, Bf16ToF32(), F32ToBf16(), and padd< Packet16f >().

padd< Packet16c >()

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::padd< Packet16c >	(	const Packet16c &	a,
		const Packet16c &	b
	)

                                                                                      {
  return a + b;
}

References a, and b.

padd< Packet16f >() [1/2]

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::padd< Packet16f >	(	const Packet16f &	a,
		const Packet16f &	b
	)

                                                                                      {
  return _mm512_add_ps(a, b);
}

References a, and b.

Referenced by padd< Packet16bf >().

padd< Packet16f >() [2/2]

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::padd< Packet16f >	(	const Packet16f &	a,
		const Packet16f &	b,
		uint16_t	umask
	)

                                                                                                      {
  __mmask16 mask = static_cast<__mmask16>(umask);
  return _mm512_maskz_add_ps(mask, a, b);
}

References a, and b.

padd< Packet16h >()

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::padd< Packet16h >	(	const Packet16h &	a,
		const Packet16h &	b
	)

                                                                                      {
  Packet16f af = half2float(a);
  Packet16f bf = half2float(b);
  Packet16f rf = padd(af, bf);
  return float2half(rf);
}

References a, b, float2half(), half2float(), and padd().

Referenced by predux_half_dowto4< Packet32h >().

padd< Packet16i >()

template<>

EIGEN_STRONG_INLINE Packet16i Eigen::internal::padd< Packet16i >	(	const Packet16i &	a,
		const Packet16i &	b
	)

                                                                                      {
  return _mm512_add_epi32(a, b);
}

References a, and b.

padd< Packet16uc >()

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::padd< Packet16uc >	(	const Packet16uc &	a,
		const Packet16uc &	b
	)

                                                                                          {
  return a + b;
}

References a, and b.

padd< Packet1cd >()

template<>

EIGEN_STRONG_INLINE Packet1cd Eigen::internal::padd< Packet1cd >	(	const Packet1cd &	a,
		const Packet1cd &	b
	)

                                                                                      {
  return Packet1cd(__lsx_vfadd_d(a.v, b.v));
}

References a, and b.

padd< Packet1cf >()

template<>

EIGEN_STRONG_INLINE Packet1cf Eigen::internal::padd< Packet1cf >	(	const Packet1cf &	a,
		const Packet1cf &	b
	)

                                                                                      {
  return Packet1cf(padd<Packet2f>(a.v, b.v));
}

References a, b, and padd< Packet2f >().

padd< Packet2cd >()

template<>

EIGEN_STRONG_INLINE Packet2cd Eigen::internal::padd< Packet2cd >	(	const Packet2cd &	a,
		const Packet2cd &	b
	)

                                                                                      {
  return Packet2cd(_mm256_add_pd(a.v, b.v));
}

References a, and b.

padd< Packet2cf >()

template<>

EIGEN_STRONG_INLINE Packet2cf Eigen::internal::padd< Packet2cf >	(	const Packet2cf &	a,
		const Packet2cf &	b
	)

                                                                                      {
  return Packet2cf(a.v + b.v);
}

References a, and b.

padd< Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::padd< Packet2d >	(	const Packet2d &	a,
		const Packet2d &	b
	)

                                                                                  {
  return __lsx_vfadd_d(a, b);
}

References a, and b.

padd< Packet2f >()

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::padd< Packet2f >	(	const Packet2f &	a,
		const Packet2f &	b
	)

                                                                                  {
  return vadd_f32(a, b);
}

References a, and b.

Referenced by padd< Packet1cf >().

padd< Packet2i >()

template<>

EIGEN_STRONG_INLINE Packet2i Eigen::internal::padd< Packet2i >	(	const Packet2i &	a,
		const Packet2i &	b
	)

                                                                                  {
  return vadd_s32(a, b);
}

References a, and b.

padd< Packet2l >()

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::padd< Packet2l >	(	const Packet2l &	a,
		const Packet2l &	b
	)

                                                                                  {
  return __lsx_vadd_d(a, b);
}

References a, and b.

padd< Packet2ui >()

template<>

EIGEN_STRONG_INLINE Packet2ui Eigen::internal::padd< Packet2ui >	(	const Packet2ui &	a,
		const Packet2ui &	b
	)

                                                                                      {
  return vadd_u32(a, b);
}

References a, and b.

padd< Packet2ul >()

template<>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::padd< Packet2ul >	(	const Packet2ul &	a,
		const Packet2ul &	b
	)

                                                                                      {
  return __lsx_vadd_d(a, b);
}

References a, and b.

padd< Packet32h >()

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::padd< Packet32h >	(	const Packet32h &	a,
		const Packet32h &	b
	)

                                                                                      {
  return _mm512_add_ph(a, b);
}

References a, and b.

padd< Packet4bf >()

template<>

EIGEN_STRONG_INLINE Packet4bf Eigen::internal::padd< Packet4bf >	(	const Packet4bf &	a,
		const Packet4bf &	b
	)

                                                                                      {
  return F32ToBf16(padd<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
}

References a, b, Bf16ToF32(), F32ToBf16(), and padd< Packet4f >().

padd< Packet4c >()

template<>

EIGEN_STRONG_INLINE Packet4c Eigen::internal::padd< Packet4c >	(	const Packet4c &	a,
		const Packet4c &	b
	)

                                                                                  {
  return vget_lane_s32(
      vreinterpret_s32_s8(vadd_s8(vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
}

References a, and b.

padd< Packet4cd >()

template<>

EIGEN_STRONG_INLINE Packet4cd Eigen::internal::padd< Packet4cd >	(	const Packet4cd &	a,
		const Packet4cd &	b
	)

                                                                                      {
  return Packet4cd(_mm512_add_pd(a.v, b.v));
}

References a, and b.

padd< Packet4cf >()

template<>

EIGEN_STRONG_INLINE Packet4cf Eigen::internal::padd< Packet4cf >	(	const Packet4cf &	a,
		const Packet4cf &	b
	)

                                                                                      {
  return Packet4cf(_mm256_add_ps(a.v, b.v));
}

References a, and b.

padd< Packet4d >()

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::padd< Packet4d >	(	const Packet4d &	a,
		const Packet4d &	b
	)

                                                                                  {
  return _mm256_add_pd(a, b);
}

References a, and b.

padd< Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::padd< Packet4f >	(	const Packet4f &	a,
		const Packet4f &	b
	)

                                                                                  {
  return a + b;
}

References a, and b.

Referenced by padd< Packet4bf >(), padd< Packet8bf >(), Eigen::internal::Packet2cf::pmul(), and predux< Packet2cf >().

padd< Packet4i >()

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::padd< Packet4i >	(	const Packet4i &	a,
		const Packet4i &	b
	)

                                                                                  {
  return a + b;
}

References a, and b.

padd< Packet4s >()

template<>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::padd< Packet4s >	(	const Packet4s &	a,
		const Packet4s &	b
	)

                                                                                  {
  return vadd_s16(a, b);
}

References a, and b.

padd< Packet4uc >()

template<>

EIGEN_STRONG_INLINE Packet4uc Eigen::internal::padd< Packet4uc >	(	const Packet4uc &	a,
		const Packet4uc &	b
	)

                                                                                      {
  return vget_lane_u32(
      vreinterpret_u32_u8(vadd_u8(vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
}

References a, and b.

padd< Packet4ui >()

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::padd< Packet4ui >	(	const Packet4ui &	a,
		const Packet4ui &	b
	)

                                                                                      {
  return a + b;
}

References a, and b.

Referenced by F32ToBf16().

padd< Packet4us >()

template<>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::padd< Packet4us >	(	const Packet4us &	a,
		const Packet4us &	b
	)

                                                                                      {
  return vadd_u16(a, b);
}

References a, and b.

padd< Packet8bf >()

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::padd< Packet8bf >	(	const Packet8bf &	a,
		const Packet8bf &	b
	)

                                                                                      {
  BF16_TO_F32_BINARY_OP_WRAPPER(padd<Packet4f>, a, b);
}

References a, b, BF16_TO_F32_BINARY_OP_WRAPPER, and padd< Packet4f >().

Referenced by plset< Packet8bf >(), and predux_half_dowto4< Packet16bf >().

padd< Packet8c >()

template<>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::padd< Packet8c >	(	const Packet8c &	a,
		const Packet8c &	b
	)

                                                                                  {
  return vadd_s8(a, b);
}

References a, and b.

padd< Packet8cf >()

template<>

EIGEN_STRONG_INLINE Packet8cf Eigen::internal::padd< Packet8cf >	(	const Packet8cf &	a,
		const Packet8cf &	b
	)

                                                                                      {
  return Packet8cf(_mm512_add_ps(a.v, b.v));
}

References a, and b.

padd< Packet8d >() [1/2]

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::padd< Packet8d >	(	const Packet8d &	a,
		const Packet8d &	b
	)

                                                                                  {
  return _mm512_add_pd(a, b);
}

References a, and b.

padd< Packet8d >() [2/2]

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::padd< Packet8d >	(	const Packet8d &	a,
		const Packet8d &	b,
		uint8_t	umask
	)

                                                                                                 {
  __mmask8 mask = static_cast<__mmask8>(umask);
  return _mm512_maskz_add_pd(mask, a, b);
}

References a, and b.

padd< Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::padd< Packet8f >	(	const Packet8f &	a,
		const Packet8f &	b
	)

                                                                                  {
  return _mm256_add_ps(a, b);
}

References a, and b.

padd< Packet8h >()

template<>

EIGEN_STRONG_INLINE Packet8h Eigen::internal::padd< Packet8h >	(	const Packet8h &	a,
		const Packet8h &	b
	)

                                                                                  {
  Packet8f af = half2float(a);
  Packet8f bf = half2float(b);
  Packet8f rf = padd(af, bf);
  return float2half(rf);
}

References a, b, float2half(), half2float(), and padd().

Referenced by predux_half_dowto4< Packet16h >().

padd< Packet8i >()

template<>

EIGEN_STRONG_INLINE Packet8i Eigen::internal::padd< Packet8i >	(	const Packet8i &	a,
		const Packet8i &	b
	)

                                                                                  {
#ifdef EIGEN_VECTORIZE_AVX2
  return _mm256_add_epi32(a, b);
#else
  __m128i lo = _mm_add_epi32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0));
  __m128i hi = _mm_add_epi32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1));
  return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1);
#endif
}

References a, and b.

padd< Packet8l >()

template<>

EIGEN_STRONG_INLINE Packet8l Eigen::internal::padd< Packet8l >	(	const Packet8l &	a,
		const Packet8l &	b
	)

                                                                                  {
  return _mm512_add_epi64(a, b);
}

References a, and b.

padd< Packet8s >()

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::padd< Packet8s >	(	const Packet8s &	a,
		const Packet8s &	b
	)

                                                                                  {
  return a + b;
}

References a, and b.

padd< Packet8uc >()

template<>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::padd< Packet8uc >	(	const Packet8uc &	a,
		const Packet8uc &	b
	)

                                                                                      {
  return vadd_u8(a, b);
}

References a, and b.

padd< Packet8ui >()

template<>

EIGEN_STRONG_INLINE Packet8ui Eigen::internal::padd< Packet8ui >	(	const Packet8ui &	a,
		const Packet8ui &	b
	)

                                                                                      {
#ifdef EIGEN_VECTORIZE_AVX2
  return _mm256_add_epi32(a, b);
#else
  __m128i lo = _mm_add_epi32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0));
  __m128i hi = _mm_add_epi32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1));
  return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1);
#endif
}

References a, and b.

padd< Packet8us >()

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::padd< Packet8us >	(	const Packet8us &	a,
		const Packet8us &	b
	)

                                                                                      {
  return a + b;
}

References a, and b.

Referenced by F32ToBf16Two().

padd< PacketXf >()

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::padd< PacketXf >	(	const PacketXf &	a,
		const PacketXf &	b
	)

                                                                                  {
  return svadd_f32_x(svptrue_b32(), a, b);
}

References a, and b.

padd< PacketXi >()

template<>

EIGEN_STRONG_INLINE PacketXi Eigen::internal::padd< PacketXi >	(	const PacketXi &	a,
		const PacketXi &	b
	)

                                                                                  {
  return svadd_s32_x(svptrue_b32(), a, b);
}

References a, and b.

padds()

template<typename Packet >

EIGEN_STRONG_INLINE Packet Eigen::internal::padds	(	const Packet &	a,
		const Packet &	b
	)

Referenced by Eigen::internal::gemm_class< Scalar, is_unit_inc >::vaddm().

padds< Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::padds< Packet2d >	(	const Packet2d &	a,
		const Packet2d &	b
	)

                                                                                   {
  return _mm_add_sd(a, b);
}

References a, and b.

padds< Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::padds< Packet4f >	(	const Packet4f &	a,
		const Packet4f &	b
	)

                                                                                   {
  return _mm_add_ss(a, b);
}

References a, and b.

paddsub()

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::paddsub ( const Packet &  a, const Packet &  b  )

inline

Returns

the addsub value of a,b

                                                                          {
  return pselect(peven_mask(a), padd(a, b), psub(a, b));
}

References a, b, padd(), peven_mask(), pselect(), and psub().

paddsub< Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::paddsub< Packet2d >	(	const Packet2d &	a,
		const Packet2d &	b
	)

                                                                                     {
  const Packet2d mask = make_packet2d(numext::bit_cast<double>(0x8000000000000000ull), 0.0);
  return padd(a, pxor(mask, b));
}

References a, b, make_packet2d(), padd(), and pxor().

paddsub< Packet2f >()

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::paddsub< Packet2f >	(	const Packet2f &	a,
		const Packet2f &	b
	)

                                                                                     {
  Packet2f mask = make_packet2f(numext::bit_cast<float>(0x80000000u), 0.0f);
  return padd(a, pxor(mask, b));
}

References a, b, make_packet2f(), padd(), and pxor().

paddsub< Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::paddsub< Packet4f >	(	const Packet4f &	a,
		const Packet4f &	b
	)

                                                                                     {
  const Packet4f mask =
      make_packet4f(numext::bit_cast<float>(0x80000000u), 0.0f, numext::bit_cast<float>(0x80000000u), 0.0f);
  return padd(a, pxor(mask, b));
}

References a, b, make_packet4f(), padd(), and pxor().

pand() [1/7]

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::pand ( const Packet &  a, const Packet &  b  )

inline

Returns

the bitwise and of a and b

                                                                       {
  return bitwise_helper<Packet>::bitwise_and(a, b);
}

References a, b, and Eigen::internal::bytewise_bitwise_helper< T >::bitwise_and().

pand() [2/7]

template<>

EIGEN_STRONG_INLINE Packet16bf Eigen::internal::pand	(	const Packet16bf &	a,
		const Packet16bf &	b
	)

                                                                              {
  return Packet16bf(pand<Packet8i>(Packet8i(a), Packet8i(b)));
}

References a, b, and pand< Packet8i >().

pand() [3/7]

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::pand	(	const Packet16h &	a,
		const Packet16h &	b
	)

                                                                           {
  return Packet16h(pand(Packet8i(a), Packet8i(b)));
}

References a, b, and pand().

pand() [4/7]

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::pand	(	const Packet32h &	a,
		const Packet32h &	b
	)

                                                                           {
  return _mm512_castsi512_ph(_mm512_and_si512(_mm512_castph_si512(a), _mm512_castph_si512(b)));
}

References a, and b.

pand() [5/7]

template<>

EIGEN_STRONG_INLINE Packet4bf Eigen::internal::pand	(	const Packet4bf &	a,
		const Packet4bf &	b
	)

                                                                           {
  return Packet4bf(pand<Packet4us>(Packet4us(a), Packet4us(b)));
}

References a, b, and pand< Packet4us >().

pand() [6/7]

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::pand	(	const Packet8bf &	a,
		const Packet8bf &	b
	)

                                                                           {
  return _mm_and_si128(a, b);
}

References a, and b.

pand() [7/7]

template<>

EIGEN_STRONG_INLINE Packet8h Eigen::internal::pand	(	const Packet8h &	a,
		const Packet8h &	b
	)

                                                                        {
  return _mm_and_si128(a, b);
}

References a, and b.

Referenced by erfc_double_large(), generic_atan(), generic_ceil(), generic_floor(), generic_pow(), generic_round(), Eigen::internal::unary_pow::handle_negative_exponent(), Eigen::internal::unary_pow::handle_nonint_nonint_errors(), Eigen::internal::minmax_coeff_visitor< Derived, is_min, NaNPropagation, isInt >::initpacket(), Eigen::internal::minmax_coeff_visitor< Derived, is_min, NaNPropagation, false >::initpacket(), Eigen::internal::minmax_coeff_visitor< Derived, is_min, PropagateNumbers, false >::packet(), Eigen::internal::minmax_coeff_visitor< Derived, is_min, NaNPropagation, false >::packet(), packetmath(), Eigen::internal::scalar_boolean_not_op< Scalar >::packetOp(), Eigen::internal::scalar_cmp_op< LhsScalar, RhsScalar, cmp_EQ, UseTypedComparators >::packetOp(), Eigen::internal::scalar_cmp_op< LhsScalar, RhsScalar, cmp_LT, UseTypedComparators >::packetOp(), Eigen::internal::scalar_cmp_op< LhsScalar, RhsScalar, cmp_LE, UseTypedComparators >::packetOp(), Eigen::internal::scalar_cmp_op< LhsScalar, RhsScalar, cmp_GT, UseTypedComparators >::packetOp(), Eigen::internal::scalar_cmp_op< LhsScalar, RhsScalar, cmp_GE, UseTypedComparators >::packetOp(), Eigen::internal::scalar_boolean_xor_op< Scalar >::packetOp(), Eigen::internal::scalar_bitwise_and_op< Scalar >::packetOp(), pand(), pand< Packet16f >(), pand< Packet4cd >(), pand< Packet8cf >(), pandnot(), patanh_double(), patanh_float(), pblueNorm(), pcmp_eq(), pcmp_lt(), pexp_complex(), pfrexp_generic(), pfrexp_generic_get_biased_exponent(), plog_impl_double(), plog_impl_float(), pround(), pround< Packet16f >(), pround< Packet32h >(), pround< Packet4d >(), pround< Packet8d >(), pround< Packet8f >(), psincos_double(), psincos_float(), psincos_inner_msa_float(), psqrt_complex(), Eigen::internal::psign_impl< Packet, std::enable_if_t<!NumTraits< typename unpacket_traits< Packet >::type >::IsComplex &&NumTraits< typename unpacket_traits< Packet >::type >::IsSigned &&NumTraits< typename unpacket_traits< Packet >::type >::IsInteger > >::run(), Eigen::internal::psignbit_impl< Packet, false, false >::run(), Eigen::internal::pselect_impl< Packet, EnableIf >::run(), and Eigen::internal::generic_fast_erfc< Scalar >::run().

pand< Packet16b >()

template<>

EIGEN_STRONG_INLINE Packet16b Eigen::internal::pand< Packet16b >	(	const Packet16b &	a,
		const Packet16b &	b
	)

                                                                                      {
  return _mm_and_si128(a, b);
}

References a, and b.

pand< Packet16c >()

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::pand< Packet16c >	(	const Packet16c &	a,
		const Packet16c &	b
	)

                                                                                      {
  return __lsx_vand_v(a, b);
}

References a, and b.

pand< Packet16f >()

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::pand< Packet16f >	(	const Packet16f &	a,
		const Packet16f &	b
	)

                                                                                      {
#ifdef EIGEN_VECTORIZE_AVX512DQ
  return _mm512_and_ps(a, b);
#else
  return _mm512_castsi512_ps(pand(_mm512_castps_si512(a), _mm512_castps_si512(b)));
#endif
}

References a, b, and pand().

pand< Packet16i >()

template<>

EIGEN_STRONG_INLINE Packet16i Eigen::internal::pand< Packet16i >	(	const Packet16i &	a,
		const Packet16i &	b
	)

                                                                                      {
  return _mm512_and_si512(a, b);
}

References a, and b.

pand< Packet16uc >()

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::pand< Packet16uc >	(	const Packet16uc &	a,
		const Packet16uc &	b
	)

                                                                                          {
  return __lsx_vand_v(a, b);
}

References a, and b.

pand< Packet1cd >()

template<>

EIGEN_STRONG_INLINE Packet1cd Eigen::internal::pand< Packet1cd >	(	const Packet1cd &	a,
		const Packet1cd &	b
	)

                                                                                      {
  Packet1cd res;
  res.v = (Packet2d)__lsx_vand_v((__m128i)a.v, (__m128i)b.v);
  return res;
}

References a, b, and res.

pand< Packet1cf >()

template<>

EIGEN_STRONG_INLINE Packet1cf Eigen::internal::pand< Packet1cf >	(	const Packet1cf &	a,
		const Packet1cf &	b
	)

                                                                                      {
  return Packet1cf(vreinterpret_f32_u32(vand_u32(vreinterpret_u32_f32(a.v), vreinterpret_u32_f32(b.v))));
}

References a, and b.

pand< Packet2cd >()

template<>

EIGEN_STRONG_INLINE Packet2cd Eigen::internal::pand< Packet2cd >	(	const Packet2cd &	a,
		const Packet2cd &	b
	)

                                                                                      {
  return Packet2cd(_mm256_and_pd(a.v, b.v));
}

References a, and b.

pand< Packet2cf >()

template<>

EIGEN_STRONG_INLINE Packet2cf Eigen::internal::pand< Packet2cf >	(	const Packet2cf &	a,
		const Packet2cf &	b
	)

                                                                                      {
  return Packet2cf(pand<Packet4f>(a.v, b.v));
}

References a, b, and pand< Packet4f >().

pand< Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pand< Packet2d >	(	const Packet2d &	a,
		const Packet2d &	b
	)

                                                                                  {
  return (Packet2d)__lsx_vand_v((__m128i)a, (__m128i)b);
}

References a, and b.

Referenced by pcmp_eq().

pand< Packet2f >()

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::pand< Packet2f >	(	const Packet2f &	a,
		const Packet2f &	b
	)

                                                                                  {
  return vreinterpret_f32_u32(vand_u32(vreinterpret_u32_f32(a), vreinterpret_u32_f32(b)));
}

References a, and b.

Referenced by pcmp_eq().

pand< Packet2i >()

template<>

EIGEN_STRONG_INLINE Packet2i Eigen::internal::pand< Packet2i >	(	const Packet2i &	a,
		const Packet2i &	b
	)

                                                                                  {
  return vand_s32(a, b);
}

References a, and b.

pand< Packet2l >()

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::pand< Packet2l >	(	const Packet2l &	a,
		const Packet2l &	b
	)

                                                                                  {
  return __lsx_vand_v(a, b);
}

References a, and b.

pand< Packet2ui >()

template<>

EIGEN_STRONG_INLINE Packet2ui Eigen::internal::pand< Packet2ui >	(	const Packet2ui &	a,
		const Packet2ui &	b
	)

                                                                                      {
  return vand_u32(a, b);
}

References a, and b.

pand< Packet2ul >()

template<>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::pand< Packet2ul >	(	const Packet2ul &	a,
		const Packet2ul &	b
	)

                                                                                      {
  return __lsx_vand_v(a, b);
}

References a, and b.

pand< Packet4c >()

template<>

EIGEN_STRONG_INLINE Packet4c Eigen::internal::pand< Packet4c >	(	const Packet4c &	a,
		const Packet4c &	b
	)

                                                                                  {
  return a & b;
}

References a, and b.

pand< Packet4cd >()

template<>

EIGEN_STRONG_INLINE Packet4cd Eigen::internal::pand< Packet4cd >	(	const Packet4cd &	a,
		const Packet4cd &	b
	)

                                                                                      {
  return Packet4cd(pand(a.v, b.v));
}

References a, b, and pand().

pand< Packet4cf >()

template<>

EIGEN_STRONG_INLINE Packet4cf Eigen::internal::pand< Packet4cf >	(	const Packet4cf &	a,
		const Packet4cf &	b
	)

                                                                                      {
  return Packet4cf(_mm256_and_ps(a.v, b.v));
}

References a, and b.

pand< Packet4d >()

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::pand< Packet4d >	(	const Packet4d &	a,
		const Packet4d &	b
	)

                                                                                  {
  return _mm256_and_pd(a, b);
}

References a, and b.

pand< Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pand< Packet4f >	(	const Packet4f &	a,
		const Packet4f &	b
	)

                                                                                  {
  return vec_and(a, b);
}

References a, and b.

Referenced by Bf16ToF32Odd(), and pand< Packet2cf >().

pand< Packet4i >()

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pand< Packet4i >	(	const Packet4i &	a,
		const Packet4i &	b
	)

                                                                                  {
  return vec_and(a, b);
}

References a, and b.

Referenced by pcmp_eq().

pand< Packet4s >()

template<>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::pand< Packet4s >	(	const Packet4s &	a,
		const Packet4s &	b
	)

                                                                                  {
  return vand_s16(a, b);
}

References a, and b.

pand< Packet4uc >()

template<>

EIGEN_STRONG_INLINE Packet4uc Eigen::internal::pand< Packet4uc >	(	const Packet4uc &	a,
		const Packet4uc &	b
	)

                                                                                      {
  return a & b;
}

References a, and b.

pand< Packet4ui >()

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::pand< Packet4ui >	(	const Packet4ui &	a,
		const Packet4ui &	b
	)

                                                                                      {
  return vec_and(a, b);
}

References a, and b.

Referenced by F32ToBf16(), pcast< Packet8bf, Packet8us >(), and pcast< Packet8us, Packet8bf >().

pand< Packet4us >()

template<>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::pand< Packet4us >	(	const Packet4us &	a,
		const Packet4us &	b
	)

                                                                                      {
  return vand_u16(a, b);
}

References a, and b.

Referenced by pand().

pand< Packet8bf >()

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::pand< Packet8bf >	(	const Packet8bf &	a,
		const Packet8bf &	b
	)

                                                                                      {
  return pand<Packet8us>(a, b);
}

References a, b, and pand< Packet8us >().

pand< Packet8c >()

template<>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::pand< Packet8c >	(	const Packet8c &	a,
		const Packet8c &	b
	)

                                                                                  {
  return vand_s8(a, b);
}

References a, and b.

pand< Packet8cf >()

template<>

EIGEN_STRONG_INLINE Packet8cf Eigen::internal::pand< Packet8cf >	(	const Packet8cf &	a,
		const Packet8cf &	b
	)

                                                                                      {
  return Packet8cf(pand(a.v, b.v));
}

References a, b, and pand().

pand< Packet8d >()

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::pand< Packet8d >	(	const Packet8d &	a,
		const Packet8d &	b
	)

                                                                                  {
#ifdef EIGEN_VECTORIZE_AVX512DQ
  return _mm512_and_pd(a, b);
#else
  Packet8d res = _mm512_undefined_pd();
  Packet4d lane0_a = _mm512_extractf64x4_pd(a, 0);
  Packet4d lane0_b = _mm512_extractf64x4_pd(b, 0);
  res = _mm512_insertf64x4(res, _mm256_and_pd(lane0_a, lane0_b), 0);
 
  Packet4d lane1_a = _mm512_extractf64x4_pd(a, 1);
  Packet4d lane1_b = _mm512_extractf64x4_pd(b, 1);
  return _mm512_insertf64x4(res, _mm256_and_pd(lane1_a, lane1_b), 1);
#endif
}

References a, b, and res.

pand< Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::pand< Packet8f >	(	const Packet8f &	a,
		const Packet8f &	b
	)

                                                                                  {
  return _mm256_and_ps(a, b);
}

References a, and b.

pand< Packet8i >()

template<>

EIGEN_STRONG_INLINE Packet8i Eigen::internal::pand< Packet8i >	(	const Packet8i &	a,
		const Packet8i &	b
	)

                                                                                  {
#ifdef EIGEN_VECTORIZE_AVX2
  return _mm256_and_si256(a, b);
#else
  return _mm256_castps_si256(_mm256_and_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b)));
#endif
}

References a, and b.

Referenced by pand().

pand< Packet8l >()

template<>

EIGEN_STRONG_INLINE Packet8l Eigen::internal::pand< Packet8l >	(	const Packet8l &	a,
		const Packet8l &	b
	)

                                                                                  {
  return _mm512_and_si512(a, b);
}

References a, and b.

pand< Packet8s >()

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::pand< Packet8s >	(	const Packet8s &	a,
		const Packet8s &	b
	)

                                                                                  {
  return __lsx_vand_v(a, b);
}

References a, and b.

pand< Packet8uc >()

template<>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::pand< Packet8uc >	(	const Packet8uc &	a,
		const Packet8uc &	b
	)

                                                                                      {
  return vand_u8(a, b);
}

References a, and b.

pand< Packet8ui >()

template<>

EIGEN_STRONG_INLINE Packet8ui Eigen::internal::pand< Packet8ui >	(	const Packet8ui &	a,
		const Packet8ui &	b
	)

                                                                                      {
#ifdef EIGEN_VECTORIZE_AVX2
  return _mm256_and_si256(a, b);
#else
  return _mm256_castps_si256(_mm256_and_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b)));
#endif
}

References a, and b.

pand< Packet8us >()

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::pand< Packet8us >	(	const Packet8us &	a,
		const Packet8us &	b
	)

                                                                                      {
  return vec_and(a, b);
}

References a, and b.

Referenced by F32ToBf16Two(), pabs(), and pand< Packet8bf >().

pand< PacketXf >()

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::pand< PacketXf >	(	const PacketXf &	a,
		const PacketXf &	b
	)

                                                                                  {
  return svreinterpret_f32_u32(svand_u32_x(svptrue_b32(), svreinterpret_u32_f32(a), svreinterpret_u32_f32(b)));
}

References a, and b.

pand< PacketXi >()

template<>

EIGEN_STRONG_INLINE PacketXi Eigen::internal::pand< PacketXi >	(	const PacketXi &	a,
		const PacketXi &	b
	)

                                                                                  {
  return svand_s32_x(svptrue_b32(), a, b);
}

References a, and b.

pandnot() [1/7]

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::pandnot ( const Packet &  a, const Packet &  b  )

inline

Returns

the bitwise and of a and not b

                                                                          {
  return pand(a, pnot(b));
}

References a, b, pand(), and pnot().

pandnot() [2/7]

template<>

EIGEN_STRONG_INLINE Packet16bf Eigen::internal::pandnot	(	const Packet16bf &	a,
		const Packet16bf &	b
	)

                                                                                 {
  return Packet16bf(pandnot<Packet8i>(Packet8i(a), Packet8i(b)));
}

References a, b, and pandnot< Packet8i >().

pandnot() [3/7]

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::pandnot	(	const Packet16h &	a,
		const Packet16h &	b
	)

                                                                              {
  return Packet16h(pandnot(Packet8i(a), Packet8i(b)));
}

References a, b, and pandnot().

pandnot() [4/7]

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::pandnot	(	const Packet32h &	a,
		const Packet32h &	b
	)

                                                                              {
  return _mm512_castsi512_ph(_mm512_andnot_si512(_mm512_castph_si512(b), _mm512_castph_si512(a)));
}

References a, and b.

pandnot() [5/7]

template<>

EIGEN_STRONG_INLINE Packet4bf Eigen::internal::pandnot	(	const Packet4bf &	a,
		const Packet4bf &	b
	)

                                                                              {
  return Packet4bf(pandnot<Packet4us>(Packet4us(a), Packet4us(b)));
}

References a, b, and pandnot< Packet4us >().

pandnot() [6/7]

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::pandnot	(	const Packet8bf &	a,
		const Packet8bf &	b
	)

                                                                              {
  return _mm_andnot_si128(b, a);
}

References a, and b.

pandnot() [7/7]

template<>

EIGEN_STRONG_INLINE Packet8h Eigen::internal::pandnot	(	const Packet8h &	a,
		const Packet8h &	b
	)

                                                                           {
  return _mm_andnot_si128(b, a);
}

References a, and b.

Referenced by Eigen::internal::any_visitor< Scalar >::any_predux(), Eigen::internal::count_visitor< Scalar >::count_redux(), generic_pow(), generic_rint(), generic_round(), generic_trunc(), Eigen::internal::unary_pow::handle_nonint_nonint_errors(), packetmath_boolean_mask_ops(), Eigen::internal::scalar_bitwise_not_op< Scalar >::packetOp(), Eigen::internal::scalar_cmp_op< LhsScalar, RhsScalar, cmp_UNORD, UseTypedComparators >::packetOp(), Eigen::internal::scalar_cmp_op< LhsScalar, RhsScalar, cmp_NEQ, UseTypedComparators >::packetOp(), Eigen::internal::scalar_boolean_and_op< Scalar >::packetOp(), Eigen::internal::scalar_boolean_or_op< Scalar >::packetOp(), pandnot(), pandnot< Packet16f >(), pandnot< Packet4cd >(), pandnot< Packet8cf >(), pandnot< Packet8d >(), pasin_float(), pblueNorm(), pexp_complex(), phypot_complex(), pisnan(), prsqrt_float_common(), psqrt_complex(), Eigen::internal::psign_impl< Packet, std::enable_if_t<!NumTraits< typename unpacket_traits< Packet >::type >::IsComplex &&!NumTraits< typename unpacket_traits< Packet >::type >::IsInteger > >::run(), Eigen::internal::psign_impl< Packet, std::enable_if_t<!NumTraits< typename unpacket_traits< Packet >::type >::IsComplex &&!NumTraits< typename unpacket_traits< Packet >::type >::IsSigned &&NumTraits< typename unpacket_traits< Packet >::type >::IsInteger > >::run(), Eigen::internal::psign_impl< Packet, std::enable_if_t< NumTraits< typename unpacket_traits< Packet >::type >::IsComplex &&unpacket_traits< Packet >::vectorizable > >::run(), and Eigen::internal::pselect_impl< Packet, EnableIf >::run().

pandnot< Packet16c >()

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::pandnot< Packet16c >	(	const Packet16c &	a,
		const Packet16c &	b
	)

                                                                                         {
  return __lsx_vandn_v(b, a);
}

References a, and b.

pandnot< Packet16f >()

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::pandnot< Packet16f >	(	const Packet16f &	a,
		const Packet16f &	b
	)

                                                                                         {
#ifdef EIGEN_VECTORIZE_AVX512DQ
  return _mm512_andnot_ps(b, a);
#else
  return _mm512_castsi512_ps(pandnot(_mm512_castps_si512(a), _mm512_castps_si512(b)));
#endif
}

References a, b, and pandnot().

pandnot< Packet16i >()

template<>

EIGEN_STRONG_INLINE Packet16i Eigen::internal::pandnot< Packet16i >	(	const Packet16i &	a,
		const Packet16i &	b
	)

                                                                                         {
  return _mm512_andnot_si512(b, a);
}

References a, and b.

pandnot< Packet16uc >()

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::pandnot< Packet16uc >	(	const Packet16uc &	a,
		const Packet16uc &	b
	)

                                                                                             {
  return __lsx_vandn_v(b, a);
}

References a, and b.

pandnot< Packet1cd >()

template<>

EIGEN_STRONG_INLINE Packet1cd Eigen::internal::pandnot< Packet1cd >	(	const Packet1cd &	a,
		const Packet1cd &	b
	)

                                                                                         {
  Packet1cd res;
  res.v = (Packet2d)__lsx_vandn_v((__m128i)b.v, (__m128i)a.v);
  return res;
}

References a, b, and res.

pandnot< Packet1cf >()

template<>

EIGEN_STRONG_INLINE Packet1cf Eigen::internal::pandnot< Packet1cf >	(	const Packet1cf &	a,
		const Packet1cf &	b
	)

                                                                                         {
  return Packet1cf(vreinterpret_f32_u32(vbic_u32(vreinterpret_u32_f32(a.v), vreinterpret_u32_f32(b.v))));
}

References a, and b.

pandnot< Packet2cd >()

template<>

EIGEN_STRONG_INLINE Packet2cd Eigen::internal::pandnot< Packet2cd >	(	const Packet2cd &	a,
		const Packet2cd &	b
	)

                                                                                         {
  return Packet2cd(_mm256_andnot_pd(b.v, a.v));
}

References a, and b.

pandnot< Packet2cf >()

template<>

EIGEN_STRONG_INLINE Packet2cf Eigen::internal::pandnot< Packet2cf >	(	const Packet2cf &	a,
		const Packet2cf &	b
	)

                                                                                         {
  return Packet2cf(pandnot<Packet4f>(a.v, b.v));
}

References a, b, and pandnot< Packet4f >().

pandnot< Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pandnot< Packet2d >	(	const Packet2d &	a,
		const Packet2d &	b
	)

                                                                                     {
  return (Packet2d)__lsx_vandn_v((__m128i)b, (__m128i)a);
}

References a, and b.

pandnot< Packet2f >()

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::pandnot< Packet2f >	(	const Packet2f &	a,
		const Packet2f &	b
	)

                                                                                     {
  return vreinterpret_f32_u32(vbic_u32(vreinterpret_u32_f32(a), vreinterpret_u32_f32(b)));
}

References a, and b.

pandnot< Packet2i >()

template<>

EIGEN_STRONG_INLINE Packet2i Eigen::internal::pandnot< Packet2i >	(	const Packet2i &	a,
		const Packet2i &	b
	)

                                                                                     {
  return vbic_s32(a, b);
}

References a, and b.

pandnot< Packet2l >()

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::pandnot< Packet2l >	(	const Packet2l &	a,
		const Packet2l &	b
	)

                                                                                     {
  return __lsx_vandn_v(b, a);
}

References a, and b.

pandnot< Packet2ui >()

template<>

EIGEN_STRONG_INLINE Packet2ui Eigen::internal::pandnot< Packet2ui >	(	const Packet2ui &	a,
		const Packet2ui &	b
	)

                                                                                         {
  return vbic_u32(a, b);
}

References a, and b.

pandnot< Packet2ul >()

template<>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::pandnot< Packet2ul >	(	const Packet2ul &	a,
		const Packet2ul &	b
	)

                                                                                         {
  return __lsx_vandn_v(b, a);
}

References a, and b.

pandnot< Packet4c >()

template<>

EIGEN_STRONG_INLINE Packet4c Eigen::internal::pandnot< Packet4c >	(	const Packet4c &	a,
		const Packet4c &	b
	)

                                                                                     {
  return a & ~b;
}

References a, and b.

pandnot< Packet4cd >()

template<>

EIGEN_STRONG_INLINE Packet4cd Eigen::internal::pandnot< Packet4cd >	(	const Packet4cd &	a,
		const Packet4cd &	b
	)

                                                                                         {
  return Packet4cd(pandnot(a.v, b.v));
}

References a, b, and pandnot().

pandnot< Packet4cf >()

template<>

EIGEN_STRONG_INLINE Packet4cf Eigen::internal::pandnot< Packet4cf >	(	const Packet4cf &	a,
		const Packet4cf &	b
	)

                                                                                         {
  return Packet4cf(_mm256_andnot_ps(b.v, a.v));
}

References a, and b.

pandnot< Packet4d >()

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::pandnot< Packet4d >	(	const Packet4d &	a,
		const Packet4d &	b
	)

                                                                                     {
  return _mm256_andnot_pd(b, a);
}

References a, and b.

pandnot< Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pandnot< Packet4f >	(	const Packet4f &	a,
		const Packet4f &	b
	)

                                                                                     {
  return vec_andc(a, b);
}

References a, and b.

Referenced by pandnot< Packet2cf >().

pandnot< Packet4i >()

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pandnot< Packet4i >	(	const Packet4i &	a,
		const Packet4i &	b
	)

                                                                                     {
  return vec_andc(a, b);
}

References a, and b.

pandnot< Packet4s >()

template<>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::pandnot< Packet4s >	(	const Packet4s &	a,
		const Packet4s &	b
	)

                                                                                     {
  return vbic_s16(a, b);
}

References a, and b.

pandnot< Packet4uc >()

template<>

EIGEN_STRONG_INLINE Packet4uc Eigen::internal::pandnot< Packet4uc >	(	const Packet4uc &	a,
		const Packet4uc &	b
	)

                                                                                         {
  return a & ~b;
}

References a, and b.

pandnot< Packet4ui >()

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::pandnot< Packet4ui >	(	const Packet4ui &	a,
		const Packet4ui &	b
	)

                                                                                         {
  return __lsx_vandn_v(b, a);
}

References a, and b.

Referenced by F32ToBf16().

pandnot< Packet4us >()

template<>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::pandnot< Packet4us >	(	const Packet4us &	a,
		const Packet4us &	b
	)

                                                                                         {
  return vbic_u16(a, b);
}

References a, and b.

Referenced by pandnot().

pandnot< Packet8c >()

template<>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::pandnot< Packet8c >	(	const Packet8c &	a,
		const Packet8c &	b
	)

                                                                                     {
  return vbic_s8(a, b);
}

References a, and b.

pandnot< Packet8cf >()

template<>

EIGEN_STRONG_INLINE Packet8cf Eigen::internal::pandnot< Packet8cf >	(	const Packet8cf &	a,
		const Packet8cf &	b
	)

                                                                                         {
  return Packet8cf(pandnot(a.v, b.v));
}

References a, b, and pandnot().

pandnot< Packet8d >()

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::pandnot< Packet8d >	(	const Packet8d &	a,
		const Packet8d &	b
	)

                                                                                     {
#ifdef EIGEN_VECTORIZE_AVX512DQ
  return _mm512_andnot_pd(b, a);
#else
  return _mm512_castsi512_pd(pandnot(_mm512_castpd_si512(a), _mm512_castpd_si512(b)));
#endif
}

References a, b, and pandnot().

pandnot< Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::pandnot< Packet8f >	(	const Packet8f &	a,
		const Packet8f &	b
	)

                                                                                     {
  return _mm256_andnot_ps(b, a);
}

References a, and b.

pandnot< Packet8i >()

template<>

EIGEN_STRONG_INLINE Packet8i Eigen::internal::pandnot< Packet8i >	(	const Packet8i &	a,
		const Packet8i &	b
	)

                                                                                     {
#ifdef EIGEN_VECTORIZE_AVX2
  return _mm256_andnot_si256(b, a);
#else
  return _mm256_castps_si256(_mm256_andnot_ps(_mm256_castsi256_ps(b), _mm256_castsi256_ps(a)));
#endif
}

References a, and b.

Referenced by pandnot().

pandnot< Packet8l >()

template<>

EIGEN_STRONG_INLINE Packet8l Eigen::internal::pandnot< Packet8l >	(	const Packet8l &	a,
		const Packet8l &	b
	)

                                                                                     {
  return _mm512_andnot_si512(b, a);
}

References a, and b.

pandnot< Packet8s >()

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::pandnot< Packet8s >	(	const Packet8s &	a,
		const Packet8s &	b
	)

                                                                                     {
  return __lsx_vandn_v(b, a);
}

References a, and b.

pandnot< Packet8uc >()

template<>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::pandnot< Packet8uc >	(	const Packet8uc &	a,
		const Packet8uc &	b
	)

                                                                                         {
  return vbic_u8(a, b);
}

References a, and b.

pandnot< Packet8ui >()

template<>

EIGEN_STRONG_INLINE Packet8ui Eigen::internal::pandnot< Packet8ui >	(	const Packet8ui &	a,
		const Packet8ui &	b
	)

                                                                                         {
#ifdef EIGEN_VECTORIZE_AVX2
  return _mm256_andnot_si256(b, a);
#else
  return _mm256_castps_si256(_mm256_andnot_ps(_mm256_castsi256_ps(b), _mm256_castsi256_ps(a)));
#endif
}

References a, and b.

pandnot< Packet8us >()

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::pandnot< Packet8us >	(	const Packet8us &	a,
		const Packet8us &	b
	)

                                                                                         {
  return __lsx_vandn_v(b, a);
}

References a, and b.

Referenced by F32ToBf16Two().

pandnot< PacketXf >()

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::pandnot< PacketXf >	(	const PacketXf &	a,
		const PacketXf &	b
	)

                                                                                     {
  return svreinterpret_f32_u32(svbic_u32_x(svptrue_b32(), svreinterpret_u32_f32(a), svreinterpret_u32_f32(b)));
}

References a, and b.

pandnot< PacketXi >()

template<>

EIGEN_STRONG_INLINE PacketXi Eigen::internal::pandnot< PacketXi >	(	const PacketXi &	a,
		const PacketXi &	b
	)

                                                                                     {
  return svbic_s32_x(svptrue_b32(), a, b);
}

References a, and b.

parallelize_gemm()

template<bool Condition, typename Functor , typename Index >

EIGEN_STRONG_INLINE void Eigen::internal::parallelize_gemm	(	const Functor &	func,
		Index	rows,
		Index	cols,
		Index	,
		bool
	)

                                                  {
  func(0, rows, 0, cols);
}

References cols, func(), and rows.

Referenced by Eigen::internal::generic_product_impl< Lhs, Rhs, DenseShape, DenseShape, GemmProduct >::scaleAndAddTo().

parg()

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::parg ( const Packet &  a )

inline

Returns

the phase angle of a

                                                      {
  using numext::arg;
  return arg(a);
}

References a.

Referenced by Eigen::internal::scalar_arg_op< Scalar >::packetOp().

parithmetic_shift_right() [1/30]

template<int N>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::parithmetic_shift_right

(

const Packet16c &

a

)

                                                                          {
  return __lsx_vsrai_b((__m128i)a, N);
}

References a, and N.

parithmetic_shift_right() [2/30]

template<int N>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::parithmetic_shift_right

(

const Packet16uc &

a

)

                                                                            {
  return __lsx_vsrli_b((__m128i)a, N);
}

References a, and N.

parithmetic_shift_right() [3/30]

template<int N>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::parithmetic_shift_right

(

const Packet2l &

a

)

                                                                        {
  return __lsx_vsrai_d((__m128i)a, N);
}

References a, and N.

parithmetic_shift_right() [4/30]

template<int N>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::parithmetic_shift_right

(

const Packet2ul &

a

)

                                                                          {
  return __lsx_vsrli_d((__m128i)a, N);
}

References a, and N.

parithmetic_shift_right() [5/30]

template<int N>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::parithmetic_shift_right

(

const Packet4i &

a

)

                                                                        {
  return vec_sra(a, reinterpret_cast<Packet4ui>(pset1<Packet4i>(N)));
}

References a, N, and pset1< Packet4i >().

parithmetic_shift_right() [6/30]

template<int N>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::parithmetic_shift_right

(

const Packet4ui &

a

)

                                                                          {
  return __lsx_vsrli_w((__m128i)a, N);
}

References a, and N.

parithmetic_shift_right() [7/30]

template<int N>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::parithmetic_shift_right

(

const Packet8s &

a

)

                                                                        {
  return __lsx_vsrai_h((__m128i)a, N);
}

References a, and N.

parithmetic_shift_right() [8/30]

template<int N>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::parithmetic_shift_right

(

const Packet8us &

a

)

                                                                          {
  return __lsx_vsrli_h((__m128i)a, N);
}

References a, and N.

parithmetic_shift_right() [9/30]

template<int N, typename T >

EIGEN_DEVICE_FUNC T Eigen::internal::parithmetic_shift_right ( const T &  a )

inline

Returns

a arithmetically shifted by N bits to the right

                                                               {
  return numext::arithmetic_shift_right(a, N);
}

References a, Eigen::numext::arithmetic_shift_right(), and N.

parithmetic_shift_right() [10/30]

template<int N>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::parithmetic_shift_right

(

Packet16c

a

)

                                                                   {
  return vshrq_n_s8(a, N);
}

References a, and N.

parithmetic_shift_right() [11/30]

template<int N>

EIGEN_STRONG_INLINE Packet16i Eigen::internal::parithmetic_shift_right

(

Packet16i

a

)

                                                                   {
  return _mm512_srai_epi32(a, N);
}

References a, and N.

parithmetic_shift_right() [12/30]

template<int N>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::parithmetic_shift_right

(

Packet16uc

a

)

                                                                     {
  return vshrq_n_u8(a, N);
}

References a, and N.

parithmetic_shift_right() [13/30]

template<int N>

EIGEN_STRONG_INLINE Packet2i Eigen::internal::parithmetic_shift_right

(

Packet2i

a

)

                                                                 {
  return vshr_n_s32(a, N);
}

References a, and N.

parithmetic_shift_right() [14/30]

template<int N>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::parithmetic_shift_right

(

Packet2l

a

)

                                                                 {
  return vshrq_n_s64(a, N);
}

References a, and N.

parithmetic_shift_right() [15/30]

template<int N>

EIGEN_STRONG_INLINE Packet2ui Eigen::internal::parithmetic_shift_right

(

Packet2ui

a

)

                                                                   {
  return vshr_n_u32(a, N);
}

References a, and N.

parithmetic_shift_right() [16/30]

template<int N>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::parithmetic_shift_right

(

Packet2ul

a

)

                                                                   {
  return vshrq_n_u64(a, N);
}

References a, and N.

parithmetic_shift_right() [17/30]

template<int N>

EIGEN_STRONG_INLINE Packet4c Eigen::internal::parithmetic_shift_right

(

Packet4c &

a

)

                                                                  {
  return vget_lane_s32(vreinterpret_s32_s8(vshr_n_s8(vreinterpret_s8_s32(vdup_n_s32(a)), N)), 0);
}

References a, and N.

parithmetic_shift_right() [18/30]

template<int N>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::parithmetic_shift_right

(

Packet4i

a

)

                                                                 {
  return vshrq_n_s32(a, N);
}

References a, and N.

parithmetic_shift_right() [19/30]

template<int N>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::parithmetic_shift_right

(

Packet4s

a

)

                                                                 {
  return vshr_n_s16(a, N);
}

References a, and N.

parithmetic_shift_right() [20/30]

template<int N>

EIGEN_STRONG_INLINE Packet4uc Eigen::internal::parithmetic_shift_right

(

Packet4uc &

a

)

                                                                    {
  return vget_lane_u32(vreinterpret_u32_u8(vshr_n_u8(vreinterpret_u8_u32(vdup_n_u32(a)), N)), 0);
}

References a, and N.

parithmetic_shift_right() [21/30]

template<int N>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::parithmetic_shift_right

(

Packet4ui

a

)

                                                                   {
  return vshrq_n_u32(a, N);
}

References a, and N.

parithmetic_shift_right() [22/30]

template<int N>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::parithmetic_shift_right

(

Packet4us

a

)

                                                                   {
  return vshr_n_u16(a, N);
}

References a, and N.

parithmetic_shift_right() [23/30]

template<int N>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::parithmetic_shift_right

(

Packet8c

a

)

                                                                 {
  return vshr_n_s8(a, N);
}

References a, and N.

parithmetic_shift_right() [24/30]

template<int N>

EIGEN_STRONG_INLINE Packet8i Eigen::internal::parithmetic_shift_right

(

Packet8i

a

)

                                                                 {
#ifdef EIGEN_VECTORIZE_AVX2
  return _mm256_srai_epi32(a, N);
#else
  __m128i lo = _mm_srai_epi32(_mm256_extractf128_si256(a, 0), N);
  __m128i hi = _mm_srai_epi32(_mm256_extractf128_si256(a, 1), N);
  return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1);
#endif
}

References a, and N.

parithmetic_shift_right() [25/30]

template<int N>

EIGEN_STRONG_INLINE Packet8l Eigen::internal::parithmetic_shift_right

(

Packet8l

a

)

                                                                 {
  return _mm512_srai_epi64(a, N);
}

References a, and N.

parithmetic_shift_right() [26/30]

template<int N>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::parithmetic_shift_right

(

Packet8s

a

)

                                                                 {
  return vshrq_n_s16(a, N);
}

References a, and N.

parithmetic_shift_right() [27/30]

template<int N>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::parithmetic_shift_right

(

Packet8uc

a

)

                                                                   {
  return vshr_n_u8(a, N);
}

References a, and N.

parithmetic_shift_right() [28/30]

template<int N>

EIGEN_STRONG_INLINE Packet8ui Eigen::internal::parithmetic_shift_right

(

Packet8ui

a

)

                                                                   {
  return (Packet8ui)plogical_shift_right<N>((Packet8i)a);
}

References a.

parithmetic_shift_right() [29/30]

template<int N>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::parithmetic_shift_right

(

Packet8us

a

)

                                                                   {
  return vshrq_n_u16(a, N);
}

References a, and N.

parithmetic_shift_right() [30/30]

template<int N>

EIGEN_STRONG_INLINE PacketXi Eigen::internal::parithmetic_shift_right

(

PacketXi

a

)

                                                                 {
  return svasrd_n_s32_x(svptrue_b32(), a, N);
}

References a, and N.

partial_lu_inplace()

template<typename MatrixType , typename TranspositionType >

void Eigen::internal::partial_lu_inplace	(	MatrixType &	lu,
		TranspositionType &	row_transpositions,
		typename TranspositionType::StorageIndex &	nb_transpositions
	)

performs the LU decomposition with partial pivoting in-place.

                                                                                   {
  // Special-case of zero matrix.
  if (lu.rows() == 0 || lu.cols() == 0) {
    nb_transpositions = 0;
    return;
  }
  eigen_assert(lu.cols() == row_transpositions.size());
  eigen_assert(row_transpositions.size() < 2 ||
               (&row_transpositions.coeffRef(1) - &row_transpositions.coeffRef(0)) == 1);
 
  partial_lu_impl<typename MatrixType::Scalar, MatrixType::Flags & RowMajorBit ? RowMajor : ColMajor,
                  typename TranspositionType::StorageIndex,
                  internal::min_size_prefer_fixed(MatrixType::RowsAtCompileTime, MatrixType::ColsAtCompileTime)>::
      blocked_lu(lu.rows(), lu.cols(), &lu.coeffRef(0, 0), lu.outerStride(), &row_transpositions.coeffRef(0),
                 nb_transpositions);
}

References Eigen::ColMajor, eigen_assert, lu(), min_size_prefer_fixed(), Eigen::RowMajor, and Eigen::RowMajorBit.

Referenced by Eigen::PartialPivLU< MatrixType_, PermutationIndex_ >::compute().

pasin()

template<typename Packet >

EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::pasin

(

const Packet &

a

)

Returns

the arc sine of a (coeff-wise)

                                                                                   {
  EIGEN_USING_STD(asin);
  return asin(a);
}

References a, Eigen::bfloat16_impl::asin(), and EIGEN_USING_STD.

Referenced by packetmath_real(), and Eigen::internal::scalar_asin_op< Scalar >::packetOp().

pasin_float()

template<typename Packet >

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::pasin_float

(

const Packet &

x

)

Returns

asin(x) for single precision float

                                                                                           {
  typedef typename unpacket_traits<Packet>::type Scalar;
  static_assert(std::is_same<Scalar, float>::value, "Scalar type must be float");
 
  constexpr float kPiOverTwo = static_cast<float>(EIGEN_PI / 2);
 
  const Packet cst_half = pset1<Packet>(0.5f);
  const Packet cst_one = pset1<Packet>(1.0f);
  const Packet cst_two = pset1<Packet>(2.0f);
  const Packet cst_pi_over_two = pset1<Packet>(kPiOverTwo);
 
  const Packet abs_x = pabs(x_in);
  const Packet sign_mask = pandnot(x_in, abs_x);
  const Packet invalid_mask = pcmp_lt(cst_one, abs_x);
 
  // For arguments |x| > 0.5, we map x back to [0:0.5] using
  // the transformation x_large = sqrt(0.5*(1-x)), and use the
  // identity
  //   asin(x) = pi/2 - 2 * asin( sqrt( 0.5 * (1 - x)))
 
  const Packet x_large = psqrt(pnmadd(cst_half, abs_x, cst_half));
  const Packet large_mask = pcmp_lt(cst_half, abs_x);
  const Packet x = pselect(large_mask, x_large, abs_x);
  const Packet x2 = pmul(x, x);
 
  // For |x| < 0.5 approximate asin(x)/x by an 8th order polynomial with
  // even terms only.
  constexpr float alpha[] = {5.08838854730129241943359375e-2f, 3.95139865577220916748046875e-2f,
                             7.550220191478729248046875e-2f, 0.16664917767047882080078125f, 1.00000011920928955078125f};
  Packet p = ppolevl<Packet, 4>::run(x2, alpha);
  p = pmul(p, x);
 
  const Packet p_large = pnmadd(cst_two, p, cst_pi_over_two);
  p = pselect(large_mask, p_large, p);
  // Flip the sign for negative arguments.
  p = pxor(p, sign_mask);
  // Return NaN for arguments outside [-1:1].
  return por(invalid_mask, p);
}

References alpha, EIGEN_PI, p, pabs(), pandnot(), pcmp_lt(), pmul(), pnmadd(), por(), pselect(), psqrt(), pxor(), Eigen::internal::ppolevl< Packet, N >::run(), Eigen::value, plotDoE::x, and Global_parameters::x2().

patan()

template<typename Packet >

EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::patan

(

const Packet &

a

)

Returns

the arc tangent of a (coeff-wise)

                                                                                   {
  EIGEN_USING_STD(atan);
  return atan(a);
}

References a, Eigen::bfloat16_impl::atan(), and EIGEN_USING_STD.

Referenced by packetmath_real(), and Eigen::internal::scalar_atan_op< Scalar >::packetOp().

patan2()

template<typename Packet , std::enable_if_t< is_scalar< Packet >::value, int > = 0>

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet Eigen::internal::patan2	(	const Packet &	y,
		const Packet &	x
	)

Returns

the 2-argument arc tangent of y and x (coeff-wise)

                                                                                      {
  return numext::atan2(y, x);
}

References Eigen::numext::atan2(), plotDoE::x, and y.

Referenced by Eigen::internal::scalar_atan2_op< LhsScalar, RhsScalar >::packetOp(), and plog_complex().

patanh()

template<typename Packet >

EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::patanh

(

const Packet &

a

)

Returns

the arc tangent of a (coeff-wise)

                                                                                    {
  EIGEN_USING_STD(atanh);
  return atanh(a);
}

References a, Eigen::bfloat16_impl::atanh(), and EIGEN_USING_STD.

Referenced by packetmath_real(), and Eigen::internal::scalar_atanh_op< Scalar >::packetOp().

patanh_double()

template<typename Packet >

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::patanh_double

(

const Packet &

x

)

Returns

atanh(x) for double precision float

                                                                                          {
  typedef typename unpacket_traits<Packet>::type Scalar;
  static_assert(std::is_same<Scalar, double>::value, "Scalar type must be double");
  // For x in [-0.5:0.5] we use a rational approximation of the form
  // R(x) = x + x^3*P(x^2)/Q(x^2), where P is or order 4 and Q is of order 5.
  constexpr double alpha[] = {3.3071338469301391e-03, -4.7129526768798737e-02, 1.8185306179826699e-01,
                              -2.5949536095445679e-01, 1.2306328729812676e-01};
 
  constexpr double beta[] = {-3.8679974580640881e-03, 7.6391885763341910e-02,  -4.2828141436397615e-01,
                             9.8733495886883648e-01,  -1.0000000000000000e+00, 3.6918986189438030e-01};
 
  const Packet x2 = pmul(x, x);
  const Packet x3 = pmul(x, x2);
  Packet p = ppolevl<Packet, 4>::run(x2, alpha);
  Packet q = ppolevl<Packet, 5>::run(x2, beta);
  Packet y_small = pmadd(x3, pdiv(p, q), x);
 
  // For |x| in ]0.5:1.0] we use atanh = 0.5*ln((1+x)/(1-x));
  const Packet half = pset1<Packet>(0.5);
  const Packet one = pset1<Packet>(1.0);
  Packet y_large = pdiv(padd(one, x), psub(one, x));
  y_large = pmul(half, plog(y_large));
 
  const Packet x_gt_half = pcmp_le(half, pabs(x));
  const Packet x_eq_one = pcmp_eq(one, pabs(x));
  const Packet x_gt_one = pcmp_lt(one, pabs(x));
  const Packet sign_mask = pset1<Packet>(-0.0);
  const Packet x_sign = pand(sign_mask, x);
  const Packet inf = pset1<Packet>(std::numeric_limits<double>::infinity());
  return por(x_gt_one, pselect(x_eq_one, por(x_sign, inf), pselect(x_gt_half, y_large, y_small)));
}

References alpha, beta, constants::inf, p, pabs(), padd(), pand(), pcmp_eq(), pcmp_le(), pcmp_lt(), pdiv(), plog(), pmadd(), pmul(), por(), pselect(), psub(), Eigen::numext::q, Eigen::internal::ppolevl< Packet, N >::run(), Eigen::value, plotDoE::x, and Global_parameters::x2().

patanh_float()

template<typename Packet >

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::patanh_float

(

const Packet &

x

)

Returns

atanh(x) for single precision float

                                                                                         {
  typedef typename unpacket_traits<Packet>::type Scalar;
  static_assert(std::is_same<Scalar, float>::value, "Scalar type must be float");
 
  // For |x| in [0:0.5] we use a polynomial approximation of the form
  // P(x) = x + x^3*(alpha[4] + x^2 * (alpha[3] + x^2 * (... x^2 * alpha[0]) ... )).
  constexpr float alpha[] = {0.1819281280040740966796875f, 8.2311116158962249755859375e-2f,
                             0.14672131836414337158203125f, 0.1997792422771453857421875f, 0.3333373963832855224609375f};
  const Packet x2 = pmul(x, x);
  const Packet x3 = pmul(x, x2);
  Packet p = ppolevl<Packet, 4>::run(x2, alpha);
  p = pmadd(x3, p, x);
 
  // For |x| in ]0.5:1.0] we use atanh = 0.5*ln((1+x)/(1-x));
  const Packet half = pset1<Packet>(0.5f);
  const Packet one = pset1<Packet>(1.0f);
  Packet r = pdiv(padd(one, x), psub(one, x));
  r = pmul(half, plog(r));
 
  const Packet x_gt_half = pcmp_le(half, pabs(x));
  const Packet x_eq_one = pcmp_eq(one, pabs(x));
  const Packet x_gt_one = pcmp_lt(one, pabs(x));
  const Packet sign_mask = pset1<Packet>(-0.0f);
  const Packet x_sign = pand(sign_mask, x);
  const Packet inf = pset1<Packet>(std::numeric_limits<float>::infinity());
  return por(x_gt_one, pselect(x_eq_one, por(x_sign, inf), pselect(x_gt_half, r, p)));
}

References alpha, constants::inf, p, pabs(), padd(), pand(), pcmp_eq(), pcmp_le(), pcmp_lt(), pdiv(), plog(), pmadd(), pmul(), por(), pselect(), psub(), UniformPSDSelfTest::r, Eigen::internal::ppolevl< Packet, N >::run(), Eigen::value, plotDoE::x, and Global_parameters::x2().

pbessel_i0()

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::pbessel_i0

(

const Packet &

x

)

Returns

the exponentially scaled modified Bessel function of order zero i0(a) (coeff-wise)

                                                                                                          {
  return numext::bessel_i0(x);
}

References Eigen::bessel_i0(), and plotDoE::x.

Referenced by packetmath_real(), and Eigen::internal::scalar_bessel_i0_op< Scalar >::packetOp().

pbessel_i0e()

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::pbessel_i0e

(

const Packet &

x

)

Returns

the exponentially scaled modified Bessel function of order zero i0e(a) (coeff-wise)

                                                                                                           {
  return numext::bessel_i0e(x);
}

References Eigen::bessel_i0e(), and plotDoE::x.

Referenced by packetmath_real(), and Eigen::internal::scalar_bessel_i0e_op< Scalar >::packetOp().

pbessel_i1()

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::pbessel_i1

(

const Packet &

x

)

Returns

the exponentially scaled modified Bessel function of order one i1(a) (coeff-wise)

                                                                                                          {
  return numext::bessel_i1(x);
}

References Eigen::bessel_i1(), and plotDoE::x.

Referenced by packetmath_real(), and Eigen::internal::scalar_bessel_i1_op< Scalar >::packetOp().

pbessel_i1e()

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::pbessel_i1e

(

const Packet &

x

)

Returns

the exponentially scaled modified Bessel function of order one i1e(a) (coeff-wise)

                                                                                                           {
  return numext::bessel_i1e(x);
}

References Eigen::bessel_i1e(), and plotDoE::x.

Referenced by packetmath_real(), and Eigen::internal::scalar_bessel_i1e_op< Scalar >::packetOp().

pbessel_j0()

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::pbessel_j0

(

const Packet &

x

)

Returns

the exponentially scaled modified Bessel function of order zero j0(a) (coeff-wise)

                                                                                                          {
  return numext::bessel_j0(x);
}

References Eigen::bessel_j0(), and plotDoE::x.

Referenced by packetmath_real(), and Eigen::internal::scalar_bessel_j0_op< Scalar >::packetOp().

pbessel_j1()

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::pbessel_j1

(

const Packet &

x

)

Returns

the exponentially scaled modified Bessel function of order zero j1(a) (coeff-wise)

                                                                                                          {
  return numext::bessel_j1(x);
}

References Eigen::bessel_j1(), and plotDoE::x.

Referenced by packetmath_real(), and Eigen::internal::scalar_bessel_j1_op< Scalar >::packetOp().

pbessel_k0()

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::pbessel_k0

(

const Packet &

x

)

Returns

the exponentially scaled modified Bessel function of order zero k0(a) (coeff-wise)

                                                                                                          {
  return numext::bessel_k0(x);
}

References Eigen::bessel_k0(), and plotDoE::x.

Referenced by packetmath_real(), and Eigen::internal::scalar_bessel_k0_op< Scalar >::packetOp().

pbessel_k0e()

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::pbessel_k0e

(

const Packet &

x

)

Returns

the exponentially scaled modified Bessel function of order zero k0e(a) (coeff-wise)

                                                                                                           {
  return numext::bessel_k0e(x);
}

References Eigen::bessel_k0e(), and plotDoE::x.

Referenced by packetmath_real(), and Eigen::internal::scalar_bessel_k0e_op< Scalar >::packetOp().

pbessel_k1()

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::pbessel_k1

(

const Packet &

x

)

Returns

the exponentially scaled modified Bessel function of order one k1e(a) (coeff-wise)

                                                                                                          {
  return numext::bessel_k1(x);
}

References Eigen::bessel_k1(), and plotDoE::x.

Referenced by packetmath_real(), and Eigen::internal::scalar_bessel_k1_op< Scalar >::packetOp().

pbessel_k1e()

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::pbessel_k1e

(

const Packet &

x

)

Returns

the exponentially scaled modified Bessel function of order one k1e(a) (coeff-wise)

                                                                                                           {
  return numext::bessel_k1e(x);
}

References Eigen::bessel_k1e(), and plotDoE::x.

Referenced by packetmath_real(), and Eigen::internal::scalar_bessel_k1e_op< Scalar >::packetOp().

pbessel_y0()

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::pbessel_y0

(

const Packet &

x

)

Returns

the exponentially scaled modified Bessel function of order one y0(a) (coeff-wise)

                                                                                                          {
  return numext::bessel_y0(x);
}

References Eigen::bessel_y0(), and plotDoE::x.

Referenced by Eigen::internal::scalar_bessel_y0_op< Scalar >::packetOp().

pbessel_y1()

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::pbessel_y1

(

const Packet &

x

)

Returns

the exponentially scaled modified Bessel function of order one y1(a) (coeff-wise)

                                                                                                          {
  return numext::bessel_y1(x);
}

References Eigen::bessel_y1(), and plotDoE::x.

Referenced by packetmath_real(), and Eigen::internal::scalar_bessel_y1_op< Scalar >::packetOp().

pbetainc()

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet Eigen::internal::pbetainc	(	const Packet &	a,
		const Packet &	b,
		const Packet &	x
	)

Returns

the complementary incomplete gamma function betainc(a, b, x)

                                                                                                         {
  using numext::betainc;
  return betainc(a, b, x);
}

References a, b, Eigen::betainc(), and plotDoE::x.

Referenced by Eigen::internal::scalar_betainc_op< Scalar >::packetOp().

pblend() [1/16]

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::pblend	(	const Selector< 16 > &	ifPacket,
		const Packet16c &	thenPacket,
		const Packet16c &	elsePacket
	)

                                                                  {
  Packet16uc select = {ifPacket.select[0],  ifPacket.select[1],  ifPacket.select[2],  ifPacket.select[3],
                       ifPacket.select[4],  ifPacket.select[5],  ifPacket.select[6],  ifPacket.select[7],
                       ifPacket.select[8],  ifPacket.select[9],  ifPacket.select[10], ifPacket.select[11],
                       ifPacket.select[12], ifPacket.select[13], ifPacket.select[14], ifPacket.select[15]};
 
  Packet16uc mask = reinterpret_cast<Packet16uc>(pnegate(reinterpret_cast<Packet16c>(select)));
  return vec_sel(elsePacket, thenPacket, mask);
}

References pnegate(), and Eigen::internal::Selector< N >::select.

pblend() [2/16]

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::pblend	(	const Selector< 16 > &	ifPacket,
		const Packet16f &	thenPacket,
		const Packet16f &	elsePacket
	)

                                                                  {
  __mmask16 m = avx512_blend_mask(ifPacket);
  return _mm512_mask_blend_ps(m, elsePacket, thenPacket);
}

References avx512_blend_mask(), and m.

pblend() [3/16]

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::pblend	(	const Selector< 16 > &	ifPacket,
		const Packet16uc &	thenPacket,
		const Packet16uc &	elsePacket
	)

                                                                    {
  Packet16uc select = {ifPacket.select[0],  ifPacket.select[1],  ifPacket.select[2],  ifPacket.select[3],
                       ifPacket.select[4],  ifPacket.select[5],  ifPacket.select[6],  ifPacket.select[7],
                       ifPacket.select[8],  ifPacket.select[9],  ifPacket.select[10], ifPacket.select[11],
                       ifPacket.select[12], ifPacket.select[13], ifPacket.select[14], ifPacket.select[15]};
 
  Packet16uc mask = reinterpret_cast<Packet16uc>(pnegate(reinterpret_cast<Packet16c>(select)));
  return vec_sel(elsePacket, thenPacket, mask);
}

References pnegate(), and Eigen::internal::Selector< N >::select.

pblend() [4/16]

template<>

EIGEN_STRONG_INLINE Packet2cf Eigen::internal::pblend	(	const Selector< 2 > &	ifPacket,
		const Packet2cf &	thenPacket,
		const Packet2cf &	elsePacket
	)

                                                                  {
  return (Packet2cf)(Packet4f)pblend<Packet2d>(ifPacket, (Packet2d)thenPacket.v, (Packet2d)elsePacket.v);
}

pblend() [5/16]

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pblend	(	const Selector< 2 > &	ifPacket,
		const Packet2d &	thenPacket,
		const Packet2d &	elsePacket
	)

                                                                {
  Packet2ul select = {ifPacket.select[0], ifPacket.select[1]};
  Packet2l mask = __builtin_msa_ceqi_d((Packet2l)select, 0);
  return (Packet2d)__builtin_msa_bsel_v((v16u8)mask, (v16u8)thenPacket, (v16u8)elsePacket);
}

References Eigen::internal::Selector< N >::select.

pblend() [6/16]

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::pblend	(	const Selector< 2 > &	ifPacket,
		const Packet2l &	thenPacket,
		const Packet2l &	elsePacket
	)

                                                                {
  const __m128i true_mask = sse_blend_mask(ifPacket);
  return pselect<Packet2l>(true_mask, thenPacket, elsePacket);
}

References sse_blend_mask().

pblend() [7/16]

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::pblend	(	const Selector< 4 > &	ifPacket,
		const Packet4d &	thenPacket,
		const Packet4d &	elsePacket
	)

                                                                {
  const __m256d true_mask = _mm256_castsi256_pd(avx_blend_mask(ifPacket));
  return pselect<Packet4d>(true_mask, thenPacket, elsePacket);
}

References avx_blend_mask(), and pselect< Packet4d >().

pblend() [8/16]

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pblend	(	const Selector< 4 > &	ifPacket,
		const Packet4f &	thenPacket,
		const Packet4f &	elsePacket
	)

                                                                {
  return pblend4<Packet4f>(ifPacket, thenPacket, elsePacket);
}

pblend() [9/16]

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pblend	(	const Selector< 4 > &	ifPacket,
		const Packet4i &	thenPacket,
		const Packet4i &	elsePacket
	)

                                                                {
  return pblend4<Packet4i>(ifPacket, thenPacket, elsePacket);
}

Referenced by Eigen::TensorEvaluator< const TensorSelectOp< IfArgType, ThenArgType, ElseArgType >, Device >::packet(), packetmath(), and pblend().

pblend() [10/16]

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::pblend	(	const Selector< 4 > &	ifPacket,
		const Packet4ui &	thenPacket,
		const Packet4ui &	elsePacket
	)

                                                                  {
  return (Packet4ui)pblend(ifPacket, (Packet4i)thenPacket, (Packet4i)elsePacket);
}

References pblend().

pblend() [11/16]

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::pblend	(	const Selector< 8 > &	ifPacket,
		const Packet8bf &	thenPacket,
		const Packet8bf &	elsePacket
	)

                                                                  {
  return pblend<Packet8us>(ifPacket, thenPacket, elsePacket);
}

pblend() [12/16]

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::pblend	(	const Selector< 8 > &	ifPacket,
		const Packet8d &	thenPacket,
		const Packet8d &	elsePacket
	)

                                                                {
  __mmask8 m = avx512_blend_mask(ifPacket);
  return _mm512_mask_blend_pd(m, elsePacket, thenPacket);
}

References avx512_blend_mask(), and m.

pblend() [13/16]

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::pblend	(	const Selector< 8 > &	ifPacket,
		const Packet8f &	thenPacket,
		const Packet8f &	elsePacket
	)

                                                                {
  const __m256 true_mask = _mm256_castsi256_ps(avx_blend_mask(ifPacket));
  return pselect<Packet8f>(true_mask, thenPacket, elsePacket);
}

References avx_blend_mask(), and pselect< Packet8f >().

pblend() [14/16]

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::pblend	(	const Selector< 8 > &	ifPacket,
		const Packet8s &	thenPacket,
		const Packet8s &	elsePacket
	)

                                                                {
  Packet8us select = {ifPacket.select[0], ifPacket.select[1], ifPacket.select[2], ifPacket.select[3],
                      ifPacket.select[4], ifPacket.select[5], ifPacket.select[6], ifPacket.select[7]};
  Packet8us mask = reinterpret_cast<Packet8us>(pnegate(reinterpret_cast<Packet8s>(select)));
  Packet8s result = vec_sel(elsePacket, thenPacket, mask);
  return result;
}

References pnegate(), and Eigen::internal::Selector< N >::select.

pblend() [15/16]

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::pblend	(	const Selector< 8 > &	ifPacket,
		const Packet8us &	thenPacket,
		const Packet8us &	elsePacket
	)

                                                                  {
  Packet8us select = {ifPacket.select[0], ifPacket.select[1], ifPacket.select[2], ifPacket.select[3],
                      ifPacket.select[4], ifPacket.select[5], ifPacket.select[6], ifPacket.select[7]};
  Packet8us mask = reinterpret_cast<Packet8us>(pnegate(reinterpret_cast<Packet8s>(select)));
  return vec_sel(elsePacket, thenPacket, mask);
}

References pnegate(), and Eigen::internal::Selector< N >::select.

pblend() [16/16]

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::pblend ( const Selector< unpacket_traits< Packet >::size > &  ifPacket, const Packet &  thenPacket, const Packet &  elsePacket  )

inline

                                                                                           {
  return ifPacket.select[0] ? thenPacket : elsePacket;
}

pblend4()

template<typename Packet >

EIGEN_STRONG_INLINE Packet Eigen::internal::pblend4	(	const Selector< 4 > &	ifPacket,
		const Packet &	thenPacket,
		const Packet &	elsePacket
	)

                                                                                                                    {
  Packet4ui select = {ifPacket.select[0], ifPacket.select[1], ifPacket.select[2], ifPacket.select[3]};
  Packet4ui mask = reinterpret_cast<Packet4ui>(pnegate(reinterpret_cast<Packet4i>(select)));
  return vec_sel(elsePacket, thenPacket, mask);
}

References pnegate(), and Eigen::internal::Selector< N >::select.

pbroadcast2()

template<typename Packet >

EIGEN_DEVICE_FUNC void Eigen::internal::pbroadcast2 ( const typename unpacket_traits< Packet >::type *  a, Packet &  a0, Packet &  a1  )

inline

equivalent to

a0 = pload1(a+0);
a1 = pload1(a+1);

See also

pset1, pload1, ploaddup, pbroadcast4

                                                                                                                 {
  a0 = pload1<Packet>(a + 0);
  a1 = pload1<Packet>(a + 1);
}

References a.

pbroadcast4()

template<typename Packet >

EIGEN_DEVICE_FUNC void Eigen::internal::pbroadcast4 ( const typename unpacket_traits< Packet >::type *  a, Packet &  a0, Packet &  a1, Packet &  a2, Packet &  a3  )

inline

equivalent to

a0 = pload1(a+0);
a1 = pload1(a+1);
a2 = pload1(a+2);
a3 = pload1(a+3);

See also

pset1, pload1, ploaddup, pbroadcast2

                                                                  {
  a0 = pload1<Packet>(a + 0);
  a1 = pload1<Packet>(a + 1);
  a2 = pload1<Packet>(a + 2);
  a3 = pload1<Packet>(a + 3);
}

References a.

Referenced by Eigen::internal::gebp_traits< LhsScalar_, RhsScalar_, ConjLhs_, ConjRhs_, Arch, PacketSize_ >::loadRhs(), Eigen::internal::gebp_traits< std::complex< RealScalar >, RealScalar, ConjLhs_, false, Arch, PacketSize_ >::loadRhs(), and Eigen::internal::gebp_traits< RealScalar, std::complex< RealScalar >, false, ConjRhs_, Arch, PacketSize_ >::loadRhs().

pbroadcast4< Packet2d >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pbroadcast4< Packet2d >	(	const double *	a,
		Packet2d &	a0,
		Packet2d &	a1,
		Packet2d &	a2,
		Packet2d &	a3
	)

                                                             {
#ifdef EIGEN_VECTORIZE_SSE3
  a0 = _mm_loaddup_pd(a + 0);
  a1 = _mm_loaddup_pd(a + 1);
  a2 = _mm_loaddup_pd(a + 2);
  a3 = _mm_loaddup_pd(a + 3);
#else
  a1 = pload<Packet2d>(a);
  a0 = vec2d_swizzle1(a1, 0, 0);
  a1 = vec2d_swizzle1(a1, 1, 1);
  a3 = pload<Packet2d>(a + 2);
  a2 = vec2d_swizzle1(a3, 0, 0);
  a3 = vec2d_swizzle1(a3, 1, 1);
#endif
}

References a, pload< Packet2d >(), and vec2d_swizzle1.

pbroadcast4< Packet4f >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pbroadcast4< Packet4f >	(	const float *	a,
		Packet4f &	a0,
		Packet4f &	a1,
		Packet4f &	a2,
		Packet4f &	a3
	)

                                                                                                                       {
  pbroadcast4_common<Packet4f>(a, a0, a1, a2, a3);
}

References a.

Referenced by pbroadcastN< Packet4f, 4, true >().

pbroadcast4< Packet4i >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pbroadcast4< Packet4i >	(	const int *	a,
		Packet4i &	a0,
		Packet4i &	a1,
		Packet4i &	a2,
		Packet4i &	a3
	)

                                                                                                                     {
  pbroadcast4_common<Packet4i>(a, a0, a1, a2, a3);
}

References a.

pbroadcast4_common()

template<typename Packet >

EIGEN_STRONG_INLINE void Eigen::internal::pbroadcast4_common	(	const __UNPACK_TYPE__(Packet) *	a,
		Packet &	a0,
		Packet &	a1,
		Packet &	a2,
		Packet &	a3
	)

                                                        {
  a3 = pload<Packet>(a);
  a0 = vec_splat(a3, 0);
  a1 = vec_splat(a3, 1);
  a2 = vec_splat(a3, 2);
  a3 = vec_splat(a3, 3);
}

References a.

pbroadcastN()

template<typename Packet , int N, bool real>

EIGEN_ALWAYS_INLINE void Eigen::internal::pbroadcastN	(	const __UNPACK_TYPE__(Packet) *	ap0,
		const __UNPACK_TYPE__(Packet) *	ap1,
		const __UNPACK_TYPE__(Packet) *	ap2,
		Packet &	a0,
		Packet &	a1,
		Packet &	a2,
		Packet &	a3
	)

                                                 {
  a0 = pset1<Packet>(ap0[0]);
  if (N == 4) {
    a1 = pset1<Packet>(ap0[1]);
    a2 = pset1<Packet>(ap0[2]);
    a3 = pset1<Packet>(ap0[3]);
    EIGEN_UNUSED_VARIABLE(ap1);
    EIGEN_UNUSED_VARIABLE(ap2);
  } else {
    if (N > 1) {
      a1 = pset1<Packet>(ap1[0]);
    } else {
      EIGEN_UNUSED_VARIABLE(a1);
      EIGEN_UNUSED_VARIABLE(ap1);
    }
    if (N > 2) {
      a2 = pset1<Packet>(ap2[0]);
    } else {
      EIGEN_UNUSED_VARIABLE(a2);
      EIGEN_UNUSED_VARIABLE(ap2);
    }
  }
}

References EIGEN_UNUSED_VARIABLE, and N.

pbroadcastN< Packet2d, 4, false >()

template<>

EIGEN_ALWAYS_INLINE void Eigen::internal::pbroadcastN< Packet2d, 4, false >	(	const double *	ap0,
		const double *	,
		const double *	,
		Packet2d &	a0,
		Packet2d &	a1,
		Packet2d &	a2,
		Packet2d &	a3
	)

                                                                                                   {
  a1 = pload<Packet2d>(ap0);
  a3 = pload<Packet2d>(ap0 + 2);
  a0 = vec_splat(a1, 0);
  a1 = vec_splat(a1, 1);
  a2 = vec_splat(a3, 0);
  a3 = vec_splat(a3, 1);
}

References pload< Packet2d >().

pbroadcastN< Packet4f, 4, false >()

template<>

EIGEN_ALWAYS_INLINE void Eigen::internal::pbroadcastN< Packet4f, 4, false >	(	const float *	ap0,
		const float *	ap1,
		const float *	ap2,
		Packet4f &	a0,
		Packet4f &	a1,
		Packet4f &	a2,
		Packet4f &	a3
	)

                                                                                                                 {
  pbroadcastN<Packet4f, 4, true>(ap0, ap1, ap2, a0, a1, a2, a3);
}

References pbroadcastN< Packet4f, 4, true >().

pbroadcastN< Packet4f, 4, true >()

template<>

EIGEN_ALWAYS_INLINE void Eigen::internal::pbroadcastN< Packet4f, 4, true >	(	const float *	ap0,
		const float *	,
		const float *	,
		Packet4f &	a0,
		Packet4f &	a1,
		Packet4f &	a2,
		Packet4f &	a3
	)

                                                                                                  {
  pbroadcast4<Packet4f>(ap0, a0, a1, a2, a3);
}

References pbroadcast4< Packet4f >().

Referenced by pbroadcastN< Packet4f, 4, false >().

pcarg()

template<typename Packet , std::enable_if_t< is_scalar< Packet >::value, int > = 0>

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet Eigen::internal::pcarg

(

const Packet &

a

)

Returns

the argument of a as a complex number

                                                                    {
  return Packet(numext::arg(a));
}

References a.

Referenced by Eigen::internal::scalar_carg_op< Scalar >::packetOp().

pcast() [1/4]

template<typename SrcPacket , typename TgtPacket >

EIGEN_DEVICE_FUNC TgtPacket Eigen::internal::pcast ( const SrcPacket &  a )

inline

Returns

static_cast<TgtType>(a) (coeff-wise)

                                                             {
  return pcast_generic<SrcPacket, TgtPacket>::run(a);
}

References a, and run().

pcast() [2/4]

template<typename SrcPacket , typename TgtPacket >

EIGEN_DEVICE_FUNC TgtPacket Eigen::internal::pcast ( const SrcPacket &  a, const SrcPacket &  b  )

inline

                                                                                 {
  return pcast_generic<SrcPacket, TgtPacket>::run(a, b);
}

References a, b, and run().

pcast() [3/4]

template<typename SrcPacket , typename TgtPacket >

EIGEN_DEVICE_FUNC TgtPacket Eigen::internal::pcast ( const SrcPacket &  a, const SrcPacket &  b, const SrcPacket &  c, const SrcPacket &  d  )

inline

                                                             {
  return pcast_generic<SrcPacket, TgtPacket>::run(a, b, c, d);
}

References a, b, calibrate::c, and run().

pcast() [4/4]

template<typename SrcPacket , typename TgtPacket >

EIGEN_DEVICE_FUNC TgtPacket Eigen::internal::pcast ( const SrcPacket &  a, const SrcPacket &  b, const SrcPacket &  c, const SrcPacket &  d, const SrcPacket &  e, const SrcPacket &  f, const SrcPacket &  g, const SrcPacket &  h  )

inline

                                                             {
  return pcast_generic<SrcPacket, TgtPacket>::run(a, b, c, d, e, f, g, h);
}

References a, b, calibrate::c, e(), f(), and run().

pcast< float, Packet1cf >()

template<>

EIGEN_STRONG_INLINE Packet1cf Eigen::internal::pcast< float, Packet1cf >

(

const float &

a

)

                                                                      {
  return Packet1cf(vset_lane_f32(a, vdup_n_f32(0.f), 0));
}

References a.

pcast< Packet16b, Packet16f >()

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::pcast< Packet16b, Packet16f >

(

const Packet16b &

a

)

                                                                              {
  return _mm512_cvtepi32_ps(_mm512_and_si512(_mm512_cvtepi8_epi32(a), _mm512_set1_epi32(1)));
}

References a.

pcast< Packet16b, Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pcast< Packet16b, Packet4f >

(

const Packet16b &

a

)

                                                                            {
  const __m128 cst_one = _mm_set_ps1(1.0f);
#ifdef EIGEN_VECTORIZE_SSE4_1
  __m128i a_extended = _mm_cvtepi8_epi32(a);
  __m128i abcd = _mm_cmpeq_epi32(a_extended, _mm_setzero_si128());
#else
  __m128i abcd_efhg_ijkl_mnop = _mm_cmpeq_epi8(a, _mm_setzero_si128());
  __m128i aabb_ccdd_eeff_gghh = _mm_unpacklo_epi8(abcd_efhg_ijkl_mnop, abcd_efhg_ijkl_mnop);
  __m128i abcd = _mm_unpacklo_epi8(aabb_ccdd_eeff_gghh, aabb_ccdd_eeff_gghh);
#endif
  __m128 result = _mm_andnot_ps(_mm_castsi128_ps(abcd), cst_one);
  return result;
}

References a.

pcast< Packet16b, Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::pcast< Packet16b, Packet8f >

(

const Packet16b &

a

)

                                                                            {
  const __m256 cst_one = _mm256_set1_ps(1.0f);
#ifdef EIGEN_VECTORIZE_AVX2
  __m256i a_extended = _mm256_cvtepi8_epi32(a);
  __m256i abcd_efgh = _mm256_cmpeq_epi32(a_extended, _mm256_setzero_si256());
#else
  __m128i abcd_efhg_ijkl_mnop = _mm_cmpeq_epi8(a, _mm_setzero_si128());
  __m128i aabb_ccdd_eeff_gghh = _mm_unpacklo_epi8(abcd_efhg_ijkl_mnop, abcd_efhg_ijkl_mnop);
  __m128i aaaa_bbbb_cccc_dddd = _mm_unpacklo_epi8(aabb_ccdd_eeff_gghh, aabb_ccdd_eeff_gghh);
  __m128i eeee_ffff_gggg_hhhh = _mm_unpackhi_epi8(aabb_ccdd_eeff_gghh, aabb_ccdd_eeff_gghh);
  __m256i abcd_efgh = _mm256_setr_m128i(aaaa_bbbb_cccc_dddd, eeee_ffff_gggg_hhhh);
#endif
  __m256 result = _mm256_andnot_ps(_mm256_castsi256_ps(abcd_efgh), cst_one);
  return result;
}

References a.

pcast< Packet16bf, Packet16f >()

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::pcast< Packet16bf, Packet16f >

(

const Packet16bf &

a

)

                                                                                {
  return Bf16ToF32(a);
}

References a, and Bf16ToF32().

pcast< Packet16c, Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pcast< Packet16c, Packet2d >

(

const Packet16c &

a

)

                                                                            {
  Packet8s tmp1 = __lsx_vsllwil_h_b((__m128i)a, 0);
  Packet4i tmp2 = __lsx_vsllwil_w_h((__m128i)tmp1, 0);
  return __lsx_vffint_d_l(__lsx_vsllwil_d_w((__m128i)tmp2, 0));
}

References a.

pcast< Packet16c, Packet2l >()

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::pcast< Packet16c, Packet2l >

(

const Packet16c &

a

)

                                                                            {
  Packet8s tmp1 = __lsx_vsllwil_h_b((__m128i)a, 0);
  Packet4i tmp2 = __lsx_vsllwil_w_h((__m128i)tmp1, 0);
  return __lsx_vsllwil_d_w((__m128i)tmp2, 0);
}

References a.

pcast< Packet16c, Packet2ul >()

template<>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::pcast< Packet16c, Packet2ul >

(

const Packet16c &

a

)

                                                                              {
  Packet8s tmp1 = __lsx_vsllwil_h_b((__m128i)a, 0);
  Packet4i tmp2 = __lsx_vsllwil_w_h((__m128i)tmp1, 0);
  return (Packet2ul)__lsx_vsllwil_d_w((__m128i)tmp2, 0);
}

References a.

pcast< Packet16c, Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pcast< Packet16c, Packet4f >

(

const Packet16c &

a

)

                                                                            {
  Packet8s tmp1 = __lsx_vsllwil_h_b((__m128i)a, 0);
  Packet4i tmp2 = __lsx_vsllwil_w_h((__m128i)tmp1, 0);
  return __lsx_vffint_s_w(tmp2);
}

References a.

pcast< Packet16c, Packet4i >()

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pcast< Packet16c, Packet4i >

(

const Packet16c &

a

)

                                                                            {
  Packet8s tmp1 = __lsx_vsllwil_h_b((__m128i)a, 0);
  return __lsx_vsllwil_w_h((__m128i)tmp1, 0);
}

References a.

pcast< Packet16c, Packet4ui >()

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::pcast< Packet16c, Packet4ui >

(

const Packet16c &

a

)

                                                                              {
  Packet8s tmp1 = __lsx_vsllwil_h_b((__m128i)a, 0);
  return (Packet4ui)__lsx_vsllwil_w_h((__m128i)tmp1, 0);
}

References a.

pcast< Packet16c, Packet8s >()

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::pcast< Packet16c, Packet8s >

(

const Packet16c &

a

)

                                                                            {
  return __lsx_vsllwil_h_b((__m128i)a, 0);
}

References a.

pcast< Packet16c, Packet8us >()

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::pcast< Packet16c, Packet8us >

(

const Packet16c &

a

)

                                                                              {
  return (Packet8us)__lsx_vsllwil_h_b((__m128i)a, 0);
}

References a.

pcast< Packet16f, Packet16b >()

template<>

EIGEN_STRONG_INLINE Packet16b Eigen::internal::pcast< Packet16f, Packet16b >

(

const Packet16f &

a

)

                                                                              {
  __mmask16 mask = _mm512_cmpneq_ps_mask(a, pzero(a));
  return _mm512_maskz_cvtepi32_epi8(mask, _mm512_set1_epi32(1));
}

References a, and pzero().

pcast< Packet16f, Packet16bf >()

template<>

EIGEN_STRONG_INLINE Packet16bf Eigen::internal::pcast< Packet16f, Packet16bf >

(

const Packet16f &

a

)

                                                                                {
  return F32ToBf16(a);
}

References a, and F32ToBf16().

pcast< Packet16f, Packet16h >()

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::pcast< Packet16f, Packet16h >

(

const Packet16f &

a

)

                                                                              {
  return float2half(a);
}

References a, and float2half().

pcast< Packet16f, Packet16i >()

template<>

EIGEN_STRONG_INLINE Packet16i Eigen::internal::pcast< Packet16f, Packet16i >

(

const Packet16f &

a

)

                                                                              {
  return _mm512_cvttps_epi32(a);
}

References a.

pcast< Packet16f, Packet8d >()

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::pcast< Packet16f, Packet8d >

(

const Packet16f &

a

)

                                                                            {
  return _mm512_cvtps_pd(_mm512_castps512_ps256(a));
}

References a.

pcast< Packet16h, Packet16f >()

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::pcast< Packet16h, Packet16f >

(

const Packet16h &

a

)

                                                                              {
  return half2float(a);
}

References a, and half2float().

pcast< Packet16i, Packet16f >()

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::pcast< Packet16i, Packet16f >

(

const Packet16i &

a

)

                                                                              {
  return _mm512_cvtepi32_ps(a);
}

References a.

pcast< Packet16i, Packet8d >()

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::pcast< Packet16i, Packet8d >

(

const Packet16i &

a

)

                                                                            {
  return _mm512_cvtepi32_pd(_mm512_castsi512_si256(a));
}

References a.

pcast< Packet16uc, Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pcast< Packet16uc, Packet2d >

(

const Packet16uc &

a

)

                                                                              {
  Packet8us tmp1 = __lsx_vsllwil_hu_bu((__m128i)a, 0);
  Packet4ui tmp2 = __lsx_vsllwil_wu_hu((__m128i)tmp1, 0);
  return __lsx_vffint_d_lu(__lsx_vsllwil_du_wu((__m128i)tmp2, 0));
}

References a.

pcast< Packet16uc, Packet2l >()

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::pcast< Packet16uc, Packet2l >

(

const Packet16uc &

a

)

                                                                              {
  Packet8us tmp1 = __lsx_vsllwil_hu_bu((__m128i)a, 0);
  Packet4ui tmp2 = __lsx_vsllwil_wu_hu((__m128i)tmp1, 0);
  return (Packet2l)__lsx_vsllwil_du_wu((__m128i)tmp2, 0);
}

References a.

pcast< Packet16uc, Packet2ul >()

template<>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::pcast< Packet16uc, Packet2ul >

(

const Packet16uc &

a

)

                                                                                {
  Packet8us tmp1 = __lsx_vsllwil_hu_bu((__m128i)a, 0);
  Packet4ui tmp2 = __lsx_vsllwil_wu_hu((__m128i)tmp1, 0);
  return __lsx_vsllwil_du_wu((__m128i)tmp2, 0);
}

References a.

pcast< Packet16uc, Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pcast< Packet16uc, Packet4f >

(

const Packet16uc &

a

)

                                                                              {
  Packet8us tmp1 = __lsx_vsllwil_hu_bu((__m128i)a, 0);
  Packet4ui tmp2 = __lsx_vsllwil_wu_hu((__m128i)tmp1, 0);
  return __lsx_vffint_s_wu(tmp2);
}

References a.

pcast< Packet16uc, Packet4i >()

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pcast< Packet16uc, Packet4i >

(

const Packet16uc &

a

)

                                                                              {
  Packet8us tmp1 = __lsx_vsllwil_hu_bu((__m128i)a, 0);
  return (Packet4i)__lsx_vsllwil_wu_hu((__m128i)tmp1, 0);
}

References a.

pcast< Packet16uc, Packet4ui >()

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::pcast< Packet16uc, Packet4ui >

(

const Packet16uc &

a

)

                                                                                {
  Packet8us tmp1 = __lsx_vsllwil_hu_bu((__m128i)a, 0);
  return __lsx_vsllwil_wu_hu((__m128i)tmp1, 0);
}

References a.

pcast< Packet16uc, Packet8s >()

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::pcast< Packet16uc, Packet8s >

(

const Packet16uc &

a

)

                                                                              {
  return (Packet8s)__lsx_vsllwil_hu_bu((__m128i)a, 0);
}

References a.

pcast< Packet16uc, Packet8us >()

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::pcast< Packet16uc, Packet8us >

(

const Packet16uc &

a

)

                                                                                {
  return __lsx_vsllwil_hu_bu((__m128i)a, 0);
}

References a.

pcast< Packet2d, Packet16c >()

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::pcast< Packet2d, Packet16c >	(	const Packet2d &	a,
		const Packet2d &	b,
		const Packet2d &	c,
		const Packet2d &	d,
		const Packet2d &	e,
		const Packet2d &	f,
		const Packet2d &	g,
		const Packet2d &	h
	)

                                                                                               {
  const Packet8s abcd = pcast<Packet2d, Packet8s>(a, b, c, d);
  const Packet8s efgh = pcast<Packet2d, Packet8s>(e, f, g, h);
  return __lsx_vssrlni_b_h((__m128i)abcd, (__m128i)efgh, 0);
}

References a, b, calibrate::c, e(), f(), and pcast< Packet2d, Packet8s >().

pcast< Packet2d, Packet16uc >()

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::pcast< Packet2d, Packet16uc >	(	const Packet2d &	a,
		const Packet2d &	b,
		const Packet2d &	c,
		const Packet2d &	d,
		const Packet2d &	e,
		const Packet2d &	f,
		const Packet2d &	g,
		const Packet2d &	h
	)

                                                                                                 {
  const Packet8us abcd = pcast<Packet2d, Packet8us>(a, b, c, d);
  const Packet8us efgh = pcast<Packet2d, Packet8us>(e, f, g, h);
  return __lsx_vssrlni_bu_h((__m128i)abcd, (__m128i)efgh, 0);
}

References a, b, calibrate::c, e(), f(), and pcast< Packet2d, Packet8us >().

pcast< Packet2d, Packet2l >()

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::pcast< Packet2d, Packet2l >

(

const Packet2d &

a

)

                                                                          {
  return __lsx_vftint_l_d(a);
}

References a.

pcast< Packet2d, Packet2ul >()

template<>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::pcast< Packet2d, Packet2ul >

(

const Packet2d &

a

)

                                                                            {
  return __lsx_vftint_lu_d(a);
}

References a.

pcast< Packet2d, Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pcast< Packet2d, Packet4f >	(	const Packet2d &	a,
		const Packet2d &	b
	)

                                                                                             {
  return __lsx_vfcvt_s_d(b, a);
}

References a, and b.

pcast< Packet2d, Packet4i >()

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pcast< Packet2d, Packet4i >	(	const Packet2d &	a,
		const Packet2d &	b
	)

                                                                                             {
  return __lsx_vssrlni_w_d(__lsx_vftint_l_d(a), __lsx_vftint_l_d(b), 0);
}

References a, and b.

pcast< Packet2d, Packet4ui >()

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::pcast< Packet2d, Packet4ui >	(	const Packet2d &	a,
		const Packet2d &	b
	)

                                                                                               {
  return __lsx_vssrlni_wu_d(__lsx_vftint_lu_d(a), __lsx_vftint_lu_d(b), 0);
}

References a, and b.

pcast< Packet2d, Packet8s >()

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::pcast< Packet2d, Packet8s >	(	const Packet2d &	a,
		const Packet2d &	b,
		const Packet2d &	c,
		const Packet2d &	d
	)

                                                                          {
  Packet4i tmp1 = __lsx_vssrlni_w_d(__lsx_vftint_l_d(a), __lsx_vftint_l_d(b), 0);
  Packet4i tmp2 = __lsx_vssrlni_w_d(__lsx_vftint_l_d(c), __lsx_vftint_l_d(d), 0);
  return __lsx_vssrlni_h_w((__m128i)tmp1, (__m128i)tmp2, 0);
}

References a, b, and calibrate::c.

Referenced by pcast< Packet2d, Packet16c >().

pcast< Packet2d, Packet8us >()

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::pcast< Packet2d, Packet8us >	(	const Packet2d &	a,
		const Packet2d &	b,
		const Packet2d &	c,
		const Packet2d &	d
	)

                                                                            {
  Packet4ui tmp1 = __lsx_vssrlni_wu_d(__lsx_vftint_lu_d(a), __lsx_vftint_lu_d(b), 0);
  Packet4ui tmp2 = __lsx_vssrlni_wu_d(__lsx_vftint_lu_d(c), __lsx_vftint_lu_d(d), 0);
  return __lsx_vssrlni_hu_w((__m128i)tmp1, (__m128i)tmp2, 0);
}

References a, b, and calibrate::c.

Referenced by pcast< Packet2d, Packet16uc >().

pcast< Packet2f, Packet2cf >()

template<>

EIGEN_STRONG_INLINE Packet2cf Eigen::internal::pcast< Packet2f, Packet2cf >

(

const Packet2f &

a

)

                                                                            {
  return Packet2cf(vreinterpretq_f32_u64(vmovl_u32(vreinterpret_u32_f32(a))));
}

References a.

pcast< Packet2f, Packet2i >()

template<>

EIGEN_STRONG_INLINE Packet2i Eigen::internal::pcast< Packet2f, Packet2i >

(

const Packet2f &

a

)

                                                                          {
  return vcvt_s32_f32(a);
}

References a.

pcast< Packet2f, Packet2l >()

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::pcast< Packet2f, Packet2l >

(

const Packet2f &

a

)

                                                                          {
  return vmovl_s32(vcvt_s32_f32(a));
}

References a.

pcast< Packet2f, Packet2ui >()

template<>

EIGEN_STRONG_INLINE Packet2ui Eigen::internal::pcast< Packet2f, Packet2ui >

(

const Packet2f &

a

)

                                                                            {
  return vcvt_u32_f32(a);
}

References a.

pcast< Packet2f, Packet2ul >()

template<>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::pcast< Packet2f, Packet2ul >

(

const Packet2f &

a

)

                                                                            {
  // Discard second half of input.
  return vmovl_u32(vcvt_u32_f32(a));
}

References a.

pcast< Packet2f, Packet4s >()

template<>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::pcast< Packet2f, Packet4s >	(	const Packet2f &	a,
		const Packet2f &	b
	)

                                                                                             {
  return vmovn_s32(vcombine_s32(vcvt_s32_f32(a), vcvt_s32_f32(b)));
}

References a, and b.

Referenced by pcast< Packet2f, Packet8c >().

pcast< Packet2f, Packet4us >()

template<>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::pcast< Packet2f, Packet4us >	(	const Packet2f &	a,
		const Packet2f &	b
	)

                                                                                               {
  return vmovn_u32(vcombine_u32(vcvt_u32_f32(a), vcvt_u32_f32(b)));
}

References a, and b.

pcast< Packet2f, Packet8c >()

template<>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::pcast< Packet2f, Packet8c >	(	const Packet2f &	a,
		const Packet2f &	b,
		const Packet2f &	c,
		const Packet2f &	d
	)

                                                                          {
  const int16x4_t ab_s16 = pcast<Packet2f, Packet4s>(a, b);
  const int16x4_t cd_s16 = pcast<Packet2f, Packet4s>(c, d);
  return vmovn_s16(vcombine_s16(ab_s16, cd_s16));
}

References a, b, calibrate::c, and pcast< Packet2f, Packet4s >().

Referenced by pcast< Packet2f, Packet8uc >().

pcast< Packet2f, Packet8uc >()

template<>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::pcast< Packet2f, Packet8uc >	(	const Packet2f &	a,
		const Packet2f &	b,
		const Packet2f &	c,
		const Packet2f &	d
	)

                                                                            {
  return preinterpret<Packet8uc>(pcast<Packet2f, Packet8c>(a, b, c, d));
}

References a, b, calibrate::c, and pcast< Packet2f, Packet8c >().

pcast< Packet2i, Packet2f >()

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::pcast< Packet2i, Packet2f >

(

const Packet2i &

a

)

                                                                          {
  return vcvt_f32_s32(a);
}

References a.

pcast< Packet2i, Packet2l >()

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::pcast< Packet2i, Packet2l >

(

const Packet2i &

a

)

                                                                          {
  return vmovl_s32(a);
}

References a.

Referenced by pcast< Packet2i, Packet2ul >().

pcast< Packet2i, Packet2ul >()

template<>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::pcast< Packet2i, Packet2ul >

(

const Packet2i &

a

)

                                                                            {
  return preinterpret<Packet2ul>(pcast<Packet2i, Packet2l>(a));
}

References a, and pcast< Packet2i, Packet2l >().

pcast< Packet2i, Packet4s >()

template<>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::pcast< Packet2i, Packet4s >	(	const Packet2i &	a,
		const Packet2i &	b
	)

                                                                                             {
  return vmovn_s32(vcombine_s32(a, b));
}

References a, and b.

pcast< Packet2i, Packet4us >()

template<>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::pcast< Packet2i, Packet4us >	(	const Packet2i &	a,
		const Packet2i &	b
	)

                                                                                               {
  return vmovn_u32(vreinterpretq_u32_s32(vcombine_s32(a, b)));
}

References a, and b.

pcast< Packet2i, Packet8c >()

template<>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::pcast< Packet2i, Packet8c >	(	const Packet2i &	a,
		const Packet2i &	b,
		const Packet2i &	c,
		const Packet2i &	d
	)

                                                                          {
  const int16x4_t ab_s16 = vmovn_s32(vcombine_s32(a, b));
  const int16x4_t cd_s16 = vmovn_s32(vcombine_s32(c, d));
  return vmovn_s16(vcombine_s16(ab_s16, cd_s16));
}

References a, b, and calibrate::c.

Referenced by pcast< Packet2i, Packet8uc >().

pcast< Packet2i, Packet8uc >()

template<>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::pcast< Packet2i, Packet8uc >	(	const Packet2i &	a,
		const Packet2i &	b,
		const Packet2i &	c,
		const Packet2i &	d
	)

                                                                            {
  return preinterpret<Packet8uc>(pcast<Packet2i, Packet8c>(a, b, c, d));
}

References a, b, calibrate::c, and pcast< Packet2i, Packet8c >().

pcast< Packet2l, Packet16c >()

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::pcast< Packet2l, Packet16c >	(	const Packet2l &	a,
		const Packet2l &	b,
		const Packet2l &	c,
		const Packet2l &	d,
		const Packet2l &	e,
		const Packet2l &	f,
		const Packet2l &	g,
		const Packet2l &	h
	)

                                                                                               {
  const Packet8s abcd = pcast<Packet2l, Packet8s>(a, b, c, d);
  const Packet8s efgh = pcast<Packet2l, Packet8s>(e, f, g, h);
  return __lsx_vssrlni_b_h((__m128i)abcd, (__m128i)efgh, 0);
}

References a, b, calibrate::c, e(), f(), and pcast< Packet2l, Packet8s >().

pcast< Packet2l, Packet16uc >()

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::pcast< Packet2l, Packet16uc >	(	const Packet2l &	a,
		const Packet2l &	b,
		const Packet2l &	c,
		const Packet2l &	d,
		const Packet2l &	e,
		const Packet2l &	f,
		const Packet2l &	g,
		const Packet2l &	h
	)

                                                                                                 {
  const Packet8us abcd = pcast<Packet2l, Packet8us>(a, b, c, d);
  const Packet8us efgh = pcast<Packet2l, Packet8us>(e, f, g, h);
  return __lsx_vssrlni_bu_h((__m128i)abcd, (__m128i)efgh, 0);
}

References a, b, calibrate::c, e(), f(), and pcast< Packet2l, Packet8us >().

pcast< Packet2l, Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pcast< Packet2l, Packet2d >

(

const Packet2l &

a

)

                                                                          {
  return __lsx_vffint_d_l(a);
}

References a.

pcast< Packet2l, Packet2f >()

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::pcast< Packet2l, Packet2f >

(

const Packet2l &

a

)

                                                                          {
#if EIGEN_ARCH_ARM64
  return vcvt_f32_f64(vcvtq_f64_s64(a));
#else
  EIGEN_ALIGN_MAX int64_t lvals[2];
  pstore(lvals, a);
  EIGEN_ALIGN_MAX float fvals[2] = {static_cast<float>(lvals[0]), static_cast<float>(lvals[1])};
  return pload<Packet2f>(fvals);
#endif
}

References a, EIGEN_ALIGN_MAX, pload< Packet2f >(), and pstore().

pcast< Packet2l, Packet2i >()

template<>

EIGEN_STRONG_INLINE Packet2i Eigen::internal::pcast< Packet2l, Packet2i >

(

const Packet2l &

a

)

                                                                          {
  return vmovn_s64(a);
}

References a.

pcast< Packet2l, Packet2ui >()

template<>

EIGEN_STRONG_INLINE Packet2ui Eigen::internal::pcast< Packet2l, Packet2ui >

(

const Packet2l &

a

)

                                                                            {
  return vmovn_u64(vreinterpretq_u64_s64(a));
}

References a.

pcast< Packet2l, Packet4c >()

template<>

EIGEN_STRONG_INLINE Packet4c Eigen::internal::pcast< Packet2l, Packet4c >	(	const Packet2l &	a,
		const Packet2l &	b
	)

                                                                                             {
  const int16x4_t ab_s16 = pcast<Packet2l, Packet4s>(a, b);
  const int16x8_t abab_s16 = vcombine_s16(ab_s16, ab_s16);
  const int8x8_t abab_s8 = vmovn_s16(abab_s16);
  return vget_lane_s32(vreinterpret_s32_s8(abab_s8), 0);
}

References a, b, and pcast< Packet2l, Packet4s >().

Referenced by pcast< Packet2l, Packet4uc >().

pcast< Packet2l, Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pcast< Packet2l, Packet4f >	(	const Packet2l &	a,
		const Packet2l &	b
	)

                                                                                             {
  return __lsx_vffint_s_w(__lsx_vssrlni_w_d((__m128i)a, (__m128i)b, 0));
}

References a, and b.

pcast< Packet2l, Packet4i >()

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pcast< Packet2l, Packet4i >	(	const Packet2l &	a,
		const Packet2l &	b
	)

                                                                                             {
  return __lsx_vssrlni_w_d((__m128i)a, (__m128i)b, 0);
}

References a, and b.

Referenced by pcast< Packet2l, Packet4s >().

pcast< Packet2l, Packet4s >()

template<>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::pcast< Packet2l, Packet4s >	(	const Packet2l &	a,
		const Packet2l &	b
	)

                                                                                             {
  const int32x4_t ab_s32 = pcast<Packet2l, Packet4i>(a, b);
  return vmovn_s32(ab_s32);
}

References a, b, and pcast< Packet2l, Packet4i >().

Referenced by pcast< Packet2l, Packet4c >(), and pcast< Packet2l, Packet4us >().

pcast< Packet2l, Packet4uc >()

template<>

EIGEN_STRONG_INLINE Packet4uc Eigen::internal::pcast< Packet2l, Packet4uc >	(	const Packet2l &	a,
		const Packet2l &	b
	)

                                                                                               {
  return static_cast<Packet4uc>(pcast<Packet2l, Packet4c>(a, b));
}

References a, b, and pcast< Packet2l, Packet4c >().

pcast< Packet2l, Packet4ui >()

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::pcast< Packet2l, Packet4ui >	(	const Packet2l &	a,
		const Packet2l &	b
	)

                                                                                               {
  return (Packet4ui)__lsx_vssrlni_w_d((__m128i)a, (__m128i)b, 0);
}

References a, and b.

pcast< Packet2l, Packet4us >()

template<>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::pcast< Packet2l, Packet4us >	(	const Packet2l &	a,
		const Packet2l &	b
	)

                                                                                               {
  return preinterpret<Packet4us>(pcast<Packet2l, Packet4s>(a, b));
}

References a, b, and pcast< Packet2l, Packet4s >().

pcast< Packet2l, Packet8c >()

template<>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::pcast< Packet2l, Packet8c >	(	const Packet2l &	a,
		const Packet2l &	b,
		const Packet2l &	c,
		const Packet2l &	d
	)

                                                                          {
  const int16x8_t abcd_s16 = pcast<Packet2l, Packet8s>(a, b, c, d);
  return vmovn_s16(abcd_s16);
}

References a, b, calibrate::c, and pcast< Packet2l, Packet8s >().

Referenced by pcast< Packet2l, Packet8uc >().

pcast< Packet2l, Packet8s >()

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::pcast< Packet2l, Packet8s >	(	const Packet2l &	a,
		const Packet2l &	b,
		const Packet2l &	c,
		const Packet2l &	d
	)

                                                                          {
  Packet4i tmp1 = __lsx_vssrlni_w_d((__m128i)a, (__m128i)b, 0);
  Packet4i tmp2 = __lsx_vssrlni_w_d((__m128i)c, (__m128i)d, 0);
  return __lsx_vssrlni_h_w((__m128i)tmp1, (__m128i)tmp2, 0);
}

References a, b, and calibrate::c.

Referenced by pcast< Packet2l, Packet16c >(), and pcast< Packet2l, Packet8c >().

pcast< Packet2l, Packet8uc >()

template<>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::pcast< Packet2l, Packet8uc >	(	const Packet2l &	a,
		const Packet2l &	b,
		const Packet2l &	c,
		const Packet2l &	d
	)

                                                                            {
  return preinterpret<Packet8uc>(pcast<Packet2l, Packet8c>(a, b, c, d));
}

References a, b, calibrate::c, and pcast< Packet2l, Packet8c >().

pcast< Packet2l, Packet8us >()

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::pcast< Packet2l, Packet8us >	(	const Packet2l &	a,
		const Packet2l &	b,
		const Packet2l &	c,
		const Packet2l &	d
	)

                                                                            {
  Packet4i tmp1 = __lsx_vssrlni_w_d((__m128i)a, (__m128i)b, 0);
  Packet4i tmp2 = __lsx_vssrlni_w_d((__m128i)c, (__m128i)d, 0);
  return (Packet8us)__lsx_vssrlni_h_w((__m128i)tmp1, (__m128i)tmp2, 0);
}

References a, b, and calibrate::c.

Referenced by pcast< Packet2l, Packet16uc >().

pcast< Packet2ui, Packet2f >()

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::pcast< Packet2ui, Packet2f >

(

const Packet2ui &

a

)

                                                                            {
  return vcvt_f32_u32(a);
}

References a.

pcast< Packet2ui, Packet2l >()

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::pcast< Packet2ui, Packet2l >

(

const Packet2ui &

a

)

                                                                            {
  return preinterpret<Packet2l>(pcast<Packet2ui, Packet2ul>(a));
}

References a, and pcast< Packet2ui, Packet2ul >().

pcast< Packet2ui, Packet2ul >()

template<>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::pcast< Packet2ui, Packet2ul >

(

const Packet2ui &

a

)

                                                                              {
  return vmovl_u32(a);
}

References a.

Referenced by pcast< Packet2ui, Packet2l >().

pcast< Packet2ui, Packet4s >()

template<>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::pcast< Packet2ui, Packet4s >	(	const Packet2ui &	a,
		const Packet2ui &	b
	)

                                                                                                {
  return preinterpret<Packet4s>(pcast<Packet2ui, Packet4us>(a, b));
}

References a, b, and pcast< Packet2ui, Packet4us >().

pcast< Packet2ui, Packet4us >()

template<>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::pcast< Packet2ui, Packet4us >	(	const Packet2ui &	a,
		const Packet2ui &	b
	)

                                                                                                  {
  return vmovn_u32(vcombine_u32(a, b));
}

References a, and b.

Referenced by pcast< Packet2ui, Packet4s >().

pcast< Packet2ui, Packet8c >()

template<>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::pcast< Packet2ui, Packet8c >	(	const Packet2ui &	a,
		const Packet2ui &	b,
		const Packet2ui &	c,
		const Packet2ui &	d
	)

                                                                            {
  return preinterpret<Packet8c>(pcast<Packet2ui, Packet8uc>(a, b, c, d));
}

References a, b, calibrate::c, and pcast< Packet2ui, Packet8uc >().

pcast< Packet2ui, Packet8uc >()

template<>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::pcast< Packet2ui, Packet8uc >	(	const Packet2ui &	a,
		const Packet2ui &	b,
		const Packet2ui &	c,
		const Packet2ui &	d
	)

                                                                              {
  const uint16x4_t ab_u16 = vmovn_u32(vcombine_u32(a, b));
  const uint16x4_t cd_u16 = vmovn_u32(vcombine_u32(c, d));
  return vmovn_u16(vcombine_u16(ab_u16, cd_u16));
}

References a, b, and calibrate::c.

Referenced by pcast< Packet2ui, Packet8c >().

pcast< Packet2ul, Packet16c >()

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::pcast< Packet2ul, Packet16c >	(	const Packet2ul &	a,
		const Packet2ul &	b,
		const Packet2ul &	c,
		const Packet2ul &	d,
		const Packet2ul &	e,
		const Packet2ul &	f,
		const Packet2ul &	g,
		const Packet2ul &	h
	)

                                                                                                  {
  const Packet8us abcd = pcast<Packet2ul, Packet8us>(a, b, c, d);
  const Packet8us efgh = pcast<Packet2ul, Packet8us>(e, f, g, h);
  return __lsx_vssrlni_bu_h((__m128i)abcd, (__m128i)efgh, 0);
}

References a, b, calibrate::c, e(), f(), and pcast< Packet2ul, Packet8us >().

pcast< Packet2ul, Packet16uc >()

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::pcast< Packet2ul, Packet16uc >	(	const Packet2ul &	a,
		const Packet2ul &	b,
		const Packet2ul &	c,
		const Packet2ul &	d,
		const Packet2ul &	e,
		const Packet2ul &	f,
		const Packet2ul &	g,
		const Packet2ul &	h
	)

                                                                                                    {
  const Packet8s abcd = pcast<Packet2ul, Packet8s>(a, b, c, d);
  const Packet8s efgh = pcast<Packet2ul, Packet8s>(e, f, g, h);
  return __lsx_vssrlni_b_h((__m128i)abcd, (__m128i)efgh, 0);
}

References a, b, calibrate::c, e(), f(), and pcast< Packet2ul, Packet8s >().

pcast< Packet2ul, Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pcast< Packet2ul, Packet2d >

(

const Packet2ul &

a

)

                                                                            {
  return __lsx_vffint_d_lu(a);
}

References a.

pcast< Packet2ul, Packet2f >()

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::pcast< Packet2ul, Packet2f >

(

const Packet2ul &

a

)

                                                                            {
#if EIGEN_ARCH_ARM64
  return vcvt_f32_f64(vcvtq_f64_u64(a));
#else
  EIGEN_ALIGN_MAX uint64_t uvals[2];
  pstore(uvals, a);
  EIGEN_ALIGN_MAX float fvals[2] = {static_cast<float>(uvals[0]), static_cast<float>(uvals[1])};
  return pload<Packet2f>(fvals);
#endif
}

References a, EIGEN_ALIGN_MAX, pload< Packet2f >(), and pstore().

pcast< Packet2ul, Packet2i >()

template<>

EIGEN_STRONG_INLINE Packet2i Eigen::internal::pcast< Packet2ul, Packet2i >

(

const Packet2ul &

a

)

                                                                            {
  return preinterpret<Packet2i>(pcast<Packet2ul, Packet2ui>(a));
}

References a, and pcast< Packet2ul, Packet2ui >().

pcast< Packet2ul, Packet2ui >()

template<>

EIGEN_STRONG_INLINE Packet2ui Eigen::internal::pcast< Packet2ul, Packet2ui >

(

const Packet2ul &

a

)

                                                                              {
  return vmovn_u64(a);
}

References a.

Referenced by pcast< Packet2ul, Packet2i >().

pcast< Packet2ul, Packet4c >()

template<>

EIGEN_STRONG_INLINE Packet4c Eigen::internal::pcast< Packet2ul, Packet4c >	(	const Packet2ul &	a,
		const Packet2ul &	b
	)

                                                                                                {
  return static_cast<Packet4c>(pcast<Packet2ul, Packet4uc>(a, b));
}

References a, b, and pcast< Packet2ul, Packet4uc >().

pcast< Packet2ul, Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pcast< Packet2ul, Packet4f >	(	const Packet2ul &	a,
		const Packet2ul &	b
	)

                                                                                                {
  return __lsx_vffint_s_wu(__lsx_vssrlni_w_d((__m128i)a, (__m128i)b, 0));
}

References a, and b.

pcast< Packet2ul, Packet4i >()

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pcast< Packet2ul, Packet4i >	(	const Packet2ul &	a,
		const Packet2ul &	b
	)

                                                                                                {
  return (Packet4i)__lsx_vssrlni_wu_d((__m128i)a, (__m128i)b, 0);
}

References a, and b.

pcast< Packet2ul, Packet4s >()

template<>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::pcast< Packet2ul, Packet4s >	(	const Packet2ul &	a,
		const Packet2ul &	b
	)

                                                                                                {
  return preinterpret<Packet4s>(pcast<Packet2ul, Packet4us>(a, b));
}

References a, b, and pcast< Packet2ul, Packet4us >().

pcast< Packet2ul, Packet4uc >()

template<>

EIGEN_STRONG_INLINE Packet4uc Eigen::internal::pcast< Packet2ul, Packet4uc >	(	const Packet2ul &	a,
		const Packet2ul &	b
	)

                                                                                                  {
  const uint16x4_t ab_u16 = pcast<Packet2ul, Packet4us>(a, b);
  const uint16x8_t abab_u16 = vcombine_u16(ab_u16, ab_u16);
  const uint8x8_t abab_u8 = vmovn_u16(abab_u16);
  return vget_lane_u32(vreinterpret_u32_u8(abab_u8), 0);
}

References a, b, and pcast< Packet2ul, Packet4us >().

Referenced by pcast< Packet2ul, Packet4c >().

pcast< Packet2ul, Packet4ui >()

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::pcast< Packet2ul, Packet4ui >	(	const Packet2ul &	a,
		const Packet2ul &	b
	)

                                                                                                  {
  return __lsx_vssrlni_wu_d((__m128i)a, (__m128i)b, 0);
}

References a, and b.

pcast< Packet2ul, Packet4us >()

template<>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::pcast< Packet2ul, Packet4us >	(	const Packet2ul &	a,
		const Packet2ul &	b
	)

                                                                                                  {
  return vmovn_u32(vcombine_u32(vmovn_u64(a), vmovn_u64(b)));
}

References a, and b.

Referenced by pcast< Packet2ul, Packet4s >(), and pcast< Packet2ul, Packet4uc >().

pcast< Packet2ul, Packet8c >()

template<>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::pcast< Packet2ul, Packet8c >	(	const Packet2ul &	a,
		const Packet2ul &	b,
		const Packet2ul &	c,
		const Packet2ul &	d
	)

                                                                            {
  return preinterpret<Packet8c>(pcast<Packet2ul, Packet8uc>(a, b, c, d));
}

References a, b, calibrate::c, and pcast< Packet2ul, Packet8uc >().

pcast< Packet2ul, Packet8s >()

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::pcast< Packet2ul, Packet8s >	(	const Packet2ul &	a,
		const Packet2ul &	b,
		const Packet2ul &	c,
		const Packet2ul &	d
	)

                                                                            {
  Packet4ui tmp1 = __lsx_vssrlni_wu_d((__m128i)a, (__m128i)b, 0);
  Packet4ui tmp2 = __lsx_vssrlni_wu_d((__m128i)c, (__m128i)d, 0);
  return (Packet8s)__lsx_vssrlni_hu_w((__m128i)tmp1, (__m128i)tmp2, 0);
}

References a, b, and calibrate::c.

Referenced by pcast< Packet2ul, Packet16uc >().

pcast< Packet2ul, Packet8uc >()

template<>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::pcast< Packet2ul, Packet8uc >	(	const Packet2ul &	a,
		const Packet2ul &	b,
		const Packet2ul &	c,
		const Packet2ul &	d
	)

                                                                              {
  const uint16x8_t abcd_u16 = pcast<Packet2ul, Packet8us>(a, b, c, d);
  return vmovn_u16(abcd_u16);
}

References a, b, calibrate::c, and pcast< Packet2ul, Packet8us >().

Referenced by pcast< Packet2ul, Packet8c >().

pcast< Packet2ul, Packet8us >()

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::pcast< Packet2ul, Packet8us >	(	const Packet2ul &	a,
		const Packet2ul &	b,
		const Packet2ul &	c,
		const Packet2ul &	d
	)

                                                                              {
  Packet4ui tmp1 = __lsx_vssrlni_wu_d((__m128i)a, (__m128i)b, 0);
  Packet4ui tmp2 = __lsx_vssrlni_wu_d((__m128i)c, (__m128i)d, 0);
  return __lsx_vssrlni_hu_w((__m128i)tmp1, (__m128i)tmp2, 0);
}

References a, b, and calibrate::c.

Referenced by pcast< Packet2ul, Packet16c >(), and pcast< Packet2ul, Packet8uc >().

pcast< Packet4c, Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pcast< Packet4c, Packet4f >

(

const Packet4c &

a

)

                                                                          {
  return vcvtq_f32_s32(vmovl_s16(vget_low_s16(vmovl_s8(vreinterpret_s8_s32(vdup_n_s32(a))))));
}

References a.

pcast< Packet4c, Packet4i >()

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pcast< Packet4c, Packet4i >

(

const Packet4c &

a

)

                                                                          {
  return pcast<Packet8c, Packet4i>(vreinterpret_s8_s32(vdup_n_s32(a)));
}

References a, and pcast< Packet8c, Packet4i >().

Referenced by pcast< Packet4c, Packet4ui >().

pcast< Packet4c, Packet4s >()

template<>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::pcast< Packet4c, Packet4s >

(

const Packet4c &

a

)

                                                                          {
  return pcast<Packet8c, Packet4s>(vreinterpret_s8_s32(vdup_n_s32(a)));
}

References a, and pcast< Packet8c, Packet4s >().

Referenced by pcast< Packet4c, Packet4us >().

pcast< Packet4c, Packet4ui >()

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::pcast< Packet4c, Packet4ui >

(

const Packet4c &

a

)

                                                                            {
  return preinterpret<Packet4ui>(pcast<Packet4c, Packet4i>(a));
}

References a, and pcast< Packet4c, Packet4i >().

pcast< Packet4c, Packet4us >()

template<>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::pcast< Packet4c, Packet4us >

(

const Packet4c &

a

)

                                                                            {
  return preinterpret<Packet4us>(pcast<Packet4c, Packet4s>(a));
}

References a, and pcast< Packet4c, Packet4s >().

pcast< Packet4d, Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pcast< Packet4d, Packet4f >

(

const Packet4d &

a

)

                                                                          {
  return _mm256_cvtpd_ps(a);
}

References a.

pcast< Packet4d, Packet4i >()

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pcast< Packet4d, Packet4i >

(

const Packet4d &

a

)

                                                                          {
  return _mm256_cvttpd_epi32(a);
}

References a.

pcast< Packet4d, Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::pcast< Packet4d, Packet8f >	(	const Packet4d &	a,
		const Packet4d &	b
	)

                                                                                             {
  return _mm256_set_m128(_mm256_cvtpd_ps(b), _mm256_cvtpd_ps(a));
}

References a, and b.

pcast< Packet4d, Packet8i >()

template<>

EIGEN_STRONG_INLINE Packet8i Eigen::internal::pcast< Packet4d, Packet8i >	(	const Packet4d &	a,
		const Packet4d &	b
	)

                                                                                             {
  return _mm256_set_m128i(_mm256_cvttpd_epi32(b), _mm256_cvttpd_epi32(a));
}

References a, and b.

pcast< Packet4f, Packet16b >()

template<>

EIGEN_STRONG_INLINE Packet16b Eigen::internal::pcast< Packet4f, Packet16b >	(	const Packet4f &	a,
		const Packet4f &	b,
		const Packet4f &	c,
		const Packet4f &	d
	)

                                                                            {
  __m128 zero = pzero(a);
  __m128 nonzero_a = _mm_cmpneq_ps(a, zero);
  __m128 nonzero_b = _mm_cmpneq_ps(b, zero);
  __m128 nonzero_c = _mm_cmpneq_ps(c, zero);
  __m128 nonzero_d = _mm_cmpneq_ps(d, zero);
  __m128i ab_bytes = _mm_packs_epi32(_mm_castps_si128(nonzero_a), _mm_castps_si128(nonzero_b));
  __m128i cd_bytes = _mm_packs_epi32(_mm_castps_si128(nonzero_c), _mm_castps_si128(nonzero_d));
  __m128i merged = _mm_packs_epi16(ab_bytes, cd_bytes);
  return _mm_and_si128(merged, _mm_set1_epi8(1));
}

References a, b, calibrate::c, pzero(), and zero().

pcast< Packet4f, Packet16c >()

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::pcast< Packet4f, Packet16c >	(	const Packet4f &	a,
		const Packet4f &	b,
		const Packet4f &	c,
		const Packet4f &	d
	)

                                                                            {
  Packet8s tmp1 = __lsx_vssrlni_h_w(__lsx_vftint_w_s(a), __lsx_vftint_w_s(b), 0);
  Packet8s tmp2 = __lsx_vssrlni_h_w(__lsx_vftint_w_s(c), __lsx_vftint_w_s(d), 0);
  return __lsx_vssrlni_b_h((__m128i)tmp1, (__m128i)tmp2, 0);
}

References a, b, and calibrate::c.

pcast< Packet4f, Packet16uc >()

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::pcast< Packet4f, Packet16uc >	(	const Packet4f &	a,
		const Packet4f &	b,
		const Packet4f &	c,
		const Packet4f &	d
	)

                                                                              {
  Packet8us tmp1 = __lsx_vssrlni_hu_w(__lsx_vftint_wu_s(a), __lsx_vftint_wu_s(b), 0);
  Packet8us tmp2 = __lsx_vssrlni_hu_w(__lsx_vftint_wu_s(c), __lsx_vftint_wu_s(d), 0);
  return __lsx_vssrlni_bu_h((__m128i)tmp1, (__m128i)tmp2, 0);
}

References a, b, and calibrate::c.

pcast< Packet4f, Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pcast< Packet4f, Packet2d >

(

const Packet4f &

a

)

                                                                          {
  return __lsx_vfcvtl_d_s(a);
}

References a.

pcast< Packet4f, Packet2l >()

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::pcast< Packet4f, Packet2l >

(

const Packet4f &

a

)

                                                                          {
  Packet2d tmp = __lsx_vfcvtl_d_s(a);
  return __lsx_vftint_l_d(tmp);
}

References a, and tmp.

pcast< Packet4f, Packet2ul >()

template<>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::pcast< Packet4f, Packet2ul >

(

const Packet4f &

a

)

                                                                            {
  Packet2d tmp = __lsx_vfcvtl_d_s(a);
  return __lsx_vftint_lu_d(tmp);
}

References a, and tmp.

pcast< Packet4f, Packet4c >()

template<>

EIGEN_STRONG_INLINE Packet4c Eigen::internal::pcast< Packet4f, Packet4c >

(

const Packet4f &

a

)

                                                                          {
  const int32x4_t a_s32x4 = vcvtq_s32_f32(a);
  const int16x4_t a_s16x4 = vmovn_s32(a_s32x4);
  const int16x8_t aa_s16x8 = vcombine_s16(a_s16x4, a_s16x4);
  const int8x8_t aa_s8x8 = vmovn_s16(aa_s16x8);
  return vget_lane_s32(vreinterpret_s32_s8(aa_s8x8), 0);
}

References a.

Referenced by pcast< Packet4f, Packet4uc >().

pcast< Packet4f, Packet4d >()

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::pcast< Packet4f, Packet4d >

(

const Packet4f &

a

)

                                                                          {
  return _mm256_cvtps_pd(a);
}

References a.

pcast< Packet4f, Packet4i >()

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pcast< Packet4f, Packet4i >

(

const Packet4f &

a

)

                                                                          {
  return vec_cts(a, 0);
}

References a.

pcast< Packet4f, Packet4s >()

template<>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::pcast< Packet4f, Packet4s >

(

const Packet4f &

a

)

                                                                          {
  return vmovn_s32(vcvtq_s32_f32(a));
}

References a.

pcast< Packet4f, Packet4uc >()

template<>

EIGEN_STRONG_INLINE Packet4uc Eigen::internal::pcast< Packet4f, Packet4uc >

(

const Packet4f &

a

)

                                                                            {
  return static_cast<Packet4uc>(pcast<Packet4f, Packet4c>(a));
}

References a, and pcast< Packet4f, Packet4c >().

pcast< Packet4f, Packet4ui >()

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::pcast< Packet4f, Packet4ui >

(

const Packet4f &

a

)

                                                                            {
  return vec_ctu(a, 0);
}

References a.

Referenced by pcast< Packet8bf, Packet8us >().

pcast< Packet4f, Packet4us >()

template<>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::pcast< Packet4f, Packet4us >

(

const Packet4f &

a

)

                                                                            {
  return vmovn_u32(vcvtq_u32_f32(a));
}

References a.

pcast< Packet4f, Packet8bf >()

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::pcast< Packet4f, Packet8bf >	(	const Packet4f &	a,
		const Packet4f &	b
	)

                                                                                               {
  return F32ToBf16Both(a, b);
}

References a, b, and F32ToBf16Both().

pcast< Packet4f, Packet8c >()

template<>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::pcast< Packet4f, Packet8c >	(	const Packet4f &	a,
		const Packet4f &	b
	)

                                                                                             {
  const int16x8_t ab_s16 = pcast<Packet4f, Packet8s>(a, b);
  return vmovn_s16(ab_s16);
}

References a, b, and pcast< Packet4f, Packet8s >().

Referenced by pcast< Packet4f, Packet8uc >().

pcast< Packet4f, Packet8s >()

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::pcast< Packet4f, Packet8s >	(	const Packet4f &	a,
		const Packet4f &	b
	)

                                                                                             {
  return __lsx_vssrlni_h_w(__lsx_vftint_w_s(a), __lsx_vftint_w_s(b), 0);
}

References a, and b.

Referenced by pcast< Packet4f, Packet8c >().

pcast< Packet4f, Packet8uc >()

template<>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::pcast< Packet4f, Packet8uc >	(	const Packet4f &	a,
		const Packet4f &	b
	)

                                                                                               {
  return preinterpret<Packet8uc>(pcast<Packet4f, Packet8c>(a, b));
}

References a, b, and pcast< Packet4f, Packet8c >().

pcast< Packet4f, Packet8us >()

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::pcast< Packet4f, Packet8us >	(	const Packet4f &	a,
		const Packet4f &	b
	)

                                                                                               {
  return __lsx_vssrlni_hu_w(__lsx_vftint_wu_s(a), __lsx_vftint_wu_s(b), 0);
}

References a, and b.

pcast< Packet4i, Packet16c >()

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::pcast< Packet4i, Packet16c >	(	const Packet4i &	a,
		const Packet4i &	b,
		const Packet4i &	c,
		const Packet4i &	d
	)

                                                                            {
  Packet8s tmp1 = __lsx_vssrlni_h_w((__m128i)a, (__m128i)b, 0);
  Packet8s tmp2 = __lsx_vssrlni_h_w((__m128i)c, (__m128i)d, 0);
  return __lsx_vssrlni_b_h((__m128i)tmp1, (__m128i)tmp2, 0);
}

References a, b, and calibrate::c.

pcast< Packet4i, Packet16uc >()

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::pcast< Packet4i, Packet16uc >	(	const Packet4i &	a,
		const Packet4i &	b,
		const Packet4i &	c,
		const Packet4i &	d
	)

                                                                              {
  Packet8s tmp1 = __lsx_vssrlni_h_w((__m128i)a, (__m128i)b, 0);
  Packet8s tmp2 = __lsx_vssrlni_h_w((__m128i)c, (__m128i)d, 0);
  return (Packet16uc)__lsx_vssrlni_b_h((__m128i)tmp1, (__m128i)tmp2, 0);
}

References a, b, and calibrate::c.

pcast< Packet4i, Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pcast< Packet4i, Packet2d >

(

const Packet4i &

a

)

                                                                          {
  return __lsx_vffint_d_l(__lsx_vsllwil_d_w((__m128i)a, 0));
}

References a.

pcast< Packet4i, Packet2l >()

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::pcast< Packet4i, Packet2l >

(

const Packet4i &

a

)

                                                                          {
  return __lsx_vsllwil_d_w((__m128i)a, 0);
}

References a.

pcast< Packet4i, Packet2ul >()

template<>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::pcast< Packet4i, Packet2ul >

(

const Packet4i &

a

)

                                                                            {
  return (Packet2ul)__lsx_vsllwil_d_w((__m128i)a, 0);
}

References a.

pcast< Packet4i, Packet4c >()

template<>

EIGEN_STRONG_INLINE Packet4c Eigen::internal::pcast< Packet4i, Packet4c >

(

const Packet4i &

a

)

                                                                          {
  const int16x4_t a_s16x4 = vmovn_s32(a);
  const int16x8_t aa_s16x8 = vcombine_s16(a_s16x4, a_s16x4);
  const int8x8_t aa_s8x8 = vmovn_s16(aa_s16x8);
  return vget_lane_s32(vreinterpret_s32_s8(aa_s8x8), 0);
}

References a.

Referenced by pcast< Packet4i, Packet4uc >().

pcast< Packet4i, Packet4d >()

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::pcast< Packet4i, Packet4d >

(

const Packet4i &

a

)

                                                                          {
  return _mm256_cvtepi32_pd(a);
}

References a.

pcast< Packet4i, Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pcast< Packet4i, Packet4f >

(

const Packet4i &

a

)

                                                                          {
  return vec_ctf(a, 0);
}

References a.

pcast< Packet4i, Packet4s >()

template<>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::pcast< Packet4i, Packet4s >

(

const Packet4i &

a

)

                                                                          {
  return vmovn_s32(a);
}

References a.

pcast< Packet4i, Packet4uc >()

template<>

EIGEN_STRONG_INLINE Packet4uc Eigen::internal::pcast< Packet4i, Packet4uc >

(

const Packet4i &

a

)

                                                                            {
  return static_cast<Packet4uc>(pcast<Packet4i, Packet4c>(a));
}

References a, and pcast< Packet4i, Packet4c >().

pcast< Packet4i, Packet4us >()

template<>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::pcast< Packet4i, Packet4us >

(

const Packet4i &

a

)

                                                                            {
  return vmovn_u32(vreinterpretq_u32_s32(a));
}

References a.

pcast< Packet4i, Packet8c >()

template<>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::pcast< Packet4i, Packet8c >	(	const Packet4i &	a,
		const Packet4i &	b
	)

                                                                                             {
  const int16x8_t ab_s16 = pcast<Packet4i, Packet8s>(a, b);
  return vmovn_s16(ab_s16);
}

References a, b, and pcast< Packet4i, Packet8s >().

Referenced by pcast< Packet4i, Packet8uc >().

pcast< Packet4i, Packet8s >()

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::pcast< Packet4i, Packet8s >	(	const Packet4i &	a,
		const Packet4i &	b
	)

                                                                                             {
  return __lsx_vssrlni_h_w((__m128i)a, (__m128i)b, 0);
}

References a, and b.

Referenced by pcast< Packet4i, Packet8c >().

pcast< Packet4i, Packet8uc >()

template<>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::pcast< Packet4i, Packet8uc >	(	const Packet4i &	a,
		const Packet4i &	b
	)

                                                                                               {
  return preinterpret<Packet8uc>(pcast<Packet4i, Packet8c>(a, b));
}

References a, b, and pcast< Packet4i, Packet8c >().

pcast< Packet4i, Packet8us >()

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::pcast< Packet4i, Packet8us >	(	const Packet4i &	a,
		const Packet4i &	b
	)

                                                                                               {
  return (Packet8us)__lsx_vssrlni_h_w((__m128i)a, (__m128i)b, 0);
}

References a, and b.

pcast< Packet4s, Packet2f >()

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::pcast< Packet4s, Packet2f >

(

const Packet4s &

a

)

                                                                          {
  // Discard second half of input.
  return vcvt_f32_s32(vget_low_s32(vmovl_s16(a)));
}

References a.

pcast< Packet4s, Packet2i >()

template<>

EIGEN_STRONG_INLINE Packet2i Eigen::internal::pcast< Packet4s, Packet2i >

(

const Packet4s &

a

)

                                                                          {
  // Discard second half of input.
  return vget_low_s32(vmovl_s16(a));
}

References a.

Referenced by pcast< Packet4s, Packet2ui >().

pcast< Packet4s, Packet2ui >()

template<>

EIGEN_STRONG_INLINE Packet2ui Eigen::internal::pcast< Packet4s, Packet2ui >

(

const Packet4s &

a

)

                                                                            {
  return preinterpret<Packet2ui>(pcast<Packet4s, Packet2i>(a));
}

References a, and pcast< Packet4s, Packet2i >().

pcast< Packet4s, Packet4c >()

template<>

EIGEN_STRONG_INLINE Packet4c Eigen::internal::pcast< Packet4s, Packet4c >

(

const Packet4s &

a

)

                                                                          {
  const int8x8_t aa_s8x8 = pcast<Packet4s, Packet8c>(a, a);
  return vget_lane_s32(vreinterpret_s32_s8(aa_s8x8), 0);
}

References a, and pcast< Packet4s, Packet8c >().

Referenced by pcast< Packet4s, Packet4uc >().

pcast< Packet4s, Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pcast< Packet4s, Packet4f >

(

const Packet4s &

a

)

                                                                          {
  return vcvtq_f32_s32(vmovl_s16(a));
}

References a.

pcast< Packet4s, Packet4i >()

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pcast< Packet4s, Packet4i >

(

const Packet4s &

a

)

                                                                          {
  return vmovl_s16(a);
}

References a.

Referenced by pcast< Packet4s, Packet4ui >().

pcast< Packet4s, Packet4uc >()

template<>

EIGEN_STRONG_INLINE Packet4uc Eigen::internal::pcast< Packet4s, Packet4uc >

(

const Packet4s &

a

)

                                                                            {
  return static_cast<Packet4uc>(pcast<Packet4s, Packet4c>(a));
}

References a, and pcast< Packet4s, Packet4c >().

pcast< Packet4s, Packet4ui >()

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::pcast< Packet4s, Packet4ui >

(

const Packet4s &

a

)

                                                                            {
  return preinterpret<Packet4ui>(pcast<Packet4s, Packet4i>(a));
}

References a, and pcast< Packet4s, Packet4i >().

pcast< Packet4s, Packet8c >()

template<>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::pcast< Packet4s, Packet8c >	(	const Packet4s &	a,
		const Packet4s &	b
	)

                                                                                             {
  return vmovn_s16(vcombine_s16(a, b));
}

References a, and b.

Referenced by pcast< Packet4s, Packet4c >(), and pcast< Packet4s, Packet8uc >().

pcast< Packet4s, Packet8uc >()

template<>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::pcast< Packet4s, Packet8uc >	(	const Packet4s &	a,
		const Packet4s &	b
	)

                                                                                               {
  return preinterpret<Packet8uc>(pcast<Packet4s, Packet8c>(a, b));
}

References a, b, and pcast< Packet4s, Packet8c >().

pcast< Packet4uc, Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pcast< Packet4uc, Packet4f >

(

const Packet4uc &

a

)

                                                                            {
  return vcvtq_f32_u32(vmovl_u16(vget_low_u16(vmovl_u8(vreinterpret_u8_u32(vdup_n_u32(a))))));
}

References a.

pcast< Packet4uc, Packet4i >()

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pcast< Packet4uc, Packet4i >

(

const Packet4uc &

a

)

                                                                            {
  return preinterpret<Packet4i>(pcast<Packet4uc, Packet4ui>(a));
}

References a, and pcast< Packet4uc, Packet4ui >().

pcast< Packet4uc, Packet4s >()

template<>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::pcast< Packet4uc, Packet4s >

(

const Packet4uc &

a

)

                                                                            {
  return preinterpret<Packet4s>(pcast<Packet4uc, Packet4us>(a));
}

References a, and pcast< Packet4uc, Packet4us >().

pcast< Packet4uc, Packet4ui >()

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::pcast< Packet4uc, Packet4ui >

(

const Packet4uc &

a

)

                                                                              {
  return pcast<Packet8uc, Packet4ui>(vreinterpret_u8_u32(vdup_n_u32(a)));
}

References a, and pcast< Packet8uc, Packet4ui >().

Referenced by pcast< Packet4uc, Packet4i >().

pcast< Packet4uc, Packet4us >()

template<>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::pcast< Packet4uc, Packet4us >

(

const Packet4uc &

a

)

                                                                              {
  return vget_low_u16(vmovl_u8(vreinterpret_u8_u32(vdup_n_u32(a))));
}

References a.

Referenced by pcast< Packet4uc, Packet4s >().

pcast< Packet4ui, Packet16c >()

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::pcast< Packet4ui, Packet16c >	(	const Packet4ui &	a,
		const Packet4ui &	b,
		const Packet4ui &	c,
		const Packet4ui &	d
	)

                                                                              {
  Packet8us tmp1 = __lsx_vssrlni_hu_w((__m128i)a, (__m128i)b, 0);
  Packet8us tmp2 = __lsx_vssrlni_hu_w((__m128i)c, (__m128i)d, 0);
  return (Packet16c)__lsx_vssrlni_bu_h((__m128i)tmp1, (__m128i)tmp2, 0);
}

References a, b, and calibrate::c.

pcast< Packet4ui, Packet16uc >()

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::pcast< Packet4ui, Packet16uc >	(	const Packet4ui &	a,
		const Packet4ui &	b,
		const Packet4ui &	c,
		const Packet4ui &	d
	)

                                                                                {
  Packet8us tmp1 = __lsx_vssrlni_hu_w((__m128i)a, (__m128i)b, 0);
  Packet8us tmp2 = __lsx_vssrlni_hu_w((__m128i)c, (__m128i)d, 0);
  return __lsx_vssrlni_bu_h((__m128i)tmp1, (__m128i)tmp2, 0);
}

References a, b, and calibrate::c.

pcast< Packet4ui, Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pcast< Packet4ui, Packet2d >

(

const Packet4ui &

a

)

                                                                            {
  return __lsx_vffint_d_lu(__lsx_vsllwil_du_wu((__m128i)a, 0));
}

References a.

pcast< Packet4ui, Packet2l >()

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::pcast< Packet4ui, Packet2l >

(

const Packet4ui &

a

)

                                                                            {
  return (Packet2l)__lsx_vsllwil_du_wu((__m128i)a, 0);
}

References a.

pcast< Packet4ui, Packet2ul >()

template<>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::pcast< Packet4ui, Packet2ul >

(

const Packet4ui &

a

)

                                                                              {
  return __lsx_vsllwil_du_wu((__m128i)a, 0);
}

References a.

pcast< Packet4ui, Packet4c >()

template<>

EIGEN_STRONG_INLINE Packet4c Eigen::internal::pcast< Packet4ui, Packet4c >

(

const Packet4ui &

a

)

                                                                            {
  return static_cast<Packet4c>(pcast<Packet4ui, Packet4uc>(a));
}

References a, and pcast< Packet4ui, Packet4uc >().

pcast< Packet4ui, Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pcast< Packet4ui, Packet4f >

(

const Packet4ui &

a

)

                                                                            {
  return vec_ctf(a, 0);
}

References a.

Referenced by pcast< Packet8us, Packet8bf >().

pcast< Packet4ui, Packet4s >()

template<>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::pcast< Packet4ui, Packet4s >

(

const Packet4ui &

a

)

                                                                            {
  return preinterpret<Packet4s>(pcast<Packet4ui, Packet4us>(a));
}

References a, and pcast< Packet4ui, Packet4us >().

pcast< Packet4ui, Packet4uc >()

template<>

EIGEN_STRONG_INLINE Packet4uc Eigen::internal::pcast< Packet4ui, Packet4uc >

(

const Packet4ui &

a

)

                                                                              {
  const uint16x4_t a_u16x4 = vmovn_u32(a);
  const uint16x8_t aa_u16x8 = vcombine_u16(a_u16x4, a_u16x4);
  const uint8x8_t aa_u8x8 = vmovn_u16(aa_u16x8);
  return vget_lane_u32(vreinterpret_u32_u8(aa_u8x8), 0);
}

References a.

Referenced by pcast< Packet4ui, Packet4c >().

pcast< Packet4ui, Packet4us >()

template<>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::pcast< Packet4ui, Packet4us >

(

const Packet4ui &

a

)

                                                                              {
  return vmovn_u32(a);
}

References a.

Referenced by pcast< Packet4ui, Packet4s >().

pcast< Packet4ui, Packet8c >()

template<>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::pcast< Packet4ui, Packet8c >	(	const Packet4ui &	a,
		const Packet4ui &	b
	)

                                                                                                {
  return preinterpret<Packet8c>(pcast<Packet4ui, Packet8uc>(a, b));
}

References a, b, and pcast< Packet4ui, Packet8uc >().

pcast< Packet4ui, Packet8s >()

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::pcast< Packet4ui, Packet8s >	(	const Packet4ui &	a,
		const Packet4ui &	b
	)

                                                                                                {
  return (Packet8s)__lsx_vssrlni_hu_w((__m128i)a, (__m128i)b, 0);
}

References a, and b.

pcast< Packet4ui, Packet8uc >()

template<>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::pcast< Packet4ui, Packet8uc >	(	const Packet4ui &	a,
		const Packet4ui &	b
	)

                                                                                                  {
  const uint16x8_t ab_u16 = vcombine_u16(vmovn_u32(a), vmovn_u32(b));
  return vmovn_u16(ab_u16);
}

References a, and b.

Referenced by pcast< Packet4ui, Packet8c >().

pcast< Packet4ui, Packet8us >()

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::pcast< Packet4ui, Packet8us >	(	const Packet4ui &	a,
		const Packet4ui &	b
	)

                                                                                                  {
  return __lsx_vssrlni_hu_w((__m128i)a, (__m128i)b, 0);
}

References a, and b.

pcast< Packet4us, Packet2f >()

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::pcast< Packet4us, Packet2f >

(

const Packet4us &

a

)

                                                                            {
  // Discard second half of input.
  return vcvt_f32_u32(vget_low_u32(vmovl_u16(a)));
}

References a.

pcast< Packet4us, Packet2i >()

template<>

EIGEN_STRONG_INLINE Packet2i Eigen::internal::pcast< Packet4us, Packet2i >

(

const Packet4us &

a

)

                                                                            {
  return preinterpret<Packet2i>(pcast<Packet4us, Packet2ui>(a));
}

References a, and pcast< Packet4us, Packet2ui >().

pcast< Packet4us, Packet2ui >()

template<>

EIGEN_STRONG_INLINE Packet2ui Eigen::internal::pcast< Packet4us, Packet2ui >

(

const Packet4us &

a

)

                                                                              {
  // Discard second half of input.
  return vget_low_u32(vmovl_u16(a));
}

References a.

Referenced by pcast< Packet4us, Packet2i >().

pcast< Packet4us, Packet4c >()

template<>

EIGEN_STRONG_INLINE Packet4c Eigen::internal::pcast< Packet4us, Packet4c >

(

const Packet4us &

a

)

                                                                            {
  return static_cast<Packet4c>(pcast<Packet4us, Packet4uc>(a));
}

References a, and pcast< Packet4us, Packet4uc >().

pcast< Packet4us, Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pcast< Packet4us, Packet4f >

(

const Packet4us &

a

)

                                                                            {
  return vcvtq_f32_u32(vmovl_u16(a));
}

References a.

pcast< Packet4us, Packet4i >()

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pcast< Packet4us, Packet4i >

(

const Packet4us &

a

)

                                                                            {
  return preinterpret<Packet4i>(pcast<Packet4us, Packet4ui>(a));
}

References a, and pcast< Packet4us, Packet4ui >().

pcast< Packet4us, Packet4uc >()

template<>

EIGEN_STRONG_INLINE Packet4uc Eigen::internal::pcast< Packet4us, Packet4uc >

(

const Packet4us &

a

)

                                                                              {
  uint8x8_t aa_u8x8 = pcast<Packet4us, Packet8uc>(a, a);
  return vget_lane_u32(vreinterpret_u32_u8(aa_u8x8), 0);
}

References a, and pcast< Packet4us, Packet8uc >().

Referenced by pcast< Packet4us, Packet4c >().

pcast< Packet4us, Packet4ui >()

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::pcast< Packet4us, Packet4ui >

(

const Packet4us &

a

)

                                                                              {
  return vmovl_u16(a);
}

References a.

Referenced by pcast< Packet4us, Packet4i >().

pcast< Packet4us, Packet8c >()

template<>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::pcast< Packet4us, Packet8c >	(	const Packet4us &	a,
		const Packet4us &	b
	)

                                                                                                {
  return preinterpret<Packet8c>(pcast<Packet4us, Packet8uc>(a, b));
}

References a, b, and pcast< Packet4us, Packet8uc >().

pcast< Packet4us, Packet8uc >()

template<>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::pcast< Packet4us, Packet8uc >	(	const Packet4us &	a,
		const Packet4us &	b
	)

                                                                                                  {
  return vmovn_u16(vcombine_u16(a, b));
}

References a, and b.

Referenced by pcast< Packet4us, Packet4uc >(), and pcast< Packet4us, Packet8c >().

pcast< Packet8bf, Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pcast< Packet8bf, Packet4f >

(

const Packet8bf &

a

)

                                                                            {
  Packet8us z = pset1<Packet8us>(0);
#ifdef _BIG_ENDIAN
  return reinterpret_cast<Packet4f>(vec_mergeh(a.m_val, z));
#else
  return reinterpret_cast<Packet4f>(vec_mergeh(z, a.m_val));
#endif
}

References a, and pset1< Packet8us >().

pcast< Packet8bf, Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::pcast< Packet8bf, Packet8f >

(

const Packet8bf &

a

)

                                                                            {
  return Bf16ToF32(a);
}

References a, and Bf16ToF32().

pcast< Packet8bf, Packet8us >()

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::pcast< Packet8bf, Packet8us >

(

const Packet8bf &

a

)

                                                                              {
  Packet4f float_even = Bf16ToF32Even(a);
  Packet4f float_odd = Bf16ToF32Odd(a);
  Packet4ui int_even = pcast<Packet4f, Packet4ui>(float_even);
  Packet4ui int_odd = pcast<Packet4f, Packet4ui>(float_odd);
  const EIGEN_DECLARE_CONST_FAST_Packet4ui(low_mask, 0x0000FFFF);
  Packet4ui low_even = pand<Packet4ui>(int_even, p4ui_low_mask);
  Packet4ui low_odd = pand<Packet4ui>(int_odd, p4ui_low_mask);
 
  // Check values that are bigger than USHRT_MAX (0xFFFF)
  Packet4bi overflow_selector;
  if (vec_any_gt(int_even, p4ui_low_mask)) {
    overflow_selector = vec_cmpgt(int_even, p4ui_low_mask);
    low_even = vec_sel(low_even, p4ui_low_mask, overflow_selector);
  }
  if (vec_any_gt(int_odd, p4ui_low_mask)) {
    overflow_selector = vec_cmpgt(int_odd, p4ui_low_mask);
    low_odd = vec_sel(low_even, p4ui_low_mask, overflow_selector);
  }
 
  return pmerge(low_even, low_odd);
}

References a, Bf16ToF32Even(), Bf16ToF32Odd(), EIGEN_DECLARE_CONST_FAST_Packet4ui(), pand< Packet4ui >(), pcast< Packet4f, Packet4ui >(), and pmerge().

pcast< Packet8c, Packet2f >()

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::pcast< Packet8c, Packet2f >

(

const Packet8c &

a

)

                                                                          {
  // Discard all but first 2 bytes.
  return vcvt_f32_s32(vget_low_s32(vmovl_s16(vget_low_s16(vmovl_s8(a)))));
}

References a.

pcast< Packet8c, Packet2i >()

template<>

EIGEN_STRONG_INLINE Packet2i Eigen::internal::pcast< Packet8c, Packet2i >

(

const Packet8c &

a

)

                                                                          {
  // Discard all but first 2 bytes.
  return vget_low_s32(vmovl_s16(vget_low_s16(vmovl_s8(a))));
}

References a.

Referenced by pcast< Packet8c, Packet2ui >().

pcast< Packet8c, Packet2ui >()

template<>

EIGEN_STRONG_INLINE Packet2ui Eigen::internal::pcast< Packet8c, Packet2ui >

(

const Packet8c &

a

)

                                                                            {
  return preinterpret<Packet2ui>(pcast<Packet8c, Packet2i>(a));
}

References a, and pcast< Packet8c, Packet2i >().

pcast< Packet8c, Packet4i >()

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pcast< Packet8c, Packet4i >

(

const Packet8c &

a

)

                                                                          {
  return vmovl_s16(vget_low_s16(vmovl_s8(a)));
}

References a.

Referenced by pcast< Packet4c, Packet4i >().

pcast< Packet8c, Packet4s >()

template<>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::pcast< Packet8c, Packet4s >

(

const Packet8c &

a

)

                                                                          {
  // Discard second half of input.
  return vget_low_s16(vmovl_s8(a));
}

References a.

Referenced by pcast< Packet4c, Packet4s >(), and pcast< Packet8c, Packet4us >().

pcast< Packet8c, Packet4us >()

template<>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::pcast< Packet8c, Packet4us >

(

const Packet8c &

a

)

                                                                            {
  return preinterpret<Packet4us>(pcast<Packet8c, Packet4s>(a));
}

References a, and pcast< Packet8c, Packet4s >().

pcast< Packet8c, Packet8s >()

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::pcast< Packet8c, Packet8s >

(

const Packet8c &

a

)

                                                                          {
  return vmovl_s8(a);
}

References a.

Referenced by pcast< Packet8c, Packet8us >().

pcast< Packet8c, Packet8us >()

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::pcast< Packet8c, Packet8us >

(

const Packet8c &

a

)

                                                                            {
  return preinterpret<Packet8us>(pcast<Packet8c, Packet8s>(a));
}

References a, and pcast< Packet8c, Packet8s >().

pcast< Packet8d, Packet16f >()

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::pcast< Packet8d, Packet16f >	(	const Packet8d &	a,
		const Packet8d &	b
	)

                                                                                               {
  return cat256(_mm512_cvtpd_ps(a), _mm512_cvtpd_ps(b));
}

References a, b, and cat256().

pcast< Packet8d, Packet16i >()

template<>

EIGEN_STRONG_INLINE Packet16i Eigen::internal::pcast< Packet8d, Packet16i >	(	const Packet8d &	a,
		const Packet8d &	b
	)

                                                                                               {
  return cat256i(_mm512_cvttpd_epi32(a), _mm512_cvttpd_epi32(b));
}

References a, b, and cat256i().

pcast< Packet8d, Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::pcast< Packet8d, Packet8f >

(

const Packet8d &

a

)

                                                                          {
  return _mm512_cvtpd_ps(a);
}

References a.

pcast< Packet8d, Packet8i >()

template<>

EIGEN_STRONG_INLINE Packet8i Eigen::internal::pcast< Packet8d, Packet8i >

(

const Packet8d &

a

)

                                                                          {
  return _mm512_cvtpd_epi32(a);
}

References a.

pcast< Packet8d, Packet8l >()

template<>

EIGEN_STRONG_INLINE Packet8l Eigen::internal::pcast< Packet8d, Packet8l >

(

const Packet8d &

a

)

                                                                          {
#if defined(EIGEN_VECTORIZE_AVX512DQ) && defined(EIGEN_VECTORIZE_AVX512VL)
  return _mm512_cvttpd_epi64(a);
#else
  constexpr int kTotalBits = sizeof(double) * CHAR_BIT, kMantissaBits = std::numeric_limits<double>::digits - 1,
                kExponentBits = kTotalBits - kMantissaBits - 1, kBias = (1 << (kExponentBits - 1)) - 1;
 
  const __m512i cst_one = _mm512_set1_epi64(1);
  const __m512i cst_total_bits = _mm512_set1_epi64(kTotalBits);
  const __m512i cst_bias = _mm512_set1_epi64(kBias);
 
  __m512i a_bits = _mm512_castpd_si512(a);
  // shift left by 1 to clear the sign bit, and shift right by kMantissaBits + 1 to recover biased exponent
  __m512i biased_e = _mm512_srli_epi64(_mm512_slli_epi64(a_bits, 1), kMantissaBits + 1);
  __m512i e = _mm512_sub_epi64(biased_e, cst_bias);
 
  // shift to the left by kExponentBits + 1 to clear the sign and exponent bits
  __m512i shifted_mantissa = _mm512_slli_epi64(a_bits, kExponentBits + 1);
  // shift to the right by kTotalBits - e to convert the significand to an integer
  __m512i result_significand = _mm512_srlv_epi64(shifted_mantissa, _mm512_sub_epi64(cst_total_bits, e));
 
  // add the implied bit
  __m512i result_exponent = _mm512_sllv_epi64(cst_one, e);
  // e <= 0 is interpreted as a large positive shift (2's complement), which also conveniently results in zero
  __m512i result = _mm512_add_epi64(result_significand, result_exponent);
  // handle negative arguments
  __mmask8 sign_mask = _mm512_cmplt_epi64_mask(a_bits, _mm512_setzero_si512());
  result = _mm512_mask_sub_epi64(result, sign_mask, _mm512_setzero_si512(), result);
  return result;
#endif
}

References a, and e().

pcast< Packet8f, Packet16b >()

template<>

EIGEN_STRONG_INLINE Packet16b Eigen::internal::pcast< Packet8f, Packet16b >	(	const Packet8f &	a,
		const Packet8f &	b
	)

                                                                                               {
  __m256 nonzero_a = _mm256_cmp_ps(a, pzero(a), _CMP_NEQ_UQ);
  __m256 nonzero_b = _mm256_cmp_ps(b, pzero(b), _CMP_NEQ_UQ);
  constexpr char kFF = '\255';
#ifndef EIGEN_VECTORIZE_AVX2
  __m128i shuffle_mask128_a_lo = _mm_set_epi8(kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, 12, 8, 4, 0);
  __m128i shuffle_mask128_a_hi = _mm_set_epi8(kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, 12, 8, 4, 0, kFF, kFF, kFF, kFF);
  __m128i shuffle_mask128_b_lo = _mm_set_epi8(kFF, kFF, kFF, kFF, 12, 8, 4, 0, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF);
  __m128i shuffle_mask128_b_hi = _mm_set_epi8(12, 8, 4, 0, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF);
  __m128i a_hi = _mm_shuffle_epi8(_mm256_extractf128_si256(_mm256_castps_si256(nonzero_a), 1), shuffle_mask128_a_hi);
  __m128i a_lo = _mm_shuffle_epi8(_mm256_extractf128_si256(_mm256_castps_si256(nonzero_a), 0), shuffle_mask128_a_lo);
  __m128i b_hi = _mm_shuffle_epi8(_mm256_extractf128_si256(_mm256_castps_si256(nonzero_b), 1), shuffle_mask128_b_hi);
  __m128i b_lo = _mm_shuffle_epi8(_mm256_extractf128_si256(_mm256_castps_si256(nonzero_b), 0), shuffle_mask128_b_lo);
  __m128i merged = _mm_or_si128(_mm_or_si128(b_lo, b_hi), _mm_or_si128(a_lo, a_hi));
  return _mm_and_si128(merged, _mm_set1_epi8(1));
#else
  __m256i a_shuffle_mask = _mm256_set_epi8(kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, 12, 8, 4, 0, kFF, kFF, kFF, kFF, kFF,
                                           kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, 12, 8, 4, 0);
  __m256i b_shuffle_mask = _mm256_set_epi8(12, 8, 4, 0, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF,
                                           kFF, kFF, kFF, 12, 8, 4, 0, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF);
  __m256i a_shuff = _mm256_shuffle_epi8(_mm256_castps_si256(nonzero_a), a_shuffle_mask);
  __m256i b_shuff = _mm256_shuffle_epi8(_mm256_castps_si256(nonzero_b), b_shuffle_mask);
  __m256i a_or_b = _mm256_or_si256(a_shuff, b_shuff);
  __m256i merged = _mm256_or_si256(a_or_b, _mm256_castsi128_si256(_mm256_extractf128_si256(a_or_b, 1)));
  return _mm256_castsi256_si128(_mm256_and_si256(merged, _mm256_set1_epi8(1)));
#endif
}

References a, b, and pzero().

pcast< Packet8f, Packet4d >()

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::pcast< Packet8f, Packet4d >

(

const Packet8f &

a

)

                                                                          {
  return _mm256_cvtps_pd(_mm256_castps256_ps128(a));
}

References a.

pcast< Packet8f, Packet8bf >()

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::pcast< Packet8f, Packet8bf >

(

const Packet8f &

a

)

                                                                            {
  return F32ToBf16(a);
}

References a, and F32ToBf16().

pcast< Packet8f, Packet8d >()

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::pcast< Packet8f, Packet8d >

(

const Packet8f &

a

)

                                                                          {
  return _mm512_cvtps_pd(a);
}

References a.

pcast< Packet8f, Packet8h >()

template<>

EIGEN_STRONG_INLINE Packet8h Eigen::internal::pcast< Packet8f, Packet8h >

(

const Packet8f &

a

)

                                                                          {
  return float2half(a);
}

References a, and float2half().

pcast< Packet8f, Packet8i >()

template<>

EIGEN_STRONG_INLINE Packet8i Eigen::internal::pcast< Packet8f, Packet8i >

(

const Packet8f &

a

)

                                                                          {
  return _mm256_cvttps_epi32(a);
}

References a.

pcast< Packet8h, Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::pcast< Packet8h, Packet8f >

(

const Packet8h &

a

)

                                                                          {
  return half2float(a);
}

References a, and half2float().

pcast< Packet8i, Packet4d >()

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::pcast< Packet8i, Packet4d >

(

const Packet8i &

a

)

                                                                          {
  return _mm256_cvtepi32_pd(_mm256_castsi256_si128(a));
}

References a.

pcast< Packet8i, Packet8d >()

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::pcast< Packet8i, Packet8d >

(

const Packet8i &

a

)

                                                                          {
  return _mm512_cvtepi32_pd(a);
}

References a.

pcast< Packet8i, Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::pcast< Packet8i, Packet8f >

(

const Packet8i &

a

)

                                                                          {
  return _mm256_cvtepi32_ps(a);
}

References a.

pcast< Packet8l, Packet8d >()

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::pcast< Packet8l, Packet8d >

(

const Packet8l &

a

)

                                                                          {
#if defined(EIGEN_VECTORIZE_AVX512DQ) && defined(EIGEN_VECTORIZE_AVX512VL)
  return _mm512_cvtepi64_pd(a);
#else
  EIGEN_ALIGN64 int64_t aux[8];
  pstore(aux, a);
  return _mm512_set_pd(static_cast<double>(aux[7]), static_cast<double>(aux[6]), static_cast<double>(aux[5]),
                       static_cast<double>(aux[4]), static_cast<double>(aux[3]), static_cast<double>(aux[2]),
                       static_cast<double>(aux[1]), static_cast<double>(aux[0]));
#endif
}

References a, EIGEN_ALIGN64, and pstore().

pcast< Packet8s, Packet16c >()

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::pcast< Packet8s, Packet16c >	(	const Packet8s &	a,
		const Packet8s &	b
	)

                                                                                               {
  return __lsx_vssrlni_b_h((__m128i)a, (__m128i)b, 0);
}

References a, and b.

pcast< Packet8s, Packet16uc >()

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::pcast< Packet8s, Packet16uc >	(	const Packet8s &	a,
		const Packet8s &	b
	)

                                                                                                 {
  return (Packet16uc)__lsx_vssrlni_b_h((__m128i)a, (__m128i)b, 0);
}

References a, and b.

pcast< Packet8s, Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pcast< Packet8s, Packet2d >

(

const Packet8s &

a

)

                                                                          {
  Packet4i tmp = __lsx_vsllwil_w_h((__m128i)a, 0);
  return __lsx_vffint_d_l(__lsx_vsllwil_d_w((__m128i)tmp, 0));
}

References a, and tmp.

pcast< Packet8s, Packet2l >()

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::pcast< Packet8s, Packet2l >

(

const Packet8s &

a

)

                                                                          {
  Packet4i tmp1 = __lsx_vsllwil_w_h((__m128i)a, 0);
  return __lsx_vsllwil_d_w((__m128i)tmp1, 0);
}

References a.

pcast< Packet8s, Packet2ul >()

template<>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::pcast< Packet8s, Packet2ul >

(

const Packet8s &

a

)

                                                                            {
  Packet4i tmp1 = __lsx_vsllwil_w_h((__m128i)a, 0);
  return (Packet2ul)__lsx_vsllwil_d_w((__m128i)tmp1, 0);
}

References a.

pcast< Packet8s, Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pcast< Packet8s, Packet4f >

(

const Packet8s &

a

)

                                                                          {
  Packet4i tmp1 = __lsx_vsllwil_w_h((__m128i)a, 0);
  return __lsx_vffint_s_w(tmp1);
}

References a.

pcast< Packet8s, Packet4i >()

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pcast< Packet8s, Packet4i >

(

const Packet8s &

a

)

                                                                          {
  return __lsx_vsllwil_w_h((__m128i)a, 0);
}

References a.

pcast< Packet8s, Packet4ui >()

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::pcast< Packet8s, Packet4ui >

(

const Packet8s &

a

)

                                                                            {
  return (Packet4ui)__lsx_vsllwil_w_h((__m128i)a, 0);
}

References a.

pcast< Packet8s, Packet8c >()

template<>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::pcast< Packet8s, Packet8c >

(

const Packet8s &

a

)

                                                                          {
  return vmovn_s16(a);
}

References a.

Referenced by pcast< Packet8s, Packet8uc >().

pcast< Packet8s, Packet8uc >()

template<>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::pcast< Packet8s, Packet8uc >

(

const Packet8s &

a

)

                                                                            {
  return preinterpret<Packet8uc>(pcast<Packet8s, Packet8c>(a));
}

References a, and pcast< Packet8s, Packet8c >().

pcast< Packet8uc, Packet2f >()

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::pcast< Packet8uc, Packet2f >

(

const Packet8uc &

a

)

                                                                            {
  // Discard all but first 2 bytes.
  return vcvt_f32_u32(vget_low_u32(vmovl_u16(vget_low_u16(vmovl_u8(a)))));
}

References a.

pcast< Packet8uc, Packet2i >()

template<>

EIGEN_STRONG_INLINE Packet2i Eigen::internal::pcast< Packet8uc, Packet2i >

(

const Packet8uc &

a

)

                                                                            {
  return preinterpret<Packet2i>(pcast<Packet8uc, Packet2ui>(a));
}

References a, and pcast< Packet8uc, Packet2ui >().

pcast< Packet8uc, Packet2ui >()

template<>

EIGEN_STRONG_INLINE Packet2ui Eigen::internal::pcast< Packet8uc, Packet2ui >

(

const Packet8uc &

a

)

                                                                              {
  // Discard all but first 2 bytes.
  return vget_low_u32(vmovl_u16(vget_low_u16(vmovl_u8(a))));
}

References a.

Referenced by pcast< Packet8uc, Packet2i >().

pcast< Packet8uc, Packet4ui >()

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::pcast< Packet8uc, Packet4ui >

(

const Packet8uc &

a

)

                                                                              {
  return vmovl_u16(vget_low_u16(vmovl_u8(a)));
}

References a.

Referenced by pcast< Packet4uc, Packet4ui >().

pcast< Packet8uc, Packet8s >()

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::pcast< Packet8uc, Packet8s >

(

const Packet8uc &

a

)

                                                                            {
  return preinterpret<Packet8s>(pcast<Packet8uc, Packet8us>(a));
}

References a, and pcast< Packet8uc, Packet8us >().

pcast< Packet8uc, Packet8us >()

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::pcast< Packet8uc, Packet8us >

(

const Packet8uc &

a

)

                                                                              {
  return vmovl_u8(a);
}

References a.

Referenced by pcast< Packet8uc, Packet8s >().

pcast< Packet8us, Packet16c >()

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::pcast< Packet8us, Packet16c >	(	const Packet8us &	a,
		const Packet8us &	b
	)

                                                                                                  {
  return (Packet16c)__lsx_vssrlni_bu_h((__m128i)a, (__m128i)b, 0);
}

References a, and b.

pcast< Packet8us, Packet16uc >()

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::pcast< Packet8us, Packet16uc >	(	const Packet8us &	a,
		const Packet8us &	b
	)

                                                                                                    {
  return __lsx_vssrlni_bu_h((__m128i)a, (__m128i)b, 0);
}

References a, and b.

pcast< Packet8us, Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pcast< Packet8us, Packet2d >

(

const Packet8us &

a

)

                                                                            {
  Packet4ui tmp = __lsx_vsllwil_wu_hu((__m128i)a, 0);
  return __lsx_vffint_d_lu(__lsx_vsllwil_du_wu((__m128i)tmp, 0));
}

References a, and tmp.

pcast< Packet8us, Packet2l >()

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::pcast< Packet8us, Packet2l >

(

const Packet8us &

a

)

                                                                            {
  Packet4ui tmp1 = __lsx_vsllwil_wu_hu((__m128i)a, 0);
  return (Packet2l)__lsx_vsllwil_du_wu((__m128i)tmp1, 0);
}

References a.

pcast< Packet8us, Packet2ul >()

template<>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::pcast< Packet8us, Packet2ul >

(

const Packet8us &

a

)

                                                                              {
  Packet4ui tmp1 = __lsx_vsllwil_wu_hu((__m128i)a, 0);
  return __lsx_vsllwil_du_wu((__m128i)tmp1, 0);
}

References a.

pcast< Packet8us, Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pcast< Packet8us, Packet4f >

(

const Packet8us &

a

)

                                                                            {
  Packet4ui tmp1 = __lsx_vsllwil_wu_hu((__m128i)a, 0);
  return __lsx_vffint_s_wu(tmp1);
}

References a.

pcast< Packet8us, Packet4i >()

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pcast< Packet8us, Packet4i >

(

const Packet8us &

a

)

                                                                            {
  return (Packet4i)__lsx_vsllwil_wu_hu((__m128i)a, 0);
}

References a.

pcast< Packet8us, Packet4ui >()

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::pcast< Packet8us, Packet4ui >

(

const Packet8us &

a

)

                                                                              {
  return __lsx_vsllwil_wu_hu((__m128i)a, 0);
}

References a.

pcast< Packet8us, Packet8bf >()

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::pcast< Packet8us, Packet8bf >

(

const Packet8us &

a

)

                                                                              {
  // short -> int -> float -> bfloat16
  const EIGEN_DECLARE_CONST_FAST_Packet4ui(low_mask, 0x0000FFFF);
  Packet4ui int_cast = reinterpret_cast<Packet4ui>(a);
  Packet4ui int_even = pand<Packet4ui>(int_cast, p4ui_low_mask);
  Packet4ui int_odd = plogical_shift_right<16>(int_cast);
  Packet4f float_even = pcast<Packet4ui, Packet4f>(int_even);
  Packet4f float_odd = pcast<Packet4ui, Packet4f>(int_odd);
  return F32ToBf16(float_even, float_odd);
}

References a, EIGEN_DECLARE_CONST_FAST_Packet4ui(), F32ToBf16(), pand< Packet4ui >(), and pcast< Packet4ui, Packet4f >().

pcast< Packet8us, Packet8c >()

template<>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::pcast< Packet8us, Packet8c >

(

const Packet8us &

a

)

                                                                            {
  return preinterpret<Packet8c>(pcast<Packet8us, Packet8uc>(a));
}

References a, and pcast< Packet8us, Packet8uc >().

pcast< Packet8us, Packet8uc >()

template<>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::pcast< Packet8us, Packet8uc >

(

const Packet8us &

a

)

                                                                              {
  return vmovn_u16(a);
}

References a.

Referenced by pcast< Packet8us, Packet8c >().

pcast< PacketXf, PacketXi >()

template<>

EIGEN_STRONG_INLINE PacketXi Eigen::internal::pcast< PacketXf, PacketXi >

(

const PacketXf &

a

)

                                                                          {
  return svcvt_s32_f32_x(svptrue_b32(), a);
}

References a.

pcast< PacketXi, PacketXf >()

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::pcast< PacketXi, PacketXf >

(

const PacketXi &

a

)

                                                                          {
  return svcvt_f32_s32_x(svptrue_b32(), a);
}

References a.

pcbrt()

template<typename Packet >

EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::pcbrt

(

const Packet &

a

)

Returns

the cube-root of a (coeff-wise)

                                                                                   {
  return numext::cbrt(a);
}

References a, and Eigen::numext::cbrt().

pceil() [1/3]

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet Eigen::internal::pceil

(

const Packet &

a

)

Returns

the ceil of a (coeff-wise)

                                                                    {
  return nearest_integer_packetop_impl<Packet>::run_ceil(a);
}

References a, and Eigen::internal::nearest_integer_packetop_impl< Packet, IsScalar, IsInteger >::run_ceil().

pceil() [2/3]

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pceil

(

const Packet2d &

a

)

                                                      {
  return __lsx_vfrintrp_d(a);
}

References a.

pceil() [3/3]

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pceil

(

const Packet4f &

a

)

                                                      {
  return __lsx_vfrintrp_s(a);
}

References a.

Referenced by packetmath_real(), and Eigen::internal::scalar_ceil_op< Scalar >::packetOp().

pceil< Packet16bf >()

template<>

EIGEN_STRONG_INLINE Packet16bf Eigen::internal::pceil< Packet16bf >

(

const Packet16bf &

a

)

                                                                      {
  return F32ToBf16(pceil<Packet16f>(Bf16ToF32(a)));
}

References a, Bf16ToF32(), F32ToBf16(), and pceil< Packet16f >().

pceil< Packet16f >()

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::pceil< Packet16f >

(

const Packet16f &

a

)

                                                                   {
  return _mm512_roundscale_ps(a, _MM_FROUND_TO_POS_INF);
}

References a.

Referenced by pceil< Packet16bf >(), and pceil< Packet16h >().

pceil< Packet16h >()

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::pceil< Packet16h >

(

const Packet16h &

a

)

                                                                   {
  return float2half(pceil<Packet16f>(half2float(a)));
}

References a, float2half(), half2float(), and pceil< Packet16f >().

pceil< Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pceil< Packet2d >

(

const Packet2d &

a

)

                                                                {
  Packet2d v = a;
  int32_t old_mode, new_mode;
  asm volatile(
      "cfcmsa  %[old_mode], $1\n"
      "ori     %[new_mode], %[old_mode], 3\n"
      "xori    %[new_mode], %[new_mode], 1\n"  // 2 = round towards +INFINITY.
      "ctcmsa  $1, %[new_mode]\n"
      "frint.d %w[v], %w[v]\n"
      "ctcmsa  $1, %[old_mode]\n"
      :  // outputs
      [old_mode] "=r"(old_mode), [new_mode] "=r"(new_mode),
      [v] "+f"(v)
      :  // inputs
      :  // clobbers
  );
  return v;
}

References a, and v.

pceil< Packet32h >()

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::pceil< Packet32h >

(

const Packet32h &

a

)

                                                                   {
  return _mm512_roundscale_ph(a, _MM_FROUND_TO_POS_INF);
}

References a.

pceil< Packet4bf >()

template<>

EIGEN_STRONG_INLINE Packet4bf Eigen::internal::pceil< Packet4bf >

(

const Packet4bf &

a

)

                                                                   {
  return F32ToBf16(pceil<Packet4f>(Bf16ToF32(a)));
}

References a, Bf16ToF32(), F32ToBf16(), and pceil< Packet4f >().

pceil< Packet4d >()

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::pceil< Packet4d >

(

const Packet4d &

a

)

                                                                {
  return _mm256_ceil_pd(a);
}

References a.

pceil< Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pceil< Packet4f >

(

const Packet4f &

a

)

                                                                {
  return vec_ceil(a);
}

References a.

Referenced by pceil< Packet4bf >(), and pceil< Packet8bf >().

pceil< Packet8bf >()

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::pceil< Packet8bf >

(

const Packet8bf &

a

)

                                                                   {
  BF16_TO_F32_UNARY_OP_WRAPPER(pceil<Packet4f>, a);
}

References a, BF16_TO_F32_UNARY_OP_WRAPPER, and pceil< Packet4f >().

pceil< Packet8d >()

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::pceil< Packet8d >

(

const Packet8d &

a

)

                                                                {
  return _mm512_roundscale_pd(a, _MM_FROUND_TO_POS_INF);
}

References a.

pceil< Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::pceil< Packet8f >

(

const Packet8f &

a

)

                                                                {
  return _mm256_ceil_ps(a);
}

References a.

Referenced by pceil< Packet8h >().

pceil< Packet8h >()

template<>

EIGEN_STRONG_INLINE Packet8h Eigen::internal::pceil< Packet8h >

(

const Packet8h &

a

)

                                                                {
  return float2half(pceil<Packet8f>(half2float(a)));
}

References a, float2half(), half2float(), and pceil< Packet8f >().

PCG_XSH_RS_generator()

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE unsigned Eigen::internal::PCG_XSH_RS_generator	(	uint64_t *	state,
		uint64_t	stream
	)

                                                                                                      {
  // TODO: Unify with the implementation in the non blocking thread pool.
  uint64_t current = *state;
  // Update the internal state
  *state = current * 6364136223846793005ULL + (stream << 1 | 1);
  // Generate the random output (using the PCG-XSH-RS scheme)
  return static_cast<unsigned>((current ^ (current >> 22)) >> (22 + (current >> 61)));
}

Referenced by RandomToTypeUniform(), RandomToTypeUniform< double >(), RandomToTypeUniform< Eigen::bfloat16 >(), RandomToTypeUniform< Eigen::half >(), and RandomToTypeUniform< float >().

PCG_XSH_RS_state()

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE uint64_t Eigen::internal::PCG_XSH_RS_state

(

uint64_t

seed

)

                                                                               {
  seed = seed ? seed : get_random_seed();
  return seed * 6364136223846793005ULL + 0xda3e39cb94b95bdbULL;
}

References get_random_seed().

Referenced by Eigen::internal::NormalRandomGenerator< T >::NormalRandomGenerator(), and Eigen::internal::UniformRandomGenerator< T >::UniformRandomGenerator().

pcmp_eq() [1/31]

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::pcmp_eq ( const Packet &  a, const Packet &  b  )

inline

Returns

a == b as a bit mask

                                                                          {
  return a == b ? ptrue(a) : pzero(a);
}

References a, b, ptrue(), and pzero().

pcmp_eq() [2/31]

template<>

EIGEN_STRONG_INLINE Packet16b Eigen::internal::pcmp_eq	(	const Packet16b &	a,
		const Packet16b &	b
	)

                                                                              {
  // Mask out invalid bool bits to avoid UB.
  const Packet16b kBoolMask = pset1<Packet16b>(true);
  return _mm_and_si128(_mm_cmpeq_epi8(a, b), kBoolMask);
}

References a, b, and pset1< Packet16b >().

pcmp_eq() [3/31]

template<>

EIGEN_STRONG_INLINE Packet16bf Eigen::internal::pcmp_eq	(	const Packet16bf &	a,
		const Packet16bf &	b
	)

                                                                                 {
  return Pack32To16(pcmp_eq(Bf16ToF32(a), Bf16ToF32(b)));
}

References a, b, Bf16ToF32(), Pack32To16(), and pcmp_eq().

pcmp_eq() [4/31]

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::pcmp_eq	(	const Packet16c &	a,
		const Packet16c &	b
	)

                                                                              {
  return reinterpret_cast<Packet16c>(vec_cmpeq(a, b));
}

References a, and b.

pcmp_eq() [5/31]

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::pcmp_eq	(	const Packet16f &	a,
		const Packet16f &	b
	)

                                                                              {
  __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_EQ_OQ);
  return _mm512_castsi512_ps(_mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, int32_t(-1)));
}

References a, and b.

pcmp_eq() [6/31]

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::pcmp_eq	(	const Packet16h &	a,
		const Packet16h &	b
	)

                                                                              {
  Packet16f af = half2float(a);
  Packet16f bf = half2float(b);
  return Pack32To16(pcmp_eq(af, bf));
}

References a, b, half2float(), Pack32To16(), and pcmp_eq().

pcmp_eq() [7/31]

template<>

EIGEN_STRONG_INLINE Packet16i Eigen::internal::pcmp_eq	(	const Packet16i &	a,
		const Packet16i &	b
	)

                                                                              {
  __mmask16 mask = _mm512_cmp_epi32_mask(a, b, _MM_CMPINT_EQ);
  return _mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, int32_t(-1));
}

References a, and b.

pcmp_eq() [8/31]

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::pcmp_eq	(	const Packet16uc &	a,
		const Packet16uc &	b
	)

                                                                                 {
  return reinterpret_cast<Packet16uc>(vec_cmpeq(a, b));
}

References a, and b.

pcmp_eq() [9/31]

template<>

EIGEN_STRONG_INLINE Packet1cd Eigen::internal::pcmp_eq	(	const Packet1cd &	a,
		const Packet1cd &	b
	)

                                                                              {
  Packet2d eq = (Packet2d)__lsx_vfcmp_ceq_d(a.v, b.v);
  return Packet1cd(pand<Packet2d>(eq, preverse(eq)));
}

References a, b, pand< Packet2d >(), and preverse().

pcmp_eq() [10/31]

template<>

EIGEN_STRONG_INLINE Packet1cf Eigen::internal::pcmp_eq	(	const Packet1cf &	a,
		const Packet1cf &	b
	)

                                                                              {
  // Compare real and imaginary parts of a and b to get the mask vector:
  // [re(a[0])==re(b[0]), im(a[0])==im(b[0])]
  Packet2f eq = pcmp_eq<Packet2f>(a.v, b.v);
  // Swap real/imag elements in the mask in to get:
  // [im(a[0])==im(b[0]), re(a[0])==re(b[0])]
  Packet2f eq_swapped = vrev64_f32(eq);
  // Return re(a)==re(b) && im(a)==im(b) by computing bitwise AND of eq and eq_swapped
  return Packet1cf(pand<Packet2f>(eq, eq_swapped));
}

References a, b, pand< Packet2f >(), and pcmp_eq< Packet2f >().

pcmp_eq() [11/31]

template<>

EIGEN_STRONG_INLINE Packet2cd Eigen::internal::pcmp_eq	(	const Packet2cd &	a,
		const Packet2cd &	b
	)

                                                                              {
  __m256d eq = _mm256_cmp_pd(a.v, b.v, _CMP_EQ_OQ);
  return Packet2cd(pand(eq, _mm256_permute_pd(eq, 0x5)));
}

References a, b, and pand().

pcmp_eq() [12/31]

template<>

EIGEN_STRONG_INLINE Packet2cf Eigen::internal::pcmp_eq	(	const Packet2cf &	a,
		const Packet2cf &	b
	)

                                                                              {
  Packet4f eq = reinterpret_cast<Packet4f>(vec_cmpeq(a.v, b.v));
  return Packet2cf(vec_and(eq, vec_perm(eq, eq, p16uc_COMPLEX32_REV)));
}

References a, b, and p16uc_COMPLEX32_REV.

Referenced by Eigen::internal::all_visitor< Scalar >::all_predux(), Eigen::internal::any_visitor< Scalar >::any_predux(), Eigen::internal::count_visitor< Scalar >::count_redux(), generic_expm1(), generic_log1p(), generic_ndtri(), generic_pow(), Eigen::internal::unary_pow::handle_negative_exponent(), Eigen::internal::unary_pow::handle_nonint_nonint_errors(), Eigen::internal::minmax_coeff_visitor< Derived, is_min, NaNPropagation, isInt >::initpacket(), Eigen::internal::minmax_coeff_visitor< Derived, is_min, NaNPropagation, false >::initpacket(), Eigen::internal::minmax_coeff_visitor< Derived, is_min, PropagateNumbers, false >::packet(), Eigen::internal::minmax_coeff_visitor< Derived, is_min, NaNPropagation, false >::packet(), packetmath_boolean_mask_ops(), Eigen::internal::scalar_boolean_not_op< Scalar >::packetOp(), Eigen::internal::scalar_cmp_op< LhsScalar, RhsScalar, cmp_EQ, UseTypedComparators >::packetOp(), Eigen::internal::scalar_cmp_op< LhsScalar, RhsScalar, cmp_NEQ, UseTypedComparators >::packetOp(), Eigen::internal::scalar_boolean_and_op< Scalar >::packetOp(), Eigen::internal::scalar_boolean_or_op< Scalar >::packetOp(), Eigen::internal::scalar_boolean_xor_op< Scalar >::packetOp(), Eigen::internal::scalar_boolean_select_op< ThenScalar, ElseScalar, ConditionScalar >::packetOp(), Eigen::internal::scalar_logistic_op_impl< T, EnableIf >::packetOp(), patanh_double(), patanh_float(), pcmp_eq(), pcmp_le(), pcmp_lt(), pexp_complex(), pfrexp_generic(), phypot_complex(), pisinf(), pisnan(), plog_complex(), plog_impl_double(), plog_impl_float(), pminmax_propagate_nan(), pminmax_propagate_numbers(), prsqrt_float_common(), psincos_double(), psincos_float(), psqrt_complex(), psqrt_float_common(), Eigen::internal::psign_impl< Packet, std::enable_if_t<!NumTraits< typename unpacket_traits< Packet >::type >::IsComplex &&!NumTraits< typename unpacket_traits< Packet >::type >::IsSigned &&NumTraits< typename unpacket_traits< Packet >::type >::IsInteger > >::run(), Eigen::internal::psign_impl< Packet, std::enable_if_t< NumTraits< typename unpacket_traits< Packet >::type >::IsComplex &&unpacket_traits< Packet >::vectorizable > >::run(), Eigen::internal::psignbit_impl< Packet, false, false >::run(), Eigen::internal::generic_reciprocal_newton_step< Packet, Steps >::run(), Eigen::internal::generic_sqrt_newton_step< Packet, Steps >::run(), Eigen::internal::pminmax_impl< PropagateNaN >::run(), Eigen::internal::pminmax_impl< PropagateNumbers >::run(), and Eigen::internal::maybe_raise_div_by_zero< Packet, true >::run().

pcmp_eq() [13/31]

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pcmp_eq	(	const Packet2d &	a,
		const Packet2d &	b
	)

                                                                           {
  return _mm_cmpeq_pd(a, b);
}

References a, and b.

pcmp_eq() [14/31]

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::pcmp_eq	(	const Packet2l &	a,
		const Packet2l &	b
	)

                                                                           {
#ifdef EIGEN_VECTORIZE_SSE4_1
  return _mm_cmpeq_epi64(a, b);
#else
  Packet4i tmp = pcmp_eq<Packet4i>(Packet4i(a), Packet4i(b));
  return Packet2l(pand<Packet4i>(tmp, _mm_shuffle_epi32(tmp, (shuffle_mask<1, 0, 3, 2>::mask))));
#endif
}

References a, b, pand< Packet4i >(), pcmp_eq< Packet4i >(), and tmp.

pcmp_eq() [15/31]

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::pcmp_eq	(	const Packet32h &	a,
		const Packet32h &	b
	)

                                                                              {
  __mmask32 mask = _mm512_cmp_ph_mask(a, b, _CMP_EQ_OQ);
  return _mm512_castsi512_ph(_mm512_mask_set1_epi16(_mm512_set1_epi32(0), mask, static_cast<short>(0xffffu)));
}

References a, and b.

pcmp_eq() [16/31]

template<>

EIGEN_STRONG_INLINE Packet4cd Eigen::internal::pcmp_eq	(	const Packet4cd &	a,
		const Packet4cd &	b
	)

                                                                              {
  __m512d eq = pcmp_eq<Packet8d>(a.v, b.v);
  return Packet4cd(pand(eq, _mm512_permute_pd(eq, 0x55)));
}

References a, b, and pand().

pcmp_eq() [17/31]

template<>

EIGEN_STRONG_INLINE Packet4cf Eigen::internal::pcmp_eq	(	const Packet4cf &	a,
		const Packet4cf &	b
	)

                                                                              {
  __m256 eq = _mm256_cmp_ps(a.v, b.v, _CMP_EQ_OQ);
  return Packet4cf(_mm256_and_ps(eq, _mm256_permute_ps(eq, 0xb1)));
}

References a, and b.

pcmp_eq() [18/31]

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::pcmp_eq	(	const Packet4d &	a,
		const Packet4d &	b
	)

                                                                           {
  return _mm256_cmp_pd(a, b, _CMP_EQ_OQ);
}

References a, and b.

pcmp_eq() [19/31]

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pcmp_eq	(	const Packet4f &	a,
		const Packet4f &	b
	)

                                                                           {
  return reinterpret_cast<Packet4f>(vec_cmpeq(a, b));
}

References a, and b.

pcmp_eq() [20/31]

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pcmp_eq	(	const Packet4i &	a,
		const Packet4i &	b
	)

                                                                           {
  return reinterpret_cast<Packet4i>(vec_cmpeq(a, b));
}

References a, and b.

pcmp_eq() [21/31]

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::pcmp_eq	(	const Packet4ui &	a,
		const Packet4ui &	b
	)

                                                                              {
  return _mm_cmpeq_epi32(a, b);
}

References a, and b.

pcmp_eq() [22/31]

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::pcmp_eq	(	const Packet8bf &	a,
		const Packet8bf &	b
	)

                                                                              {
  BF16_TO_F32_BINARY_OP_WRAPPER_BOOL(pcmp_eq<Packet4f>, a, b);
}

References a, b, BF16_TO_F32_BINARY_OP_WRAPPER_BOOL, and pcmp_eq< Packet4f >().

pcmp_eq() [23/31]

template<>

EIGEN_STRONG_INLINE Packet8cf Eigen::internal::pcmp_eq	(	const Packet8cf &	a,
		const Packet8cf &	b
	)

                                                                              {
  __m512 eq = pcmp_eq<Packet16f>(a.v, b.v);
  return Packet8cf(pand(eq, _mm512_permute_ps(eq, 0xB1)));
}

References a, b, and pand().

pcmp_eq() [24/31]

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::pcmp_eq	(	const Packet8d &	a,
		const Packet8d &	b
	)

                                                                           {
  __mmask8 mask = _mm512_cmp_pd_mask(a, b, _CMP_EQ_OQ);
  return _mm512_castsi512_pd(_mm512_mask_set1_epi64(_mm512_setzero_epi32(), mask, 0xffffffffffffffffu));
}

References a, and b.

pcmp_eq() [25/31]

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::pcmp_eq	(	const Packet8f &	a,
		const Packet8f &	b
	)

                                                                           {
  return _mm256_cmp_ps(a, b, _CMP_EQ_OQ);
}

References a, and b.

pcmp_eq() [26/31]

template<>

EIGEN_STRONG_INLINE Packet8h Eigen::internal::pcmp_eq	(	const Packet8h &	a,
		const Packet8h &	b
	)

                                                                           {
  return Pack16To8(pcmp_eq(half2float(a), half2float(b)));
}

References a, b, half2float(), Pack16To8(), and pcmp_eq().

pcmp_eq() [27/31]

template<>

EIGEN_STRONG_INLINE Packet8i Eigen::internal::pcmp_eq	(	const Packet8i &	a,
		const Packet8i &	b
	)

                                                                           {
#ifdef EIGEN_VECTORIZE_AVX2
  return _mm256_cmpeq_epi32(a, b);
#else
  __m128i lo = _mm_cmpeq_epi32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0));
  __m128i hi = _mm_cmpeq_epi32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1));
  return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1);
#endif
}

References a, and b.

pcmp_eq() [28/31]

template<>

EIGEN_STRONG_INLINE Packet8l Eigen::internal::pcmp_eq	(	const Packet8l &	a,
		const Packet8l &	b
	)

                                                                           {
  __mmask8 mask = _mm512_cmp_epi64_mask(a, b, _MM_CMPINT_EQ);
  return _mm512_mask_set1_epi64(_mm512_setzero_si512(), mask, int64_t(-1));
}

References a, and b.

pcmp_eq() [29/31]

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::pcmp_eq	(	const Packet8s &	a,
		const Packet8s &	b
	)

                                                                           {
  return reinterpret_cast<Packet8s>(vec_cmpeq(a, b));
}

References a, and b.

pcmp_eq() [30/31]

template<>

EIGEN_STRONG_INLINE Packet8ui Eigen::internal::pcmp_eq	(	const Packet8ui &	a,
		const Packet8ui &	b
	)

                                                                              {
#ifdef EIGEN_VECTORIZE_AVX2
  return _mm256_cmpeq_epi32(a, b);
#else
  __m128i lo = _mm_cmpeq_epi32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0));
  __m128i hi = _mm_cmpeq_epi32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1));
  return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1);
#endif
}

References a, and b.

pcmp_eq() [31/31]

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::pcmp_eq	(	const Packet8us &	a,
		const Packet8us &	b
	)

                                                                              {
  return reinterpret_cast<Packet8us>(vec_cmpeq(a, b));
}

References a, and b.

pcmp_eq< Packet16c >()

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::pcmp_eq< Packet16c >	(	const Packet16c &	a,
		const Packet16c &	b
	)

                                                                                         {
  return __lsx_vseq_b(a, b);
}

References a, and b.

pcmp_eq< Packet16uc >()

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::pcmp_eq< Packet16uc >	(	const Packet16uc &	a,
		const Packet16uc &	b
	)

                                                                                             {
  return __lsx_vseq_b(a, b);
}

References a, and b.

pcmp_eq< Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pcmp_eq< Packet2d >	(	const Packet2d &	a,
		const Packet2d &	b
	)

                                                                                     {
  return (Packet2d)__lsx_vfcmp_seq_d(a, b);
}

References a, and b.

pcmp_eq< Packet2f >()

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::pcmp_eq< Packet2f >	(	const Packet2f &	a,
		const Packet2f &	b
	)

                                                                                     {
  return vreinterpret_f32_u32(vceq_f32(a, b));
}

References a, and b.

Referenced by pcmp_eq().

pcmp_eq< Packet2i >()

template<>

EIGEN_STRONG_INLINE Packet2i Eigen::internal::pcmp_eq< Packet2i >	(	const Packet2i &	a,
		const Packet2i &	b
	)

                                                                                     {
  return vreinterpret_s32_u32(vceq_s32(a, b));
}

References a, and b.

pcmp_eq< Packet2l >()

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::pcmp_eq< Packet2l >	(	const Packet2l &	a,
		const Packet2l &	b
	)

                                                                                     {
  return __lsx_vseq_d(a, b);
}

References a, and b.

pcmp_eq< Packet2ui >()

template<>

EIGEN_STRONG_INLINE Packet2ui Eigen::internal::pcmp_eq< Packet2ui >	(	const Packet2ui &	a,
		const Packet2ui &	b
	)

                                                                                         {
  return vceq_u32(a, b);
}

References a, and b.

pcmp_eq< Packet2ul >()

template<>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::pcmp_eq< Packet2ul >	(	const Packet2ul &	a,
		const Packet2ul &	b
	)

                                                                                         {
  return __lsx_vseq_d(a, b);
}

References a, and b.

pcmp_eq< Packet4bf >()

template<>

EIGEN_STRONG_INLINE Packet4bf Eigen::internal::pcmp_eq< Packet4bf >	(	const Packet4bf &	a,
		const Packet4bf &	b
	)

                                                                                         {
  return F32MaskToBf16Mask(pcmp_eq<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
}

References a, b, Bf16ToF32(), F32MaskToBf16Mask(), and pcmp_eq< Packet4f >().

pcmp_eq< Packet4c >()

template<>

EIGEN_STRONG_INLINE Packet4c Eigen::internal::pcmp_eq< Packet4c >	(	const Packet4c &	a,
		const Packet4c &	b
	)

                                                                                     {
  return vget_lane_s32(
      vreinterpret_s32_u8(vceq_s8(vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
}

References a, and b.

pcmp_eq< Packet4f >()

template<>

Packet4f EIGEN_STRONG_INLINE Eigen::internal::pcmp_eq< Packet4f >	(	const Packet4f &	a,
		const Packet4f &	b
	)

                                                                                     {
  return (Packet4f)__lsx_vfcmp_seq_s(a, b);
}

References a, and b.

Referenced by pcmp_eq(), and pcmp_eq< Packet4bf >().

pcmp_eq< Packet4i >()

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pcmp_eq< Packet4i >	(	const Packet4i &	a,
		const Packet4i &	b
	)

                                                                                     {
  return __lsx_vseq_w(a, b);
}

References a, and b.

Referenced by pcmp_eq(), and pcmp_lt().

pcmp_eq< Packet4s >()

template<>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::pcmp_eq< Packet4s >	(	const Packet4s &	a,
		const Packet4s &	b
	)

                                                                                     {
  return vreinterpret_s16_u16(vceq_s16(a, b));
}

References a, and b.

pcmp_eq< Packet4uc >()

template<>

EIGEN_STRONG_INLINE Packet4uc Eigen::internal::pcmp_eq< Packet4uc >	(	const Packet4uc &	a,
		const Packet4uc &	b
	)

                                                                                         {
  return vget_lane_u32(
      vreinterpret_u32_u8(vceq_u8(vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
}

References a, and b.

pcmp_eq< Packet4ui >()

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::pcmp_eq< Packet4ui >	(	const Packet4ui &	a,
		const Packet4ui &	b
	)

                                                                                         {
  return __lsx_vseq_w(a, b);
}

References a, and b.

pcmp_eq< Packet4us >()

template<>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::pcmp_eq< Packet4us >	(	const Packet4us &	a,
		const Packet4us &	b
	)

                                                                                         {
  return vceq_u16(a, b);
}

References a, and b.

pcmp_eq< Packet8c >()

template<>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::pcmp_eq< Packet8c >	(	const Packet8c &	a,
		const Packet8c &	b
	)

                                                                                     {
  return vreinterpret_s8_u8(vceq_s8(a, b));
}

References a, and b.

pcmp_eq< Packet8s >()

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::pcmp_eq< Packet8s >	(	const Packet8s &	a,
		const Packet8s &	b
	)

                                                                                     {
  return __lsx_vseq_h(a, b);
}

References a, and b.

pcmp_eq< Packet8uc >()

template<>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::pcmp_eq< Packet8uc >	(	const Packet8uc &	a,
		const Packet8uc &	b
	)

                                                                                         {
  return vceq_u8(a, b);
}

References a, and b.

pcmp_eq< Packet8us >()

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::pcmp_eq< Packet8us >	(	const Packet8us &	a,
		const Packet8us &	b
	)

                                                                                         {
  return __lsx_vseq_h(a, b);
}

References a, and b.

pcmp_eq< PacketXf >()

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::pcmp_eq< PacketXf >	(	const PacketXf &	a,
		const PacketXf &	b
	)

                                                                                     {
  return svreinterpret_f32_u32(svdup_n_u32_z(svcmpeq_f32(svptrue_b32(), a, b), 0xffffffffu));
}

References a, and b.

pcmp_eq< PacketXi >()

template<>

EIGEN_STRONG_INLINE PacketXi Eigen::internal::pcmp_eq< PacketXi >	(	const PacketXi &	a,
		const PacketXi &	b
	)

                                                                                     {
  return svdup_n_s32_z(svcmpeq_s32(svptrue_b32(), a, b), 0xffffffffu);
}

References a, and b.

pcmp_le() [1/19]

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::pcmp_le ( const Packet &  a, const Packet &  b  )

inline

Returns

a <= b as a bit mask

                                                                          {
  return a <= b ? ptrue(a) : pzero(a);
}

References a, b, ptrue(), and pzero().

pcmp_le() [2/19]

template<>

EIGEN_STRONG_INLINE Packet16bf Eigen::internal::pcmp_le	(	const Packet16bf &	a,
		const Packet16bf &	b
	)

                                                                                 {
  return Pack32To16(pcmp_le(Bf16ToF32(a), Bf16ToF32(b)));
}

References a, b, Bf16ToF32(), Pack32To16(), and pcmp_le().

pcmp_le() [3/19]

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::pcmp_le	(	const Packet16f &	a,
		const Packet16f &	b
	)

                                                                              {
  __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_LE_OQ);
  return _mm512_castsi512_ps(_mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, int32_t(-1)));
}

References a, and b.

pcmp_le() [4/19]

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::pcmp_le	(	const Packet16h &	a,
		const Packet16h &	b
	)

                                                                              {
  return Pack32To16(pcmp_le(half2float(a), half2float(b)));
}

References a, b, half2float(), Pack32To16(), and pcmp_le().

pcmp_le() [5/19]

template<>

EIGEN_STRONG_INLINE Packet16i Eigen::internal::pcmp_le	(	const Packet16i &	a,
		const Packet16i &	b
	)

                                                                              {
  __mmask16 mask = _mm512_cmp_epi32_mask(a, b, _MM_CMPINT_LE);
  return _mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, int32_t(-1));
}

References a, and b.

pcmp_le() [6/19]

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pcmp_le	(	const Packet2d &	a,
		const Packet2d &	b
	)

                                                                           {
  return _mm_cmple_pd(a, b);
}

References a, and b.

pcmp_le() [7/19]

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::pcmp_le	(	const Packet2l &	a,
		const Packet2l &	b
	)

                                                                           {
  return por(pcmp_lt(a, b), pcmp_eq(a, b));
}

References a, b, pcmp_eq(), pcmp_lt(), and por().

pcmp_le() [8/19]

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::pcmp_le	(	const Packet32h &	a,
		const Packet32h &	b
	)

                                                                              {
  __mmask32 mask = _mm512_cmp_ph_mask(a, b, _CMP_LE_OQ);
  return _mm512_castsi512_ph(_mm512_mask_set1_epi16(_mm512_set1_epi32(0), mask, static_cast<short>(0xffffu)));
}

References a, and b.

pcmp_le() [9/19]

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::pcmp_le	(	const Packet4d &	a,
		const Packet4d &	b
	)

                                                                           {
  return _mm256_cmp_pd(a, b, _CMP_LE_OQ);
}

References a, and b.

pcmp_le() [10/19]

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pcmp_le	(	const Packet4f &	a,
		const Packet4f &	b
	)

                                                                           {
  return reinterpret_cast<Packet4f>(vec_cmple(a, b));
}

References a, and b.

Referenced by generic_ndtri(), generic_pow(), generic_round(), Eigen::internal::unary_pow::handle_nonint_nonint_errors(), Eigen::internal::scalar_cmp_op< LhsScalar, RhsScalar, cmp_LE, UseTypedComparators >::packetOp(), Eigen::internal::scalar_cmp_op< LhsScalar, RhsScalar, cmp_GE, UseTypedComparators >::packetOp(), Eigen::internal::scalar_cmp_op< LhsScalar, RhsScalar, cmp_UNORD, UseTypedComparators >::packetOp(), patanh_double(), patanh_float(), pcmp_le(), pdiv_float_common(), psincos_double(), psincos_float(), packetmath_boolean_mask_ops_notcomplex_test< Scalar, Packet, std::enable_if_t< internal::packet_traits< Scalar >::HasCmp &&!internal::is_same< Scalar, bool >::value > >::run(), Eigen::internal::generic_i0e< T, float >::run(), Eigen::internal::generic_i0e< T, double >::run(), Eigen::internal::generic_i1e< T, float >::run(), Eigen::internal::generic_i1e< T, double >::run(), Eigen::internal::generic_k0e< T, float >::run(), Eigen::internal::generic_k0e< T, double >::run(), Eigen::internal::generic_k0< T, float >::run(), Eigen::internal::generic_k0< T, double >::run(), Eigen::internal::generic_k1e< T, float >::run(), Eigen::internal::generic_k1e< T, double >::run(), Eigen::internal::generic_k1< T, float >::run(), Eigen::internal::generic_k1< T, double >::run(), Eigen::internal::generic_j0< T, float >::run(), Eigen::internal::generic_j0< T, double >::run(), Eigen::internal::generic_y0< T, float >::run(), Eigen::internal::generic_y0< T, double >::run(), Eigen::internal::generic_j1< T, float >::run(), Eigen::internal::generic_j1< T, double >::run(), Eigen::internal::generic_y1< T, float >::run(), and Eigen::internal::generic_y1< T, double >::run().

pcmp_le() [11/19]

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pcmp_le	(	const Packet4i &	a,
		const Packet4i &	b
	)

                                                                           {
  return por(pcmp_lt(a, b), pcmp_eq(a, b));
}

References a, b, pcmp_eq(), pcmp_lt(), and por().

pcmp_le() [12/19]

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::pcmp_le	(	const Packet4ui &	a,
		const Packet4ui &	b
	)

                                                                              {
#ifdef EIGEN_VECTORIZE_SSE4_1
  return pcmp_eq(a, pmin(a, b));
#else
  return (Packet4ui)pcmp_le((Packet4i)psub(a, pset1<Packet4ui>(0x80000000UL)),
                            (Packet4i)psub(b, pset1<Packet4ui>(0x80000000UL)));
#endif
}

References a, b, pcmp_eq(), pcmp_le(), pmin(), pset1< Packet4ui >(), and psub().

pcmp_le() [13/19]

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::pcmp_le	(	const Packet8bf &	a,
		const Packet8bf &	b
	)

                                                                              {
  BF16_TO_F32_BINARY_OP_WRAPPER_BOOL(pcmp_le<Packet4f>, a, b);
}

References a, b, BF16_TO_F32_BINARY_OP_WRAPPER_BOOL, and pcmp_le< Packet4f >().

pcmp_le() [14/19]

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::pcmp_le	(	const Packet8d &	a,
		const Packet8d &	b
	)

                                                                           {
  __mmask8 mask = _mm512_cmp_pd_mask(a, b, _CMP_LE_OQ);
  return _mm512_castsi512_pd(_mm512_mask_set1_epi64(_mm512_setzero_epi32(), mask, 0xffffffffffffffffu));
}

References a, and b.

pcmp_le() [15/19]

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::pcmp_le	(	const Packet8f &	a,
		const Packet8f &	b
	)

                                                                           {
  return _mm256_cmp_ps(a, b, _CMP_LE_OQ);
}

References a, and b.

pcmp_le() [16/19]

template<>

EIGEN_STRONG_INLINE Packet8h Eigen::internal::pcmp_le	(	const Packet8h &	a,
		const Packet8h &	b
	)

                                                                           {
  return Pack16To8(pcmp_le(half2float(a), half2float(b)));
}

References a, b, half2float(), Pack16To8(), and pcmp_le().

pcmp_le() [17/19]

template<>

EIGEN_STRONG_INLINE Packet8i Eigen::internal::pcmp_le	(	const Packet8i &	a,
		const Packet8i &	b
	)

                                                                           {
#ifdef EIGEN_VECTORIZE_AVX2
  return _mm256_xor_si256(_mm256_cmpgt_epi32(a, b), _mm256_set1_epi32(-1));
#else
  __m128i lo = _mm_cmpgt_epi32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0));
  lo = _mm_xor_si128(lo, _mm_set1_epi32(-1));
  __m128i hi = _mm_cmpgt_epi32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1));
  hi = _mm_xor_si128(hi, _mm_set1_epi32(-1));
  return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1);
#endif
}

References a, and b.

pcmp_le() [18/19]

template<>

EIGEN_STRONG_INLINE Packet8l Eigen::internal::pcmp_le	(	const Packet8l &	a,
		const Packet8l &	b
	)

                                                                           {
  __mmask8 mask = _mm512_cmp_epi64_mask(a, b, _MM_CMPINT_LE);
  return _mm512_mask_set1_epi64(_mm512_setzero_si512(), mask, int64_t(-1));
}

References a, and b.

pcmp_le() [19/19]

template<>

EIGEN_STRONG_INLINE Packet8ui Eigen::internal::pcmp_le	(	const Packet8ui &	a,
		const Packet8ui &	b
	)

                                                                              {
  return pcmp_eq(a, pmin(a, b));
}

References a, b, pcmp_eq(), and pmin().

pcmp_le< Packet16c >()

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::pcmp_le< Packet16c >	(	const Packet16c &	a,
		const Packet16c &	b
	)

                                                                                         {
  return __lsx_vsle_b(a, b);
}

References a, and b.

pcmp_le< Packet16uc >()

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::pcmp_le< Packet16uc >	(	const Packet16uc &	a,
		const Packet16uc &	b
	)

                                                                                             {
  return __lsx_vsle_bu(a, b);
}

References a, and b.

pcmp_le< Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pcmp_le< Packet2d >	(	const Packet2d &	a,
		const Packet2d &	b
	)

                                                                                     {
  return (Packet2d)__lsx_vfcmp_cle_d(a, b);
}

References a, and b.

pcmp_le< Packet2f >()

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::pcmp_le< Packet2f >	(	const Packet2f &	a,
		const Packet2f &	b
	)

                                                                                     {
  return vreinterpret_f32_u32(vcle_f32(a, b));
}

References a, and b.

pcmp_le< Packet2i >()

template<>

EIGEN_STRONG_INLINE Packet2i Eigen::internal::pcmp_le< Packet2i >	(	const Packet2i &	a,
		const Packet2i &	b
	)

                                                                                     {
  return vreinterpret_s32_u32(vcle_s32(a, b));
}

References a, and b.

pcmp_le< Packet2l >()

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::pcmp_le< Packet2l >	(	const Packet2l &	a,
		const Packet2l &	b
	)

                                                                                     {
  return __lsx_vsle_d(a, b);
}

References a, and b.

pcmp_le< Packet2ui >()

template<>

EIGEN_STRONG_INLINE Packet2ui Eigen::internal::pcmp_le< Packet2ui >	(	const Packet2ui &	a,
		const Packet2ui &	b
	)

                                                                                         {
  return vcle_u32(a, b);
}

References a, and b.

pcmp_le< Packet2ul >()

template<>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::pcmp_le< Packet2ul >	(	const Packet2ul &	a,
		const Packet2ul &	b
	)

                                                                                         {
  return __lsx_vsle_du(a, b);
}

References a, and b.

pcmp_le< Packet4bf >()

template<>

EIGEN_STRONG_INLINE Packet4bf Eigen::internal::pcmp_le< Packet4bf >	(	const Packet4bf &	a,
		const Packet4bf &	b
	)

                                                                                         {
  return F32MaskToBf16Mask(pcmp_le<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
}

References a, b, Bf16ToF32(), F32MaskToBf16Mask(), and pcmp_le< Packet4f >().

pcmp_le< Packet4c >()

template<>

EIGEN_STRONG_INLINE Packet4c Eigen::internal::pcmp_le< Packet4c >	(	const Packet4c &	a,
		const Packet4c &	b
	)

                                                                                     {
  return vget_lane_s32(
      vreinterpret_s32_u8(vcle_s8(vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
}

References a, and b.

pcmp_le< Packet4f >()

template<>

Packet4f EIGEN_STRONG_INLINE Eigen::internal::pcmp_le< Packet4f >	(	const Packet4f &	a,
		const Packet4f &	b
	)

                                                                                     {
  return (Packet4f)__lsx_vfcmp_cle_s(a, b);
}

References a, and b.

Referenced by pcmp_le(), and pcmp_le< Packet4bf >().

pcmp_le< Packet4i >()

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pcmp_le< Packet4i >	(	const Packet4i &	a,
		const Packet4i &	b
	)

                                                                                     {
  return __lsx_vsle_w(a, b);
}

References a, and b.

pcmp_le< Packet4s >()

template<>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::pcmp_le< Packet4s >	(	const Packet4s &	a,
		const Packet4s &	b
	)

                                                                                     {
  return vreinterpret_s16_u16(vcle_s16(a, b));
}

References a, and b.

pcmp_le< Packet4uc >()

template<>

EIGEN_STRONG_INLINE Packet4uc Eigen::internal::pcmp_le< Packet4uc >	(	const Packet4uc &	a,
		const Packet4uc &	b
	)

                                                                                         {
  return vget_lane_u32(
      vreinterpret_u32_u8(vcle_u8(vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
}

References a, and b.

pcmp_le< Packet4ui >()

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::pcmp_le< Packet4ui >	(	const Packet4ui &	a,
		const Packet4ui &	b
	)

                                                                                         {
  return __lsx_vsle_wu(a, b);
}

References a, and b.

pcmp_le< Packet4us >()

template<>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::pcmp_le< Packet4us >	(	const Packet4us &	a,
		const Packet4us &	b
	)

                                                                                         {
  return vcle_u16(a, b);
}

References a, and b.

pcmp_le< Packet8c >()

template<>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::pcmp_le< Packet8c >	(	const Packet8c &	a,
		const Packet8c &	b
	)

                                                                                     {
  return vreinterpret_s8_u8(vcle_s8(a, b));
}

References a, and b.

pcmp_le< Packet8s >()

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::pcmp_le< Packet8s >	(	const Packet8s &	a,
		const Packet8s &	b
	)

                                                                                     {
  return __lsx_vsle_h(a, b);
}

References a, and b.

pcmp_le< Packet8uc >()

template<>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::pcmp_le< Packet8uc >	(	const Packet8uc &	a,
		const Packet8uc &	b
	)

                                                                                         {
  return vcle_u8(a, b);
}

References a, and b.

pcmp_le< Packet8us >()

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::pcmp_le< Packet8us >	(	const Packet8us &	a,
		const Packet8us &	b
	)

                                                                                         {
  return __lsx_vsle_hu(a, b);
}

References a, and b.

pcmp_le< PacketXf >()

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::pcmp_le< PacketXf >	(	const PacketXf &	a,
		const PacketXf &	b
	)

                                                                                     {
  return svreinterpret_f32_u32(svdup_n_u32_z(svcmple_f32(svptrue_b32(), a, b), 0xffffffffu));
}

References a, and b.

pcmp_le< PacketXi >()

template<>

EIGEN_STRONG_INLINE PacketXi Eigen::internal::pcmp_le< PacketXi >	(	const PacketXi &	a,
		const PacketXi &	b
	)

                                                                                     {
  return svdup_n_s32_z(svcmple_s32(svptrue_b32(), a, b), 0xffffffffu);
}

References a, and b.

pcmp_lt() [1/23]

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::pcmp_lt ( const Packet &  a, const Packet &  b  )

inline

Returns

a < b as a bit mask

                                                                          {
  return a < b ? ptrue(a) : pzero(a);
}

References a, b, ptrue(), and pzero().

pcmp_lt() [2/23]

template<>

EIGEN_STRONG_INLINE Packet16bf Eigen::internal::pcmp_lt	(	const Packet16bf &	a,
		const Packet16bf &	b
	)

                                                                                 {
  return Pack32To16(pcmp_lt(Bf16ToF32(a), Bf16ToF32(b)));
}

References a, b, Bf16ToF32(), Pack32To16(), and pcmp_lt().

pcmp_lt() [3/23]

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::pcmp_lt	(	const Packet16c &	a,
		const Packet16c &	b
	)

                                                                              {
  return reinterpret_cast<Packet16c>(vec_cmplt(a, b));
}

References a, and b.

pcmp_lt() [4/23]

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::pcmp_lt	(	const Packet16f &	a,
		const Packet16f &	b
	)

                                                                              {
  __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_LT_OQ);
  return _mm512_castsi512_ps(_mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, int32_t(-1)));
}

References a, and b.

pcmp_lt() [5/23]

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::pcmp_lt	(	const Packet16h &	a,
		const Packet16h &	b
	)

                                                                              {
  return Pack32To16(pcmp_lt(half2float(a), half2float(b)));
}

References a, b, half2float(), Pack32To16(), and pcmp_lt().

pcmp_lt() [6/23]

template<>

EIGEN_STRONG_INLINE Packet16i Eigen::internal::pcmp_lt	(	const Packet16i &	a,
		const Packet16i &	b
	)

                                                                              {
  __mmask16 mask = _mm512_cmp_epi32_mask(a, b, _MM_CMPINT_LT);
  return _mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, int32_t(-1));
}

References a, and b.

pcmp_lt() [7/23]

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::pcmp_lt	(	const Packet16uc &	a,
		const Packet16uc &	b
	)

                                                                                 {
  return reinterpret_cast<Packet16uc>(vec_cmplt(a, b));
}

References a, and b.

pcmp_lt() [8/23]

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pcmp_lt	(	const Packet2d &	a,
		const Packet2d &	b
	)

                                                                           {
  return _mm_cmplt_pd(a, b);
}

References a, and b.

pcmp_lt() [9/23]

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::pcmp_lt	(	const Packet2l &	a,
		const Packet2l &	b
	)

                                                                           {
#ifdef EIGEN_VECTORIZE_SSE4_2
  return _mm_cmpgt_epi64(b, a);
#else
  Packet4i eq = pcmp_eq<Packet4i>(Packet4i(a), Packet4i(b));
  Packet2l hi_eq = Packet2l(_mm_shuffle_epi32(eq, (shuffle_mask<1, 1, 3, 3>::mask)));
  Packet4i lt = pcmp_lt<Packet4i>(Packet4i(a), Packet4i(b));
  Packet2l hi_lt = Packet2l(_mm_shuffle_epi32(lt, (shuffle_mask<1, 1, 3, 3>::mask)));
  Packet2l lo_lt = Packet2l(_mm_shuffle_epi32(lt, (shuffle_mask<0, 0, 2, 2>::mask)));
  // return hi(a) < hi(b) || (hi(a) == hi(b) && lo(a) < lo(b))
  return por(hi_lt, pand(hi_eq, lo_lt));
#endif
}

References a, b, pand(), pcmp_eq< Packet4i >(), pcmp_lt< Packet4i >(), and por().

pcmp_lt() [10/23]

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::pcmp_lt	(	const Packet32h &	a,
		const Packet32h &	b
	)

                                                                              {
  __mmask32 mask = _mm512_cmp_ph_mask(a, b, _CMP_LT_OQ);
  return _mm512_castsi512_ph(_mm512_mask_set1_epi16(_mm512_set1_epi32(0), mask, static_cast<short>(0xffffu)));
}

References a, and b.

pcmp_lt() [11/23]

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::pcmp_lt	(	const Packet4d &	a,
		const Packet4d &	b
	)

                                                                           {
  return _mm256_cmp_pd(a, b, _CMP_LT_OQ);
}

References a, and b.

pcmp_lt() [12/23]

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pcmp_lt	(	const Packet4f &	a,
		const Packet4f &	b
	)

                                                                           {
  return _mm_cmplt_ps(a, b);
}

References a, and b.

pcmp_lt() [13/23]

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pcmp_lt	(	const Packet4i &	a,
		const Packet4i &	b
	)

                                                                           {
  return reinterpret_cast<Packet4i>(vec_cmplt(a, b));
}

References a, and b.

Referenced by erfc_double_large(), generic_atan(), generic_ceil(), generic_floor(), generic_ndtri(), generic_ndtri_lt_exp_neg_two(), generic_pow(), generic_pow_impl(), generic_rint(), pabsdiff(), Eigen::internal::scalar_logistic_op< float >::packetOp(), Eigen::internal::scalar_isfinite_op< Scalar, true >::packetOp(), Eigen::internal::scalar_cmp_op< LhsScalar, RhsScalar, cmp_LT, UseTypedComparators >::packetOp(), Eigen::internal::scalar_cmp_op< LhsScalar, RhsScalar, cmp_GT, UseTypedComparators >::packetOp(), Eigen::internal::linspaced_op_impl< Scalar, false >::packetOp(), pasin_float(), patanh_double(), patanh_float(), pcmp_le(), pcmp_lt(), pexp_double(), pexp_float(), pfrexp_generic(), plog_impl_double(), plog_impl_float(), psqrt_complex(), packetmath_boolean_mask_ops_notcomplex_test< Scalar, Packet, std::enable_if_t< internal::packet_traits< Scalar >::HasCmp &&!internal::is_same< Scalar, bool >::value > >::run(), Eigen::internal::psign_impl< Packet, std::enable_if_t<!NumTraits< typename unpacket_traits< Packet >::type >::IsComplex &&!NumTraits< typename unpacket_traits< Packet >::type >::IsInteger > >::run(), Eigen::internal::psign_impl< Packet, std::enable_if_t<!NumTraits< typename unpacket_traits< Packet >::type >::IsComplex &&NumTraits< typename unpacket_traits< Packet >::type >::IsSigned &&NumTraits< typename unpacket_traits< Packet >::type >::IsInteger > >::run(), Eigen::internal::psignbit_impl< Packet, false, true >::run(), Eigen::internal::generic_i1e< T, float >::run(), Eigen::internal::generic_i1e< T, double >::run(), Eigen::internal::generic_j0< T, float >::run(), Eigen::internal::generic_j0< T, double >::run(), Eigen::internal::generic_j1< T, float >::run(), Eigen::internal::generic_j1< T, double >::run(), Eigen::internal::generic_y1< T, float >::run(), Eigen::internal::generic_fast_erf< Scalar >::run(), Eigen::internal::generic_fast_erfc< Scalar >::run(), and twosum().

pcmp_lt() [14/23]

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::pcmp_lt	(	const Packet4ui &	a,
		const Packet4ui &	b
	)

                                                                              {
#ifdef EIGEN_VECTORIZE_SSE4_1
  return pxor(pcmp_eq(a, pmax(a, b)), ptrue(a));
#else
  return (Packet4ui)pcmp_lt((Packet4i)psub(a, pset1<Packet4ui>(0x80000000UL)),
                            (Packet4i)psub(b, pset1<Packet4ui>(0x80000000UL)));
#endif
}

References a, b, pcmp_eq(), pcmp_lt(), pmax(), pset1< Packet4ui >(), psub(), ptrue(), and pxor().

pcmp_lt() [15/23]

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::pcmp_lt	(	const Packet8bf &	a,
		const Packet8bf &	b
	)

                                                                              {
  BF16_TO_F32_BINARY_OP_WRAPPER_BOOL(pcmp_lt<Packet4f>, a, b);
}

References a, b, BF16_TO_F32_BINARY_OP_WRAPPER_BOOL, and pcmp_lt< Packet4f >().

pcmp_lt() [16/23]

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::pcmp_lt	(	const Packet8d &	a,
		const Packet8d &	b
	)

                                                                           {
  __mmask8 mask = _mm512_cmp_pd_mask(a, b, _CMP_LT_OQ);
  return _mm512_castsi512_pd(_mm512_mask_set1_epi64(_mm512_setzero_epi32(), mask, 0xffffffffffffffffu));
}

References a, and b.

pcmp_lt() [17/23]

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::pcmp_lt	(	const Packet8f &	a,
		const Packet8f &	b
	)

                                                                           {
  return _mm256_cmp_ps(a, b, _CMP_LT_OQ);
}

References a, and b.

pcmp_lt() [18/23]

template<>

EIGEN_STRONG_INLINE Packet8h Eigen::internal::pcmp_lt	(	const Packet8h &	a,
		const Packet8h &	b
	)

                                                                           {
  return Pack16To8(pcmp_lt(half2float(a), half2float(b)));
}

References a, b, half2float(), Pack16To8(), and pcmp_lt().

pcmp_lt() [19/23]

template<>

EIGEN_STRONG_INLINE Packet8i Eigen::internal::pcmp_lt	(	const Packet8i &	a,
		const Packet8i &	b
	)

                                                                           {
#ifdef EIGEN_VECTORIZE_AVX2
  return _mm256_cmpgt_epi32(b, a);
#else
  __m128i lo = _mm_cmpgt_epi32(_mm256_extractf128_si256(b, 0), _mm256_extractf128_si256(a, 0));
  __m128i hi = _mm_cmpgt_epi32(_mm256_extractf128_si256(b, 1), _mm256_extractf128_si256(a, 1));
  return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1);
#endif
}

References a, and b.

pcmp_lt() [20/23]

template<>

EIGEN_STRONG_INLINE Packet8l Eigen::internal::pcmp_lt	(	const Packet8l &	a,
		const Packet8l &	b
	)

                                                                           {
  __mmask8 mask = _mm512_cmp_epi64_mask(a, b, _MM_CMPINT_LT);
  return _mm512_mask_set1_epi64(_mm512_setzero_si512(), mask, int64_t(-1));
}

References a, and b.

pcmp_lt() [21/23]

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::pcmp_lt	(	const Packet8s &	a,
		const Packet8s &	b
	)

                                                                           {
  return reinterpret_cast<Packet8s>(vec_cmplt(a, b));
}

References a, and b.

pcmp_lt() [22/23]

template<>

EIGEN_STRONG_INLINE Packet8ui Eigen::internal::pcmp_lt	(	const Packet8ui &	a,
		const Packet8ui &	b
	)

                                                                              {
  return pxor(pcmp_eq(a, pmax(a, b)), ptrue(a));
}

References a, b, pcmp_eq(), pmax(), ptrue(), and pxor().

pcmp_lt() [23/23]

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::pcmp_lt	(	const Packet8us &	a,
		const Packet8us &	b
	)

                                                                              {
  return reinterpret_cast<Packet8us>(vec_cmplt(a, b));
}

References a, and b.

pcmp_lt< Packet16c >()

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::pcmp_lt< Packet16c >	(	const Packet16c &	a,
		const Packet16c &	b
	)

                                                                                         {
  return __lsx_vslt_b(a, b);
}

References a, and b.

pcmp_lt< Packet16uc >()

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::pcmp_lt< Packet16uc >	(	const Packet16uc &	a,
		const Packet16uc &	b
	)

                                                                                             {
  return __lsx_vslt_bu(a, b);
}

References a, and b.

pcmp_lt< Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pcmp_lt< Packet2d >	(	const Packet2d &	a,
		const Packet2d &	b
	)

                                                                                     {
  return (Packet2d)__lsx_vfcmp_clt_d(a, b);
}

References a, and b.

pcmp_lt< Packet2f >()

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::pcmp_lt< Packet2f >	(	const Packet2f &	a,
		const Packet2f &	b
	)

                                                                                     {
  return vreinterpret_f32_u32(vclt_f32(a, b));
}

References a, and b.

pcmp_lt< Packet2i >()

template<>

EIGEN_STRONG_INLINE Packet2i Eigen::internal::pcmp_lt< Packet2i >	(	const Packet2i &	a,
		const Packet2i &	b
	)

                                                                                     {
  return vreinterpret_s32_u32(vclt_s32(a, b));
}

References a, and b.

pcmp_lt< Packet2l >()

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::pcmp_lt< Packet2l >	(	const Packet2l &	a,
		const Packet2l &	b
	)

                                                                                     {
  return __lsx_vslt_d(a, b);
}

References a, and b.

pcmp_lt< Packet2ui >()

template<>

EIGEN_STRONG_INLINE Packet2ui Eigen::internal::pcmp_lt< Packet2ui >	(	const Packet2ui &	a,
		const Packet2ui &	b
	)

                                                                                         {
  return vclt_u32(a, b);
}

References a, and b.

pcmp_lt< Packet2ul >()

template<>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::pcmp_lt< Packet2ul >	(	const Packet2ul &	a,
		const Packet2ul &	b
	)

                                                                                         {
  return __lsx_vslt_du(a, b);
}

References a, and b.

pcmp_lt< Packet4bf >()

template<>

EIGEN_STRONG_INLINE Packet4bf Eigen::internal::pcmp_lt< Packet4bf >	(	const Packet4bf &	a,
		const Packet4bf &	b
	)

                                                                                         {
  return F32MaskToBf16Mask(pcmp_lt<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
}

References a, b, Bf16ToF32(), F32MaskToBf16Mask(), and pcmp_lt< Packet4f >().

pcmp_lt< Packet4c >()

template<>

EIGEN_STRONG_INLINE Packet4c Eigen::internal::pcmp_lt< Packet4c >	(	const Packet4c &	a,
		const Packet4c &	b
	)

                                                                                     {
  return vget_lane_s32(
      vreinterpret_s32_u8(vclt_s8(vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
}

References a, and b.

pcmp_lt< Packet4f >()

template<>

Packet4f EIGEN_STRONG_INLINE Eigen::internal::pcmp_lt< Packet4f >	(	const Packet4f &	a,
		const Packet4f &	b
	)

                                                                                     {
  return (Packet4f)__lsx_vfcmp_clt_s(a, b);
}

References a, and b.

Referenced by pcmp_lt(), and pcmp_lt< Packet4bf >().

pcmp_lt< Packet4i >()

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pcmp_lt< Packet4i >	(	const Packet4i &	a,
		const Packet4i &	b
	)

                                                                                     {
  return __lsx_vslt_w(a, b);
}

References a, and b.

Referenced by pcmp_lt().

pcmp_lt< Packet4s >()

template<>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::pcmp_lt< Packet4s >	(	const Packet4s &	a,
		const Packet4s &	b
	)

                                                                                     {
  return vreinterpret_s16_u16(vclt_s16(a, b));
}

References a, and b.

pcmp_lt< Packet4uc >()

template<>

EIGEN_STRONG_INLINE Packet4uc Eigen::internal::pcmp_lt< Packet4uc >	(	const Packet4uc &	a,
		const Packet4uc &	b
	)

                                                                                         {
  return vget_lane_u32(
      vreinterpret_u32_u8(vclt_u8(vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
}

References a, and b.

pcmp_lt< Packet4ui >()

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::pcmp_lt< Packet4ui >	(	const Packet4ui &	a,
		const Packet4ui &	b
	)

                                                                                         {
  return __lsx_vslt_wu(a, b);
}

References a, and b.

pcmp_lt< Packet4us >()

template<>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::pcmp_lt< Packet4us >	(	const Packet4us &	a,
		const Packet4us &	b
	)

                                                                                         {
  return vclt_u16(a, b);
}

References a, and b.

pcmp_lt< Packet8c >()

template<>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::pcmp_lt< Packet8c >	(	const Packet8c &	a,
		const Packet8c &	b
	)

                                                                                     {
  return vreinterpret_s8_u8(vclt_s8(a, b));
}

References a, and b.

pcmp_lt< Packet8s >()

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::pcmp_lt< Packet8s >	(	const Packet8s &	a,
		const Packet8s &	b
	)

                                                                                     {
  return __lsx_vslt_h(a, b);
}

References a, and b.

pcmp_lt< Packet8uc >()

template<>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::pcmp_lt< Packet8uc >	(	const Packet8uc &	a,
		const Packet8uc &	b
	)

                                                                                         {
  return vclt_u8(a, b);
}

References a, and b.

pcmp_lt< Packet8us >()

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::pcmp_lt< Packet8us >	(	const Packet8us &	a,
		const Packet8us &	b
	)

                                                                                         {
  return __lsx_vslt_hu(a, b);
}

References a, and b.

pcmp_lt< PacketXf >()

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::pcmp_lt< PacketXf >	(	const PacketXf &	a,
		const PacketXf &	b
	)

                                                                                     {
  return svreinterpret_f32_u32(svdup_n_u32_z(svcmplt_f32(svptrue_b32(), a, b), 0xffffffffu));
}

References a, and b.

pcmp_lt< PacketXi >()

template<>

EIGEN_STRONG_INLINE PacketXi Eigen::internal::pcmp_lt< PacketXi >	(	const PacketXi &	a,
		const PacketXi &	b
	)

                                                                                     {
  return svdup_n_s32_z(svcmplt_s32(svptrue_b32(), a, b), 0xffffffffu);
}

References a, and b.

pcmp_lt_or_nan() [1/12]

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::pcmp_lt_or_nan ( const Packet &  a, const Packet &  b  )

inline

Returns

a < b or a==NaN or b==NaN as a bit mask

                                                                                 {
  return a >= b ? pzero(a) : ptrue(a);
}

References a, b, ptrue(), and pzero().

pcmp_lt_or_nan() [2/12]

template<>

EIGEN_STRONG_INLINE Packet16bf Eigen::internal::pcmp_lt_or_nan	(	const Packet16bf &	a,
		const Packet16bf &	b
	)

                                                                                        {
  return Pack32To16(pcmp_lt_or_nan(Bf16ToF32(a), Bf16ToF32(b)));
}

References a, b, Bf16ToF32(), Pack32To16(), and pcmp_lt_or_nan().

pcmp_lt_or_nan() [3/12]

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::pcmp_lt_or_nan	(	const Packet16f &	a,
		const Packet16f &	b
	)

                                                                                     {
  __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_NGE_UQ);
  return _mm512_castsi512_ps(_mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, int32_t(-1)));
}

References a, and b.

pcmp_lt_or_nan() [4/12]

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::pcmp_lt_or_nan	(	const Packet16h &	a,
		const Packet16h &	b
	)

                                                                                     {
  return Pack32To16(pcmp_lt_or_nan(half2float(a), half2float(b)));
}

References a, b, half2float(), Pack32To16(), and pcmp_lt_or_nan().

pcmp_lt_or_nan() [5/12]

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pcmp_lt_or_nan	(	const Packet2d &	a,
		const Packet2d &	b
	)

                                                                                  {
  return _mm_cmpnge_pd(a, b);
}

References a, and b.

pcmp_lt_or_nan() [6/12]

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::pcmp_lt_or_nan	(	const Packet32h &	a,
		const Packet32h &	b
	)

                                                                                     {
  __mmask32 mask = _mm512_cmp_ph_mask(a, b, _CMP_NGE_UQ);
  return _mm512_castsi512_ph(_mm512_mask_set1_epi16(_mm512_set1_epi16(0), mask, static_cast<short>(0xffffu)));
}

References a, and b.

pcmp_lt_or_nan() [7/12]

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::pcmp_lt_or_nan	(	const Packet4d &	a,
		const Packet4d &	b
	)

                                                                                  {
  return _mm256_cmp_pd(a, b, _CMP_NGE_UQ);
}

References a, and b.

pcmp_lt_or_nan() [8/12]

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pcmp_lt_or_nan	(	const Packet4f &	a,
		const Packet4f &	b
	)

                                                                                  {
  Packet4f c = reinterpret_cast<Packet4f>(vec_cmpge(a, b));
  return vec_nor(c, c);
}

References a, b, and calibrate::c.

Referenced by packetmath_boolean_mask_ops_real(), pcmp_lt_or_nan(), plog_impl_double(), and plog_impl_float().

pcmp_lt_or_nan() [9/12]

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::pcmp_lt_or_nan	(	const Packet8bf &	a,
		const Packet8bf &	b
	)

                                                                                     {
  BF16_TO_F32_BINARY_OP_WRAPPER_BOOL(pcmp_lt_or_nan<Packet4f>, a, b);
}

References a, b, BF16_TO_F32_BINARY_OP_WRAPPER_BOOL, and pcmp_lt_or_nan< Packet4f >().

pcmp_lt_or_nan() [10/12]

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::pcmp_lt_or_nan	(	const Packet8d &	a,
		const Packet8d &	b
	)

                                                                                  {
  __mmask8 mask = _mm512_cmp_pd_mask(a, b, _CMP_NGE_UQ);
  return _mm512_castsi512_pd(_mm512_mask_set1_epi64(_mm512_setzero_epi32(), mask, 0xffffffffffffffffu));
}

References a, and b.

pcmp_lt_or_nan() [11/12]

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::pcmp_lt_or_nan	(	const Packet8f &	a,
		const Packet8f &	b
	)

                                                                                  {
  return _mm256_cmp_ps(a, b, _CMP_NGE_UQ);
}

References a, and b.

pcmp_lt_or_nan() [12/12]

template<>

EIGEN_STRONG_INLINE Packet8h Eigen::internal::pcmp_lt_or_nan	(	const Packet8h &	a,
		const Packet8h &	b
	)

                                                                                  {
  return Pack16To8(pcmp_lt_or_nan(half2float(a), half2float(b)));
}

References a, b, half2float(), Pack16To8(), and pcmp_lt_or_nan().

pcmp_lt_or_nan< Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pcmp_lt_or_nan< Packet2d >	(	const Packet2d &	a,
		const Packet2d &	b
	)

                                                                                            {
  return (Packet2d)__lsx_vfcmp_sult_d(a, b);
}

References a, and b.

pcmp_lt_or_nan< Packet2f >()

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::pcmp_lt_or_nan< Packet2f >	(	const Packet2f &	a,
		const Packet2f &	b
	)

                                                                                            {
  return vreinterpret_f32_u32(vmvn_u32(vcge_f32(a, b)));
}

References a, and b.

pcmp_lt_or_nan< Packet4bf >()

template<>

EIGEN_STRONG_INLINE Packet4bf Eigen::internal::pcmp_lt_or_nan< Packet4bf >	(	const Packet4bf &	a,
		const Packet4bf &	b
	)

                                                                                                {
  return F32MaskToBf16Mask(pcmp_lt_or_nan<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
}

References a, b, Bf16ToF32(), F32MaskToBf16Mask(), and pcmp_lt_or_nan< Packet4f >().

pcmp_lt_or_nan< Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pcmp_lt_or_nan< Packet4f >	(	const Packet4f &	a,
		const Packet4f &	b
	)

                                                                                            {
  return (Packet4f)__lsx_vfcmp_sult_s(a, b);
}

References a, and b.

Referenced by pcmp_lt_or_nan(), and pcmp_lt_or_nan< Packet4bf >().

pcmp_lt_or_nan< PacketXf >()

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::pcmp_lt_or_nan< PacketXf >	(	const PacketXf &	a,
		const PacketXf &	b
	)

                                                                                            {
  return svreinterpret_f32_u32(svdup_n_u32_z(svnot_b_z(svptrue_b32(), svcmpge_f32(svptrue_b32(), a, b)), 0xffffffffu));
}

References a, and b.

pconj() [1/41]

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::pconj ( const Packet &  a )

inline

Returns

conj(a) (coeff-wise)

                                                       {
  return numext::conj(a);
}

References a, and conj().

pconj() [2/41]

template<>

EIGEN_STRONG_INLINE Packet16bf Eigen::internal::pconj

(

const Packet16bf &

a

)

                                                          {
  return a;
}

References a.

pconj() [3/41]

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::pconj

(

const Packet16c &

a

)

                                                        {
  return a;
}

References a.

pconj() [4/41]

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::pconj

(

const Packet16f &

a

)

                                                        {
  return a;
}

References a.

pconj() [5/41]

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::pconj

(

const Packet16h &

a

)

                                                        {
  return a;
}

References a.

pconj() [6/41]

template<>

EIGEN_STRONG_INLINE Packet16i Eigen::internal::pconj

(

const Packet16i &

a

)

                                                        {
  return a;
}

References a.

pconj() [7/41]

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::pconj

(

const Packet16uc &

a

)

                                                          {
  return a;
}

References a.

pconj() [8/41]

template<>

EIGEN_STRONG_INLINE Packet1cd Eigen::internal::pconj

(

const Packet1cd &

a

)

                                                        {
  const uint64_t tmp[2] = {0x0000000000000000u, 0x8000000000000000u};
  __m128i mask = __lsx_vld(tmp, 0);
  Packet1cd res;
  res.v = (Packet2d)__lsx_vxor_v((__m128i)a.v, mask);
  return res;
}

References a, res, and tmp.

pconj() [9/41]

template<>

EIGEN_STRONG_INLINE Packet1cf Eigen::internal::pconj

(

const Packet1cf &

a

)

                                                        {
  const Packet2ui b = Packet2ui(vreinterpret_u32_f32(a.v));
  return Packet1cf(vreinterpret_f32_u32(veor_u32(b, p2ui_CONJ_XOR())));
}

References a, b, and p2ui_CONJ_XOR().

pconj() [10/41]

template<>

EIGEN_STRONG_INLINE Packet2cd Eigen::internal::pconj

(

const Packet2cd &

a

)

                                                        {
  const __m256d mask = _mm256_castsi256_pd(_mm256_set_epi32(0x80000000, 0x0, 0x0, 0x0, 0x80000000, 0x0, 0x0, 0x0));
  return Packet2cd(_mm256_xor_pd(a.v, mask));
}

References a.

pconj() [11/41]

template<>

EIGEN_STRONG_INLINE Packet2cf Eigen::internal::pconj

(

const Packet2cf &

a

)

                                                        {
  return Packet2cf(pxor<Packet4f>(a.v, reinterpret_cast<Packet4f>(p4ui_CONJ_XOR)));
}

References a, p4ui_CONJ_XOR, and pxor< Packet4f >().

Referenced by Eigen::internal::gebp_traits< std::complex< RealScalar >, std::complex< RealScalar >, ConjLhs_, ConjRhs_, Arch, PacketSize_ >::acc(), Eigen::internal::conditional_conj< Scalar, true >::packet(), packetmath(), Eigen::internal::scalar_conjugate_op< Scalar >::packetOp(), Eigen::internal::conj_if< true >::pconj(), pdiv_complex(), Eigen::internal::conj_helper< Packet, Packet, ConjLhs, ConjRhs >::pmadd(), Eigen::internal::conj_helper< Packet, Packet, true, true >::pmadd(), Eigen::internal::conj_helper< Packet, Packet, ConjLhs, ConjRhs >::pmul(), and Eigen::internal::conj_helper< Packet, Packet, true, true >::pmul().

pconj() [12/41]

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pconj

(

const Packet2d &

a

)

                                                      {
  return a;
}

References a.

pconj() [13/41]

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::pconj

(

const Packet2f &

a

)

                                                      {
  return a;
}

References a.

pconj() [14/41]

template<>

EIGEN_STRONG_INLINE Packet2i Eigen::internal::pconj

(

const Packet2i &

a

)

                                                      {
  return a;
}

References a.

pconj() [15/41]

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::pconj

(

const Packet2l &

a

)

                                                      {
  return a;
}

References a.

pconj() [16/41]

template<>

EIGEN_STRONG_INLINE Packet2ui Eigen::internal::pconj

(

const Packet2ui &

a

)

                                                        {
  return a;
}

References a.

pconj() [17/41]

template<>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::pconj

(

const Packet2ul &

a

)

                                                        {
  return a;
}

References a.

pconj() [18/41]

template<>

EIGEN_STRONG_INLINE Packet4bf Eigen::internal::pconj

(

const Packet4bf &

a

)

                                                        {
  return a;
}

References a.

pconj() [19/41]

template<>

EIGEN_STRONG_INLINE Packet4c Eigen::internal::pconj

(

const Packet4c &

a

)

                                                      {
  return a;
}

References a.

pconj() [20/41]

template<>

EIGEN_STRONG_INLINE Packet4cd Eigen::internal::pconj

(

const Packet4cd &

a

)

                                                        {
  const __m512d mask = _mm512_castsi512_pd(_mm512_set_epi32(0x80000000, 0x0, 0x0, 0x0, 0x80000000, 0x0, 0x0, 0x0,
                                                            0x80000000, 0x0, 0x0, 0x0, 0x80000000, 0x0, 0x0, 0x0));
  return Packet4cd(pxor(a.v, mask));
}

References a, and pxor().

pconj() [21/41]

template<>

EIGEN_STRONG_INLINE Packet4cf Eigen::internal::pconj

(

const Packet4cf &

a

)

                                                        {
  const __m256 mask = _mm256_castsi256_ps(_mm256_setr_epi32(0x00000000, 0x80000000, 0x00000000, 0x80000000, 0x00000000,
                                                            0x80000000, 0x00000000, 0x80000000));
  return Packet4cf(_mm256_xor_ps(a.v, mask));
}

References a.

pconj() [22/41]

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::pconj

(

const Packet4d &

a

)

                                                      {
  return a;
}

References a.

pconj() [23/41]

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pconj

(

const Packet4f &

a

)

                                                      {
  return a;
}

References a.

pconj() [24/41]

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pconj

(

const Packet4i &

a

)

                                                      {
  return a;
}

References a.

pconj() [25/41]

template<>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::pconj

(

const Packet4s &

a

)

                                                      {
  return a;
}

References a.

pconj() [26/41]

template<>

EIGEN_STRONG_INLINE Packet4uc Eigen::internal::pconj

(

const Packet4uc &

a

)

                                                        {
  return a;
}

References a.

pconj() [27/41]

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::pconj

(

const Packet4ui &

a

)

                                                        {
  return a;
}

References a.

pconj() [28/41]

template<>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::pconj

(

const Packet4us &

a

)

                                                        {
  return a;
}

References a.

pconj() [29/41]

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::pconj

(

const Packet8bf &

a

)

                                                        {
  return a;
}

References a.

pconj() [30/41]

template<>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::pconj

(

const Packet8c &

a

)

                                                      {
  return a;
}

References a.

pconj() [31/41]

template<>

EIGEN_STRONG_INLINE Packet8cf Eigen::internal::pconj

(

const Packet8cf &

a

)

                                                        {
  const __m512 mask = _mm512_castsi512_ps(_mm512_setr_epi32(
      0x00000000, 0x80000000, 0x00000000, 0x80000000, 0x00000000, 0x80000000, 0x00000000, 0x80000000, 0x00000000,
      0x80000000, 0x00000000, 0x80000000, 0x00000000, 0x80000000, 0x00000000, 0x80000000));
  return Packet8cf(pxor(a.v, mask));
}

References a, and pxor().

pconj() [32/41]

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::pconj

(

const Packet8d &

a

)

                                                      {
  return a;
}

References a.

pconj() [33/41]

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::pconj

(

const Packet8f &

a

)

                                                      {
  return a;
}

References a.

pconj() [34/41]

template<>

EIGEN_STRONG_INLINE Packet8h Eigen::internal::pconj

(

const Packet8h &

a

)

                                                      {
  return a;
}

References a.

pconj() [35/41]

template<>

EIGEN_STRONG_INLINE Packet8i Eigen::internal::pconj

(

const Packet8i &

a

)

                                                      {
  return a;
}

References a.

pconj() [36/41]

template<>

EIGEN_STRONG_INLINE Packet8l Eigen::internal::pconj

(

const Packet8l &

a

)

                                                      {
  return a;
}

References a.

pconj() [37/41]

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::pconj

(

const Packet8s &

a

)

                                                      {
  return a;
}

References a.

pconj() [38/41]

template<>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::pconj

(

const Packet8uc &

a

)

                                                        {
  return a;
}

References a.

pconj() [39/41]

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::pconj

(

const Packet8us &

a

)

                                                        {
  return a;
}

References a.

pconj() [40/41]

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::pconj

(

const PacketXf &

a

)

                                                      {
  return a;
}

References a.

pconj() [41/41]

template<>

EIGEN_STRONG_INLINE PacketXi Eigen::internal::pconj

(

const PacketXi &

a

)

                                                      {
  return a;
}

References a.

pconj< Packet32h >()

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::pconj< Packet32h >

(

const Packet32h &

a

)

                                                                   {
  return a;
}

References a.

pcos()

template<typename Packet >

EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::pcos

(

const Packet &

a

)

Returns

the cosine of a (coeff-wise)

                                                                                  {
  EIGEN_USING_STD(cos);
  return cos(a);
}

References a, cos(), and EIGEN_USING_STD.

Referenced by packetmath_real(), Eigen::internal::scalar_cos_op< Scalar >::packetOp(), pcos< Packet32h >(), Eigen::internal::generic_j0< T, float >::run(), Eigen::internal::generic_j0< T, double >::run(), Eigen::internal::generic_y0< T, double >::run(), Eigen::internal::generic_j1< T, float >::run(), Eigen::internal::generic_j1< T, double >::run(), and Eigen::internal::generic_y1< T, double >::run().

pcos< Packet16h >()

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::pcos< Packet16h >

(

const Packet16h &

)

pcos< Packet32h >()

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::pcos< Packet32h >

(

const Packet32h &

a

)

                                                                  {
  Packet16h low;
  Packet16h high;
  extract2Packet16h(a, low, high);
 
  Packet16h lowOut = pcos(low);
  Packet16h highOut = pcos(high);
 
  return combine2Packet16h(lowOut, highOut);
}

References a, combine2Packet16h(), extract2Packet16h(), and pcos().

pcos< Packet4f >()

template<>

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet4f Eigen::internal::pcos< Packet4f >

(

const Packet4f &

x

)

                                                                                               {
  return psincos_inner_msa_float</* sine */ false>(x);
}

References psincos_inner_msa_float(), and plotDoE::x.

pcos< Packet8bf >()

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::pcos< Packet8bf >

(

const Packet8bf &

a

)

                                                                  {
  BF16_TO_F32_UNARY_OP_WRAPPER(pcos_float, a);
}

References a, BF16_TO_F32_UNARY_OP_WRAPPER, and pcos_float().

pcos< PacketXf >()

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::pcos< PacketXf >

(

const PacketXf &

x

)

                                                               {
  return pcos_float(x);
}

References pcos_float(), and plotDoE::x.

pcos_double()

template<typename Packet >

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::pcos_double

(

const Packet &

x

)

Returns

cos(x) for double precision float

                                                                                        {
  return psincos_double<false>(x);
}

References plotDoE::x.

pcos_float()

template<typename Packet >

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::pcos_float

(

const Packet &

x

)

Returns

cos(x) for single precision float

                                                                                       {
  return psincos_float<false>(x);
}

References plotDoE::x.

Referenced by pcos< Packet8bf >(), and pcos< PacketXf >().

pcosh()

template<typename Packet >

EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::pcosh

(

const Packet &

a

)

Returns

the hyperbolic cosine of a (coeff-wise)

                                                                                   {
  EIGEN_USING_STD(cosh);
  return cosh(a);
}

References a, Eigen::bfloat16_impl::cosh(), and EIGEN_USING_STD.

Referenced by Eigen::internal::scalar_cosh_op< Scalar >::packetOp().

pcplxflip() [1/3]

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::pcplxflip ( const Packet &  a )

inline

Returns

a with real and imaginary part flipped (for complex type only)

                                                           {
  return Packet(numext::imag(a), numext::real(a));
}

References a, and imag().

pcplxflip() [2/3]

EIGEN_STRONG_INLINE Packet1cd Eigen::internal::pcplxflip

(

const Packet1cd &

x

)

                                                               {
  return Packet1cd(preverse(Packet2d(x.v)));
}

References preverse(), and plotDoE::x.

pcplxflip() [3/3]

EIGEN_STRONG_INLINE Packet2cf Eigen::internal::pcplxflip

(

const Packet2cf &

x

)

                                                               {
  return Packet2cf(vec4f_swizzle1(x.v, 1, 0, 3, 2));
}

References vec4f_swizzle1(), and plotDoE::x.

Referenced by Eigen::internal::gebp_traits< std::complex< RealScalar >, std::complex< RealScalar >, ConjLhs_, ConjRhs_, Arch, PacketSize_ >::acc(), packetmath_complex(), pcplxconjflip(), pcplxflipconj(), pcplxflipnegate(), pdiv_complex(), pexp_complex(), phypot_complex(), plog_complex(), psqrt_complex(), and Eigen::internal::psign_impl< Packet, std::enable_if_t< NumTraits< typename unpacket_traits< Packet >::type >::IsComplex &&unpacket_traits< Packet >::vectorizable > >::run().

pcplxflip< Packet1cf >()

template<>

EIGEN_STRONG_INLINE Packet1cf Eigen::internal::pcplxflip< Packet1cf >

(

const Packet1cf &

a

)

                                                                       {
  return Packet1cf(vrev64_f32(a.v));
}

References a.

pcplxflip< Packet2cd >()

template<>

EIGEN_STRONG_INLINE Packet2cd Eigen::internal::pcplxflip< Packet2cd >

(

const Packet2cd &

x

)

                                                                       {
  return Packet2cd(_mm256_shuffle_pd(x.v, x.v, 0x5));
}

References plotDoE::x.

pcplxflip< Packet2cf >()

template<>

EIGEN_STRONG_INLINE Packet2cf Eigen::internal::pcplxflip< Packet2cf >

(

const Packet2cf &

x

)

                                                                       {
  return Packet2cf(vec_perm(x.v, x.v, p16uc_COMPLEX32_REV));
}

References p16uc_COMPLEX32_REV, and plotDoE::x.

pcplxflip< Packet4cd >()

template<>

EIGEN_STRONG_INLINE Packet4cd Eigen::internal::pcplxflip< Packet4cd >

(

const Packet4cd &

x

)

                                                                       {
  return Packet4cd(_mm512_permute_pd(x.v, 0x55));
}

References plotDoE::x.

pcplxflip< Packet4cf >()

template<>

EIGEN_STRONG_INLINE Packet4cf Eigen::internal::pcplxflip< Packet4cf >

(

const Packet4cf &

x

)

                                                                       {
  return Packet4cf(_mm256_shuffle_ps(x.v, x.v, _MM_SHUFFLE(2, 3, 0, 1)));
}

References plotDoE::x.

pcplxflip< Packet8cf >()

template<>

EIGEN_STRONG_INLINE Packet8cf Eigen::internal::pcplxflip< Packet8cf >

(

const Packet8cf &

x

)

                                                                       {
  return Packet8cf(_mm512_shuffle_ps(x.v, x.v, _MM_SHUFFLE(2, 3, 0, 1)));
}

References plotDoE::x.

pdigamma()

template<typename Packet >

EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::pdigamma

(

const Packet &

a

)

Returns

the derivative of lgamma, psi(a) (coeff-wise)

                                                                                      {
  using numext::digamma;
  return digamma(a);
}

References a.

Referenced by Eigen::internal::scalar_digamma_op< Scalar >::packetOp().

pdiv()

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::pdiv ( const Packet &  a, const Packet &  b  )

inline

Returns

a / b (coeff-wise)

                                                                       {
  return a / b;
}

References a, and b.

Referenced by Eigen::internal::div_assign_op< DstScalar, SrcScalar >::assignPacket(), doubleword_div_fp(), erf_over_x_double_small(), erfc_double_large(), Eigen::TensorSycl::internal::OpDefiner< Eigen::internal::MeanReducer< CoeffReturnType >, CoeffReturnType, Index, true >::finalise_op(), Eigen::internal::MeanReducer< T >::finalizePacket(), generic_expm1(), generic_log1p(), generic_ndtri_gt_exp_neg_two(), generic_ndtri_lt_exp_neg_two(), Eigen::internal::Packet1cd::operator/=(), packetmath(), Eigen::internal::scalar_logistic_op< float >::packetOp(), Eigen::internal::scalar_quotient_op< LhsScalar, RhsScalar >::packetOp(), Eigen::internal::scalar_logistic_op_impl< T, EnableIf >::packetOp(), patanh_double(), patanh_float(), pdiv< Packet16h >(), pdiv< Packet8h >(), pdiv_complex(), pexp< Packet2d >(), pexp_double(), phypot_complex(), plog_impl_double(), plog_impl_float(), preciprocal(), psincos_double(), psqrt_complex(), ptanh< Packet4f >(), ptanh_double(), ptanh_float(), Eigen::internal::compute_inverse_size4< Architecture::Target, double, MatrixType, ResultType >::run(), Eigen::internal::psign_impl< Packet, std::enable_if_t< NumTraits< typename unpacket_traits< Packet >::type >::IsComplex &&unpacket_traits< Packet >::vectorizable > >::run(), Eigen::internal::patan_reduced< Scalar >::run(), Eigen::internal::unary_pow::reciprocate< Packet, ScalarExponent, ReciprocateIfExponentIsNegative >::run(), Eigen::internal::generic_i0e< T, float >::run(), Eigen::internal::generic_i0e< T, double >::run(), Eigen::internal::generic_i1e< T, float >::run(), Eigen::internal::generic_i1e< T, double >::run(), Eigen::internal::generic_k0e< T, float >::run(), Eigen::internal::generic_k0e< T, double >::run(), Eigen::internal::generic_k0< T, float >::run(), Eigen::internal::generic_k0< T, double >::run(), Eigen::internal::generic_k1e< T, float >::run(), Eigen::internal::generic_k1e< T, double >::run(), Eigen::internal::generic_k1< T, float >::run(), Eigen::internal::generic_k1< T, double >::run(), Eigen::internal::generic_j0< T, float >::run(), Eigen::internal::generic_j0< T, double >::run(), Eigen::internal::generic_y0< T, float >::run(), Eigen::internal::generic_y0< T, double >::run(), Eigen::internal::generic_j1< T, float >::run(), Eigen::internal::generic_j1< T, double >::run(), Eigen::internal::generic_y1< T, float >::run(), Eigen::internal::generic_y1< T, double >::run(), Eigen::internal::generic_fast_erf< Scalar >::run(), and Eigen::internal::generic_fast_erfc< Scalar >::run().

pdiv< Packet16bf >()

template<>

EIGEN_STRONG_INLINE Packet16bf Eigen::internal::pdiv< Packet16bf >	(	const Packet16bf &	a,
		const Packet16bf &	b
	)

                                                                                          {
  return F32ToBf16(pdiv<Packet16f>(Bf16ToF32(a), Bf16ToF32(b)));
}

References a, b, Bf16ToF32(), F32ToBf16(), and pdiv< Packet16f >().

pdiv< Packet16c >()

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::pdiv< Packet16c >	(	const Packet16c &	,
		const Packet16c &
	)

                                                                                      {
  eigen_assert(false && "packet integer division are not supported by NEON");
  return pset1<Packet16c>(0);
}

References eigen_assert, and pset1< Packet16c >().

pdiv< Packet16f >()

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::pdiv< Packet16f >	(	const Packet16f &	a,
		const Packet16f &	b
	)

                                                                                      {
  return _mm512_div_ps(a, b);
}

References a, and b.

Referenced by pdiv< Packet16bf >().

pdiv< Packet16h >()

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::pdiv< Packet16h >	(	const Packet16h &	a,
		const Packet16h &	b
	)

                                                                                      {
  Packet16f af = half2float(a);
  Packet16f bf = half2float(b);
  Packet16f rf = pdiv(af, bf);
  return float2half(rf);
}

References a, b, float2half(), half2float(), and pdiv().

pdiv< Packet16i >()

template<>

EIGEN_STRONG_INLINE Packet16i Eigen::internal::pdiv< Packet16i >	(	const Packet16i &	a,
		const Packet16i &	b
	)

                                                                                      {
  Packet8i q_lo = pdiv<Packet8i>(_mm512_extracti64x4_epi64(a, 0), _mm512_extracti64x4_epi64(b, 0));
  Packet8i q_hi = pdiv<Packet8i>(_mm512_extracti64x4_epi64(a, 1), _mm512_extracti64x4_epi64(b, 1));
  return _mm512_inserti64x4(_mm512_castsi256_si512(q_lo), q_hi, 1);
}

References a, b, and pdiv< Packet8i >().

pdiv< Packet16uc >()

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::pdiv< Packet16uc >	(	const Packet16uc &	a,
		const Packet16uc &	b
	)

                                                                                          {
  return __lsx_vdiv_bu(a, b);
}

References a, and b.

pdiv< Packet1cd >()

template<>

EIGEN_STRONG_INLINE Packet1cd Eigen::internal::pdiv< Packet1cd >	(	const Packet1cd &	a,
		const Packet1cd &	b
	)

                                                                                      {
  return pdiv_complex(a, b);
}

References a, b, and pdiv_complex().

pdiv< Packet1cf >()

template<>

EIGEN_STRONG_INLINE Packet1cf Eigen::internal::pdiv< Packet1cf >	(	const Packet1cf &	a,
		const Packet1cf &	b
	)

                                                                                      {
  return pdiv_complex(a, b);
}

References a, b, and pdiv_complex().

pdiv< Packet2cd >()

template<>

EIGEN_STRONG_INLINE Packet2cd Eigen::internal::pdiv< Packet2cd >	(	const Packet2cd &	a,
		const Packet2cd &	b
	)

                                                                                      {
  return pdiv_complex(a, b);
}

References a, b, and pdiv_complex().

pdiv< Packet2cf >()

template<>

EIGEN_STRONG_INLINE Packet2cf Eigen::internal::pdiv< Packet2cf >	(	const Packet2cf &	a,
		const Packet2cf &	b
	)

                                                                                      {
  return pdiv_complex(a, b);
}

References a, b, and pdiv_complex().

pdiv< Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pdiv< Packet2d >	(	const Packet2d &	a,
		const Packet2d &	b
	)

                                                                                  {
  return __lsx_vfdiv_d(a, b);
}

References a, and b.

pdiv< Packet2f >()

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::pdiv< Packet2f >	(	const Packet2f &	a,
		const Packet2f &	b
	)

                                                                                  {
  return pdiv_float_common(a, b);
}

References a, b, and pdiv_float_common().

pdiv< Packet2i >()

template<>

EIGEN_STRONG_INLINE Packet2i Eigen::internal::pdiv< Packet2i >	(	const Packet2i &	,
		const Packet2i &
	)

                                                                                  {
  eigen_assert(false && "packet integer division are not supported by NEON");
  return pset1<Packet2i>(0);
}

References eigen_assert, and pset1< Packet2i >().

pdiv< Packet2l >()

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::pdiv< Packet2l >	(	const Packet2l &	a,
		const Packet2l &	b
	)

                                                                                  {
  return __lsx_vdiv_d(a, b);
}

References a, and b.

pdiv< Packet2ui >()

template<>

EIGEN_STRONG_INLINE Packet2ui Eigen::internal::pdiv< Packet2ui >	(	const Packet2ui &	,
		const Packet2ui &
	)

                                                                                      {
  eigen_assert(false && "packet integer division are not supported by NEON");
  return pset1<Packet2ui>(0);
}

References eigen_assert, and pset1< Packet2ui >().

pdiv< Packet2ul >()

template<>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::pdiv< Packet2ul >	(	const Packet2ul &	a,
		const Packet2ul &	b
	)

                                                                                      {
  return __lsx_vdiv_du(a, b);
}

References a, and b.

pdiv< Packet32h >()

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::pdiv< Packet32h >	(	const Packet32h &	a,
		const Packet32h &	b
	)

                                                                                      {
  return _mm512_div_ph(a, b);
}

References a, and b.

pdiv< Packet4bf >()

template<>

EIGEN_STRONG_INLINE Packet4bf Eigen::internal::pdiv< Packet4bf >	(	const Packet4bf &	a,
		const Packet4bf &	b
	)

                                                                                      {
  return F32ToBf16(pdiv<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
}

References a, b, Bf16ToF32(), F32ToBf16(), and pdiv< Packet4f >().

pdiv< Packet4c >()

template<>

EIGEN_STRONG_INLINE Packet4c Eigen::internal::pdiv< Packet4c >	(	const Packet4c &	,
		const Packet4c &
	)

                                                                                  {
  eigen_assert(false && "packet integer division are not supported by NEON");
  return pset1<Packet4c>(0);
}

References eigen_assert, and pset1< Packet4c >().

pdiv< Packet4cd >()

template<>

EIGEN_STRONG_INLINE Packet4cd Eigen::internal::pdiv< Packet4cd >	(	const Packet4cd &	a,
		const Packet4cd &	b
	)

                                                                                      {
  return pdiv_complex(a, b);
}

References a, b, and pdiv_complex().

pdiv< Packet4cf >()

template<>

EIGEN_STRONG_INLINE Packet4cf Eigen::internal::pdiv< Packet4cf >	(	const Packet4cf &	a,
		const Packet4cf &	b
	)

                                                                                      {
  return pdiv_complex(a, b);
}

References a, b, and pdiv_complex().

pdiv< Packet4d >()

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::pdiv< Packet4d >	(	const Packet4d &	a,
		const Packet4d &	b
	)

                                                                                  {
  return _mm256_div_pd(a, b);
}

References a, and b.

pdiv< Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pdiv< Packet4f >	(	const Packet4f &	a,
		const Packet4f &	b
	)

                                                                                  {
#ifndef __VSX__  // VSX actually provides a div instruction
  Packet4f t, y_0, y_1;
 
  // Altivec does not offer a divide instruction, we have to do a reciprocal approximation
  y_0 = vec_re(b);
 
  // Do one Newton-Raphson iteration to get the needed accuracy
  t = vec_nmsub(y_0, b, p4f_ONE);
  y_1 = vec_madd(y_0, t, y_0);
 
  return vec_madd(a, y_1, p4f_MZERO);
#else
  return vec_div(a, b);
#endif
}

References a, b, p4f_MZERO, p4f_ONE, and plotPSD::t.

Referenced by pdiv< Packet4bf >(), and pdiv< Packet8bf >().

pdiv< Packet4i >()

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pdiv< Packet4i >	(	const Packet4i &	a,
		const Packet4i &	b
	)

                                                                                  {
#if defined(_ARCH_PWR10) && (EIGEN_COMP_LLVM || EIGEN_GNUC_STRICT_AT_LEAST(11, 0, 0))
  return vec_div(a, b);
#else
  EIGEN_UNUSED_VARIABLE(a);
  EIGEN_UNUSED_VARIABLE(b);
  eigen_assert(false && "packet integer division are not supported by AltiVec");
  return pset1<Packet4i>(0);
#endif
}

References a, b, eigen_assert, EIGEN_UNUSED_VARIABLE, and pset1< Packet4i >().

Referenced by pdiv< Packet8i >().

pdiv< Packet4s >()

template<>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::pdiv< Packet4s >	(	const Packet4s &	,
		const Packet4s &
	)

                                                                                  {
  eigen_assert(false && "packet integer division are not supported by NEON");
  return pset1<Packet4s>(0);
}

References eigen_assert, and pset1< Packet4s >().

pdiv< Packet4uc >()

template<>

EIGEN_STRONG_INLINE Packet4uc Eigen::internal::pdiv< Packet4uc >	(	const Packet4uc &	,
		const Packet4uc &
	)

                                                                                      {
  eigen_assert(false && "packet integer division are not supported by NEON");
  return pset1<Packet4uc>(0);
}

References eigen_assert, and pset1< Packet4uc >().

pdiv< Packet4ui >()

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::pdiv< Packet4ui >	(	const Packet4ui &	a,
		const Packet4ui &	b
	)

                                                                                      {
  return __lsx_vdiv_wu(a, b);
}

References a, and b.

pdiv< Packet4us >()

template<>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::pdiv< Packet4us >	(	const Packet4us &	,
		const Packet4us &
	)

                                                                                      {
  eigen_assert(false && "packet integer division are not supported by NEON");
  return pset1<Packet4us>(0);
}

References eigen_assert, and pset1< Packet4us >().

pdiv< Packet8bf >()

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::pdiv< Packet8bf >	(	const Packet8bf &	a,
		const Packet8bf &	b
	)

                                                                                      {
  BF16_TO_F32_BINARY_OP_WRAPPER(pdiv<Packet4f>, a, b);
}

References a, b, BF16_TO_F32_BINARY_OP_WRAPPER, and pdiv< Packet4f >().

pdiv< Packet8c >()

template<>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::pdiv< Packet8c >	(	const Packet8c &	,
		const Packet8c &
	)

                                                                                  {
  eigen_assert(false && "packet integer division are not supported by NEON");
  return pset1<Packet8c>(0);
}

References eigen_assert, and pset1< Packet8c >().

pdiv< Packet8cf >()

template<>

EIGEN_STRONG_INLINE Packet8cf Eigen::internal::pdiv< Packet8cf >	(	const Packet8cf &	a,
		const Packet8cf &	b
	)

                                                                                      {
  return pdiv_complex(a, b);
}

References a, b, and pdiv_complex().

pdiv< Packet8d >()

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::pdiv< Packet8d >	(	const Packet8d &	a,
		const Packet8d &	b
	)

                                                                                  {
  return _mm512_div_pd(a, b);
}

References a, and b.

pdiv< Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::pdiv< Packet8f >	(	const Packet8f &	a,
		const Packet8f &	b
	)

                                                                                  {
  return _mm256_div_ps(a, b);
}

References a, and b.

pdiv< Packet8h >()

template<>

EIGEN_STRONG_INLINE Packet8h Eigen::internal::pdiv< Packet8h >	(	const Packet8h &	a,
		const Packet8h &	b
	)

                                                                                  {
  Packet8f af = half2float(a);
  Packet8f bf = half2float(b);
  Packet8f rf = pdiv(af, bf);
  return float2half(rf);
}

References a, b, float2half(), half2float(), and pdiv().

pdiv< Packet8i >()

template<>

EIGEN_STRONG_INLINE Packet8i Eigen::internal::pdiv< Packet8i >	(	const Packet8i &	a,
		const Packet8i &	b
	)

                                                                                  {
#ifdef EIGEN_VECTORIZE_AVX512
  return _mm512_cvttpd_epi32(_mm512_div_pd(_mm512_cvtepi32_pd(a), _mm512_cvtepi32_pd(b)));
#else
  Packet4i lo = pdiv<Packet4i>(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0));
  Packet4i hi = pdiv<Packet4i>(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1));
  return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), hi, 1);
#endif
}

References a, b, and pdiv< Packet4i >().

Referenced by pdiv< Packet16i >().

pdiv< Packet8s >()

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::pdiv< Packet8s >	(	const Packet8s &	a,
		const Packet8s &	b
	)

                                                                                  {
  return __lsx_vdiv_h(a, b);
}

References a, and b.

pdiv< Packet8uc >()

template<>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::pdiv< Packet8uc >	(	const Packet8uc &	,
		const Packet8uc &
	)

                                                                                      {
  eigen_assert(false && "packet integer division are not supported by NEON");
  return pset1<Packet8uc>(0);
}

References eigen_assert, and pset1< Packet8uc >().

pdiv< Packet8us >()

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::pdiv< Packet8us >	(	const Packet8us &	a,
		const Packet8us &	b
	)

                                                                                      {
  return __lsx_vdiv_hu(a, b);
}

References a, and b.

pdiv< PacketXf >()

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::pdiv< PacketXf >	(	const PacketXf &	a,
		const PacketXf &	b
	)

                                                                                  {
  return svdiv_f32_x(svptrue_b32(), a, b);
}

References a, and b.

pdiv< PacketXi >()

template<>

EIGEN_STRONG_INLINE PacketXi Eigen::internal::pdiv< PacketXi >	(	const PacketXi &	a,
		const PacketXi &	b
	)

                                                                                  {
  return svdiv_s32_x(svptrue_b32(), a, b);
}

References a, and b.

pdiv_complex()

template<typename Packet >

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::pdiv_complex	(	const Packet &	x,
		const Packet &	y
	)

Returns

x / y for complex types

                                                                                                          {
  typedef typename unpacket_traits<Packet>::as_real RealPacket;
  // In the following we annotate the code for the case where the inputs
  // are a pair length-2 SIMD vectors representing a single pair of complex
  // numbers x = a + i*b, y = c + i*d.
  const RealPacket y_abs = pabs(y.v);                        // |c|, |d|
  const RealPacket y_abs_flip = pcplxflip(Packet(y_abs)).v;  // |d|, |c|
  const RealPacket y_max = pmax(y_abs, y_abs_flip);          // max(|c|, |d|), max(|c|, |d|)
  const RealPacket y_scaled = pdiv(y.v, y_max);              // c / max(|c|, |d|), d / max(|c|, |d|)
  // Compute scaled denominator.
  const RealPacket y_scaled_sq = pmul(y_scaled, y_scaled);  // c'**2, d'**2
  const RealPacket denom = padd(y_scaled_sq, pcplxflip(Packet(y_scaled_sq)).v);
  Packet result_scaled = pmul(x, pconj(Packet(y_scaled)));  // a * c' + b * d', -a * d + b * c
  // Divide elementwise by denom.
  result_scaled = Packet(pdiv(result_scaled.v, denom));
  // Rescale result
  return Packet(pdiv(result_scaled.v, y_max));
}

References pabs(), padd(), pconj(), pcplxflip(), pdiv(), pmax(), pmul(), v, Eigen::internal::Packet2cf::v, plotDoE::x, and y.

Referenced by Eigen::internal::Packet2cf::operator/(), pdiv< Packet1cd >(), pdiv< Packet1cf >(), pdiv< Packet2cd >(), pdiv< Packet2cf >(), pdiv< Packet4cd >(), pdiv< Packet4cf >(), and pdiv< Packet8cf >().

pdiv_float_common()

template<typename Packet >

EIGEN_STRONG_INLINE Packet Eigen::internal::pdiv_float_common	(	const Packet &	a,
		const Packet &	b
	)

                                                                               {
  // if b is large, NEON intrinsics will flush preciprocal(b) to zero
  // avoid underflow with the following manipulation:
  // a / b = f * (a * reciprocal(f * b))
 
  const Packet cst_one = pset1<Packet>(1.0f);
  const Packet cst_quarter = pset1<Packet>(0.25f);
  const Packet cst_thresh = pset1<Packet>(NumTraits<float>::highest() / 4.0f);
 
  Packet b_will_underflow = pcmp_le(cst_thresh, pabs(b));
  Packet f = pselect(b_will_underflow, cst_quarter, cst_one);
  Packet result = pmul(f, pmul(a, preciprocal(pmul(b, f))));
  return result;
}

References a, b, f(), pabs(), pcmp_le(), pmul(), preciprocal(), and pselect().

Referenced by pdiv< Packet2f >().

perf()

template<typename Packet >

EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::perf

(

const Packet &

a

)

Returns

the erf(a) (coeff-wise)

                                                                                  {
  using numext::erf;
  return erf(a);
}

References a.

Referenced by packetmath_real(), and Eigen::internal::scalar_erf_op< Scalar >::packetOp().

perfc()

template<typename Packet >

EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::perfc

(

const Packet &

a

)

Returns

the erfc(a) (coeff-wise)

                                                                                   {
  using numext::erfc;
  return erfc(a);
}

References a.

Referenced by packetmath_real(), and Eigen::internal::scalar_erfc_op< Scalar >::packetOp().

permute_symm_to_fullsymm()

template<int Mode, bool NonHermitian, typename MatrixType , int DestOrder>

void Eigen::internal::permute_symm_to_fullsymm	(	const MatrixType &	mat,
		SparseMatrix< typename MatrixType::Scalar, DestOrder, typename MatrixType::StorageIndex > &	_dest,
		const typename MatrixType::StorageIndex *	perm = 0
	)

                                                 {
  typedef typename MatrixType::StorageIndex StorageIndex;
  typedef typename MatrixType::Scalar Scalar;
  typedef SparseMatrix<Scalar, DestOrder, StorageIndex> Dest;
  typedef Matrix<StorageIndex, Dynamic, 1> VectorI;
  typedef evaluator<MatrixType> MatEval;
  typedef typename evaluator<MatrixType>::InnerIterator MatIterator;
 
  MatEval matEval(mat);
  Dest& dest(_dest.derived());
  enum { StorageOrderMatch = int(Dest::IsRowMajor) == int(MatrixType::IsRowMajor) };
 
  Index size = mat.rows();
  VectorI count;
  count.resize(size);
  count.setZero();
  dest.resize(size, size);
  for (Index j = 0; j < size; ++j) {
    Index jp = perm ? perm[j] : j;
    for (MatIterator it(matEval, j); it; ++it) {
      Index i = it.index();
      Index r = it.row();
      Index c = it.col();
      Index ip = perm ? perm[i] : i;
      if (Mode == int(Upper | Lower))
        count[StorageOrderMatch ? jp : ip]++;
      else if (r == c)
        count[ip]++;
      else if ((Mode == Lower && r > c) || (Mode == Upper && r < c)) {
        count[ip]++;
        count[jp]++;
      }
    }
  }
  Index nnz = count.sum();
 
  // reserve space
  dest.resizeNonZeros(nnz);
  dest.outerIndexPtr()[0] = 0;
  for (Index j = 0; j < size; ++j) dest.outerIndexPtr()[j + 1] = dest.outerIndexPtr()[j] + count[j];
  for (Index j = 0; j < size; ++j) count[j] = dest.outerIndexPtr()[j];
 
  // copy data
  for (StorageIndex j = 0; j < size; ++j) {
    for (MatIterator it(matEval, j); it; ++it) {
      StorageIndex i = internal::convert_index<StorageIndex>(it.index());
      Index r = it.row();
      Index c = it.col();
 
      StorageIndex jp = perm ? perm[j] : j;
      StorageIndex ip = perm ? perm[i] : i;
 
      if (Mode == int(Upper | Lower)) {
        Index k = count[StorageOrderMatch ? jp : ip]++;
        dest.innerIndexPtr()[k] = StorageOrderMatch ? ip : jp;
        dest.valuePtr()[k] = it.value();
      } else if (r == c) {
        Index k = count[ip]++;
        dest.innerIndexPtr()[k] = ip;
        dest.valuePtr()[k] = it.value();
      } else if (((Mode & Lower) == Lower && r > c) || ((Mode & Upper) == Upper && r < c)) {
        if (!StorageOrderMatch) std::swap(ip, jp);
        Index k = count[jp]++;
        dest.innerIndexPtr()[k] = ip;
        dest.valuePtr()[k] = it.value();
        k = count[ip]++;
        dest.innerIndexPtr()[k] = jp;
        dest.valuePtr()[k] = (NonHermitian ? it.value() : numext::conj(it.value()));
      }
    }
  }
}

References calibrate::c, conj(), Eigen::SparseMatrixBase< Derived >::derived(), i, int(), j, k, Eigen::Lower, UniformPSDSelfTest::r, Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::rows(), size, swap(), and Eigen::Upper.

permute_symm_to_symm() [1/2]

template<int SrcMode, int DstMode, bool NonHermitian, typename MatrixType , int DestOrder>

void Eigen::internal::permute_symm_to_symm	(	const MatrixType &	mat,
		SparseMatrix< typename MatrixType::Scalar, DestOrder, typename MatrixType::StorageIndex > &	_dest,
		const typename MatrixType::StorageIndex *	perm = 0
	)

permute_symm_to_symm() [2/2]

template<int SrcMode_, int DstMode_, bool NonHermitian, typename MatrixType , int DstOrder>

void Eigen::internal::permute_symm_to_symm	(	const MatrixType &	mat,
		SparseMatrix< typename MatrixType::Scalar, DstOrder, typename MatrixType::StorageIndex > &	_dest,
		const typename MatrixType::StorageIndex *	perm
	)

                                                                       {
  typedef typename MatrixType::StorageIndex StorageIndex;
  typedef typename MatrixType::Scalar Scalar;
  SparseMatrix<Scalar, DstOrder, StorageIndex>& dest(_dest.derived());
  typedef Matrix<StorageIndex, Dynamic, 1> VectorI;
  typedef evaluator<MatrixType> MatEval;
  typedef typename evaluator<MatrixType>::InnerIterator MatIterator;
 
  enum {
    SrcOrder = MatrixType::IsRowMajor ? RowMajor : ColMajor,
    StorageOrderMatch = int(SrcOrder) == int(DstOrder),
    DstMode = DstOrder == RowMajor ? (DstMode_ == Upper ? Lower : Upper) : DstMode_,
    SrcMode = SrcOrder == RowMajor ? (SrcMode_ == Upper ? Lower : Upper) : SrcMode_
  };
 
  MatEval matEval(mat);
 
  Index size = mat.rows();
  VectorI count(size);
  count.setZero();
  dest.resize(size, size);
  for (StorageIndex j = 0; j < size; ++j) {
    StorageIndex jp = perm ? perm[j] : j;
    for (MatIterator it(matEval, j); it; ++it) {
      StorageIndex i = it.index();
      if ((int(SrcMode) == int(Lower) && i < j) || (int(SrcMode) == int(Upper) && i > j)) continue;
 
      StorageIndex ip = perm ? perm[i] : i;
      count[int(DstMode) == int(Lower) ? (std::min)(ip, jp) : (std::max)(ip, jp)]++;
    }
  }
  dest.outerIndexPtr()[0] = 0;
  for (Index j = 0; j < size; ++j) dest.outerIndexPtr()[j + 1] = dest.outerIndexPtr()[j] + count[j];
  dest.resizeNonZeros(dest.outerIndexPtr()[size]);
  for (Index j = 0; j < size; ++j) count[j] = dest.outerIndexPtr()[j];
 
  for (StorageIndex j = 0; j < size; ++j) {
    for (MatIterator it(matEval, j); it; ++it) {
      StorageIndex i = it.index();
      if ((int(SrcMode) == int(Lower) && i < j) || (int(SrcMode) == int(Upper) && i > j)) continue;
 
      StorageIndex jp = perm ? perm[j] : j;
      StorageIndex ip = perm ? perm[i] : i;
 
      Index k = count[int(DstMode) == int(Lower) ? (std::min)(ip, jp) : (std::max)(ip, jp)]++;
      dest.innerIndexPtr()[k] = int(DstMode) == int(Lower) ? (std::max)(ip, jp) : (std::min)(ip, jp);
 
      if (!StorageOrderMatch) std::swap(ip, jp);
      if (((int(DstMode) == int(Lower) && ip < jp) || (int(DstMode) == int(Upper) && ip > jp)))
        dest.valuePtr()[k] = (NonHermitian ? it.value() : numext::conj(it.value()));
      else
        dest.valuePtr()[k] = it.value();
    }
  }
}

References Eigen::ColMajor, conj(), Eigen::SparseMatrixBase< Derived >::derived(), i, Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::innerIndexPtr(), int(), j, k, Eigen::Lower, max, min, Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::outerIndexPtr(), Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::resize(), Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::resizeNonZeros(), Eigen::RowMajor, Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::rows(), size, swap(), Eigen::Upper, and Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::valuePtr().

peven_mask() [1/14]

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::peven_mask ( const Packet &  )

inline

Returns

a packet with constant coefficients a, e.g.: (x, 0, x, 0), where x is the value of all 1-bits.

                                                            {
  typedef typename unpacket_traits<Packet>::type Scalar;
  const size_t n = unpacket_traits<Packet>::size;
  EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar elements[n];
  for (size_t i = 0; i < n; ++i) {
    memset(elements + i, ((i & 1) == 0 ? 0xff : 0), sizeof(Scalar));
  }
  return ploadu<Packet>(elements);
}

References EIGEN_ALIGN_TO_BOUNDARY, i, and n.

peven_mask() [2/14]

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::peven_mask

(

const Packet16f &

)

                                                             {
  return _mm512_castsi512_ps(_mm512_set_epi32(0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1));
}

peven_mask() [3/14]

template<>

EIGEN_STRONG_INLINE Packet16i Eigen::internal::peven_mask

(

const Packet16i &

)

                                                             {
  return _mm512_set_epi32(0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1);
}

peven_mask() [4/14]

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::peven_mask

(

const Packet2d &

)

                                                           {
  return _mm_castsi128_pd(_mm_set_epi32(0, 0, -1, -1));
}

peven_mask() [5/14]

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::peven_mask

(

const Packet2l &

)

                                                           {
  return _mm_set_epi32(0, 0, -1, -1);
}

peven_mask() [6/14]

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::peven_mask

(

const Packet4d &

)

                                                           {
  return _mm256_castsi256_pd(_mm256_set_epi32(0, 0, -1, -1, 0, 0, -1, -1));
}

peven_mask() [7/14]

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::peven_mask

(

const Packet4f &

)

                                                           {
  return _mm_castsi128_ps(_mm_set_epi32(0, -1, 0, -1));
}

peven_mask() [8/14]

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::peven_mask

(

const Packet4i &

)

                                                           {
  return _mm_set_epi32(0, -1, 0, -1);
}

peven_mask() [9/14]

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::peven_mask

(

const Packet4ui &

)

                                                             {
  return _mm_set_epi32(0, -1, 0, -1);
}

peven_mask() [10/14]

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::peven_mask

(

const Packet8d &

)

                                                           {
  return _mm512_castsi512_pd(_mm512_set_epi32(0, 0, -1, -1, 0, 0, -1, -1, 0, 0, -1, -1, 0, 0, -1, -1));
}

peven_mask() [11/14]

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::peven_mask

(

const Packet8f &

)

                                                           {
  return _mm256_castsi256_ps(_mm256_set_epi32(0, -1, 0, -1, 0, -1, 0, -1));
}

Referenced by paddsub(), pexp_complex(), phypot_complex(), plog_complex(), psincos_double(), psincos_float(), and psqrt_complex().

peven_mask() [12/14]

template<>

EIGEN_STRONG_INLINE Packet8i Eigen::internal::peven_mask

(

const Packet8i &

)

                                                           {
  return _mm256_set_epi32(0, -1, 0, -1, 0, -1, 0, -1);
}

peven_mask() [13/14]

template<>

EIGEN_STRONG_INLINE Packet8l Eigen::internal::peven_mask

(

const Packet8l &

)

                                                           {
  return _mm512_set_epi32(0, 0, -1, -1, 0, 0, -1, -1, 0, 0, -1, -1, 0, 0, -1, -1);
}

peven_mask() [14/14]

template<>

EIGEN_STRONG_INLINE Packet8ui Eigen::internal::peven_mask

(

const Packet8ui &

)

                                                             {
  return _mm256_set_epi32(0, -1, 0, -1, 0, -1, 0, -1);
}

pexp() [1/3]

template<typename Packet >

EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::pexp

(

const Packet &

a

)

Returns

the exp of a (coeff-wise)

                                                                                  {
  return numext::exp(a);
}

References a, and Eigen::numext::exp().

pexp() [2/3]

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pexp

(

const Packet2d &

_x

)

                                                      {
  return pexp_double(_x);
}

References pexp_double().

pexp() [3/3]

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pexp

(

const Packet4f &

_x

)

                                                      {
  return pexp_float(_x);
}

References pexp_float().

Referenced by erfc_double_large(), generic_exp2(), generic_expm1(), Eigen::internal::scalar_exp_op< Scalar >::operator()(), packetmath_real(), Eigen::internal::scalar_exp_op< Scalar >::packetOp(), Eigen::internal::scalar_logistic_op_impl< T, EnableIf >::packetOp(), exp_complex_test_impl< Scalar, Packet, HasExp >::pexp1(), pexp< Packet32h >(), pexp_complex(), Eigen::internal::generic_i0< T, ScalarType >::run(), Eigen::internal::generic_i1< T, ScalarType >::run(), Eigen::internal::generic_k0e< T, float >::run(), Eigen::internal::generic_k0e< T, double >::run(), Eigen::internal::generic_k0< T, float >::run(), Eigen::internal::generic_k0< T, double >::run(), Eigen::internal::generic_k1e< T, float >::run(), Eigen::internal::generic_k1e< T, double >::run(), Eigen::internal::generic_k1< T, float >::run(), Eigen::internal::generic_k1< T, double >::run(), Eigen::internal::generic_fast_erfc< Scalar >::run(), and exp_complex_test_impl< Scalar, Packet, HasExp >::run().

pexp2()

template<typename Packet >

EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::pexp2

(

const Packet &

a

)

Returns

the exp2 of a (coeff-wise)

                                                                                   {
  return numext::exp2(a);
}

References a, and Eigen::numext::exp2().

Referenced by Eigen::internal::scalar_exp2_op< Scalar >::operator()(), packetmath_real(), and Eigen::internal::scalar_exp2_op< Scalar >::packetOp().

pexp2< Packet8bf >()

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::pexp2< Packet8bf >

(

const Packet8bf &

a

)

                                                                   {
  BF16_TO_F32_UNARY_OP_WRAPPER(generic_exp2, a);
}

References a, BF16_TO_F32_UNARY_OP_WRAPPER, and generic_exp2().

pexp< Packet16h >()

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::pexp< Packet16h >

(

const Packet16h &

)

pexp< Packet1cf >()

template<>

EIGEN_STRONG_INLINE Packet1cf Eigen::internal::pexp< Packet1cf >

(

const Packet1cf &

a

)

                                                                  {
  return pexp_complex(a);
}

References a, and pexp_complex().

pexp< Packet2cf >() [1/2]

template<>

EIGEN_STRONG_INLINE Packet2cf Eigen::internal::pexp< Packet2cf >

(

const Packet2cf &

a

)

                                                                  {
  return pexp_complex<Packet2cf>(a);
}

References a.

pexp< Packet2cf >() [2/2]

template<>

EIGEN_STRONG_INLINE Packet2cf Eigen::internal::pexp< Packet2cf >	(	const Packet2cf &	a,
		const Packet2cf &	b
	)

                                                                                      {
  return pexp_complex(a, b);
}

References a, b, and pexp_complex().

pexp< Packet2d >()

template<>

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet2d Eigen::internal::pexp< Packet2d >

(

const Packet2d &

_x

)

                                                                                                {
  // Limiting double-precision pexp's argument to [-1024, +1024] lets pexp
  // reach 0 and INFINITY naturally.
  static EIGEN_DECLARE_CONST_Packet2d(exp_lo, -1024.0);
  static EIGEN_DECLARE_CONST_Packet2d(exp_hi, +1024.0);
  static EIGEN_DECLARE_CONST_Packet2d(cephes_LOG2EF, 1.4426950408889634073599);
  static EIGEN_DECLARE_CONST_Packet2d(cephes_exp_C1, 0.693145751953125);
  static EIGEN_DECLARE_CONST_Packet2d(cephes_exp_C2, 1.42860682030941723212e-6);
  static EIGEN_DECLARE_CONST_Packet2d(cephes_exp_p0, 1.26177193074810590878e-4);
  static EIGEN_DECLARE_CONST_Packet2d(cephes_exp_p1, 3.02994407707441961300e-2);
  static EIGEN_DECLARE_CONST_Packet2d(cephes_exp_p2, 9.99999999999999999910e-1);
  static EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q0, 3.00198505138664455042e-6);
  static EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q1, 2.52448340349684104192e-3);
  static EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q2, 2.27265548208155028766e-1);
  static EIGEN_DECLARE_CONST_Packet2d(cephes_exp_q3, 2.00000000000000000009e0);
  static EIGEN_DECLARE_CONST_Packet2d(half, 0.5);
  static EIGEN_DECLARE_CONST_Packet2d(1, 1.0);
  static EIGEN_DECLARE_CONST_Packet2d(2, 2.0);
 
  Packet2d x = _x;
 
  // Clamp x.
  x = (Packet2d)__builtin_msa_bsel_v((v16u8)__builtin_msa_fclt_d(x, p2d_exp_lo), (v16u8)x, (v16u8)p2d_exp_lo);
  x = (Packet2d)__builtin_msa_bsel_v((v16u8)__builtin_msa_fclt_d(p2d_exp_hi, x), (v16u8)x, (v16u8)p2d_exp_hi);
 
  // Round to nearest integer by adding 0.5 (with x's sign) and truncating.
  Packet2d x2_add = (Packet2d)__builtin_msa_binsli_d((v2u64)p2d_half, (v2u64)x, 0);
  Packet2d x2 = pmadd(x, p2d_cephes_LOG2EF, x2_add);
  Packet2l x2_long = __builtin_msa_ftrunc_s_d(x2);
  Packet2d x2_long_d = __builtin_msa_ffint_s_d(x2_long);
 
  x = __builtin_msa_fmsub_d(x, x2_long_d, p2d_cephes_exp_C1);
  x = __builtin_msa_fmsub_d(x, x2_long_d, p2d_cephes_exp_C2);
 
  x2 = pmul(x, x);
 
  Packet2d px = p2d_cephes_exp_p0;
  px = pmadd(px, x2, p2d_cephes_exp_p1);
  px = pmadd(px, x2, p2d_cephes_exp_p2);
  px = pmul(px, x);
 
  Packet2d qx = p2d_cephes_exp_q0;
  qx = pmadd(qx, x2, p2d_cephes_exp_q1);
  qx = pmadd(qx, x2, p2d_cephes_exp_q2);
  qx = pmadd(qx, x2, p2d_cephes_exp_q3);
 
  x = pdiv(px, psub(qx, px));
  x = pmadd(p2d_2, x, p2d_1);
 
  // x *= 2**exponent.
  x = __builtin_msa_fexp2_d(x, x2_long);
 
  return x;
}

References EIGEN_DECLARE_CONST_Packet2d(), pdiv(), pmadd(), pmul(), psub(), px, plotDoE::x, and Global_parameters::x2().

pexp< Packet32h >()

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::pexp< Packet32h >

(

const Packet32h &

a

)

                                                                  {
  Packet16h low;
  Packet16h high;
  extract2Packet16h(a, low, high);
 
  Packet16h lowOut = pexp(low);
  Packet16h highOut = pexp(high);
 
  return combine2Packet16h(lowOut, highOut);
}

References a, combine2Packet16h(), extract2Packet16h(), and pexp().

pexp< Packet4cf >()

template<>

EIGEN_STRONG_INLINE Packet4cf Eigen::internal::pexp< Packet4cf >

(

const Packet4cf &

a

)

                                                                  {
  return pexp_complex<Packet4cf>(a);
}

References a.

pexp< Packet4f >()

template<>

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet4f Eigen::internal::pexp< Packet4f >

(

const Packet4f &

_x

)

                                                                                                {
  // Limiting single-precision pexp's argument to [-128, +128] lets pexp
  // reach 0 and INFINITY naturally.
  static EIGEN_DECLARE_CONST_Packet4f(exp_lo, -128.0f);
  static EIGEN_DECLARE_CONST_Packet4f(exp_hi, +128.0f);
  static EIGEN_DECLARE_CONST_Packet4f(cephes_LOG2EF, 1.44269504088896341f);
  static EIGEN_DECLARE_CONST_Packet4f(cephes_exp_C1, 0.693359375f);
  static EIGEN_DECLARE_CONST_Packet4f(cephes_exp_C2, -2.12194440e-4f);
  static EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p0, 1.9875691500e-4f);
  static EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p1, 1.3981999507e-3f);
  static EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p2, 8.3334519073e-3f);
  static EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p3, 4.1665795894e-2f);
  static EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p4, 1.6666665459e-1f);
  static EIGEN_DECLARE_CONST_Packet4f(cephes_exp_p5, 5.0000001201e-1f);
  static EIGEN_DECLARE_CONST_Packet4f(half, 0.5f);
  static EIGEN_DECLARE_CONST_Packet4f(1, 1.0f);
 
  Packet4f x = _x;
 
  // Clamp x.
  x = (Packet4f)__builtin_msa_bsel_v((v16u8)__builtin_msa_fclt_w(x, p4f_exp_lo), (v16u8)x, (v16u8)p4f_exp_lo);
  x = (Packet4f)__builtin_msa_bsel_v((v16u8)__builtin_msa_fclt_w(p4f_exp_hi, x), (v16u8)x, (v16u8)p4f_exp_hi);
 
  // Round to nearest integer by adding 0.5 (with x's sign) and truncating.
  Packet4f x2_add = (Packet4f)__builtin_msa_binsli_w((v4u32)p4f_half, (v4u32)x, 0);
  Packet4f x2 = pmadd(x, p4f_cephes_LOG2EF, x2_add);
  Packet4i x2_int = __builtin_msa_ftrunc_s_w(x2);
  Packet4f x2_int_f = __builtin_msa_ffint_s_w(x2_int);
 
  x = __builtin_msa_fmsub_w(x, x2_int_f, p4f_cephes_exp_C1);
  x = __builtin_msa_fmsub_w(x, x2_int_f, p4f_cephes_exp_C2);
 
  Packet4f z = pmul(x, x);
 
  Packet4f y = p4f_cephes_exp_p0;
  y = pmadd(y, x, p4f_cephes_exp_p1);
  y = pmadd(y, x, p4f_cephes_exp_p2);
  y = pmadd(y, x, p4f_cephes_exp_p3);
  y = pmadd(y, x, p4f_cephes_exp_p4);
  y = pmadd(y, x, p4f_cephes_exp_p5);
  y = pmadd(y, z, x);
  y = padd(y, p4f_1);
 
  // y *= 2**exponent.
  y = __builtin_msa_fexp2_w(y, x2_int);
 
  return y;
}

References EIGEN_DECLARE_CONST_Packet4f(), padd(), pmadd(), pmul(), plotDoE::x, Global_parameters::x2(), and y.

pexp< Packet8bf >()

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::pexp< Packet8bf >

(

const Packet8bf &

a

)

                                                                  {
  BF16_TO_F32_UNARY_OP_WRAPPER(pexp_float, a);
}

References a, BF16_TO_F32_UNARY_OP_WRAPPER, and pexp_float().

pexp< Packet8cf >()

template<>

EIGEN_STRONG_INLINE Packet8cf Eigen::internal::pexp< Packet8cf >

(

const Packet8cf &

a

)

                                                                  {
  return pexp_complex<Packet8cf>(a);
}

References a.

pexp< PacketXf >()

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::pexp< PacketXf >

(

const PacketXf &

x

)

                                                               {
  return pexp_float(x);
}

References pexp_float(), and plotDoE::x.

pexp_complex()

template<typename Packet >

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::pexp_complex

(

const Packet &

x

)

Returns

exp(x) for complex types

                                                                                         {
  typedef typename unpacket_traits<Packet>::as_real RealPacket;
  typedef typename unpacket_traits<Packet>::type Scalar;
  typedef typename Scalar::value_type RealScalar;
  const RealPacket even_mask = peven_mask(a.v);
  const RealPacket odd_mask = pcplxflip(Packet(even_mask)).v;
 
  // Let a = x + iy.
  // exp(a) = exp(x) * cis(y), plus some special edge-case handling.
 
  // exp(x):
  RealPacket x = pand(a.v, even_mask);
  x = por(x, pcplxflip(Packet(x)).v);
  RealPacket expx = pexp(x);  // exp(x);
 
  // cis(y):
  RealPacket y = pand(odd_mask, a.v);
  y = por(y, pcplxflip(Packet(y)).v);
  RealPacket cisy = psincos_float<false, RealPacket, true>(y);
  cisy = pcplxflip(Packet(cisy)).v;  // cos(y) + i * sin(y)
 
  const RealPacket cst_pos_inf = pset1<RealPacket>(NumTraits<RealScalar>::infinity());
  const RealPacket cst_neg_inf = pset1<RealPacket>(-NumTraits<RealScalar>::infinity());
 
  // If x is -inf, we know that cossin(y) is bounded,
  //   so the result is (0, +/-0), where the sign of the imaginary part comes
  //   from the sign of cossin(y).
  RealPacket cisy_sign = por(pandnot(cisy, pabs(cisy)), pset1<RealPacket>(RealScalar(1)));
  cisy = pselect(pcmp_eq(x, cst_neg_inf), cisy_sign, cisy);
 
  // If x is inf, and cos(y) has unknown sign (y is inf or NaN), the result
  // is (+/-inf, NaN), where the signs are undetermined (take the sign of y).
  RealPacket y_sign = por(pandnot(y, pabs(y)), pset1<RealPacket>(RealScalar(1)));
  cisy = pselect(pand(pcmp_eq(x, cst_pos_inf), pisnan(cisy)), pand(y_sign, even_mask), cisy);
  Packet result = Packet(pmul(expx, cisy));
 
  // If y is +/- 0, the input is real, so take the real result for consistency.
  result = pselect(Packet(pcmp_eq(y, pzero(y))), Packet(por(pand(expx, even_mask), pand(y, odd_mask))), result);
 
  return result;
}

References a, Eigen::expx, pabs(), pand(), pandnot(), pcmp_eq(), pcplxflip(), peven_mask(), pexp(), pisnan(), pmul(), por(), pselect(), pzero(), v, Eigen::internal::Packet2cf::v, plotDoE::x, and y.

Referenced by pexp< Packet1cf >(), and pexp< Packet2cf >().

pexp_double()

template<typename Packet >

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::pexp_double

(

const Packet

_x

)

Returns

exp(x) for double precision real numbers

                                                                                        {
  Packet x = _x;
  const Packet cst_zero = pset1<Packet>(0.0);
  const Packet cst_1 = pset1<Packet>(1.0);
  const Packet cst_2 = pset1<Packet>(2.0);
  const Packet cst_half = pset1<Packet>(0.5);
 
  const Packet cst_exp_hi = pset1<Packet>(709.784);
  const Packet cst_exp_lo = pset1<Packet>(-745.519);
  const Packet cst_pldexp_threshold = pset1<Packet>(708.0);
  const Packet cst_cephes_LOG2EF = pset1<Packet>(1.4426950408889634073599);
  const Packet cst_cephes_exp_p0 = pset1<Packet>(1.26177193074810590878e-4);
  const Packet cst_cephes_exp_p1 = pset1<Packet>(3.02994407707441961300e-2);
  const Packet cst_cephes_exp_p2 = pset1<Packet>(9.99999999999999999910e-1);
  const Packet cst_cephes_exp_q0 = pset1<Packet>(3.00198505138664455042e-6);
  const Packet cst_cephes_exp_q1 = pset1<Packet>(2.52448340349684104192e-3);
  const Packet cst_cephes_exp_q2 = pset1<Packet>(2.27265548208155028766e-1);
  const Packet cst_cephes_exp_q3 = pset1<Packet>(2.00000000000000000009e0);
  const Packet cst_cephes_exp_C1 = pset1<Packet>(0.693145751953125);
  const Packet cst_cephes_exp_C2 = pset1<Packet>(1.42860682030941723212e-6);
 
  Packet tmp, fx;
 
  // clamp x
  Packet zero_mask = pcmp_lt(_x, cst_exp_lo);
  x = pmin(x, cst_exp_hi);
  // Express exp(x) as exp(g + n*log(2)).
  fx = pmadd(cst_cephes_LOG2EF, x, cst_half);
 
  // Get the integer modulus of log(2), i.e. the "n" described above.
  fx = pfloor(fx);
 
  // Get the remainder modulo log(2), i.e. the "g" described above. Subtract
  // n*log(2) out in two steps, i.e. n*C1 + n*C2, C1+C2=log2 to get the last
  // digits right.
  tmp = pmul(fx, cst_cephes_exp_C1);
  Packet z = pmul(fx, cst_cephes_exp_C2);
  x = psub(x, tmp);
  x = psub(x, z);
 
  Packet x2 = pmul(x, x);
 
  // Evaluate the numerator polynomial of the rational interpolant.
  Packet px = cst_cephes_exp_p0;
  px = pmadd(px, x2, cst_cephes_exp_p1);
  px = pmadd(px, x2, cst_cephes_exp_p2);
  px = pmul(px, x);
 
  // Evaluate the denominator polynomial of the rational interpolant.
  Packet qx = cst_cephes_exp_q0;
  qx = pmadd(qx, x2, cst_cephes_exp_q1);
  qx = pmadd(qx, x2, cst_cephes_exp_q2);
  qx = pmadd(qx, x2, cst_cephes_exp_q3);
 
  // I don't really get this bit, copied from the SSE2 routines, so...
  // TODO(gonnet): Figure out what is going on here, perhaps find a better
  // rational interpolant?
  x = pdiv(px, psub(qx, px));
  x = pmadd(cst_2, x, cst_1);
 
  // Construct the result 2^n * exp(g) = e * x. The max is used to catch
  // non-finite values in the input.
  const Packet fast_pldexp_unsafe = pcmp_lt(cst_pldexp_threshold, pabs(_x));
  if (!predux_any(fast_pldexp_unsafe)) {
    // For |x| <= 708, we know the result is not zero or inf, and we can safely use
    // the fast version of pldexp.
    return pmax(pldexp_fast(x, fx), _x);
  }
  return pselect(zero_mask, cst_zero, pmax(pldexp(x, fx), _x));
}

References pabs(), pcmp_lt(), pdiv(), pfloor(), pldexp(), pldexp_fast(), pmadd(), pmax(), pmin(), pmul(), predux_any(), pselect(), psub(), px, tmp, plotDoE::x, and Global_parameters::x2().

Referenced by pexp().

pexp_float()

template<typename Packet >

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::pexp_float

(

const Packet

_x

)

Returns

exp(x) for single precision float

                                                                                       {
  const Packet cst_zero = pset1<Packet>(0.0f);
  const Packet cst_one = pset1<Packet>(1.0f);
  const Packet cst_half = pset1<Packet>(0.5f);
  const Packet cst_exp_hi = pset1<Packet>(88.723f);
  const Packet cst_exp_lo = pset1<Packet>(-104.f);
  const Packet cst_pldexp_threshold = pset1<Packet>(87.0);
 
  const Packet cst_cephes_LOG2EF = pset1<Packet>(1.44269504088896341f);
  const Packet cst_p2 = pset1<Packet>(0.49999988079071044921875f);
  const Packet cst_p3 = pset1<Packet>(0.16666518151760101318359375f);
  const Packet cst_p4 = pset1<Packet>(4.166965186595916748046875e-2f);
  const Packet cst_p5 = pset1<Packet>(8.36894474923610687255859375e-3f);
  const Packet cst_p6 = pset1<Packet>(1.37449637986719608306884765625e-3f);
 
  // Clamp x.
  Packet zero_mask = pcmp_lt(_x, cst_exp_lo);
  Packet x = pmin(_x, cst_exp_hi);
 
  // Express exp(x) as exp(m*ln(2) + r), start by extracting
  // m = floor(x/ln(2) + 0.5).
  Packet m = pfloor(pmadd(x, cst_cephes_LOG2EF, cst_half));
 
  // Get r = x - m*ln(2). If no FMA instructions are available, m*ln(2) is
  // subtracted out in two parts, m*C1+m*C2 = m*ln(2), to avoid accumulating
  // truncation errors.
  const Packet cst_cephes_exp_C1 = pset1<Packet>(-0.693359375f);
  const Packet cst_cephes_exp_C2 = pset1<Packet>(2.12194440e-4f);
  Packet r = pmadd(m, cst_cephes_exp_C1, x);
  r = pmadd(m, cst_cephes_exp_C2, r);
 
  // Evaluate the 6th order polynomial approximation to exp(r)
  // with r in the interval [-ln(2)/2;ln(2)/2].
  const Packet r2 = pmul(r, r);
  Packet p_even = pmadd(r2, cst_p6, cst_p4);
  const Packet p_odd = pmadd(r2, cst_p5, cst_p3);
  p_even = pmadd(r2, p_even, cst_p2);
  const Packet p_low = padd(r, cst_one);
  Packet y = pmadd(r, p_odd, p_even);
  y = pmadd(r2, y, p_low);
 
  // Return 2^m * exp(r).
  const Packet fast_pldexp_unsafe = pcmp_lt(cst_pldexp_threshold, pabs(x));
  if (!predux_any(fast_pldexp_unsafe)) {
    // For |x| <= 87, we know the result is not zero or inf, and we can safely use
    // the fast version of pldexp.
    return pmax(pldexp_fast(y, m), _x);
  }
  return pselect(zero_mask, cst_zero, pmax(pldexp(y, m), _x));
}

References m, pabs(), padd(), pcmp_lt(), pfloor(), pldexp(), pldexp_fast(), pmadd(), pmax(), pmin(), pmul(), predux_any(), pselect(), UniformPSDSelfTest::r, plotDoE::x, and y.

Referenced by pexp(), pexp< Packet8bf >(), and pexp< PacketXf >().

pexpm1()

template<typename Packet >

EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::pexpm1

(

const Packet &

a

)

Returns

the expm1 of a (coeff-wise)

                                                                                    {
  return numext::expm1(a);
}

References a, and Eigen::bfloat16_impl::expm1().

Referenced by packetmath_real(), Eigen::internal::scalar_expm1_op< Scalar >::packetOp(), and pexpm1< Packet32h >().

pexpm1< Packet16h >()

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::pexpm1< Packet16h >

(

const Packet16h &

)

pexpm1< Packet32h >()

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::pexpm1< Packet32h >

(

const Packet32h &

a

)

                                                                    {
  Packet16h low;
  Packet16h high;
  extract2Packet16h(a, low, high);
 
  Packet16h lowOut = pexpm1(low);
  Packet16h highOut = pexpm1(high);
 
  return combine2Packet16h(lowOut, highOut);
}

References a, combine2Packet16h(), extract2Packet16h(), and pexpm1().

pfirst() [1/2]

template<typename Packet >

EIGEN_DEVICE_FUNC unpacket_traits<Packet>::type Eigen::internal::pfirst ( const Packet &  a )

inline

Returns

the first element of a packet

                                                                                      {
  return a;
}

References a.

pfirst() [2/2]

template<>

EIGEN_STRONG_INLINE bfloat16 Eigen::internal::pfirst

(

const Packet8bf &

a

)

                                                        {
  return Eigen::bfloat16_impl::raw_uint16_to_bfloat16((pfirst<Packet8us>(a)));
}

References a, pfirst< Packet8us >(), and Eigen::bfloat16_impl::raw_uint16_to_bfloat16().

Referenced by packetmath(), exp_complex_test_impl< Scalar, Packet, HasExp >::pexp1(), pfirst< Packet4cf >(), pfirst< Packet8cf >(), predux< Packet16b >(), predux< Packet1cd >(), predux< Packet4f >(), predux< Packet4i >(), predux_max< Packet16c >(), predux_max< Packet16f >(), predux_max< Packet16uc >(), predux_max< Packet2d >(), predux_max< Packet2l >(), predux_max< Packet2ul >(), predux_max< Packet4d >(), predux_max< Packet4ui >(), predux_max< Packet8d >(), predux_max< Packet8f >(), predux_max< Packet8s >(), predux_max< Packet8us >(), predux_min< Packet16c >(), predux_min< Packet16f >(), predux_min< Packet16uc >(), predux_min< Packet2d >(), predux_min< Packet2l >(), predux_min< Packet2ul >(), predux_min< Packet4d >(), predux_min< Packet4ui >(), predux_min< Packet8d >(), predux_min< Packet8f >(), predux_min< Packet8s >(), predux_min< Packet8us >(), predux_mul< Packet16c >(), predux_mul< Packet16f >(), predux_mul< Packet16uc >(), predux_mul< Packet1cd >(), predux_mul< Packet4d >(), predux_mul< Packet4f >(), predux_mul< Packet8d >(), predux_mul< Packet8f >(), predux_mul< Packet8s >(), predux_mul< Packet8us >(), eigen_optimization_barrier_test< Packet, std::enable_if_t<!NumTraits< Packet >::IsComplex &&!internal::is_same< Packet, Eigen::half >::value &&!internal::is_same< Packet, Eigen::bfloat16 >::value > >::run(), and Eigen::numext::sqrt().

pfirst< Packet16b >()

template<>

EIGEN_STRONG_INLINE bool Eigen::internal::pfirst< Packet16b >

(

const Packet16b &

a

)

                                                               {
  int x = _mm_cvtsi128_si32(a);
  return static_cast<bool>(x & 1);
}

References a, and plotDoE::x.

pfirst< Packet16bf >()

template<>

EIGEN_STRONG_INLINE bfloat16 Eigen::internal::pfirst< Packet16bf >

(

const Packet16bf &

from

)

                                                                        {
  bfloat16 t;
  t.value = static_cast<unsigned short>(_mm256_extract_epi16(from, 0));
  return t;
}

References plotPSD::t.

pfirst< Packet16c >()

template<>

EIGEN_STRONG_INLINE int8_t Eigen::internal::pfirst< Packet16c >

(

const Packet16c &

a

)

                                                                      {
  return pfirst_common<Packet16c>(a);
}

References a.

pfirst< Packet16f >()

template<>

EIGEN_STRONG_INLINE float Eigen::internal::pfirst< Packet16f >

(

const Packet16f &

a

)

                                                                {
  return _mm512_cvtss_f32(a);
}

References a.

pfirst< Packet16h >()

template<>

EIGEN_STRONG_INLINE Eigen::half Eigen::internal::pfirst< Packet16h >

(

const Packet16h &

from

)

                                                                       {
  return half_impl::raw_uint16_to_half(static_cast<unsigned short>(_mm256_extract_epi16(from, 0)));
}

References Eigen::half_impl::raw_uint16_to_half().

pfirst< Packet16i >()

template<>

EIGEN_STRONG_INLINE int Eigen::internal::pfirst< Packet16i >

(

const Packet16i &

a

)

                                                              {
#if EIGEN_GNUC_STRICT_LESS_THAN(11, 0, 0)
  return _mm_cvtsi128_si32(_mm512_castsi512_si128(a));
#else
  return _mm512_cvtsi512_si32(a);
#endif
}

References a.

pfirst< Packet16uc >()

template<>

EIGEN_STRONG_INLINE uint8_t Eigen::internal::pfirst< Packet16uc >

(

const Packet16uc &

a

)

                                                                          {
  return pfirst_common<Packet16uc>(a);
}

References a.

pfirst< Packet1cd >()

template<>

EIGEN_STRONG_INLINE std::complex< double > Eigen::internal::pfirst< Packet1cd >

(

const Packet1cd &

a

)

                                                                             {
  EIGEN_ALIGN16 double res[2];
  __lsx_vst(a.v, res, 0);
  return std::complex<double>(res[0], res[1]);
}

References a, EIGEN_ALIGN16, and res.

pfirst< Packet1cf >()

template<>

EIGEN_STRONG_INLINE std::complex<float> Eigen::internal::pfirst< Packet1cf >

(

const Packet1cf &

a

)

                                                                            {
  EIGEN_ALIGN16 std::complex<float> x;
  vst1_f32(reinterpret_cast<float*>(&x), a.v);
  return x;
}

References a, EIGEN_ALIGN16, and plotDoE::x.

pfirst< Packet2cd >()

template<>

EIGEN_STRONG_INLINE std::complex<double> Eigen::internal::pfirst< Packet2cd >

(

const Packet2cd &

a

)

                                                                             {
  __m128d low = _mm256_extractf128_pd(a.v, 0);
  EIGEN_ALIGN16 double res[2];
  _mm_store_pd(res, low);
  return std::complex<double>(res[0], res[1]);
}

References a, EIGEN_ALIGN16, and res.

pfirst< Packet2cf >()

template<>

EIGEN_STRONG_INLINE std::complex< float > Eigen::internal::pfirst< Packet2cf >

(

const Packet2cf &

a

)

                                                                            {
  EIGEN_ALIGN16 std::complex<float> res[2];
  pstore((float*)&res, a.v);
 
  return res[0];
}

References a, EIGEN_ALIGN16, pstore(), and res.

Referenced by predux< Packet2cf >(), and predux_mul< Packet2cf >().

pfirst< Packet2d >()

template<>

EIGEN_STRONG_INLINE double Eigen::internal::pfirst< Packet2d >

(

const Packet2d &

a

)

                                                               {
  double v;
  __lsx_vstelm_d(a, &v, 0, 0);
  return v;
}

References a, and v.

Referenced by predux< Packet2d >(), and predux_mul< Packet2d >().

pfirst< Packet2f >()

template<>

EIGEN_STRONG_INLINE float Eigen::internal::pfirst< Packet2f >

(

const Packet2f &

a

)

                                                              {
  return vget_lane_f32(a, 0);
}

References a.

pfirst< Packet2i >()

template<>

EIGEN_STRONG_INLINE int32_t Eigen::internal::pfirst< Packet2i >

(

const Packet2i &

a

)

                                                                {
  return vget_lane_s32(a, 0);
}

References a.

pfirst< Packet2l >()

template<>

EIGEN_STRONG_INLINE int64_t Eigen::internal::pfirst< Packet2l >

(

const Packet2l &

a

)

                                                                {
  return __lsx_vpickve2gr_d((__m128i)a, 0);
}

References a.

pfirst< Packet2ui >()

template<>

EIGEN_STRONG_INLINE uint32_t Eigen::internal::pfirst< Packet2ui >

(

const Packet2ui &

a

)

                                                                   {
  return vget_lane_u32(a, 0);
}

References a.

pfirst< Packet2ul >()

template<>

EIGEN_STRONG_INLINE uint64_t Eigen::internal::pfirst< Packet2ul >

(

const Packet2ul &

a

)

                                                                   {
  return __lsx_vpickve2gr_du((__m128i)a, 0);
}

References a.

pfirst< Packet32h >()

template<>

EIGEN_STRONG_INLINE Eigen::half Eigen::internal::pfirst< Packet32h >

(

const Packet32h &

from

)

                                                                       {
#ifdef EIGEN_VECTORIZE_AVX512DQ
  return half_impl::raw_uint16_to_half(
      static_cast<unsigned short>(_mm256_extract_epi16(_mm512_extracti32x8_epi32(_mm512_castph_si512(from), 0), 0)));
#else
  Eigen::half dest[32];
  _mm512_storeu_ph(dest, from);
  return dest[0];
#endif
}

References Eigen::half_impl::raw_uint16_to_half().

pfirst< Packet4bf >()

template<>

EIGEN_STRONG_INLINE bfloat16 Eigen::internal::pfirst< Packet4bf >

(

const Packet4bf &

from

)

                                                                      {
  return bfloat16_impl::raw_uint16_to_bfloat16(static_cast<uint16_t>(pfirst<Packet4us>(Packet4us(from))));
}

References pfirst< Packet4us >(), and Eigen::bfloat16_impl::raw_uint16_to_bfloat16().

pfirst< Packet4c >()

template<>

EIGEN_STRONG_INLINE int8_t Eigen::internal::pfirst< Packet4c >

(

const Packet4c &

a

)

                                                               {
  return static_cast<int8_t>(a & 0xff);
}

References a.

pfirst< Packet4cd >()

template<>

EIGEN_STRONG_INLINE std::complex<double> Eigen::internal::pfirst< Packet4cd >

(

const Packet4cd &

a

)

                                                                             {
  __m128d low = extract128<0>(a.v);
  EIGEN_ALIGN16 double res[2];
  _mm_store_pd(res, low);
  return std::complex<double>(res[0], res[1]);
}

References a, EIGEN_ALIGN16, and res.

pfirst< Packet4cf >()

template<>

EIGEN_STRONG_INLINE std::complex<float> Eigen::internal::pfirst< Packet4cf >

(

const Packet4cf &

a

)

                                                                            {
  return pfirst(Packet2cf(_mm256_castps256_ps128(a.v)));
}

References a, and pfirst().

pfirst< Packet4d >()

template<>

EIGEN_STRONG_INLINE double Eigen::internal::pfirst< Packet4d >

(

const Packet4d &

a

)

                                                               {
  return _mm_cvtsd_f64(_mm256_castpd256_pd128(a));
}

References a.

pfirst< Packet4f >()

template<>

EIGEN_STRONG_INLINE float Eigen::internal::pfirst< Packet4f >

(

const Packet4f &

a

)

                                                              {
  EIGEN_ALIGN16 float x;
  vec_ste(a, 0, &x);
  return x;
}

References a, EIGEN_ALIGN16, and plotDoE::x.

pfirst< Packet4i >()

template<>

EIGEN_STRONG_INLINE int Eigen::internal::pfirst< Packet4i >

(

const Packet4i &

a

)

                                                            {
  EIGEN_ALIGN16 int x;
  vec_ste(a, 0, &x);
  return x;
}

References a, EIGEN_ALIGN16, and plotDoE::x.

Referenced by predux< Packet16b >(), and predux_mul< Packet16b >().

pfirst< Packet4s >()

template<>

EIGEN_STRONG_INLINE int16_t Eigen::internal::pfirst< Packet4s >

(

const Packet4s &

a

)

                                                                {
  return vget_lane_s16(a, 0);
}

References a.

pfirst< Packet4uc >()

template<>

EIGEN_STRONG_INLINE uint8_t Eigen::internal::pfirst< Packet4uc >

(

const Packet4uc &

a

)

                                                                  {
  return static_cast<uint8_t>(a & 0xff);
}

References a.

pfirst< Packet4ui >()

template<>

EIGEN_STRONG_INLINE uint32_t Eigen::internal::pfirst< Packet4ui >

(

const Packet4ui &

a

)

                                                                   {
  return __lsx_vpickve2gr_wu((__m128i)a, 0);
}

References a.

pfirst< Packet4us >()

template<>

EIGEN_STRONG_INLINE uint16_t Eigen::internal::pfirst< Packet4us >

(

const Packet4us &

a

)

                                                                   {
  return vget_lane_u16(a, 0);
}

References a.

Referenced by pfirst< Packet4bf >().

pfirst< Packet8bf >()

template<>

EIGEN_STRONG_INLINE bfloat16 Eigen::internal::pfirst< Packet8bf >

(

const Packet8bf &

from

)

                                                                      {
  return numext::bit_cast<bfloat16>(static_cast<numext::uint16_t>(_mm_extract_epi16(from, 0)));
}

pfirst< Packet8c >()

template<>

EIGEN_STRONG_INLINE int8_t Eigen::internal::pfirst< Packet8c >

(

const Packet8c &

a

)

                                                               {
  return vget_lane_s8(a, 0);
}

References a.

pfirst< Packet8cf >()

template<>

EIGEN_STRONG_INLINE std::complex<float> Eigen::internal::pfirst< Packet8cf >

(

const Packet8cf &

a

)

                                                                            {
  return pfirst(Packet2cf(_mm512_castps512_ps128(a.v)));
}

References a, and pfirst().

pfirst< Packet8d >()

template<>

EIGEN_STRONG_INLINE double Eigen::internal::pfirst< Packet8d >

(

const Packet8d &

a

)

                                                               {
  return _mm512_cvtsd_f64(a);
}

References a.

pfirst< Packet8f >()

template<>

EIGEN_STRONG_INLINE float Eigen::internal::pfirst< Packet8f >

(

const Packet8f &

a

)

                                                              {
  return _mm_cvtss_f32(_mm256_castps256_ps128(a));
}

References a.

pfirst< Packet8h >()

template<>

EIGEN_STRONG_INLINE Eigen::half Eigen::internal::pfirst< Packet8h >

(

const Packet8h &

from

)

                                                                     {
  return numext::bit_cast<Eigen::half>(static_cast<numext::uint16_t>(_mm_extract_epi16(from, 0)));
}

pfirst< Packet8i >()

template<>

EIGEN_STRONG_INLINE int Eigen::internal::pfirst< Packet8i >

(

const Packet8i &

a

)

                                                            {
  return _mm_cvtsi128_si32(_mm256_castsi256_si128(a));
}

References a.

pfirst< Packet8l >()

template<>

EIGEN_STRONG_INLINE int64_t Eigen::internal::pfirst< Packet8l >

(

const Packet8l &

a

)

                                                                {
  int64_t x = _mm_extract_epi64_0(_mm512_extracti32x4_epi32(a, 0));
  return x;
}

References _mm_extract_epi64_0(), a, and plotDoE::x.

pfirst< Packet8s >()

template<>

EIGEN_STRONG_INLINE int16_t Eigen::internal::pfirst< Packet8s >

(

const Packet8s &

a

)

                                                                  {
  return pfirst_common<Packet8s>(a);
}

References a.

pfirst< Packet8uc >()

template<>

EIGEN_STRONG_INLINE uint8_t Eigen::internal::pfirst< Packet8uc >

(

const Packet8uc &

a

)

                                                                  {
  return vget_lane_u8(a, 0);
}

References a.

pfirst< Packet8ui >()

template<>

EIGEN_STRONG_INLINE uint32_t Eigen::internal::pfirst< Packet8ui >

(

const Packet8ui &

a

)

                                                                   {
  return numext::bit_cast<uint32_t>(_mm_cvtsi128_si32(_mm256_castsi256_si128(a)));
}

References a.

pfirst< Packet8us >()

template<>

EIGEN_STRONG_INLINE uint16_t Eigen::internal::pfirst< Packet8us >

(

const Packet8us &

a

)

                                                                             {
  return pfirst_common<Packet8us>(a);
}

References a.

Referenced by pfirst().

pfirst< PacketXf >()

template<>

EIGEN_STRONG_INLINE float Eigen::internal::pfirst< PacketXf >

(

const PacketXf &

a

)

                                                              {
  // svlasta returns the first element if all predicate bits are 0
  return svlasta_f32(svpfalse_b(), a);
}

References a.

Referenced by predux_mul< PacketXf >().

pfirst< PacketXi >()

template<>

EIGEN_STRONG_INLINE numext::int32_t Eigen::internal::pfirst< PacketXi >

(

const PacketXi &

a

)

                                                                      {
  // svlasta returns the first element if all predicate bits are 0
  return svlasta_s32(svpfalse_b(), a);
}

References a.

Referenced by predux_mul< PacketXi >().

pfloor() [1/3]

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet Eigen::internal::pfloor

(

const Packet &

a

)

Returns

the floor of a (coeff-wise)

                                                                     {
  return nearest_integer_packetop_impl<Packet>::run_floor(a);
}

References a, and Eigen::internal::nearest_integer_packetop_impl< Packet, IsScalar, IsInteger >::run_floor().

pfloor() [2/3]

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pfloor

(

const Packet2d &

a

)

                                                       {
  return __lsx_vfrintrm_d(a);
}

References a.

pfloor() [3/3]

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pfloor

(

const Packet4f &

a

)

                                                       {
  return __lsx_vfrintrm_s(a);
}

References a.

Referenced by generic_pow(), packetmath_real(), Eigen::internal::scalar_logistic_op< float >::packetOp(), Eigen::internal::scalar_floor_op< Scalar >::packetOp(), pexp_double(), pexp_float(), and psincos_double().

pfloor< Packet16bf >()

template<>

EIGEN_STRONG_INLINE Packet16bf Eigen::internal::pfloor< Packet16bf >

(

const Packet16bf &

a

)

                                                                       {
  return F32ToBf16(pfloor<Packet16f>(Bf16ToF32(a)));
}

References a, Bf16ToF32(), F32ToBf16(), and pfloor< Packet16f >().

pfloor< Packet16f >()

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::pfloor< Packet16f >

(

const Packet16f &

a

)

                                                                    {
  return _mm512_roundscale_ps(a, _MM_FROUND_TO_NEG_INF);
}

References a.

Referenced by pfloor< Packet16bf >(), and pfloor< Packet16h >().

pfloor< Packet16h >()

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::pfloor< Packet16h >

(

const Packet16h &

a

)

                                                                    {
  return float2half(pfloor<Packet16f>(half2float(a)));
}

References a, float2half(), half2float(), and pfloor< Packet16f >().

pfloor< Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pfloor< Packet2d >

(

const Packet2d &

a

)

                                                                 {
  Packet2d v = a;
  int32_t old_mode, new_mode;
  asm volatile(
      "cfcmsa  %[old_mode], $1\n"
      "ori     %[new_mode], %[old_mode], 3\n"  // 3 = round towards -INFINITY.
      "ctcmsa  $1, %[new_mode]\n"
      "frint.d %w[v], %w[v]\n"
      "ctcmsa  $1, %[old_mode]\n"
      :  // outputs
      [old_mode] "=r"(old_mode), [new_mode] "=r"(new_mode),
      [v] "+f"(v)
      :  // inputs
      :  // clobbers
  );
  return v;
}

References a, and v.

pfloor< Packet32h >()

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::pfloor< Packet32h >

(

const Packet32h &

a

)

                                                                    {
  return _mm512_roundscale_ph(a, _MM_FROUND_TO_NEG_INF);
}

References a.

pfloor< Packet4bf >()

template<>

EIGEN_STRONG_INLINE Packet4bf Eigen::internal::pfloor< Packet4bf >

(

const Packet4bf &

a

)

                                                                    {
  return F32ToBf16(pfloor<Packet4f>(Bf16ToF32(a)));
}

References a, Bf16ToF32(), F32ToBf16(), and pfloor< Packet4f >().

pfloor< Packet4d >()

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::pfloor< Packet4d >

(

const Packet4d &

a

)

                                                                 {
  return _mm256_floor_pd(a);
}

References a.

pfloor< Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pfloor< Packet4f >

(

const Packet4f &

a

)

                                                                 {
  return vec_floor(a);
}

References a.

Referenced by pfloor< Packet4bf >(), and pfloor< Packet8bf >().

pfloor< Packet8bf >()

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::pfloor< Packet8bf >

(

const Packet8bf &

a

)

                                                                    {
  BF16_TO_F32_UNARY_OP_WRAPPER(pfloor<Packet4f>, a);
}

References a, BF16_TO_F32_UNARY_OP_WRAPPER, and pfloor< Packet4f >().

pfloor< Packet8d >()

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::pfloor< Packet8d >

(

const Packet8d &

a

)

                                                                 {
  return _mm512_roundscale_pd(a, _MM_FROUND_TO_NEG_INF);
}

References a.

pfloor< Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::pfloor< Packet8f >

(

const Packet8f &

a

)

                                                                 {
  return _mm256_floor_ps(a);
}

References a.

Referenced by pfloor< Packet8h >().

pfloor< Packet8h >()

template<>

EIGEN_STRONG_INLINE Packet8h Eigen::internal::pfloor< Packet8h >

(

const Packet8h &

a

)

                                                                 {
  return float2half(pfloor<Packet8f>(half2float(a)));
}

References a, float2half(), half2float(), and pfloor< Packet8f >().

pfloor< PacketXf >()

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::pfloor< PacketXf >

(

const PacketXf &

a

)

                                                                 {
  return svrintm_f32_x(svptrue_b32(), a);
}

References a.

pfrexp() [1/6]

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::pfrexp ( const Packet &  a, Packet &  exponent  )

inline

Returns

the significant and exponent of the underlying floating point numbers See https://en.cppreference.com/w/cpp/numeric/math/frexp

                                                                          {
  int exp;
  EIGEN_USING_STD(frexp);
  Packet result = static_cast<Packet>(frexp(a, &exp));
  exponent = static_cast<Packet>(exp);
  return result;
}

References a, EIGEN_USING_STD, and Eigen::bfloat16_impl::exp().

pfrexp() [2/6]

template<>

EIGEN_STRONG_INLINE Packet16bf Eigen::internal::pfrexp	(	const Packet16bf &	a,
		Packet16bf &	exponent
	)

                                                                                 {
  Packet16f fexponent;
  const Packet16bf out = F32ToBf16(pfrexp<Packet16f>(Bf16ToF32(a), fexponent));
  exponent = F32ToBf16(fexponent);
  return out;
}

References a, Bf16ToF32(), F32ToBf16(), out(), and pfrexp< Packet16f >().

pfrexp() [3/6]

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::pfrexp	(	const Packet16h &	a,
		Packet16h &	exponent
	)

                                                                              {
  Packet16f fexponent;
  const Packet16h out = float2half(pfrexp<Packet16f>(half2float(a), fexponent));
  exponent = float2half(fexponent);
  return out;
}

References a, float2half(), half2float(), out(), and pfrexp< Packet16f >().

pfrexp() [4/6]

template<>

EIGEN_STRONG_INLINE Packet4bf Eigen::internal::pfrexp	(	const Packet4bf &	a,
		Packet4bf &	exponent
	)

                                                                              {
  Packet4f fexponent;
  const Packet4bf out = F32ToBf16(pfrexp<Packet4f>(Bf16ToF32(a), fexponent));
  exponent = F32ToBf16(fexponent);
  return out;
}

References a, Bf16ToF32(), F32ToBf16(), out(), and pfrexp< Packet4f >().

pfrexp() [5/6]

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::pfrexp	(	const Packet8bf &	a,
		Packet8bf &	exponent
	)

                                                                              {
  Packet8f fexponent;
  const Packet8bf out = F32ToBf16(pfrexp<Packet8f>(Bf16ToF32(a), fexponent));
  exponent = F32ToBf16(fexponent);
  return out;
}

References a, Bf16ToF32(), F32ToBf16(), out(), and pfrexp< Packet8f >().

pfrexp() [6/6]

template<>

EIGEN_STRONG_INLINE Packet8h Eigen::internal::pfrexp	(	const Packet8h &	a,
		Packet8h &	exponent
	)

                                                                           {
  Packet8f fexponent;
  const Packet8h out = float2half(pfrexp<Packet8f>(half2float(a), fexponent));
  exponent = float2half(fexponent);
  return out;
}

References a, float2half(), half2float(), out(), and pfrexp< Packet8f >().

Referenced by generic_pow_impl(), packetmath_real(), pfrexp< Packet32h >(), plog_impl_double(), and plog_impl_float().

pfrexp< Packet16f >()

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::pfrexp< Packet16f >	(	const Packet16f &	a,
		Packet16f &	exponent
	)

                                                                                         {
  return pfrexp_generic(a, exponent);
}

References a, and pfrexp_generic().

Referenced by pfrexp().

pfrexp< Packet16h >()

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::pfrexp< Packet16h >	(	const Packet16h &	,
		Packet16h &
	)

pfrexp< Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pfrexp< Packet2d >	(	const Packet2d &	a,
		Packet2d &	exponent
	)

                                                                                     {
  return pfrexp_generic(a, exponent);
}

References a, and pfrexp_generic().

pfrexp< Packet2f >()

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::pfrexp< Packet2f >	(	const Packet2f &	a,
		Packet2f &	exponent
	)

                                                                                     {
  return pfrexp_generic(a, exponent);
}

References a, and pfrexp_generic().

pfrexp< Packet32h >()

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::pfrexp< Packet32h >	(	const Packet32h &	a,
		Packet32h &	exponent
	)

                                                                                         {
  Packet16h low;
  Packet16h high;
  extract2Packet16h(a, low, high);
 
  Packet16h exp1 = _mm256_undefined_si256();
  Packet16h exp2 = _mm256_undefined_si256();
 
  Packet16h lowOut = pfrexp(low, exp1);
  Packet16h highOut = pfrexp(high, exp2);
 
  exponent = combine2Packet16h(exp1, exp2);
 
  return combine2Packet16h(lowOut, highOut);
}

References a, combine2Packet16h(), Eigen::bfloat16_impl::exp2(), extract2Packet16h(), and pfrexp().

pfrexp< Packet4d >()

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::pfrexp< Packet4d >	(	const Packet4d &	a,
		Packet4d &	exponent
	)

                                                                                     {
  return pfrexp_generic(a, exponent);
}

References a, and pfrexp_generic().

pfrexp< Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pfrexp< Packet4f >	(	const Packet4f &	a,
		Packet4f &	exponent
	)

                                                                                     {
  return pfrexp_generic(a, exponent);
}

References a, and pfrexp_generic().

Referenced by pfrexp(), and pfrexp< Packet8bf >().

pfrexp< Packet8bf >()

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::pfrexp< Packet8bf >	(	const Packet8bf &	a,
		Packet8bf &	e
	)

                                                                                  {
  Packet4f a_even = Bf16ToF32Even(a);
  Packet4f a_odd = Bf16ToF32Odd(a);
  Packet4f e_even;
  Packet4f e_odd;
  Packet4f op_even = pfrexp<Packet4f>(a_even, e_even);
  Packet4f op_odd = pfrexp<Packet4f>(a_odd, e_odd);
  e = F32ToBf16(e_even, e_odd);
  return F32ToBf16(op_even, op_odd);
}

References a, Bf16ToF32Even(), Bf16ToF32Odd(), e(), F32ToBf16(), and pfrexp< Packet4f >().

pfrexp< Packet8d >()

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::pfrexp< Packet8d >	(	const Packet8d &	a,
		Packet8d &	exponent
	)

                                                                                     {
  return pfrexp_generic(a, exponent);
}

References a, and pfrexp_generic().

pfrexp< Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::pfrexp< Packet8f >	(	const Packet8f &	a,
		Packet8f &	exponent
	)

                                                                                     {
  return pfrexp_generic(a, exponent);
}

References a, and pfrexp_generic().

Referenced by pfrexp().

pfrexp< PacketXf >()

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::pfrexp< PacketXf >	(	const PacketXf &	a,
		PacketXf &	exponent
	)

                                                                                     {
  return pfrexp_generic(a, exponent);
}

References a, and pfrexp_generic().

pfrexp_generic()

template<typename Packet >

EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Packet Eigen::internal::pfrexp_generic	(	const Packet &	a,
		Packet &	exponent
	)

Default implementation of pfrexp. It is expected to be called by implementers of template<> pfrexp.

                                                                                               {
  typedef typename unpacket_traits<Packet>::type Scalar;
  typedef typename make_unsigned<typename make_integer<Scalar>::type>::type ScalarUI;
  static constexpr int TotalBits = sizeof(Scalar) * CHAR_BIT, MantissaBits = numext::numeric_limits<Scalar>::digits - 1,
                       ExponentBits = TotalBits - MantissaBits - 1;
 
  EIGEN_CONSTEXPR ScalarUI scalar_sign_mantissa_mask =
      ~(((ScalarUI(1) << ExponentBits) - ScalarUI(1)) << MantissaBits);  // ~0x7f800000
  const Packet sign_mantissa_mask = pset1frombits<Packet>(static_cast<ScalarUI>(scalar_sign_mantissa_mask));
  const Packet half = pset1<Packet>(Scalar(0.5));
  const Packet zero = pzero(a);
  const Packet normal_min = pset1<Packet>((numext::numeric_limits<Scalar>::min)());  // Minimum normal value, 2^-126
 
  // To handle denormals, normalize by multiplying by 2^(int(MantissaBits)+1).
  const Packet is_denormal = pcmp_lt(pabs(a), normal_min);
  EIGEN_CONSTEXPR ScalarUI scalar_normalization_offset = ScalarUI(MantissaBits + 1);  // 24
  // The following cannot be constexpr because bfloat16(uint16_t) is not constexpr.
  const Scalar scalar_normalization_factor = Scalar(ScalarUI(1) << int(scalar_normalization_offset));  // 2^24
  const Packet normalization_factor = pset1<Packet>(scalar_normalization_factor);
  const Packet normalized_a = pselect(is_denormal, pmul(a, normalization_factor), a);
 
  // Determine exponent offset: -126 if normal, -126-24 if denormal
  const Scalar scalar_exponent_offset = -Scalar((ScalarUI(1) << (ExponentBits - 1)) - ScalarUI(2));  // -126
  Packet exponent_offset = pset1<Packet>(scalar_exponent_offset);
  const Packet normalization_offset = pset1<Packet>(-Scalar(scalar_normalization_offset));  // -24
  exponent_offset = pselect(is_denormal, padd(exponent_offset, normalization_offset), exponent_offset);
 
  // Determine exponent and mantissa from normalized_a.
  exponent = pfrexp_generic_get_biased_exponent(normalized_a);
  // Zero, Inf and NaN return 'a' unmodified, exponent is zero
  // (technically the exponent is unspecified for inf/NaN, but GCC/Clang set it to zero)
  const Scalar scalar_non_finite_exponent = Scalar((ScalarUI(1) << ExponentBits) - ScalarUI(1));  // 255
  const Packet non_finite_exponent = pset1<Packet>(scalar_non_finite_exponent);
  const Packet is_zero_or_not_finite = por(pcmp_eq(a, zero), pcmp_eq(exponent, non_finite_exponent));
  const Packet m = pselect(is_zero_or_not_finite, a, por(pand(normalized_a, sign_mantissa_mask), half));
  exponent = pselect(is_zero_or_not_finite, zero, padd(exponent, exponent_offset));
  return m;
}

References a, EIGEN_CONSTEXPR, m, min, pabs(), padd(), pand(), pcmp_eq(), pcmp_lt(), pfrexp_generic_get_biased_exponent(), pmul(), por(), pselect(), pzero(), compute_granudrum_aor::type, and zero().

Referenced by pfrexp< Packet16f >(), pfrexp< Packet2d >(), pfrexp< Packet2f >(), pfrexp< Packet4d >(), pfrexp< Packet4f >(), pfrexp< Packet8d >(), pfrexp< Packet8f >(), and pfrexp< PacketXf >().

pfrexp_generic_get_biased_exponent() [1/4]

template<typename Packet >

EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Packet Eigen::internal::pfrexp_generic_get_biased_exponent

(

const Packet &

a

)

                                                                                                 {
  typedef typename unpacket_traits<Packet>::type Scalar;
  typedef typename unpacket_traits<Packet>::integer_packet PacketI;
  static constexpr int mantissa_bits = numext::numeric_limits<Scalar>::digits - 1;
  return pcast<PacketI, Packet>(plogical_shift_right<mantissa_bits>(preinterpret<PacketI>(pabs(a))));
}

References a, and pabs().

pfrexp_generic_get_biased_exponent() [2/4]

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pfrexp_generic_get_biased_exponent

(

const Packet2d &

a

)

                                                                                   {
  const Packet2d cst_exp_mask = pset1frombits<Packet2d>(static_cast<uint64_t>(0x7ff0000000000000ull));
  __m128i a_expo = _mm_srli_epi64(_mm_castpd_si128(pand(a, cst_exp_mask)), 52);
  return _mm_cvtepi32_pd(vec4i_swizzle1(a_expo, 0, 2, 1, 3));
}

References a, pand(), pset1frombits< Packet2d >(), and vec4i_swizzle1.

pfrexp_generic_get_biased_exponent() [3/4]

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::pfrexp_generic_get_biased_exponent

(

const Packet4d &

a

)

                                                                                   {
  const Packet4d cst_exp_mask = pset1frombits<Packet4d>(static_cast<uint64_t>(0x7ff0000000000000ull));
  __m256i a_expo = _mm256_castpd_si256(pand(a, cst_exp_mask));
#ifdef EIGEN_VECTORIZE_AVX2
  a_expo = _mm256_srli_epi64(a_expo, 52);
  __m128i lo = _mm256_extractf128_si256(a_expo, 0);
  __m128i hi = _mm256_extractf128_si256(a_expo, 1);
#else
  __m128i lo = _mm256_extractf128_si256(a_expo, 0);
  __m128i hi = _mm256_extractf128_si256(a_expo, 1);
  lo = _mm_srli_epi64(lo, 52);
  hi = _mm_srli_epi64(hi, 52);
#endif
  Packet2d exponent_lo = _mm_cvtepi32_pd(vec4i_swizzle1(lo, 0, 2, 1, 3));
  Packet2d exponent_hi = _mm_cvtepi32_pd(vec4i_swizzle1(hi, 0, 2, 1, 3));
  Packet4d exponent = _mm256_insertf128_pd(_mm256_setzero_pd(), exponent_lo, 0);
  exponent = _mm256_insertf128_pd(exponent, exponent_hi, 1);
  return exponent;
}

References a, pand(), pset1frombits< Packet4d >(), and vec4i_swizzle1.

Referenced by pfrexp_generic().

pfrexp_generic_get_biased_exponent() [4/4]

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::pfrexp_generic_get_biased_exponent

(

const Packet8d &

a

)

                                                                                   {
  const Packet8d cst_exp_mask = pset1frombits<Packet8d>(static_cast<uint64_t>(0x7ff0000000000000ull));
#ifdef EIGEN_VECTORIZE_AVX512DQ
  return _mm512_cvtepi64_pd(_mm512_srli_epi64(_mm512_castpd_si512(pand(a, cst_exp_mask)), 52));
#else
  return _mm512_cvtepi32_pd(_mm512_cvtepi64_epi32(_mm512_srli_epi64(_mm512_castpd_si512(pand(a, cst_exp_mask)), 52)));
#endif
}

References a, pand(), and pset1frombits< Packet8d >().

pgamma_sample_der_alpha()

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet Eigen::internal::pgamma_sample_der_alpha	(	const Packet &	alpha,
		const Packet &	sample
	)

Returns

compute the derivative of the sample of Gamma(alpha, 1) random variable with respect to the parameter a gamma_sample_der_alpha(alpha, sample)

                                                                                                                {
  using numext::gamma_sample_der_alpha;
  return gamma_sample_der_alpha(alpha, sample);
}

References alpha, and Eigen::gamma_sample_der_alpha().

Referenced by Eigen::internal::scalar_gamma_sample_der_alpha_op< Scalar >::packetOp().

pgather() [1/2]

template<typename Scalar , typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::pgather ( const Packet &  src, const Scalar *  from, Index  stride, typename unpacket_traits< Packet >::mask_t  umask  )

inline

Referenced by Eigen::internal::gemm_class< Scalar, is_unit_inc >::vaddm(), and Eigen::internal::gemm_class< Scalar, is_unit_inc >::vfmaddm().

pgather() [2/2]

template<typename Scalar , typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::pgather ( const Scalar *  from, Index    )

inline

                                                                     {
  return ploadu<Packet>(from);
}

pgather< bfloat16, Packet16bf >()

template<>

EIGEN_STRONG_INLINE Packet16bf Eigen::internal::pgather< bfloat16, Packet16bf >	(	const bfloat16 *	from,
		Index	stride
	)

                                                                                                 {
  return _mm256_set_epi16(
      from[15 * stride].value, from[14 * stride].value, from[13 * stride].value, from[12 * stride].value,
      from[11 * stride].value, from[10 * stride].value, from[9 * stride].value, from[8 * stride].value,
      from[7 * stride].value, from[6 * stride].value, from[5 * stride].value, from[4 * stride].value,
      from[3 * stride].value, from[2 * stride].value, from[1 * stride].value, from[0 * stride].value);
}

References Eigen::value.

pgather< bfloat16, Packet4bf >()

template<>

EIGEN_STRONG_INLINE Packet4bf Eigen::internal::pgather< bfloat16, Packet4bf >	(	const bfloat16 *	from,
		Index	stride
	)

                                                                                               {
  return Packet4bf(pgather<uint16_t, Packet4us>(reinterpret_cast<const uint16_t*>(from), stride));
}

References pgather< uint16_t, Packet4us >().

pgather< bfloat16, Packet8bf >()

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::pgather< bfloat16, Packet8bf >	(	const bfloat16 *	from,
		Index	stride
	)

                                                                                                                 {
  return pgather_common<Packet8bf>(from, stride);
}

Referenced by loadBF16fromResult().

pgather< bool, Packet16b >()

template<>

EIGEN_STRONG_INLINE Packet16b Eigen::internal::pgather< bool, Packet16b >	(	const bool *	from,
		Index	stride
	)

                                                                                       {
  return _mm_set_epi8(from[15 * stride], from[14 * stride], from[13 * stride], from[12 * stride], from[11 * stride],
                      from[10 * stride], from[9 * stride], from[8 * stride], from[7 * stride], from[6 * stride],
                      from[5 * stride], from[4 * stride], from[3 * stride], from[2 * stride], from[1 * stride],
                      from[0 * stride]);
}

pgather< double, Packet2d >()

template<>

EIGEN_DEVICE_FUNC Packet2d Eigen::internal::pgather< double, Packet2d > ( const double *  from, Index  stride  )

inline

                                                                                                           {
  Packet2d v = {from[0], from[stride]};
  return v;
}

References v.

pgather< double, Packet4d >()

template<>

EIGEN_DEVICE_FUNC Packet4d Eigen::internal::pgather< double, Packet4d > ( const double *  from, Index  stride  )

inline

                                                                                              {
  return _mm256_set_pd(from[3 * stride], from[2 * stride], from[1 * stride], from[0 * stride]);
}

pgather< double, Packet8d >() [1/2]

template<>

EIGEN_DEVICE_FUNC Packet8d Eigen::internal::pgather< double, Packet8d > ( const double *  from, Index  stride  )

inline

                                                                                              {
  Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
  Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
  Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
 
  return _mm512_i32gather_pd(indices, from, 8);
}

pgather< double, Packet8d >() [2/2]

template<>

EIGEN_DEVICE_FUNC Packet8d Eigen::internal::pgather< double, Packet8d > ( const Packet8d &  src, const double *  from, Index  stride, uint8_t  umask  )

inline

                                                                           {
  Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
  Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
  Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
  __mmask8 mask = static_cast<__mmask8>(umask);
 
  return _mm512_mask_i32gather_pd(src, mask, indices, from, 8);
}

Referenced by pgather< std::complex< float >, Packet8cf >().

pgather< Eigen::half, Packet16h >()

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::pgather< Eigen::half, Packet16h >	(	const Eigen::half *	from,
		Index	stride
	)

                                                                                                 {
  return _mm256_set_epi16(from[15 * stride].x, from[14 * stride].x, from[13 * stride].x, from[12 * stride].x,
                          from[11 * stride].x, from[10 * stride].x, from[9 * stride].x, from[8 * stride].x,
                          from[7 * stride].x, from[6 * stride].x, from[5 * stride].x, from[4 * stride].x,
                          from[3 * stride].x, from[2 * stride].x, from[1 * stride].x, from[0 * stride].x);
}

References plotDoE::x.

pgather< Eigen::half, Packet32h >()

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::pgather< Eigen::half, Packet32h >	(	const Eigen::half *	from,
		Index	stride
	)

                                                                                                 {
  return _mm512_castsi512_ph(_mm512_set_epi16(
      from[31 * stride].x, from[30 * stride].x, from[29 * stride].x, from[28 * stride].x, from[27 * stride].x,
      from[26 * stride].x, from[25 * stride].x, from[24 * stride].x, from[23 * stride].x, from[22 * stride].x,
      from[21 * stride].x, from[20 * stride].x, from[19 * stride].x, from[18 * stride].x, from[17 * stride].x,
      from[16 * stride].x, from[15 * stride].x, from[14 * stride].x, from[13 * stride].x, from[12 * stride].x,
      from[11 * stride].x, from[10 * stride].x, from[9 * stride].x, from[8 * stride].x, from[7 * stride].x,
      from[6 * stride].x, from[5 * stride].x, from[4 * stride].x, from[3 * stride].x, from[2 * stride].x,
      from[1 * stride].x, from[0 * stride].x));
}

References plotDoE::x.

pgather< Eigen::half, Packet8h >()

template<>

EIGEN_STRONG_INLINE Packet8h Eigen::internal::pgather< Eigen::half, Packet8h >	(	const Eigen::half *	from,
		Index	stride
	)

                                                                                               {
  const numext::uint16_t s0 = numext::bit_cast<numext::uint16_t>(from[0 * stride]);
  const numext::uint16_t s1 = numext::bit_cast<numext::uint16_t>(from[1 * stride]);
  const numext::uint16_t s2 = numext::bit_cast<numext::uint16_t>(from[2 * stride]);
  const numext::uint16_t s3 = numext::bit_cast<numext::uint16_t>(from[3 * stride]);
  const numext::uint16_t s4 = numext::bit_cast<numext::uint16_t>(from[4 * stride]);
  const numext::uint16_t s5 = numext::bit_cast<numext::uint16_t>(from[5 * stride]);
  const numext::uint16_t s6 = numext::bit_cast<numext::uint16_t>(from[6 * stride]);
  const numext::uint16_t s7 = numext::bit_cast<numext::uint16_t>(from[7 * stride]);
  return _mm_set_epi16(s7, s6, s5, s4, s3, s2, s1, s0);
}

pgather< float, Packet16f >() [1/2]

template<>

EIGEN_DEVICE_FUNC Packet16f Eigen::internal::pgather< float, Packet16f > ( const float *  from, Index  stride  )

inline

                                                                                              {
  Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
  Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
  Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
 
  return _mm512_i32gather_ps(indices, from, 4);
}

pgather< float, Packet16f >() [2/2]

template<>

EIGEN_DEVICE_FUNC Packet16f Eigen::internal::pgather< float, Packet16f > ( const Packet16f &  src, const float *  from, Index  stride, uint16_t  umask  )

inline

                                                                             {
  Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
  Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
  Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
  __mmask16 mask = static_cast<__mmask16>(umask);
 
  return _mm512_mask_i32gather_ps(src, mask, indices, from, 4);
}

pgather< float, Packet2f >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2f Eigen::internal::pgather< float, Packet2f >	(	const float *	from,
		Index	stride
	)

                                                                                                         {
  Packet2f res = vld1_dup_f32(from);
  res = vld1_lane_f32(from + 1 * stride, res, 1);
  return res;
}

References res.

pgather< float, Packet4f >()

template<>

EIGEN_DEVICE_FUNC Packet4f Eigen::internal::pgather< float, Packet4f > ( const float *  from, Index  stride  )

inline

                                                                                                         {
  return pgather_common<Packet4f>(from, stride);
}

pgather< float, Packet8f >()

template<>

EIGEN_DEVICE_FUNC Packet8f Eigen::internal::pgather< float, Packet8f > ( const float *  from, Index  stride  )

inline

                                                                                            {
  return _mm256_set_ps(from[7 * stride], from[6 * stride], from[5 * stride], from[4 * stride], from[3 * stride],
                       from[2 * stride], from[1 * stride], from[0 * stride]);
}

pgather< float, PacketXf >()

template<>

EIGEN_DEVICE_FUNC PacketXf Eigen::internal::pgather< float, PacketXf > ( const float *  from, Index  stride  )

inline

                                                                                            {
  // Indice format: {base=0, base+stride, base+stride*2, base+stride*3, ...}
  svint32_t indices = svindex_s32(0, stride);
  return svld1_gather_s32index_f32(svptrue_b32(), from, indices);
}

pgather< int, Packet16i >()

template<>

EIGEN_DEVICE_FUNC Packet16i Eigen::internal::pgather< int, Packet16i > ( const int *  from, Index  stride  )

inline

                                                                                          {
  Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
  Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
  Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
  return _mm512_i32gather_epi32(indices, from, 4);
}

pgather< int, Packet4i >()

template<>

EIGEN_DEVICE_FUNC Packet4i Eigen::internal::pgather< int, Packet4i > ( const int *  from, Index  stride  )

inline

                                                                                                     {
  return pgather_common<Packet4i>(from, stride);
}

pgather< int, Packet8i >()

template<>

EIGEN_DEVICE_FUNC Packet8i Eigen::internal::pgather< int, Packet8i > ( const int *  from, Index  stride  )

inline

                                                                                        {
  return _mm256_set_epi32(from[7 * stride], from[6 * stride], from[5 * stride], from[4 * stride], from[3 * stride],
                          from[2 * stride], from[1 * stride], from[0 * stride]);
}

Referenced by pgather< uint32_t, Packet8ui >().

pgather< int16_t, Packet4s >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4s Eigen::internal::pgather< int16_t, Packet4s >	(	const int16_t *	from,
		Index	stride
	)

                                                                                                             {
  Packet4s res = vld1_dup_s16(from);
  res = vld1_lane_s16(from + 1 * stride, res, 1);
  res = vld1_lane_s16(from + 2 * stride, res, 2);
  res = vld1_lane_s16(from + 3 * stride, res, 3);
  return res;
}

References res.

pgather< int16_t, Packet8s >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8s Eigen::internal::pgather< int16_t, Packet8s >	(	const int16_t *	from,
		Index	stride
	)

                                                                                                             {
  int16_t v[8] __attribute__((aligned(16)));
  v[0] = from[0];
  v[1] = from[stride];
  v[2] = from[2 * stride];
  v[3] = from[3 * stride];
  v[4] = from[4 * stride];
  v[5] = from[5 * stride];
  v[6] = from[6 * stride];
  v[7] = from[7 * stride];
  return __lsx_vld(v, 0);
}

References v.

pgather< int32_t, Packet2i >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2i Eigen::internal::pgather< int32_t, Packet2i >	(	const int32_t *	from,
		Index	stride
	)

                                                                                                             {
  Packet2i res = vld1_dup_s32(from);
  res = vld1_lane_s32(from + 1 * stride, res, 1);
  return res;
}

References res.

pgather< int32_t, Packet4i >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4i Eigen::internal::pgather< int32_t, Packet4i > ( const int32_t *  from, Index  stride  )

inline

                                                                                                             {
  int32_t v[4] __attribute__((aligned(16)));
  v[0] = from[0];
  v[1] = from[stride];
  v[2] = from[2 * stride];
  v[3] = from[3 * stride];
  return __lsx_vld(v, 0);
}

References v.

pgather< int64_t, Packet2l >()

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::pgather< int64_t, Packet2l >	(	const int64_t *	from,
		Index	stride
	)

                                                                                                             {
  int64_t v[2] __attribute__((aligned(16)));
  v[0] = from[0];
  v[1] = from[stride];
  return __lsx_vld(v, 0);
}

References v.

pgather< int64_t, Packet8l >()

template<>

EIGEN_DEVICE_FUNC Packet8l Eigen::internal::pgather< int64_t, Packet8l > ( const int64_t *  from, Index  stride  )

inline

                                                                                                {
  Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
  Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
  Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
 
  return _mm512_i32gather_epi64(indices, from, 8);
}

pgather< int8_t, Packet16c >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16c Eigen::internal::pgather< int8_t, Packet16c >	(	const int8_t *	from,
		Index	stride
	)

                                                                                                             {
  int8_t v[16] __attribute__((aligned(16)));
  v[0] = from[0];
  v[1] = from[stride];
  v[2] = from[2 * stride];
  v[3] = from[3 * stride];
  v[4] = from[4 * stride];
  v[5] = from[5 * stride];
  v[6] = from[6 * stride];
  v[7] = from[7 * stride];
  v[8] = from[8 * stride];
  v[9] = from[9 * stride];
  v[10] = from[10 * stride];
  v[11] = from[11 * stride];
  v[12] = from[12 * stride];
  v[13] = from[13 * stride];
  v[14] = from[14 * stride];
  v[15] = from[15 * stride];
  return __lsx_vld(v, 0);
}

References v.

pgather< int8_t, Packet4c >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4c Eigen::internal::pgather< int8_t, Packet4c >	(	const int8_t *	from,
		Index	stride
	)

                                                                                                           {
  Packet4c res;
  for (int i = 0; i != 4; i++) reinterpret_cast<int8_t*>(&res)[i] = *(from + i * stride);
  return res;
}

References i, and res.

pgather< int8_t, Packet8c >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8c Eigen::internal::pgather< int8_t, Packet8c >	(	const int8_t *	from,
		Index	stride
	)

                                                                                                           {
  Packet8c res = vld1_dup_s8(from);
  res = vld1_lane_s8(from + 1 * stride, res, 1);
  res = vld1_lane_s8(from + 2 * stride, res, 2);
  res = vld1_lane_s8(from + 3 * stride, res, 3);
  res = vld1_lane_s8(from + 4 * stride, res, 4);
  res = vld1_lane_s8(from + 5 * stride, res, 5);
  res = vld1_lane_s8(from + 6 * stride, res, 6);
  res = vld1_lane_s8(from + 7 * stride, res, 7);
  return res;
}

References res.

pgather< numext::int32_t, PacketXi >()

template<>

EIGEN_DEVICE_FUNC PacketXi Eigen::internal::pgather< numext::int32_t, PacketXi > ( const numext::int32_t *  from, Index  stride  )

inline

                                                                                                            {
  // Indice format: {base=0, base+stride, base+stride*2, base+stride*3, ...}
  svint32_t indices = svindex_s32(0, stride);
  return svld1_gather_s32index_s32(svptrue_b32(), from, indices);
}

pgather< short int, Packet8s >()

template<>

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet8s Eigen::internal::pgather< short int, Packet8s >	(	const short int *	from,
		Index	stride
	)

                                                                                                                 {
  return pgather_common<Packet8s>(from, stride);
}

pgather< signed char, Packet16c >()

template<>

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet16c Eigen::internal::pgather< signed char, Packet16c >	(	const signed char *	from,
		Index	stride
	)

                                                                                                                       {
  return pgather_common<Packet16c>(from, stride);
}

pgather< std::complex< double >, Packet1cd >() [1/3]

template<>

EIGEN_DEVICE_FUNC Packet1cd Eigen::internal::pgather< std::complex< double >, Packet1cd > ( const std::complex< double > *  from, Index stride   __attribute__(unused)  )

inline

                                                                                                                  {
  EIGEN_MSA_DEBUG;
 
  Packet1cd res;
  res.v[0] = std::real(from[0]);
  res.v[1] = std::imag(from[0]);
  return res;
}

References EIGEN_MSA_DEBUG, imag(), and res.

pgather< std::complex< double >, Packet1cd >() [2/3]

template<>

EIGEN_DEVICE_FUNC Packet1cd Eigen::internal::pgather< std::complex< double >, Packet1cd > ( const std::complex< double > *  from, Index stride  EIGEN_UNUSED  )

inline

                                                                                                       {
  return pload<Packet1cd>(from);
}

References pload< Packet1cd >().

pgather< std::complex< double >, Packet1cd >() [3/3]

template<>

EIGEN_DEVICE_FUNC Packet1cd Eigen::internal::pgather< std::complex< double >, Packet1cd > ( const std::complex< double > *  from, Index     )

inline

                                                                                     {
  Packet1cd res;
  __m128i tmp = __lsx_vld((void*)from, 0);
  res.v = (__m128d)tmp;
  return res;
}

References res, and tmp.

pgather< std::complex< double >, Packet2cd >()

template<>

EIGEN_DEVICE_FUNC Packet2cd Eigen::internal::pgather< std::complex< double >, Packet2cd > ( const std::complex< double > *  from, Index  stride  )

inline

                                                                                          {
  return Packet2cd(_mm256_set_pd(std::imag(from[1 * stride]), std::real(from[1 * stride]), std::imag(from[0 * stride]),
                                 std::real(from[0 * stride])));
}

References imag().

pgather< std::complex< double >, Packet4cd >()

template<>

EIGEN_DEVICE_FUNC Packet4cd Eigen::internal::pgather< std::complex< double >, Packet4cd > ( const std::complex< double > *  from, Index  stride  )

inline

                                                                                          {
  return Packet4cd(_mm512_insertf64x4(
      _mm512_castpd256_pd512(_mm256_insertf128_pd(_mm256_castpd128_pd256(ploadu<Packet1cd>(from + 0 * stride).v),
                                                  ploadu<Packet1cd>(from + 1 * stride).v, 1)),
      _mm256_insertf128_pd(_mm256_castpd128_pd256(ploadu<Packet1cd>(from + 2 * stride).v),
                           ploadu<Packet1cd>(from + 3 * stride).v, 1),
      1));
}

References ploadu< Packet1cd >(), and v.

pgather< std::complex< float >, Packet1cf >()

template<>

EIGEN_DEVICE_FUNC Packet1cf Eigen::internal::pgather< std::complex< float >, Packet1cf > ( const std::complex< float > *  from, Index  stride  )

inline

                                                                                         {
  const Packet2f tmp = vdup_n_f32(std::real(from[0 * stride]));
  return Packet1cf(vset_lane_f32(std::imag(from[0 * stride]), tmp, 1));
}

References imag(), and tmp.

pgather< std::complex< float >, Packet2cf >()

template<>

EIGEN_DEVICE_FUNC Packet2cf Eigen::internal::pgather< std::complex< float >, Packet2cf > ( const std::complex< float > *  from, Index  stride  )

inline

                                                                                                      {
  return pgather_complex_size2<std::complex<float>, Packet2cf>(from, stride);
}

pgather< std::complex< float >, Packet4cf >()

template<>

EIGEN_DEVICE_FUNC Packet4cf Eigen::internal::pgather< std::complex< float >, Packet4cf > ( const std::complex< float > *  from, Index  stride  )

inline

                                                                                         {
  return Packet4cf(_mm256_set_ps(std::imag(from[3 * stride]), std::real(from[3 * stride]), std::imag(from[2 * stride]),
                                 std::real(from[2 * stride]), std::imag(from[1 * stride]), std::real(from[1 * stride]),
                                 std::imag(from[0 * stride]), std::real(from[0 * stride])));
}

References imag().

pgather< std::complex< float >, Packet8cf >()

template<>

EIGEN_DEVICE_FUNC Packet8cf Eigen::internal::pgather< std::complex< float >, Packet8cf > ( const std::complex< float > *  from, Index  stride  )

inline

                                                                                         {
  return Packet8cf(_mm512_castpd_ps(pgather<double, Packet8d>((const double*)(const void*)from, stride)));
}

References pgather< double, Packet8d >().

pgather< uint16_t, Packet4us >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4us Eigen::internal::pgather< uint16_t, Packet4us >	(	const uint16_t *	from,
		Index	stride
	)

                                                                                                                 {
  Packet4us res = vld1_dup_u16(from);
  res = vld1_lane_u16(from + 1 * stride, res, 1);
  res = vld1_lane_u16(from + 2 * stride, res, 2);
  res = vld1_lane_u16(from + 3 * stride, res, 3);
  return res;
}

References res.

Referenced by pgather< bfloat16, Packet4bf >().

pgather< uint16_t, Packet8us >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8us Eigen::internal::pgather< uint16_t, Packet8us >	(	const uint16_t *	from,
		Index	stride
	)

                                                                                                                 {
  uint16_t v[8] __attribute__((aligned(16)));
  v[0] = from[0];
  v[1] = from[stride];
  v[2] = from[2 * stride];
  v[3] = from[3 * stride];
  v[4] = from[4 * stride];
  v[5] = from[5 * stride];
  v[6] = from[6 * stride];
  v[7] = from[7 * stride];
  return __lsx_vld(v, 0);
}

References v.

pgather< uint32_t, Packet2ui >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ui Eigen::internal::pgather< uint32_t, Packet2ui >	(	const uint32_t *	from,
		Index	stride
	)

                                                                                                                 {
  Packet2ui res = vld1_dup_u32(from);
  res = vld1_lane_u32(from + 1 * stride, res, 1);
  return res;
}

References res.

pgather< uint32_t, Packet4ui >()

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::pgather< uint32_t, Packet4ui >	(	const uint32_t *	from,
		Index	stride
	)

                                                                                                                 {
  uint32_t v[4] __attribute__((aligned(16)));
  v[0] = from[0];
  v[1] = from[stride];
  v[2] = from[2 * stride];
  v[3] = from[3 * stride];
  return __lsx_vld(v, 0);
}

References v.

pgather< uint32_t, Packet8ui >()

template<>

EIGEN_DEVICE_FUNC Packet8ui Eigen::internal::pgather< uint32_t, Packet8ui > ( const uint32_t *  from, Index  stride  )

inline

                                                                                                    {
  return (Packet8ui)pgather<int, Packet8i>((int*)from, stride);
}

References pgather< int, Packet8i >().

pgather< uint64_t, Packet2ul >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ul Eigen::internal::pgather< uint64_t, Packet2ul >	(	const uint64_t *	from,
		Index	stride
	)

                                                                                                                 {
  uint64_t v[2] __attribute__((aligned(16)));
  v[0] = from[0];
  v[1] = from[stride];
  return __lsx_vld(v, 0);
}

References v.

pgather< uint8_t, Packet16uc >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16uc Eigen::internal::pgather< uint8_t, Packet16uc >	(	const uint8_t *	from,
		Index	stride
	)

                                                                                                                 {
  uint8_t v[16] __attribute__((aligned(16)));
  v[0] = from[0];
  v[1] = from[stride];
  v[2] = from[2 * stride];
  v[3] = from[3 * stride];
  v[4] = from[4 * stride];
  v[5] = from[5 * stride];
  v[6] = from[6 * stride];
  v[7] = from[7 * stride];
  v[8] = from[8 * stride];
  v[9] = from[9 * stride];
  v[10] = from[10 * stride];
  v[11] = from[11 * stride];
  v[12] = from[12 * stride];
  v[13] = from[13 * stride];
  v[14] = from[14 * stride];
  v[15] = from[15 * stride];
  return __lsx_vld(v, 0);
}

References v.

pgather< uint8_t, Packet4uc >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4uc Eigen::internal::pgather< uint8_t, Packet4uc >	(	const uint8_t *	from,
		Index	stride
	)

                                                                                                               {
  Packet4uc res;
  for (int i = 0; i != 4; i++) reinterpret_cast<uint8_t*>(&res)[i] = *(from + i * stride);
  return res;
}

References i, and res.

pgather< uint8_t, Packet8uc >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8uc Eigen::internal::pgather< uint8_t, Packet8uc >	(	const uint8_t *	from,
		Index	stride
	)

                                                                                                               {
  Packet8uc res = vld1_dup_u8(from);
  res = vld1_lane_u8(from + 1 * stride, res, 1);
  res = vld1_lane_u8(from + 2 * stride, res, 2);
  res = vld1_lane_u8(from + 3 * stride, res, 3);
  res = vld1_lane_u8(from + 4 * stride, res, 4);
  res = vld1_lane_u8(from + 5 * stride, res, 5);
  res = vld1_lane_u8(from + 6 * stride, res, 6);
  res = vld1_lane_u8(from + 7 * stride, res, 7);
  return res;
}

References res.

pgather< unsigned char, Packet16uc >()

template<>

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet16uc Eigen::internal::pgather< unsigned char, Packet16uc >	(	const unsigned char *	from,
		Index	stride
	)

                                                                                                  {
  return pgather_common<Packet16uc>(from, stride);
}

pgather< unsigned short int, Packet8us >()

template<>

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet8us Eigen::internal::pgather< unsigned short int, Packet8us >	(	const unsigned short int *	from,
		Index	stride
	)

                                                                                                     {
  return pgather_common<Packet8us>(from, stride);
}

pgather_common()

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet Eigen::internal::pgather_common	(	const __UNPACK_TYPE__(Packet) *	from,
		Index	stride,
		const Index	n = unpacket_traits<Packet>::size
	)

                                                                                                         {
  EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) a[unpacket_traits<Packet>::size];
  eigen_internal_assert(n <= unpacket_traits<Packet>::size && "number of elements will gather past end of packet");
  if (stride == 1) {
    if (n == unpacket_traits<Packet>::size) {
      return ploadu<Packet>(from);
    } else {
      return ploadu_partial<Packet>(from, n);
    }
  } else {
    LOAD_STORE_UNROLL_16
    for (Index i = 0; i < n; i++) {
      a[i] = from[i * stride];
    }
    // Leave rest of the array uninitialized
    return pload_ignore<Packet>(a);
  }
}

References __UNPACK_TYPE__(), a, EIGEN_ALIGN16, eigen_internal_assert, i, LOAD_STORE_UNROLL_16, and n.

pgather_complex_size2()

template<typename Scalar , typename Packet >

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet Eigen::internal::pgather_complex_size2	(	const Scalar *	from,
		Index	stride,
		const Index	n = 2
	)

                                                                                      {
  eigen_internal_assert(n <= unpacket_traits<Packet>::size && "number of elements will gather past end of packet");
  EIGEN_ALIGN16 Scalar af[2];
  for (Index i = 0; i < n; i++) {
    af[i] = from[i * stride];
  }
  return pload_ignore<Packet>(af);
}

References EIGEN_ALIGN16, eigen_internal_assert, i, and n.

pgather_partial()

template<typename Scalar , typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::pgather_partial ( const Scalar *  from, Index  stride, const Index  n  )

inline

                                                                                                 {
  const Index packet_size = unpacket_traits<Packet>::size;
  EIGEN_ALIGN_MAX Scalar elements[packet_size] = {Scalar(0)};
  for (Index i = 0; i < numext::mini(n, packet_size); i++) {
    elements[i] = from[i * stride];
  }
  return pload<Packet>(elements);
}

References EIGEN_ALIGN_MAX, i, Eigen::numext::mini(), and n.

pgather_partial< bfloat16, Packet8bf >()

template<>

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet8bf Eigen::internal::pgather_partial< bfloat16, Packet8bf >	(	const bfloat16 *	from,
		Index	stride,
		const Index	n
	)

                                                                                                    {
  return pgather_common<Packet8bf>(from, stride, n);
}

References n.

pgather_partial< float, Packet4f >()

template<>

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet4f Eigen::internal::pgather_partial< float, Packet4f >	(	const float *	from,
		Index	stride,
		const Index	n
	)

                                                                                               {
  return pgather_common<Packet4f>(from, stride, n);
}

References n.

pgather_partial< int, Packet4i >()

template<>

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet4i Eigen::internal::pgather_partial< int, Packet4i >	(	const int *	from,
		Index	stride,
		const Index	n
	)

                                                                                             {
  return pgather_common<Packet4i>(from, stride, n);
}

References n.

pgather_partial< short int, Packet8s >()

template<>

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet8s Eigen::internal::pgather_partial< short int, Packet8s >	(	const short int *	from,
		Index	stride,
		const Index	n
	)

                                                                                                   {
  return pgather_common<Packet8s>(from, stride, n);
}

References n.

pgather_partial< signed char, Packet16c >()

template<>

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet16c Eigen::internal::pgather_partial< signed char, Packet16c >	(	const signed char *	from,
		Index	stride,
		const Index	n
	)

                                                                                                                     {
  return pgather_common<Packet16c>(from, stride, n);
}

References n.

pgather_partial< std::complex< float >, Packet2cf >()

template<>

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet2cf Eigen::internal::pgather_partial< std::complex< float >, Packet2cf >	(	const std::complex< float > *	from,
		Index	stride,
		const Index	n
	)

                                                                                                          {
  return pgather_complex_size2<std::complex<float>, Packet2cf>(from, stride, n);
}

References n.

pgather_partial< unsigned char, Packet16uc >()

template<>

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet16uc Eigen::internal::pgather_partial< unsigned char, Packet16uc >	(	const unsigned char *	from,
		Index	stride,
		const Index	n
	)

                                                                                                           {
  return pgather_common<Packet16uc>(from, stride, n);
}

References n.

pgather_partial< unsigned short int, Packet8us >()

template<>

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet8us Eigen::internal::pgather_partial< unsigned short int, Packet8us >	(	const unsigned short int *	from,
		Index	stride,
		const Index	n
	)

                                                                                                            {
  return pgather_common<Packet8us>(from, stride, n);
}

References n.

pger()

template<int N, typename Scalar , typename Packet , bool NegativeAccumulate>

EIGEN_ALWAYS_INLINE void Eigen::internal::pger	(	PacketBlock< Packet, N > *	acc,
		const Scalar *	lhs,
		const Packet *	rhsV
	)

                                                                                                  {
  Packet lhsV = pload<Packet>(lhs);
 
  pger_common<Packet, NegativeAccumulate, N>(acc, lhsV, rhsV);
}

pger_common()

template<typename Packet , bool NegativeAccumulate, int N>

EIGEN_ALWAYS_INLINE void Eigen::internal::pger_common	(	PacketBlock< Packet, N > *	acc,
		const Packet &	lhsV,
		const Packet *	rhsV
	)

                                                                                                          {
  if (NegativeAccumulate) {
    for (int M = 0; M < N; M++) {
      acc->packet[M] = vec_nmsub(lhsV, rhsV[M], acc->packet[M]);
    }
  } else {
    for (int M = 0; M < N; M++) {
      acc->packet[M] = vec_madd(lhsV, rhsV[M], acc->packet[M]);
    }
  }
}

References N, and Eigen::internal::PacketBlock< Packet, N >::packet.

pgerc()

template<int N, typename Scalar , typename Packet , bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>

EIGEN_ALWAYS_INLINE void Eigen::internal::pgerc	(	PacketBlock< Packet, N > *	accReal,
		PacketBlock< Packet, N > *	accImag,
		const Scalar *	lhs_ptr,
		const Scalar *	lhs_ptr_imag,
		const Packet *	rhsV,
		const Packet *	rhsVi
	)

                                                                                                    {
  Packet lhsV = ploadLhs<Packet>(lhs_ptr);
  Packet lhsVi;
  if (!LhsIsReal)
    lhsVi = ploadLhs<Packet>(lhs_ptr_imag);
  else
    EIGEN_UNUSED_VARIABLE(lhs_ptr_imag);
 
  pgerc_common<N, Packet, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(accReal, accImag, lhsV, lhsVi, rhsV, rhsVi);
}

References EIGEN_UNUSED_VARIABLE.

pgerc_common()

template<int N, typename Packet , bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>

EIGEN_ALWAYS_INLINE void Eigen::internal::pgerc_common	(	PacketBlock< Packet, N > *	accReal,
		PacketBlock< Packet, N > *	accImag,
		const Packet &	lhsV,
		Packet &	lhsVi,
		const Packet *	rhsV,
		const Packet *	rhsVi
	)

                                                                                                                  {
  pger_common<Packet, false, N>(accReal, lhsV, rhsV);
  if (LhsIsReal) {
    pger_common<Packet, ConjugateRhs, N>(accImag, lhsV, rhsVi);
    EIGEN_UNUSED_VARIABLE(lhsVi);
  } else {
    if (!RhsIsReal) {
      pger_common<Packet, ConjugateLhs == ConjugateRhs, N>(accReal, lhsVi, rhsVi);
      pger_common<Packet, ConjugateRhs, N>(accImag, lhsV, rhsVi);
    } else {
      EIGEN_UNUSED_VARIABLE(rhsVi);
    }
    pger_common<Packet, ConjugateLhs, N>(accImag, lhsVi, rhsV);
  }
}

References EIGEN_UNUSED_VARIABLE.

pgercMMA()

template<typename Packet , typename RhsPacket , bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>

EIGEN_ALWAYS_INLINE void Eigen::internal::pgercMMA	(	__vector_quad *	accReal,
		__vector_quad *	accImag,
		const Packet &	lhsV,
		Packet &	lhsVi,
		const RhsPacket &	rhsV,
		RhsPacket &	rhsVi
	)

                                                                           {
  pgerMMA<Packet, RhsPacket, false>(accReal, rhsV, lhsV);
  if (LhsIsReal) {
    pgerMMA<Packet, RhsPacket, ConjugateRhs>(accImag, rhsVi, lhsV);
    EIGEN_UNUSED_VARIABLE(lhsVi);
  } else {
    if (!RhsIsReal) {
      pgerMMA<Packet, RhsPacket, ConjugateLhs == ConjugateRhs>(accReal, rhsVi, lhsVi);
      pgerMMA<Packet, RhsPacket, ConjugateRhs>(accImag, rhsVi, lhsV);
    } else {
      EIGEN_UNUSED_VARIABLE(rhsVi);
    }
    pgerMMA<Packet, RhsPacket, ConjugateLhs>(accImag, rhsV, lhsVi);
  }
}

References EIGEN_UNUSED_VARIABLE.

pgerMMA() [1/2]

template<typename LhsPacket , typename RhsPacket , bool NegativeAccumulate>

EIGEN_ALWAYS_INLINE void Eigen::internal::pgerMMA	(	__vector_quad *	acc,
		const __vector_pair &	a,
		const Packet2d &	b
	)

                                                                                                {
  if (NegativeAccumulate) {
    __builtin_mma_xvf64gernp(acc, (__vector_pair)a, (__vector unsigned char)b);
  } else {
    __builtin_mma_xvf64gerpp(acc, (__vector_pair)a, (__vector unsigned char)b);
  }
}

References a, and b.

pgerMMA() [2/2]

template<typename LhsPacket , typename RhsPacket , bool NegativeAccumulate>

EIGEN_ALWAYS_INLINE void Eigen::internal::pgerMMA	(	__vector_quad *	acc,
		const RhsPacket &	a,
		const LhsPacket &	b
	)

                                                                                             {
  if (NegativeAccumulate) {
    __builtin_mma_xvf32gernp(acc, (__vector unsigned char)a, (__vector unsigned char)b);
  } else {
    __builtin_mma_xvf32gerpp(acc, (__vector unsigned char)a, (__vector unsigned char)b);
  }
}

References a, and b.

phypot_complex()

template<typename Packet >

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::phypot_complex

(

const Packet &

a

)

                                                                                           {
  typedef typename unpacket_traits<Packet>::type Scalar;
  typedef typename Scalar::value_type RealScalar;
  typedef typename unpacket_traits<Packet>::as_real RealPacket;
 
  const RealPacket cst_zero_rp = pset1<RealPacket>(static_cast<RealScalar>(0.0));
  const RealPacket cst_minus_one_rp = pset1<RealPacket>(static_cast<RealScalar>(-1.0));
  const RealPacket cst_two_rp = pset1<RealPacket>(static_cast<RealScalar>(2.0));
  const RealPacket evenmask = peven_mask(a.v);
 
  RealPacket a_abs = pabs(a.v);
  RealPacket a_flip = pcplxflip(Packet(a_abs)).v;       // |b|, |a|
  RealPacket a_all = pselect(evenmask, a_abs, a_flip);  // |a|, |a|
  RealPacket b_all = pselect(evenmask, a_flip, a_abs);  // |b|, |b|
 
  RealPacket a2 = pmul(a.v, a.v);                    // |a^2, b^2|
  RealPacket a2_flip = pcplxflip(Packet(a2)).v;      // |b^2, a^2|
  RealPacket h = psqrt(padd(a2, a2_flip));           // |sqrt(a^2 + b^2), sqrt(a^2 + b^2)|
  RealPacket h_sq = pmul(h, h);                      // |a^2 + b^2, a^2 + b^2|
  RealPacket a_sq = pselect(evenmask, a2, a2_flip);  // |a^2, a^2|
  RealPacket m_h_sq = pmul(h_sq, cst_minus_one_rp);
  RealPacket m_a_sq = pmul(a_sq, cst_minus_one_rp);
  RealPacket x = psub(psub(pmadd(h, h, m_h_sq), pmadd(b_all, b_all, psub(a_sq, h_sq))), pmadd(a_all, a_all, m_a_sq));
  h = psub(h, pdiv(x, pmul(cst_two_rp, h)));  // |h - x/(2*h), h - x/(2*h)|
 
  // handle zero-case
  RealPacket iszero = pcmp_eq(por(a_abs, a_flip), cst_zero_rp);
 
  h = pandnot(h, iszero);  // |sqrt(a^2+b^2), sqrt(a^2+b^2)|
  return Packet(h);        // |sqrt(a^2+b^2), sqrt(a^2+b^2)|
}

References a, pabs(), padd(), pandnot(), pcmp_eq(), pcplxflip(), pdiv(), peven_mask(), pmadd(), pmul(), por(), pselect(), psqrt(), psub(), Eigen::internal::Packet2cf::v, and plotDoE::x.

Referenced by plog_complex().

pigamma()

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet Eigen::internal::pigamma	(	const Packet &	a,
		const Packet &	x
	)

Returns

the incomplete gamma function igamma(a, x)

                                                                                       {
  using numext::igamma;
  return igamma(a, x);
}

References a, Eigen::igamma(), and plotDoE::x.

Referenced by Eigen::internal::scalar_igamma_op< Scalar >::packetOp().

pigamma_der_a()

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet Eigen::internal::pigamma_der_a	(	const Packet &	a,
		const Packet &	x
	)

Returns

the derivative of the incomplete gamma function igamma_der_a(a, x)

                                                                                             {
  using numext::igamma_der_a;
  return igamma_der_a(a, x);
}

References a, Eigen::igamma_der_a(), and plotDoE::x.

Referenced by Eigen::internal::scalar_igamma_der_a_op< Scalar >::packetOp().

pigammac()

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet Eigen::internal::pigammac	(	const Packet &	a,
		const Packet &	x
	)

Returns

the complementary incomplete gamma function igammac(a, x)

                                                                                        {
  using numext::igammac;
  return igammac(a, x);
}

References a, Eigen::igammac(), and plotDoE::x.

Referenced by Eigen::internal::scalar_igammac_op< Scalar >::packetOp().

pisinf()

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::pisinf ( const Packet &  a )

inline

Returns

isinf(a)

                                                        {
  using Scalar = typename unpacket_traits<Packet>::type;
  constexpr Scalar inf = NumTraits<Scalar>::infinity();
  return pcmp_eq(pabs(a), pset1<Packet>(inf));
}

References a, constants::inf, pabs(), and pcmp_eq().

Referenced by Eigen::internal::scalar_isinf_op< Scalar, true >::packetOp().

pisnan() [1/3]

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::pisnan ( const Packet &  a )

inline

Returns

isnan(a)

                                                        {
  return pandnot(ptrue(a), pcmp_eq(a, a));
}

References a, pandnot(), pcmp_eq(), and ptrue().

pisnan() [2/3]

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::pisnan

(

const Packet16f &

a

)

                                                         {
  __mmask16 mask = _mm512_cmp_ps_mask(a, a, _CMP_UNORD_Q);
  return _mm512_castsi512_ps(_mm512_maskz_set1_epi32(mask, int32_t(-1)));
}

References a.

pisnan() [3/3]

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::pisnan

(

const Packet8f &

a

)

                                                       {
  return _mm256_cmp_ps(a, a, _CMP_UNORD_Q);
}

References a.

Referenced by generic_pow(), Eigen::internal::scalar_isnan_op< Scalar, true >::packetOp(), pexp_complex(), psqrt_complex(), and Eigen::internal::generic_rsqrt_newton_step< Packet, Steps >::run().

plain_enum_asserts()

template<typename A , typename B >

constexpr void Eigen::internal::plain_enum_asserts ( A  , B    )

inlineconstexpr

                                               {
  static_assert(is_int_or_enum_v<A>, "Argument a must be an integer or enum");
  static_assert(is_int_or_enum_v<B>, "Argument b must be an integer or enum");
}

Referenced by enum_eq_not_dynamic(), enum_ge_not_dynamic(), enum_gt_not_dynamic(), enum_le_not_dynamic(), enum_lt_not_dynamic(), max_size_prefer_dynamic(), min_size_prefer_dynamic(), min_size_prefer_fixed(), plain_enum_max(), and plain_enum_min().

plain_enum_max()

template<typename A , typename B >

constexpr int Eigen::internal::plain_enum_max ( A  a, B  b  )

inlineconstexpr

Gets the maximum of two values which may be integers or enums

                                              {
  plain_enum_asserts(a, b);
  return ((int)a >= (int)b) ? (int)a : (int)b;
}

References a, b, int(), and plain_enum_asserts().

Referenced by Eigen::internal::block_evaluator< ArgType, BlockRows, BlockCols, InnerPanel, true >::block_evaluator(), max_size_prefer_dynamic(), Eigen::internal::reshaped_evaluator< ArgType, Rows, Cols, Order, true >::reshaped_evaluator(), vectorization_logic< Scalar, Enable >::run(), vectorization_logic_half< Scalar, Enable >::run(), Eigen::internal::redux_impl< Func, Evaluator, LinearVectorizedTraversal, NoUnrolling >::run(), Eigen::internal::triangular_solve_matrix< Scalar, Index, OnTheLeft, Mode, Conjugate, TriStorageOrder, ColMajor, OtherInnerStride >::run(), and Eigen::internal::triangular_solve_matrix< Scalar, Index, OnTheRight, Mode, Conjugate, TriStorageOrder, ColMajor, OtherInnerStride >::run().

plain_enum_min()

template<typename A , typename B >

constexpr int Eigen::internal::plain_enum_min ( A  a, B  b  )

inlineconstexpr

Gets the minimum of two values which may be integers or enums

                                              {
  plain_enum_asserts(a, b);
  return ((int)a <= (int)b) ? (int)a : (int)b;
}

References a, b, int(), and plain_enum_asserts().

Referenced by min_size_prefer_dynamic(), min_size_prefer_fixed(), Eigen::internal::unary_evaluator< CwiseUnaryOp< core_cast_op< SrcType, DstType >, ArgType >, IndexBased >::packet(), Eigen::internal::diagonal_product_evaluator_base< MatrixType, DiagonalType, Derived, ProductOrder >::packet_impl(), vectorization_logic< Scalar, Enable >::run(), Eigen::internal::gemv_dense_selector< OnTheRight, ColMajor, true >::run(), Eigen::internal::selfadjoint_product_impl< Lhs, LhsMode, false, Rhs, 0, true >::run(), and Eigen::internal::Assignment< DstXprType, Inverse< XprType >, internal::assign_op< typename DstXprType::Scalar, typename XprType::Scalar >, Dense2Dense >::run().

pldexp() [1/6]

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::pldexp ( const Packet &  a, const Packet &  exponent  )

inline

Returns

a * 2^((int)exponent) See https://en.cppreference.com/w/cpp/numeric/math/ldexp

                                                                                {
  EIGEN_USING_STD(ldexp)
  return static_cast<Packet>(ldexp(a, static_cast<int>(exponent)));
}

References a, and EIGEN_USING_STD.

pldexp() [2/6]

template<>

EIGEN_STRONG_INLINE Packet16bf Eigen::internal::pldexp	(	const Packet16bf &	a,
		const Packet16bf &	exponent
	)

                                                                                       {
  return F32ToBf16(pldexp<Packet16f>(Bf16ToF32(a), Bf16ToF32(exponent)));
}

References a, Bf16ToF32(), F32ToBf16(), and pldexp< Packet16f >().

pldexp() [3/6]

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::pldexp	(	const Packet16h &	a,
		const Packet16h &	exponent
	)

                                                                                    {
  return float2half(pldexp<Packet16f>(half2float(a), half2float(exponent)));
}

References a, float2half(), half2float(), and pldexp< Packet16f >().

pldexp() [4/6]

template<>

EIGEN_STRONG_INLINE Packet4bf Eigen::internal::pldexp	(	const Packet4bf &	a,
		const Packet4bf &	exponent
	)

                                                                                    {
  return F32ToBf16(pldexp<Packet4f>(Bf16ToF32(a), Bf16ToF32(exponent)));
}

References a, Bf16ToF32(), F32ToBf16(), and pldexp< Packet4f >().

pldexp() [5/6]

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::pldexp	(	const Packet8bf &	a,
		const Packet8bf &	exponent
	)

                                                                                    {
  return F32ToBf16(pldexp<Packet8f>(Bf16ToF32(a), Bf16ToF32(exponent)));
}

References a, Bf16ToF32(), F32ToBf16(), and pldexp< Packet8f >().

pldexp() [6/6]

template<>

EIGEN_STRONG_INLINE Packet8h Eigen::internal::pldexp	(	const Packet8h &	a,
		const Packet8h &	exponent
	)

                                                                                 {
  return float2half(pldexp<Packet8f>(half2float(a), half2float(exponent)));
}

References a, float2half(), half2float(), and pldexp< Packet8f >().

Referenced by generic_pow_impl(), packetmath_real(), pexp_double(), pexp_float(), and pldexp< Packet32h >().

pldexp< Packet16f >()

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::pldexp< Packet16f >	(	const Packet16f &	a,
		const Packet16f &	exponent
	)

                                                                                               {
  return pldexp_generic(a, exponent);
}

References a, and pldexp_generic().

Referenced by pldexp().

pldexp< Packet16h >()

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::pldexp< Packet16h >	(	const Packet16h &	,
		const Packet16h &
	)

pldexp< Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pldexp< Packet2d >	(	const Packet2d &	a,
		const Packet2d &	exponent
	)

                                                                                           {
  return pldexp_generic(a, exponent);
}

References a, and pldexp_generic().

pldexp< Packet2f >()

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::pldexp< Packet2f >	(	const Packet2f &	a,
		const Packet2f &	exponent
	)

                                                                                           {
  return pldexp_generic(a, exponent);
}

References a, and pldexp_generic().

pldexp< Packet32h >()

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::pldexp< Packet32h >	(	const Packet32h &	a,
		const Packet32h &	exponent
	)

                                                                                               {
  Packet16h low;
  Packet16h high;
  extract2Packet16h(a, low, high);
 
  Packet16h exp1;
  Packet16h exp2;
  extract2Packet16h(exponent, exp1, exp2);
 
  Packet16h lowOut = pldexp(low, exp1);
  Packet16h highOut = pldexp(high, exp2);
 
  return combine2Packet16h(lowOut, highOut);
}

References a, combine2Packet16h(), Eigen::bfloat16_impl::exp2(), extract2Packet16h(), and pldexp().

pldexp< Packet4d >()

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::pldexp< Packet4d >	(	const Packet4d &	a,
		const Packet4d &	exponent
	)

                                                                                           {
  // Clamp exponent to [-2099, 2099]
  const Packet4d max_exponent = pset1<Packet4d>(2099.0);
  const Packet4i e = _mm256_cvtpd_epi32(pmin(pmax(exponent, pnegate(max_exponent)), max_exponent));
 
  // Split 2^e into four factors and multiply.
  const Packet4i bias = pset1<Packet4i>(1023);
  Packet4i b = parithmetic_shift_right<2>(e);  // floor(e/4)
 
  // 2^b
  Packet4i hi = vec4i_swizzle1(padd(b, bias), 0, 2, 1, 3);
  Packet4i lo = _mm_slli_epi64(hi, 52);
  hi = _mm_slli_epi64(_mm_srli_epi64(hi, 32), 52);
  Packet4d c = _mm256_castsi256_pd(_mm256_insertf128_si256(_mm256_castsi128_si256(lo), hi, 1));
  Packet4d out = pmul(pmul(pmul(a, c), c), c);  // a * 2^(3b)
 
  // 2^(e - 3b)
  b = psub(psub(psub(e, b), b), b);  // e - 3b
  hi = vec4i_swizzle1(padd(b, bias), 0, 2, 1, 3);
  lo = _mm_slli_epi64(hi, 52);
  hi = _mm_slli_epi64(_mm_srli_epi64(hi, 32), 52);
  c = _mm256_castsi256_pd(_mm256_insertf128_si256(_mm256_castsi128_si256(lo), hi, 1));
  out = pmul(out, c);  // a * 2^e
  return out;
}

References a, b, calibrate::c, e(), out(), padd(), pmax(), pmin(), pmul(), pnegate(), pset1< Packet4d >(), pset1< Packet4i >(), psub(), and vec4i_swizzle1.

pldexp< Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pldexp< Packet4f >	(	const Packet4f &	a,
		const Packet4f &	exponent
	)

                                                                                           {
  return pldexp_generic(a, exponent);
}

References a, and pldexp_generic().

Referenced by pldexp(), and pldexp< Packet8bf >().

pldexp< Packet8bf >()

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::pldexp< Packet8bf >	(	const Packet8bf &	a,
		const Packet8bf &	exponent
	)

                                                                                               {
  BF16_TO_F32_BINARY_OP_WRAPPER(pldexp<Packet4f>, a, exponent);
}

References a, BF16_TO_F32_BINARY_OP_WRAPPER, and pldexp< Packet4f >().

pldexp< Packet8d >()

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::pldexp< Packet8d >	(	const Packet8d &	a,
		const Packet8d &	exponent
	)

                                                                                           {
  // Clamp exponent to [-2099, 2099]
  const Packet8d max_exponent = pset1<Packet8d>(2099.0);
  const Packet8i e = _mm512_cvtpd_epi32(pmin(pmax(exponent, pnegate(max_exponent)), max_exponent));
 
  // Split 2^e into four factors and multiply.
  const Packet8i bias = pset1<Packet8i>(1023);
  Packet8i b = parithmetic_shift_right<2>(e);  // floor(e/4)
 
  // 2^b
  const Packet8i permute_idx = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7);
  Packet8i hi = _mm256_permutevar8x32_epi32(padd(b, bias), permute_idx);
  Packet8i lo = _mm256_slli_epi64(hi, 52);
  hi = _mm256_slli_epi64(_mm256_srli_epi64(hi, 32), 52);
  Packet8d c = _mm512_castsi512_pd(_mm512_inserti64x4(_mm512_castsi256_si512(lo), hi, 1));
  Packet8d out = pmul(pmul(pmul(a, c), c), c);  // a * 2^(3b)
 
  // 2^(e - 3b)
  b = psub(psub(psub(e, b), b), b);  // e - 3b
  hi = _mm256_permutevar8x32_epi32(padd(b, bias), permute_idx);
  lo = _mm256_slli_epi64(hi, 52);
  hi = _mm256_slli_epi64(_mm256_srli_epi64(hi, 32), 52);
  c = _mm512_castsi512_pd(_mm512_inserti64x4(_mm512_castsi256_si512(lo), hi, 1));
  out = pmul(out, c);  // a * 2^e
  return out;
}

References a, b, calibrate::c, e(), out(), padd(), pmax(), pmin(), pmul(), pnegate(), pset1< Packet8d >(), pset1< Packet8i >(), and psub().

pldexp< Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::pldexp< Packet8f >	(	const Packet8f &	a,
		const Packet8f &	exponent
	)

                                                                                           {
  return pldexp_generic(a, exponent);
}

References a, and pldexp_generic().

Referenced by pldexp().

pldexp< PacketXf >()

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::pldexp< PacketXf >	(	const PacketXf &	a,
		const PacketXf &	exponent
	)

                                                                                           {
  return pldexp_generic(a, exponent);
}

References a, and pldexp_generic().

pldexp_fast()

template<typename Packet >

EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Packet Eigen::internal::pldexp_fast	(	const Packet &	a,
		const Packet &	exponent
	)

                                                                                                  {
  typedef typename unpacket_traits<Packet>::integer_packet PacketI;
  typedef typename unpacket_traits<Packet>::type Scalar;
  typedef typename unpacket_traits<PacketI>::type ScalarI;
  static constexpr int TotalBits = sizeof(Scalar) * CHAR_BIT, MantissaBits = numext::numeric_limits<Scalar>::digits - 1,
                       ExponentBits = TotalBits - MantissaBits - 1;
 
  const Packet bias = pset1<Packet>(Scalar((ScalarI(1) << (ExponentBits - 1)) - ScalarI(1)));  // 127
  const Packet limit = pset1<Packet>(Scalar((ScalarI(1) << ExponentBits) - ScalarI(1)));       // 255
  // restrict biased exponent between 0 and 255 for float.
  const PacketI e = pcast<Packet, PacketI>(pmin(pmax(padd(exponent, bias), pzero(limit)), limit));  // exponent + 127
  // return a * (2^e)
  return pmul(a, preinterpret<Packet>(plogical_shift_left<MantissaBits>(e)));
}

References a, e(), padd(), pmax(), pmin(), pmul(), and pzero().

Referenced by generic_pow_impl(), Eigen::internal::scalar_logistic_op< float >::packetOp(), pexp_double(), and pexp_float().

pldexp_fast< Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pldexp_fast< Packet2d >	(	const Packet2d &	a,
		const Packet2d &	exponent
	)

                                                                                                {
  // Clamp exponent to [-1023, 1024]
  const Packet2d min_exponent = pset1<Packet2d>(-1023.0);
  const Packet2d max_exponent = pset1<Packet2d>(1024.0);
  const Packet2d e = pmin(pmax(exponent, min_exponent), max_exponent);
 
  // Convert e to integer and swizzle to low-order bits.
  const Packet4i ei = vec4i_swizzle1(_mm_cvtpd_epi32(e), 0, 3, 1, 3);
 
  // Compute 2^e multiply:
  const Packet4i bias = _mm_set_epi32(0, 1023, 0, 1023);
  const Packet2d c = _mm_castsi128_pd(_mm_slli_epi64(padd(ei, bias), 52));  // 2^e
  return pmul(a, c);
}

References a, calibrate::c, e(), padd(), pmax(), pmin(), pmul(), pset1< Packet2d >(), and vec4i_swizzle1.

pldexp_fast< Packet4d >()

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::pldexp_fast< Packet4d >	(	const Packet4d &	a,
		const Packet4d &	exponent
	)

                                                                                                {
  // Clamp exponent to [-1024, 1024]
  const Packet4d min_exponent = pset1<Packet4d>(-1023.0);
  const Packet4d max_exponent = pset1<Packet4d>(1024.0);
  const Packet4i e = _mm256_cvtpd_epi32(pmin(pmax(exponent, min_exponent), max_exponent));
  const Packet4i bias = pset1<Packet4i>(1023);
 
  // 2^e
  Packet4i hi = vec4i_swizzle1(padd(e, bias), 0, 2, 1, 3);
  const Packet4i lo = _mm_slli_epi64(hi, 52);
  hi = _mm_slli_epi64(_mm_srli_epi64(hi, 32), 52);
  const Packet4d c = _mm256_castsi256_pd(_mm256_insertf128_si256(_mm256_castsi128_si256(lo), hi, 1));
  return pmul(a, c);  // a * 2^e
}

References a, calibrate::c, e(), padd(), pmax(), pmin(), pmul(), pset1< Packet4d >(), pset1< Packet4i >(), and vec4i_swizzle1.

pldexp_generic()

template<typename Packet >

EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Packet Eigen::internal::pldexp_generic	(	const Packet &	a,
		const Packet &	exponent
	)

Default implementation of pldexp. It is expected to be called by implementers of template<> pldexp.

                                                                                                     {
  // We want to return a * 2^exponent, allowing for all possible integer
  // exponents without overflowing or underflowing in intermediate
  // computations.
  //
  // Since 'a' and the output can be denormal, the maximum range of 'exponent'
  // to consider for a float is:
  //   -255-23 -> 255+23
  // Below -278 any finite float 'a' will become zero, and above +278 any
  // finite float will become inf, including when 'a' is the smallest possible
  // denormal.
  //
  // Unfortunately, 2^(278) cannot be represented using either one or two
  // finite normal floats, so we must split the scale factor into at least
  // three parts. It turns out to be faster to split 'exponent' into four
  // factors, since [exponent>>2] is much faster to compute that [exponent/3].
  //
  // Set e = min(max(exponent, -278), 278);
  //     b = floor(e/4);
  //   out = ((((a * 2^(b)) * 2^(b)) * 2^(b)) * 2^(e-3*b))
  //
  // This will avoid any intermediate overflows and correctly handle 0, inf,
  // NaN cases.
  typedef typename unpacket_traits<Packet>::integer_packet PacketI;
  typedef typename unpacket_traits<Packet>::type Scalar;
  typedef typename unpacket_traits<PacketI>::type ScalarI;
  static constexpr int TotalBits = sizeof(Scalar) * CHAR_BIT, MantissaBits = numext::numeric_limits<Scalar>::digits - 1,
                       ExponentBits = TotalBits - MantissaBits - 1;
 
  const Packet max_exponent = pset1<Packet>(Scalar((ScalarI(1) << ExponentBits) + ScalarI(MantissaBits - 1)));  // 278
  const PacketI bias = pset1<PacketI>((ScalarI(1) << (ExponentBits - 1)) - ScalarI(1));                         // 127
  const PacketI e = pcast<Packet, PacketI>(pmin(pmax(exponent, pnegate(max_exponent)), max_exponent));
  PacketI b = parithmetic_shift_right<2>(e);                                          // floor(e/4);
  Packet c = preinterpret<Packet>(plogical_shift_left<MantissaBits>(padd(b, bias)));  // 2^b
  Packet out = pmul(pmul(pmul(a, c), c), c);                                          // a * 2^(3b)
  b = pnmadd(pset1<PacketI>(3), b, e);                                                // e - 3b
  c = preinterpret<Packet>(plogical_shift_left<MantissaBits>(padd(b, bias)));         // 2^(e-3*b)
  out = pmul(out, c);
  return out;
}

References a, b, calibrate::c, e(), out(), padd(), pmax(), pmin(), pmul(), pnegate(), and pnmadd().

Referenced by pldexp< Packet16f >(), pldexp< Packet2d >(), pldexp< Packet2f >(), pldexp< Packet4f >(), pldexp< Packet8f >(), and pldexp< PacketXf >().

plgamma()

template<typename Packet >

EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::plgamma

(

const Packet &

a

)

Returns

the ln(|gamma(a)|) (coeff-wise)

                                                                                     {
  using numext::lgamma;
  return lgamma(a);
}

References a.

Referenced by packetmath_real(), and Eigen::internal::scalar_lgamma_op< Scalar >::packetOp().

pload()

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::pload ( const typename unpacket_traits< Packet >::type *  from )

inline

Returns

a packet version of *from, from must be properly aligned

                                                                                        {
  return *from;
}

pload1()

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::pload1 ( const typename unpacket_traits< Packet >::type *  a )

inline

Returns

a packet with constant coefficients a[0], e.g.: (a[0],a[0],a[0],a[0])

                                                                                      {
  return pset1<Packet>(*a);
}

References a.

pload1< Packet16f >()

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::pload1< Packet16f >

(

const float *

from

)

                                                                   {
#if (EIGEN_COMP_GNUC != 0) || (EIGEN_COMP_CLANG != 0)
  // Inline asm here helps reduce some register spilling in TRSM kernels.
  // See note in unrolls::gemm::microKernel in TrsmKernel.h
  Packet16f ret;
  __asm__("vbroadcastss %[mem], %[dst]" : [dst] "=v"(ret) : [mem] "m"(*from));
  return ret;
#else
  return _mm512_broadcastss_ps(_mm_load_ps1(from));
#endif
}

References ret.

pload1< Packet4d >()

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::pload1< Packet4d >

(

const double *

from

)

                                                                  {
  return _mm256_broadcast_sd(from);
}

pload1< Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pload1< Packet4f >

(

const float *

from

)

                                                                 {
  EIGEN_MSA_DEBUG;
 
  float f = *from;
  Packet4f v = {f, f, f, f};
  return v;
}

References EIGEN_MSA_DEBUG, f(), and v.

Referenced by ploadquad< Packet16b >().

pload1< Packet4i >()

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pload1< Packet4i >

(

const int32_t *

from

)

                                                                   {
  EIGEN_MSA_DEBUG;
 
  return __builtin_msa_fill_w(*from);
}

References EIGEN_MSA_DEBUG.

pload1< Packet8d >()

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::pload1< Packet8d >

(

const double *

from

)

                                                                  {
#if (EIGEN_COMP_GNUC != 0) || (EIGEN_COMP_CLANG != 0)
  Packet8d ret;
  __asm__("vbroadcastsd %[mem], %[dst]" : [dst] "=v"(ret) : [mem] "m"(*from));
  return ret;
#else
  return _mm512_set1_pd(*from);
#endif
}

References ret.

Referenced by Eigen::internal::gemm_class< Scalar, is_unit_inc >::a_load().

pload1< Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::pload1< Packet8f >

(

const float *

from

)

                                                                 {
  return _mm256_broadcast_ss(from);
}

pload2()

EIGEN_STRONG_INLINE Packet2cf Eigen::internal::pload2	(	const std::complex< float > &	from0,
		const std::complex< float > &	from1
	)

                                                                                                     {
  Packet4f res0, res1;
#ifdef EIGEN_VECTORIZE_VSX
  // Load two std::complex<float> from memory and combine
  __asm__("lxsdx %x0,%y1" : "=wa"(res0) : "Z"(from0));
  __asm__("lxsdx %x0,%y1" : "=wa"(res1) : "Z"(from1));
#ifdef _BIG_ENDIAN
  __asm__("xxpermdi %x0, %x1, %x2, 0" : "=wa"(res0) : "wa"(res0), "wa"(res1));
#else
  __asm__("xxpermdi %x0, %x2, %x1, 0" : "=wa"(res0) : "wa"(res0), "wa"(res1));
#endif
#else
  *reinterpret_cast<std::complex<float>*>(&res0) = from0;
  *reinterpret_cast<std::complex<float>*>(&res1) = from1;
  res0 = vec_perm(res0, res1, p16uc_TRANSPOSE64_HI);
#endif
  return Packet2cf(res0);
}

References p16uc_TRANSPOSE64_HI.

Referenced by Eigen::internal::dhs_cpack< Scalar, DataMapper, Packet, PacketC, StorageOrder, Conjugate, PanelMode, UseLhs >::operator()().

pload< Packet16b >()

template<>

EIGEN_STRONG_INLINE Packet16b Eigen::internal::pload< Packet16b >

(

const bool *

from

)

                                                                 {
  EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_si128(reinterpret_cast<const __m128i*>(from));
}

References EIGEN_DEBUG_ALIGNED_LOAD.

pload< Packet16bf >()

template<>

EIGEN_STRONG_INLINE Packet16bf Eigen::internal::pload< Packet16bf >

(

const bfloat16 *

from

)

                                                                       {
  return _mm256_load_si256(reinterpret_cast<const __m256i*>(from));
}

pload< Packet16c >() [1/2]

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::pload< Packet16c >

(

const int8_t *

from

)

                                                                   {
  EIGEN_DEBUG_ALIGNED_LOAD return __lsx_vld(from, 0);
}

References EIGEN_DEBUG_ALIGNED_LOAD.

pload< Packet16c >() [2/2]

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::pload< Packet16c >

(

const signed char *

from

)

                                                                        {
  return pload_common<Packet16c>(from);
}

Referenced by ploaddup< Packet16c >(), and ploadquad< Packet16c >().

pload< Packet16f >()

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::pload< Packet16f >

(

const float *

from

)

                                                                  {
  EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_ps(from);
}

References EIGEN_DEBUG_ALIGNED_LOAD.

Referenced by pload< Packet8cf >().

pload< Packet16h >()

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::pload< Packet16h >

(

const Eigen::half *

from

)

                                                                      {
  return _mm256_load_si256(reinterpret_cast<const __m256i*>(from));
}

Referenced by ptranspose().

pload< Packet16i >()

template<>

EIGEN_STRONG_INLINE Packet16i Eigen::internal::pload< Packet16i >

(

const int *

from

)

                                                                {
  EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_epi64(from);
}

References EIGEN_DEBUG_ALIGNED_LOAD.

pload< Packet16uc >() [1/2]

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::pload< Packet16uc >

(

const uint8_t *

from

)

                                                                      {
  EIGEN_DEBUG_ALIGNED_LOAD return __lsx_vld(from, 0);
}

References EIGEN_DEBUG_ALIGNED_LOAD.

pload< Packet16uc >() [2/2]

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::pload< Packet16uc >

(

const unsigned char *

from

)

                                                                            {
  return pload_common<Packet16uc>(from);
}

Referenced by ploaddup< Packet16uc >(), and ploadquad< Packet16uc >().

pload< Packet1cd >()

template<>

EIGEN_STRONG_INLINE Packet1cd Eigen::internal::pload< Packet1cd >

(

const std::complex< double > *

from

)

                                                                               {
  EIGEN_DEBUG_ALIGNED_LOAD return Packet1cd(pload<Packet2d>((const double*)from));
}

References EIGEN_DEBUG_ALIGNED_LOAD, and pload< Packet2d >().

Referenced by pgather< std::complex< double >, Packet1cd >().

pload< Packet1cf >()

template<>

EIGEN_STRONG_INLINE Packet1cf Eigen::internal::pload< Packet1cf >

(

const std::complex< float > *

from

)

                                                                              {
  EIGEN_DEBUG_ALIGNED_LOAD return Packet1cf(pload<Packet2f>((const float*)from));
}

References EIGEN_DEBUG_ALIGNED_LOAD, and pload< Packet2f >().

pload< Packet2cd >()

template<>

EIGEN_STRONG_INLINE Packet2cd Eigen::internal::pload< Packet2cd >

(

const std::complex< double > *

from

)

                                                                               {
  EIGEN_DEBUG_ALIGNED_LOAD return Packet2cd(_mm256_load_pd((const double*)from));
}

References EIGEN_DEBUG_ALIGNED_LOAD.

pload< Packet2cf >()

template<>

EIGEN_STRONG_INLINE Packet2cf Eigen::internal::pload< Packet2cf >

(

const std::complex< float > *

from

)

                                                                              {
  return Packet2cf(pload<Packet4f>((const float*)from));
}

References pload< Packet4f >().

pload< Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pload< Packet2d >

(

const double *

from

)

                                                                 {
  EIGEN_DEBUG_ALIGNED_LOAD return (Packet2d)__lsx_vld(from, 0);
}

References EIGEN_DEBUG_ALIGNED_LOAD.

Referenced by pbroadcast4< Packet2d >(), pbroadcastN< Packet2d, 4, false >(), pload< Packet1cd >(), and Eigen::internal::compute_inverse_size4< Architecture::Target, double, MatrixType, ResultType >::run().

pload< Packet2f >()

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::pload< Packet2f >

(

const float *

from

)

                                                                {
  EIGEN_DEBUG_ALIGNED_LOAD return vld1_f32(from);
}

References EIGEN_DEBUG_ALIGNED_LOAD.

Referenced by pcast< Packet2l, Packet2f >(), pcast< Packet2ul, Packet2f >(), and pload< Packet1cf >().

pload< Packet2i >()

template<>

EIGEN_STRONG_INLINE Packet2i Eigen::internal::pload< Packet2i >

(

const int32_t *

from

)

                                                                  {
  EIGEN_DEBUG_ALIGNED_LOAD return vld1_s32(from);
}

References EIGEN_DEBUG_ALIGNED_LOAD.

pload< Packet2l >()

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::pload< Packet2l >

(

const int64_t *

from

)

                                                                  {
  EIGEN_DEBUG_ALIGNED_LOAD return __lsx_vld(from, 0);
}

References EIGEN_DEBUG_ALIGNED_LOAD.

pload< Packet2ui >()

template<>

EIGEN_STRONG_INLINE Packet2ui Eigen::internal::pload< Packet2ui >

(

const uint32_t *

from

)

                                                                     {
  EIGEN_DEBUG_ALIGNED_LOAD return vld1_u32(from);
}

References EIGEN_DEBUG_ALIGNED_LOAD.

pload< Packet2ul >()

template<>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::pload< Packet2ul >

(

const uint64_t *

from

)

                                                                     {
  EIGEN_DEBUG_ALIGNED_LOAD return __lsx_vld(from, 0);
}

References EIGEN_DEBUG_ALIGNED_LOAD.

pload< Packet32h >()

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::pload< Packet32h >

(

const Eigen::half *

from

)

                                                                      {
  EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_ph(from);
}

References EIGEN_DEBUG_ALIGNED_LOAD.

pload< Packet4bf >()

template<>

EIGEN_STRONG_INLINE Packet4bf Eigen::internal::pload< Packet4bf >

(

const bfloat16 *

from

)

                                                                     {
  return Packet4bf(pload<Packet4us>(reinterpret_cast<const uint16_t*>(from)));
}

References pload< Packet4us >().

pload< Packet4c >()

template<>

EIGEN_STRONG_INLINE Packet4c Eigen::internal::pload< Packet4c >

(

const int8_t *

from

)

                                                                 {
  Packet4c res;
  memcpy(&res, from, sizeof(Packet4c));
  return res;
}

References res.

Referenced by ploaddup< Packet4c >().

pload< Packet4cd >()

template<>

EIGEN_STRONG_INLINE Packet4cd Eigen::internal::pload< Packet4cd >

(

const std::complex< double > *

from

)

                                                                               {
  EIGEN_DEBUG_ALIGNED_LOAD return Packet4cd(pload<Packet8d>((const double*)from));
}

References EIGEN_DEBUG_ALIGNED_LOAD, and pload< Packet8d >().

pload< Packet4cf >()

template<>

EIGEN_STRONG_INLINE Packet4cf Eigen::internal::pload< Packet4cf >

(

const std::complex< float > *

from

)

                                                                              {
  EIGEN_DEBUG_ALIGNED_LOAD return Packet4cf(_mm256_load_ps(&numext::real_ref(*from)));
}

References EIGEN_DEBUG_ALIGNED_LOAD, and Eigen::numext::real_ref().

pload< Packet4d >()

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::pload< Packet4d >

(

const double *

from

)

                                                                 {
  EIGEN_DEBUG_ALIGNED_LOAD return _mm256_load_pd(from);
}

References EIGEN_DEBUG_ALIGNED_LOAD.

pload< Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pload< Packet4f >

(

const float *

from

)

                                                                {
  return pload_common<Packet4f>(from);
}

Referenced by pload< Packet2cf >(), pset1< Packet2cf >(), and Eigen::internal::compute_inverse_size4< Architecture::Target, float, MatrixType, ResultType >::run().

pload< Packet4i >() [1/2]

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pload< Packet4i >

(

const int *

from

)

                                                              {
  return pload_common<Packet4i>(from);
}

Referenced by ploaddup< Packet4i >().

pload< Packet4i >() [2/2]

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pload< Packet4i >

(

const int32_t *

from

)

                                                                  {
  EIGEN_DEBUG_ALIGNED_LOAD return __lsx_vld(from, 0);
}

References EIGEN_DEBUG_ALIGNED_LOAD.

pload< Packet4s >()

template<>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::pload< Packet4s >

(

const int16_t *

from

)

                                                                  {
  EIGEN_DEBUG_ALIGNED_LOAD return vld1_s16(from);
}

References EIGEN_DEBUG_ALIGNED_LOAD.

pload< Packet4uc >()

template<>

EIGEN_STRONG_INLINE Packet4uc Eigen::internal::pload< Packet4uc >

(

const uint8_t *

from

)

                                                                    {
  Packet4uc res;
  memcpy(&res, from, sizeof(Packet4uc));
  return res;
}

References res.

Referenced by ploaddup< Packet4uc >().

pload< Packet4ui >()

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::pload< Packet4ui >

(

const uint32_t *

from

)

                                                                     {
  EIGEN_DEBUG_ALIGNED_LOAD return __lsx_vld(from, 0);
}

References EIGEN_DEBUG_ALIGNED_LOAD.

Referenced by ploaddup< Packet4ui >().

pload< Packet4us >()

template<>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::pload< Packet4us >

(

const uint16_t *

from

)

                                                                     {
  EIGEN_DEBUG_ALIGNED_LOAD return vld1_u16(from);
}

References EIGEN_DEBUG_ALIGNED_LOAD.

Referenced by pload< Packet4bf >().

pload< Packet8bf >()

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::pload< Packet8bf >

(

const bfloat16 *

from

)

                                                                     {
  return pload_common<Packet8us>(reinterpret_cast<const unsigned short int*>(from));
}

Referenced by plset< Packet8bf >(), and loadColData_impl< RhsMapper, linear >::run().

pload< Packet8c >()

template<>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::pload< Packet8c >

(

const int8_t *

from

)

                                                                 {
  EIGEN_DEBUG_ALIGNED_LOAD return vld1_s8(from);
}

References EIGEN_DEBUG_ALIGNED_LOAD.

pload< Packet8cf >()

template<>

EIGEN_STRONG_INLINE Packet8cf Eigen::internal::pload< Packet8cf >

(

const std::complex< float > *

from

)

                                                                              {
  EIGEN_DEBUG_ALIGNED_LOAD return Packet8cf(pload<Packet16f>(&numext::real_ref(*from)));
}

References EIGEN_DEBUG_ALIGNED_LOAD, pload< Packet16f >(), and Eigen::numext::real_ref().

pload< Packet8d >()

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::pload< Packet8d >

(

const double *

from

)

                                                                 {
  EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_pd(from);
}

References EIGEN_DEBUG_ALIGNED_LOAD.

Referenced by pload< Packet4cd >().

pload< Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::pload< Packet8f >

(

const float *

from

)

                                                                {
  EIGEN_DEBUG_ALIGNED_LOAD return _mm256_load_ps(from);
}

References EIGEN_DEBUG_ALIGNED_LOAD.

pload< Packet8h >()

template<>

EIGEN_STRONG_INLINE Packet8h Eigen::internal::pload< Packet8h >

(

const Eigen::half *

from

)

                                                                    {
  return _mm_load_si128(reinterpret_cast<const __m128i*>(from));
}

Referenced by ptranspose().

pload< Packet8i >()

template<>

EIGEN_STRONG_INLINE Packet8i Eigen::internal::pload< Packet8i >

(

const int *

from

)

                                                              {
  EIGEN_DEBUG_ALIGNED_LOAD return _mm256_load_si256(reinterpret_cast<const __m256i*>(from));
}

References EIGEN_DEBUG_ALIGNED_LOAD.

pload< Packet8l >()

template<>

EIGEN_STRONG_INLINE Packet8l Eigen::internal::pload< Packet8l >

(

const int64_t *

from

)

                                                                  {
  EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_epi64(from);
}

References EIGEN_DEBUG_ALIGNED_LOAD.

pload< Packet8s >() [1/2]

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::pload< Packet8s >

(

const int16_t *

from

)

                                                                  {
  EIGEN_DEBUG_ALIGNED_LOAD return __lsx_vld(from, 0);
}

References EIGEN_DEBUG_ALIGNED_LOAD.

pload< Packet8s >() [2/2]

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::pload< Packet8s >

(

const short int *

from

)

                                                                    {
  return pload_common<Packet8s>(from);
}

Referenced by ploaddup< Packet8s >(), and ploadquad< Packet8s >().

pload< Packet8uc >()

template<>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::pload< Packet8uc >

(

const uint8_t *

from

)

                                                                    {
  EIGEN_DEBUG_ALIGNED_LOAD return vld1_u8(from);
}

References EIGEN_DEBUG_ALIGNED_LOAD.

pload< Packet8ui >()

template<>

EIGEN_STRONG_INLINE Packet8ui Eigen::internal::pload< Packet8ui >

(

const uint32_t *

from

)

                                                                     {
  EIGEN_DEBUG_ALIGNED_LOAD return _mm256_load_si256(reinterpret_cast<const __m256i*>(from));
}

References EIGEN_DEBUG_ALIGNED_LOAD.

pload< Packet8us >() [1/2]

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::pload< Packet8us >

(

const uint16_t *

from

)

                                                                     {
  EIGEN_DEBUG_ALIGNED_LOAD return __lsx_vld(from, 0);
}

References EIGEN_DEBUG_ALIGNED_LOAD.

pload< Packet8us >() [2/2]

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::pload< Packet8us >

(

const unsigned short int *

from

)

                                                                               {
  return pload_common<Packet8us>(from);
}

Referenced by ploaddup< Packet8us >(), and ploadquad< Packet8us >().

pload< PacketXf >()

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::pload< PacketXf >

(

const float *

from

)

                                                                {
  EIGEN_DEBUG_ALIGNED_LOAD return svld1_f32(svptrue_b32(), from);
}

References EIGEN_DEBUG_ALIGNED_LOAD.

pload< PacketXi >()

template<>

EIGEN_STRONG_INLINE PacketXi Eigen::internal::pload< PacketXi >

(

const numext::int32_t *

from

)

                                                                        {
  EIGEN_DEBUG_ALIGNED_LOAD return svld1_s32(svptrue_b32(), from);
}

References EIGEN_DEBUG_ALIGNED_LOAD.

pload_common()

template<typename Packet >

EIGEN_STRONG_INLINE Packet Eigen::internal::pload_common

(

const __UNPACK_TYPE__(Packet) *

from

)

                                                                              {
  // some versions of GCC throw "unused-but-set-parameter".
  // ignoring these warnings for now.
  EIGEN_UNUSED_VARIABLE(from);
  EIGEN_DEBUG_ALIGNED_LOAD
#ifdef EIGEN_VECTORIZE_VSX
  return vec_xl(0, const_cast<__UNPACK_TYPE__(Packet)*>(from));
#else
  return vec_ld(0, from);
#endif
}

References __UNPACK_TYPE__(), EIGEN_DEBUG_ALIGNED_LOAD, and EIGEN_UNUSED_VARIABLE.

pload_ignore()

template<typename Packet >

EIGEN_ALWAYS_INLINE Packet Eigen::internal::pload_ignore

(

const __UNPACK_TYPE__(Packet) *

from

)

                                                                              {
  // some versions of GCC throw "unused-but-set-parameter".
  // ignoring these warnings for now.
  EIGEN_UNUSED_VARIABLE(from);
  EIGEN_DEBUG_ALIGNED_LOAD
  // Ignore partial input memory initialized
#if !EIGEN_COMP_LLVM
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
#endif
#ifdef EIGEN_VECTORIZE_VSX
  return vec_xl(0, const_cast<__UNPACK_TYPE__(Packet)*>(from));
#else
  return vec_ld(0, from);
#endif
#if !EIGEN_COMP_LLVM
#pragma GCC diagnostic pop
#endif
}

References __UNPACK_TYPE__(), EIGEN_DEBUG_ALIGNED_LOAD, and EIGEN_UNUSED_VARIABLE.

pload_ignore< Packet2cf >()

template<>

EIGEN_ALWAYS_INLINE Packet2cf Eigen::internal::pload_ignore< Packet2cf >

(

const std::complex< float > *

from

)

                                                                                     {
  Packet2cf res;
  res.v = pload_ignore<Packet4f>(reinterpret_cast<const float*>(from));
  return res;
}

References res.

pload_ignore< Packet8bf >()

template<>

EIGEN_ALWAYS_INLINE Packet8bf Eigen::internal::pload_ignore< Packet8bf >

(

const bfloat16 *

from

)

                                                                            {
  return pload_ignore<Packet8us>(reinterpret_cast<const unsigned short int*>(from));
}

pload_partial()

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::pload_partial ( const typename unpacket_traits< Packet >::type *  from, const Index  n, const Index  offset = 0  )

inline

Returns

n elements of a packet version of *from, from must be properly aligned offset indicates the starting element in which to load and offset + n <= unpacket_traits::size All elements before offset and after the last element loaded will initialized with zero

                                                                      {
  const Index packet_size = unpacket_traits<Packet>::size;
  eigen_assert(n + offset <= packet_size && "number of elements plus offset will read past end of packet");
  typedef typename unpacket_traits<Packet>::type Scalar;
  EIGEN_ALIGN_MAX Scalar elements[packet_size] = {Scalar(0)};
  for (Index i = offset; i < numext::mini(n + offset, packet_size); i++) {
    elements[i] = from[i - offset];
  }
  return pload<Packet>(elements);
}

References EIGEN_ALIGN_MAX, eigen_assert, i, Eigen::numext::mini(), and n.

pload_partial< Packet16c >()

template<>

EIGEN_ALWAYS_INLINE Packet16c Eigen::internal::pload_partial< Packet16c >	(	const signed char *	from,
		const Index	n,
		const Index	offset
	)

                                                                                                                   {
  return pload_partial_common<Packet16c>(from, n, offset);
}

References n.

pload_partial< Packet16uc >()

template<>

EIGEN_ALWAYS_INLINE Packet16uc Eigen::internal::pload_partial< Packet16uc >	(	const unsigned char *	from,
		const Index	n,
		const Index	offset
	)

                                                                                                                       {
  return pload_partial_common<Packet16uc>(from, n, offset);
}

References n.

pload_partial< Packet2cf >()

template<>

EIGEN_ALWAYS_INLINE Packet2cf Eigen::internal::pload_partial< Packet2cf >	(	const std::complex< float > *	from,
		const Index	n,
		const Index	offset
	)

                                                                           {
  return Packet2cf(pload_partial<Packet4f>((const float*)from, n * 2, offset * 2));
}

References n, and pload_partial< Packet4f >().

pload_partial< Packet4f >()

template<>

EIGEN_ALWAYS_INLINE Packet4f Eigen::internal::pload_partial< Packet4f >	(	const float *	from,
		const Index	n,
		const Index	offset
	)

                                                                                                           {
  return pload_partial_common<Packet4f>(from, n, offset);
}

References n.

Referenced by pload_partial< Packet2cf >().

pload_partial< Packet4i >()

template<>

EIGEN_ALWAYS_INLINE Packet4i Eigen::internal::pload_partial< Packet4i >	(	const int *	from,
		const Index	n,
		const Index	offset
	)

                                                                                                         {
  return pload_partial_common<Packet4i>(from, n, offset);
}

References n.

pload_partial< Packet8bf >()

template<>

EIGEN_ALWAYS_INLINE Packet8bf Eigen::internal::pload_partial< Packet8bf >	(	const bfloat16 *	from,
		const Index	n,
		const Index	offset
	)

                                                                                                                {
  return pload_partial_common<Packet8us>(reinterpret_cast<const unsigned short int*>(from), n, offset);
}

References n.

pload_partial< Packet8s >()

template<>

EIGEN_ALWAYS_INLINE Packet8s Eigen::internal::pload_partial< Packet8s >	(	const short int *	from,
		const Index	n,
		const Index	offset
	)

                                                                                                               {
  return pload_partial_common<Packet8s>(from, n, offset);
}

References n.

pload_partial< Packet8us >()

template<>

EIGEN_ALWAYS_INLINE Packet8us Eigen::internal::pload_partial< Packet8us >	(	const unsigned short int *	from,
		const Index	n,
		const Index	offset
	)

                                                                           {
  return pload_partial_common<Packet8us>(from, n, offset);
}

References n.

pload_partial_common()

template<typename Packet >

EIGEN_ALWAYS_INLINE Packet Eigen::internal::pload_partial_common	(	const __UNPACK_TYPE__(Packet) *	from,
		const Index	n,
		const Index	offset
	)

                                                                    {
  // some versions of GCC throw "unused-but-set-parameter".
  // ignoring these warnings for now.
  const Index packet_size = unpacket_traits<Packet>::size;
  eigen_internal_assert(n + offset <= packet_size && "number of elements plus offset will read past end of packet");
  const Index size = sizeof(__UNPACK_TYPE__(Packet));
#ifdef _ARCH_PWR9
  EIGEN_UNUSED_VARIABLE(packet_size);
  EIGEN_DEBUG_ALIGNED_LOAD
  EIGEN_UNUSED_VARIABLE(from);
  Packet load = vec_xl_len(const_cast<__UNPACK_TYPE__(Packet)*>(from), n * size);
  if (offset) {
    Packet16uc shift = pset1<Packet16uc>(offset * 8 * size);
#ifdef _BIG_ENDIAN
    load = Packet(vec_sro(Packet16uc(load), shift));
#else
    load = Packet(vec_slo(Packet16uc(load), shift));
#endif
  }
  return load;
#else
  if (n) {
    EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) load[packet_size];
    unsigned char* load2 = reinterpret_cast<unsigned char*>(load + offset);
    unsigned char* from2 = reinterpret_cast<unsigned char*>(const_cast<__UNPACK_TYPE__(Packet)*>(from));
    Index n2 = n * size;
    if (16 <= n2) {
      pstoreu(load2, ploadu<Packet16uc>(from2));
    } else {
      memcpy((void*)load2, (void*)from2, n2);
    }
    return pload_ignore<Packet>(load);
  } else {
    return Packet(pset1<Packet16uc>(0));
  }
#endif
}

References __UNPACK_TYPE__(), EIGEN_ALIGN16, EIGEN_DEBUG_ALIGNED_LOAD, eigen_internal_assert, EIGEN_UNUSED_VARIABLE, load(), n, ploadu< Packet16uc >(), pset1< Packet16uc >(), pstoreu(), and size.

ploaddup()

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet Eigen::internal::ploaddup

(

const typename unpacket_traits< Packet >::type *

from

)

Returns

a packet with elements of *from duplicated. For instance, for a packet of 8 elements, 4 scalars will be read from *from and duplicated to form: {from[0],from[0],from[1],from[1],from[2],from[2],from[3],from[3]} Currently, this function is only used for scalar * complex products.

                                                                                                        {
  return *from;
}

ploaddup< Packet16b >()

template<>

EIGEN_STRONG_INLINE Packet16b Eigen::internal::ploaddup< Packet16b >

(

const bool *

from

)

                                                                    {
  __m128i tmp = _mm_castpd_si128(pload1<Packet2d>(reinterpret_cast<const double*>(from)));
  return _mm_unpacklo_epi8(tmp, tmp);
}

References tmp.

ploaddup< Packet16bf >()

template<>

EIGEN_STRONG_INLINE Packet16bf Eigen::internal::ploaddup< Packet16bf >

(

const bfloat16 *

from

)

                                                                          {
  unsigned short a = from[0].value;
  unsigned short b = from[1].value;
  unsigned short c = from[2].value;
  unsigned short d = from[3].value;
  unsigned short e = from[4].value;
  unsigned short f = from[5].value;
  unsigned short g = from[6].value;
  unsigned short h = from[7].value;
  return _mm256_set_epi16(h, h, g, g, f, f, e, e, d, d, c, c, b, b, a, a);
}

References a, b, calibrate::c, e(), f(), and Eigen::bfloat16_impl::__bfloat16_raw::value.

ploaddup< Packet16c >() [1/2]

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::ploaddup< Packet16c >

(

const int8_t *

from

)

                                                                      {
  Packet16c tmp = pload<Packet16c>(from);
  return __lsx_vilvl_b(tmp, tmp);
}

References pload< Packet16c >(), and tmp.

ploaddup< Packet16c >() [2/2]

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::ploaddup< Packet16c >

(

const signed char *

from

)

                                                                           {
  Packet16c p;
  if ((std::ptrdiff_t(from) % 16) == 0)
    p = pload<Packet16c>(from);
  else
    p = ploadu<Packet16c>(from);
  return vec_mergeh(p, p);
}

References p, pload< Packet16c >(), and ploadu< Packet16c >().

ploaddup< Packet16f >()

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::ploaddup< Packet16f >

(

const float *

from

)

                                                                     {
  // an unaligned load is required here as there is no requirement
  // on the alignment of input pointer 'from'
  __m256i low_half = _mm256_castps_si256(_mm256_loadu_ps(from));
  __m512 even_elements = _mm512_castsi512_ps(_mm512_cvtepu32_epi64(low_half));
  __m512 pairs = _mm512_permute_ps(even_elements, _MM_SHUFFLE(2, 2, 0, 0));
  return pairs;
}

ploaddup< Packet16h >()

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::ploaddup< Packet16h >

(

const Eigen::half *

from

)

                                                                         {
  unsigned short a = from[0].x;
  unsigned short b = from[1].x;
  unsigned short c = from[2].x;
  unsigned short d = from[3].x;
  unsigned short e = from[4].x;
  unsigned short f = from[5].x;
  unsigned short g = from[6].x;
  unsigned short h = from[7].x;
  return _mm256_set_epi16(h, h, g, g, f, f, e, e, d, d, c, c, b, b, a, a);
}

References a, b, calibrate::c, e(), f(), and Eigen::half_impl::__half_raw::x.

ploaddup< Packet16i >()

template<>

EIGEN_STRONG_INLINE Packet16i Eigen::internal::ploaddup< Packet16i >

(

const int *

from

)

                                                                   {
  __m256i low_half = _mm256_load_si256(reinterpret_cast<const __m256i*>(from));
  __m512 even_elements = _mm512_castsi512_ps(_mm512_cvtepu32_epi64(low_half));
  __m512 pairs = _mm512_permute_ps(even_elements, _MM_SHUFFLE(2, 2, 0, 0));
  return _mm512_castps_si512(pairs);
}

ploaddup< Packet16uc >() [1/2]

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::ploaddup< Packet16uc >

(

const uint8_t *

from

)

                                                                         {
  Packet16uc tmp = pload<Packet16uc>(from);
  return __lsx_vilvl_b(tmp, tmp);
}

References pload< Packet16uc >(), and tmp.

ploaddup< Packet16uc >() [2/2]

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::ploaddup< Packet16uc >

(

const unsigned char *

from

)

                                                                               {
  Packet16uc p;
  if ((std::ptrdiff_t(from) % 16) == 0)
    p = pload<Packet16uc>(from);
  else
    p = ploadu<Packet16uc>(from);
  return vec_mergeh(p, p);
}

References p, pload< Packet16uc >(), and ploadu< Packet16uc >().

ploaddup< Packet1cd >()

template<>

EIGEN_STRONG_INLINE Packet1cd Eigen::internal::ploaddup< Packet1cd >

(

const std::complex< double > *

from

)

                                                                                  {
  return pset1<Packet1cd>(*from);
}

References pset1< Packet1cd >().

ploaddup< Packet1cf >()

template<>

EIGEN_STRONG_INLINE Packet1cf Eigen::internal::ploaddup< Packet1cf >

(

const std::complex< float > *

from

)

                                                                                 {
  return pset1<Packet1cf>(*from);
}

References pset1< Packet1cf >().

ploaddup< Packet2cd >()

template<>

EIGEN_STRONG_INLINE Packet2cd Eigen::internal::ploaddup< Packet2cd >

(

const std::complex< double > *

from

)

                                                                                  {
  return pset1<Packet2cd>(*from);
}

References pset1< Packet2cd >().

Referenced by ploaddup< Packet4cd >().

ploaddup< Packet2cf >()

template<>

EIGEN_STRONG_INLINE Packet2cf Eigen::internal::ploaddup< Packet2cf >

(

const std::complex< float > *

from

)

                                                                                 {
  return pset1<Packet2cf>(*from);
}

References pset1< Packet2cf >().

Referenced by ploaddup< Packet4cf >().

ploaddup< Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::ploaddup< Packet2d >

(

const double *

from

)

                                                                    {
  return pset1<Packet2d>(from[0]);
}

References pset1< Packet2d >().

Referenced by pload_complex_full(), and pload_realimag_combine().

ploaddup< Packet2f >()

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::ploaddup< Packet2f >

(

const float *

from

)

                                                                   {
  return vld1_dup_f32(from);
}

ploaddup< Packet2i >()

template<>

EIGEN_STRONG_INLINE Packet2i Eigen::internal::ploaddup< Packet2i >

(

const int32_t *

from

)

                                                                     {
  return vld1_dup_s32(from);
}

ploaddup< Packet2l >()

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::ploaddup< Packet2l >

(

const int64_t *

from

)

                                                                     {
  return pset1<Packet2l>(from[0]);
}

References pset1< Packet2l >().

ploaddup< Packet2ui >()

template<>

EIGEN_STRONG_INLINE Packet2ui Eigen::internal::ploaddup< Packet2ui >

(

const uint32_t *

from

)

                                                                        {
  return vld1_dup_u32(from);
}

ploaddup< Packet2ul >()

template<>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::ploaddup< Packet2ul >

(

const uint64_t *

from

)

                                                                        {
  return pset1<Packet2ul>(from[0]);
}

References pset1< Packet2ul >().

ploaddup< Packet32h >()

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::ploaddup< Packet32h >

(

const Eigen::half *

from

)

                                                                         {
  __m512h a = _mm512_castph256_ph512(_mm256_loadu_ph(from));
  return _mm512_permutexvar_ph(_mm512_set_epi16(15, 15, 14, 14, 13, 13, 12, 12, 11, 11, 10, 10, 9, 9, 8, 8, 7, 7, 6, 6,
                                                5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0, 0),
                               a);
}

References a.

ploaddup< Packet4bf >()

template<>

EIGEN_STRONG_INLINE Packet4bf Eigen::internal::ploaddup< Packet4bf >

(

const bfloat16 *

from

)

                                                                        {
  return Packet4bf(ploaddup<Packet4us>(reinterpret_cast<const uint16_t*>(from)));
}

References ploaddup< Packet4us >().

ploaddup< Packet4c >()

template<>

EIGEN_STRONG_INLINE Packet4c Eigen::internal::ploaddup< Packet4c >

(

const int8_t *

from

)

                                                                    {
  const int8x8_t a = vreinterpret_s8_s32(vdup_n_s32(pload<Packet4c>(from)));
  return vget_lane_s32(vreinterpret_s32_s8(vzip_s8(a, a).val[0]), 0);
}

References a, pload< Packet4c >(), and calibrate::val.

ploaddup< Packet4cd >()

template<>

EIGEN_STRONG_INLINE Packet4cd Eigen::internal::ploaddup< Packet4cd >

(

const std::complex< double > *

from

)

                                                                                  {
  return Packet4cd(
      _mm512_insertf64x4(_mm512_castpd256_pd512(ploaddup<Packet2cd>(from).v), ploaddup<Packet2cd>(from + 1).v, 1));
}

References ploaddup< Packet2cd >(), and v.

ploaddup< Packet4cf >()

template<>

EIGEN_STRONG_INLINE Packet4cf Eigen::internal::ploaddup< Packet4cf >

(

const std::complex< float > *

from

)

                                                                                 {
  // FIXME The following might be optimized using _mm256_movedup_pd
  Packet2cf a = ploaddup<Packet2cf>(from);
  Packet2cf b = ploaddup<Packet2cf>(from + 1);
  return Packet4cf(_mm256_insertf128_ps(_mm256_castps128_ps256(a.v), b.v, 1));
}

References a, b, and ploaddup< Packet2cf >().

ploaddup< Packet4d >()

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::ploaddup< Packet4d >

(

const double *

from

)

                                                                    {
  Packet4d tmp = _mm256_broadcast_pd((const __m128d*)(const void*)from);
  return _mm256_permute_pd(tmp, 3 << 2);
}

References tmp.

ploaddup< Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::ploaddup< Packet4f >

(

const float *

from

)

                                                                   {
  return ploaddup_common<Packet4f>(from);
}

ploaddup< Packet4i >() [1/2]

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::ploaddup< Packet4i >

(

const int *

from

)

                                                                 {
  return ploaddup_common<Packet4i>(from);
}

ploaddup< Packet4i >() [2/2]

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::ploaddup< Packet4i >

(

const int32_t *

from

)

                                                                     {
  Packet4i tmp = pload<Packet4i>(from);
  return __lsx_vilvl_w(tmp, tmp);
}

References pload< Packet4i >(), and tmp.

ploaddup< Packet4s >()

template<>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::ploaddup< Packet4s >

(

const int16_t *

from

)

                                                                     {
  return vreinterpret_s16_u32(
      vzip_u32(vreinterpret_u32_s16(vld1_dup_s16(from)), vreinterpret_u32_s16(vld1_dup_s16(from + 1))).val[0]);
}

References calibrate::val.

ploaddup< Packet4uc >()

template<>

EIGEN_STRONG_INLINE Packet4uc Eigen::internal::ploaddup< Packet4uc >

(

const uint8_t *

from

)

                                                                       {
  const uint8x8_t a = vreinterpret_u8_u32(vdup_n_u32(pload<Packet4uc>(from)));
  return vget_lane_u32(vreinterpret_u32_u8(vzip_u8(a, a).val[0]), 0);
}

References a, pload< Packet4uc >(), and calibrate::val.

ploaddup< Packet4ui >()

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::ploaddup< Packet4ui >

(

const uint32_t *

from

)

                                                                        {
  Packet4ui tmp = pload<Packet4ui>(from);
  return __lsx_vilvl_w(tmp, tmp);
}

References pload< Packet4ui >(), and tmp.

ploaddup< Packet4us >()

template<>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::ploaddup< Packet4us >

(

const uint16_t *

from

)

                                                                        {
  return vreinterpret_u16_u32(
      vzip_u32(vreinterpret_u32_u16(vld1_dup_u16(from)), vreinterpret_u32_u16(vld1_dup_u16(from + 1))).val[0]);
}

References calibrate::val.

Referenced by ploaddup< Packet4bf >().

ploaddup< Packet8bf >()

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::ploaddup< Packet8bf >

(

const bfloat16 *

from

)

                                                                        {
  return ploaddup<Packet8us>(reinterpret_cast<const unsigned short int*>(from));
}

References ploaddup< Packet8us >().

ploaddup< Packet8c >()

template<>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::ploaddup< Packet8c >

(

const int8_t *

from

)

                                                                    {
  const int8x8_t a = vld1_s8(from);
  return vzip_s8(a, a).val[0];
}

References a.

ploaddup< Packet8cf >()

template<>

EIGEN_STRONG_INLINE Packet8cf Eigen::internal::ploaddup< Packet8cf >

(

const std::complex< float > *

from

)

                                                                                 {
  return Packet8cf(_mm512_castpd_ps(ploaddup<Packet8d>((const double*)(const void*)from)));
}

References ploaddup< Packet8d >().

ploaddup< Packet8d >()

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::ploaddup< Packet8d >

(

const double *

from

)

                                                                    {
  Packet8d tmp = _mm512_castpd256_pd512(ploadu<Packet4d>(from));
  const Packet8l scatter_mask = _mm512_set_epi64(3, 3, 2, 2, 1, 1, 0, 0);
  return _mm512_permutexvar_pd(scatter_mask, tmp);
}

References ploadu< Packet4d >(), and tmp.

Referenced by ploaddup< Packet8cf >().

ploaddup< Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::ploaddup< Packet8f >

(

const float *

from

)

                                                                   {
  // TODO try to find a way to avoid the need of a temporary register
  //   Packet8f tmp  = _mm256_castps128_ps256(_mm_loadu_ps(from));
  //   tmp = _mm256_insertf128_ps(tmp, _mm_movehl_ps(_mm256_castps256_ps128(tmp),_mm256_castps256_ps128(tmp)), 1);
  //   return _mm256_unpacklo_ps(tmp,tmp);
 
  // _mm256_insertf128_ps is very slow on Haswell, thus:
  Packet8f tmp = _mm256_broadcast_ps((const __m128*)(const void*)from);
  // mimic an "inplace" permutation of the lower 128bits using a blend
  tmp = _mm256_blend_ps(
      tmp, _mm256_castps128_ps256(_mm_permute_ps(_mm256_castps256_ps128(tmp), _MM_SHUFFLE(1, 0, 1, 0))), 15);
  // then we can perform a consistent permutation on the global register to get everything in shape:
  return _mm256_permute_ps(tmp, _MM_SHUFFLE(3, 3, 2, 2));
}

References tmp.

ploaddup< Packet8h >()

template<>

EIGEN_STRONG_INLINE Packet8h Eigen::internal::ploaddup< Packet8h >

(

const Eigen::half *

from

)

                                                                       {
  const numext::uint16_t a = numext::bit_cast<numext::uint16_t>(from[0]);
  const numext::uint16_t b = numext::bit_cast<numext::uint16_t>(from[1]);
  const numext::uint16_t c = numext::bit_cast<numext::uint16_t>(from[2]);
  const numext::uint16_t d = numext::bit_cast<numext::uint16_t>(from[3]);
  return _mm_set_epi16(d, d, c, c, b, b, a, a);
}

References a, b, and calibrate::c.

ploaddup< Packet8i >()

template<>

EIGEN_STRONG_INLINE Packet8i Eigen::internal::ploaddup< Packet8i >

(

const int *

from

)

                                                                 {
#ifdef EIGEN_VECTORIZE_AVX2
  const Packet8i a = _mm256_castsi128_si256(ploadu<Packet4i>(from));
  return _mm256_permutevar8x32_epi32(a, _mm256_setr_epi32(0, 0, 1, 1, 2, 2, 3, 3));
#else
  __m256 tmp = _mm256_broadcast_ps((const __m128*)(const void*)from);
  // mimic an "inplace" permutation of the lower 128bits using a blend
  tmp = _mm256_blend_ps(
      tmp, _mm256_castps128_ps256(_mm_permute_ps(_mm256_castps256_ps128(tmp), _MM_SHUFFLE(1, 0, 1, 0))), 15);
  // then we can perform a consistent permutation on the global register to get everything in shape:
  return _mm256_castps_si256(_mm256_permute_ps(tmp, _MM_SHUFFLE(3, 3, 2, 2)));
#endif
}

References a, ploadu< Packet4i >(), and tmp.

ploaddup< Packet8l >()

template<>

EIGEN_STRONG_INLINE Packet8l Eigen::internal::ploaddup< Packet8l >

(

const int64_t *

from

)

                                                                     {
  Packet8l tmp = _mm512_castsi256_si512(ploadu<Packet4l>(from));
  const Packet8l scatter_mask = _mm512_set_epi64(3, 3, 2, 2, 1, 1, 0, 0);
  return _mm512_permutexvar_epi64(scatter_mask, tmp);
}

References tmp.

ploaddup< Packet8s >() [1/2]

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::ploaddup< Packet8s >

(

const int16_t *

from

)

                                                                     {
  Packet8s tmp = pload<Packet8s>(from);
  return __lsx_vilvl_h(tmp, tmp);
}

References pload< Packet8s >(), and tmp.

ploaddup< Packet8s >() [2/2]

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::ploaddup< Packet8s >

(

const short int *

from

)

                                                                       {
  Packet8s p;
  if ((std::ptrdiff_t(from) % 16) == 0)
    p = pload<Packet8s>(from);
  else
    p = ploadu<Packet8s>(from);
  return vec_mergeh(p, p);
}

References p, pload< Packet8s >(), and ploadu< Packet8s >().

ploaddup< Packet8uc >()

template<>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::ploaddup< Packet8uc >

(

const uint8_t *

from

)

                                                                       {
  const uint8x8_t a = vld1_u8(from);
  return vzip_u8(a, a).val[0];
}

References a.

ploaddup< Packet8ui >()

template<>

EIGEN_STRONG_INLINE Packet8ui Eigen::internal::ploaddup< Packet8ui >

(

const uint32_t *

from

)

                                                                        {
#ifdef EIGEN_VECTORIZE_AVX2
  const Packet8ui a = _mm256_castsi128_si256(ploadu<Packet4ui>(from));
  return _mm256_permutevar8x32_epi32(a, _mm256_setr_epi32(0, 0, 1, 1, 2, 2, 3, 3));
#else
  __m256 tmp = _mm256_broadcast_ps((const __m128*)(const void*)from);
  // mimic an "inplace" permutation of the lower 128bits using a blend
  tmp = _mm256_blend_ps(
      tmp, _mm256_castps128_ps256(_mm_permute_ps(_mm256_castps256_ps128(tmp), _MM_SHUFFLE(1, 0, 1, 0))), 15);
  // then we can perform a consistent permutation on the global register to get
  // everything in shape:
  return _mm256_castps_si256(_mm256_permute_ps(tmp, _MM_SHUFFLE(3, 3, 2, 2)));
#endif
}

References a, ploadu< Packet4ui >(), and tmp.

ploaddup< Packet8us >() [1/2]

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::ploaddup< Packet8us >

(

const uint16_t *

from

)

                                                                        {
  Packet8us tmp = pload<Packet8us>(from);
  return __lsx_vilvl_h(tmp, tmp);
}

References pload< Packet8us >(), and tmp.

ploaddup< Packet8us >() [2/2]

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::ploaddup< Packet8us >

(

const unsigned short int *

from

)

                                                                                  {
  Packet8us p;
  if ((std::ptrdiff_t(from) % 16) == 0)
    p = pload<Packet8us>(from);
  else
    p = ploadu<Packet8us>(from);
  return vec_mergeh(p, p);
}

References p, pload< Packet8us >(), and ploadu< Packet8us >().

Referenced by ploaddup< Packet8bf >().

ploaddup< PacketXf >()

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::ploaddup< PacketXf >

(

const float *

from

)

                                                                   {
  svuint32_t indices = svindex_u32(0, 1);  // index {base=0, base+step=1, base+step*2, ...}
  indices = svzip1_u32(indices, indices);  // index in the format {a0, a0, a1, a1, a2, a2, ...}
  return svld1_gather_u32index_f32(svptrue_b32(), from, indices);
}

ploaddup< PacketXi >()

template<>

EIGEN_STRONG_INLINE PacketXi Eigen::internal::ploaddup< PacketXi >

(

const numext::int32_t *

from

)

                                                                           {
  svuint32_t indices = svindex_u32(0, 1);  // index {base=0, base+step=1, base+step*2, ...}
  indices = svzip1_u32(indices, indices);  // index in the format {a0, a0, a1, a1, a2, a2, ...}
  return svld1_gather_u32index_s32(svptrue_b32(), from, indices);
}

ploaddup_common()

template<typename Packet >

EIGEN_STRONG_INLINE Packet Eigen::internal::ploaddup_common

(

const __UNPACK_TYPE__(Packet) *

from

)

                                                                                 {
  Packet p;
  if ((std::ptrdiff_t(from) % 16) == 0)
    p = pload<Packet>(from);
  else
    p = ploadu<Packet>(from);
  return vec_mergeh(p, p);
}

References p.

ploadl()

template<typename Packet >

EIGEN_STRONG_INLINE Packet Eigen::internal::ploadl

(

const typename unpacket_traits< Packet >::type *

from

)

ploadl< Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::ploadl< Packet2d >

(

const double *

from

)

                                                                  {
  EIGEN_DEBUG_UNALIGNED_LOAD return _mm_load_sd(from);
}

References EIGEN_DEBUG_UNALIGNED_LOAD.

ploadl< Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::ploadl< Packet4f >

(

const float *

from

)

                                                                 {
  EIGEN_DEBUG_UNALIGNED_LOAD return _mm_castpd_ps(_mm_load_sd(reinterpret_cast<const double*>(from)));
}

References EIGEN_DEBUG_UNALIGNED_LOAD.

ploadLhs()

template<typename Packet >

EIGEN_ALWAYS_INLINE Packet Eigen::internal::ploadLhs

(

const __UNPACK_TYPE__(Packet) *

lhs

)

                                                                         {
  return ploadu<Packet>(lhs);
}

ploadLhsMMA()

EIGEN_ALWAYS_INLINE void Eigen::internal::ploadLhsMMA	(	const double *	lhs,
		__vector_pair &	lhsV
	)

{ ploadRhsMMA(lhs, lhsV); }

References ploadRhsMMA().

ploadquad() [1/3]

template<>

EIGEN_STRONG_INLINE Packet16bf Eigen::internal::ploadquad

(

const bfloat16 *

from

)

                                                               {
  unsigned short a = from[0].value;
  unsigned short b = from[1].value;
  unsigned short c = from[2].value;
  unsigned short d = from[3].value;
  return _mm256_set_epi16(d, d, d, d, c, c, c, c, b, b, b, b, a, a, a, a);
}

References a, b, calibrate::c, and Eigen::bfloat16_impl::__bfloat16_raw::value.

ploadquad() [2/3]

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::ploadquad

(

const Eigen::half *

from

)

                                                               {
  unsigned short a = from[0].x;
  unsigned short b = from[1].x;
  unsigned short c = from[2].x;
  unsigned short d = from[3].x;
  return _mm256_set_epi16(d, d, d, d, c, c, c, c, b, b, b, b, a, a, a, a);
}

References a, b, calibrate::c, and Eigen::half_impl::__half_raw::x.

ploadquad() [3/3]

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::ploadquad ( const typename unpacket_traits< Packet >::type *  from )

inline

Returns

a packet with elements of *from quadrupled. For instance, for a packet of 8 elements, 2 scalars will be read from *from and replicated to form: {from[0],from[0],from[0],from[0],from[1],from[1],from[1],from[1]} Currently, this function is only used in matrix products. For packet-size smaller or equal to 4, this function is equivalent to pload1

                                                                                            {
  return pload1<Packet>(from);
}

ploadquad< Packet16b >()

template<>

EIGEN_STRONG_INLINE Packet16b Eigen::internal::ploadquad< Packet16b >

(

const bool *

from

)

                                                                     {
  __m128i tmp = _mm_castps_si128(pload1<Packet4f>(reinterpret_cast<const float*>(from)));
  tmp = _mm_unpacklo_epi8(tmp, tmp);
  return _mm_unpacklo_epi16(tmp, tmp);
}

References pload1< Packet4f >(), and tmp.

ploadquad< Packet16c >() [1/2]

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::ploadquad< Packet16c >

(

const int8_t *

from

)

                                                                       {
  int8_t tmp[16] = {*from,       *from,       *from,       *from,       *(from + 1), *(from + 1),
                    *(from + 1), *(from + 1), *(from + 2), *(from + 2), *(from + 2), *(from + 2),
                    *(from + 3), *(from + 3), *(from + 3), *(from + 3)};
  return __lsx_vld(tmp, 0);
}

References tmp.

ploadquad< Packet16c >() [2/2]

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::ploadquad< Packet16c >

(

const signed char *

from

)

                                                                            {
  Packet16c p;
  if ((std::ptrdiff_t(from) % 16) == 0)
    p = pload<Packet16c>(from);
  else
    p = ploadu<Packet16c>(from);
  return vec_perm(p, p, p16uc_QUADRUPLICATE16);
}

References p, p16uc_QUADRUPLICATE16, pload< Packet16c >(), and ploadu< Packet16c >().

ploadquad< Packet16f >()

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::ploadquad< Packet16f >

(

const float *

from

)

                                                                      {
  Packet16f tmp = _mm512_castps128_ps512(ploadu<Packet4f>(from));
  const Packet16i scatter_mask = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);
  return _mm512_permutexvar_ps(scatter_mask, tmp);
}

References ploadu< Packet4f >(), and tmp.

ploadquad< Packet16i >()

template<>

EIGEN_STRONG_INLINE Packet16i Eigen::internal::ploadquad< Packet16i >

(

const int *

from

)

                                                                    {
  Packet16i tmp = _mm512_castsi128_si512(ploadu<Packet4i>(from));
  const Packet16i scatter_mask = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);
  return _mm512_permutexvar_epi32(scatter_mask, tmp);
}

References ploadu< Packet4i >(), and tmp.

ploadquad< Packet16uc >() [1/2]

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::ploadquad< Packet16uc >

(

const uint8_t *

from

)

                                                                          {
  uint8_t tmp[16] = {*from,       *from,       *from,       *from,       *(from + 1), *(from + 1),
                     *(from + 1), *(from + 1), *(from + 2), *(from + 2), *(from + 2), *(from + 2),
                     *(from + 3), *(from + 3), *(from + 3), *(from + 3)};
  return __lsx_vld(tmp, 0);
}

References tmp.

ploadquad< Packet16uc >() [2/2]

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::ploadquad< Packet16uc >

(

const unsigned char *

from

)

                                                                                {
  Packet16uc p;
  if ((std::ptrdiff_t(from) % 16) == 0)
    p = pload<Packet16uc>(from);
  else
    p = ploadu<Packet16uc>(from);
  return vec_perm(p, p, p16uc_QUADRUPLICATE16);
}

References p, p16uc_QUADRUPLICATE16, pload< Packet16uc >(), and ploadu< Packet16uc >().

ploadquad< Packet32h >()

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::ploadquad< Packet32h >

(

const Eigen::half *

from

)

                                                                          {
  __m512h a = _mm512_castph128_ph512(_mm_loadu_ph(from));
  return _mm512_permutexvar_ph(
      _mm512_set_epi16(7, 7, 7, 7, 6, 6, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0),
      a);
}

References a.

ploadquad< Packet4c >()

template<>

EIGEN_STRONG_INLINE Packet4c Eigen::internal::ploadquad< Packet4c >

(

const int8_t *

from

)

                                                                     {
  return vget_lane_s32(vreinterpret_s32_s8(vld1_dup_s8(from)), 0);
}

ploadquad< Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::ploadquad< Packet4f >

(

const float *

from

)

                                                                    {
  return (__m128)__lsx_vldrepl_w(from, 0);
}

ploadquad< Packet4i >()

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::ploadquad< Packet4i >

(

const int32_t *

from

)

                                                                      {
  int32_t tmp[4] = {*from, *from, *from, *from};
  return __lsx_vld(tmp, 0);
}

References tmp.

ploadquad< Packet4uc >()

template<>

EIGEN_STRONG_INLINE Packet4uc Eigen::internal::ploadquad< Packet4uc >

(

const uint8_t *

from

)

                                                                        {
  return vget_lane_u32(vreinterpret_u32_u8(vld1_dup_u8(from)), 0);
}

ploadquad< Packet4ui >()

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::ploadquad< Packet4ui >

(

const uint32_t *

from

)

                                                                         {
  uint32_t tmp[4] = {*from, *from, *from, *from};
  return __lsx_vld(tmp, 0);
}

References tmp.

ploadquad< Packet8bf >()

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::ploadquad< Packet8bf >

(

const bfloat16 *

from

)

                                                                         {
  return ploadquad<Packet8us>(reinterpret_cast<const unsigned short int*>(from));
}

References ploadquad< Packet8us >().

ploadquad< Packet8c >()

template<>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::ploadquad< Packet8c >

(

const int8_t *

from

)

                                                                     {
  return vreinterpret_s8_u32(
      vzip_u32(vreinterpret_u32_s8(vld1_dup_s8(from)), vreinterpret_u32_s8(vld1_dup_s8(from + 1))).val[0]);
}

References calibrate::val.

ploadquad< Packet8cf >()

template<>

EIGEN_STRONG_INLINE Packet8cf Eigen::internal::ploadquad< Packet8cf >

(

const std::complex< float > *

from

)

                                                                                  {
  return Packet8cf(_mm512_castpd_ps(ploadquad<Packet8d>((const double*)(const void*)from)));
}

References ploadquad< Packet8d >().

ploadquad< Packet8d >()

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::ploadquad< Packet8d >

(

const double *

from

)

                                                                     {
  __m256d lane0 = _mm256_set1_pd(*from);
  __m256d lane1 = _mm256_set1_pd(*(from + 1));
  __m512d tmp = _mm512_undefined_pd();
  tmp = _mm512_insertf64x4(tmp, lane0, 0);
  return _mm512_insertf64x4(tmp, lane1, 1);
}

References tmp.

Referenced by ploadquad< Packet8cf >().

ploadquad< Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::ploadquad< Packet8f >

(

const float *

from

)

                                                                    {
  Packet8f tmp = _mm256_castps128_ps256(_mm_broadcast_ss(from));
  return _mm256_insertf128_ps(tmp, _mm_broadcast_ss(from + 1), 1);
}

References tmp.

ploadquad< Packet8h >()

template<>

EIGEN_STRONG_INLINE Packet8h Eigen::internal::ploadquad< Packet8h >

(

const Eigen::half *

from

)

                                                                        {
  const numext::uint16_t a = numext::bit_cast<numext::uint16_t>(from[0]);
  const numext::uint16_t b = numext::bit_cast<numext::uint16_t>(from[1]);
  return _mm_set_epi16(b, b, b, b, a, a, a, a);
}

References a, and b.

ploadquad< Packet8i >()

template<>

EIGEN_STRONG_INLINE Packet8i Eigen::internal::ploadquad< Packet8i >

(

const int *

from

)

                                                                  {
  return _mm256_insertf128_si256(_mm256_set1_epi32(*from), _mm_set1_epi32(*(from + 1)), 1);
}

ploadquad< Packet8l >()

template<>

EIGEN_STRONG_INLINE Packet8l Eigen::internal::ploadquad< Packet8l >

(

const int64_t *

from

)

                                                                      {
  __m256i lane0 = _mm256_set1_epi64x(*from);
  __m256i lane1 = _mm256_set1_epi64x(*(from + 1));
  __m512i tmp = _mm512_undefined_epi32();
  tmp = _mm512_inserti64x4(tmp, lane0, 0);
  return _mm512_inserti64x4(tmp, lane1, 1);
}

References tmp.

ploadquad< Packet8s >() [1/2]

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::ploadquad< Packet8s >

(

const int16_t *

from

)

                                                                      {
  int16_t tmp[8] = {*from, *from, *from, *from, *(from + 1), *(from + 1), *(from + 1), *(from + 1)};
  return __lsx_vld(tmp, 0);
}

References tmp.

ploadquad< Packet8s >() [2/2]

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::ploadquad< Packet8s >

(

const short int *

from

)

                                                                        {
  Packet8s p;
  if ((std::ptrdiff_t(from) % 16) == 0)
    p = pload<Packet8s>(from);
  else
    p = ploadu<Packet8s>(from);
  return vec_perm(p, p, p16uc_QUADRUPLICATE16_HI);
}

References p, p16uc_QUADRUPLICATE16_HI, pload< Packet8s >(), and ploadu< Packet8s >().

ploadquad< Packet8uc >()

template<>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::ploadquad< Packet8uc >

(

const uint8_t *

from

)

                                                                        {
  return vreinterpret_u8_u32(
      vzip_u32(vreinterpret_u32_u8(vld1_dup_u8(from)), vreinterpret_u32_u8(vld1_dup_u8(from + 1))).val[0]);
}

References calibrate::val.

ploadquad< Packet8ui >()

template<>

EIGEN_STRONG_INLINE Packet8ui Eigen::internal::ploadquad< Packet8ui >

(

const uint32_t *

from

)

                                                                         {
  return _mm256_insertf128_si256(_mm256_set1_epi32(*from), _mm_set1_epi32(*(from + 1)), 1);
}

ploadquad< Packet8us >() [1/2]

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::ploadquad< Packet8us >

(

const uint16_t *

from

)

                                                                         {
  uint16_t tmp[8] = {*from, *from, *from, *from, *(from + 1), *(from + 1), *(from + 1), *(from + 1)};
  return __lsx_vld(tmp, 0);
}

References tmp.

ploadquad< Packet8us >() [2/2]

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::ploadquad< Packet8us >

(

const unsigned short int *

from

)

                                                                                   {
  Packet8us p;
  if ((std::ptrdiff_t(from) % 16) == 0)
    p = pload<Packet8us>(from);
  else
    p = ploadu<Packet8us>(from);
  return vec_perm(p, p, p16uc_QUADRUPLICATE16_HI);
}

References p, p16uc_QUADRUPLICATE16_HI, pload< Packet8us >(), and ploadu< Packet8us >().

Referenced by ploadquad< Packet8bf >().

ploadquad< PacketXf >()

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::ploadquad< PacketXf >

(

const float *

from

)

                                                                    {
  svuint32_t indices = svindex_u32(0, 1);  // index {base=0, base+step=1, base+step*2, ...}
  indices = svzip1_u32(indices, indices);  // index in the format {a0, a0, a1, a1, a2, a2, ...}
  indices = svzip1_u32(indices, indices);  // index in the format {a0, a0, a0, a0, a1, a1, a1, a1, ...}
  return svld1_gather_u32index_f32(svptrue_b32(), from, indices);
}

ploadquad< PacketXi >()

template<>

EIGEN_STRONG_INLINE PacketXi Eigen::internal::ploadquad< PacketXi >

(

const numext::int32_t *

from

)

                                                                            {
  svuint32_t indices = svindex_u32(0, 1);  // index {base=0, base+step=1, base+step*2, ...}
  indices = svzip1_u32(indices, indices);  // index in the format {a0, a0, a1, a1, a2, a2, ...}
  indices = svzip1_u32(indices, indices);  // index in the format {a0, a0, a0, a0, a1, a1, a1, a1, ...}
  return svld1_gather_u32index_s32(svptrue_b32(), from, indices);
}

ploadRhs()

template<typename Packet >

EIGEN_ALWAYS_INLINE Packet Eigen::internal::ploadRhs

(

const __UNPACK_TYPE__(Packet) *

rhs

)

                                                                         {
  return ploadu<Packet>(rhs);
}

ploadRhsMMA() [1/2]

template<>

EIGEN_ALWAYS_INLINE void Eigen::internal::ploadRhsMMA	(	const double *	rhs,
		__vector_pair &	rhsV
	)

                                                                             {
#if EIGEN_COMP_LLVM
  __builtin_vsx_assemble_pair(
      &rhsV, reinterpret_cast<__vector unsigned char>(ploadRhs<Packet2d>(rhs + (sizeof(Packet2d) / sizeof(double)))),
      reinterpret_cast<__vector unsigned char>(ploadRhs<Packet2d>(rhs)));
#else
  rhsV = *reinterpret_cast<__vector_pair*>(const_cast<double*>(rhs));
#endif
}

ploadRhsMMA() [2/2]

template<typename Scalar , typename Packet >

EIGEN_ALWAYS_INLINE void Eigen::internal::ploadRhsMMA	(	const Scalar *	rhs,
		Packet &	rhsV
	)

                                                                      {
  rhsV = ploadRhs<Packet>(rhs);
}

Referenced by ploadLhsMMA().

ploads()

template<typename Packet >

EIGEN_STRONG_INLINE Packet Eigen::internal::ploads

(

const typename unpacket_traits< Packet >::type *

from

)

ploads< Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::ploads< Packet2d >

(

const double *

from

)

                                                                  {
  EIGEN_DEBUG_UNALIGNED_LOAD return _mm_load_sd(from);
}

References EIGEN_DEBUG_UNALIGNED_LOAD.

ploads< Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::ploads< Packet4f >

(

const float *

from

)

                                                                 {
  EIGEN_DEBUG_UNALIGNED_LOAD return _mm_load_ss(from);
}

References EIGEN_DEBUG_UNALIGNED_LOAD.

ploadt()

template<typename Packet , int Alignment>

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet Eigen::internal::ploadt

(

const typename unpacket_traits< Packet >::type *

from

)

Returns

a packet version of *from. The pointer from must be aligned on a Alignment bytes boundary.

                                                                                                      {
  if (Alignment >= unpacket_traits<Packet>::alignment)
    return pload<Packet>(from);
  else
    return ploadu<Packet>(from);
}

ploadt_partial()

template<typename Packet , int Alignment>

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet Eigen::internal::ploadt_partial	(	const typename unpacket_traits< Packet >::type *	from,
		const Index	n,
		const Index	offset = 0
	)

Returns

n elements of a packet version of *from. The pointer from must be aligned on a Alignment bytes boundary.

                                                                                                   {
  if (Alignment >= unpacket_traits<Packet>::alignment)
    return pload_partial<Packet>(from, n, offset);
  else
    return ploadu_partial<Packet>(from, n, offset);
}

References n.

ploadt_ro()

template<typename Packet , int LoadMode>

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet Eigen::internal::ploadt_ro

(

const typename unpacket_traits< Packet >::type *

from

)

Returns

a packet version of *from. Unlike ploadt, ploadt_ro takes advantage of the read-only memory path on the hardware if available to speedup the loading of data that won't be modified by the current computation.

                                                                                                         {
  return ploadt<Packet, LoadMode>(from);
}

ploadu() [1/2]

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::ploadu ( const typename unpacket_traits< Packet >::type *  from )

inline

Returns

a packet version of *from, (un-aligned load)

                                                                                         {
  return *from;
}

Referenced by benchVec().

ploadu() [2/2]

template<typename Packet >

EIGEN_DEVICE_FUNC std::enable_if_t<unpacket_traits<Packet>::masked_load_available, Packet> Eigen::internal::ploadu ( const typename unpacket_traits< Packet >::type *  from, typename unpacket_traits< Packet >::mask_t  umask  )

inline

Returns

a packet version of *from, (un-aligned masked load) There is no generic implementation. We only have implementations for specialized cases. Generic case should not be called.

ploadu< Packet16b >()

template<>

EIGEN_STRONG_INLINE Packet16b Eigen::internal::ploadu< Packet16b >

(

const bool *

from

)

                                                                  {
  EIGEN_DEBUG_UNALIGNED_LOAD
  return _mm_loadu_si128(reinterpret_cast<const __m128i*>(from));
}

References EIGEN_DEBUG_UNALIGNED_LOAD.

ploadu< Packet16bf >()

template<>

EIGEN_STRONG_INLINE Packet16bf Eigen::internal::ploadu< Packet16bf >

(

const bfloat16 *

from

)

                                                                        {
  return _mm256_loadu_si256(reinterpret_cast<const __m256i*>(from));
}

ploadu< Packet16c >() [1/2]

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::ploadu< Packet16c >

(

const int8_t *

from

)

                                                                    {
  EIGEN_DEBUG_UNALIGNED_LOAD return __lsx_vld(from, 0);
}

References EIGEN_DEBUG_UNALIGNED_LOAD.

ploadu< Packet16c >() [2/2]

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::ploadu< Packet16c >

(

const signed char *

from

)

                                                                         {
  return ploadu_common<Packet16c>(from);
}

Referenced by ploaddup< Packet16c >(), and ploadquad< Packet16c >().

ploadu< Packet16f >() [1/2]

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::ploadu< Packet16f >

(

const float *

from

)

                                                                   {
  EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_ps(from);
}

References EIGEN_DEBUG_UNALIGNED_LOAD.

Referenced by ploadu< Packet8cf >().

ploadu< Packet16f >() [2/2]

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::ploadu< Packet16f >	(	const float *	from,
		uint16_t	umask
	)

                                                                                   {
  __mmask16 mask = static_cast<__mmask16>(umask);
  EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_maskz_loadu_ps(mask, from);
}

References EIGEN_DEBUG_UNALIGNED_LOAD.

ploadu< Packet16h >()

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::ploadu< Packet16h >

(

const Eigen::half *

from

)

                                                                       {
  return _mm256_loadu_si256(reinterpret_cast<const __m256i*>(from));
}

ploadu< Packet16i >()

template<>

EIGEN_STRONG_INLINE Packet16i Eigen::internal::ploadu< Packet16i >

(

const int *

from

)

                                                                 {
  EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_epi32(from);
}

References EIGEN_DEBUG_UNALIGNED_LOAD.

ploadu< Packet16uc >() [1/2]

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::ploadu< Packet16uc >

(

const uint8_t *

from

)

                                                                       {
  EIGEN_DEBUG_UNALIGNED_LOAD return __lsx_vld(from, 0);
}

References EIGEN_DEBUG_UNALIGNED_LOAD.

ploadu< Packet16uc >() [2/2]

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::ploadu< Packet16uc >

(

const unsigned char *

from

)

                                                                             {
  return ploadu_common<Packet16uc>(from);
}

Referenced by pload_partial_common(), ploaddup< Packet16uc >(), ploadquad< Packet16uc >(), ploadu_partial_common(), pstore_partial_common(), and pstoreu_partial_common().

ploadu< Packet1cd >()

template<>

EIGEN_STRONG_INLINE Packet1cd Eigen::internal::ploadu< Packet1cd >

(

const std::complex< double > *

from

)

                                                                                {
  EIGEN_DEBUG_UNALIGNED_LOAD return Packet1cd(ploadu<Packet2d>((const double*)from));
}

References EIGEN_DEBUG_UNALIGNED_LOAD, and ploadu< Packet2d >().

Referenced by pgather< std::complex< double >, Packet4cd >(), pload_complex_full(), pload_realimag_combine(), pload_realimag_combine_row(), and pset1< Packet1cd >().

ploadu< Packet1cf >()

template<>

EIGEN_STRONG_INLINE Packet1cf Eigen::internal::ploadu< Packet1cf >

(

const std::complex< float > *

from

)

                                                                               {
  EIGEN_DEBUG_UNALIGNED_LOAD return Packet1cf(ploadu<Packet2f>((const float*)from));
}

References EIGEN_DEBUG_UNALIGNED_LOAD, and ploadu< Packet2f >().

ploadu< Packet2cd >()

template<>

EIGEN_STRONG_INLINE Packet2cd Eigen::internal::ploadu< Packet2cd >

(

const std::complex< double > *

from

)

                                                                                {
  EIGEN_DEBUG_UNALIGNED_LOAD return Packet2cd(_mm256_loadu_pd((const double*)from));
}

References EIGEN_DEBUG_UNALIGNED_LOAD.

ploadu< Packet2cf >()

template<>

EIGEN_STRONG_INLINE Packet2cf Eigen::internal::ploadu< Packet2cf >

(

const std::complex< float > *

from

)

                                                                               {
  return Packet2cf(ploadu<Packet4f>((const float*)from));
}

References ploadu< Packet4f >().

Referenced by pload_complex_full_row(), and pload_realimag_combine_row().

ploadu< Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::ploadu< Packet2d >

(

const double *

from

)

                                                                  {
  EIGEN_DEBUG_UNALIGNED_LOAD return (Packet2d)__lsx_vld(from, 0);
}

References EIGEN_DEBUG_UNALIGNED_LOAD.

Referenced by pload_complex(), pload_realimag(), and ploadu< Packet1cd >().

ploadu< Packet2f >()

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::ploadu< Packet2f >

(

const float *

from

)

                                                                 {
  EIGEN_DEBUG_UNALIGNED_LOAD return vld1_f32(from);
}

References EIGEN_DEBUG_UNALIGNED_LOAD.

Referenced by ploadu< Packet1cf >().

ploadu< Packet2i >()

template<>

EIGEN_STRONG_INLINE Packet2i Eigen::internal::ploadu< Packet2i >

(

const int32_t *

from

)

                                                                   {
  EIGEN_DEBUG_UNALIGNED_LOAD return vld1_s32(from);
}

References EIGEN_DEBUG_UNALIGNED_LOAD.

ploadu< Packet2l >()

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::ploadu< Packet2l >

(

const int64_t *

from

)

                                                                   {
  EIGEN_DEBUG_UNALIGNED_LOAD return __lsx_vld(from, 0);
}

References EIGEN_DEBUG_UNALIGNED_LOAD.

ploadu< Packet2ui >()

template<>

EIGEN_STRONG_INLINE Packet2ui Eigen::internal::ploadu< Packet2ui >

(

const uint32_t *

from

)

                                                                      {
  EIGEN_DEBUG_UNALIGNED_LOAD return vld1_u32(from);
}

References EIGEN_DEBUG_UNALIGNED_LOAD.

ploadu< Packet2ul >()

template<>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::ploadu< Packet2ul >

(

const uint64_t *

from

)

                                                                      {
  EIGEN_DEBUG_UNALIGNED_LOAD return __lsx_vld(from, 0);
}

References EIGEN_DEBUG_UNALIGNED_LOAD.

ploadu< Packet32h >()

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::ploadu< Packet32h >

(

const Eigen::half *

from

)

                                                                       {
  EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_ph(from);
}

References EIGEN_DEBUG_UNALIGNED_LOAD.

ploadu< Packet4bf >()

template<>

EIGEN_STRONG_INLINE Packet4bf Eigen::internal::ploadu< Packet4bf >

(

const bfloat16 *

from

)

                                                                      {
  return Packet4bf(ploadu<Packet4us>(reinterpret_cast<const uint16_t*>(from)));
}

References ploadu< Packet4us >().

ploadu< Packet4c >()

template<>

EIGEN_STRONG_INLINE Packet4c Eigen::internal::ploadu< Packet4c >

(

const int8_t *

from

)

                                                                  {
  Packet4c res;
  memcpy(&res, from, sizeof(Packet4c));
  return res;
}

References res.

ploadu< Packet4cd >()

template<>

EIGEN_STRONG_INLINE Packet4cd Eigen::internal::ploadu< Packet4cd >

(

const std::complex< double > *

from

)

                                                                                {
  EIGEN_DEBUG_UNALIGNED_LOAD return Packet4cd(ploadu<Packet8d>((const double*)from));
}

References EIGEN_DEBUG_UNALIGNED_LOAD, and ploadu< Packet8d >().

ploadu< Packet4cf >()

template<>

EIGEN_STRONG_INLINE Packet4cf Eigen::internal::ploadu< Packet4cf >

(

const std::complex< float > *

from

)

                                                                               {
  EIGEN_DEBUG_UNALIGNED_LOAD return Packet4cf(_mm256_loadu_ps(&numext::real_ref(*from)));
}

References EIGEN_DEBUG_UNALIGNED_LOAD, and Eigen::numext::real_ref().

ploadu< Packet4d >()

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::ploadu< Packet4d >

(

const double *

from

)

                                                                  {
  EIGEN_DEBUG_UNALIGNED_LOAD return _mm256_loadu_pd(from);
}

References EIGEN_DEBUG_UNALIGNED_LOAD.

Referenced by Eigen::internal::gemm_class< Scalar, is_unit_inc >::a_load(), and ploaddup< Packet8d >().

ploadu< Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::ploadu< Packet4f >

(

const float *

from

)

                                                                 {
  return ploadu_common<Packet4f>(from);
}

Referenced by Eigen::internal::gemm_class< Scalar, is_unit_inc >::a_load(), convertF32toBF16VSX(), KLoop(), loadAndMultiplyF32(), loadTwoRhsFloat32(), outputVecCol(), outputVecResults(), pload_complex(), pload_real_full(), pload_real_row(), pload_realimag_row(), ploadquad< Packet16f >(), ploadu< Packet2cf >(), pset1< Packet2cf >(), Eigen::internal::quat_product< Architecture::Target, Derived, OtherDerived, float >::run(), and Eigen::internal::quat_conj< Architecture::Target, Derived, float >::run().

ploadu< Packet4i >() [1/2]

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::ploadu< Packet4i >

(

const int *

from

)

                                                               {
  return ploadu_common<Packet4i>(from);
}

Referenced by ploaddup< Packet8i >(), and ploadquad< Packet16i >().

ploadu< Packet4i >() [2/2]

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::ploadu< Packet4i >

(

const int32_t *

from

)

                                                                   {
  EIGEN_DEBUG_UNALIGNED_LOAD return __lsx_vld(from, 0);
}

References EIGEN_DEBUG_UNALIGNED_LOAD.

ploadu< Packet4s >()

template<>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::ploadu< Packet4s >

(

const int16_t *

from

)

                                                                   {
  EIGEN_DEBUG_UNALIGNED_LOAD return vld1_s16(from);
}

References EIGEN_DEBUG_UNALIGNED_LOAD.

ploadu< Packet4uc >()

template<>

EIGEN_STRONG_INLINE Packet4uc Eigen::internal::ploadu< Packet4uc >

(

const uint8_t *

from

)

                                                                     {
  Packet4uc res;
  memcpy(&res, from, sizeof(Packet4uc));
  return res;
}

References res.

ploadu< Packet4ui >()

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::ploadu< Packet4ui >

(

const uint32_t *

from

)

                                                                      {
  EIGEN_DEBUG_UNALIGNED_LOAD return __lsx_vld(from, 0);
}

References EIGEN_DEBUG_UNALIGNED_LOAD.

Referenced by ploaddup< Packet8ui >().

ploadu< Packet4us >()

template<>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::ploadu< Packet4us >

(

const uint16_t *

from

)

                                                                      {
  EIGEN_DEBUG_UNALIGNED_LOAD return vld1_u16(from);
}

References EIGEN_DEBUG_UNALIGNED_LOAD.

Referenced by ploadu< Packet4bf >().

ploadu< Packet8bf >()

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::ploadu< Packet8bf >

(

const bfloat16 *

from

)

                                                                      {
  return ploadu_common<Packet8us>(reinterpret_cast<const unsigned short int*>(from));
}

Referenced by convertArrayPointerBF16toF32DupOne(), KLoop(), loadBF16fromResult(), and loadBfloat16().

ploadu< Packet8c >()

template<>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::ploadu< Packet8c >

(

const int8_t *

from

)

                                                                  {
  EIGEN_DEBUG_UNALIGNED_LOAD return vld1_s8(from);
}

References EIGEN_DEBUG_UNALIGNED_LOAD.

ploadu< Packet8cf >()

template<>

EIGEN_STRONG_INLINE Packet8cf Eigen::internal::ploadu< Packet8cf >

(

const std::complex< float > *

from

)

                                                                               {
  EIGEN_DEBUG_UNALIGNED_LOAD return Packet8cf(ploadu<Packet16f>(&numext::real_ref(*from)));
}

References EIGEN_DEBUG_UNALIGNED_LOAD, ploadu< Packet16f >(), and Eigen::numext::real_ref().

ploadu< Packet8d >() [1/2]

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::ploadu< Packet8d >

(

const double *

from

)

                                                                  {
  EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_pd(from);
}

References EIGEN_DEBUG_UNALIGNED_LOAD.

Referenced by ploadu< Packet4cd >().

ploadu< Packet8d >() [2/2]

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::ploadu< Packet8d >	(	const double *	from,
		uint8_t	umask
	)

                                                                                 {
  __mmask8 mask = static_cast<__mmask8>(umask);
  EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_maskz_loadu_pd(mask, from);
}

References EIGEN_DEBUG_UNALIGNED_LOAD.

ploadu< Packet8f >() [1/2]

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::ploadu< Packet8f >

(

const float *

from

)

                                                                 {
  EIGEN_DEBUG_UNALIGNED_LOAD return _mm256_loadu_ps(from);
}

References EIGEN_DEBUG_UNALIGNED_LOAD.

ploadu< Packet8f >() [2/2]

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::ploadu< Packet8f >	(	const float *	from,
		uint8_t	umask
	)

                                                                                {
#ifdef EIGEN_VECTORIZE_AVX512
  __mmask16 mask = static_cast<__mmask16>(umask & 0x00FF);
  EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_castps512_ps256(_mm512_maskz_loadu_ps(mask, from));
#else
  Packet8i mask = _mm256_set1_epi8(static_cast<char>(umask));
  const Packet8i bit_mask =
      _mm256_set_epi32(0xffffff7f, 0xffffffbf, 0xffffffdf, 0xffffffef, 0xfffffff7, 0xfffffffb, 0xfffffffd, 0xfffffffe);
  mask = por<Packet8i>(mask, bit_mask);
  mask = pcmp_eq<Packet8i>(mask, _mm256_set1_epi32(0xffffffff));
  EIGEN_DEBUG_UNALIGNED_LOAD return _mm256_maskload_ps(from, mask);
#endif
}

References EIGEN_DEBUG_UNALIGNED_LOAD, and por< Packet8i >().

ploadu< Packet8h >()

template<>

EIGEN_STRONG_INLINE Packet8h Eigen::internal::ploadu< Packet8h >

(

const Eigen::half *

from

)

                                                                     {
  return _mm_loadu_si128(reinterpret_cast<const __m128i*>(from));
}

ploadu< Packet8i >()

template<>

EIGEN_STRONG_INLINE Packet8i Eigen::internal::ploadu< Packet8i >

(

const int *

from

)

                                                               {
  EIGEN_DEBUG_UNALIGNED_LOAD return _mm256_loadu_si256(reinterpret_cast<const __m256i*>(from));
}

References EIGEN_DEBUG_UNALIGNED_LOAD.

ploadu< Packet8l >()

template<>

EIGEN_STRONG_INLINE Packet8l Eigen::internal::ploadu< Packet8l >

(

const int64_t *

from

)

                                                                   {
  EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_epi64(from);
}

References EIGEN_DEBUG_UNALIGNED_LOAD.

ploadu< Packet8s >() [1/2]

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::ploadu< Packet8s >

(

const int16_t *

from

)

                                                                   {
  EIGEN_DEBUG_UNALIGNED_LOAD return __lsx_vld(from, 0);
}

References EIGEN_DEBUG_UNALIGNED_LOAD.

ploadu< Packet8s >() [2/2]

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::ploadu< Packet8s >

(

const short int *

from

)

                                                                     {
  return ploadu_common<Packet8s>(from);
}

Referenced by ploaddup< Packet8s >(), and ploadquad< Packet8s >().

ploadu< Packet8uc >()

template<>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::ploadu< Packet8uc >

(

const uint8_t *

from

)

                                                                     {
  EIGEN_DEBUG_UNALIGNED_LOAD return vld1_u8(from);
}

References EIGEN_DEBUG_UNALIGNED_LOAD.

ploadu< Packet8ui >()

template<>

EIGEN_STRONG_INLINE Packet8ui Eigen::internal::ploadu< Packet8ui >

(

const uint32_t *

from

)

                                                                      {
  EIGEN_DEBUG_UNALIGNED_LOAD return _mm256_loadu_si256(reinterpret_cast<const __m256i*>(from));
}

References EIGEN_DEBUG_UNALIGNED_LOAD.

ploadu< Packet8us >() [1/2]

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::ploadu< Packet8us >

(

const uint16_t *

from

)

                                                                      {
  EIGEN_DEBUG_UNALIGNED_LOAD return __lsx_vld(from, 0);
}

References EIGEN_DEBUG_UNALIGNED_LOAD.

ploadu< Packet8us >() [2/2]

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::ploadu< Packet8us >

(

const unsigned short int *

from

)

                                                                                {
  return ploadu_common<Packet8us>(from);
}

Referenced by ploaddup< Packet8us >(), and ploadquad< Packet8us >().

ploadu< PacketXf >()

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::ploadu< PacketXf >

(

const float *

from

)

                                                                 {
  EIGEN_DEBUG_UNALIGNED_LOAD return svld1_f32(svptrue_b32(), from);
}

References EIGEN_DEBUG_UNALIGNED_LOAD.

ploadu< PacketXi >()

template<>

EIGEN_STRONG_INLINE PacketXi Eigen::internal::ploadu< PacketXi >

(

const numext::int32_t *

from

)

                                                                         {
  EIGEN_DEBUG_UNALIGNED_LOAD return svld1_s32(svptrue_b32(), from);
}

References EIGEN_DEBUG_UNALIGNED_LOAD.

ploadu_common()

template<typename Packet >

EIGEN_STRONG_INLINE Packet Eigen::internal::ploadu_common

(

const __UNPACK_TYPE__(Packet) *

from

)

                                                                               {
  EIGEN_DEBUG_ALIGNED_LOAD
#if defined(EIGEN_VECTORIZE_VSX) || !defined(_BIG_ENDIAN)
  EIGEN_DEBUG_UNALIGNED_LOAD
  return vec_xl(0, const_cast<__UNPACK_TYPE__(Packet)*>(from));
#else
  Packet16uc MSQ, LSQ;
  Packet16uc mask;
  MSQ = vec_ld(0, (unsigned char*)from);   // most significant quadword
  LSQ = vec_ld(15, (unsigned char*)from);  // least significant quadword
  mask = vec_lvsl(0, from);                // create the permute mask
  // TODO: Add static_cast here
  return (Packet)vec_perm(MSQ, LSQ, mask);  // align the data
#endif
}

References __UNPACK_TYPE__(), EIGEN_DEBUG_ALIGNED_LOAD, and EIGEN_DEBUG_UNALIGNED_LOAD.

ploadu_partial()

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::ploadu_partial ( const typename unpacket_traits< Packet >::type *  from, const Index  n, const Index  offset = 0  )

inline

Returns

n elements of a packet version of *from, (un-aligned load) All elements after the last element loaded will initialized with zero

                                                                       {
  const Index packet_size = unpacket_traits<Packet>::size;
  eigen_assert(n + offset <= packet_size && "number of elements plus offset will read past end of packet");
  typedef typename unpacket_traits<Packet>::type Scalar;
  EIGEN_ALIGN_MAX Scalar elements[packet_size] = {Scalar(0)};
  for (Index i = offset; i < numext::mini(n + offset, packet_size); i++) {
    elements[i] = from[i - offset];
  }
  return pload<Packet>(elements);
}

References EIGEN_ALIGN_MAX, eigen_assert, i, Eigen::numext::mini(), and n.

ploadu_partial< Packet16c >()

template<>

EIGEN_ALWAYS_INLINE Packet16c Eigen::internal::ploadu_partial< Packet16c >	(	const signed char *	from,
		const Index	n,
		const Index	offset
	)

                                                                                                                    {
  return ploadu_partial_common<Packet16c>(from, n, offset);
}

References n.

ploadu_partial< Packet16uc >()

template<>

EIGEN_ALWAYS_INLINE Packet16uc Eigen::internal::ploadu_partial< Packet16uc >	(	const unsigned char *	from,
		const Index	n,
		const Index	offset
	)

                                                                              {
  return ploadu_partial_common<Packet16uc>(from, n, offset);
}

References n.

ploadu_partial< Packet2cf >()

template<>

EIGEN_ALWAYS_INLINE Packet2cf Eigen::internal::ploadu_partial< Packet2cf >	(	const std::complex< float > *	from,
		const Index	n,
		const Index	offset
	)

                                                                            {
  return Packet2cf(ploadu_partial<Packet4f>((const float*)from, n * 2, offset * 2));
}

References n, and ploadu_partial< Packet4f >().

ploadu_partial< Packet4f >()

template<>

EIGEN_ALWAYS_INLINE Packet4f Eigen::internal::ploadu_partial< Packet4f >	(	const float *	from,
		const Index	n,
		const Index	offset
	)

                                                                                                            {
  return ploadu_partial_common<Packet4f>(from, n, offset);
}

References n.

Referenced by ploadu_partial< Packet2cf >().

ploadu_partial< Packet4i >()

template<>

EIGEN_ALWAYS_INLINE Packet4i Eigen::internal::ploadu_partial< Packet4i >	(	const int *	from,
		const Index	n,
		const Index	offset
	)

                                                                                                          {
  return ploadu_partial_common<Packet4i>(from, n, offset);
}

References n.

ploadu_partial< Packet8bf >()

template<>

EIGEN_ALWAYS_INLINE Packet8bf Eigen::internal::ploadu_partial< Packet8bf >	(	const bfloat16 *	from,
		const Index	n,
		const Index	offset
	)

                                                                                                                 {
  return ploadu_partial_common<Packet8us>(reinterpret_cast<const unsigned short int*>(from), n, offset);
}

References n.

ploadu_partial< Packet8s >()

template<>

EIGEN_ALWAYS_INLINE Packet8s Eigen::internal::ploadu_partial< Packet8s >	(	const short int *	from,
		const Index	n,
		const Index	offset
	)

                                                                                                                {
  return ploadu_partial_common<Packet8s>(from, n, offset);
}

References n.

ploadu_partial< Packet8us >()

template<>

EIGEN_ALWAYS_INLINE Packet8us Eigen::internal::ploadu_partial< Packet8us >	(	const unsigned short int *	from,
		const Index	n,
		const Index	offset
	)

                                                                            {
  return ploadu_partial_common<Packet8us>(from, n, offset);
}

References n.

ploadu_partial_common()

template<typename Packet >

EIGEN_ALWAYS_INLINE Packet Eigen::internal::ploadu_partial_common	(	const __UNPACK_TYPE__(Packet) *	from,
		const Index	n,
		const Index	offset
	)

                                                                     {
  const Index packet_size = unpacket_traits<Packet>::size;
  eigen_internal_assert(n + offset <= packet_size && "number of elements plus offset will read past end of packet");
  const Index size = sizeof(__UNPACK_TYPE__(Packet));
#ifdef _ARCH_PWR9
  EIGEN_UNUSED_VARIABLE(packet_size);
  EIGEN_DEBUG_ALIGNED_LOAD
  EIGEN_DEBUG_UNALIGNED_LOAD
  Packet load = vec_xl_len(const_cast<__UNPACK_TYPE__(Packet)*>(from), n * size);
  if (offset) {
    Packet16uc shift = pset1<Packet16uc>(offset * 8 * size);
#ifdef _BIG_ENDIAN
    load = Packet(vec_sro(Packet16uc(load), shift));
#else
    load = Packet(vec_slo(Packet16uc(load), shift));
#endif
  }
  return load;
#else
  if (n) {
    EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) load[packet_size];
    unsigned char* load2 = reinterpret_cast<unsigned char*>(load + offset);
    unsigned char* from2 = reinterpret_cast<unsigned char*>(const_cast<__UNPACK_TYPE__(Packet)*>(from));
    Index n2 = n * size;
    if (16 <= n2) {
      pstoreu(load2, ploadu<Packet16uc>(from2));
    } else {
      memcpy((void*)load2, (void*)from2, n2);
    }
    return pload_ignore<Packet>(load);
  } else {
    return Packet(pset1<Packet16uc>(0));
  }
#endif
}

References __UNPACK_TYPE__(), EIGEN_ALIGN16, EIGEN_DEBUG_ALIGNED_LOAD, EIGEN_DEBUG_UNALIGNED_LOAD, eigen_internal_assert, EIGEN_UNUSED_VARIABLE, load(), n, ploadu< Packet16uc >(), pset1< Packet16uc >(), pstoreu(), and size.

plog()

template<typename Packet >

EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::plog

(

const Packet &

a

)

Returns

the log of a (coeff-wise)

                                                                                  {
  EIGEN_USING_STD(log);
  return log(a);
}

References a, EIGEN_USING_STD, and Eigen::bfloat16_impl::log().

Referenced by generic_expm1(), generic_log1p(), generic_ndtri_lt_exp_neg_two(), packetmath_complex(), packetmath_real(), Eigen::internal::scalar_log_op< Scalar >::packetOp(), patanh_double(), patanh_float(), plog2(), plog< Packet32h >(), plog_complex(), Eigen::internal::generic_k0e< T, float >::run(), Eigen::internal::generic_k0e< T, double >::run(), Eigen::internal::generic_k0< T, float >::run(), Eigen::internal::generic_k0< T, double >::run(), Eigen::internal::generic_k1e< T, float >::run(), Eigen::internal::generic_k1e< T, double >::run(), Eigen::internal::generic_k1< T, float >::run(), Eigen::internal::generic_k1< T, double >::run(), Eigen::internal::generic_y0< T, float >::run(), Eigen::internal::generic_y0< T, double >::run(), Eigen::internal::generic_y1< T, float >::run(), and Eigen::internal::generic_y1< T, double >::run().

plog10()

template<typename Packet >

EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::plog10

(

const Packet &

a

)

Returns

the log10 of a (coeff-wise)

                                                                                    {
  EIGEN_USING_STD(log10);
  return log10(a);
}

References a, EIGEN_USING_STD, and Eigen::bfloat16_impl::log10().

Referenced by Eigen::internal::scalar_log10_op< Scalar >::packetOp().

plog1p()

template<typename Packet >

EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::plog1p

(

const Packet &

a

)

Returns

the log1p of a (coeff-wise)

                                                                                    {
  return numext::log1p(a);
}

References a, and Eigen::bfloat16_impl::log1p().

Referenced by packetmath_real(), Eigen::internal::scalar_log1p_op< Scalar >::packetOp(), and plog1p< Packet32h >().

plog1p< Packet16h >()

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::plog1p< Packet16h >

(

const Packet16h &

)

plog1p< Packet32h >()

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::plog1p< Packet32h >

(

const Packet32h &

a

)

                                                                    {
  Packet16h low;
  Packet16h high;
  extract2Packet16h(a, low, high);
 
  Packet16h lowOut = plog1p(low);
  Packet16h highOut = plog1p(high);
 
  return combine2Packet16h(lowOut, highOut);
}

References a, combine2Packet16h(), extract2Packet16h(), and plog1p().

plog2()

template<typename Packet >

EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::plog2

(

const Packet &

a

)

Returns

the log2 of a (coeff-wise)

                                                                                   {
  using Scalar = typename internal::unpacket_traits<Packet>::type;
  using RealScalar = typename NumTraits<Scalar>::Real;
  return pmul(pset1<Packet>(Scalar(RealScalar(EIGEN_LOG2E))), plog(a));
}

References a, EIGEN_LOG2E, plog(), and pmul().

Referenced by Eigen::internal::accurate_log2< Scalar >::operator()(), packetmath_real(), Eigen::internal::scalar_log2_op< Scalar >::packetOp(), and plog2< Packet32h >().

plog2< Packet16h >()

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::plog2< Packet16h >

(

const Packet16h &

)

plog2< Packet32h >()

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::plog2< Packet32h >

(

const Packet32h &

a

)

                                                                   {
  Packet16h low;
  Packet16h high;
  extract2Packet16h(a, low, high);
 
  Packet16h lowOut = plog2(low);
  Packet16h highOut = plog2(high);
 
  return combine2Packet16h(lowOut, highOut);
}

References a, combine2Packet16h(), extract2Packet16h(), and plog2().

plog2_double()

template<typename Packet >

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::plog2_double

(

const Packet

_x

)

Returns

log2(x) for single precision float

                                                                                         {
  return plog_impl_double<Packet, /* base2 */ true>(_x);
}

References plog_impl_double().

plog2_float()

template<typename Packet >

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::plog2_float

(

const Packet

_x

)

Returns

log2(x) for single precision float

                                                                                        {
  return plog_impl_float<Packet, /* base2 */ true>(_x);
}

References plog_impl_float().

plog< Packet16h >()

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::plog< Packet16h >

(

const Packet16h &

)

plog< Packet1cd >() [1/2]

template<>

EIGEN_STRONG_INLINE Packet1cd Eigen::internal::plog< Packet1cd >

(

const Packet1cd &

a

)

                                                                  {
  return plog_complex(a);
}

References a, and plog_complex().

plog< Packet1cd >() [2/2]

template<>

EIGEN_STRONG_INLINE Packet1cd Eigen::internal::plog< Packet1cd >	(	const Packet1cd &	a,
		const Packet1cd &	b
	)

                                                                                      {
  return plog_complex(a, b);
}

References a, b, and plog_complex().

plog< Packet1cf >()

template<>

EIGEN_STRONG_INLINE Packet1cf Eigen::internal::plog< Packet1cf >

(

const Packet1cf &

a

)

                                                                  {
  return plog_complex(a);
}

References a, and plog_complex().

plog< Packet2cd >()

template<>

EIGEN_STRONG_INLINE Packet2cd Eigen::internal::plog< Packet2cd >

(

const Packet2cd &

a

)

                                                                  {
  return plog_complex<Packet2cd>(a);
}

References a.

plog< Packet2cf >() [1/2]

template<>

EIGEN_STRONG_INLINE Packet2cf Eigen::internal::plog< Packet2cf >

(

const Packet2cf &

a

)

                                                                  {
  return plog_complex<Packet2cf>(a);
}

References a.

plog< Packet2cf >() [2/2]

template<>

EIGEN_STRONG_INLINE Packet2cf Eigen::internal::plog< Packet2cf >	(	const Packet2cf &	a,
		const Packet2cf &	b
	)

                                                                                      {
  return plog_complex(a, b);
}

References a, b, and plog_complex().

plog< Packet32h >()

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::plog< Packet32h >

(

const Packet32h &

a

)

                                                                  {
  Packet16h low;
  Packet16h high;
  extract2Packet16h(a, low, high);
 
  Packet16h lowOut = plog(low);
  Packet16h highOut = plog(high);
 
  return combine2Packet16h(lowOut, highOut);
}

References a, combine2Packet16h(), extract2Packet16h(), and plog().

plog< Packet4cd >()

template<>

EIGEN_STRONG_INLINE Packet4cd Eigen::internal::plog< Packet4cd >

(

const Packet4cd &

a

)

                                                                  {
  return plog_complex<Packet4cd>(a);
}

References a.

plog< Packet4cf >()

template<>

EIGEN_STRONG_INLINE Packet4cf Eigen::internal::plog< Packet4cf >

(

const Packet4cf &

a

)

                                                                  {
  return plog_complex<Packet4cf>(a);
}

References a.

plog< Packet4f >()

template<>

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet4f Eigen::internal::plog< Packet4f >

(

const Packet4f &

_x

)

                                                                                                {
  static EIGEN_DECLARE_CONST_Packet4f(cephes_SQRTHF, 0.707106781186547524f);
  static EIGEN_DECLARE_CONST_Packet4f(cephes_log_p0, 7.0376836292e-2f);
  static EIGEN_DECLARE_CONST_Packet4f(cephes_log_p1, -1.1514610310e-1f);
  static EIGEN_DECLARE_CONST_Packet4f(cephes_log_p2, 1.1676998740e-1f);
  static EIGEN_DECLARE_CONST_Packet4f(cephes_log_p3, -1.2420140846e-1f);
  static EIGEN_DECLARE_CONST_Packet4f(cephes_log_p4, +1.4249322787e-1f);
  static EIGEN_DECLARE_CONST_Packet4f(cephes_log_p5, -1.6668057665e-1f);
  static EIGEN_DECLARE_CONST_Packet4f(cephes_log_p6, +2.0000714765e-1f);
  static EIGEN_DECLARE_CONST_Packet4f(cephes_log_p7, -2.4999993993e-1f);
  static EIGEN_DECLARE_CONST_Packet4f(cephes_log_p8, +3.3333331174e-1f);
  static EIGEN_DECLARE_CONST_Packet4f(cephes_log_q1, -2.12194440e-4f);
  static EIGEN_DECLARE_CONST_Packet4f(cephes_log_q2, 0.693359375f);
  static EIGEN_DECLARE_CONST_Packet4f(half, 0.5f);
  static EIGEN_DECLARE_CONST_Packet4f(1, 1.0f);
 
  // Convert negative argument into NAN (quiet negative, to be specific).
  Packet4f zero = (Packet4f)__builtin_msa_ldi_w(0);
  Packet4i neg_mask = __builtin_msa_fclt_w(_x, zero);
  Packet4i zero_mask = __builtin_msa_fceq_w(_x, zero);
  Packet4f non_neg_x_or_nan = padd(_x, (Packet4f)neg_mask);  // Add 0.0 or NAN.
  Packet4f x = non_neg_x_or_nan;
 
  // Extract exponent from x = mantissa * 2**exponent, where 1.0 <= mantissa < 2.0.
  // N.B. the exponent is one less of what frexpf() would return.
  Packet4i e_int = __builtin_msa_ftint_s_w(__builtin_msa_flog2_w(x));
  // Multiply x by 2**(-exponent-1) to get 0.5 <= x < 1.0 as from frexpf().
  x = __builtin_msa_fexp2_w(x, (Packet4i)__builtin_msa_nori_b((v16u8)e_int, 0));
 
  /*
     if (x < SQRTHF) {
       x = x + x - 1.0;
     } else {
       e += 1;
       x = x - 1.0;
     }
  */
  Packet4f xx = padd(x, x);
  Packet4i ge_mask = __builtin_msa_fcle_w(p4f_cephes_SQRTHF, x);
  e_int = psub(e_int, ge_mask);
  x = (Packet4f)__builtin_msa_bsel_v((v16u8)ge_mask, (v16u8)xx, (v16u8)x);
  x = psub(x, p4f_1);
  Packet4f e = __builtin_msa_ffint_s_w(e_int);
 
  Packet4f x2 = pmul(x, x);
  Packet4f x3 = pmul(x2, x);
 
  Packet4f y, y1, y2;
  y = pmadd(p4f_cephes_log_p0, x, p4f_cephes_log_p1);
  y1 = pmadd(p4f_cephes_log_p3, x, p4f_cephes_log_p4);
  y2 = pmadd(p4f_cephes_log_p6, x, p4f_cephes_log_p7);
  y = pmadd(y, x, p4f_cephes_log_p2);
  y1 = pmadd(y1, x, p4f_cephes_log_p5);
  y2 = pmadd(y2, x, p4f_cephes_log_p8);
  y = pmadd(y, x3, y1);
  y = pmadd(y, x3, y2);
  y = pmul(y, x3);
 
  y = pmadd(e, p4f_cephes_log_q1, y);
  x = __builtin_msa_fmsub_w(x, x2, p4f_half);
  x = padd(x, y);
  x = pmadd(e, p4f_cephes_log_q2, x);
 
  // x is now the logarithm result candidate. We still need to handle the
  // extreme arguments of zero and positive infinity, though.
  // N.B. if the argument is +INFINITY, x is NAN because the polynomial terms
  // contain infinities of both signs (see the coefficients and code above).
  // INFINITY - INFINITY is NAN.
 
  // If the argument is +INFINITY, make it the new result candidate.
  // To achieve that we choose the smaller of the result candidate and the
  // argument.
  // This is correct for all finite pairs of values (the logarithm is smaller
  // than the argument).
  // This is also correct in the special case when the argument is +INFINITY
  // and the result candidate is NAN. This is because the fmin.df instruction
  // prefers non-NANs to NANs.
  x = __builtin_msa_fmin_w(x, non_neg_x_or_nan);
 
  // If the argument is zero (including -0.0), the result becomes -INFINITY.
  Packet4i neg_infs = __builtin_msa_slli_w(zero_mask, 23);
  x = (Packet4f)__builtin_msa_bsel_v((v16u8)zero_mask, (v16u8)x, (v16u8)neg_infs);
 
  return x;
}

References e(), EIGEN_DECLARE_CONST_Packet4f(), padd(), pmadd(), pmul(), psub(), plotDoE::x, Global_parameters::x2(), y, and zero().

plog< Packet8bf >()

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::plog< Packet8bf >

(

const Packet8bf &

a

)

                                                                  {
  BF16_TO_F32_UNARY_OP_WRAPPER(plog_float, a);
}

References a, BF16_TO_F32_UNARY_OP_WRAPPER, and plog_float().

plog< Packet8cf >()

template<>

EIGEN_STRONG_INLINE Packet8cf Eigen::internal::plog< Packet8cf >

(

const Packet8cf &

a

)

                                                                  {
  return plog_complex<Packet8cf>(a);
}

References a.

plog< PacketXf >()

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::plog< PacketXf >

(

const PacketXf &

x

)

                                                               {
  return plog_float(x);
}

References plog_float(), and plotDoE::x.

plog_complex()

template<typename Packet >

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::plog_complex

(

const Packet &

x

)

Returns

log(x) for complex types

                                                                                         {
  typedef typename unpacket_traits<Packet>::type Scalar;
  typedef typename Scalar::value_type RealScalar;
  typedef typename unpacket_traits<Packet>::as_real RealPacket;
 
  RealPacket real_mask_rp = peven_mask(x.v);
  Packet real_mask(real_mask_rp);
 
  // Real part
  RealPacket x_flip = pcplxflip(x).v;  // b, a
  Packet x_norm = phypot_complex(x);   // sqrt(a^2 + b^2), sqrt(a^2 + b^2)
  RealPacket xlogr = plog(x_norm.v);   // log(sqrt(a^2 + b^2)), log(sqrt(a^2 + b^2))
 
  // Imag part
  RealPacket ximg = patan2(x.v, x_flip);  // atan2(a, b), atan2(b, a)
 
  const RealPacket cst_pos_inf = pset1<RealPacket>(NumTraits<RealScalar>::infinity());
  RealPacket x_abs = pabs(x.v);
  RealPacket is_x_pos_inf = pcmp_eq(x_abs, cst_pos_inf);
  RealPacket is_y_pos_inf = pcplxflip(Packet(is_x_pos_inf)).v;
  RealPacket is_any_inf = por(is_x_pos_inf, is_y_pos_inf);
  RealPacket xreal = pselect(is_any_inf, cst_pos_inf, xlogr);
 
  Packet xres = pselect(real_mask, Packet(xreal), Packet(ximg));  // log(sqrt(a^2 + b^2)), atan2(b, a)
  return xres;
}

References pabs(), patan2(), pcmp_eq(), pcplxflip(), peven_mask(), phypot_complex(), plog(), por(), pselect(), Eigen::internal::Packet2cf::v, and plotDoE::x.

Referenced by plog< Packet1cd >(), plog< Packet1cf >(), and plog< Packet2cf >().

plog_double()

template<typename Packet >

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::plog_double

(

const Packet

_x

)

Returns

log(x) for single precision float

                                                                                        {
  return plog_impl_double<Packet, /* base2 */ false>(_x);
}

References plog_impl_double().

plog_float()

template<typename Packet >

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::plog_float

(

const Packet

_x

)

Returns

log(x) for single precision float

                                                                                       {
  return plog_impl_float<Packet, /* base2 */ false>(_x);
}

References plog_impl_float().

Referenced by plog< Packet8bf >(), and plog< PacketXf >().

plog_impl_double()

template<typename Packet , bool base2>

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::plog_impl_double

(

const Packet

_x

)

                                                                                             {
  Packet x = _x;
 
  const Packet cst_1 = pset1<Packet>(1.0);
  const Packet cst_neg_half = pset1<Packet>(-0.5);
  const Packet cst_minus_inf = pset1frombits<Packet>(static_cast<uint64_t>(0xfff0000000000000ull));
  const Packet cst_pos_inf = pset1frombits<Packet>(static_cast<uint64_t>(0x7ff0000000000000ull));
 
  // Polynomial Coefficients for log(1+x) = x - x**2/2 + x**3 P(x)/Q(x)
  //                             1/sqrt(2) <= x < sqrt(2)
  const Packet cst_cephes_SQRTHF = pset1<Packet>(0.70710678118654752440E0);
  const Packet cst_cephes_log_p0 = pset1<Packet>(1.01875663804580931796E-4);
  const Packet cst_cephes_log_p1 = pset1<Packet>(4.97494994976747001425E-1);
  const Packet cst_cephes_log_p2 = pset1<Packet>(4.70579119878881725854E0);
  const Packet cst_cephes_log_p3 = pset1<Packet>(1.44989225341610930846E1);
  const Packet cst_cephes_log_p4 = pset1<Packet>(1.79368678507819816313E1);
  const Packet cst_cephes_log_p5 = pset1<Packet>(7.70838733755885391666E0);
 
  const Packet cst_cephes_log_q0 = pset1<Packet>(1.0);
  const Packet cst_cephes_log_q1 = pset1<Packet>(1.12873587189167450590E1);
  const Packet cst_cephes_log_q2 = pset1<Packet>(4.52279145837532221105E1);
  const Packet cst_cephes_log_q3 = pset1<Packet>(8.29875266912776603211E1);
  const Packet cst_cephes_log_q4 = pset1<Packet>(7.11544750618563894466E1);
  const Packet cst_cephes_log_q5 = pset1<Packet>(2.31251620126765340583E1);
 
  Packet e;
  // extract significant in the range [0.5,1) and exponent
  x = pfrexp(x, e);
 
  // Shift the inputs from the range [0.5,1) to [sqrt(1/2),sqrt(2))
  // and shift by -1. The values are then centered around 0, which improves
  // the stability of the polynomial evaluation.
  //   if( x < SQRTHF ) {
  //     e -= 1;
  //     x = x + x - 1.0;
  //   } else { x = x - 1.0; }
  Packet mask = pcmp_lt(x, cst_cephes_SQRTHF);
  Packet tmp = pand(x, mask);
  x = psub(x, cst_1);
  e = psub(e, pand(cst_1, mask));
  x = padd(x, tmp);
 
  Packet x2 = pmul(x, x);
  Packet x3 = pmul(x2, x);
 
  // Evaluate the polynomial approximant , probably to improve instruction-level parallelism.
  // y = x - 0.5*x^2 + x^3 * polevl( x, P, 5 ) / p1evl( x, Q, 5 ) );
  Packet y, y1, y_;
  y = pmadd(cst_cephes_log_p0, x, cst_cephes_log_p1);
  y1 = pmadd(cst_cephes_log_p3, x, cst_cephes_log_p4);
  y = pmadd(y, x, cst_cephes_log_p2);
  y1 = pmadd(y1, x, cst_cephes_log_p5);
  y_ = pmadd(y, x3, y1);
 
  y = pmadd(cst_cephes_log_q0, x, cst_cephes_log_q1);
  y1 = pmadd(cst_cephes_log_q3, x, cst_cephes_log_q4);
  y = pmadd(y, x, cst_cephes_log_q2);
  y1 = pmadd(y1, x, cst_cephes_log_q5);
  y = pmadd(y, x3, y1);
 
  y_ = pmul(y_, x3);
  y = pdiv(y_, y);
 
  y = pmadd(cst_neg_half, x2, y);
  x = padd(x, y);
 
  // Add the logarithm of the exponent back to the result of the interpolation.
  if (base2) {
    const Packet cst_log2e = pset1<Packet>(static_cast<double>(EIGEN_LOG2E));
    x = pmadd(x, cst_log2e, e);
  } else {
    const Packet cst_ln2 = pset1<Packet>(static_cast<double>(EIGEN_LN2));
    x = pmadd(e, cst_ln2, x);
  }
 
  Packet invalid_mask = pcmp_lt_or_nan(_x, pzero(_x));
  Packet iszero_mask = pcmp_eq(_x, pzero(_x));
  Packet pos_inf_mask = pcmp_eq(_x, cst_pos_inf);
  // Filter out invalid inputs, i.e.:
  //  - negative arg will be NAN
  //  - 0 will be -INF
  //  - +INF will be +INF
  return pselect(iszero_mask, cst_minus_inf, por(pselect(pos_inf_mask, cst_pos_inf, x), invalid_mask));
}

References e(), EIGEN_LN2, EIGEN_LOG2E, padd(), pand(), pcmp_eq(), pcmp_lt(), pcmp_lt_or_nan(), pdiv(), pfrexp(), pmadd(), pmul(), por(), pselect(), psub(), pzero(), tmp, plotDoE::x, Global_parameters::x2(), and y.

Referenced by plog2_double(), and plog_double().

plog_impl_float()

template<typename Packet , bool base2>

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::plog_impl_float

(

const Packet

_x

)

                                                                                            {
  const Packet cst_1 = pset1<Packet>(1.0f);
  const Packet cst_minus_inf = pset1frombits<Packet>(static_cast<Eigen::numext::uint32_t>(0xff800000u));
  const Packet cst_pos_inf = pset1frombits<Packet>(static_cast<Eigen::numext::uint32_t>(0x7f800000u));
 
  const Packet cst_cephes_SQRTHF = pset1<Packet>(0.707106781186547524f);
  Packet e, x;
  // extract significant in the range [0.5,1) and exponent
  x = pfrexp(_x, e);
 
  // part2: Shift the inputs from the range [0.5,1) to [sqrt(1/2),sqrt(2))
  // and shift by -1. The values are then centered around 0, which improves
  // the stability of the polynomial evaluation.
  //   if( x < SQRTHF ) {
  //     e -= 1;
  //     x = x + x - 1.0;
  //   } else { x = x - 1.0; }
  Packet mask = pcmp_lt(x, cst_cephes_SQRTHF);
  Packet tmp = pand(x, mask);
  x = psub(x, cst_1);
  e = psub(e, pand(cst_1, mask));
  x = padd(x, tmp);
 
  // Polynomial coefficients for rational r(x) = p(x)/q(x)
  // approximating log(1+x) on [sqrt(0.5)-1;sqrt(2)-1].
  constexpr float alpha[] = {0.18256296349849254f, 1.0000000190281063f, 1.0000000190281136f};
  constexpr float beta[] = {0.049616247954120038f, 0.59923249590823520f, 1.4999999999999927f, 1.0f};
 
  Packet p = ppolevl<Packet, 2>::run(x, alpha);
  p = pmul(x, p);
  Packet q = ppolevl<Packet, 3>::run(x, beta);
  x = pdiv(p, q);
 
  // Add the logarithm of the exponent back to the result of the interpolation.
  if (base2) {
    const Packet cst_log2e = pset1<Packet>(static_cast<float>(EIGEN_LOG2E));
    x = pmadd(x, cst_log2e, e);
  } else {
    const Packet cst_ln2 = pset1<Packet>(static_cast<float>(EIGEN_LN2));
    x = pmadd(e, cst_ln2, x);
  }
 
  Packet invalid_mask = pcmp_lt_or_nan(_x, pzero(_x));
  Packet iszero_mask = pcmp_eq(_x, pzero(_x));
  Packet pos_inf_mask = pcmp_eq(_x, cst_pos_inf);
  // Filter out invalid inputs, i.e.:
  //  - negative arg will be NAN
  //  - 0 will be -INF
  //  - +INF will be +INF
  return pselect(iszero_mask, cst_minus_inf, por(pselect(pos_inf_mask, cst_pos_inf, x), invalid_mask));
}

References alpha, beta, e(), EIGEN_LN2, EIGEN_LOG2E, p, padd(), pand(), pcmp_eq(), pcmp_lt(), pcmp_lt_or_nan(), pdiv(), pfrexp(), pmadd(), pmul(), por(), pselect(), psub(), pzero(), Eigen::numext::q, Eigen::internal::ppolevl< Packet, N >::run(), tmp, and plotDoE::x.

Referenced by plog2_float(), and plog_float().

plogical_shift_left() [1/31]

template<int N>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::plogical_shift_left

(

const Packet16c &

a

)

                                                                      {
  return __lsx_vslli_b((__m128i)a, N);
}

References a, and N.

plogical_shift_left() [2/31]

template<int N>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::plogical_shift_left

(

const Packet16uc &

a

)

                                                                        {
  return __lsx_vslli_b((__m128i)a, N);
}

References a, and N.

plogical_shift_left() [3/31]

template<int N>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::plogical_shift_left

(

const Packet2l &

a

)

                                                                    {
  return __lsx_vslli_d((__m128i)a, N);
}

References a, and N.

plogical_shift_left() [4/31]

template<int N>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::plogical_shift_left

(

const Packet2ul &

a

)

                                                                      {
  return __lsx_vslli_d((__m128i)a, N);
}

References a, and N.

plogical_shift_left() [5/31]

template<int N>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::plogical_shift_left

(

const Packet4f &

a

)

                                                                    {
  const EIGEN_DECLARE_CONST_FAST_Packet4ui(mask, N);
  Packet4ui r = vec_sl(reinterpret_cast<Packet4ui>(a), p4ui_mask);
  return reinterpret_cast<Packet4f>(r);
}

References a, EIGEN_DECLARE_CONST_FAST_Packet4ui(), N, and UniformPSDSelfTest::r.

plogical_shift_left() [6/31]

template<int N>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::plogical_shift_left

(

const Packet4i &

a

)

                                                                    {
  return vec_sl(a, reinterpret_cast<Packet4ui>(pset1<Packet4i>(N)));
}

References a, N, and pset1< Packet4i >().

plogical_shift_left() [7/31]

template<int N>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::plogical_shift_left

(

const Packet4ui &

a

)

                                                                      {
  const EIGEN_DECLARE_CONST_FAST_Packet4ui(mask, N);
  return vec_sl(a, p4ui_mask);
}

References a, EIGEN_DECLARE_CONST_FAST_Packet4ui(), and N.

plogical_shift_left() [8/31]

template<int N>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::plogical_shift_left

(

const Packet8s &

a

)

                                                                    {
  return __lsx_vslli_h((__m128i)a, N);
}

References a, and N.

plogical_shift_left() [9/31]

template<int N>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::plogical_shift_left

(

const Packet8us &

a

)

                                                                      {
  const EIGEN_DECLARE_CONST_FAST_Packet8us(mask, N);
  return vec_sl(a, p8us_mask);
}

References a, EIGEN_DECLARE_CONST_FAST_Packet8us(), and N.

plogical_shift_left() [10/31]

template<int N, typename T >

EIGEN_DEVICE_FUNC T Eigen::internal::plogical_shift_left ( const T &  a )

inline

Returns

a shifted by N bits to the left

                                                           {
  return numext::logical_shift_left(a, N);
}

References a, Eigen::numext::logical_shift_left(), and N.

plogical_shift_left() [11/31]

template<int N>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::plogical_shift_left

(

Packet16c

a

)

                                                               {
  return vshlq_n_s8(a, N);
}

References a, and N.

plogical_shift_left() [12/31]

template<int N>

EIGEN_STRONG_INLINE Packet16i Eigen::internal::plogical_shift_left

(

Packet16i

a

)

                                                               {
  return _mm512_slli_epi32(a, N);
}

References a, and N.

plogical_shift_left() [13/31]

template<int N>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::plogical_shift_left

(

Packet16uc

a

)

                                                                 {
  return vshlq_n_u8(a, N);
}

References a, and N.

plogical_shift_left() [14/31]

template<int N>

EIGEN_STRONG_INLINE Packet2i Eigen::internal::plogical_shift_left

(

Packet2i

a

)

                                                             {
  return vshl_n_s32(a, N);
}

References a, and N.

plogical_shift_left() [15/31]

template<int N>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::plogical_shift_left

(

Packet2l

a

)

                                                             {
  return vshlq_n_s64(a, N);
}

References a, and N.

plogical_shift_left() [16/31]

template<int N>

EIGEN_STRONG_INLINE Packet2ui Eigen::internal::plogical_shift_left

(

Packet2ui

a

)

                                                               {
  return vshl_n_u32(a, N);
}

References a, and N.

plogical_shift_left() [17/31]

template<int N>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::plogical_shift_left

(

Packet2ul

a

)

                                                               {
  return vshlq_n_u64(a, N);
}

References a, and N.

plogical_shift_left() [18/31]

template<int N>

EIGEN_STRONG_INLINE Packet4c Eigen::internal::plogical_shift_left

(

Packet4c &

a

)

                                                              {
  return vget_lane_s32(vreinterpret_s32_s8(vshl_n_s8(vreinterpret_s8_s32(vdup_n_s32(a)), N)), 0);
}

References a, and N.

plogical_shift_left() [19/31]

template<int N>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::plogical_shift_left

(

Packet4i

a

)

                                                             {
  return vshlq_n_s32(a, N);
}

References a, and N.

plogical_shift_left() [20/31]

template<int N>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::plogical_shift_left

(

Packet4s

a

)

                                                             {
  return vshl_n_s16(a, N);
}

References a, and N.

plogical_shift_left() [21/31]

template<int N>

EIGEN_STRONG_INLINE Packet4uc Eigen::internal::plogical_shift_left

(

Packet4uc &

a

)

                                                                {
  return vget_lane_u32(vreinterpret_u32_u8(vshl_n_u8(vreinterpret_u8_u32(vdup_n_u32(a)), N)), 0);
}

References a, and N.

plogical_shift_left() [22/31]

template<int N>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::plogical_shift_left

(

Packet4ui

a

)

                                                               {
  return vshlq_n_u32(a, N);
}

References a, and N.

plogical_shift_left() [23/31]

template<int N>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::plogical_shift_left

(

Packet4us

a

)

                                                               {
  return vshl_n_u16(a, N);
}

References a, and N.

plogical_shift_left() [24/31]

template<int N>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::plogical_shift_left

(

Packet8c

a

)

                                                             {
  return vshl_n_s8(a, N);
}

References a, and N.

plogical_shift_left() [25/31]

template<int N>

EIGEN_STRONG_INLINE Packet8i Eigen::internal::plogical_shift_left

(

Packet8i

a

)

                                                             {
#ifdef EIGEN_VECTORIZE_AVX2
  return _mm256_slli_epi32(a, N);
#else
  __m128i lo = _mm_slli_epi32(_mm256_extractf128_si256(a, 0), N);
  __m128i hi = _mm_slli_epi32(_mm256_extractf128_si256(a, 1), N);
  return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1);
#endif
}

References a, and N.

plogical_shift_left() [26/31]

template<int N>

EIGEN_STRONG_INLINE Packet8l Eigen::internal::plogical_shift_left

(

Packet8l

a

)

                                                             {
  return _mm512_slli_epi64(a, N);
}

References a, and N.

plogical_shift_left() [27/31]

template<int N>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::plogical_shift_left

(

Packet8s

a

)

                                                             {
  return vshlq_n_s16(a, N);
}

References a, and N.

plogical_shift_left() [28/31]

template<int N>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::plogical_shift_left

(

Packet8uc

a

)

                                                               {
  return vshl_n_u8(a, N);
}

References a, and N.

plogical_shift_left() [29/31]

template<int N>

EIGEN_STRONG_INLINE Packet8ui Eigen::internal::plogical_shift_left

(

Packet8ui

a

)

                                                               {
  return (Packet8ui)plogical_shift_left<N>((Packet8i)a);
}

References a.

plogical_shift_left() [30/31]

template<int N>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::plogical_shift_left

(

Packet8us

a

)

                                                               {
  return vshlq_n_u16(a, N);
}

References a, and N.

plogical_shift_left() [31/31]

template<int N>

EIGEN_STRONG_INLINE PacketXi Eigen::internal::plogical_shift_left

(

PacketXi

a

)

                                                             {
  return svlsl_n_s32_x(svptrue_b32(), a, N);
}

References a, and N.

plogical_shift_right() [1/31]

template<int N>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::plogical_shift_right

(

const Packet16c &

a

)

                                                                       {
  return __lsx_vsrli_b((__m128i)a, N);
}

References a, and N.

plogical_shift_right() [2/31]

template<int N>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::plogical_shift_right

(

const Packet16uc &

a

)

                                                                         {
  return __lsx_vsrli_b((__m128i)a, N);
}

References a, and N.

plogical_shift_right() [3/31]

template<int N>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::plogical_shift_right

(

const Packet2l &

a

)

                                                                     {
  return __lsx_vsrli_d((__m128i)a, N);
}

References a, and N.

plogical_shift_right() [4/31]

template<int N>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::plogical_shift_right

(

const Packet2ul &

a

)

                                                                       {
  return __lsx_vsrli_d((__m128i)a, N);
}

References a, and N.

plogical_shift_right() [5/31]

template<int N>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::plogical_shift_right

(

const Packet4f &

a

)

                                                                     {
  const EIGEN_DECLARE_CONST_FAST_Packet4ui(mask, N);
  Packet4ui r = vec_sr(reinterpret_cast<Packet4ui>(a), p4ui_mask);
  return reinterpret_cast<Packet4f>(r);
}

References a, EIGEN_DECLARE_CONST_FAST_Packet4ui(), N, and UniformPSDSelfTest::r.

plogical_shift_right() [6/31]

template<int N>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::plogical_shift_right

(

const Packet4i &

a

)

                                                                     {
  return vec_sr(a, reinterpret_cast<Packet4ui>(pset1<Packet4i>(N)));
}

References a, N, and pset1< Packet4i >().

plogical_shift_right() [7/31]

template<int N>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::plogical_shift_right

(

const Packet4ui &

a

)

                                                                       {
  const EIGEN_DECLARE_CONST_FAST_Packet4ui(mask, N);
  return vec_sr(a, p4ui_mask);
}

References a, EIGEN_DECLARE_CONST_FAST_Packet4ui(), and N.

plogical_shift_right() [8/31]

template<int N>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::plogical_shift_right

(

const Packet8s &

a

)

                                                                     {
  return __lsx_vsrli_h((__m128i)a, N);
}

References a, and N.

plogical_shift_right() [9/31]

template<int N>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::plogical_shift_right

(

const Packet8us &

a

)

                                                                       {
  const EIGEN_DECLARE_CONST_FAST_Packet8us(mask, N);
  return vec_sr(a, p8us_mask);
}

References a, EIGEN_DECLARE_CONST_FAST_Packet8us(), and N.

plogical_shift_right() [10/31]

template<int N, typename T >

EIGEN_DEVICE_FUNC T Eigen::internal::plogical_shift_right ( const T &  a )

inline

Returns

a logically shifted by N bits to the right

                                                            {
  return numext::logical_shift_right(a, N);
}

References a, Eigen::numext::logical_shift_right(), and N.

plogical_shift_right() [11/31]

template<int N>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::plogical_shift_right

(

Packet16c

a

)

                                                                {
  return vreinterpretq_s8_u8(vshrq_n_u8(vreinterpretq_u8_s8(a), N));
}

References a, and N.

plogical_shift_right() [12/31]

template<int N>

EIGEN_STRONG_INLINE Packet16i Eigen::internal::plogical_shift_right

(

Packet16i

a

)

                                                                {
  return _mm512_srli_epi32(a, N);
}

References a, and N.

plogical_shift_right() [13/31]

template<int N>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::plogical_shift_right

(

Packet16uc

a

)

                                                                  {
  return vshrq_n_u8(a, N);
}

References a, and N.

plogical_shift_right() [14/31]

template<int N>

EIGEN_STRONG_INLINE Packet2i Eigen::internal::plogical_shift_right

(

Packet2i

a

)

                                                              {
  return vreinterpret_s32_u32(vshr_n_u32(vreinterpret_u32_s32(a), N));
}

References a, and N.

plogical_shift_right() [15/31]

template<int N>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::plogical_shift_right

(

Packet2l

a

)

                                                              {
  return vreinterpretq_s64_u64(vshrq_n_u64(vreinterpretq_u64_s64(a), N));
}

References a, and N.

plogical_shift_right() [16/31]

template<int N>

EIGEN_STRONG_INLINE Packet2ui Eigen::internal::plogical_shift_right

(

Packet2ui

a

)

                                                                {
  return vshr_n_u32(a, N);
}

References a, and N.

plogical_shift_right() [17/31]

template<int N>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::plogical_shift_right

(

Packet2ul

a

)

                                                                {
  return vshrq_n_u64(a, N);
}

References a, and N.

plogical_shift_right() [18/31]

template<int N>

EIGEN_STRONG_INLINE Packet4c Eigen::internal::plogical_shift_right

(

Packet4c &

a

)

                                                               {
  return vget_lane_s32(vreinterpret_s32_u8(vshr_n_u8(vreinterpret_u8_s32(vdup_n_s32(a)), N)), 0);
}

References a, and N.

plogical_shift_right() [19/31]

template<int N>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::plogical_shift_right

(

Packet4i

a

)

                                                              {
  return vreinterpretq_s32_u32(vshrq_n_u32(vreinterpretq_u32_s32(a), N));
}

References a, and N.

plogical_shift_right() [20/31]

template<int N>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::plogical_shift_right

(

Packet4s

a

)

                                                              {
  return vreinterpret_s16_u16(vshr_n_u16(vreinterpret_u16_s16(a), N));
}

References a, and N.

plogical_shift_right() [21/31]

template<int N>

EIGEN_STRONG_INLINE Packet4uc Eigen::internal::plogical_shift_right

(

Packet4uc &

a

)

                                                                 {
  return vget_lane_u32(vreinterpret_u32_s8(vshr_n_s8(vreinterpret_s8_u32(vdup_n_u32(a)), N)), 0);
}

References a, and N.

plogical_shift_right() [22/31]

template<int N>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::plogical_shift_right

(

Packet4ui

a

)

                                                                {
  return vshrq_n_u32(a, N);
}

References a, and N.

plogical_shift_right() [23/31]

template<int N>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::plogical_shift_right

(

Packet4us

a

)

                                                                {
  return vshr_n_u16(a, N);
}

References a, and N.

plogical_shift_right() [24/31]

template<int N>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::plogical_shift_right

(

Packet8c

a

)

                                                              {
  return vreinterpret_s8_u8(vshr_n_u8(vreinterpret_u8_s8(a), N));
}

References a, and N.

plogical_shift_right() [25/31]

template<int N>

EIGEN_STRONG_INLINE Packet8i Eigen::internal::plogical_shift_right

(

Packet8i

a

)

                                                              {
#ifdef EIGEN_VECTORIZE_AVX2
  return _mm256_srli_epi32(a, N);
#else
  __m128i lo = _mm_srli_epi32(_mm256_extractf128_si256(a, 0), N);
  __m128i hi = _mm_srli_epi32(_mm256_extractf128_si256(a, 1), N);
  return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1);
#endif
}

References a, and N.

plogical_shift_right() [26/31]

template<int N>

EIGEN_STRONG_INLINE Packet8l Eigen::internal::plogical_shift_right

(

Packet8l

a

)

                                                              {
  return _mm512_srli_epi64(a, N);
}

References a, and N.

plogical_shift_right() [27/31]

template<int N>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::plogical_shift_right

(

Packet8s

a

)

                                                              {
  return vreinterpretq_s16_u16(vshrq_n_u16(vreinterpretq_u16_s16(a), N));
}

References a, and N.

plogical_shift_right() [28/31]

template<int N>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::plogical_shift_right

(

Packet8uc

a

)

                                                                {
  return vshr_n_u8(a, N);
}

References a, and N.

plogical_shift_right() [29/31]

template<int N>

EIGEN_STRONG_INLINE Packet8ui Eigen::internal::plogical_shift_right

(

Packet8ui

a

)

                                                                {
  return (Packet8ui)plogical_shift_right<N>((Packet8i)a);
}

References a.

plogical_shift_right() [30/31]

template<int N>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::plogical_shift_right

(

Packet8us

a

)

                                                                {
  return vshrq_n_u16(a, N);
}

References a, and N.

plogical_shift_right() [31/31]

template<int N>

EIGEN_STRONG_INLINE PacketXi Eigen::internal::plogical_shift_right

(

PacketXi

a

)

                                                              {
  return svreinterpret_s32_u32(svlsr_n_u32_x(svptrue_b32(), svreinterpret_u32_s32(a), N));
}

References a, and N.

plset()

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet Eigen::internal::plset

(

const typename unpacket_traits< Packet >::type &

a

)

Returns a packet with coefficients (a,a+1,...,a+packet_size-1).

                                                                                                  {
  return a;
}

References a.

plset< Packet16bf >()

template<>

EIGEN_STRONG_INLINE Packet16bf Eigen::internal::plset< Packet16bf >

(

const bfloat16 &

a

)

                                                                    {
  return F32ToBf16(plset<Packet16f>(static_cast<float>(a)));
}

References a, F32ToBf16(), and plset< Packet16f >().

plset< Packet16c >() [1/2]

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::plset< Packet16c >

(

const int8_t &

a

)

                                                                {
  const int8_t countdown[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
  return __lsx_vadd_b(pset1<Packet16c>(a), __lsx_vld(countdown, 0));
}

References a, and pset1< Packet16c >().

plset< Packet16c >() [2/2]

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::plset< Packet16c >

(

const signed char &

a

)

                                                                     {
  return pset1<Packet16c>(a) + p16c_COUNTDOWN;
}

References a, p16c_COUNTDOWN, and pset1< Packet16c >().

plset< Packet16f >()

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::plset< Packet16f >

(

const float &

a

)

                                                               {
  return _mm512_add_ps(_mm512_set1_ps(a), _mm512_set_ps(15.0f, 14.0f, 13.0f, 12.0f, 11.0f, 10.0f, 9.0f, 8.0f, 7.0f,
                                                        6.0f, 5.0f, 4.0f, 3.0f, 2.0f, 1.0f, 0.0f));
}

References a.

Referenced by plset< Packet16bf >(), and plset< Packet16h >().

plset< Packet16h >()

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::plset< Packet16h >

(

const half &

a

)

                                                              {
  return float2half(plset<Packet16f>(static_cast<float>(a)));
}

References a, float2half(), and plset< Packet16f >().

plset< Packet16i >()

template<>

EIGEN_STRONG_INLINE Packet16i Eigen::internal::plset< Packet16i >

(

const int &

a

)

                                                             {
  return _mm512_add_epi32(_mm512_set1_epi32(a), _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0));
}

References a.

plset< Packet16uc >() [1/2]

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::plset< Packet16uc >

(

const uint8_t &

a

)

                                                                   {
  const uint8_t countdown[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
  return __lsx_vadd_b(pset1<Packet16uc>(a), __lsx_vld(countdown, 0));
}

References a, and pset1< Packet16uc >().

plset< Packet16uc >() [2/2]

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::plset< Packet16uc >

(

const unsigned char &

a

)

                                                                         {
  return pset1<Packet16uc>(a) + p16uc_COUNTDOWN;
}

References a, p16uc_COUNTDOWN, and pset1< Packet16uc >().

plset< Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::plset< Packet2d >

(

const double &

a

)

                                                              {
  static const Packet2d countdown = {0.0f, 1.0f};
  return __lsx_vfadd_d(pset1<Packet2d>(a), countdown);
}

References a, and pset1< Packet2d >().

plset< Packet2f >()

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::plset< Packet2f >

(

const float &

a

)

                                                             {
  const float c[] = {0.0f, 1.0f};
  return vadd_f32(pset1<Packet2f>(a), vld1_f32(c));
}

References a, calibrate::c, and pset1< Packet2f >().

plset< Packet2i >()

template<>

EIGEN_STRONG_INLINE Packet2i Eigen::internal::plset< Packet2i >

(

const int32_t &

a

)

                                                               {
  const int32_t c[] = {0, 1};
  return vadd_s32(pset1<Packet2i>(a), vld1_s32(c));
}

References a, calibrate::c, and pset1< Packet2i >().

plset< Packet2l >()

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::plset< Packet2l >

(

const int64_t &

a

)

                                                               {
  const int64_t countdown[] = {0, 1};
  return __lsx_vadd_d(pset1<Packet2l>(a), __lsx_vld(countdown, 0));
}

References a, and pset1< Packet2l >().

plset< Packet2ui >()

template<>

EIGEN_STRONG_INLINE Packet2ui Eigen::internal::plset< Packet2ui >

(

const uint32_t &

a

)

                                                                  {
  const uint32_t c[] = {0, 1};
  return vadd_u32(pset1<Packet2ui>(a), vld1_u32(c));
}

References a, calibrate::c, and pset1< Packet2ui >().

plset< Packet2ul >()

template<>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::plset< Packet2ul >

(

const uint64_t &

a

)

                                                                  {
  const uint64_t countdown[] = {0, 1};
  return __lsx_vadd_d(pset1<Packet2ul>(a), __lsx_vld(countdown, 0));
}

References a, and pset1< Packet2ul >().

plset< Packet32h >()

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::plset< Packet32h >

(

const half &

a

)

                                                              {
  return _mm512_add_ph(pset1<Packet32h>(a), _mm512_set_ph(31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17,
                                                          16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0));
}

References a, and pset1< Packet32h >().

plset< Packet4bf >()

template<>

EIGEN_STRONG_INLINE Packet4bf Eigen::internal::plset< Packet4bf >

(

const bfloat16 &

a

)

                                                                  {
  return F32ToBf16(plset<Packet4f>(static_cast<float>(a)));
}

References a, F32ToBf16(), and plset< Packet4f >().

plset< Packet4c >()

template<>

EIGEN_STRONG_INLINE Packet4c Eigen::internal::plset< Packet4c >

(

const int8_t &

a

)

                                                              {
  return vget_lane_s32(vreinterpret_s32_s8(vadd_s8(vreinterpret_s8_u32(vdup_n_u32(0x03020100)), vdup_n_s8(a))), 0);
}

References a.

plset< Packet4d >()

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::plset< Packet4d >

(

const double &

a

)

                                                              {
  return padd(pset1<Packet4d>(a), _mm256_set_pd(3.0, 2.0, 1.0, 0.0));
}

References a, padd(), and pset1< Packet4d >().

plset< Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::plset< Packet4f >

(

const float &

a

)

                                                             {
  return pset1<Packet4f>(a) + p4f_COUNTDOWN;
}

References a, p4f_COUNTDOWN, and pset1< Packet4f >().

Referenced by plset< Packet4bf >().

plset< Packet4i >() [1/2]

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::plset< Packet4i >

(

const int &

a

)

                                                           {
  return pset1<Packet4i>(a) + p4i_COUNTDOWN;
}

References a, p4i_COUNTDOWN, and pset1< Packet4i >().

plset< Packet4i >() [2/2]

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::plset< Packet4i >

(

const int32_t &

a

)

                                                               {
  const int32_t countdown[] = {0, 1, 2, 3};
  return __lsx_vadd_w(pset1<Packet4i>(a), __lsx_vld(countdown, 0));
}

References a, and pset1< Packet4i >().

plset< Packet4s >()

template<>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::plset< Packet4s >

(

const int16_t &

a

)

                                                               {
  const int16_t c[] = {0, 1, 2, 3};
  return vadd_s16(pset1<Packet4s>(a), vld1_s16(c));
}

References a, calibrate::c, and pset1< Packet4s >().

plset< Packet4uc >()

template<>

EIGEN_STRONG_INLINE Packet4uc Eigen::internal::plset< Packet4uc >

(

const uint8_t &

a

)

                                                                 {
  return vget_lane_u32(vreinterpret_u32_u8(vadd_u8(vreinterpret_u8_u32(vdup_n_u32(0x03020100)), vdup_n_u8(a))), 0);
}

References a.

plset< Packet4ui >()

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::plset< Packet4ui >

(

const uint32_t &

a

)

                                                                  {
  const uint32_t countdown[] = {0, 1, 2, 3};
  return __lsx_vadd_w(pset1<Packet4ui>(a), __lsx_vld(countdown, 0));
}

References a, and pset1< Packet4ui >().

plset< Packet4us >()

template<>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::plset< Packet4us >

(

const uint16_t &

a

)

                                                                  {
  const uint16_t c[] = {0, 1, 2, 3};
  return vadd_u16(pset1<Packet4us>(a), vld1_u16(c));
}

References a, calibrate::c, and pset1< Packet4us >().

plset< Packet8bf >()

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::plset< Packet8bf >

(

const bfloat16 &

a

)

                                                                  {
  bfloat16 countdown[8] = {bfloat16(0), bfloat16(1), bfloat16(2), bfloat16(3),
                           bfloat16(4), bfloat16(5), bfloat16(6), bfloat16(7)};
  return padd<Packet8bf>(pset1<Packet8bf>(a), pload<Packet8bf>(countdown));
}

References a, padd< Packet8bf >(), pload< Packet8bf >(), and pset1< Packet8bf >().

plset< Packet8c >()

template<>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::plset< Packet8c >

(

const int8_t &

a

)

                                                              {
  const int8_t c[] = {0, 1, 2, 3, 4, 5, 6, 7};
  return vadd_s8(pset1<Packet8c>(a), vld1_s8(c));
}

References a, calibrate::c, and pset1< Packet8c >().

plset< Packet8d >()

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::plset< Packet8d >

(

const double &

a

)

                                                              {
  return _mm512_add_pd(_mm512_set1_pd(a), _mm512_set_pd(7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, 0.0));
}

References a.

plset< Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::plset< Packet8f >

(

const float &

a

)

                                                             {
  return padd(pset1<Packet8f>(a), _mm256_set_ps(7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, 0.0));
}

References a, padd(), and pset1< Packet8f >().

Referenced by plset< Packet8h >().

plset< Packet8h >()

template<>

EIGEN_STRONG_INLINE Packet8h Eigen::internal::plset< Packet8h >

(

const half &

a

)

                                                            {
  return float2half(plset<Packet8f>(static_cast<float>(a)));
}

References a, float2half(), and plset< Packet8f >().

plset< Packet8i >()

template<>

EIGEN_STRONG_INLINE Packet8i Eigen::internal::plset< Packet8i >

(

const int &

a

)

                                                           {
  return padd(pset1<Packet8i>(a), (Packet8i)_mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0));
}

References a, padd(), and pset1< Packet8i >().

plset< Packet8l >()

template<>

EIGEN_STRONG_INLINE Packet8l Eigen::internal::plset< Packet8l >

(

const int64_t &

a

)

                                                               {
  return _mm512_add_epi64(_mm512_set1_epi64(a), _mm512_set_epi64(7, 6, 5, 4, 3, 2, 1, 0));
}

References a.

plset< Packet8s >() [1/2]

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::plset< Packet8s >

(

const int16_t &

a

)

                                                               {
  const int16_t countdown[] = {0, 1, 2, 3, 4, 5, 6, 7};
  return __lsx_vadd_h(pset1<Packet8s>(a), __lsx_vld(countdown, 0));
}

References a, and pset1< Packet8s >().

plset< Packet8s >() [2/2]

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::plset< Packet8s >

(

const short int &

a

)

                                                                 {
  return pset1<Packet8s>(a) + p8s_COUNTDOWN;
}

References a, p8s_COUNTDOWN, and pset1< Packet8s >().

plset< Packet8uc >()

template<>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::plset< Packet8uc >

(

const uint8_t &

a

)

                                                                 {
  const uint8_t c[] = {0, 1, 2, 3, 4, 5, 6, 7};
  return vadd_u8(pset1<Packet8uc>(a), vld1_u8(c));
}

References a, calibrate::c, and pset1< Packet8uc >().

plset< Packet8ui >()

template<>

EIGEN_STRONG_INLINE Packet8ui Eigen::internal::plset< Packet8ui >

(

const uint32_t &

a

)

                                                                  {
  return padd(pset1<Packet8ui>(a), (Packet8ui)_mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0));
}

References a, padd(), and pset1< Packet8ui >().

plset< Packet8us >() [1/2]

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::plset< Packet8us >

(

const uint16_t &

a

)

                                                                  {
  const uint16_t countdown[] = {0, 1, 2, 3, 4, 5, 6, 7};
  return __lsx_vadd_h(pset1<Packet8us>(a), __lsx_vld(countdown, 0));
}

References a, and pset1< Packet8us >().

plset< Packet8us >() [2/2]

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::plset< Packet8us >

(

const unsigned short int &

a

)

                                                                            {
  return pset1<Packet8us>(a) + p8us_COUNTDOWN;
}

References a, p8us_COUNTDOWN, and pset1< Packet8us >().

plset< PacketXf >()

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::plset< PacketXf >

(

const float &

a

)

                                                             {
  float c[packet_traits<float>::size];
  for (int i = 0; i < packet_traits<float>::size; i++) c[i] = i;
  return svadd_f32_x(svptrue_b32(), pset1<PacketXf>(a), svld1_f32(svptrue_b32(), c));
}

References a, calibrate::c, i, pset1< PacketXf >(), and size.

plset< PacketXi >()

template<>

EIGEN_STRONG_INLINE PacketXi Eigen::internal::plset< PacketXi >

(

const numext::int32_t &

a

)

                                                                     {
  numext::int32_t c[packet_traits<numext::int32_t>::size];
  for (int i = 0; i < packet_traits<numext::int32_t>::size; i++) c[i] = i;
  return svadd_s32_x(svptrue_b32(), pset1<PacketXi>(a), svld1_s32(svptrue_b32(), c));
}

References a, calibrate::c, i, pset1< PacketXi >(), and size.

pmadd() [1/23]

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::pmadd ( const Packet &  a, const Packet &  b, const Packet &  c  )

inline

Returns

a * b + c (coeff-wise)

                                                                                         {
  return padd(pmul(a, b), c);
}

References a, b, calibrate::c, padd(), and pmul().

pmadd() [2/23]

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::pmadd	(	const Packet16c &	a,
		const Packet16c &	b,
		const Packet16c &	c
	)

                                                                                                {
  return __lsx_vmadd_b(c, a, b);
}

References a, b, and calibrate::c.

pmadd() [3/23]

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::pmadd	(	const Packet16uc &	a,
		const Packet16uc &	b,
		const Packet16uc &	c
	)

                                                                                                    {
  return __lsx_vmadd_b(c, a, b);
}

References a, b, and calibrate::c.

pmadd() [4/23]

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pmadd	(	const Packet2d &	a,
		const Packet2d &	b,
		const Packet2d &	c
	)

                                                                                            {
  return __lsx_vfmadd_d(a, b, c);
}

References a, b, and calibrate::c.

pmadd() [5/23]

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::pmadd	(	const Packet2f &	a,
		const Packet2f &	b,
		const Packet2f &	c
	)

                                                                                            {
  return vmla_f32(c, a, b);
}

References a, b, and calibrate::c.

pmadd() [6/23]

template<>

EIGEN_STRONG_INLINE Packet2i Eigen::internal::pmadd	(	const Packet2i &	a,
		const Packet2i &	b,
		const Packet2i &	c
	)

                                                                                            {
  return vmla_s32(c, a, b);
}

References a, b, and calibrate::c.

pmadd() [7/23]

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::pmadd	(	const Packet2l &	a,
		const Packet2l &	b,
		const Packet2l &	c
	)

                                                                                            {
  return __lsx_vmadd_d(c, a, b);
}

References a, b, and calibrate::c.

pmadd() [8/23]

template<>

EIGEN_STRONG_INLINE Packet2ui Eigen::internal::pmadd	(	const Packet2ui &	a,
		const Packet2ui &	b,
		const Packet2ui &	c
	)

                                                                                                {
  return vmla_u32(c, a, b);
}

References a, b, and calibrate::c.

pmadd() [9/23]

template<>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::pmadd	(	const Packet2ul &	a,
		const Packet2ul &	b,
		const Packet2ul &	c
	)

                                                                                                {
  return __lsx_vmadd_d(c, a, b);
}

References a, b, and calibrate::c.

pmadd() [10/23]

template<>

EIGEN_STRONG_INLINE Packet4c Eigen::internal::pmadd	(	const Packet4c &	a,
		const Packet4c &	b,
		const Packet4c &	c
	)

                                                                                            {
  return vget_lane_s32(
      vreinterpret_s32_s8(vmla_s8(vreinterpret_s8_s32(vdup_n_s32(c)), vreinterpret_s8_s32(vdup_n_s32(a)),
                                  vreinterpret_s8_s32(vdup_n_s32(b)))),
      0);
}

References a, b, and calibrate::c.

pmadd() [11/23]

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pmadd	(	const Packet4f &	a,
		const Packet4f &	b,
		const Packet4f &	c
	)

                                                                                            {
  return vec_madd(a, b, c);
}

References a, b, and calibrate::c.

Referenced by Eigen::internal::gebp_traits< LhsScalar_, RhsScalar_, ConjLhs_, ConjRhs_, Arch, PacketSize_ >::acc(), Eigen::internal::gebp_traits< std::complex< RealScalar >, std::complex< RealScalar >, ConjLhs_, ConjRhs_, Arch, PacketSize_ >::acc(), Eigen::internal::scalar_inner_product_op< Scalar, Scalar, Conj >::coeff(), Eigen::TensorSycl::internal::TensorContractionKernel< OutScalar, LhsScalar, RhsScalar, OutAccessor, LhsMapper, RhsMapper, StorageIndex, Properties, TripleDim, Vectorizable, input_mapper_properties, IsFinal, contraction_tp >::compute_block_per_tile(), Eigen::TensorSycl::internal::GeneralVectorTensor< OutScalar, OutAccessor, VectorMapper, TensorMapper, StorageIndex, Properties, KFactor, Vectorizable, is_lhs_vec, IsFinal >::compute_panel(), Eigen::internal::determinant_impl< Derived, 4 >::det3(), erfc_double_large(), generic_ndtri_gt_exp_neg_two(), KLoop(), loadAndMultiplyF32(), Eigen::internal::gebp_traits< std::complex< RealScalar >, std::complex< RealScalar >, ConjLhs_, ConjRhs_, Arch, PacketSize_ >::madd(), Eigen::internal::gebp_traits< std::complex< RealScalar >, RealScalar, ConjLhs_, false, Arch, PacketSize_ >::madd_impl(), Eigen::internal::gebp_traits< RealScalar, std::complex< RealScalar >, false, ConjRhs_, Arch, PacketSize_ >::madd_impl(), multVecVSX(), Eigen::TensorSycl::internal::GeneralScalarContraction< OutScalar, LhsScalar, RhsScalar, OutAccessor, LhsMapper, RhsMapper, StorageIndex, Vectorizable >::operator()(), Eigen::internal::accurate_log2< double >::operator()(), outputVecCol(), outputVecResults(), Eigen::internal::scalar_inner_product_op< Scalar, Scalar, Conj >::packet(), packetmath(), Eigen::internal::scalar_logistic_op< float >::packetOp(), Eigen::internal::equalspaced_op< Scalar >::packetOp(), pacos_float(), patanh_double(), patanh_float(), pexp< Packet2d >(), pexp< Packet4f >(), pexp_double(), pexp_float(), phypot_complex(), plog< Packet4f >(), plog_impl_double(), plog_impl_float(), Eigen::internal::conj_helper< Packet, Packet, ConjLhs, ConjRhs >::pmadd(), Eigen::internal::conj_helper< Packet, Packet, true, true >::pmadd(), pmadd_complex(), pnmsub(), psincos_double(), psincos_float(), psincos_inner_msa_float(), ptanh< Packet4f >(), ptanh_double(), ptanh_float(), Eigen::internal::determinant_impl< Derived, 4 >::run(), Eigen::internal::generic_rsqrt_newton_step< Packet, Steps >::run(), Eigen::internal::generic_sqrt_newton_step< Packet, Steps >::run(), Eigen::internal::ppolevl< Packet, N >::run(), Eigen::internal::generic_i0e< T, float >::run(), Eigen::internal::generic_i0e< T, double >::run(), Eigen::internal::generic_i1e< T, float >::run(), Eigen::internal::generic_i1e< T, double >::run(), Eigen::internal::generic_k0e< T, float >::run(), Eigen::internal::generic_k0e< T, double >::run(), Eigen::internal::generic_k0< T, float >::run(), Eigen::internal::generic_k0< T, double >::run(), Eigen::internal::generic_k1e< T, float >::run(), Eigen::internal::generic_k1e< T, double >::run(), Eigen::internal::generic_k1< T, float >::run(), Eigen::internal::generic_k1< T, double >::run(), Eigen::internal::generic_j0< T, float >::run(), Eigen::internal::generic_j0< T, double >::run(), Eigen::internal::generic_y0< T, float >::run(), Eigen::internal::generic_y0< T, double >::run(), Eigen::internal::generic_j1< T, float >::run(), Eigen::internal::generic_j1< T, double >::run(), Eigen::internal::generic_y1< T, float >::run(), Eigen::internal::generic_y1< T, double >::run(), Eigen::internal::generic_fast_erfc< Scalar >::run(), Eigen::internal::etor_product_packet_impl< RowMajor, UnrollingIndex, Lhs, Rhs, Packet, LoadMode >::run(), Eigen::internal::etor_product_packet_impl< ColMajor, UnrollingIndex, Lhs, Rhs, Packet, LoadMode >::run(), Eigen::internal::etor_product_packet_impl< RowMajor, Dynamic, Lhs, Rhs, Packet, LoadMode >::run(), Eigen::internal::etor_product_packet_impl< ColMajor, Dynamic, Lhs, Rhs, Packet, LoadMode >::run(), Eigen::internal::general_matrix_vector_product< Index, LhsScalar, LhsMapper, ColMajor, ConjugateLhs, RhsScalar, RhsMapper, ConjugateRhs, Version >::run(), Eigen::internal::pchebevl< Packet, N >::run(), storeMaddData(), trig_reduce_medium_double(), trig_reduce_small_double(), twoprod(), twoprod_low(), Eigen::internal::gemm_class< Scalar, is_unit_inc >::vfmadd(), and Eigen::internal::gemm_class< Scalar, is_unit_inc >::vfmaddm().

pmadd() [12/23]

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pmadd	(	const Packet4i &	a,
		const Packet4i &	b,
		const Packet4i &	c
	)

                                                                                            {
  return a * b + c;
}

References a, b, and calibrate::c.

pmadd() [13/23]

template<>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::pmadd	(	const Packet4s &	a,
		const Packet4s &	b,
		const Packet4s &	c
	)

                                                                                            {
  return vmla_s16(c, a, b);
}

References a, b, and calibrate::c.

pmadd() [14/23]

template<>

EIGEN_STRONG_INLINE Packet4uc Eigen::internal::pmadd	(	const Packet4uc &	a,
		const Packet4uc &	b,
		const Packet4uc &	c
	)

                                                                                                {
  return vget_lane_u32(
      vreinterpret_u32_u8(vmla_u8(vreinterpret_u8_u32(vdup_n_u32(c)), vreinterpret_u8_u32(vdup_n_u32(a)),
                                  vreinterpret_u8_u32(vdup_n_u32(b)))),
      0);
}

References a, b, and calibrate::c.

pmadd() [15/23]

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::pmadd	(	const Packet4ui &	a,
		const Packet4ui &	b,
		const Packet4ui &	c
	)

                                                                                                {
  return __lsx_vmadd_w(c, a, b);
}

References a, b, and calibrate::c.

pmadd() [16/23]

template<>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::pmadd	(	const Packet4us &	a,
		const Packet4us &	b,
		const Packet4us &	c
	)

                                                                                                {
  return vmla_u16(c, a, b);
}

References a, b, and calibrate::c.

pmadd() [17/23]

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::pmadd	(	const Packet8bf &	a,
		const Packet8bf &	b,
		const Packet8bf &	c
	)

                                                                                                {
  Packet4f a_even = Bf16ToF32Even(a);
  Packet4f a_odd = Bf16ToF32Odd(a);
  Packet4f b_even = Bf16ToF32Even(b);
  Packet4f b_odd = Bf16ToF32Odd(b);
  Packet4f c_even = Bf16ToF32Even(c);
  Packet4f c_odd = Bf16ToF32Odd(c);
  Packet4f pmadd_even = pmadd<Packet4f>(a_even, b_even, c_even);
  Packet4f pmadd_odd = pmadd<Packet4f>(a_odd, b_odd, c_odd);
  return F32ToBf16(pmadd_even, pmadd_odd);
}

References a, b, Bf16ToF32Even(), Bf16ToF32Odd(), calibrate::c, and F32ToBf16().

pmadd() [18/23]

template<>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::pmadd	(	const Packet8c &	a,
		const Packet8c &	b,
		const Packet8c &	c
	)

                                                                                            {
  return vmla_s8(c, a, b);
}

References a, b, and calibrate::c.

pmadd() [19/23]

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::pmadd	(	const Packet8s &	a,
		const Packet8s &	b,
		const Packet8s &	c
	)

                                                                                            {
  return vec_madd(a, b, c);
}

References a, b, and calibrate::c.

pmadd() [20/23]

template<>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::pmadd	(	const Packet8uc &	a,
		const Packet8uc &	b,
		const Packet8uc &	c
	)

                                                                                                {
  return vmla_u8(c, a, b);
}

References a, b, and calibrate::c.

pmadd() [21/23]

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::pmadd	(	const Packet8us &	a,
		const Packet8us &	b,
		const Packet8us &	c
	)

                                                                                                {
  return vec_madd(a, b, c);
}

References a, b, and calibrate::c.

pmadd() [22/23]

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::pmadd	(	const PacketXf &	a,
		const PacketXf &	b,
		const PacketXf &	c
	)

                                                                                            {
  return svmla_f32_x(svptrue_b32(), c, a, b);
}

References a, b, and calibrate::c.

pmadd() [23/23]

template<>

EIGEN_STRONG_INLINE PacketXi Eigen::internal::pmadd	(	const PacketXi &	a,
		const PacketXi &	b,
		const PacketXi &	c
	)

                                                                                            {
  return svmla_s32_x(svptrue_b32(), c, a, b);
}

References a, b, and calibrate::c.

pmadd< Packet1cd >()

template<>

EIGEN_STRONG_INLINE Packet1cd Eigen::internal::pmadd< Packet1cd >	(	const Packet1cd &	a,
		const Packet1cd &	b,
		const Packet1cd &	c
	)

                                                                                                           {
  Packet1cd result, t0, t1, t2;
  t1 = pzero(t1);
  t0.v = (__m128d)__lsx_vpackev_d((__m128i)a.v, (__m128i)a.v);
  t2.v = __lsx_vfmadd_d(t0.v, b.v, c.v);
  result.v = __lsx_vfadd_d(t2.v, t1.v);
  t1.v = __lsx_vfsub_d(t1.v, a.v);
  t1.v = (__m128d)__lsx_vpackod_d((__m128i)a.v, (__m128i)t1.v);
  t2.v = (__m128d)__lsx_vshuf4i_d((__m128i)t2.v, (__m128i)b.v, 0xb);
  result.v = __lsx_vfmadd_d(t1.v, t2.v, result.v);
  return result;
}

References a, b, calibrate::c, pzero(), and Eigen::internal::Packet1cd::v.

pmadd< Packet2cf >()

template<>

EIGEN_STRONG_INLINE Packet2cf Eigen::internal::pmadd< Packet2cf >	(	const Packet2cf &	a,
		const Packet2cf &	b,
		const Packet2cf &	c
	)

                                                                                                           {
  Packet2cf result, t0, t1, t2;
  t1 = pzero(t1);
  t0.v = (__m128)__lsx_vpackev_w((__m128i)a.v, (__m128i)a.v);
  t2.v = __lsx_vfmadd_s(t0.v, b.v, c.v);
  result.v = __lsx_vfadd_s(t2.v, t1.v);
  t1.v = __lsx_vfsub_s(t1.v, a.v);
  t1.v = (__m128)__lsx_vpackod_w((__m128i)a.v, (__m128i)t1.v);
  t2.v = (__m128)__lsx_vshuf4i_w((__m128i)b.v, 0xb1);
  result.v = __lsx_vfmadd_s(t1.v, t2.v, result.v);
  return result;
}

References a, b, calibrate::c, and pzero().

pmax() [1/2]

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::pmax ( const Packet &  a, const Packet &  b  )

inline

Returns

the max of a and b (coeff-wise) If a or b is NaN, the return value is implementation defined.

                                                                       {
  return numext::maxi(a, b);
}

References a, b, and Eigen::numext::maxi().

Referenced by generic_exp2(), packetmath_notcomplex(), Eigen::internal::scalar_clamp_op< Scalar >::packetOp(), pcmp_lt(), pdiv_complex(), pexp_double(), pexp_float(), pldexp< Packet4d >(), pldexp< Packet8d >(), pldexp_fast(), pldexp_fast< Packet2d >(), pldexp_fast< Packet4d >(), pldexp_generic(), psqrt_complex(), ptanh_double(), ptanh_float(), Eigen::internal::psign_impl< Packet, std::enable_if_t< NumTraits< typename unpacket_traits< Packet >::type >::IsComplex &&unpacket_traits< Packet >::vectorizable > >::run(), Eigen::internal::generic_fast_erf< Scalar >::run(), and Eigen::internal::generic_fast_erfc< Scalar >::run().

pmax() [2/2]

template<int NaNPropagation, typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::pmax ( const Packet &  a, const Packet &  b  )

inline

Returns

the max of a and b (coeff-wise). NaNPropagation determines the NaN propagation semantics.

                                                                       {
  return pminmax_impl<NaNPropagation>::run(a, b, EIGEN_BINARY_OP_NAN_PROPAGATION(Packet, (pmax<Packet>)));
}

References a, b, EIGEN_BINARY_OP_NAN_PROPAGATION, and Eigen::internal::pminmax_impl< NaNPropagation >::run().

pmax< Packet16bf >()

template<>

EIGEN_STRONG_INLINE Packet16bf Eigen::internal::pmax< Packet16bf >	(	const Packet16bf &	a,
		const Packet16bf &	b
	)

                                                                                          {
  return F32ToBf16(pmax<Packet16f>(Bf16ToF32(a), Bf16ToF32(b)));
}

References a, b, Bf16ToF32(), F32ToBf16(), and pmax< Packet16f >().

pmax< Packet16c >()

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::pmax< Packet16c >	(	const Packet16c &	a,
		const Packet16c &	b
	)

                                                                                      {
  return vec_max(a, b);
}

References a, and b.

pmax< Packet16f >()

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::pmax< Packet16f >	(	const Packet16f &	a,
		const Packet16f &	b
	)

                                                                                      {
  // Arguments are reversed to match NaN propagation behavior of std::max.
  return _mm512_max_ps(b, a);
}

References a, and b.

Referenced by pmax< Packet16bf >(), pmax< Packet16h >(), pmax< PropagateNaN, Packet16f >(), and pmax< PropagateNumbers, Packet16f >().

pmax< Packet16h >()

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::pmax< Packet16h >	(	const Packet16h &	a,
		const Packet16h &	b
	)

                                                                                      {
  return float2half(pmax<Packet16f>(half2float(a), half2float(b)));
}

References a, b, float2half(), half2float(), and pmax< Packet16f >().

pmax< Packet16i >()

template<>

EIGEN_STRONG_INLINE Packet16i Eigen::internal::pmax< Packet16i >	(	const Packet16i &	a,
		const Packet16i &	b
	)

                                                                                      {
  return _mm512_max_epi32(b, a);
}

References a, and b.

pmax< Packet16uc >()

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::pmax< Packet16uc >	(	const Packet16uc &	a,
		const Packet16uc &	b
	)

                                                                                          {
  return vec_max(a, b);
}

References a, and b.

pmax< Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pmax< Packet2d >	(	const Packet2d &	a,
		const Packet2d &	b
	)

                                                                                  {
  Packet2l aNaN = __lsx_vfcmp_cun_d(a, a);
  Packet2l aMaxOrNaN = por<Packet2l>(__lsx_vfcmp_clt_d(b, a), aNaN);
  return (Packet2d)__lsx_vbitsel_v((__m128i)b, (__m128i)a, aMaxOrNaN);
}

References a, b, and por< Packet2l >().

Referenced by pmax< PropagateNaN, Packet2d >(), and pmax< PropagateNumbers, Packet2d >().

pmax< Packet2f >()

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::pmax< Packet2f >	(	const Packet2f &	a,
		const Packet2f &	b
	)

                                                                                  {
  return vmax_f32(a, b);
}

References a, and b.

Referenced by pmax< PropagateNaN, Packet2f >().

pmax< Packet2i >()

template<>

EIGEN_STRONG_INLINE Packet2i Eigen::internal::pmax< Packet2i >	(	const Packet2i &	a,
		const Packet2i &	b
	)

                                                                                  {
  return vmax_s32(a, b);
}

References a, and b.

pmax< Packet2l >()

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::pmax< Packet2l >	(	const Packet2l &	a,
		const Packet2l &	b
	)

                                                                                  {
  return __lsx_vmax_d(a, b);
}

References a, and b.

pmax< Packet2ui >()

template<>

EIGEN_STRONG_INLINE Packet2ui Eigen::internal::pmax< Packet2ui >	(	const Packet2ui &	a,
		const Packet2ui &	b
	)

                                                                                      {
  return vmax_u32(a, b);
}

References a, and b.

pmax< Packet2ul >()

template<>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::pmax< Packet2ul >	(	const Packet2ul &	a,
		const Packet2ul &	b
	)

                                                                                      {
  return __lsx_vmax_du(a, b);
}

References a, and b.

pmax< Packet32h >()

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::pmax< Packet32h >	(	const Packet32h &	a,
		const Packet32h &	b
	)

                                                                                      {
  return _mm512_max_ph(a, b);
}

References a, and b.

pmax< Packet4bf >()

template<>

EIGEN_STRONG_INLINE Packet4bf Eigen::internal::pmax< Packet4bf >	(	const Packet4bf &	a,
		const Packet4bf &	b
	)

                                                                                      {
  return F32ToBf16(pmax<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
}

References a, b, Bf16ToF32(), F32ToBf16(), and pmax< Packet4f >().

pmax< Packet4c >()

template<>

EIGEN_STRONG_INLINE Packet4c Eigen::internal::pmax< Packet4c >	(	const Packet4c &	a,
		const Packet4c &	b
	)

                                                                                  {
  return vget_lane_s32(
      vreinterpret_s32_s8(vmax_s8(vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
}

References a, and b.

pmax< Packet4d >()

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::pmax< Packet4d >	(	const Packet4d &	a,
		const Packet4d &	b
	)

                                                                                  {
#if EIGEN_GNUC_STRICT_LESS_THAN(6, 3, 0)
  // See pmin above
  Packet4d res;
  asm("vmaxpd %[a], %[b], %[res]" : [res] "=x"(res) : [a] "x"(a), [b] "x"(b));
  return res;
#else
  // Arguments are swapped to match NaN propagation behavior of std::max.
  return _mm256_max_pd(b, a);
#endif
}

References a, b, and res.

Referenced by pmax< PropagateNaN, Packet4d >(), and pmax< PropagateNumbers, Packet4d >().

pmax< Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pmax< Packet4f >	(	const Packet4f &	a,
		const Packet4f &	b
	)

                                                                                  {
#ifdef EIGEN_VECTORIZE_VSX
  // NOTE: about 10% slower than vec_max, but consistent with std::max and SSE regarding NaN
  Packet4f ret;
  __asm__("xvcmpgtsp %x0,%x2,%x1\n\txxsel %x0,%x1,%x2,%x0" : "=&wa"(ret) : "wa"(a), "wa"(b));
  return ret;
#else
  return vec_max(a, b);
#endif
}

References a, b, and ret.

Referenced by pmax< Packet4bf >(), pmax< Packet8bf >(), pmax< PropagateNaN, Packet4f >(), and pmax< PropagateNumbers, Packet4f >().

pmax< Packet4i >()

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pmax< Packet4i >	(	const Packet4i &	a,
		const Packet4i &	b
	)

                                                                                  {
  return vec_max(a, b);
}

References a, and b.

pmax< Packet4s >()

template<>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::pmax< Packet4s >	(	const Packet4s &	a,
		const Packet4s &	b
	)

                                                                                  {
  return vmax_s16(a, b);
}

References a, and b.

pmax< Packet4uc >()

template<>

EIGEN_STRONG_INLINE Packet4uc Eigen::internal::pmax< Packet4uc >	(	const Packet4uc &	a,
		const Packet4uc &	b
	)

                                                                                      {
  return vget_lane_u32(
      vreinterpret_u32_u8(vmax_u8(vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
}

References a, and b.

pmax< Packet4ui >()

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::pmax< Packet4ui >	(	const Packet4ui &	a,
		const Packet4ui &	b
	)

                                                                                      {
  return __lsx_vmax_wu(a, b);
}

References a, and b.

pmax< Packet4us >()

template<>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::pmax< Packet4us >	(	const Packet4us &	a,
		const Packet4us &	b
	)

                                                                                      {
  return vmax_u16(a, b);
}

References a, and b.

pmax< Packet8bf >()

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::pmax< Packet8bf >	(	const Packet8bf &	a,
		const Packet8bf &	b
	)

                                                                                      {
  BF16_TO_F32_BINARY_OP_WRAPPER(pmax<Packet4f>, a, b);
}

References a, b, BF16_TO_F32_BINARY_OP_WRAPPER, and pmax< Packet4f >().

pmax< Packet8c >()

template<>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::pmax< Packet8c >	(	const Packet8c &	a,
		const Packet8c &	b
	)

                                                                                  {
  return vmax_s8(a, b);
}

References a, and b.

pmax< Packet8d >()

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::pmax< Packet8d >	(	const Packet8d &	a,
		const Packet8d &	b
	)

                                                                                  {
  // Arguments are reversed to match NaN propagation behavior of std::max.
  return _mm512_max_pd(b, a);
}

References a, and b.

Referenced by pmax< PropagateNaN, Packet8d >(), and pmax< PropagateNumbers, Packet8d >().

pmax< Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::pmax< Packet8f >	(	const Packet8f &	a,
		const Packet8f &	b
	)

                                                                                  {
#if EIGEN_GNUC_STRICT_LESS_THAN(6, 3, 0)
  // See pmin above
  Packet8f res;
  asm("vmaxps %[a], %[b], %[res]" : [res] "=x"(res) : [a] "x"(a), [b] "x"(b));
  return res;
#else
  // Arguments are swapped to match NaN propagation behavior of std::max.
  return _mm256_max_ps(b, a);
#endif
}

References a, b, and res.

Referenced by pmax< Packet8h >(), pmax< PropagateNaN, Packet8f >(), and pmax< PropagateNumbers, Packet8f >().

pmax< Packet8h >()

template<>

EIGEN_STRONG_INLINE Packet8h Eigen::internal::pmax< Packet8h >	(	const Packet8h &	a,
		const Packet8h &	b
	)

                                                                                  {
  return float2half(pmax<Packet8f>(half2float(a), half2float(b)));
}

References a, b, float2half(), half2float(), and pmax< Packet8f >().

pmax< Packet8i >()

template<>

EIGEN_STRONG_INLINE Packet8i Eigen::internal::pmax< Packet8i >	(	const Packet8i &	a,
		const Packet8i &	b
	)

                                                                                  {
#ifdef EIGEN_VECTORIZE_AVX2
  return _mm256_max_epi32(a, b);
#else
  __m128i lo = _mm_max_epi32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0));
  __m128i hi = _mm_max_epi32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1));
  return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1);
#endif
}

References a, and b.

pmax< Packet8l >()

template<>

EIGEN_STRONG_INLINE Packet8l Eigen::internal::pmax< Packet8l >	(	const Packet8l &	a,
		const Packet8l &	b
	)

                                                                                  {
  return _mm512_max_epi64(b, a);
}

References a, and b.

pmax< Packet8s >()

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::pmax< Packet8s >	(	const Packet8s &	a,
		const Packet8s &	b
	)

                                                                                  {
  return vec_max(a, b);
}

References a, and b.

pmax< Packet8uc >()

template<>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::pmax< Packet8uc >	(	const Packet8uc &	a,
		const Packet8uc &	b
	)

                                                                                      {
  return vmax_u8(a, b);
}

References a, and b.

pmax< Packet8ui >()

template<>

EIGEN_STRONG_INLINE Packet8ui Eigen::internal::pmax< Packet8ui >	(	const Packet8ui &	a,
		const Packet8ui &	b
	)

                                                                                      {
#ifdef EIGEN_VECTORIZE_AVX2
  return _mm256_max_epu32(a, b);
#else
  __m128i lo = _mm_max_epu32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0));
  __m128i hi = _mm_max_epu32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1));
  return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1);
#endif
}

References a, and b.

pmax< Packet8us >()

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::pmax< Packet8us >	(	const Packet8us &	a,
		const Packet8us &	b
	)

                                                                                      {
  return vec_max(a, b);
}

References a, and b.

pmax< PacketXf >()

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::pmax< PacketXf >	(	const PacketXf &	a,
		const PacketXf &	b
	)

                                                                                  {
  return svmax_f32_x(svptrue_b32(), a, b);
}

References a, and b.

Referenced by pmax< PropagateNaN, PacketXf >().

pmax< PacketXi >()

template<>

EIGEN_STRONG_INLINE PacketXi Eigen::internal::pmax< PacketXi >	(	const PacketXi &	a,
		const PacketXi &	b
	)

                                                                                  {
  return svmax_s32_x(svptrue_b32(), a, b);
}

References a, and b.

pmax< PropagateNaN, Packet16f >()

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::pmax< PropagateNaN, Packet16f >	(	const Packet16f &	a,
		const Packet16f &	b
	)

                                                                                                    {
  return pminmax_propagate_nan(a, b, pmax<Packet16f>);
}

References a, b, pmax< Packet16f >(), and pminmax_propagate_nan().

pmax< PropagateNaN, Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pmax< PropagateNaN, Packet2d >	(	const Packet2d &	a,
		const Packet2d &	b
	)

                                                                                                {
  return pmax<Packet2d>(a, b);
}

References a, b, and pmax< Packet2d >().

pmax< PropagateNaN, Packet2f >()

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::pmax< PropagateNaN, Packet2f >	(	const Packet2f &	a,
		const Packet2f &	b
	)

                                                                                                {
  return pmax<Packet2f>(a, b);
}

References a, b, and pmax< Packet2f >().

pmax< PropagateNaN, Packet4bf >()

template<>

EIGEN_STRONG_INLINE Packet4bf Eigen::internal::pmax< PropagateNaN, Packet4bf >	(	const Packet4bf &	a,
		const Packet4bf &	b
	)

                                                                                                    {
  return F32ToBf16(pmax<PropagateNaN, Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
}

References a, b, Bf16ToF32(), F32ToBf16(), and pmax< PropagateNaN, Packet4f >().

pmax< PropagateNaN, Packet4d >()

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::pmax< PropagateNaN, Packet4d >	(	const Packet4d &	a,
		const Packet4d &	b
	)

                                                                                                {
  return pminmax_propagate_nan(a, b, pmax<Packet4d>);
}

References a, b, pmax< Packet4d >(), and pminmax_propagate_nan().

pmax< PropagateNaN, Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pmax< PropagateNaN, Packet4f >	(	const Packet4f &	a,
		const Packet4f &	b
	)

                                                                                                {
  return pmax<Packet4f>(a, b);
}

References a, b, and pmax< Packet4f >().

Referenced by pmax< PropagateNaN, Packet4bf >().

pmax< PropagateNaN, Packet8d >()

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::pmax< PropagateNaN, Packet8d >	(	const Packet8d &	a,
		const Packet8d &	b
	)

                                                                                                {
  return pminmax_propagate_nan(a, b, pmax<Packet8d>);
}

References a, b, pmax< Packet8d >(), and pminmax_propagate_nan().

pmax< PropagateNaN, Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::pmax< PropagateNaN, Packet8f >	(	const Packet8f &	a,
		const Packet8f &	b
	)

                                                                                                {
  return pminmax_propagate_nan(a, b, pmax<Packet8f>);
}

References a, b, pmax< Packet8f >(), and pminmax_propagate_nan().

pmax< PropagateNaN, PacketXf >()

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::pmax< PropagateNaN, PacketXf >	(	const PacketXf &	a,
		const PacketXf &	b
	)

                                                                                                {
  return pmax<PacketXf>(a, b);
}

References a, b, and pmax< PacketXf >().

pmax< PropagateNumbers, Packet16f >()

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::pmax< PropagateNumbers, Packet16f >	(	const Packet16f &	a,
		const Packet16f &	b
	)

                                                                                                        {
  return pminmax_propagate_numbers(a, b, pmax<Packet16f>);
}

References a, b, pmax< Packet16f >(), and pminmax_propagate_numbers().

pmax< PropagateNumbers, Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pmax< PropagateNumbers, Packet2d >	(	const Packet2d &	a,
		const Packet2d &	b
	)

                                                                                                    {
  return pminmax_propagate_numbers(a, b, pmax<Packet2d>);
}

References a, b, pmax< Packet2d >(), and pminmax_propagate_numbers().

pmax< PropagateNumbers, Packet4bf >()

template<>

EIGEN_STRONG_INLINE Packet4bf Eigen::internal::pmax< PropagateNumbers, Packet4bf >	(	const Packet4bf &	a,
		const Packet4bf &	b
	)

                                                                                                        {
  return F32ToBf16(pmax<PropagateNumbers, Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
}

References a, b, Bf16ToF32(), F32ToBf16(), and pmax< PropagateNumbers, Packet4f >().

pmax< PropagateNumbers, Packet4d >()

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::pmax< PropagateNumbers, Packet4d >	(	const Packet4d &	a,
		const Packet4d &	b
	)

                                                                                                    {
  return pminmax_propagate_numbers(a, b, pmax<Packet4d>);
}

References a, b, pmax< Packet4d >(), and pminmax_propagate_numbers().

pmax< PropagateNumbers, Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pmax< PropagateNumbers, Packet4f >	(	const Packet4f &	a,
		const Packet4f &	b
	)

                                                                                                    {
  return pminmax_propagate_numbers(a, b, pmax<Packet4f>);
}

References a, b, pmax< Packet4f >(), and pminmax_propagate_numbers().

Referenced by pmax< PropagateNumbers, Packet4bf >().

pmax< PropagateNumbers, Packet8d >()

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::pmax< PropagateNumbers, Packet8d >	(	const Packet8d &	a,
		const Packet8d &	b
	)

                                                                                                    {
  return pminmax_propagate_numbers(a, b, pmax<Packet8d>);
}

References a, b, pmax< Packet8d >(), and pminmax_propagate_numbers().

pmax< PropagateNumbers, Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::pmax< PropagateNumbers, Packet8f >	(	const Packet8f &	a,
		const Packet8f &	b
	)

                                                                                                    {
  return pminmax_propagate_numbers(a, b, pmax<Packet8f>);
}

References a, b, pmax< Packet8f >(), and pminmax_propagate_numbers().

pmax< PropagateNumbers, PacketXf >()

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::pmax< PropagateNumbers, PacketXf >	(	const PacketXf &	a,
		const PacketXf &	b
	)

                                                                                                    {
  return svmaxnm_f32_x(svptrue_b32(), a, b);
}

References a, and b.

pmerge()

EIGEN_ALWAYS_INLINE Packet8us Eigen::internal::pmerge	(	Packet4ui	even,
		Packet4ui	odd
	)

                                                                    {
#ifdef _BIG_ENDIAN
  return vec_perm(reinterpret_cast<Packet8us>(odd), reinterpret_cast<Packet8us>(even), p16uc_MERGEO16);
#else
  return vec_perm(reinterpret_cast<Packet8us>(even), reinterpret_cast<Packet8us>(odd), p16uc_MERGEE16);
#endif
}

References p16uc_MERGEE16, and p16uc_MERGEO16.

Referenced by F32ToBf16(), F32ToBf16Bool(), and pcast< Packet8bf, Packet8us >().

pmin() [1/2]

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::pmin ( const Packet &  a, const Packet &  b  )

inline

Returns

the min of a and b (coeff-wise). If a or b is NaN, the return value is implementation defined.

                                                                       {
  return numext::mini(a, b);
}

References a, b, and Eigen::numext::mini().

Referenced by generic_exp2(), packetmath_notcomplex(), Eigen::internal::scalar_logistic_op< float >::packetOp(), Eigen::internal::scalar_clamp_op< Scalar >::packetOp(), pcmp_le(), pexp_double(), pexp_float(), pldexp< Packet4d >(), pldexp< Packet8d >(), pldexp_fast(), pldexp_fast< Packet2d >(), pldexp_fast< Packet4d >(), pldexp_generic(), psqrt_complex(), ptanh_double(), ptanh_float(), Eigen::internal::psign_impl< Packet, std::enable_if_t< NumTraits< typename unpacket_traits< Packet >::type >::IsComplex &&unpacket_traits< Packet >::vectorizable > >::run(), Eigen::internal::generic_fast_erf< Scalar >::run(), and Eigen::internal::generic_fast_erfc< Scalar >::run().

pmin() [2/2]

template<int NaNPropagation, typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::pmin ( const Packet &  a, const Packet &  b  )

inline

Returns

the min of a and b (coeff-wise). NaNPropagation determines the NaN propagation semantics.

                                                                       {
  return pminmax_impl<NaNPropagation>::run(a, b, EIGEN_BINARY_OP_NAN_PROPAGATION(Packet, (pmin<Packet>)));
}

References a, b, EIGEN_BINARY_OP_NAN_PROPAGATION, and Eigen::internal::pminmax_impl< NaNPropagation >::run().

pmin< Packet16bf >()

template<>

EIGEN_STRONG_INLINE Packet16bf Eigen::internal::pmin< Packet16bf >	(	const Packet16bf &	a,
		const Packet16bf &	b
	)

                                                                                          {
  return F32ToBf16(pmin<Packet16f>(Bf16ToF32(a), Bf16ToF32(b)));
}

References a, b, Bf16ToF32(), F32ToBf16(), and pmin< Packet16f >().

pmin< Packet16c >()

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::pmin< Packet16c >	(	const Packet16c &	a,
		const Packet16c &	b
	)

                                                                                      {
  return vec_min(a, b);
}

References a, and b.

pmin< Packet16f >()

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::pmin< Packet16f >	(	const Packet16f &	a,
		const Packet16f &	b
	)

                                                                                      {
  // Arguments are reversed to match NaN propagation behavior of std::min.
  return _mm512_min_ps(b, a);
}

References a, and b.

Referenced by pmin< Packet16bf >(), pmin< Packet16h >(), pmin< PropagateNaN, Packet16f >(), and pmin< PropagateNumbers, Packet16f >().

pmin< Packet16h >()

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::pmin< Packet16h >	(	const Packet16h &	a,
		const Packet16h &	b
	)

                                                                                      {
  return float2half(pmin<Packet16f>(half2float(a), half2float(b)));
}

References a, b, float2half(), half2float(), and pmin< Packet16f >().

pmin< Packet16i >()

template<>

EIGEN_STRONG_INLINE Packet16i Eigen::internal::pmin< Packet16i >	(	const Packet16i &	a,
		const Packet16i &	b
	)

                                                                                      {
  return _mm512_min_epi32(b, a);
}

References a, and b.

pmin< Packet16uc >()

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::pmin< Packet16uc >	(	const Packet16uc &	a,
		const Packet16uc &	b
	)

                                                                                          {
  return vec_min(a, b);
}

References a, and b.

pmin< Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pmin< Packet2d >	(	const Packet2d &	a,
		const Packet2d &	b
	)

                                                                                  {
  Packet2l aNaN = __lsx_vfcmp_cun_d(a, a);
  Packet2l aMinOrNaN = por<Packet2l>(__lsx_vfcmp_clt_d(a, b), aNaN);
  return (Packet2d)__lsx_vbitsel_v((__m128i)b, (__m128i)a, aMinOrNaN);
}

References a, b, and por< Packet2l >().

Referenced by pmin< PropagateNaN, Packet2d >(), and pmin< PropagateNumbers, Packet2d >().

pmin< Packet2f >()

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::pmin< Packet2f >	(	const Packet2f &	a,
		const Packet2f &	b
	)

                                                                                  {
  return vmin_f32(a, b);
}

References a, and b.

Referenced by pmin< PropagateNaN, Packet2f >().

pmin< Packet2i >()

template<>

EIGEN_STRONG_INLINE Packet2i Eigen::internal::pmin< Packet2i >	(	const Packet2i &	a,
		const Packet2i &	b
	)

                                                                                  {
  return vmin_s32(a, b);
}

References a, and b.

pmin< Packet2l >()

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::pmin< Packet2l >	(	const Packet2l &	a,
		const Packet2l &	b
	)

                                                                                  {
  return __lsx_vmin_d(a, b);
}

References a, and b.

pmin< Packet2ui >()

template<>

EIGEN_STRONG_INLINE Packet2ui Eigen::internal::pmin< Packet2ui >	(	const Packet2ui &	a,
		const Packet2ui &	b
	)

                                                                                      {
  return vmin_u32(a, b);
}

References a, and b.

pmin< Packet2ul >()

template<>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::pmin< Packet2ul >	(	const Packet2ul &	a,
		const Packet2ul &	b
	)

                                                                                      {
  return __lsx_vmin_du(a, b);
}

References a, and b.

pmin< Packet32h >()

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::pmin< Packet32h >	(	const Packet32h &	a,
		const Packet32h &	b
	)

                                                                                      {
  return _mm512_min_ph(a, b);
}

References a, and b.

pmin< Packet4bf >()

template<>

EIGEN_STRONG_INLINE Packet4bf Eigen::internal::pmin< Packet4bf >	(	const Packet4bf &	a,
		const Packet4bf &	b
	)

                                                                                      {
  return F32ToBf16(pmin<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
}

References a, b, Bf16ToF32(), F32ToBf16(), and pmin< Packet4f >().

pmin< Packet4c >()

template<>

EIGEN_STRONG_INLINE Packet4c Eigen::internal::pmin< Packet4c >	(	const Packet4c &	a,
		const Packet4c &	b
	)

                                                                                  {
  return vget_lane_s32(
      vreinterpret_s32_s8(vmin_s8(vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
}

References a, and b.

pmin< Packet4d >()

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::pmin< Packet4d >	(	const Packet4d &	a,
		const Packet4d &	b
	)

                                                                                  {
#if EIGEN_GNUC_STRICT_LESS_THAN(6, 3, 0)
  // See pmin above
  Packet4d res;
  asm("vminpd %[a], %[b], %[res]" : [res] "=x"(res) : [a] "x"(a), [b] "x"(b));
  return res;
#else
  // Arguments are swapped to match NaN propagation behavior of std::min.
  return _mm256_min_pd(b, a);
#endif
}

References a, b, and res.

Referenced by pmin< PropagateNaN, Packet4d >(), and pmin< PropagateNumbers, Packet4d >().

pmin< Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pmin< Packet4f >	(	const Packet4f &	a,
		const Packet4f &	b
	)

                                                                                  {
#ifdef EIGEN_VECTORIZE_VSX
  // NOTE: about 10% slower than vec_min, but consistent with std::min and SSE regarding NaN
  Packet4f ret;
  __asm__("xvcmpgesp %x0,%x1,%x2\n\txxsel %x0,%x1,%x2,%x0" : "=&wa"(ret) : "wa"(a), "wa"(b));
  return ret;
#else
  return vec_min(a, b);
#endif
}

References a, b, and ret.

Referenced by pmin< Packet4bf >(), pmin< Packet8bf >(), pmin< PropagateNaN, Packet4f >(), and pmin< PropagateNumbers, Packet4f >().

pmin< Packet4i >()

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pmin< Packet4i >	(	const Packet4i &	a,
		const Packet4i &	b
	)

                                                                                  {
  return vec_min(a, b);
}

References a, and b.

pmin< Packet4s >()

template<>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::pmin< Packet4s >	(	const Packet4s &	a,
		const Packet4s &	b
	)

                                                                                  {
  return vmin_s16(a, b);
}

References a, and b.

pmin< Packet4uc >()

template<>

EIGEN_STRONG_INLINE Packet4uc Eigen::internal::pmin< Packet4uc >	(	const Packet4uc &	a,
		const Packet4uc &	b
	)

                                                                                      {
  return vget_lane_u32(
      vreinterpret_u32_u8(vmin_u8(vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
}

References a, and b.

pmin< Packet4ui >()

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::pmin< Packet4ui >	(	const Packet4ui &	a,
		const Packet4ui &	b
	)

                                                                                      {
  return __lsx_vmin_wu(a, b);
}

References a, and b.

pmin< Packet4us >()

template<>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::pmin< Packet4us >	(	const Packet4us &	a,
		const Packet4us &	b
	)

                                                                                      {
  return vmin_u16(a, b);
}

References a, and b.

pmin< Packet8bf >()

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::pmin< Packet8bf >	(	const Packet8bf &	a,
		const Packet8bf &	b
	)

                                                                                      {
  BF16_TO_F32_BINARY_OP_WRAPPER(pmin<Packet4f>, a, b);
}

References a, b, BF16_TO_F32_BINARY_OP_WRAPPER, and pmin< Packet4f >().

pmin< Packet8c >()

template<>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::pmin< Packet8c >	(	const Packet8c &	a,
		const Packet8c &	b
	)

                                                                                  {
  return vmin_s8(a, b);
}

References a, and b.

pmin< Packet8d >()

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::pmin< Packet8d >	(	const Packet8d &	a,
		const Packet8d &	b
	)

                                                                                  {
  // Arguments are reversed to match NaN propagation behavior of std::min.
  return _mm512_min_pd(b, a);
}

References a, and b.

Referenced by pmin< PropagateNaN, Packet8d >(), and pmin< PropagateNumbers, Packet8d >().

pmin< Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::pmin< Packet8f >	(	const Packet8f &	a,
		const Packet8f &	b
	)

                                                                                  {
#if EIGEN_GNUC_STRICT_LESS_THAN(6, 3, 0)
  // There appears to be a bug in GCC, by which the optimizer may flip
  // the argument order in calls to _mm_min_ps/_mm_max_ps, so we have to
  // resort to inline ASM here. This is supposed to be fixed in gcc6.3,
  // see also: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=72867
  Packet8f res;
  asm("vminps %[a], %[b], %[res]" : [res] "=x"(res) : [a] "x"(a), [b] "x"(b));
  return res;
#else
  // Arguments are swapped to match NaN propagation behavior of std::min.
  return _mm256_min_ps(b, a);
#endif
}

References a, b, and res.

Referenced by pmin< Packet8h >(), pmin< PropagateNaN, Packet8f >(), and pmin< PropagateNumbers, Packet8f >().

pmin< Packet8h >()

template<>

EIGEN_STRONG_INLINE Packet8h Eigen::internal::pmin< Packet8h >	(	const Packet8h &	a,
		const Packet8h &	b
	)

                                                                                  {
  return float2half(pmin<Packet8f>(half2float(a), half2float(b)));
}

References a, b, float2half(), half2float(), and pmin< Packet8f >().

pmin< Packet8i >()

template<>

EIGEN_STRONG_INLINE Packet8i Eigen::internal::pmin< Packet8i >	(	const Packet8i &	a,
		const Packet8i &	b
	)

                                                                                  {
#ifdef EIGEN_VECTORIZE_AVX2
  return _mm256_min_epi32(a, b);
#else
  __m128i lo = _mm_min_epi32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0));
  __m128i hi = _mm_min_epi32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1));
  return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1);
#endif
}

References a, and b.

pmin< Packet8l >()

template<>

EIGEN_STRONG_INLINE Packet8l Eigen::internal::pmin< Packet8l >	(	const Packet8l &	a,
		const Packet8l &	b
	)

                                                                                  {
  return _mm512_min_epi64(b, a);
}

References a, and b.

pmin< Packet8s >()

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::pmin< Packet8s >	(	const Packet8s &	a,
		const Packet8s &	b
	)

                                                                                  {
  return vec_min(a, b);
}

References a, and b.

pmin< Packet8uc >()

template<>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::pmin< Packet8uc >	(	const Packet8uc &	a,
		const Packet8uc &	b
	)

                                                                                      {
  return vmin_u8(a, b);
}

References a, and b.

pmin< Packet8ui >()

template<>

EIGEN_STRONG_INLINE Packet8ui Eigen::internal::pmin< Packet8ui >	(	const Packet8ui &	a,
		const Packet8ui &	b
	)

                                                                                      {
#ifdef EIGEN_VECTORIZE_AVX2
  return _mm256_min_epu32(a, b);
#else
  __m128i lo = _mm_min_epu32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0));
  __m128i hi = _mm_min_epu32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1));
  return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1);
#endif
}

References a, and b.

pmin< Packet8us >()

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::pmin< Packet8us >	(	const Packet8us &	a,
		const Packet8us &	b
	)

                                                                                      {
  return vec_min(a, b);
}

References a, and b.

pmin< PacketXf >()

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::pmin< PacketXf >	(	const PacketXf &	a,
		const PacketXf &	b
	)

                                                                                  {
  return svmin_f32_x(svptrue_b32(), a, b);
}

References a, and b.

Referenced by pmin< PropagateNaN, PacketXf >().

pmin< PacketXi >()

template<>

EIGEN_STRONG_INLINE PacketXi Eigen::internal::pmin< PacketXi >	(	const PacketXi &	a,
		const PacketXi &	b
	)

                                                                                  {
  return svmin_s32_x(svptrue_b32(), a, b);
}

References a, and b.

pmin< PropagateNaN, Packet16f >()

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::pmin< PropagateNaN, Packet16f >	(	const Packet16f &	a,
		const Packet16f &	b
	)

                                                                                                    {
  return pminmax_propagate_nan(a, b, pmin<Packet16f>);
}

References a, b, pmin< Packet16f >(), and pminmax_propagate_nan().

pmin< PropagateNaN, Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pmin< PropagateNaN, Packet2d >	(	const Packet2d &	a,
		const Packet2d &	b
	)

                                                                                                {
  return pmin<Packet2d>(a, b);
}

References a, b, and pmin< Packet2d >().

pmin< PropagateNaN, Packet2f >()

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::pmin< PropagateNaN, Packet2f >	(	const Packet2f &	a,
		const Packet2f &	b
	)

                                                                                                {
  return pmin<Packet2f>(a, b);
}

References a, b, and pmin< Packet2f >().

pmin< PropagateNaN, Packet4bf >()

template<>

EIGEN_STRONG_INLINE Packet4bf Eigen::internal::pmin< PropagateNaN, Packet4bf >	(	const Packet4bf &	a,
		const Packet4bf &	b
	)

                                                                                                    {
  return F32ToBf16(pmin<PropagateNaN, Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
}

References a, b, Bf16ToF32(), F32ToBf16(), and pmin< PropagateNaN, Packet4f >().

pmin< PropagateNaN, Packet4d >()

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::pmin< PropagateNaN, Packet4d >	(	const Packet4d &	a,
		const Packet4d &	b
	)

                                                                                                {
  return pminmax_propagate_nan(a, b, pmin<Packet4d>);
}

References a, b, pmin< Packet4d >(), and pminmax_propagate_nan().

pmin< PropagateNaN, Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pmin< PropagateNaN, Packet4f >	(	const Packet4f &	a,
		const Packet4f &	b
	)

                                                                                                {
  return pmin<Packet4f>(a, b);
}

References a, b, and pmin< Packet4f >().

Referenced by pmin< PropagateNaN, Packet4bf >().

pmin< PropagateNaN, Packet8d >()

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::pmin< PropagateNaN, Packet8d >	(	const Packet8d &	a,
		const Packet8d &	b
	)

                                                                                                {
  return pminmax_propagate_nan(a, b, pmin<Packet8d>);
}

References a, b, pmin< Packet8d >(), and pminmax_propagate_nan().

pmin< PropagateNaN, Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::pmin< PropagateNaN, Packet8f >	(	const Packet8f &	a,
		const Packet8f &	b
	)

                                                                                                {
  return pminmax_propagate_nan(a, b, pmin<Packet8f>);
}

References a, b, pmin< Packet8f >(), and pminmax_propagate_nan().

pmin< PropagateNaN, PacketXf >()

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::pmin< PropagateNaN, PacketXf >	(	const PacketXf &	a,
		const PacketXf &	b
	)

                                                                                                {
  return pmin<PacketXf>(a, b);
}

References a, b, and pmin< PacketXf >().

pmin< PropagateNumbers, Packet16f >()

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::pmin< PropagateNumbers, Packet16f >	(	const Packet16f &	a,
		const Packet16f &	b
	)

                                                                                                        {
  return pminmax_propagate_numbers(a, b, pmin<Packet16f>);
}

References a, b, pmin< Packet16f >(), and pminmax_propagate_numbers().

pmin< PropagateNumbers, Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pmin< PropagateNumbers, Packet2d >	(	const Packet2d &	a,
		const Packet2d &	b
	)

                                                                                                    {
  return pminmax_propagate_numbers(a, b, pmin<Packet2d>);
}

References a, b, pmin< Packet2d >(), and pminmax_propagate_numbers().

pmin< PropagateNumbers, Packet4bf >()

template<>

EIGEN_STRONG_INLINE Packet4bf Eigen::internal::pmin< PropagateNumbers, Packet4bf >	(	const Packet4bf &	a,
		const Packet4bf &	b
	)

                                                                                                        {
  return F32ToBf16(pmin<PropagateNumbers, Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
}

References a, b, Bf16ToF32(), F32ToBf16(), and pmin< PropagateNumbers, Packet4f >().

pmin< PropagateNumbers, Packet4d >()

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::pmin< PropagateNumbers, Packet4d >	(	const Packet4d &	a,
		const Packet4d &	b
	)

                                                                                                    {
  return pminmax_propagate_numbers(a, b, pmin<Packet4d>);
}

References a, b, pmin< Packet4d >(), and pminmax_propagate_numbers().

pmin< PropagateNumbers, Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pmin< PropagateNumbers, Packet4f >	(	const Packet4f &	a,
		const Packet4f &	b
	)

                                                                                                    {
  return pminmax_propagate_numbers(a, b, pmin<Packet4f>);
}

References a, b, pmin< Packet4f >(), and pminmax_propagate_numbers().

Referenced by pmin< PropagateNumbers, Packet4bf >().

pmin< PropagateNumbers, Packet8d >()

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::pmin< PropagateNumbers, Packet8d >	(	const Packet8d &	a,
		const Packet8d &	b
	)

                                                                                                    {
  return pminmax_propagate_numbers(a, b, pmin<Packet8d>);
}

References a, b, pmin< Packet8d >(), and pminmax_propagate_numbers().

pmin< PropagateNumbers, Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::pmin< PropagateNumbers, Packet8f >	(	const Packet8f &	a,
		const Packet8f &	b
	)

                                                                                                    {
  return pminmax_propagate_numbers(a, b, pmin<Packet8f>);
}

References a, b, pmin< Packet8f >(), and pminmax_propagate_numbers().

pmin< PropagateNumbers, PacketXf >()

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::pmin< PropagateNumbers, PacketXf >	(	const PacketXf &	a,
		const PacketXf &	b
	)

                                                                                                    {
  return svminnm_f32_x(svptrue_b32(), a, b);
}

References a, and b.

pminmax_propagate_nan()

template<typename Packet , typename Op >

EIGEN_STRONG_INLINE Packet Eigen::internal::pminmax_propagate_nan	(	const Packet &	a,
		const Packet &	b,
		Op	op
	)

                                                                                          {
  // In this implementation, we take advantage of the fact that pmin/pmax for SSE
  // always return a if either a or b is NaN.
  Packet not_nan_mask_a = pcmp_eq(a, a);
  Packet m = op(b, a);
  return pselect<Packet>(not_nan_mask_a, m, a);
}

References a, b, m, op, and pcmp_eq().

Referenced by pmax< PropagateNaN, Packet16f >(), pmax< PropagateNaN, Packet4d >(), pmax< PropagateNaN, Packet8d >(), pmax< PropagateNaN, Packet8f >(), pmin< PropagateNaN, Packet16f >(), pmin< PropagateNaN, Packet4d >(), pmin< PropagateNaN, Packet8d >(), and pmin< PropagateNaN, Packet8f >().

pminmax_propagate_numbers()

template<typename Packet , typename Op >

EIGEN_STRONG_INLINE Packet Eigen::internal::pminmax_propagate_numbers	(	const Packet &	a,
		const Packet &	b,
		Op	op
	)

                                                                                              {
  // In this implementation, we take advantage of the fact that pmin/pmax for SSE
  // always return a if either a or b is NaN.
  Packet not_nan_mask_a = pcmp_eq(a, a);
  Packet m = op(a, b);
  return pselect<Packet>(not_nan_mask_a, m, b);
}

References a, b, m, op, and pcmp_eq().

Referenced by pmax< PropagateNumbers, Packet16f >(), pmax< PropagateNumbers, Packet2d >(), pmax< PropagateNumbers, Packet4d >(), pmax< PropagateNumbers, Packet4f >(), pmax< PropagateNumbers, Packet8d >(), pmax< PropagateNumbers, Packet8f >(), pmin< PropagateNumbers, Packet16f >(), pmin< PropagateNumbers, Packet2d >(), pmin< PropagateNumbers, Packet4d >(), pmin< PropagateNumbers, Packet4f >(), pmin< PropagateNumbers, Packet8d >(), and pmin< PropagateNumbers, Packet8f >().

pmsub() [1/7]

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::pmsub ( const Packet &  a, const Packet &  b, const Packet &  c  )

inline

Returns

a * b - c (coeff-wise)

                                                                                         {
  return psub(pmul(a, b), c);
}

References a, b, calibrate::c, pmul(), and psub().

pmsub() [2/7]

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::pmsub	(	const Packet16c &	a,
		const Packet16c &	b,
		const Packet16c &	c
	)

                                                                                                {
  return __lsx_vmadd_b(pnegate(c), a, b);
}

References a, b, calibrate::c, and pnegate().

pmsub() [3/7]

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pmsub	(	const Packet2d &	a,
		const Packet2d &	b,
		const Packet2d &	c
	)

                                                                                            {
  return __lsx_vfmsub_d(a, b, c);
}

References a, b, and calibrate::c.

pmsub() [4/7]

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::pmsub	(	const Packet2l &	a,
		const Packet2l &	b,
		const Packet2l &	c
	)

                                                                                            {
  return __lsx_vmadd_d(pnegate(c), a, b);
}

References a, b, calibrate::c, and pnegate().

pmsub() [5/7]

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pmsub	(	const Packet4f &	a,
		const Packet4f &	b,
		const Packet4f &	c
	)

                                                                                            {
  return __lsx_vfmsub_s(a, b, c);
}

References a, b, and calibrate::c.

Referenced by packetmath().

pmsub() [6/7]

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pmsub	(	const Packet4i &	a,
		const Packet4i &	b,
		const Packet4i &	c
	)

                                                                                            {
  return __lsx_vmadd_w(pnegate(c), a, b);
}

References a, b, calibrate::c, and pnegate().

pmsub() [7/7]

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::pmsub	(	const Packet8s &	a,
		const Packet8s &	b,
		const Packet8s &	c
	)

                                                                                            {
  return __lsx_vmadd_h(pnegate(c), a, b);
}

References a, b, calibrate::c, and pnegate().

pmul() [1/8]

template<>

EIGEN_DEVICE_FUNC bool Eigen::internal::pmul ( const bool &  a, const bool &  b  )

inline

                                                                 {
  return a && b;
}

References a, and b.

pmul() [2/8]

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::pmul ( const Packet &  a, const Packet &  b  )

inline

Returns

a * b (coeff-wise)

                                                                       {
  return a * b;
}

References a, and b.

pmul() [3/8]

template<>

EIGEN_STRONG_INLINE Packet1cd Eigen::internal::pmul	(	const Packet1cd &	a,
		const Packet1cd &	b
	)

                                                                           {
  __m128d tmp1 = _mm_mul_pd(_mm_unpackhi_pd(a.v, a.v), vec2d_swizzle1(b.v, 1, 0));
#ifdef EIGEN_VECTORIZE_SSE3
  __m128d tmp2 = _mm_movedup_pd(a.v);
#else
  __m128d tmp2 = _mm_unpacklo_pd(a.v, a.v);
#endif
#ifdef EIGEN_VECTORIZE_FMA
  __m128d result = _mm_fmaddsub_pd(tmp2, b.v, tmp1);
#else
#ifdef EIGEN_VECTORIZE_SSE3
  __m128d result = _mm_addsub_pd(_mm_mul_pd(tmp2, b.v), tmp1);
#else
  const __m128d mask = _mm_setr_pd(-0.0, 0.0);
  __m128d result = _mm_add_pd(_mm_mul_pd(tmp2, b.v), _mm_xor_pd(tmp1, mask));
#endif
#endif
  return Packet1cd(result);
}

References a, b, and vec2d_swizzle1.

pmul() [4/8]

template<>

EIGEN_STRONG_INLINE Packet2cd Eigen::internal::pmul	(	const Packet2cd &	a,
		const Packet2cd &	b
	)

                                                                           {
  __m256d tmp1 = _mm256_mul_pd(_mm256_permute_pd(a.v, 0xF), _mm256_permute_pd(b.v, 0x5));
  __m256d tmp2 = _mm256_movedup_pd(a.v);
#ifdef EIGEN_VECTORIZE_FMA
  __m256d result = _mm256_fmaddsub_pd(tmp2, b.v, tmp1);
#else
  __m256d result = _mm256_addsub_pd(_mm256_mul_pd(tmp2, b.v), tmp1);
#endif
  return Packet2cd(result);
}

References a, and b.

pmul() [5/8]

template<>

EIGEN_STRONG_INLINE Packet2cf Eigen::internal::pmul	(	const Packet2cf &	a,
		const Packet2cf &	b
	)

                                                                           {
#ifdef EIGEN_VECTORIZE_SSE3
  __m128 tmp1 = _mm_mul_ps(_mm_movehdup_ps(a.v), vec4f_swizzle1(b.v, 1, 0, 3, 2));
  __m128 tmp2 = _mm_moveldup_ps(a.v);
#else
  __m128 tmp1 = _mm_mul_ps(vec4f_swizzle1(a.v, 1, 1, 3, 3), vec4f_swizzle1(b.v, 1, 0, 3, 2));
  __m128 tmp2 = vec4f_swizzle1(a.v, 0, 0, 2, 2);
#endif
#ifdef EIGEN_VECTORIZE_FMA
  __m128 result = _mm_fmaddsub_ps(tmp2, b.v, tmp1);
#else
#ifdef EIGEN_VECTORIZE_SSE3
  __m128 result = _mm_addsub_ps(_mm_mul_ps(tmp2, b.v), tmp1);
#else
  const __m128 mask = _mm_setr_ps(-0.0f, 0.0f, -0.0f, 0.0f);
  __m128 result = _mm_add_ps(_mm_mul_ps(tmp2, b.v), _mm_xor_ps(tmp1, mask));
#endif
#endif
  return Packet2cf(result);
}

References a, b, and vec4f_swizzle1().

pmul() [6/8]

template<>

EIGEN_STRONG_INLINE Packet4cf Eigen::internal::pmul	(	const Packet4cf &	a,
		const Packet4cf &	b
	)

                                                                           {
  __m256 tmp1 = _mm256_mul_ps(_mm256_movehdup_ps(a.v), _mm256_permute_ps(b.v, _MM_SHUFFLE(2, 3, 0, 1)));
  __m256 tmp2 = _mm256_moveldup_ps(a.v);
#ifdef EIGEN_VECTORIZE_FMA
  __m256 result = _mm256_fmaddsub_ps(tmp2, b.v, tmp1);
#else
  __m256 result = _mm256_addsub_ps(_mm256_mul_ps(tmp2, b.v), tmp1);
#endif
  return Packet4cf(result);
}

References a, and b.

Referenced by Eigen::internal::mul_assign_op< DstScalar, SrcScalar >::assignPacket(), Eigen::internal::scalar_inner_product_op< Scalar, Scalar, Conj >::coeff(), erfc_double_large(), generic_exp2(), generic_expm1(), generic_log1p(), generic_ndtri_gt_exp_neg_two(), generic_ndtri_lt_exp_neg_two(), generic_pow(), generic_pow_impl(), Eigen::internal::unary_pow::int_pow(), Eigen::internal::gebp_traits< std::complex< RealScalar >, RealScalar, ConjLhs_, false, Arch, PacketSize_ >::madd_impl(), Eigen::internal::gebp_traits< RealScalar, std::complex< RealScalar >, false, ConjRhs_, Arch, PacketSize_ >::madd_impl(), Eigen::internal::Packet1cd::operator*=(), Eigen::internal::scalar_inner_product_op< Scalar, Scalar, Conj >::packet(), Eigen::internal::diagonal_product_evaluator_base< MatrixType, DiagonalType, Derived, ProductOrder >::packet_impl(), packetmath(), Eigen::internal::scalar_logistic_op< float >::packetOp(), Eigen::internal::scalar_abs2_op< Scalar >::packetOp(), Eigen::internal::squared_norm_functor< Scalar, IsComplex >::packetOp(), Eigen::internal::scalar_square_op< Scalar >::packetOp(), Eigen::internal::scalar_cube_op< Scalar >::packetOp(), Eigen::internal::scalar_product_op< LhsScalar, RhsScalar >::packetOp(), Eigen::internal::linspaced_op_impl< Scalar, false >::packetOp(), pacos_float(), pasin_float(), patanh_double(), patanh_float(), pblueNorm(), pdiv_complex(), pdiv_float_common(), pexp< Packet2d >(), pexp< Packet4f >(), pexp_complex(), pexp_double(), pexp_float(), pfrexp_generic(), phypot_complex(), pldexp< Packet4d >(), pldexp< Packet8d >(), pldexp_fast(), pldexp_fast< Packet2d >(), pldexp_fast< Packet4d >(), pldexp_generic(), plog2(), plog< Packet4f >(), plog_impl_double(), plog_impl_float(), pmadd(), pmsub(), Eigen::internal::conj_helper< Packet, Packet, ConjLhs, ConjRhs >::pmul(), Eigen::internal::conj_helper< Packet, Packet, true, true >::pmul(), pmul< Packet16h >(), pmul< Packet8h >(), pnmadd(), predux_mul< Packet16f >(), predux_mul< Packet2cd >(), predux_mul< Packet4cd >(), predux_mul< Packet4cf >(), predux_mul< Packet4f >(), predux_mul< Packet8cf >(), predux_mul< Packet8d >(), psincos_double(), psincos_float(), psincos_inner_msa_float(), psqrt_complex(), psqrt_float_common(), ptanh< Packet4f >(), ptanh_double(), ptanh_float(), Eigen::internal::compute_inverse_size4< Architecture::Target, float, MatrixType, ResultType >::run(), Eigen::internal::compute_inverse_size4< Architecture::Target, double, MatrixType, ResultType >::run(), Eigen::internal::psign_impl< Packet, std::enable_if_t< NumTraits< typename unpacket_traits< Packet >::type >::IsComplex &&unpacket_traits< Packet >::vectorizable > >::run(), Eigen::internal::generic_reciprocal_newton_step< Packet, Steps >::run(), Eigen::internal::generic_rsqrt_newton_step< Packet, Steps >::run(), Eigen::internal::generic_sqrt_newton_step< Packet, Steps >::run(), Eigen::internal::patan_reduced< Scalar >::run(), Eigen::internal::quat_product< Architecture::Target, Derived, OtherDerived, float >::run(), Eigen::internal::generic_i0e< T, float >::run(), Eigen::internal::generic_i0e< T, double >::run(), Eigen::internal::generic_i0< T, ScalarType >::run(), Eigen::internal::generic_i1e< T, float >::run(), Eigen::internal::generic_i1e< T, double >::run(), Eigen::internal::generic_i1< T, ScalarType >::run(), Eigen::internal::generic_k0e< T, float >::run(), Eigen::internal::generic_k0e< T, double >::run(), Eigen::internal::generic_k0< T, float >::run(), Eigen::internal::generic_k0< T, double >::run(), Eigen::internal::generic_k1e< T, float >::run(), Eigen::internal::generic_k1e< T, double >::run(), Eigen::internal::generic_k1< T, float >::run(), Eigen::internal::generic_k1< T, double >::run(), Eigen::internal::generic_j0< T, float >::run(), Eigen::internal::generic_j0< T, double >::run(), Eigen::internal::generic_y0< T, float >::run(), Eigen::internal::generic_y0< T, double >::run(), Eigen::internal::generic_j1< T, float >::run(), Eigen::internal::generic_j1< T, double >::run(), Eigen::internal::generic_y1< T, float >::run(), Eigen::internal::generic_y1< T, double >::run(), Eigen::internal::generic_fast_erf< Scalar >::run(), Eigen::internal::generic_fast_erfc< Scalar >::run(), Eigen::internal::cross3_impl< Architecture::Target, VectorLhs, VectorRhs, float, true >::run(), Eigen::internal::etor_product_packet_impl< RowMajor, 1, Lhs, Rhs, Packet, LoadMode >::run(), Eigen::internal::etor_product_packet_impl< ColMajor, 1, Lhs, Rhs, Packet, LoadMode >::run(), Eigen::internal::pchebevl< Packet, N >::run(), Eigen::internal::gemm_class< Scalar, is_unit_inc >::scale_load_c(), twoprod(), and veltkamp_splitting().

pmul() [7/8]

template<>

std::complex<double> Eigen::internal::pmul ( const std::complex< double > &  a, const std::complex< double > &  b  )

inline

                                                                                         {
  return std::complex<double>(a.real() * b.real() - a.imag() * b.imag(), a.imag() * b.real() + a.real() * b.imag());
}

References a, and b.

pmul() [8/8]

template<>

std::complex<float> Eigen::internal::pmul ( const std::complex< float > &  a, const std::complex< float > &  b  )

inline

                                                                                      {
  return std::complex<float>(a.real() * b.real() - a.imag() * b.imag(), a.imag() * b.real() + a.real() * b.imag());
}

References a, and b.

pmul< Packet16b >()

template<>

EIGEN_STRONG_INLINE Packet16b Eigen::internal::pmul< Packet16b >	(	const Packet16b &	a,
		const Packet16b &	b
	)

                                                                                      {
  return _mm_and_si128(a, b);
}

References a, and b.

pmul< Packet16bf >()

template<>

EIGEN_STRONG_INLINE Packet16bf Eigen::internal::pmul< Packet16bf >	(	const Packet16bf &	a,
		const Packet16bf &	b
	)

                                                                                          {
  return F32ToBf16(pmul<Packet16f>(Bf16ToF32(a), Bf16ToF32(b)));
}

References a, b, Bf16ToF32(), F32ToBf16(), and pmul< Packet16f >().

pmul< Packet16c >()

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::pmul< Packet16c >	(	const Packet16c &	a,
		const Packet16c &	b
	)

                                                                                      {
  return vec_mul(a, b);
}

References a, and b.

pmul< Packet16f >()

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::pmul< Packet16f >	(	const Packet16f &	a,
		const Packet16f &	b
	)

                                                                                      {
  return _mm512_mul_ps(a, b);
}

References a, and b.

Referenced by pmul< Packet16bf >().

pmul< Packet16h >()

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::pmul< Packet16h >	(	const Packet16h &	a,
		const Packet16h &	b
	)

                                                                                      {
  Packet16f af = half2float(a);
  Packet16f bf = half2float(b);
  Packet16f rf = pmul(af, bf);
  return float2half(rf);
}

References a, b, float2half(), half2float(), and pmul().

pmul< Packet16i >()

template<>

EIGEN_STRONG_INLINE Packet16i Eigen::internal::pmul< Packet16i >	(	const Packet16i &	a,
		const Packet16i &	b
	)

                                                                                      {
  return _mm512_mullo_epi32(a, b);
}

References a, and b.

pmul< Packet16uc >()

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::pmul< Packet16uc >	(	const Packet16uc &	a,
		const Packet16uc &	b
	)

                                                                                          {
  return vec_mul(a, b);
}

References a, and b.

pmul< Packet1cd >()

template<>

EIGEN_STRONG_INLINE Packet1cd Eigen::internal::pmul< Packet1cd >	(	const Packet1cd &	a,
		const Packet1cd &	b
	)

                                                                                      {
  Packet2d tmp_real = __lsx_vfmul_d(a.v, b.v);
  Packet2d real = __lsx_vfsub_d(tmp_real, preverse(tmp_real));
 
  Packet2d tmp_imag = __lsx_vfmul_d(preverse(a.v), b.v);
  Packet2d imag = (__m128d)__lsx_vfadd_d((__m128d)tmp_imag, preverse(tmp_imag));
  Packet1cd res;
  res.v = (__m128d)__lsx_vilvl_d((__m128i)imag, (__m128i)real);
  return res;
}

References a, b, Eigen::imag(), preverse(), and res.

pmul< Packet1cf >()

template<>

EIGEN_STRONG_INLINE Packet1cf Eigen::internal::pmul< Packet1cf >	(	const Packet1cf &	a,
		const Packet1cf &	b
	)

                                                                                      {
  Packet2f v1, v2;
 
  // Get the real values of a | a1_re | a1_re |
  v1 = vdup_lane_f32(a.v, 0);
  // Get the imag values of a | a1_im | a1_im |
  v2 = vdup_lane_f32(a.v, 1);
  // Multiply the real a with b
  v1 = vmul_f32(v1, b.v);
  // Multiply the imag a with b
  v2 = vmul_f32(v2, b.v);
  // Conjugate v2
  v2 = vreinterpret_f32_u32(veor_u32(vreinterpret_u32_f32(v2), p2ui_CONJ_XOR()));
  // Swap real/imag elements in v2.
  v2 = vrev64_f32(v2);
  // Add and return the result
  return Packet1cf(vadd_f32(v1, v2));
}

References a, b, p2ui_CONJ_XOR(), v1(), and v2().

pmul< Packet2cf >()

template<>

EIGEN_STRONG_INLINE Packet2cf Eigen::internal::pmul< Packet2cf >	(	const Packet2cf &	a,
		const Packet2cf &	b
	)

                                                                                      {
  Packet4f part0_tmp = (Packet4f)__lsx_vfmul_s(a.v, b.v);
  Packet4f part0 = __lsx_vfsub_s(part0_tmp, (__m128)__lsx_vshuf4i_w(part0_tmp, 0x31));
  Packet4f part1_tmp = __lsx_vfmul_s((__m128)__lsx_vshuf4i_w(a.v, 0xb1), b.v);
  Packet4f part1 = __lsx_vfadd_s(part1_tmp, (__m128)__lsx_vshuf4i_w(part1_tmp, 0x31));
  Packet2cf res;
  res.v = (Packet4f)__lsx_vpackev_w((__m128i)part1, (__m128i)part0);
  return res;
}

References a, b, and res.

Referenced by predux_mul< Packet2cf >().

pmul< Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pmul< Packet2d >	(	const Packet2d &	a,
		const Packet2d &	b
	)

                                                                                  {
  return __lsx_vfmul_d(a, b);
}

References a, and b.

Referenced by Eigen::internal::Packet1cd::operator/=().

pmul< Packet2f >()

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::pmul< Packet2f >	(	const Packet2f &	a,
		const Packet2f &	b
	)

                                                                                  {
  return vmul_f32(a, b);
}

References a, and b.

pmul< Packet2i >()

template<>

EIGEN_STRONG_INLINE Packet2i Eigen::internal::pmul< Packet2i >	(	const Packet2i &	a,
		const Packet2i &	b
	)

                                                                                  {
  return vmul_s32(a, b);
}

References a, and b.

pmul< Packet2l >()

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::pmul< Packet2l >	(	const Packet2l &	a,
		const Packet2l &	b
	)

                                                                                  {
  return __lsx_vmul_d(a, b);
}

References a, and b.

pmul< Packet2ui >()

template<>

EIGEN_STRONG_INLINE Packet2ui Eigen::internal::pmul< Packet2ui >	(	const Packet2ui &	a,
		const Packet2ui &	b
	)

                                                                                      {
  return vmul_u32(a, b);
}

References a, and b.

pmul< Packet2ul >()

template<>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::pmul< Packet2ul >	(	const Packet2ul &	a,
		const Packet2ul &	b
	)

                                                                                      {
  return __lsx_vmul_d(a, b);
}

References a, and b.

pmul< Packet32h >()

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::pmul< Packet32h >	(	const Packet32h &	a,
		const Packet32h &	b
	)

                                                                                      {
  return _mm512_mul_ph(a, b);
}

References a, and b.

pmul< Packet4bf >()

template<>

EIGEN_STRONG_INLINE Packet4bf Eigen::internal::pmul< Packet4bf >	(	const Packet4bf &	a,
		const Packet4bf &	b
	)

                                                                                      {
  return F32ToBf16(pmul<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
}

References a, b, Bf16ToF32(), F32ToBf16(), and pmul< Packet4f >().

pmul< Packet4c >()

template<>

EIGEN_STRONG_INLINE Packet4c Eigen::internal::pmul< Packet4c >	(	const Packet4c &	a,
		const Packet4c &	b
	)

                                                                                  {
  return vget_lane_s32(
      vreinterpret_s32_s8(vmul_s8(vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
}

References a, and b.

pmul< Packet4cd >()

template<>

EIGEN_STRONG_INLINE Packet4cd Eigen::internal::pmul< Packet4cd >	(	const Packet4cd &	a,
		const Packet4cd &	b
	)

                                                                                      {
  __m512d tmp1 = _mm512_shuffle_pd(a.v, a.v, 0x0);
  __m512d tmp2 = _mm512_shuffle_pd(a.v, a.v, 0xFF);
  __m512d tmp3 = _mm512_shuffle_pd(b.v, b.v, 0x55);
  __m512d odd = _mm512_mul_pd(tmp2, tmp3);
  return Packet4cd(_mm512_fmaddsub_pd(tmp1, b.v, odd));
}

References a, and b.

pmul< Packet4d >()

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::pmul< Packet4d >	(	const Packet4d &	a,
		const Packet4d &	b
	)

                                                                                  {
  return _mm256_mul_pd(a, b);
}

References a, and b.

pmul< Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pmul< Packet4f >	(	const Packet4f &	a,
		const Packet4f &	b
	)

                                                                                  {
  return vec_madd(a, b, p4f_MZERO);
}

References a, b, and p4f_MZERO.

Referenced by pmul< Packet4bf >(), and pmul< Packet8bf >().

pmul< Packet4i >()

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pmul< Packet4i >	(	const Packet4i &	a,
		const Packet4i &	b
	)

                                                                                  {
  return a * b;
}

References a, and b.

pmul< Packet4s >()

template<>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::pmul< Packet4s >	(	const Packet4s &	a,
		const Packet4s &	b
	)

                                                                                  {
  return vmul_s16(a, b);
}

References a, and b.

pmul< Packet4uc >()

template<>

EIGEN_STRONG_INLINE Packet4uc Eigen::internal::pmul< Packet4uc >	(	const Packet4uc &	a,
		const Packet4uc &	b
	)

                                                                                      {
  return vget_lane_u32(
      vreinterpret_u32_u8(vmul_u8(vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
}

References a, and b.

pmul< Packet4ui >()

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::pmul< Packet4ui >	(	const Packet4ui &	a,
		const Packet4ui &	b
	)

                                                                                      {
  return __lsx_vmul_w(a, b);
}

References a, and b.

pmul< Packet4us >()

template<>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::pmul< Packet4us >	(	const Packet4us &	a,
		const Packet4us &	b
	)

                                                                                      {
  return vmul_u16(a, b);
}

References a, and b.

pmul< Packet8bf >()

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::pmul< Packet8bf >	(	const Packet8bf &	a,
		const Packet8bf &	b
	)

                                                                                      {
  BF16_TO_F32_BINARY_OP_WRAPPER(pmul<Packet4f>, a, b);
}

References a, b, BF16_TO_F32_BINARY_OP_WRAPPER, and pmul< Packet4f >().

pmul< Packet8c >()

template<>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::pmul< Packet8c >	(	const Packet8c &	a,
		const Packet8c &	b
	)

                                                                                  {
  return vmul_s8(a, b);
}

References a, and b.

pmul< Packet8cf >()

template<>

EIGEN_STRONG_INLINE Packet8cf Eigen::internal::pmul< Packet8cf >	(	const Packet8cf &	a,
		const Packet8cf &	b
	)

                                                                                      {
  __m512 tmp2 = _mm512_mul_ps(_mm512_movehdup_ps(a.v), _mm512_permute_ps(b.v, _MM_SHUFFLE(2, 3, 0, 1)));
  return Packet8cf(_mm512_fmaddsub_ps(_mm512_moveldup_ps(a.v), b.v, tmp2));
}

References a, and b.

pmul< Packet8d >()

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::pmul< Packet8d >	(	const Packet8d &	a,
		const Packet8d &	b
	)

                                                                                  {
  return _mm512_mul_pd(a, b);
}

References a, and b.

pmul< Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::pmul< Packet8f >	(	const Packet8f &	a,
		const Packet8f &	b
	)

                                                                                  {
  return _mm256_mul_ps(a, b);
}

References a, and b.

pmul< Packet8h >()

template<>

EIGEN_STRONG_INLINE Packet8h Eigen::internal::pmul< Packet8h >	(	const Packet8h &	a,
		const Packet8h &	b
	)

                                                                                  {
  Packet8f af = half2float(a);
  Packet8f bf = half2float(b);
  Packet8f rf = pmul(af, bf);
  return float2half(rf);
}

References a, b, float2half(), half2float(), and pmul().

pmul< Packet8i >()

template<>

EIGEN_STRONG_INLINE Packet8i Eigen::internal::pmul< Packet8i >	(	const Packet8i &	a,
		const Packet8i &	b
	)

                                                                                  {
#ifdef EIGEN_VECTORIZE_AVX2
  return _mm256_mullo_epi32(a, b);
#else
  const __m128i lo = _mm_mullo_epi32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0));
  const __m128i hi = _mm_mullo_epi32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1));
  return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1);
#endif
}

References a, and b.

pmul< Packet8l >()

template<>

EIGEN_STRONG_INLINE Packet8l Eigen::internal::pmul< Packet8l >	(	const Packet8l &	a,
		const Packet8l &	b
	)

                                                                                  {
#ifdef EIGEN_VECTORIZE_AVX512DQ
  return _mm512_mullo_epi64(a, b);
#else
  return _mm512_mullox_epi64(a, b);
#endif
}

References a, and b.

pmul< Packet8s >()

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::pmul< Packet8s >	(	const Packet8s &	a,
		const Packet8s &	b
	)

                                                                                  {
  return vec_mul(a, b);
}

References a, and b.

pmul< Packet8uc >()

template<>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::pmul< Packet8uc >	(	const Packet8uc &	a,
		const Packet8uc &	b
	)

                                                                                      {
  return vmul_u8(a, b);
}

References a, and b.

pmul< Packet8ui >()

template<>

EIGEN_STRONG_INLINE Packet8ui Eigen::internal::pmul< Packet8ui >	(	const Packet8ui &	a,
		const Packet8ui &	b
	)

                                                                                      {
#ifdef EIGEN_VECTORIZE_AVX2
  return _mm256_mullo_epi32(a, b);
#else
  const __m128i lo = _mm_mullo_epi32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0));
  const __m128i hi = _mm_mullo_epi32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1));
  return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1);
#endif
}

References a, and b.

pmul< Packet8us >()

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::pmul< Packet8us >	(	const Packet8us &	a,
		const Packet8us &	b
	)

                                                                                      {
  return vec_mul(a, b);
}

References a, and b.

pmul< PacketXf >()

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::pmul< PacketXf >	(	const PacketXf &	a,
		const PacketXf &	b
	)

                                                                                  {
  return svmul_f32_x(svptrue_b32(), a, b);
}

References a, and b.

pmul< PacketXi >()

template<>

EIGEN_STRONG_INLINE PacketXi Eigen::internal::pmul< PacketXi >	(	const PacketXi &	a,
		const PacketXi &	b
	)

                                                                                  {
  return svmul_s32_x(svptrue_b32(), a, b);
}

References a, and b.

pndtri()

template<typename Packet >

EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::pndtri

(

const Packet &

a

)

Returns

the ndtri(a) (coeff-wise)

                                                                                    {
  typedef typename unpacket_traits<Packet>::type ScalarType;
  using internal::generic_ndtri;
  return generic_ndtri<Packet, ScalarType>(a);
}

References a, and generic_ndtri().

Referenced by packetmath_real(), and Eigen::internal::scalar_ndtri_op< Scalar >::packetOp().

pnegate() [1/32]

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::pnegate ( const Packet &  a )

inline

Returns

-a (coeff-wise)

                                                         {
  EIGEN_STATIC_ASSERT((!is_same<typename unpacket_traits<Packet>::type, bool>::value),
                      NEGATE IS NOT DEFINED FOR BOOLEAN TYPES)
  return numext::negate(a);
}

References a, EIGEN_STATIC_ASSERT, Eigen::test::negate(), and Eigen::value.

pnegate() [2/32]

template<>

EIGEN_STRONG_INLINE Packet16bf Eigen::internal::pnegate

(

const Packet16bf &

a

)

                                                            {
  Packet16bf sign_mask = _mm256_set1_epi16(static_cast<unsigned short>(0x8000));
  return _mm256_xor_si256(a, sign_mask);
}

References a.

pnegate() [3/32]

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::pnegate

(

const Packet16c &

a

)

                                                          {
#ifdef __POWER8_VECTOR__
  return vec_neg(a);
#else
  return reinterpret_cast<Packet16c>(p4i_ZERO) - a;
#endif
}

References a.

pnegate() [4/32]

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::pnegate

(

const Packet16f &

a

)

                                                          {
  // NOTE: MSVC seems to struggle with _mm512_set1_epi32, leading to random results.
  //       The intel docs give it a relatively high latency as well, so we're probably
  //       better off with using _mm512_set_epi32 directly anyways.
  const __m512i mask =
      _mm512_set_epi32(0x80000000, 0x80000000, 0x80000000, 0x80000000, 0x80000000, 0x80000000, 0x80000000, 0x80000000,
                       0x80000000, 0x80000000, 0x80000000, 0x80000000, 0x80000000, 0x80000000, 0x80000000, 0x80000000);
  return _mm512_castsi512_ps(_mm512_xor_epi32(_mm512_castps_si512(a), mask));
}

References a.

pnegate() [5/32]

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::pnegate

(

const Packet16h &

a

)

                                                          {
  Packet16h sign_mask = _mm256_set1_epi16(static_cast<unsigned short>(0x8000));
  return _mm256_xor_si256(a, sign_mask);
}

References a.

pnegate() [6/32]

template<>

EIGEN_STRONG_INLINE Packet16i Eigen::internal::pnegate

(

const Packet16i &

a

)

                                                          {
  return _mm512_sub_epi32(_mm512_setzero_si512(), a);
}

References a.

pnegate() [7/32]

template<>

EIGEN_STRONG_INLINE Packet1cd Eigen::internal::pnegate

(

const Packet1cd &

a

)

                                                          {
  return Packet1cd(pnegate(Packet2d(a.v)));
}

References a, and pnegate().

pnegate() [8/32]

template<>

EIGEN_STRONG_INLINE Packet1cf Eigen::internal::pnegate

(

const Packet1cf &

a

)

                                                          {
  return Packet1cf(pnegate<Packet2f>(a.v));
}

References a.

pnegate() [9/32]

template<>

EIGEN_STRONG_INLINE Packet2cd Eigen::internal::pnegate

(

const Packet2cd &

a

)

                                                          {
  return Packet2cd(pnegate(a.v));
}

References a, and pnegate().

pnegate() [10/32]

template<>

EIGEN_STRONG_INLINE Packet2cf Eigen::internal::pnegate

(

const Packet2cf &

a

)

                                                          {
  return Packet2cf(pnegate(a.v));
}

References a.

Referenced by erfc_double_large(), generic_pow(), Eigen::internal::Packet2cf::operator-(), Eigen::internal::Packet1cd::operator-(), packetmath_real(), Eigen::internal::scalar_opposite_op< Scalar >::packetOp(), pblend(), pblend4(), pldexp< Packet4d >(), pldexp< Packet8d >(), pldexp_generic(), pmsub(), pnegate(), pnmsub(), Eigen::internal::generic_i1e< T, float >::run(), Eigen::internal::generic_i1e< T, double >::run(), Eigen::internal::generic_k0e< T, float >::run(), Eigen::internal::generic_k0< T, float >::run(), Eigen::internal::generic_k0< T, double >::run(), Eigen::internal::generic_k1< T, float >::run(), Eigen::internal::generic_j1< T, float >::run(), Eigen::internal::generic_j1< T, double >::run(), Eigen::internal::generic_fast_erfc< Scalar >::run(), negate_test_impl< Scalar, Packet, HasNegate >::run_negate(), twoprod(), and twoprod_low().

pnegate() [11/32]

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pnegate

(

const Packet2d &

a

)

                                                        {
  Packet2d mask =
      make_packet2d(numext::bit_cast<double>(0x8000000000000000), numext::bit_cast<double>(0x8000000000000000));
  return (Packet2d)__lsx_vxor_v(numext::bit_cast<__m128i>(mask), numext::bit_cast<__m128i>(a));
}

References a, and make_packet2d().

pnegate() [12/32]

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::pnegate

(

const Packet2f &

a

)

                                                        {
  return vneg_f32(a);
}

References a.

pnegate() [13/32]

template<>

EIGEN_STRONG_INLINE Packet2i Eigen::internal::pnegate

(

const Packet2i &

a

)

                                                        {
  return vneg_s32(a);
}

References a.

pnegate() [14/32]

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::pnegate

(

const Packet2l &

a

)

                                                        {
  return __lsx_vneg_d(a);
}

References a.

pnegate() [15/32]

template<>

EIGEN_STRONG_INLINE Packet4c Eigen::internal::pnegate

(

const Packet4c &

a

)

                                                        {
  return vget_lane_s32(vreinterpret_s32_s8(vneg_s8(vreinterpret_s8_s32(vdup_n_s32(a)))), 0);
}

References a.

pnegate() [16/32]

template<>

EIGEN_STRONG_INLINE Packet4cd Eigen::internal::pnegate

(

const Packet4cd &

a

)

                                                          {
  return Packet4cd(pnegate(a.v));
}

References a, and pnegate().

pnegate() [17/32]

template<>

EIGEN_STRONG_INLINE Packet4cf Eigen::internal::pnegate

(

const Packet4cf &

a

)

                                                          {
  return Packet4cf(pnegate(a.v));
}

References a, and pnegate().

pnegate() [18/32]

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::pnegate

(

const Packet4d &

a

)

                                                        {
  const Packet4d mask = _mm256_castsi256_pd(_mm256_set1_epi64x(0x8000000000000000ULL));
  return _mm256_xor_pd(a, mask);
}

References a.

pnegate() [19/32]

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pnegate

(

const Packet4f &

a

)

                                                        {
#ifdef __POWER8_VECTOR__
  return vec_neg(a);
#else
  return vec_xor(a, p4f_MZERO);
#endif
}

References a, and p4f_MZERO.

pnegate() [20/32]

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pnegate

(

const Packet4i &

a

)

                                                        {
#ifdef __POWER8_VECTOR__
  return vec_neg(a);
#else
  return p4i_ZERO - a;
#endif
}

References a.

pnegate() [21/32]

template<>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::pnegate

(

const Packet4s &

a

)

                                                        {
  return vneg_s16(a);
}

References a.

pnegate() [22/32]

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::pnegate

(

const Packet8bf &

a

)

                                                          {
  Packet8bf sign_mask = _mm_set1_epi16(static_cast<numext::uint16_t>(0x8000));
  return _mm_xor_si128(a, sign_mask);
}

References a.

pnegate() [23/32]

template<>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::pnegate

(

const Packet8c &

a

)

                                                        {
  return vneg_s8(a);
}

References a.

pnegate() [24/32]

template<>

EIGEN_STRONG_INLINE Packet8cf Eigen::internal::pnegate

(

const Packet8cf &

a

)

                                                          {
  return Packet8cf(pnegate(a.v));
}

References a, and pnegate().

pnegate() [25/32]

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::pnegate

(

const Packet8d &

a

)

                                                        {
  const __m512i mask =
      _mm512_set_epi64(0x8000000000000000ULL, 0x8000000000000000ULL, 0x8000000000000000ULL, 0x8000000000000000ULL,
                       0x8000000000000000ULL, 0x8000000000000000ULL, 0x8000000000000000ULL, 0x8000000000000000ULL);
  return _mm512_castsi512_pd(_mm512_xor_epi64(_mm512_castpd_si512(a), mask));
}

References a.

pnegate() [26/32]

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::pnegate

(

const Packet8f &

a

)

                                                        {
  const Packet8f mask = _mm256_castsi256_ps(_mm256_set1_epi32(0x80000000));
  return _mm256_xor_ps(a, mask);
}

References a.

pnegate() [27/32]

template<>

EIGEN_STRONG_INLINE Packet8h Eigen::internal::pnegate

(

const Packet8h &

a

)

                                                        {
  Packet8h sign_mask = _mm_set1_epi16(static_cast<numext::uint16_t>(0x8000));
  return _mm_xor_si128(a, sign_mask);
}

References a.

pnegate() [28/32]

template<>

EIGEN_STRONG_INLINE Packet8i Eigen::internal::pnegate

(

const Packet8i &

a

)

                                                        {
  return psub(pzero(a), a);
}

References a, psub(), and pzero().

pnegate() [29/32]

template<>

EIGEN_STRONG_INLINE Packet8l Eigen::internal::pnegate

(

const Packet8l &

a

)

                                                        {
  return _mm512_sub_epi64(_mm512_setzero_si512(), a);
}

References a.

pnegate() [30/32]

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::pnegate

(

const Packet8s &

a

)

                                                        {
#ifdef __POWER8_VECTOR__
  return vec_neg(a);
#else
  return reinterpret_cast<Packet8s>(p4i_ZERO) - a;
#endif
}

References a.

pnegate() [31/32]

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::pnegate

(

const PacketXf &

a

)

                                                        {
  return svneg_f32_x(svptrue_b32(), a);
}

References a.

pnegate() [32/32]

template<>

EIGEN_STRONG_INLINE PacketXi Eigen::internal::pnegate

(

const PacketXi &

a

)

                                                        {
  return svneg_s32_x(svptrue_b32(), a);
}

References a.

pnegate< Packet32h >()

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::pnegate< Packet32h >

(

const Packet32h &

a

)

                                                                     {
  return psub(pzero(a), a);
}

References a, psub(), and pzero().

pnegate< Packet4bf >()

template<>

EIGEN_STRONG_INLINE Packet4bf Eigen::internal::pnegate< Packet4bf >

(

const Packet4bf &

a

)

                                                                     {
  return Packet4bf(pxor<Packet4us>(Packet4us(a), pset1<Packet4us>(static_cast<uint16_t>(0x8000))));
}

References a, pset1< Packet4us >(), and pxor< Packet4us >().

pnegate< Packet8bf >()

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::pnegate< Packet8bf >

(

const Packet8bf &

a

)

                                                                     {
  EIGEN_DECLARE_CONST_FAST_Packet8us(neg_mask, 0x8000);
  return pxor<Packet8us>(p8us_neg_mask, a);
}

References a, EIGEN_DECLARE_CONST_FAST_Packet8us(), and pxor< Packet8us >().

pnmadd() [1/7]

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::pnmadd ( const Packet &  a, const Packet &  b, const Packet &  c  )

inline

Returns

-(a * b) + c (coeff-wise)

                                                                                          {
  return psub(c, pmul(a, b));
}

References a, b, calibrate::c, pmul(), and psub().

pnmadd() [2/7]

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::pnmadd	(	const Packet16c &	a,
		const Packet16c &	b,
		const Packet16c &	c
	)

                                                                                                 {
  return __lsx_vmsub_b(c, a, b);
}

References a, b, and calibrate::c.

pnmadd() [3/7]

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pnmadd	(	const Packet2d &	a,
		const Packet2d &	b,
		const Packet2d &	c
	)

                                                                                             {
  return __lsx_vfnmsub_d(a, b, c);
}

References a, b, and calibrate::c.

pnmadd() [4/7]

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::pnmadd	(	const Packet2l &	a,
		const Packet2l &	b,
		const Packet2l &	c
	)

                                                                                             {
  return __lsx_vmsub_d(c, a, b);
}

References a, b, and calibrate::c.

pnmadd() [5/7]

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pnmadd	(	const Packet4f &	a,
		const Packet4f &	b,
		const Packet4f &	c
	)

                                                                                             {
  return __lsx_vfnmsub_s(a, b, c);
}

References a, b, and calibrate::c.

Referenced by erfc_double_large(), packetmath(), pasin_float(), pldexp_generic(), Eigen::internal::generic_reciprocal_newton_step< Packet, Steps >::run(), and Eigen::internal::generic_fast_erfc< Scalar >::run().

pnmadd() [6/7]

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pnmadd	(	const Packet4i &	a,
		const Packet4i &	b,
		const Packet4i &	c
	)

                                                                                             {
  return __lsx_vmsub_w(c, a, b);
}

References a, b, and calibrate::c.

pnmadd() [7/7]

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::pnmadd	(	const Packet8s &	a,
		const Packet8s &	b,
		const Packet8s &	c
	)

                                                                                             {
  return __lsx_vmsub_h(c, a, b);
}

References a, b, and calibrate::c.

pnmsub() [1/7]

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::pnmsub ( const Packet &  a, const Packet &  b, const Packet &  c  )

inline

Returns

-((a * b + c) (coeff-wise)

                                                                                          {
  return pnegate(pmadd(a, b, c));
}

References a, b, calibrate::c, pmadd(), and pnegate().

pnmsub() [2/7]

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::pnmsub	(	const Packet16c &	a,
		const Packet16c &	b,
		const Packet16c &	c
	)

                                                                                                 {
  return __lsx_vmsub_b(pnegate(c), a, b);
}

References a, b, calibrate::c, and pnegate().

pnmsub() [3/7]

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pnmsub	(	const Packet2d &	a,
		const Packet2d &	b,
		const Packet2d &	c
	)

                                                                                             {
  return __lsx_vfnmadd_d(a, b, c);
}

References a, b, and calibrate::c.

pnmsub() [4/7]

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::pnmsub	(	const Packet2l &	a,
		const Packet2l &	b,
		const Packet2l &	c
	)

                                                                                             {
  return __lsx_vmsub_d(pnegate(c), a, b);
}

References a, b, calibrate::c, and pnegate().

pnmsub() [5/7]

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pnmsub	(	const Packet4f &	a,
		const Packet4f &	b,
		const Packet4f &	c
	)

                                                                                             {
  return __lsx_vfnmadd_s(a, b, c);
}

References a, b, and calibrate::c.

Referenced by negate_test_impl< Scalar, Packet, HasNegate >::run_nmsub().

pnmsub() [6/7]

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pnmsub	(	const Packet4i &	a,
		const Packet4i &	b,
		const Packet4i &	c
	)

                                                                                             {
  return __lsx_vmsub_w(pnegate(c), a, b);
}

References a, b, calibrate::c, and pnegate().

pnmsub() [7/7]

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::pnmsub	(	const Packet8s &	a,
		const Packet8s &	b,
		const Packet8s &	c
	)

                                                                                             {
  return __lsx_vmsub_h(pnegate(c), a, b);
}

References a, b, calibrate::c, and pnegate().

pnot()

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::pnot ( const Packet &  a )

inline

Returns

the bitwise not of a

                                                      {
  return bitwise_helper<Packet>::bitwise_not(a);
}

References a, and Eigen::internal::bytewise_bitwise_helper< T >::bitwise_not().

Referenced by Eigen::internal::minmax_coeff_visitor< Derived, is_min, NaNPropagation, false >::initpacket(), Eigen::internal::minmax_coeff_visitor< Derived, is_min, NaNPropagation, false >::packet(), and pandnot().

por() [1/7]

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::por ( const Packet &  a, const Packet &  b  )

inline

Returns

the bitwise or of a and b

                                                                      {
  return bitwise_helper<Packet>::bitwise_or(a, b);
}

References a, b, and Eigen::internal::bytewise_bitwise_helper< T >::bitwise_or().

por() [2/7]

template<>

EIGEN_STRONG_INLINE Packet16bf Eigen::internal::por	(	const Packet16bf &	a,
		const Packet16bf &	b
	)

                                                                             {
  return Packet16bf(por<Packet8i>(Packet8i(a), Packet8i(b)));
}

References a, b, and por< Packet8i >().

por() [3/7]

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::por	(	const Packet16h &	a,
		const Packet16h &	b
	)

                                                                          {
  // in some cases Packet8i is a wrapper around __m256i, so we need to
  // cast to Packet8i to call the correct overload.
  return Packet16h(por(Packet8i(a), Packet8i(b)));
}

References a, b, and por().

por() [4/7]

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::por	(	const Packet32h &	a,
		const Packet32h &	b
	)

                                                                          {
  return _mm512_castsi512_ph(_mm512_or_si512(_mm512_castph_si512(a), _mm512_castph_si512(b)));
}

References a, and b.

por() [5/7]

template<>

EIGEN_STRONG_INLINE Packet4bf Eigen::internal::por	(	const Packet4bf &	a,
		const Packet4bf &	b
	)

                                                                          {
  return Packet4bf(por<Packet4us>(Packet4us(a), Packet4us(b)));
}

References a, b, and por< Packet4us >().

por() [6/7]

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::por	(	const Packet8bf &	a,
		const Packet8bf &	b
	)

                                                                          {
  return _mm_or_si128(a, b);
}

References a, and b.

por() [7/7]

template<>

EIGEN_STRONG_INLINE Packet8h Eigen::internal::por	(	const Packet8h &	a,
		const Packet8h &	b
	)

                                                                       {
  // in some cases Packet4i is a wrapper around __m128i, so we either need to
  // cast to Packet4i to directly call the intrinsics as below:
  return _mm_or_si128(a, b);
}

References a, and b.

Referenced by generic_ceil(), generic_log1p(), generic_pow(), generic_rint(), generic_round(), generic_trunc(), Eigen::internal::unary_pow::handle_nonint_nonint_errors(), packetmath(), Eigen::internal::scalar_cmp_op< LhsScalar, RhsScalar, cmp_UNORD, UseTypedComparators >::packetOp(), Eigen::internal::scalar_boolean_and_op< Scalar >::packetOp(), Eigen::internal::scalar_boolean_or_op< Scalar >::packetOp(), Eigen::internal::scalar_bitwise_or_op< Scalar >::packetOp(), pasin_float(), patanh_double(), patanh_float(), pcmp_le(), pcmp_lt(), pexp_complex(), pfrexp_generic(), phypot_complex(), plog_complex(), plog_impl_double(), plog_impl_float(), por(), por< Packet16f >(), por< Packet4cd >(), por< Packet8cf >(), por< Packet8d >(), pround(), pround< Packet16f >(), pround< Packet32h >(), pround< Packet4d >(), pround< Packet8d >(), pround< Packet8f >(), prsqrt_float_common(), psqrt_complex(), psqrt_float_common(), Eigen::internal::psign_impl< Packet, std::enable_if_t<!NumTraits< typename unpacket_traits< Packet >::type >::IsComplex &&!NumTraits< typename unpacket_traits< Packet >::type >::IsInteger > >::run(), Eigen::internal::psign_impl< Packet, std::enable_if_t<!NumTraits< typename unpacket_traits< Packet >::type >::IsComplex &&NumTraits< typename unpacket_traits< Packet >::type >::IsSigned &&NumTraits< typename unpacket_traits< Packet >::type >::IsInteger > >::run(), Eigen::internal::psignbit_impl< Packet, false, false >::run(), Eigen::internal::generic_sqrt_newton_step< Packet, Steps >::run(), and Eigen::internal::pselect_impl< Packet, EnableIf >::run().

por< Packet16b >()

template<>

EIGEN_STRONG_INLINE Packet16b Eigen::internal::por< Packet16b >	(	const Packet16b &	a,
		const Packet16b &	b
	)

                                                                                     {
  return _mm_or_si128(a, b);
}

References a, and b.

por< Packet16c >()

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::por< Packet16c >	(	const Packet16c &	a,
		const Packet16c &	b
	)

                                                                                     {
  return __lsx_vor_v(a, b);
}

References a, and b.

por< Packet16f >()

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::por< Packet16f >	(	const Packet16f &	a,
		const Packet16f &	b
	)

                                                                                     {
#ifdef EIGEN_VECTORIZE_AVX512DQ
  return _mm512_or_ps(a, b);
#else
  return _mm512_castsi512_ps(por(_mm512_castps_si512(a), _mm512_castps_si512(b)));
#endif
}

References a, b, and por().

por< Packet16i >()

template<>

EIGEN_STRONG_INLINE Packet16i Eigen::internal::por< Packet16i >	(	const Packet16i &	a,
		const Packet16i &	b
	)

                                                                                     {
  return _mm512_or_si512(a, b);
}

References a, and b.

por< Packet16uc >()

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::por< Packet16uc >	(	const Packet16uc &	a,
		const Packet16uc &	b
	)

                                                                                         {
  return __lsx_vor_v(a, b);
}

References a, and b.

por< Packet1cd >()

template<>

EIGEN_STRONG_INLINE Packet1cd Eigen::internal::por< Packet1cd >	(	const Packet1cd &	a,
		const Packet1cd &	b
	)

                                                                                     {
  Packet1cd res;
  res.v = (Packet2d)__lsx_vor_v((__m128i)a.v, (__m128i)b.v);
  return res;
}

References a, b, and res.

por< Packet1cf >()

template<>

EIGEN_STRONG_INLINE Packet1cf Eigen::internal::por< Packet1cf >	(	const Packet1cf &	a,
		const Packet1cf &	b
	)

                                                                                     {
  return Packet1cf(vreinterpret_f32_u32(vorr_u32(vreinterpret_u32_f32(a.v), vreinterpret_u32_f32(b.v))));
}

References a, and b.

por< Packet2cd >()

template<>

EIGEN_STRONG_INLINE Packet2cd Eigen::internal::por< Packet2cd >	(	const Packet2cd &	a,
		const Packet2cd &	b
	)

                                                                                     {
  return Packet2cd(_mm256_or_pd(a.v, b.v));
}

References a, and b.

por< Packet2cf >()

template<>

EIGEN_STRONG_INLINE Packet2cf Eigen::internal::por< Packet2cf >	(	const Packet2cf &	a,
		const Packet2cf &	b
	)

                                                                                     {
  return Packet2cf(por<Packet4f>(a.v, b.v));
}

References a, b, and por< Packet4f >().

por< Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::por< Packet2d >	(	const Packet2d &	a,
		const Packet2d &	b
	)

                                                                                 {
  return (Packet2d)__lsx_vor_v((__m128i)a, (__m128i)b);
}

References a, and b.

por< Packet2f >()

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::por< Packet2f >	(	const Packet2f &	a,
		const Packet2f &	b
	)

                                                                                 {
  return vreinterpret_f32_u32(vorr_u32(vreinterpret_u32_f32(a), vreinterpret_u32_f32(b)));
}

References a, and b.

por< Packet2i >()

template<>

EIGEN_STRONG_INLINE Packet2i Eigen::internal::por< Packet2i >	(	const Packet2i &	a,
		const Packet2i &	b
	)

                                                                                 {
  return vorr_s32(a, b);
}

References a, and b.

por< Packet2l >()

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::por< Packet2l >	(	const Packet2l &	a,
		const Packet2l &	b
	)

                                                                                 {
  return __lsx_vor_v(a, b);
}

References a, and b.

Referenced by pmax< Packet2d >(), and pmin< Packet2d >().

por< Packet2ui >()

template<>

EIGEN_STRONG_INLINE Packet2ui Eigen::internal::por< Packet2ui >	(	const Packet2ui &	a,
		const Packet2ui &	b
	)

                                                                                     {
  return vorr_u32(a, b);
}

References a, and b.

por< Packet2ul >()

template<>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::por< Packet2ul >	(	const Packet2ul &	a,
		const Packet2ul &	b
	)

                                                                                     {
  return __lsx_vor_v(a, b);
}

References a, and b.

por< Packet4c >()

template<>

EIGEN_STRONG_INLINE Packet4c Eigen::internal::por< Packet4c >	(	const Packet4c &	a,
		const Packet4c &	b
	)

                                                                                 {
  return a | b;
}

References a, and b.

por< Packet4cd >()

template<>

EIGEN_STRONG_INLINE Packet4cd Eigen::internal::por< Packet4cd >	(	const Packet4cd &	a,
		const Packet4cd &	b
	)

                                                                                     {
  return Packet4cd(por(a.v, b.v));
}

References a, b, and por().

por< Packet4cf >()

template<>

EIGEN_STRONG_INLINE Packet4cf Eigen::internal::por< Packet4cf >	(	const Packet4cf &	a,
		const Packet4cf &	b
	)

                                                                                     {
  return Packet4cf(_mm256_or_ps(a.v, b.v));
}

References a, and b.

por< Packet4d >()

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::por< Packet4d >	(	const Packet4d &	a,
		const Packet4d &	b
	)

                                                                                 {
  return _mm256_or_pd(a, b);
}

References a, and b.

por< Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::por< Packet4f >	(	const Packet4f &	a,
		const Packet4f &	b
	)

                                                                                 {
  return vec_or(a, b);
}

References a, and b.

Referenced by por< Packet2cf >().

por< Packet4i >()

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::por< Packet4i >	(	const Packet4i &	a,
		const Packet4i &	b
	)

                                                                                 {
  return vec_or(a, b);
}

References a, and b.

por< Packet4s >()

template<>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::por< Packet4s >	(	const Packet4s &	a,
		const Packet4s &	b
	)

                                                                                 {
  return vorr_s16(a, b);
}

References a, and b.

por< Packet4uc >()

template<>

EIGEN_STRONG_INLINE Packet4uc Eigen::internal::por< Packet4uc >	(	const Packet4uc &	a,
		const Packet4uc &	b
	)

                                                                                     {
  return a | b;
}

References a, and b.

por< Packet4ui >()

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::por< Packet4ui >	(	const Packet4ui &	a,
		const Packet4ui &	b
	)

                                                                                     {
  return __lsx_vor_v(a, b);
}

References a, and b.

por< Packet4us >()

template<>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::por< Packet4us >	(	const Packet4us &	a,
		const Packet4us &	b
	)

                                                                                     {
  return vorr_u16(a, b);
}

References a, and b.

Referenced by por().

por< Packet8bf >()

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::por< Packet8bf >	(	const Packet8bf &	a,
		const Packet8bf &	b
	)

                                                                                     {
  return por<Packet8us>(a, b);
}

References a, b, and por< Packet8us >().

por< Packet8c >()

template<>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::por< Packet8c >	(	const Packet8c &	a,
		const Packet8c &	b
	)

                                                                                 {
  return vorr_s8(a, b);
}

References a, and b.

por< Packet8cf >()

template<>

EIGEN_STRONG_INLINE Packet8cf Eigen::internal::por< Packet8cf >	(	const Packet8cf &	a,
		const Packet8cf &	b
	)

                                                                                     {
  return Packet8cf(por(a.v, b.v));
}

References a, b, and por().

por< Packet8d >()

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::por< Packet8d >	(	const Packet8d &	a,
		const Packet8d &	b
	)

                                                                                 {
#ifdef EIGEN_VECTORIZE_AVX512DQ
  return _mm512_or_pd(a, b);
#else
  return _mm512_castsi512_pd(por(_mm512_castpd_si512(a), _mm512_castpd_si512(b)));
#endif
}

References a, b, and por().

por< Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::por< Packet8f >	(	const Packet8f &	a,
		const Packet8f &	b
	)

                                                                                 {
  return _mm256_or_ps(a, b);
}

References a, and b.

por< Packet8i >()

template<>

EIGEN_STRONG_INLINE Packet8i Eigen::internal::por< Packet8i >	(	const Packet8i &	a,
		const Packet8i &	b
	)

                                                                                 {
#ifdef EIGEN_VECTORIZE_AVX2
  return _mm256_or_si256(a, b);
#else
  return _mm256_castps_si256(_mm256_or_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b)));
#endif
}

References a, and b.

Referenced by ploadu< Packet8f >(), por(), and pstoreu< float >().

por< Packet8l >()

template<>

EIGEN_STRONG_INLINE Packet8l Eigen::internal::por< Packet8l >	(	const Packet8l &	a,
		const Packet8l &	b
	)

                                                                                 {
  return _mm512_or_si512(a, b);
}

References a, and b.

por< Packet8s >()

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::por< Packet8s >	(	const Packet8s &	a,
		const Packet8s &	b
	)

                                                                                 {
  return vec_or(a, b);
}

References a, and b.

por< Packet8uc >()

template<>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::por< Packet8uc >	(	const Packet8uc &	a,
		const Packet8uc &	b
	)

                                                                                     {
  return vorr_u8(a, b);
}

References a, and b.

por< Packet8ui >()

template<>

EIGEN_STRONG_INLINE Packet8ui Eigen::internal::por< Packet8ui >	(	const Packet8ui &	a,
		const Packet8ui &	b
	)

                                                                                     {
#ifdef EIGEN_VECTORIZE_AVX2
  return _mm256_or_si256(a, b);
#else
  return _mm256_castps_si256(_mm256_or_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b)));
#endif
}

References a, and b.

por< Packet8us >()

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::por< Packet8us >	(	const Packet8us &	a,
		const Packet8us &	b
	)

                                                                                     {
  return vec_or(a, b);
}

References a, and b.

Referenced by por< Packet8bf >().

por< PacketXf >()

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::por< PacketXf >	(	const PacketXf &	a,
		const PacketXf &	b
	)

                                                                                 {
  return svreinterpret_f32_u32(svorr_u32_x(svptrue_b32(), svreinterpret_u32_f32(a), svreinterpret_u32_f32(b)));
}

References a, and b.

por< PacketXi >()

template<>

EIGEN_STRONG_INLINE PacketXi Eigen::internal::por< PacketXi >	(	const PacketXi &	a,
		const PacketXi &	b
	)

                                                                                 {
  return svorr_s32_x(svptrue_b32(), a, b);
}

References a, and b.

positive_real_hypot()

template<typename RealScalar >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE RealScalar Eigen::internal::positive_real_hypot	(	const RealScalar &	x,
		const RealScalar &	y
	)

                                                                                                               {
  // IEEE IEC 6059 special cases.
  if ((numext::isinf)(x) || (numext::isinf)(y)) return NumTraits<RealScalar>::infinity();
  if ((numext::isnan)(x) || (numext::isnan)(y)) return NumTraits<RealScalar>::quiet_NaN();
 
  EIGEN_USING_STD(sqrt);
  RealScalar p, qp;
  p = numext::maxi(x, y);
  if (numext::is_exactly_zero(p)) return RealScalar(0);
  qp = numext::mini(y, x) / p;
  return p * sqrt(RealScalar(1) + qp * qp);
}

References EIGEN_USING_STD, Eigen::numext::is_exactly_zero(), Eigen::numext::isinf(), Eigen::numext::isnan(), Eigen::numext::maxi(), Eigen::numext::mini(), p, sqrt(), plotDoE::x, and y.

Referenced by Eigen::internal::scalar_hypot_op< Scalar, Scalar >::operator()().

ppolygamma()

template<typename Packet >

EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::ppolygamma	(	const Packet &	n,
		const Packet &	x
	)

Returns

the polygamma function (coeff-wise)

                                                                                                         {
  using numext::polygamma;
  return polygamma(n, x);
}

References n, Eigen::polygamma(), and plotDoE::x.

Referenced by Eigen::internal::scalar_polygamma_op< Scalar >::packetOp().

preciprocal()

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::preciprocal ( const Packet &  a )

inline

Returns

1 / a (coeff-wise)

                                                             {
  using Scalar = typename unpacket_traits<Packet>::type;
  return pdiv(pset1<Packet>(Scalar(1)), a);
}

References a, and pdiv().

Referenced by erfc_double_large(), generic_atan(), generic_ndtri_lt_exp_neg_two(), packetmath(), packetmath_real(), Eigen::internal::scalar_inverse_op< Scalar >::packetOp(), pdiv_float_common(), Eigen::internal::compute_inverse_size4< Architecture::Target, float, MatrixType, ResultType >::run(), and Eigen::internal::generic_fast_erfc< Scalar >::run().

preciprocal< Packet2f >()

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::preciprocal< Packet2f >

(

const Packet2f &

a

)

                                                                      {
  // Compute approximate reciprocal.
  float32x2_t result = vrecpe_f32(a);
  result = vmul_f32(vrecps_f32(a, result), result);
  result = vmul_f32(vrecps_f32(a, result), result);
  return result;
}

References a.

preciprocal< Packet32h >()

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::preciprocal< Packet32h >

(

const Packet32h &

a

)

                                                                         {
  return _mm512_rcp_ph(a);
}

References a.

preciprocal< Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::preciprocal< Packet4f >

(

const Packet4f &

a

)

                                                                      {
  return __lsx_vfrecip_s(a);
}

References a.

predux()

template<typename Packet >

EIGEN_DEVICE_FUNC unpacket_traits<Packet>::type Eigen::internal::predux ( const Packet &  a )

inline

Returns

the sum of the elements of a

                                                                                      {
  return a;
}

References a.

Referenced by Eigen::internal::count_visitor< Scalar >::count_redux(), Eigen::internal::SumReducer< T >::finalizeBoth(), Eigen::internal::MeanReducer< T >::finalizeBoth(), gemv_row(), packetmath(), pblueNorm(), Eigen::internal::scalar_sum_op< LhsScalar, RhsScalar >::predux(), predux< Packet16h >(), predux< Packet2cd >(), predux< Packet4cd >(), predux< Packet4cf >(), predux< Packet4d >(), predux< Packet8cf >(), predux< Packet8f >(), predux< Packet8i >(), predux< Packet8ui >(), predux_any(), predux_mul< Packet2cd >(), predux_real(), Eigen::internal::inner_product_impl< Evaluator, true >::run(), Eigen::internal::general_matrix_vector_product< Index, LhsScalar, LhsMapper, RowMajor, ConjugateLhs, RhsScalar, RhsMapper, ConjugateRhs, Version >::run(), and Eigen::internal::selfadjoint_matrix_vector_product< Scalar, Index, StorageOrder, UpLo, ConjugateLhs, ConjugateRhs, Version >::run().

predux< Packet16b >()

template<>

EIGEN_STRONG_INLINE bool Eigen::internal::predux< Packet16b >

(

const Packet16b &

a

)

                                                               {
  Packet4i tmp = _mm_or_si128(a, _mm_unpackhi_epi64(a, a));
  return (pfirst(tmp) != 0) || (pfirst<Packet4i>(_mm_shuffle_epi32(tmp, 1)) != 0);
}

References a, pfirst(), pfirst< Packet4i >(), and tmp.

predux< Packet16bf >()

template<>

EIGEN_STRONG_INLINE bfloat16 Eigen::internal::predux< Packet16bf >

(

const Packet16bf &

p

)

                                                                     {
  return static_cast<bfloat16>(predux<Packet16f>(Bf16ToF32(p)));
}

References Bf16ToF32(), p, and predux< Packet16f >().

predux< Packet16c >()

template<>

EIGEN_STRONG_INLINE int8_t Eigen::internal::predux< Packet16c >

(

const Packet16c &

a

)

                                                                      {
  return predux_size16<Packet16c>(a);
}

References a.

predux< Packet16f >()

template<>

EIGEN_STRONG_INLINE float Eigen::internal::predux< Packet16f >

(

const Packet16f &

a

)

                                                                {
#ifdef EIGEN_VECTORIZE_AVX512DQ
  __m256 lane0 = _mm512_extractf32x8_ps(a, 0);
  __m256 lane1 = _mm512_extractf32x8_ps(a, 1);
  Packet8f x = _mm256_add_ps(lane0, lane1);
  return predux<Packet8f>(x);
#else
  __m128 lane0 = _mm512_extractf32x4_ps(a, 0);
  __m128 lane1 = _mm512_extractf32x4_ps(a, 1);
  __m128 lane2 = _mm512_extractf32x4_ps(a, 2);
  __m128 lane3 = _mm512_extractf32x4_ps(a, 3);
  __m128 sum = _mm_add_ps(_mm_add_ps(lane0, lane1), _mm_add_ps(lane2, lane3));
  return predux<Packet4f>(sum);
#endif
}

References a, predux< Packet4f >(), predux< Packet8f >(), and plotDoE::x.

Referenced by predux< Packet16bf >().

predux< Packet16h >()

template<>

EIGEN_STRONG_INLINE half Eigen::internal::predux< Packet16h >

(

const Packet16h &

from

)

                                                                  {
  Packet16f from_float = half2float(from);
  return half(predux(from_float));
}

References half2float(), and predux().

predux< Packet16i >()

template<>

EIGEN_STRONG_INLINE int Eigen::internal::predux< Packet16i >

(

const Packet16i &

a

)

                                                              {
  return _mm512_reduce_add_epi32(a);
}

References a.

predux< Packet16uc >()

template<>

EIGEN_STRONG_INLINE uint8_t Eigen::internal::predux< Packet16uc >

(

const Packet16uc &

a

)

                                                                          {
  return predux_size16<Packet16uc>(a);
}

References a.

predux< Packet1cd >()

template<>

EIGEN_STRONG_INLINE std::complex< double > Eigen::internal::predux< Packet1cd >

(

const Packet1cd &

a

)

                                                                             {
  return pfirst(a);
}

References a, and pfirst().

predux< Packet1cf >()

template<>

EIGEN_STRONG_INLINE std::complex<float> Eigen::internal::predux< Packet1cf >

(

const Packet1cf &

a

)

                                                                            {
  std::complex<float> s;
  vst1_f32((float*)&s, a.v);
  return s;
}

References a, and s.

predux< Packet2cd >()

template<>

EIGEN_STRONG_INLINE std::complex<double> Eigen::internal::predux< Packet2cd >

(

const Packet2cd &

a

)

                                                                             {
  return predux(padd(Packet1cd(_mm256_extractf128_pd(a.v, 0)), Packet1cd(_mm256_extractf128_pd(a.v, 1))));
}

References a, padd(), and predux().

predux< Packet2cf >()

template<>

EIGEN_STRONG_INLINE std::complex< float > Eigen::internal::predux< Packet2cf >

(

const Packet2cf &

a

)

                                                                            {
  Packet4f b;
  b = vec_sld(a.v, a.v, 8);
  b = padd<Packet4f>(a.v, b);
  return pfirst<Packet2cf>(Packet2cf(b));
}

References a, b, padd< Packet4f >(), and pfirst< Packet2cf >().

predux< Packet2d >()

template<>

EIGEN_STRONG_INLINE double Eigen::internal::predux< Packet2d >

(

const Packet2d &

a

)

                                                               {
  return pfirst<Packet2d>(__lsx_vfadd_d(a, preverse(a)));
}

References a, pfirst< Packet2d >(), and preverse().

predux< Packet2f >()

template<>

EIGEN_STRONG_INLINE float Eigen::internal::predux< Packet2f >

(

const Packet2f &

a

)

                                                              {
  return vget_lane_f32(vpadd_f32(a, a), 0);
}

References a.

predux< Packet2i >()

template<>

EIGEN_STRONG_INLINE int32_t Eigen::internal::predux< Packet2i >

(

const Packet2i &

a

)

                                                                {
  return vget_lane_s32(vpadd_s32(a, a), 0);
}

References a.

predux< Packet2l >()

template<>

EIGEN_STRONG_INLINE int64_t Eigen::internal::predux< Packet2l >

(

const Packet2l &

a

)

                                                                {
  return (int64_t)__lsx_vpickve2gr_d(__lsx_vhaddw_q_d(a, a), 0);
}

References a.

predux< Packet2ui >()

template<>

EIGEN_STRONG_INLINE uint32_t Eigen::internal::predux< Packet2ui >

(

const Packet2ui &

a

)

                                                                   {
  return vget_lane_u32(vpadd_u32(a, a), 0);
}

References a.

predux< Packet2ul >()

template<>

EIGEN_STRONG_INLINE uint64_t Eigen::internal::predux< Packet2ul >

(

const Packet2ul &

a

)

                                                                   {
  return (uint64_t)__lsx_vpickve2gr_d(__lsx_vhaddw_qu_du(a, a), 0);
}

References a.

predux< Packet32h >()

template<>

EIGEN_STRONG_INLINE half Eigen::internal::predux< Packet32h >

(

const Packet32h &

a

)

                                                               {
  return (half)_mm512_reduce_add_ph(a);
}

References a.

predux< Packet4bf >()

template<>

EIGEN_STRONG_INLINE bfloat16 Eigen::internal::predux< Packet4bf >

(

const Packet4bf &

a

)

                                                                   {
  return static_cast<bfloat16>(predux<Packet4f>(Bf16ToF32(a)));
}

References a, Bf16ToF32(), and predux< Packet4f >().

predux< Packet4c >()

template<>

EIGEN_STRONG_INLINE int8_t Eigen::internal::predux< Packet4c >

(

const Packet4c &

a

)

                                                               {
  const int8x8_t a_dup = vreinterpret_s8_s32(vdup_n_s32(a));
  int8x8_t sum = vpadd_s8(a_dup, a_dup);
  sum = vpadd_s8(sum, sum);
  return vget_lane_s8(sum, 0);
}

References a.

predux< Packet4cd >()

template<>

EIGEN_STRONG_INLINE std::complex<double> Eigen::internal::predux< Packet4cd >

(

const Packet4cd &

a

)

                                                                             {
  return predux(padd(Packet2cd(_mm512_extractf64x4_pd(a.v, 0)), Packet2cd(_mm512_extractf64x4_pd(a.v, 1))));
}

References a, padd(), and predux().

predux< Packet4cf >()

template<>

EIGEN_STRONG_INLINE std::complex<float> Eigen::internal::predux< Packet4cf >

(

const Packet4cf &

a

)

                                                                            {
  return predux(padd(Packet2cf(_mm256_extractf128_ps(a.v, 0)), Packet2cf(_mm256_extractf128_ps(a.v, 1))));
}

References a, padd(), and predux().

predux< Packet4d >()

template<>

EIGEN_STRONG_INLINE double Eigen::internal::predux< Packet4d >

(

const Packet4d &

a

)

                                                               {
  return predux(Packet2d(_mm_add_pd(_mm256_castpd256_pd128(a), _mm256_extractf128_pd(a, 1))));
}

References a, and predux().

Referenced by predux< Packet8d >().

predux< Packet4f >()

template<>

EIGEN_STRONG_INLINE float Eigen::internal::predux< Packet4f >

(

const Packet4f &

a

)

                                                              {
  Packet4f b, sum;
  b = vec_sld(a, a, 8);
  sum = a + b;
  b = vec_sld(sum, sum, 4);
  sum += b;
  return pfirst(sum);
}

References a, b, and pfirst().

Referenced by predux< Packet16f >(), predux< Packet4bf >(), and predux< Packet8bf >().

predux< Packet4i >()

template<>

EIGEN_STRONG_INLINE int Eigen::internal::predux< Packet4i >

(

const Packet4i &

a

)

                                                            {
  Packet4i b, sum;
  b = vec_sld(a, a, 8);
  sum = a + b;
  b = vec_sld(sum, sum, 4);
  sum += b;
  return pfirst(sum);
}

References a, b, and pfirst().

predux< Packet4s >()

template<>

EIGEN_STRONG_INLINE int16_t Eigen::internal::predux< Packet4s >

(

const Packet4s &

a

)

                                                                {
  const int16x4_t sum = vpadd_s16(a, a);
  return vget_lane_s16(vpadd_s16(sum, sum), 0);
}

References a.

predux< Packet4uc >()

template<>

EIGEN_STRONG_INLINE uint8_t Eigen::internal::predux< Packet4uc >

(

const Packet4uc &

a

)

                                                                  {
  const uint8x8_t a_dup = vreinterpret_u8_u32(vdup_n_u32(a));
  uint8x8_t sum = vpadd_u8(a_dup, a_dup);
  sum = vpadd_u8(sum, sum);
  return vget_lane_u8(sum, 0);
}

References a.

predux< Packet4ui >()

template<>

EIGEN_STRONG_INLINE uint32_t Eigen::internal::predux< Packet4ui >

(

const Packet4ui &

a

)

                                                                   {
  Packet2ul tmp = __lsx_vhaddw_du_wu(a, a);
  return (uint32_t)__lsx_vpickve2gr_d(__lsx_vhaddw_qu_du(tmp, tmp), 0);
}

References a, and tmp.

predux< Packet4us >()

template<>

EIGEN_STRONG_INLINE uint16_t Eigen::internal::predux< Packet4us >

(

const Packet4us &

a

)

                                                                   {
  const uint16x4_t sum = vpadd_u16(a, a);
  return vget_lane_u16(vpadd_u16(sum, sum), 0);
}

References a.

predux< Packet8bf >()

template<>

EIGEN_STRONG_INLINE bfloat16 Eigen::internal::predux< Packet8bf >

(

const Packet8bf &

a

)

                                                                   {
  float redux_even = predux<Packet4f>(Bf16ToF32Even(a));
  float redux_odd = predux<Packet4f>(Bf16ToF32Odd(a));
  float f32_result = redux_even + redux_odd;
  return bfloat16(f32_result);
}

References a, Bf16ToF32Even(), Bf16ToF32Odd(), and predux< Packet4f >().

predux< Packet8c >()

template<>

EIGEN_STRONG_INLINE int8_t Eigen::internal::predux< Packet8c >

(

const Packet8c &

a

)

                                                               {
  int8x8_t sum = vpadd_s8(a, a);
  sum = vpadd_s8(sum, sum);
  sum = vpadd_s8(sum, sum);
  return vget_lane_s8(sum, 0);
}

References a.

predux< Packet8cf >()

template<>

EIGEN_STRONG_INLINE std::complex<float> Eigen::internal::predux< Packet8cf >

(

const Packet8cf &

a

)

                                                                            {
  return predux(padd(Packet4cf(extract256<0>(a.v)), Packet4cf(extract256<1>(a.v))));
}

References a, padd(), and predux().

predux< Packet8d >()

template<>

EIGEN_STRONG_INLINE double Eigen::internal::predux< Packet8d >

(

const Packet8d &

a

)

                                                               {
  __m256d lane0 = _mm512_extractf64x4_pd(a, 0);
  __m256d lane1 = _mm512_extractf64x4_pd(a, 1);
  __m256d sum = _mm256_add_pd(lane0, lane1);
  return predux<Packet4d>(sum);
}

References a, and predux< Packet4d >().

predux< Packet8f >()

template<>

EIGEN_STRONG_INLINE float Eigen::internal::predux< Packet8f >

(

const Packet8f &

a

)

                                                              {
  return predux(Packet4f(_mm_add_ps(_mm256_castps256_ps128(a), _mm256_extractf128_ps(a, 1))));
}

References a, and predux().

Referenced by predux< Packet16f >(), and predux< Packet8h >().

predux< Packet8h >()

template<>

EIGEN_STRONG_INLINE half Eigen::internal::predux< Packet8h >

(

const Packet8h &

a

)

                                                                  {
  Packet8f af = half2float(a);
  float reduced = predux<Packet8f>(af);
  return Eigen::half(reduced);
}

References a, half2float(), and predux< Packet8f >().

predux< Packet8i >()

template<>

EIGEN_STRONG_INLINE int Eigen::internal::predux< Packet8i >

(

const Packet8i &

a

)

                                                            {
  return predux(Packet4i(_mm_add_epi32(_mm256_castsi256_si128(a), _mm256_extractf128_si256(a, 1))));
}

References a, and predux().

predux< Packet8l >()

template<>

EIGEN_STRONG_INLINE int64_t Eigen::internal::predux< Packet8l >

(

const Packet8l &

a

)

                                                                {
  return _mm512_reduce_add_epi64(a);
}

References a.

predux< Packet8s >()

template<>

EIGEN_STRONG_INLINE int16_t Eigen::internal::predux< Packet8s >

(

const Packet8s &

a

)

                                                                  {
  return predux_size8<Packet8s>(a);
}

References a.

predux< Packet8uc >()

template<>

EIGEN_STRONG_INLINE uint8_t Eigen::internal::predux< Packet8uc >

(

const Packet8uc &

a

)

                                                                  {
  uint8x8_t sum = vpadd_u8(a, a);
  sum = vpadd_u8(sum, sum);
  sum = vpadd_u8(sum, sum);
  return vget_lane_u8(sum, 0);
}

References a.

predux< Packet8ui >()

template<>

EIGEN_STRONG_INLINE uint32_t Eigen::internal::predux< Packet8ui >

(

const Packet8ui &

a

)

                                                                   {
  return predux(Packet4ui(_mm_add_epi32(_mm256_castsi256_si128(a), _mm256_extractf128_si256(a, 1))));
}

References a, and predux().

predux< Packet8us >()

template<>

EIGEN_STRONG_INLINE uint16_t Eigen::internal::predux< Packet8us >

(

const Packet8us &

a

)

                                                                             {
  return predux_size8<Packet8us>(a);
}

References a.

predux< PacketXf >()

template<>

EIGEN_STRONG_INLINE float Eigen::internal::predux< PacketXf >

(

const PacketXf &

a

)

                                                              {
  return svaddv_f32(svptrue_b32(), a);
}

References a.

predux< PacketXi >()

template<>

EIGEN_STRONG_INLINE numext::int32_t Eigen::internal::predux< PacketXi >

(

const PacketXi &

a

)

                                                                      {
  return static_cast<numext::int32_t>(svaddv_s32(svptrue_b32(), a));
}

References a.

predux_any() [1/16]

template<typename Packet >

EIGEN_DEVICE_FUNC bool Eigen::internal::predux_any ( const Packet &  a )

inline

Returns

true if all coeffs of a means "true" It is supposed to be called on values returned by pcmp_*.

Returns

true if any coeffs of a means "true" It is supposed to be called on values returned by pcmp_*.

                                                          {
  // Dirty but generic implementation where "true" is assumed to be non 0 and all the sames.
  // It is expected that "true" is either:
  //  - Scalar(1)
  //  - bits full of ones (NaN for floats),
  //  - or first bit equals to 1 (1 for ints, smallest denormal for floats).
  // For all these cases, taking the sum is just fine, and this boils down to a no-op for scalars.
  typedef typename unpacket_traits<Packet>::type Scalar;
  return numext::not_equal_strict(predux(a), Scalar(0));
}

References a, Eigen::numext::not_equal_strict(), and predux().

predux_any() [2/16]

template<>

EIGEN_STRONG_INLINE bool Eigen::internal::predux_any

(

const Packet16f &

a

)

                                                        {
  return _mm512_reduce_or_epi32(_mm512_castps_si512(a)) != 0;
}

References a.

predux_any() [3/16]

template<>

EIGEN_STRONG_INLINE bool Eigen::internal::predux_any

(

const Packet16i &

a

)

                                                        {
  return _mm512_reduce_or_epi32(a) != 0;
}

References a.

predux_any() [4/16]

template<>

EIGEN_STRONG_INLINE bool Eigen::internal::predux_any

(

const Packet2d &

x

)

                                                       {
  return _mm_movemask_pd(x) != 0x0;
}

References plotDoE::x.

predux_any() [5/16]

template<>

EIGEN_STRONG_INLINE bool Eigen::internal::predux_any

(

const Packet2l &

x

)

                                                       {
  return _mm_movemask_pd(_mm_castsi128_pd(x)) != 0x0;
}

References plotDoE::x.

predux_any() [6/16]

template<>

EIGEN_STRONG_INLINE bool Eigen::internal::predux_any

(

const Packet4d &

x

)

                                                       {
  return _mm256_movemask_pd(x) != 0;
}

References plotDoE::x.

predux_any() [7/16]

template<>

EIGEN_STRONG_INLINE bool Eigen::internal::predux_any

(

const Packet4f &

x

)

                                                       {
  return vec_any_ne(x, pzero(x));
}

References pzero(), and plotDoE::x.

Referenced by Eigen::internal::all_visitor< Scalar >::all_predux(), Eigen::internal::any_visitor< Scalar >::any_predux(), generic_pow_impl(), packetmath_notcomplex(), pexp_double(), pexp_float(), psincos_double(), psincos_float(), check_signbit_impl< T >::run(), Eigen::internal::generic_fast_erf< Scalar >::run(), Eigen::internal::generic_fast_erfc< Scalar >::run(), Eigen::internal::maybe_raise_div_by_zero< Packet, true >::run(), and signbit_test().

predux_any() [8/16]

template<>

EIGEN_STRONG_INLINE bool Eigen::internal::predux_any

(

const Packet4i &

x

)

                                                       {
  return _mm_movemask_ps(_mm_castsi128_ps(x)) != 0x0;
}

References plotDoE::x.

predux_any() [9/16]

template<>

EIGEN_STRONG_INLINE bool Eigen::internal::predux_any

(

const Packet4ui &

x

)

                                                        {
  return _mm_movemask_ps(_mm_castsi128_ps(x)) != 0x0;
}

References plotDoE::x.

predux_any() [10/16]

template<>

EIGEN_STRONG_INLINE bool Eigen::internal::predux_any

(

const Packet8bf &

x

)

                                                        {
  return _mm_movemask_epi8(x) != 0;
}

References plotDoE::x.

predux_any() [11/16]

template<>

EIGEN_STRONG_INLINE bool Eigen::internal::predux_any

(

const Packet8d &

a

)

                                                       {
  return _mm512_reduce_or_epi64(_mm512_castpd_si512(a)) != 0;
}

References a.

predux_any() [12/16]

template<>

EIGEN_STRONG_INLINE bool Eigen::internal::predux_any

(

const Packet8f &

x

)

                                                       {
  return _mm256_movemask_ps(x) != 0;
}

References plotDoE::x.

predux_any() [13/16]

template<>

EIGEN_STRONG_INLINE bool Eigen::internal::predux_any

(

const Packet8h &

x

)

                                                       {
  return _mm_movemask_epi8(x) != 0;
}

References plotDoE::x.

predux_any() [14/16]

template<>

EIGEN_STRONG_INLINE bool Eigen::internal::predux_any

(

const Packet8i &

x

)

                                                       {
  return _mm256_movemask_ps(_mm256_castsi256_ps(x)) != 0;
}

References plotDoE::x.

predux_any() [15/16]

template<>

EIGEN_STRONG_INLINE bool Eigen::internal::predux_any

(

const Packet8l &

a

)

                                                       {
  return _mm512_reduce_or_epi64(a) != 0;
}

References a.

predux_any() [16/16]

template<>

EIGEN_STRONG_INLINE bool Eigen::internal::predux_any

(

const Packet8ui &

x

)

                                                        {
  return _mm256_movemask_ps(_mm256_castsi256_ps(x)) != 0;
}

References plotDoE::x.

predux_half_dowto4() [1/9]

template<typename Packet >

const DoublePacket<Packet>& Eigen::internal::predux_half_dowto4

(

const DoublePacket< Packet > &

a

)

                                                                                                      {
  return a;
}

References a.

predux_half_dowto4() [2/9]

template<typename Packet >

DoublePacket<typename unpacket_traits<Packet>::half> Eigen::internal::predux_half_dowto4	(	const DoublePacket< Packet > &	a,
		std::enable_if_t< unpacket_traits< Packet >::size==16 > *	= 0
	)

                                                                                           {
  // yes, that's pretty hackish :(
  DoublePacket<typename unpacket_traits<Packet>::half> res;
  typedef std::complex<typename unpacket_traits<Packet>::type> Cplx;
  typedef typename packet_traits<Cplx>::type CplxPacket;
  res.first = predux_half_dowto4(CplxPacket(a.first)).v;
  res.second = predux_half_dowto4(CplxPacket(a.second)).v;
  return res;
}

References a, predux_half_dowto4(), res, and compute_granudrum_aor::type.

predux_half_dowto4() [3/9]

template<typename Packet >

EIGEN_DEVICE_FUNC std::conditional_t<(unpacket_traits<Packet>::size % 8) == 0, typename unpacket_traits<Packet>::half, Packet> Eigen::internal::predux_half_dowto4 ( const Packet &  a )

inline

Returns

the sum of the elements of upper and lower half of a if a is larger than 4. For a packet {a0, a1, a2, a3, a4, a5, a6, a7}, it returns a half packet {a0+a4, a1+a5, a2+a6, a3+a7} For packet-size smaller or equal to 4, this boils down to a noop.

                                    {
  return a;
}

References a.

predux_half_dowto4() [4/9]

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8c Eigen::internal::predux_half_dowto4

(

const Packet16c &

a

)

                                                                                      {
  return vadd_s8(vget_high_s8(a), vget_low_s8(a));
}

References a.

predux_half_dowto4() [5/9]

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8uc Eigen::internal::predux_half_dowto4

(

const Packet16uc &

a

)

                                                                                        {
  return vadd_u8(vget_high_u8(a), vget_low_u8(a));
}

References a.

predux_half_dowto4() [6/9]

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4c Eigen::internal::predux_half_dowto4

(

const Packet8c &

a

)

                                                                                     {
  return vget_lane_s32(vreinterpret_s32_s8(vadd_s8(a, vreinterpret_s8_s32(vrev64_s32(vreinterpret_s32_s8(a))))), 0);
}

References a.

Referenced by Eigen::internal::gebp_kernel< LhsScalar, RhsScalar, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >::operator()(), Eigen::internal::last_row_process_16_packets< LhsScalar, RhsScalar, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs, 16 >::operator()(), packetmath(), and predux_half_dowto4().

predux_half_dowto4() [7/9]

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4s Eigen::internal::predux_half_dowto4

(

const Packet8s &

a

)

                                                                                     {
  return vadd_s16(vget_high_s16(a), vget_low_s16(a));
}

References a.

predux_half_dowto4() [8/9]

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4uc Eigen::internal::predux_half_dowto4

(

const Packet8uc &

a

)

                                                                                       {
  return vget_lane_u32(vreinterpret_u32_u8(vadd_u8(a, vreinterpret_u8_u32(vrev64_u32(vreinterpret_u32_u8(a))))), 0);
}

References a.

predux_half_dowto4() [9/9]

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4us Eigen::internal::predux_half_dowto4

(

const Packet8us &

a

)

                                                                                       {
  return vadd_u16(vget_high_u16(a), vget_low_u16(a));
}

References a.

predux_half_dowto4< Packet16bf >()

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::predux_half_dowto4< Packet16bf >

(

const Packet16bf &

a

)

                                                                                  {
  Packet8bf lane0 = _mm256_extractf128_si256(a, 0);
  Packet8bf lane1 = _mm256_extractf128_si256(a, 1);
  return padd<Packet8bf>(lane0, lane1);
}

References a, and padd< Packet8bf >().

predux_half_dowto4< Packet16f >()

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::predux_half_dowto4< Packet16f >

(

const Packet16f &

a

)

                                                                               {
#ifdef EIGEN_VECTORIZE_AVX512DQ
  __m256 lane0 = _mm512_extractf32x8_ps(a, 0);
  __m256 lane1 = _mm512_extractf32x8_ps(a, 1);
  return _mm256_add_ps(lane0, lane1);
#else
  __m128 lane0 = _mm512_extractf32x4_ps(a, 0);
  __m128 lane1 = _mm512_extractf32x4_ps(a, 1);
  __m128 lane2 = _mm512_extractf32x4_ps(a, 2);
  __m128 lane3 = _mm512_extractf32x4_ps(a, 3);
  __m128 sum0 = _mm_add_ps(lane0, lane2);
  __m128 sum1 = _mm_add_ps(lane1, lane3);
  return _mm256_insertf128_ps(_mm256_castps128_ps256(sum0), sum1, 1);
#endif
}

References a.

predux_half_dowto4< Packet16h >()

template<>

EIGEN_STRONG_INLINE Packet8h Eigen::internal::predux_half_dowto4< Packet16h >

(

const Packet16h &

a

)

                                                                               {
  Packet8h lane0 = _mm256_extractf128_si256(a, 0);
  Packet8h lane1 = _mm256_extractf128_si256(a, 1);
  return padd<Packet8h>(lane0, lane1);
}

References a, and padd< Packet8h >().

predux_half_dowto4< Packet16i >()

template<>

EIGEN_STRONG_INLINE Packet8i Eigen::internal::predux_half_dowto4< Packet16i >

(

const Packet16i &

a

)

                                                                               {
#ifdef EIGEN_VECTORIZE_AVX512DQ
  __m256i lane0 = _mm512_extracti32x8_epi32(a, 0);
  __m256i lane1 = _mm512_extracti32x8_epi32(a, 1);
  return _mm256_add_epi32(lane0, lane1);
#else
  __m128i lane0 = _mm512_extracti32x4_epi32(a, 0);
  __m128i lane1 = _mm512_extracti32x4_epi32(a, 1);
  __m128i lane2 = _mm512_extracti32x4_epi32(a, 2);
  __m128i lane3 = _mm512_extracti32x4_epi32(a, 3);
  __m128i sum0 = _mm_add_epi32(lane0, lane2);
  __m128i sum1 = _mm_add_epi32(lane1, lane3);
  return _mm256_inserti128_si256(_mm256_castsi128_si256(sum0), sum1, 1);
#endif
}

References a.

predux_half_dowto4< Packet32h >()

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::predux_half_dowto4< Packet32h >

(

const Packet32h &

a

)

                                                                                {
#ifdef EIGEN_VECTORIZE_AVX512DQ
  __m256i lowHalf = _mm256_castps_si256(_mm512_extractf32x8_ps(_mm512_castph_ps(a), 0));
  __m256i highHalf = _mm256_castps_si256(_mm512_extractf32x8_ps(_mm512_castph_ps(a), 1));
 
  return Packet16h(padd<Packet16h>(lowHalf, highHalf));
#else
  Eigen::half data[32];
  _mm512_storeu_ph(data, a);
 
  __m256i lowHalf = _mm256_castph_si256(_mm256_loadu_ph(data));
  __m256i highHalf = _mm256_castph_si256(_mm256_loadu_ph(data + 16));
 
  return Packet16h(padd<Packet16h>(lowHalf, highHalf));
#endif
}

References a, data, and padd< Packet16h >().

predux_half_dowto4< Packet8cf >()

template<>

EIGEN_STRONG_INLINE Packet4cf Eigen::internal::predux_half_dowto4< Packet8cf >

(

const Packet8cf &

a

)

                                                                                {
  __m256 lane0 = extract256<0>(a.v);
  __m256 lane1 = extract256<1>(a.v);
  __m256 res = _mm256_add_ps(lane0, lane1);
  return Packet4cf(res);
}

References a, and res.

predux_half_dowto4< Packet8d >()

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::predux_half_dowto4< Packet8d >

(

const Packet8d &

a

)

                                                                             {
  __m256d lane0 = _mm512_extractf64x4_pd(a, 0);
  __m256d lane1 = _mm512_extractf64x4_pd(a, 1);
  return _mm256_add_pd(lane0, lane1);
}

References a.

predux_half_dowto4< Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::predux_half_dowto4< Packet8f >

(

const Packet8f &

a

)

                                                                             {
  return _mm_add_ps(_mm256_castps256_ps128(a), _mm256_extractf128_ps(a, 1));
}

References a.

predux_half_dowto4< Packet8i >()

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::predux_half_dowto4< Packet8i >

(

const Packet8i &

a

)

                                                                             {
  return _mm_add_epi32(_mm256_castsi256_si128(a), _mm256_extractf128_si256(a, 1));
}

References a.

predux_half_dowto4< Packet8l >()

template<>

EIGEN_STRONG_INLINE Packet4l Eigen::internal::predux_half_dowto4< Packet8l >

(

const Packet8l &

a

)

                                                                             {
  __m256i lane0 = _mm512_extracti64x4_epi64(a, 0);
  __m256i lane1 = _mm512_extracti64x4_epi64(a, 1);
  return _mm256_add_epi64(lane0, lane1);
}

References a.

predux_half_dowto4< Packet8ui >()

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::predux_half_dowto4< Packet8ui >

(

const Packet8ui &

a

)

                                                                                {
  return _mm_add_epi32(_mm256_castsi256_si128(a), _mm256_extractf128_si256(a, 1));
}

References a.

predux_helper()

template<typename Packet , typename Op >

EIGEN_DEVICE_FUNC unpacket_traits<Packet>::type Eigen::internal::predux_helper ( const Packet &  a, Op  op  )

inline

                                                                                                    {
  typedef typename unpacket_traits<Packet>::type Scalar;
  const size_t n = unpacket_traits<Packet>::size;
  EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar elements[n];
  pstoreu<Scalar>(elements, a);
  for (size_t k = n / 2; k > 0; k /= 2) {
    for (size_t i = 0; i < k; ++i) {
      elements[i] = op(elements[i], elements[i + k]);
    }
  }
  return elements[0];
}

References a, EIGEN_ALIGN_TO_BOUNDARY, i, k, n, and op.

Referenced by predux_max(), predux_min(), and predux_mul().

predux_max() [1/2]

template<typename Packet >

EIGEN_DEVICE_FUNC unpacket_traits<Packet>::type Eigen::internal::predux_max ( const Packet &  a )

inline

Returns

the min of the elements of a

                                                                                          {
  typedef typename unpacket_traits<Packet>::type Scalar;
  return predux_helper(a, EIGEN_BINARY_OP_NAN_PROPAGATION(Scalar, (pmax<PropagateFast, Scalar>)));
}

References a, EIGEN_BINARY_OP_NAN_PROPAGATION, and predux_helper().

Referenced by Eigen::internal::minmax_coeff_visitor< Derived, is_min, NaNPropagation, isInt >::initpacket(), Eigen::internal::minmax_coeff_visitor< Derived, is_min, NaNPropagation, false >::initpacket(), Eigen::internal::minmax_coeff_visitor< Derived, is_min, PropagateNumbers, false >::packet(), Eigen::internal::minmax_coeff_visitor< Derived, is_min, NaNPropagation, false >::packet(), and packetmath_notcomplex().

predux_max() [2/2]

template<int NaNPropagation, typename Packet >

EIGEN_DEVICE_FUNC unpacket_traits<Packet>::type Eigen::internal::predux_max ( const Packet &  a )

inline

                                                                                          {
  typedef typename unpacket_traits<Packet>::type Scalar;
  return predux_helper(a, EIGEN_BINARY_OP_NAN_PROPAGATION(Scalar, (pmax<NaNPropagation, Scalar>)));
}

References a, EIGEN_BINARY_OP_NAN_PROPAGATION, and predux_helper().

predux_max< Packet16bf >()

template<>

EIGEN_STRONG_INLINE bfloat16 Eigen::internal::predux_max< Packet16bf >

(

const Packet16bf &

from

)

                                                                            {
  return static_cast<bfloat16>(predux_max<Packet16f>(Bf16ToF32(from)));
}

References Bf16ToF32(), and predux_max< Packet16f >().

predux_max< Packet16c >()

template<>

EIGEN_STRONG_INLINE int8_t Eigen::internal::predux_max< Packet16c >

(

const Packet16c &

a

)

                                                                          {
  Packet16c pair, quad, octo, result;
 
  pair = vec_max(a, vec_sld(a, a, 8));
  quad = vec_max(pair, vec_sld(pair, pair, 4));
  octo = vec_max(quad, vec_sld(quad, quad, 2));
  result = vec_max(octo, vec_sld(octo, octo, 1));
 
  return pfirst(result);
}

References a, and pfirst().

predux_max< Packet16f >()

template<>

EIGEN_STRONG_INLINE float Eigen::internal::predux_max< Packet16f >

(

const Packet16f &

a

)

                                                                    {
  __m128 lane0 = _mm512_extractf32x4_ps(a, 0);
  __m128 lane1 = _mm512_extractf32x4_ps(a, 1);
  __m128 lane2 = _mm512_extractf32x4_ps(a, 2);
  __m128 lane3 = _mm512_extractf32x4_ps(a, 3);
  __m128 res = _mm_max_ps(_mm_max_ps(lane0, lane1), _mm_max_ps(lane2, lane3));
  res = _mm_max_ps(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 3, 2)));
  return pfirst(_mm_max_ps(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 0, 1))));
}

References a, pfirst(), and res.

Referenced by predux_max< Packet16bf >(), and predux_max< Packet16h >().

predux_max< Packet16h >()

template<>

EIGEN_STRONG_INLINE Eigen::half Eigen::internal::predux_max< Packet16h >

(

const Packet16h &

a

)

                                                                        {
  Packet16f af = half2float(a);
  float reduced = predux_max<Packet16f>(af);
  return Eigen::half(reduced);
}

References a, half2float(), and predux_max< Packet16f >().

predux_max< Packet16i >()

template<>

EIGEN_STRONG_INLINE int Eigen::internal::predux_max< Packet16i >

(

const Packet16i &

a

)

                                                                  {
  return _mm512_reduce_max_epi32(a);
}

References a.

predux_max< Packet16uc >()

template<>

EIGEN_STRONG_INLINE uint8_t Eigen::internal::predux_max< Packet16uc >

(

const Packet16uc &

a

)

                                                                              {
  Packet16uc pair, quad, octo, result;
 
  pair = vec_max(a, vec_sld(a, a, 8));
  quad = vec_max(pair, vec_sld(pair, pair, 4));
  octo = vec_max(quad, vec_sld(quad, quad, 2));
  result = vec_max(octo, vec_sld(octo, octo, 1));
 
  return pfirst(result);
}

References a, and pfirst().

predux_max< Packet2d >()

template<>

EIGEN_STRONG_INLINE double Eigen::internal::predux_max< Packet2d >

(

const Packet2d &

a

)

                                                                   {
  return pfirst(__lsx_vfmax_d(a, preverse(a)));
}

References a, pfirst(), and preverse().

predux_max< Packet2f >()

template<>

EIGEN_STRONG_INLINE float Eigen::internal::predux_max< Packet2f >

(

const Packet2f &

a

)

                                                                  {
  return vget_lane_f32(vpmax_f32(a, a), 0);
}

References a.

predux_max< Packet2i >()

template<>

EIGEN_STRONG_INLINE int32_t Eigen::internal::predux_max< Packet2i >

(

const Packet2i &

a

)

                                                                    {
  return vget_lane_s32(vpmax_s32(a, a), 0);
}

References a.

predux_max< Packet2l >()

template<>

EIGEN_STRONG_INLINE int64_t Eigen::internal::predux_max< Packet2l >

(

const Packet2l &

a

)

                                                                    {
  return pfirst((Packet2l)__lsx_vmax_d(a, preverse(a)));
}

References a, pfirst(), and preverse().

predux_max< Packet2ui >()

template<>

EIGEN_STRONG_INLINE uint32_t Eigen::internal::predux_max< Packet2ui >

(

const Packet2ui &

a

)

                                                                       {
  return vget_lane_u32(vpmax_u32(a, a), 0);
}

References a.

predux_max< Packet2ul >()

template<>

EIGEN_STRONG_INLINE uint64_t Eigen::internal::predux_max< Packet2ul >

(

const Packet2ul &

a

)

                                                                       {
  return pfirst((Packet2ul)__lsx_vmax_du(a, preverse(a)));
}

References a, pfirst(), and preverse().

predux_max< Packet4bf >()

template<>

EIGEN_STRONG_INLINE bfloat16 Eigen::internal::predux_max< Packet4bf >

(

const Packet4bf &

a

)

                                                                       {
  return static_cast<bfloat16>(predux_max<Packet4f>(Bf16ToF32(a)));
}

References a, Bf16ToF32(), and predux_max< Packet4f >().

predux_max< Packet4c >()

template<>

EIGEN_STRONG_INLINE int8_t Eigen::internal::predux_max< Packet4c >

(

const Packet4c &

a

)

                                                                   {
  const int8x8_t a_dup = vreinterpret_s8_s32(vdup_n_s32(a));
  int8x8_t max = vpmax_s8(a_dup, a_dup);
  max = vpmax_s8(max, max);
  return vget_lane_s8(max, 0);
}

References a, and Eigen::max().

predux_max< Packet4d >()

template<>

EIGEN_STRONG_INLINE double Eigen::internal::predux_max< Packet4d >

(

const Packet4d &

a

)

                                                                   {
  Packet4d tmp = _mm256_max_pd(a, _mm256_permute2f128_pd(a, a, 1));
  return pfirst(_mm256_max_pd(tmp, _mm256_shuffle_pd(tmp, tmp, 1)));
}

References a, pfirst(), and tmp.

predux_max< Packet4f >()

template<>

EIGEN_STRONG_INLINE float Eigen::internal::predux_max< Packet4f >

(

const Packet4f &

a

)

                                                                  {
  return predux_max4<Packet4f>(a);
}

References a.

Referenced by predux_max< Packet4bf >(), and predux_max< Packet8bf >().

predux_max< Packet4i >()

template<>

EIGEN_STRONG_INLINE int Eigen::internal::predux_max< Packet4i >

(

const Packet4i &

a

)

                                                                {
  return predux_max4<Packet4i>(a);
}

References a.

predux_max< Packet4s >()

template<>

EIGEN_STRONG_INLINE int16_t Eigen::internal::predux_max< Packet4s >

(

const Packet4s &

a

)

                                                                    {
  const int16x4_t max = vpmax_s16(a, a);
  return vget_lane_s16(vpmax_s16(max, max), 0);
}

References a, and Eigen::max().

predux_max< Packet4uc >()

template<>

EIGEN_STRONG_INLINE uint8_t Eigen::internal::predux_max< Packet4uc >

(

const Packet4uc &

a

)

                                                                      {
  const uint8x8_t a_dup = vreinterpret_u8_u32(vdup_n_u32(a));
  uint8x8_t max = vpmax_u8(a_dup, a_dup);
  max = vpmax_u8(max, max);
  return vget_lane_u8(max, 0);
}

References a, and Eigen::max().

predux_max< Packet4ui >()

template<>

EIGEN_STRONG_INLINE uint32_t Eigen::internal::predux_max< Packet4ui >

(

const Packet4ui &

a

)

                                                                       {
  Packet4ui tmp = __lsx_vmax_wu(a, __lsx_vshuf4i_w((__m128i)a, 0x4E));
  return pfirst((Packet4ui)__lsx_vmax_wu(tmp, __lsx_vshuf4i_w((__m128i)tmp, 0xB1)));
}

References a, pfirst(), and tmp.

predux_max< Packet4us >()

template<>

EIGEN_STRONG_INLINE uint16_t Eigen::internal::predux_max< Packet4us >

(

const Packet4us &

a

)

                                                                       {
  const uint16x4_t max = vpmax_u16(a, a);
  return vget_lane_u16(vpmax_u16(max, max), 0);
}

References a, and Eigen::max().

predux_max< Packet8bf >()

template<>

EIGEN_STRONG_INLINE bfloat16 Eigen::internal::predux_max< Packet8bf >

(

const Packet8bf &

a

)

                                                                       {
  float redux_even = predux_max<Packet4f>(Bf16ToF32Even(a));
  float redux_odd = predux_max<Packet4f>(Bf16ToF32Odd(a));
  float f32_result = (std::max)(redux_even, redux_odd);
  return bfloat16(f32_result);
}

References a, Bf16ToF32Even(), Bf16ToF32Odd(), max, and predux_max< Packet4f >().

predux_max< Packet8c >()

template<>

EIGEN_STRONG_INLINE int8_t Eigen::internal::predux_max< Packet8c >

(

const Packet8c &

a

)

                                                                   {
  int8x8_t max = vpmax_s8(a, a);
  max = vpmax_s8(max, max);
  max = vpmax_s8(max, max);
  return vget_lane_s8(max, 0);
}

References a, and Eigen::max().

predux_max< Packet8d >()

template<>

EIGEN_STRONG_INLINE double Eigen::internal::predux_max< Packet8d >

(

const Packet8d &

a

)

                                                                   {
  __m256d lane0 = _mm512_extractf64x4_pd(a, 0);
  __m256d lane1 = _mm512_extractf64x4_pd(a, 1);
  __m256d res = _mm256_max_pd(lane0, lane1);
  res = _mm256_max_pd(res, _mm256_permute2f128_pd(res, res, 1));
  return pfirst(_mm256_max_pd(res, _mm256_shuffle_pd(res, res, 1)));
}

References a, pfirst(), and res.

predux_max< Packet8f >()

template<>

EIGEN_STRONG_INLINE float Eigen::internal::predux_max< Packet8f >

(

const Packet8f &

a

)

                                                                  {
  Packet8f tmp = _mm256_max_ps(a, _mm256_permute2f128_ps(a, a, 1));
  tmp = _mm256_max_ps(tmp, _mm256_shuffle_ps(tmp, tmp, _MM_SHUFFLE(1, 0, 3, 2)));
  return pfirst(_mm256_max_ps(tmp, _mm256_shuffle_ps(tmp, tmp, 1)));
}

References a, pfirst(), and tmp.

Referenced by predux_max< Packet8h >().

predux_max< Packet8h >()

template<>

EIGEN_STRONG_INLINE Eigen::half Eigen::internal::predux_max< Packet8h >

(

const Packet8h &

a

)

                                                                      {
  Packet8f af = half2float(a);
  float reduced = predux_max<Packet8f>(af);
  return Eigen::half(reduced);
}

References a, half2float(), and predux_max< Packet8f >().

predux_max< Packet8l >()

template<>

EIGEN_STRONG_INLINE int64_t Eigen::internal::predux_max< Packet8l >

(

const Packet8l &

a

)

                                                                    {
  return _mm512_reduce_max_epi64(a);
}

References a.

predux_max< Packet8s >()

template<>

EIGEN_STRONG_INLINE int16_t Eigen::internal::predux_max< Packet8s >

(

const Packet8s &

a

)

                                                                      {
  Packet8s pair, quad, octo;
 
  // pair = { Max(a0,a4), Max(a1,a5), Max(a2,a6), Max(a3,a7) }
  pair = vec_max(a, vec_sld(a, a, 8));
 
  // quad = { Max(a0, a4, a2, a6), Max(a1, a5, a3, a7) }
  quad = vec_max(pair, vec_sld(pair, pair, 4));
 
  // octo = { Max(a0, a4, a2, a6, a1, a5, a3, a7) }
  octo = vec_max(quad, vec_sld(quad, quad, 2));
  return pfirst(octo);
}

References a, and pfirst().

predux_max< Packet8uc >()

template<>

EIGEN_STRONG_INLINE uint8_t Eigen::internal::predux_max< Packet8uc >

(

const Packet8uc &

a

)

                                                                      {
  uint8x8_t max = vpmax_u8(a, a);
  max = vpmax_u8(max, max);
  max = vpmax_u8(max, max);
  return vget_lane_u8(max, 0);
}

References a, and Eigen::max().

predux_max< Packet8us >()

template<>

EIGEN_STRONG_INLINE uint16_t Eigen::internal::predux_max< Packet8us >

(

const Packet8us &

a

)

                                                                                 {
  Packet8us pair, quad, octo;
 
  // pair = { Max(a0,a4), Max(a1,a5), Max(a2,a6), Max(a3,a7) }
  pair = vec_max(a, vec_sld(a, a, 8));
 
  // quad = { Max(a0, a4, a2, a6), Max(a1, a5, a3, a7) }
  quad = vec_max(pair, vec_sld(pair, pair, 4));
 
  // octo = { Max(a0, a4, a2, a6, a1, a5, a3, a7) }
  octo = vec_max(quad, vec_sld(quad, quad, 2));
  return pfirst(octo);
}

References a, and pfirst().

predux_max< PacketXf >()

template<>

EIGEN_STRONG_INLINE float Eigen::internal::predux_max< PacketXf >

(

const PacketXf &

a

)

                                                                  {
  return svmaxv_f32(svptrue_b32(), a);
}

References a.

predux_max< PacketXi >()

template<>

EIGEN_STRONG_INLINE numext::int32_t Eigen::internal::predux_max< PacketXi >

(

const PacketXi &

a

)

                                                                          {
  return svmaxv_s32(svptrue_b32(), a);
}

References a.

predux_min() [1/2]

template<typename Packet >

EIGEN_DEVICE_FUNC unpacket_traits<Packet>::type Eigen::internal::predux_min ( const Packet &  a )

inline

Returns

the min of the elements of a

                                                                                          {
  typedef typename unpacket_traits<Packet>::type Scalar;
  return predux_helper(a, EIGEN_BINARY_OP_NAN_PROPAGATION(Scalar, (pmin<PropagateFast, Scalar>)));
}

References a, EIGEN_BINARY_OP_NAN_PROPAGATION, and predux_helper().

Referenced by packetmath_notcomplex().

predux_min() [2/2]

template<int NaNPropagation, typename Packet >

EIGEN_DEVICE_FUNC unpacket_traits<Packet>::type Eigen::internal::predux_min ( const Packet &  a )

inline

                                                                                          {
  typedef typename unpacket_traits<Packet>::type Scalar;
  return predux_helper(a, EIGEN_BINARY_OP_NAN_PROPAGATION(Scalar, (pmin<NaNPropagation, Scalar>)));
}

References a, EIGEN_BINARY_OP_NAN_PROPAGATION, and predux_helper().

predux_min< Packet16bf >()

template<>

EIGEN_STRONG_INLINE bfloat16 Eigen::internal::predux_min< Packet16bf >

(

const Packet16bf &

from

)

                                                                            {
  return static_cast<bfloat16>(predux_min<Packet16f>(Bf16ToF32(from)));
}

References Bf16ToF32(), and predux_min< Packet16f >().

predux_min< Packet16c >()

template<>

EIGEN_STRONG_INLINE int8_t Eigen::internal::predux_min< Packet16c >

(

const Packet16c &

a

)

                                                                          {
  Packet16c pair, quad, octo, result;
 
  pair = vec_min(a, vec_sld(a, a, 8));
  quad = vec_min(pair, vec_sld(pair, pair, 4));
  octo = vec_min(quad, vec_sld(quad, quad, 2));
  result = vec_min(octo, vec_sld(octo, octo, 1));
 
  return pfirst(result);
}

References a, and pfirst().

predux_min< Packet16f >()

template<>

EIGEN_STRONG_INLINE float Eigen::internal::predux_min< Packet16f >

(

const Packet16f &

a

)

                                                                    {
  __m128 lane0 = _mm512_extractf32x4_ps(a, 0);
  __m128 lane1 = _mm512_extractf32x4_ps(a, 1);
  __m128 lane2 = _mm512_extractf32x4_ps(a, 2);
  __m128 lane3 = _mm512_extractf32x4_ps(a, 3);
  __m128 res = _mm_min_ps(_mm_min_ps(lane0, lane1), _mm_min_ps(lane2, lane3));
  res = _mm_min_ps(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 3, 2)));
  return pfirst(_mm_min_ps(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 0, 1))));
}

References a, pfirst(), and res.

Referenced by predux_min< Packet16bf >(), and predux_min< Packet16h >().

predux_min< Packet16h >()

template<>

EIGEN_STRONG_INLINE Eigen::half Eigen::internal::predux_min< Packet16h >

(

const Packet16h &

a

)

                                                                        {
  Packet16f af = half2float(a);
  float reduced = predux_min<Packet16f>(af);
  return Eigen::half(reduced);
}

References a, half2float(), and predux_min< Packet16f >().

predux_min< Packet16i >()

template<>

EIGEN_STRONG_INLINE int Eigen::internal::predux_min< Packet16i >

(

const Packet16i &

a

)

                                                                  {
  return _mm512_reduce_min_epi32(a);
}

References a.

predux_min< Packet16uc >()

template<>

EIGEN_STRONG_INLINE uint8_t Eigen::internal::predux_min< Packet16uc >

(

const Packet16uc &

a

)

                                                                              {
  Packet16uc pair, quad, octo, result;
 
  pair = vec_min(a, vec_sld(a, a, 8));
  quad = vec_min(pair, vec_sld(pair, pair, 4));
  octo = vec_min(quad, vec_sld(quad, quad, 2));
  result = vec_min(octo, vec_sld(octo, octo, 1));
 
  return pfirst(result);
}

References a, and pfirst().

predux_min< Packet2d >()

template<>

EIGEN_STRONG_INLINE double Eigen::internal::predux_min< Packet2d >

(

const Packet2d &

a

)

                                                                   {
  return pfirst(__lsx_vfmin_d(a, preverse(a)));
}

References a, pfirst(), and preverse().

predux_min< Packet2f >()

template<>

EIGEN_STRONG_INLINE float Eigen::internal::predux_min< Packet2f >

(

const Packet2f &

a

)

                                                                  {
  return vget_lane_f32(vpmin_f32(a, a), 0);
}

References a.

predux_min< Packet2i >()

template<>

EIGEN_STRONG_INLINE int32_t Eigen::internal::predux_min< Packet2i >

(

const Packet2i &

a

)

                                                                    {
  return vget_lane_s32(vpmin_s32(a, a), 0);
}

References a.

predux_min< Packet2l >()

template<>

EIGEN_STRONG_INLINE int64_t Eigen::internal::predux_min< Packet2l >

(

const Packet2l &

a

)

                                                                    {
  return pfirst((Packet2l)__lsx_vmin_d(a, preverse(a)));
}

References a, pfirst(), and preverse().

predux_min< Packet2ui >()

template<>

EIGEN_STRONG_INLINE uint32_t Eigen::internal::predux_min< Packet2ui >

(

const Packet2ui &

a

)

                                                                       {
  return vget_lane_u32(vpmin_u32(a, a), 0);
}

References a.

predux_min< Packet2ul >()

template<>

EIGEN_STRONG_INLINE uint64_t Eigen::internal::predux_min< Packet2ul >

(

const Packet2ul &

a

)

                                                                       {
  return pfirst((Packet2ul)__lsx_vmin_du(a, preverse(a)));
}

References a, pfirst(), and preverse().

predux_min< Packet4bf >()

template<>

EIGEN_STRONG_INLINE bfloat16 Eigen::internal::predux_min< Packet4bf >

(

const Packet4bf &

a

)

                                                                       {
  return static_cast<bfloat16>(predux_min<Packet4f>(Bf16ToF32(a)));
}

References a, Bf16ToF32(), and predux_min< Packet4f >().

predux_min< Packet4c >()

template<>

EIGEN_STRONG_INLINE int8_t Eigen::internal::predux_min< Packet4c >

(

const Packet4c &

a

)

                                                                   {
  const int8x8_t a_dup = vreinterpret_s8_s32(vdup_n_s32(a));
  int8x8_t min = vpmin_s8(a_dup, a_dup);
  min = vpmin_s8(min, min);
  return vget_lane_s8(min, 0);
}

References a, and Eigen::min().

predux_min< Packet4d >()

template<>

EIGEN_STRONG_INLINE double Eigen::internal::predux_min< Packet4d >

(

const Packet4d &

a

)

                                                                   {
  Packet4d tmp = _mm256_min_pd(a, _mm256_permute2f128_pd(a, a, 1));
  return pfirst(_mm256_min_pd(tmp, _mm256_shuffle_pd(tmp, tmp, 1)));
}

References a, pfirst(), and tmp.

predux_min< Packet4f >()

template<>

EIGEN_STRONG_INLINE float Eigen::internal::predux_min< Packet4f >

(

const Packet4f &

a

)

                                                                  {
  return predux_min4<Packet4f>(a);
}

References a.

Referenced by predux_min< Packet4bf >(), and predux_min< Packet8bf >().

predux_min< Packet4i >()

template<>

EIGEN_STRONG_INLINE int Eigen::internal::predux_min< Packet4i >

(

const Packet4i &

a

)

                                                                {
  return predux_min4<Packet4i>(a);
}

References a.

predux_min< Packet4s >()

template<>

EIGEN_STRONG_INLINE int16_t Eigen::internal::predux_min< Packet4s >

(

const Packet4s &

a

)

                                                                    {
  const int16x4_t min = vpmin_s16(a, a);
  return vget_lane_s16(vpmin_s16(min, min), 0);
}

References a, and Eigen::min().

predux_min< Packet4uc >()

template<>

EIGEN_STRONG_INLINE uint8_t Eigen::internal::predux_min< Packet4uc >

(

const Packet4uc &

a

)

                                                                      {
  const uint8x8_t a_dup = vreinterpret_u8_u32(vdup_n_u32(a));
  uint8x8_t min = vpmin_u8(a_dup, a_dup);
  min = vpmin_u8(min, min);
  return vget_lane_u8(min, 0);
}

References a, and Eigen::min().

predux_min< Packet4ui >()

template<>

EIGEN_STRONG_INLINE uint32_t Eigen::internal::predux_min< Packet4ui >

(

const Packet4ui &

a

)

                                                                       {
  Packet4ui tmp = __lsx_vmin_wu(a, __lsx_vshuf4i_w((__m128i)a, 0x4E));
  return pfirst((Packet4ui)__lsx_vmin_wu(tmp, __lsx_vshuf4i_w((__m128i)tmp, 0xB1)));
}

References a, pfirst(), and tmp.

predux_min< Packet4us >()

template<>

EIGEN_STRONG_INLINE uint16_t Eigen::internal::predux_min< Packet4us >

(

const Packet4us &

a

)

                                                                       {
  const uint16x4_t min = vpmin_u16(a, a);
  return vget_lane_u16(vpmin_u16(min, min), 0);
}

References a, and Eigen::min().

predux_min< Packet8bf >()

template<>

EIGEN_STRONG_INLINE bfloat16 Eigen::internal::predux_min< Packet8bf >

(

const Packet8bf &

a

)

                                                                       {
  float redux_even = predux_min<Packet4f>(Bf16ToF32Even(a));
  float redux_odd = predux_min<Packet4f>(Bf16ToF32Odd(a));
  float f32_result = (std::min)(redux_even, redux_odd);
  return bfloat16(f32_result);
}

References a, Bf16ToF32Even(), Bf16ToF32Odd(), min, and predux_min< Packet4f >().

predux_min< Packet8c >()

template<>

EIGEN_STRONG_INLINE int8_t Eigen::internal::predux_min< Packet8c >

(

const Packet8c &

a

)

                                                                   {
  int8x8_t min = vpmin_s8(a, a);
  min = vpmin_s8(min, min);
  min = vpmin_s8(min, min);
  return vget_lane_s8(min, 0);
}

References a, and Eigen::min().

predux_min< Packet8d >()

template<>

EIGEN_STRONG_INLINE double Eigen::internal::predux_min< Packet8d >

(

const Packet8d &

a

)

                                                                   {
  __m256d lane0 = _mm512_extractf64x4_pd(a, 0);
  __m256d lane1 = _mm512_extractf64x4_pd(a, 1);
  __m256d res = _mm256_min_pd(lane0, lane1);
  res = _mm256_min_pd(res, _mm256_permute2f128_pd(res, res, 1));
  return pfirst(_mm256_min_pd(res, _mm256_shuffle_pd(res, res, 1)));
}

References a, pfirst(), and res.

predux_min< Packet8f >()

template<>

EIGEN_STRONG_INLINE float Eigen::internal::predux_min< Packet8f >

(

const Packet8f &

a

)

                                                                  {
  Packet8f tmp = _mm256_min_ps(a, _mm256_permute2f128_ps(a, a, 1));
  tmp = _mm256_min_ps(tmp, _mm256_shuffle_ps(tmp, tmp, _MM_SHUFFLE(1, 0, 3, 2)));
  return pfirst(_mm256_min_ps(tmp, _mm256_shuffle_ps(tmp, tmp, 1)));
}

References a, pfirst(), and tmp.

Referenced by predux_min< Packet8h >().

predux_min< Packet8h >()

template<>

EIGEN_STRONG_INLINE Eigen::half Eigen::internal::predux_min< Packet8h >

(

const Packet8h &

a

)

                                                                      {
  Packet8f af = half2float(a);
  float reduced = predux_min<Packet8f>(af);
  return Eigen::half(reduced);
}

References a, half2float(), and predux_min< Packet8f >().

predux_min< Packet8l >()

template<>

EIGEN_STRONG_INLINE int64_t Eigen::internal::predux_min< Packet8l >

(

const Packet8l &

a

)

                                                                    {
  return _mm512_reduce_min_epi64(a);
}

References a.

predux_min< Packet8s >()

template<>

EIGEN_STRONG_INLINE int16_t Eigen::internal::predux_min< Packet8s >

(

const Packet8s &

a

)

                                                                      {
  Packet8s pair, quad, octo;
 
  // pair = { Min(a0,a4), Min(a1,a5), Min(a2,a6), Min(a3,a7) }
  pair = vec_min(a, vec_sld(a, a, 8));
 
  // quad = { Min(a0, a4, a2, a6), Min(a1, a5, a3, a7) }
  quad = vec_min(pair, vec_sld(pair, pair, 4));
 
  // octo = { Min(a0, a4, a2, a6, a1, a5, a3, a7) }
  octo = vec_min(quad, vec_sld(quad, quad, 2));
  return pfirst(octo);
}

References a, and pfirst().

predux_min< Packet8uc >()

template<>

EIGEN_STRONG_INLINE uint8_t Eigen::internal::predux_min< Packet8uc >

(

const Packet8uc &

a

)

                                                                      {
  uint8x8_t min = vpmin_u8(a, a);
  min = vpmin_u8(min, min);
  min = vpmin_u8(min, min);
  return vget_lane_u8(min, 0);
}

References a, and Eigen::min().

predux_min< Packet8us >()

template<>

EIGEN_STRONG_INLINE uint16_t Eigen::internal::predux_min< Packet8us >

(

const Packet8us &

a

)

                                                                                 {
  Packet8us pair, quad, octo;
 
  // pair = { Min(a0,a4), Min(a1,a5), Min(a2,a6), Min(a3,a7) }
  pair = vec_min(a, vec_sld(a, a, 8));
 
  // quad = { Min(a0, a4, a2, a6), Min(a1, a5, a3, a7) }
  quad = vec_min(pair, vec_sld(pair, pair, 4));
 
  // octo = { Min(a0, a4, a2, a6, a1, a5, a3, a7) }
  octo = vec_min(quad, vec_sld(quad, quad, 2));
  return pfirst(octo);
}

References a, and pfirst().

predux_min< PacketXf >()

template<>

EIGEN_STRONG_INLINE float Eigen::internal::predux_min< PacketXf >

(

const PacketXf &

a

)

                                                                  {
  return svminv_f32(svptrue_b32(), a);
}

References a.

predux_min< PacketXi >()

template<>

EIGEN_STRONG_INLINE numext::int32_t Eigen::internal::predux_min< PacketXi >

(

const PacketXi &

a

)

                                                                          {
  return svminv_s32(svptrue_b32(), a);
}

References a.

predux_mul()

template<typename Packet >

EIGEN_DEVICE_FUNC unpacket_traits<Packet>::type Eigen::internal::predux_mul ( const Packet &  a )

inline

Returns

the product of the elements of a

                                                                                          {
  typedef typename unpacket_traits<Packet>::type Scalar;
  return predux_helper(a, EIGEN_BINARY_OP_NAN_PROPAGATION(Scalar, (pmul<Scalar>)));
}

References a, EIGEN_BINARY_OP_NAN_PROPAGATION, and predux_helper().

Referenced by Eigen::internal::ProdReducer< T >::finalizeBoth(), packetmath(), Eigen::internal::scalar_product_op< LhsScalar, RhsScalar >::predux(), predux_mul< Packet16h >(), predux_mul< Packet4cd >(), predux_mul< Packet4cf >(), and predux_mul< Packet8cf >().

predux_mul< Packet16b >()

template<>

EIGEN_STRONG_INLINE bool Eigen::internal::predux_mul< Packet16b >

(

const Packet16b &

a

)

                                                                   {
  Packet4i tmp = _mm_and_si128(a, _mm_unpackhi_epi64(a, a));
  return ((pfirst<Packet4i>(tmp) == 0x01010101) && (pfirst<Packet4i>(_mm_shuffle_epi32(tmp, 1)) == 0x01010101));
}

References a, pfirst< Packet4i >(), and tmp.

predux_mul< Packet16bf >()

template<>

EIGEN_STRONG_INLINE bfloat16 Eigen::internal::predux_mul< Packet16bf >

(

const Packet16bf &

from

)

                                                                            {
  return static_cast<bfloat16>(predux_mul<Packet16f>(Bf16ToF32(from)));
}

References Bf16ToF32(), and predux_mul< Packet16f >().

predux_mul< Packet16c >()

template<>

EIGEN_STRONG_INLINE int8_t Eigen::internal::predux_mul< Packet16c >

(

const Packet16c &

a

)

                                                                          {
  Packet16c pair, quad, octo, result;
 
  pair = vec_mul(a, vec_sld(a, a, 8));
  quad = vec_mul(pair, vec_sld(pair, pair, 4));
  octo = vec_mul(quad, vec_sld(quad, quad, 2));
  result = vec_mul(octo, vec_sld(octo, octo, 1));
 
  return pfirst(result);
}

References a, and pfirst().

predux_mul< Packet16f >()

template<>

EIGEN_STRONG_INLINE float Eigen::internal::predux_mul< Packet16f >

(

const Packet16f &

a

)

                                                                    {
// #ifdef EIGEN_VECTORIZE_AVX512DQ
#if 0
  Packet8f lane0 = _mm512_extractf32x8_ps(a, 0);
  Packet8f lane1 = _mm512_extractf32x8_ps(a, 1);
  Packet8f res = pmul(lane0, lane1);
  res = pmul(res, _mm256_permute2f128_ps(res, res, 1));
  res = pmul(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 3, 2)));
  return pfirst(pmul(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 0, 1))));
#else
  __m128 lane0 = _mm512_extractf32x4_ps(a, 0);
  __m128 lane1 = _mm512_extractf32x4_ps(a, 1);
  __m128 lane2 = _mm512_extractf32x4_ps(a, 2);
  __m128 lane3 = _mm512_extractf32x4_ps(a, 3);
  __m128 res = pmul(pmul(lane0, lane1), pmul(lane2, lane3));
  res = pmul(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 3, 2)));
  return pfirst(pmul(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 0, 1))));
#endif
}

References a, pfirst(), pmul(), and res.

Referenced by predux_mul< Packet16bf >().

predux_mul< Packet16h >()

template<>

EIGEN_STRONG_INLINE half Eigen::internal::predux_mul< Packet16h >

(

const Packet16h &

from

)

                                                                      {
  Packet16f from_float = half2float(from);
  return half(predux_mul(from_float));
}

References half2float(), and predux_mul().

predux_mul< Packet16i >()

template<>

EIGEN_STRONG_INLINE int Eigen::internal::predux_mul< Packet16i >

(

const Packet16i &

a

)

                                                                  {
  return _mm512_reduce_mul_epi32(a);
}

References a.

predux_mul< Packet16uc >()

template<>

EIGEN_STRONG_INLINE uint8_t Eigen::internal::predux_mul< Packet16uc >

(

const Packet16uc &

a

)

                                                                              {
  Packet16uc pair, quad, octo, result;
 
  pair = vec_mul(a, vec_sld(a, a, 8));
  quad = vec_mul(pair, vec_sld(pair, pair, 4));
  octo = vec_mul(quad, vec_sld(quad, quad, 2));
  result = vec_mul(octo, vec_sld(octo, octo, 1));
 
  return pfirst(result);
}

References a, and pfirst().

predux_mul< Packet1cd >()

template<>

EIGEN_STRONG_INLINE std::complex< double > Eigen::internal::predux_mul< Packet1cd >

(

const Packet1cd &

a

)

                                                                                 {
  return pfirst(a);
}

References a, and pfirst().

predux_mul< Packet1cf >()

template<>

EIGEN_STRONG_INLINE std::complex<float> Eigen::internal::predux_mul< Packet1cf >

(

const Packet1cf &

a

)

                                                                                {
  std::complex<float> s;
  vst1_f32((float*)&s, a.v);
  return s;
}

References a, and s.

predux_mul< Packet2cd >()

template<>

EIGEN_STRONG_INLINE std::complex<double> Eigen::internal::predux_mul< Packet2cd >

(

const Packet2cd &

a

)

                                                                                 {
  return predux(pmul(Packet1cd(_mm256_extractf128_pd(a.v, 0)), Packet1cd(_mm256_extractf128_pd(a.v, 1))));
}

References a, pmul(), and predux().

predux_mul< Packet2cf >()

template<>

EIGEN_STRONG_INLINE std::complex< float > Eigen::internal::predux_mul< Packet2cf >

(

const Packet2cf &

a

)

                                                                                {
  Packet4f b;
  Packet2cf prod;
  b = vec_sld(a.v, a.v, 8);
  prod = pmul<Packet2cf>(a, Packet2cf(b));
 
  return pfirst<Packet2cf>(prod);
}

References a, b, pfirst< Packet2cf >(), pmul< Packet2cf >(), and Eigen::prod().

predux_mul< Packet2d >()

template<>

EIGEN_STRONG_INLINE double Eigen::internal::predux_mul< Packet2d >

(

const Packet2d &

a

)

                                                                   {
  return pfirst<Packet2d>(__lsx_vfmul_d(a, preverse(a)));
}

References a, pfirst< Packet2d >(), and preverse().

predux_mul< Packet2f >()

template<>

EIGEN_STRONG_INLINE float Eigen::internal::predux_mul< Packet2f >

(

const Packet2f &

a

)

                                                                  {
  return vget_lane_f32(a, 0) * vget_lane_f32(a, 1);
}

References a.

predux_mul< Packet2i >()

template<>

EIGEN_STRONG_INLINE int32_t Eigen::internal::predux_mul< Packet2i >

(

const Packet2i &

a

)

                                                                    {
  return vget_lane_s32(a, 0) * vget_lane_s32(a, 1);
}

References a.

predux_mul< Packet2l >()

template<>

EIGEN_STRONG_INLINE int64_t Eigen::internal::predux_mul< Packet2l >

(

const Packet2l &

a

)

                                                                    {
  return (int64_t)__lsx_vpickve2gr_d(__lsx_vmulwev_q_d(a, preverse(a)), 0);
}

References a, and preverse().

predux_mul< Packet2ui >()

template<>

EIGEN_STRONG_INLINE uint32_t Eigen::internal::predux_mul< Packet2ui >

(

const Packet2ui &

a

)

                                                                       {
  return vget_lane_u32(a, 0) * vget_lane_u32(a, 1);
}

References a.

predux_mul< Packet2ul >()

template<>

EIGEN_STRONG_INLINE uint64_t Eigen::internal::predux_mul< Packet2ul >

(

const Packet2ul &

a

)

                                                                       {
  return (uint64_t)__lsx_vpickve2gr_d(__lsx_vmulwev_q_du(a, preverse(a)), 0);
}

References a, and preverse().

predux_mul< Packet4bf >()

template<>

EIGEN_STRONG_INLINE bfloat16 Eigen::internal::predux_mul< Packet4bf >

(

const Packet4bf &

a

)

                                                                       {
  return static_cast<bfloat16>(predux_mul<Packet4f>(Bf16ToF32(a)));
}

References a, Bf16ToF32(), and predux_mul< Packet4f >().

predux_mul< Packet4c >()

template<>

EIGEN_STRONG_INLINE int8_t Eigen::internal::predux_mul< Packet4c >

(

const Packet4c &

a

)

                                                                   {
  int8x8_t prod = vreinterpret_s8_s32(vdup_n_s32(a));
  prod = vmul_s8(prod, vrev16_s8(prod));
  return vget_lane_s8(prod, 0) * vget_lane_s8(prod, 2);
}

References a, and Eigen::prod().

predux_mul< Packet4cd >()

template<>

EIGEN_STRONG_INLINE std::complex<double> Eigen::internal::predux_mul< Packet4cd >

(

const Packet4cd &

a

)

                                                                                 {
  return predux_mul(pmul(Packet2cd(_mm512_extractf64x4_pd(a.v, 0)), Packet2cd(_mm512_extractf64x4_pd(a.v, 1))));
}

References a, pmul(), and predux_mul().

predux_mul< Packet4cf >()

template<>

EIGEN_STRONG_INLINE std::complex<float> Eigen::internal::predux_mul< Packet4cf >

(

const Packet4cf &

a

)

                                                                                {
  return predux_mul(pmul(Packet2cf(_mm256_extractf128_ps(a.v, 0)), Packet2cf(_mm256_extractf128_ps(a.v, 1))));
}

References a, pmul(), and predux_mul().

predux_mul< Packet4d >()

template<>

EIGEN_STRONG_INLINE double Eigen::internal::predux_mul< Packet4d >

(

const Packet4d &

a

)

                                                                   {
  Packet4d tmp;
  tmp = _mm256_mul_pd(a, _mm256_permute2f128_pd(a, a, 1));
  return pfirst(_mm256_mul_pd(tmp, _mm256_shuffle_pd(tmp, tmp, 1)));
}

References a, pfirst(), and tmp.

predux_mul< Packet4f >()

template<>

EIGEN_STRONG_INLINE float Eigen::internal::predux_mul< Packet4f >

(

const Packet4f &

a

)

                                                                  {
  Packet4f prod;
  prod = pmul(a, vec_sld(a, a, 8));
  return pfirst(pmul(prod, vec_sld(prod, prod, 4)));
}

References a, pfirst(), pmul(), and Eigen::prod().

Referenced by predux_mul< Packet4bf >(), and predux_mul< Packet8bf >().

predux_mul< Packet4i >()

template<>

EIGEN_STRONG_INLINE int Eigen::internal::predux_mul< Packet4i >

(

const Packet4i &

a

)

                                                                {
  EIGEN_ALIGN16 int aux[4];
  pstore(aux, a);
  return aux[0] * aux[1] * aux[2] * aux[3];
}

References a, EIGEN_ALIGN16, and pstore().

predux_mul< Packet4s >()

template<>

EIGEN_STRONG_INLINE int16_t Eigen::internal::predux_mul< Packet4s >

(

const Packet4s &

a

)

                                                                    {
  const int16x4_t prod = vmul_s16(a, vrev32_s16(a));
  return vget_lane_s16(prod, 0) * vget_lane_s16(prod, 2);
}

References a, and Eigen::prod().

predux_mul< Packet4uc >()

template<>

EIGEN_STRONG_INLINE uint8_t Eigen::internal::predux_mul< Packet4uc >

(

const Packet4uc &

a

)

                                                                      {
  uint8x8_t prod = vreinterpret_u8_u32(vdup_n_u32(a));
  prod = vmul_u8(prod, vrev16_u8(prod));
  return vget_lane_u8(prod, 0) * vget_lane_u8(prod, 2);
}

References a, and Eigen::prod().

predux_mul< Packet4ui >()

template<>

EIGEN_STRONG_INLINE uint32_t Eigen::internal::predux_mul< Packet4ui >

(

const Packet4ui &

a

)

                                                                       {
  Packet2ul tmp = __lsx_vmulwev_d_wu(a, preverse(a));
  return (uint32_t)__lsx_vpickve2gr_d(__lsx_vmulwev_q_d(tmp, preverse(tmp)), 0);
}

References a, preverse(), and tmp.

predux_mul< Packet4us >()

template<>

EIGEN_STRONG_INLINE uint16_t Eigen::internal::predux_mul< Packet4us >

(

const Packet4us &

a

)

                                                                       {
  const uint16x4_t prod = vmul_u16(a, vrev32_u16(a));
  return vget_lane_u16(prod, 0) * vget_lane_u16(prod, 2);
}

References a, and Eigen::prod().

predux_mul< Packet8bf >()

template<>

EIGEN_STRONG_INLINE bfloat16 Eigen::internal::predux_mul< Packet8bf >

(

const Packet8bf &

a

)

                                                                       {
  float redux_even = predux_mul<Packet4f>(Bf16ToF32Even(a));
  float redux_odd = predux_mul<Packet4f>(Bf16ToF32Odd(a));
  float f32_result = redux_even * redux_odd;
  return bfloat16(f32_result);
}

References a, Bf16ToF32Even(), Bf16ToF32Odd(), and predux_mul< Packet4f >().

predux_mul< Packet8c >()

template<>

EIGEN_STRONG_INLINE int8_t Eigen::internal::predux_mul< Packet8c >

(

const Packet8c &

a

)

                                                                   {
  int8x8_t prod = vmul_s8(a, vrev16_s8(a));
  prod = vmul_s8(prod, vrev32_s8(prod));
  return vget_lane_s8(prod, 0) * vget_lane_s8(prod, 4);
}

References a, and Eigen::prod().

predux_mul< Packet8cf >()

template<>

EIGEN_STRONG_INLINE std::complex<float> Eigen::internal::predux_mul< Packet8cf >

(

const Packet8cf &

a

)

                                                                                {
  return predux_mul(pmul(Packet4cf(extract256<0>(a.v)), Packet4cf(extract256<1>(a.v))));
}

References a, pmul(), and predux_mul().

predux_mul< Packet8d >()

template<>

EIGEN_STRONG_INLINE double Eigen::internal::predux_mul< Packet8d >

(

const Packet8d &

a

)

                                                                   {
  __m256d lane0 = _mm512_extractf64x4_pd(a, 0);
  __m256d lane1 = _mm512_extractf64x4_pd(a, 1);
  __m256d res = pmul(lane0, lane1);
  res = pmul(res, _mm256_permute2f128_pd(res, res, 1));
  return pfirst(pmul(res, _mm256_shuffle_pd(res, res, 1)));
}

References a, pfirst(), pmul(), and res.

predux_mul< Packet8f >()

template<>

EIGEN_STRONG_INLINE float Eigen::internal::predux_mul< Packet8f >

(

const Packet8f &

a

)

                                                                  {
  Packet8f tmp;
  tmp = _mm256_mul_ps(a, _mm256_permute2f128_ps(a, a, 1));
  tmp = _mm256_mul_ps(tmp, _mm256_shuffle_ps(tmp, tmp, _MM_SHUFFLE(1, 0, 3, 2)));
  return pfirst(_mm256_mul_ps(tmp, _mm256_shuffle_ps(tmp, tmp, 1)));
}

References a, pfirst(), and tmp.

Referenced by predux_mul< Packet8h >().

predux_mul< Packet8h >()

template<>

EIGEN_STRONG_INLINE Eigen::half Eigen::internal::predux_mul< Packet8h >

(

const Packet8h &

a

)

                                                                      {
  Packet8f af = half2float(a);
  float reduced = predux_mul<Packet8f>(af);
  return Eigen::half(reduced);
}

References a, half2float(), and predux_mul< Packet8f >().

predux_mul< Packet8l >()

template<>

EIGEN_STRONG_INLINE int64_t Eigen::internal::predux_mul< Packet8l >

(

const Packet8l &

a

)

                                                                    {
  return _mm512_reduce_mul_epi64(a);
}

References a.

predux_mul< Packet8s >()

template<>

EIGEN_STRONG_INLINE int16_t Eigen::internal::predux_mul< Packet8s >

(

const Packet8s &

a

)

                                                                      {
  Packet8s pair, quad, octo;
 
  pair = vec_mul(a, vec_sld(a, a, 8));
  quad = vec_mul(pair, vec_sld(pair, pair, 4));
  octo = vec_mul(quad, vec_sld(quad, quad, 2));
 
  return pfirst(octo);
}

References a, and pfirst().

predux_mul< Packet8uc >()

template<>

EIGEN_STRONG_INLINE uint8_t Eigen::internal::predux_mul< Packet8uc >

(

const Packet8uc &

a

)

                                                                      {
  uint8x8_t prod = vmul_u8(a, vrev16_u8(a));
  prod = vmul_u8(prod, vrev32_u8(prod));
  return vget_lane_u8(prod, 0) * vget_lane_u8(prod, 4);
}

References a, and Eigen::prod().

predux_mul< Packet8us >()

template<>

EIGEN_STRONG_INLINE uint16_t Eigen::internal::predux_mul< Packet8us >

(

const Packet8us &

a

)

                                                                                 {
  Packet8us pair, quad, octo;
 
  pair = vec_mul(a, vec_sld(a, a, 8));
  quad = vec_mul(pair, vec_sld(pair, pair, 4));
  octo = vec_mul(quad, vec_sld(quad, quad, 2));
 
  return pfirst(octo);
}

References a, and pfirst().

predux_mul< PacketXf >()

template<>

EIGEN_STRONG_INLINE float Eigen::internal::predux_mul< PacketXf >

(

const PacketXf &

a

)

                                                                  {
  EIGEN_STATIC_ASSERT((EIGEN_ARM64_SVE_VL % 128 == 0), EIGEN_INTERNAL_ERROR_PLEASE_FILE_A_BUG_REPORT);
  // Multiply the vector by its reverse
  svfloat32_t prod = svmul_f32_x(svptrue_b32(), a, svrev_f32(a));
  svfloat32_t half_prod;
 
  // Extract the high half of the vector. Depending on the VL more reductions need to be done
  if (EIGEN_ARM64_SVE_VL >= 2048) {
    half_prod = svtbl_f32(prod, svindex_u32(32, 1));
    prod = svmul_f32_x(svptrue_b32(), prod, half_prod);
  }
  if (EIGEN_ARM64_SVE_VL >= 1024) {
    half_prod = svtbl_f32(prod, svindex_u32(16, 1));
    prod = svmul_f32_x(svptrue_b32(), prod, half_prod);
  }
  if (EIGEN_ARM64_SVE_VL >= 512) {
    half_prod = svtbl_f32(prod, svindex_u32(8, 1));
    prod = svmul_f32_x(svptrue_b32(), prod, half_prod);
  }
  if (EIGEN_ARM64_SVE_VL >= 256) {
    half_prod = svtbl_f32(prod, svindex_u32(4, 1));
    prod = svmul_f32_x(svptrue_b32(), prod, half_prod);
  }
  // Last reduction
  half_prod = svtbl_f32(prod, svindex_u32(2, 1));
  prod = svmul_f32_x(svptrue_b32(), prod, half_prod);
 
  // The reduction is done to the first element.
  return pfirst<PacketXf>(prod);
}

References a, EIGEN_STATIC_ASSERT, pfirst< PacketXf >(), and Eigen::prod().

predux_mul< PacketXi >()

template<>

EIGEN_STRONG_INLINE numext::int32_t Eigen::internal::predux_mul< PacketXi >

(

const PacketXi &

a

)

                                                                          {
  EIGEN_STATIC_ASSERT((EIGEN_ARM64_SVE_VL % 128 == 0), EIGEN_INTERNAL_ERROR_PLEASE_FILE_A_BUG_REPORT);
 
  // Multiply the vector by its reverse
  svint32_t prod = svmul_s32_x(svptrue_b32(), a, svrev_s32(a));
  svint32_t half_prod;
 
  // Extract the high half of the vector. Depending on the VL more reductions need to be done
  if (EIGEN_ARM64_SVE_VL >= 2048) {
    half_prod = svtbl_s32(prod, svindex_u32(32, 1));
    prod = svmul_s32_x(svptrue_b32(), prod, half_prod);
  }
  if (EIGEN_ARM64_SVE_VL >= 1024) {
    half_prod = svtbl_s32(prod, svindex_u32(16, 1));
    prod = svmul_s32_x(svptrue_b32(), prod, half_prod);
  }
  if (EIGEN_ARM64_SVE_VL >= 512) {
    half_prod = svtbl_s32(prod, svindex_u32(8, 1));
    prod = svmul_s32_x(svptrue_b32(), prod, half_prod);
  }
  if (EIGEN_ARM64_SVE_VL >= 256) {
    half_prod = svtbl_s32(prod, svindex_u32(4, 1));
    prod = svmul_s32_x(svptrue_b32(), prod, half_prod);
  }
  // Last reduction
  half_prod = svtbl_s32(prod, svindex_u32(2, 1));
  prod = svmul_s32_x(svptrue_b32(), prod, half_prod);
 
  // The reduction is done to the first element.
  return pfirst<PacketXi>(prod);
}

References a, EIGEN_STATIC_ASSERT, pfirst< PacketXi >(), and Eigen::prod().

preduxVecResults()

template<Index num_acc>

EIGEN_ALWAYS_INLINE void Eigen::internal::preduxVecResults

(

Packet4f(&)

acc[num_acc][4]

)

                                                                       {
  BFLOAT16_UNROLL
  for (Index k = 0; k < num_acc; k += 4) {
    preduxVecResults2<num_acc>(acc, k + 0);
    if (num_acc > (k + 2)) {
      preduxVecResults2<num_acc>(acc, k + 2);
      acc[k + 0][0] = reinterpret_cast<Packet4f>(
          vec_mergeh(reinterpret_cast<Packet2ul>(acc[k + 0][0]), reinterpret_cast<Packet2ul>(acc[k + 2][0])));
    }
  }
}

References BFLOAT16_UNROLL, and k.

preduxVecResults2()

template<Index num_acc>

EIGEN_ALWAYS_INLINE void Eigen::internal::preduxVecResults2	(	Packet4f(&)	acc[num_acc][4],
		Index	k
	)

                                                                                 {
  if (num_acc > (k + 1)) {
    acc[k][0] = vec_mergeh(acc[k][0], acc[k + 1][0]);
    acc[k][1] = vec_mergeo(acc[k][1], acc[k + 1][1]);
    acc[k][2] = vec_mergel(acc[k][2], acc[k + 1][2]);
    acc[k][3] = vec_perm(acc[k][3], acc[k + 1][3], p16uc_ELEMENT_VEC3);
 
    acc[k][0] = (acc[k][0] + acc[k][2]) + (acc[k][1] + acc[k][3]);
  } else {
    acc[k][0] = vec_mergeh(acc[k][0], acc[k][1]);
    acc[k][0] += vec_mergel(acc[k][2], acc[k][3]);
#ifdef _BIG_ENDIAN
    acc[k][0] += vec_sld(acc[k][0], acc[k][0], 12);
#else
    acc[k][0] += vec_sld(acc[k][0], acc[k][0], 4);
#endif
  }
}

References k, and p16uc_ELEMENT_VEC3.

prefetch()

template<typename Scalar >

EIGEN_DEVICE_FUNC void Eigen::internal::prefetch ( const Scalar *  addr )

inline

tries to do cache prefetching of addr

                                                           {
#if defined(EIGEN_HIP_DEVICE_COMPILE)
  // do nothing
#elif defined(EIGEN_CUDA_ARCH)
#if defined(__LP64__) || EIGEN_OS_WIN64
  // 64-bit pointer operand constraint for inlined asm
  asm(" prefetch.L1 [ %1 ];" : "=l"(addr) : "l"(addr));
#else
  // 32-bit pointer operand constraint for inlined asm
  asm(" prefetch.L1 [ %1 ];" : "=r"(addr) : "r"(addr));
#endif
#elif (!EIGEN_COMP_MSVC) && (EIGEN_COMP_GNUC || EIGEN_COMP_CLANG || EIGEN_COMP_ICC)
  __builtin_prefetch(addr);
#endif
}

Referenced by Eigen::internal::gebp_kernel< LhsScalar, RhsScalar, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >::operator()(), Eigen::internal::lhs_process_one_packet< nr, LhsProgress, RhsProgress, LhsScalar, RhsScalar, ResScalar, AccPacket, LhsPacket, RhsPacket, ResPacket, GEBPTraits, LinearMapper, DataMapper >::operator()(), Eigen::internal::BlasLinearMapper< Scalar, Index, AlignmentType >::prefetch(), Eigen::internal::blas_data_mapper< Scalar, Index, StorageOrder, AlignmentType, 1 >::prefetch(), Eigen::internal::blas_data_mapper< Scalar, Index, StorageOrder, AlignmentType, Incr >::prefetch(), and Eigen::internal::BlasLinearMapper< Scalar, Index, AlignmentType, Incr >::prefetch().

prefetch< double >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::prefetch< double >

(

const double *

addr

)

                                                              {
  _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0);
}

prefetch< float >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::prefetch< float >

(

const float *

addr

)

                                                            {
  EIGEN_PPC_PREFETCH(addr);
}

References EIGEN_PPC_PREFETCH.

prefetch< int >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::prefetch< int >

(

const int *

addr

)

                                                        {
  EIGEN_PPC_PREFETCH(addr);
}

References EIGEN_PPC_PREFETCH.

prefetch< int16_t >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::prefetch< int16_t >

(

const int16_t *

addr

)

                                                                {
  __builtin_prefetch(addr);
}

prefetch< int32_t >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::prefetch< int32_t >

(

const int32_t *

addr

)

                                                                {
  __builtin_prefetch(addr);
}

prefetch< int64_t >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::prefetch< int64_t >

(

const int64_t *

addr

)

                                                                {
  __builtin_prefetch(addr);
}

prefetch< int8_t >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::prefetch< int8_t >

(

const int8_t *

addr

)

                                                              {
  __builtin_prefetch(addr);
}

prefetch< numext::int32_t >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::prefetch< numext::int32_t >

(

const numext::int32_t *

addr

)

                                                                            {
  svprfw(svptrue_b32(), addr, SV_PLDL1KEEP);
}

prefetch< std::complex< double > >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::prefetch< std::complex< double > >

(

const std::complex< double > *

addr

)

                                                                                       {
  __builtin_prefetch(addr);
}

prefetch< std::complex< float > >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::prefetch< std::complex< float > >

(

const std::complex< float > *

addr

)

                                                                                     {
  EIGEN_PPC_PREFETCH(addr);
}

References EIGEN_PPC_PREFETCH.

prefetch< uint16_t >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::prefetch< uint16_t >

(

const uint16_t *

addr

)

                                                                  {
  __builtin_prefetch(addr);
}

prefetch< uint32_t >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::prefetch< uint32_t >

(

const uint32_t *

addr

)

                                                                  {
  _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0);
}

prefetch< uint64_t >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::prefetch< uint64_t >

(

const uint64_t *

addr

)

                                                                  {
  __builtin_prefetch(addr);
}

prefetch< uint8_t >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::prefetch< uint8_t >

(

const uint8_t *

addr

)

                                                                {
  __builtin_prefetch(addr);
}

preinterpret()

template<typename Target , typename Packet >

EIGEN_DEVICE_FUNC Target Eigen::internal::preinterpret ( const Packet &  a )

inline

Returns

reinterpret_cast<Target>(a)

                                                              {
  return preinterpret_generic<Target, Packet>::run(a);
}

References a, and run().

preinterpret< Packet16c, Packet16uc >()

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::preinterpret< Packet16c, Packet16uc >

(

const Packet16uc &

a

)

                                                                                       {
  return (__m128i)a;
}

References a.

preinterpret< Packet16f, Packet16i >()

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::preinterpret< Packet16f, Packet16i >

(

const Packet16i &

a

)

                                                                                     {
  return _mm512_castsi512_ps(a);
}

References a.

preinterpret< Packet16f, Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::preinterpret< Packet16f, Packet4f >

(

const Packet4f &

a

)

                                                                                   {
  return _mm512_castps128_ps512(a);
}

References a.

preinterpret< Packet16f, Packet8d >()

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::preinterpret< Packet16f, Packet8d >

(

const Packet8d &

a

)

                                                                                   {
  return _mm512_castpd_ps(a);
}

References a.

preinterpret< Packet16f, Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::preinterpret< Packet16f, Packet8f >

(

const Packet8f &

a

)

                                                                                   {
  return _mm512_castps256_ps512(a);
}

References a.

preinterpret< Packet16i, Packet16f >()

template<>

EIGEN_STRONG_INLINE Packet16i Eigen::internal::preinterpret< Packet16i, Packet16f >

(

const Packet16f &

a

)

                                                                                     {
  return _mm512_castps_si512(a);
}

References a.

preinterpret< Packet16uc, Packet16c >()

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::preinterpret< Packet16uc, Packet16c >

(

const Packet16c &

a

)

                                                                                       {
  return (__m128i)a;
}

References a.

preinterpret< Packet2d, Packet2l >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::preinterpret< Packet2d, Packet2l >

(

const Packet2l &

a

)

                                                                                 {
  return (__m128d)((__m128i)a);
}

References a.

preinterpret< Packet2d, Packet2ul >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::preinterpret< Packet2d, Packet2ul >

(

const Packet2ul &

a

)

                                                                                   {
  return (__m128d)((__m128i)a);
}

References a.

preinterpret< Packet2d, Packet4d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::preinterpret< Packet2d, Packet4d >

(

const Packet4d &

a

)

                                                                                 {
  return _mm256_castpd256_pd128(a);
}

References a.

preinterpret< Packet2d, Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::preinterpret< Packet2d, Packet4f >

(

const Packet4f &

a

)

                                                                                 {
  return _mm_castps_pd(a);
}

References a.

preinterpret< Packet2d, Packet4i >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::preinterpret< Packet2d, Packet4i >

(

const Packet4i &

a

)

                                                                                 {
  return (__m128d)((__m128i)a);
}

References a.

preinterpret< Packet2d, Packet8d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::preinterpret< Packet2d, Packet8d >

(

const Packet8d &

a

)

                                                                                 {
  return _mm512_castpd512_pd128(a);
}

References a.

preinterpret< Packet2f, Packet2i >()

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::preinterpret< Packet2f, Packet2i >

(

const Packet2i &

a

)

                                                                                 {
  return Packet2f(vreinterpret_f32_s32(a));
}

References a.

preinterpret< Packet2f, Packet2ui >()

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::preinterpret< Packet2f, Packet2ui >

(

const Packet2ui &

a

)

                                                                                   {
  return Packet2f(vreinterpret_f32_u32(a));
}

References a.

preinterpret< Packet2f, Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::preinterpret< Packet2f, Packet4f >

(

const Packet4f &

a

)

                                                                                 {
  return Packet2f(vget_low_f32(a));
}

References a.

preinterpret< Packet2i, Packet2f >()

template<>

EIGEN_STRONG_INLINE Packet2i Eigen::internal::preinterpret< Packet2i, Packet2f >

(

const Packet2f &

a

)

                                                                                 {
  return Packet2i(vreinterpret_s32_f32(a));
}

References a.

preinterpret< Packet2i, Packet2ui >()

template<>

EIGEN_STRONG_INLINE Packet2i Eigen::internal::preinterpret< Packet2i, Packet2ui >

(

const Packet2ui &

a

)

                                                                                   {
  return Packet2i(vreinterpret_s32_u32(a));
}

References a.

preinterpret< Packet2i, Packet4i >()

template<>

EIGEN_STRONG_INLINE Packet2i Eigen::internal::preinterpret< Packet2i, Packet4i >

(

const Packet4i &

a

)

                                                                                 {
  return Packet2i(vget_low_s32(a));
}

References a.

preinterpret< Packet2l, Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::preinterpret< Packet2l, Packet2d >

(

const Packet2d &

a

)

                                                                                 {
  return (__m128i)a;
}

References a.

preinterpret< Packet2l, Packet2ul >()

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::preinterpret< Packet2l, Packet2ul >

(

const Packet2ul &

a

)

                                                                                   {
  return Packet2l(vreinterpretq_s64_u64(a));
}

References a.

preinterpret< Packet2ui, Packet2f >()

template<>

EIGEN_STRONG_INLINE Packet2ui Eigen::internal::preinterpret< Packet2ui, Packet2f >

(

const Packet2f &

a

)

                                                                                   {
  return Packet2ui(vreinterpret_u32_f32(a));
}

References a.

preinterpret< Packet2ui, Packet2i >()

template<>

EIGEN_STRONG_INLINE Packet2ui Eigen::internal::preinterpret< Packet2ui, Packet2i >

(

const Packet2i &

a

)

                                                                                   {
  return Packet2ui(vreinterpret_u32_s32(a));
}

References a.

preinterpret< Packet2ui, Packet4ui >()

template<>

EIGEN_STRONG_INLINE Packet2ui Eigen::internal::preinterpret< Packet2ui, Packet4ui >

(

const Packet4ui &

a

)

                                                                                     {
  return Packet2ui(vget_low_u32(a));
}

References a.

preinterpret< Packet2ul, Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::preinterpret< Packet2ul, Packet2d >

(

const Packet2d &

a

)

                                                                                   {
  return (__m128i)a;
}

References a.

preinterpret< Packet2ul, Packet2l >()

template<>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::preinterpret< Packet2ul, Packet2l >

(

const Packet2l &

a

)

                                                                                   {
  return (__m128i)a;
}

References a.

preinterpret< Packet4c, Packet16c >()

template<>

EIGEN_STRONG_INLINE Packet4c Eigen::internal::preinterpret< Packet4c, Packet16c >

(

const Packet16c &

a

)

                                                                                   {
  return preinterpret<Packet4c>(preinterpret<Packet8c>(a));
}

References a.

preinterpret< Packet4c, Packet4uc >()

template<>

EIGEN_STRONG_INLINE Packet4c Eigen::internal::preinterpret< Packet4c, Packet4uc >

(

const Packet4uc &

a

)

                                                                                   {
  return static_cast<Packet4c>(a);
}

References a.

preinterpret< Packet4c, Packet8c >()

template<>

EIGEN_STRONG_INLINE Packet4c Eigen::internal::preinterpret< Packet4c, Packet8c >

(

const Packet8c &

a

)

                                                                                 {
  return Packet4c(vget_lane_s32(vreinterpret_s32_s8(a), 0));
}

References a.

preinterpret< Packet4d, Packet8d >()

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::preinterpret< Packet4d, Packet8d >

(

const Packet8d &

a

)

                                                                                 {
  return _mm512_castpd512_pd256(a);
}

References a.

preinterpret< Packet4f, Packet16f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::preinterpret< Packet4f, Packet16f >

(

const Packet16f &

a

)

                                                                                   {
  return _mm512_castps512_ps128(a);
}

References a.

preinterpret< Packet4f, Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::preinterpret< Packet4f, Packet2d >

(

const Packet2d &

a

)

                                                                                 {
  return _mm_castpd_ps(a);
}

References a.

preinterpret< Packet4f, Packet4i >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::preinterpret< Packet4f, Packet4i >

(

const Packet4i &

a

)

                                                                                 {
  return reinterpret_cast<Packet4f>(a);
}

References a.

preinterpret< Packet4f, Packet4ui >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::preinterpret< Packet4f, Packet4ui >

(

const Packet4ui &

a

)

                                                                                   {
  return (__m128)((__m128i)a);
}

References a.

preinterpret< Packet4f, Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::preinterpret< Packet4f, Packet8f >

(

const Packet8f &

a

)

                                                                                 {
  return _mm256_castps256_ps128(a);
}

References a.

preinterpret< Packet4i, Packet16i >()

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::preinterpret< Packet4i, Packet16i >

(

const Packet16i &

a

)

                                                                                   {
  return _mm512_castsi512_si128(a);
}

References a.

preinterpret< Packet4i, Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::preinterpret< Packet4i, Packet2d >

(

const Packet2d &

a

)

                                                                                 {
  return (__m128i)a;
}

References a.

preinterpret< Packet4i, Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::preinterpret< Packet4i, Packet4f >

(

const Packet4f &

a

)

                                                                                 {
  return reinterpret_cast<Packet4i>(a);
}

References a.

preinterpret< Packet4i, Packet4ui >()

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::preinterpret< Packet4i, Packet4ui >

(

const Packet4ui &

a

)

                                                                                   {
  return (__m128i)a;
}

References a.

preinterpret< Packet4i, Packet8i >()

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::preinterpret< Packet4i, Packet8i >

(

const Packet8i &

a

)

                                                                                 {
  return _mm256_castsi256_si128(a);
}

References a.

preinterpret< Packet4s, Packet4us >()

template<>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::preinterpret< Packet4s, Packet4us >

(

const Packet4us &

a

)

                                                                                   {
  return Packet4s(vreinterpret_s16_u16(a));
}

References a.

preinterpret< Packet4s, Packet8s >()

template<>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::preinterpret< Packet4s, Packet8s >

(

const Packet8s &

a

)

                                                                                 {
  return Packet4s(vget_low_s16(a));
}

References a.

preinterpret< Packet4uc, Packet16uc >()

template<>

EIGEN_STRONG_INLINE Packet4uc Eigen::internal::preinterpret< Packet4uc, Packet16uc >

(

const Packet16uc &

a

)

                                                                                       {
  return preinterpret<Packet4uc>(preinterpret<Packet8uc>(a));
}

References a.

preinterpret< Packet4uc, Packet4c >()

template<>

EIGEN_STRONG_INLINE Packet4uc Eigen::internal::preinterpret< Packet4uc, Packet4c >

(

const Packet4c &

a

)

                                                                                   {
  return static_cast<Packet4uc>(a);
}

References a.

preinterpret< Packet4uc, Packet8uc >()

template<>

EIGEN_STRONG_INLINE Packet4uc Eigen::internal::preinterpret< Packet4uc, Packet8uc >

(

const Packet8uc &

a

)

                                                                                     {
  return Packet4uc(vget_lane_u32(vreinterpret_u32_u8(a), 0));
}

References a.

preinterpret< Packet4ui, Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::preinterpret< Packet4ui, Packet4f >

(

const Packet4f &

a

)

                                                                                   {
  return (__m128i)a;
}

References a.

preinterpret< Packet4ui, Packet4i >()

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::preinterpret< Packet4ui, Packet4i >

(

const Packet4i &

a

)

                                                                                   {
  return (__m128i)a;
}

References a.

preinterpret< Packet4ui, Packet8ui >()

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::preinterpret< Packet4ui, Packet8ui >

(

const Packet8ui &

a

)

                                                                                     {
  return _mm256_castsi256_si128(a);
}

References a.

preinterpret< Packet4us, Packet4s >()

template<>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::preinterpret< Packet4us, Packet4s >

(

const Packet4s &

a

)

                                                                                   {
  return Packet4us(vreinterpret_u16_s16(a));
}

References a.

preinterpret< Packet4us, Packet8us >()

template<>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::preinterpret< Packet4us, Packet8us >

(

const Packet8us &

a

)

                                                                                     {
  return Packet4us(vget_low_u16(a));
}

References a.

preinterpret< Packet8bf, Packet16bf >()

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::preinterpret< Packet8bf, Packet16bf >

(

const Packet16bf &

a

)

                                                                                       {
  return _mm256_castsi256_si128(a);
}

References a.

preinterpret< Packet8c, Packet16c >()

template<>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::preinterpret< Packet8c, Packet16c >

(

const Packet16c &

a

)

                                                                                   {
  return Packet8c(vget_low_s8(a));
}

References a.

preinterpret< Packet8c, Packet8uc >()

template<>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::preinterpret< Packet8c, Packet8uc >

(

const Packet8uc &

a

)

                                                                                   {
  return Packet8c(vreinterpret_s8_u8(a));
}

References a.

preinterpret< Packet8d, Packet16f >()

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::preinterpret< Packet8d, Packet16f >

(

const Packet16f &

a

)

                                                                                   {
  return _mm512_castps_pd(a);
}

References a.

preinterpret< Packet8d, Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::preinterpret< Packet8d, Packet2d >

(

const Packet2d &

a

)

                                                                                 {
  return _mm512_castpd128_pd512(a);
}

References a.

preinterpret< Packet8d, Packet4d >()

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::preinterpret< Packet8d, Packet4d >

(

const Packet4d &

a

)

                                                                                 {
  return _mm512_castpd256_pd512(a);
}

References a.

preinterpret< Packet8d, Packet8l >()

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::preinterpret< Packet8d, Packet8l >

(

const Packet8l &

a

)

                                                                                 {
  return _mm512_castsi512_pd(a);
}

References a.

preinterpret< Packet8f, Packet16f >()

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::preinterpret< Packet8f, Packet16f >

(

const Packet16f &

a

)

                                                                                   {
  return _mm512_castps512_ps256(a);
}

References a.

preinterpret< Packet8f, Packet8i >()

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::preinterpret< Packet8f, Packet8i >

(

const Packet8i &

a

)

                                                                                 {
  return _mm256_castsi256_ps(a);
}

References a.

preinterpret< Packet8h, Packet16h >()

template<>

EIGEN_STRONG_INLINE Packet8h Eigen::internal::preinterpret< Packet8h, Packet16h >

(

const Packet16h &

a

)

                                                                                   {
  return _mm256_castsi256_si128(a);
}

References a.

preinterpret< Packet8i, Packet16i >()

template<>

EIGEN_STRONG_INLINE Packet8i Eigen::internal::preinterpret< Packet8i, Packet16i >

(

const Packet16i &

a

)

                                                                                   {
  return _mm512_castsi512_si256(a);
}

References a.

preinterpret< Packet8i, Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet8i Eigen::internal::preinterpret< Packet8i, Packet8f >

(

const Packet8f &

a

)

                                                                                 {
  return _mm256_castps_si256(a);
}

References a.

preinterpret< Packet8i, Packet8ui >()

template<>

EIGEN_STRONG_INLINE Packet8i Eigen::internal::preinterpret< Packet8i, Packet8ui >

(

const Packet8ui &

a

)

                                                                                   {
  return Packet8i(a);
}

References a.

preinterpret< Packet8l, Packet8d >()

template<>

EIGEN_STRONG_INLINE Packet8l Eigen::internal::preinterpret< Packet8l, Packet8d >

(

const Packet8d &

a

)

                                                                                 {
  return _mm512_castpd_si512(a);
}

References a.

preinterpret< Packet8s, Packet8us >()

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::preinterpret< Packet8s, Packet8us >

(

const Packet8us &

a

)

                                                                                   {
  return (__m128i)a;
}

References a.

preinterpret< Packet8uc, Packet16uc >()

template<>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::preinterpret< Packet8uc, Packet16uc >

(

const Packet16uc &

a

)

                                                                                       {
  return Packet8uc(vget_low_u8(a));
}

References a.

preinterpret< Packet8uc, Packet8c >()

template<>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::preinterpret< Packet8uc, Packet8c >

(

const Packet8c &

a

)

                                                                                   {
  return Packet8uc(vreinterpret_u8_s8(a));
}

References a.

preinterpret< Packet8ui, Packet8i >()

template<>

EIGEN_STRONG_INLINE Packet8ui Eigen::internal::preinterpret< Packet8ui, Packet8i >

(

const Packet8i &

a

)

                                                                                   {
  return Packet8ui(a);
}

References a.

preinterpret< Packet8us, Packet8s >()

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::preinterpret< Packet8us, Packet8s >

(

const Packet8s &

a

)

                                                                                   {
  return (__m128i)a;
}

References a.

preinterpret< PacketXf, PacketXi >()

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::preinterpret< PacketXf, PacketXi >

(

const PacketXi &

a

)

                                                                                 {
  return svreinterpret_f32_s32(a);
}

References a.

preinterpret< PacketXi, PacketXf >()

template<>

EIGEN_STRONG_INLINE PacketXi Eigen::internal::preinterpret< PacketXi, PacketXf >

(

const PacketXf &

a

)

                                                                                 {
  return svreinterpret_s32_f32(a);
}

References a.

preverse() [1/43]

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::preverse ( const Packet &  a )

inline

Returns

the reversed elements of a

                                                          {
  return a;
}

References a.

preverse() [2/43]

template<>

EIGEN_STRONG_INLINE Packet16b Eigen::internal::preverse

(

const Packet16b &

a

)

                                                           {
#ifdef EIGEN_VECTORIZE_SSSE3
  __m128i mask = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
  return _mm_shuffle_epi8(a, mask);
#else
  Packet16b tmp = _mm_shuffle_epi32(a, _MM_SHUFFLE(0, 1, 2, 3));
  tmp = _mm_shufflehi_epi16(_mm_shufflelo_epi16(tmp, _MM_SHUFFLE(2, 3, 0, 1)), _MM_SHUFFLE(2, 3, 0, 1));
  return _mm_or_si128(_mm_slli_epi16(tmp, 8), _mm_srli_epi16(tmp, 8));
#endif
}

References a, and tmp.

preverse() [3/43]

template<>

EIGEN_STRONG_INLINE Packet16bf Eigen::internal::preverse

(

const Packet16bf &

a

)

                                                             {
  __m256i m = _mm256_setr_epi8(14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1, 14, 15, 12, 13, 10, 11, 8, 9, 6, 7,
                               4, 5, 2, 3, 0, 1);
 
  Packet16bf res;
  // Swap hi and lo first because shuffle is in 128-bit lanes.
  res = _mm256_permute2x128_si256(a, a, 1);
  // Shuffle 8-bit values in src within 2*128-bit lanes.
  return _mm256_shuffle_epi8(res, m);
}

References a, m, and res.

preverse() [4/43]

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::preverse

(

const Packet16c &

a

)

                                                           {
  return vec_perm(a, a, p16uc_REVERSE8);
}

References a, and p16uc_REVERSE8.

preverse() [5/43]

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::preverse

(

const Packet16f &

a

)

                                                           {
  return _mm512_permutexvar_ps(_mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), a);
}

References a.

preverse() [6/43]

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::preverse

(

const Packet16h &

a

)

                                                           {
  __m128i m = _mm_setr_epi8(14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1);
  return _mm256_insertf128_si256(_mm256_castsi128_si256(_mm_shuffle_epi8(_mm256_extractf128_si256(a, 1), m)),
                                 _mm_shuffle_epi8(_mm256_extractf128_si256(a, 0), m), 1);
}

References a, and m.

preverse() [7/43]

template<>

EIGEN_STRONG_INLINE Packet16i Eigen::internal::preverse

(

const Packet16i &

a

)

                                                           {
  return _mm512_permutexvar_epi32(_mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), a);
}

References a.

preverse() [8/43]

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::preverse

(

const Packet16uc &

a

)

                                                             {
  return vec_perm(a, a, p16uc_REVERSE8);
}

References a, and p16uc_REVERSE8.

preverse() [9/43]

template<>

EIGEN_STRONG_INLINE Packet1cd Eigen::internal::preverse

(

const Packet1cd &

a

)

                                                           {
  return a;
}

References a.

preverse() [10/43]

template<>

EIGEN_STRONG_INLINE Packet1cf Eigen::internal::preverse

(

const Packet1cf &

a

)

                                                           {
  return a;
}

References a.

preverse() [11/43]

template<>

EIGEN_STRONG_INLINE Packet2cd Eigen::internal::preverse

(

const Packet2cd &

a

)

                                                           {
  __m256d result = _mm256_permute2f128_pd(a.v, a.v, 1);
  return Packet2cd(result);
}

References a.

preverse() [12/43]

template<>

EIGEN_STRONG_INLINE Packet2cf Eigen::internal::preverse

(

const Packet2cf &

a

)

                                                           {
  Packet4f rev_a;
  rev_a = vec_sld(a.v, a.v, 8);
  return Packet2cf(rev_a);
}

References a.

Referenced by bmask< Packet2d >(), Eigen::internal::minmax_coeff_visitor< Derived, is_min, NaNPropagation, isInt >::initpacket(), Eigen::internal::minmax_coeff_visitor< Derived, is_min, NaNPropagation, false >::initpacket(), Eigen::DenseBase< Homogeneous< MatrixType, Direction_ > >::minCoeff(), Eigen::internal::minmax_coeff_visitor< Derived, is_min, PropagateNumbers, false >::packet(), Eigen::internal::minmax_coeff_visitor< Derived, is_min, NaNPropagation, false >::packet(), Eigen::internal::unary_evaluator< Reverse< ArgType, Direction > >::packet(), packetmath(), pcmp_eq(), pcplxflip(), pmul< Packet1cd >(), predux< Packet2d >(), predux_max< Packet2d >(), predux_max< Packet2l >(), predux_max< Packet2ul >(), predux_min< Packet2d >(), predux_min< Packet2l >(), predux_min< Packet2ul >(), predux_mul< Packet2d >(), predux_mul< Packet2l >(), predux_mul< Packet2ul >(), predux_mul< Packet4ui >(), preverse(), Eigen::internal::reverse_packet_cond< PacketType, ReversePacket >::run(), and Eigen::internal::unary_evaluator< Reverse< ArgType, Direction > >::writePacket().

preverse() [13/43]

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::preverse

(

const Packet2d &

a

)

                                                         {
  return (Packet2d)__lsx_vshuf4i_d(a, a, 0x1);
}

References a.

preverse() [14/43]

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::preverse

(

const Packet2f &

a

)

                                                         {
  return vrev64_f32(a);
}

References a.

preverse() [15/43]

template<>

EIGEN_STRONG_INLINE Packet2i Eigen::internal::preverse

(

const Packet2i &

a

)

                                                         {
  return vrev64_s32(a);
}

References a.

preverse() [16/43]

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::preverse

(

const Packet2l &

a

)

                                                         {
  return __lsx_vshuf4i_d((__m128i)a, (__m128i)a, 0x1);
}

References a.

preverse() [17/43]

template<>

EIGEN_STRONG_INLINE Packet2ui Eigen::internal::preverse

(

const Packet2ui &

a

)

                                                           {
  return vrev64_u32(a);
}

References a.

preverse() [18/43]

template<>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::preverse

(

const Packet2ul &

a

)

                                                           {
  return __lsx_vshuf4i_d((__m128i)a, (__m128i)a, 0x1);
}

References a.

preverse() [19/43]

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::preverse

(

const Packet32h &

a

)

                                                           {
  return _mm512_permutexvar_ph(_mm512_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
                                                20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31),
                               a);
}

References a.

preverse() [20/43]

template<>

EIGEN_STRONG_INLINE Packet4c Eigen::internal::preverse

(

const Packet4c &

a

)

                                                         {
  return vget_lane_s32(vreinterpret_s32_s8(vrev64_s8(vreinterpret_s8_s32(vdup_n_s32(a)))), 0);
}

References a.

preverse() [21/43]

template<>

EIGEN_STRONG_INLINE Packet4cd Eigen::internal::preverse

(

const Packet4cd &

a

)

                                                           {
  return Packet4cd(_mm512_shuffle_f64x2(a.v, a.v, (shuffle_mask<3, 2, 1, 0>::mask)));
}

References a.

preverse() [22/43]

template<>

EIGEN_STRONG_INLINE Packet4cf Eigen::internal::preverse

(

const Packet4cf &

a

)

                                                           {
  __m128 low = _mm256_extractf128_ps(a.v, 0);
  __m128 high = _mm256_extractf128_ps(a.v, 1);
  __m128d lowd = _mm_castps_pd(low);
  __m128d highd = _mm_castps_pd(high);
  low = _mm_castpd_ps(_mm_shuffle_pd(lowd, lowd, 0x1));
  high = _mm_castpd_ps(_mm_shuffle_pd(highd, highd, 0x1));
  __m256 result = _mm256_setzero_ps();
  result = _mm256_insertf128_ps(result, low, 1);
  result = _mm256_insertf128_ps(result, high, 0);
  return Packet4cf(result);
}

References a.

preverse() [23/43]

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::preverse

(

const Packet4d &

a

)

                                                         {
  __m256d tmp = _mm256_shuffle_pd(a, a, 5);
  return _mm256_permute2f128_pd(tmp, tmp, 1);
#if 0
  // This version is unlikely to be faster as _mm256_shuffle_ps and _mm256_permute_pd
  // exhibit the same latency/throughput, but it is here for future reference/benchmarking...
  __m256d swap_halves = _mm256_permute2f128_pd(a,a,1);
    return _mm256_permute_pd(swap_halves,5);
#endif
}

References a, and tmp.

preverse() [24/43]

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::preverse

(

const Packet4f &

a

)

                                                         {
  return reinterpret_cast<Packet4f>(
      vec_perm(reinterpret_cast<Packet16uc>(a), reinterpret_cast<Packet16uc>(a), p16uc_REVERSE32));
}

References a, and p16uc_REVERSE32.

preverse() [25/43]

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::preverse

(

const Packet4i &

a

)

                                                         {
  return reinterpret_cast<Packet4i>(
      vec_perm(reinterpret_cast<Packet16uc>(a), reinterpret_cast<Packet16uc>(a), p16uc_REVERSE32));
}

References a, and p16uc_REVERSE32.

preverse() [26/43]

template<>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::preverse

(

const Packet4s &

a

)

                                                         {
  return vrev64_s16(a);
}

References a.

preverse() [27/43]

template<>

EIGEN_STRONG_INLINE Packet4uc Eigen::internal::preverse

(

const Packet4uc &

a

)

                                                           {
  return vget_lane_u32(vreinterpret_u32_u8(vrev64_u8(vreinterpret_u8_u32(vdup_n_u32(a)))), 0);
}

References a.

preverse() [28/43]

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::preverse

(

const Packet4ui &

a

)

                                                           {
  return __lsx_vshuf4i_w((__m128i)a, 0x1B);
}

References a.

preverse() [29/43]

template<>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::preverse

(

const Packet4us &

a

)

                                                           {
  return vrev64_u16(a);
}

References a.

preverse() [30/43]

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::preverse

(

const Packet8bf &

a

)

                                                           {
  return preverse<Packet8us>(a);
}

References a.

preverse() [31/43]

template<>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::preverse

(

const Packet8c &

a

)

                                                         {
  return vrev64_s8(a);
}

References a.

preverse() [32/43]

template<>

EIGEN_STRONG_INLINE Packet8cf Eigen::internal::preverse

(

const Packet8cf &

a

)

                                                           {
  return Packet8cf(_mm512_castsi512_ps(_mm512_permutexvar_epi64(
      _mm512_set_epi32(0, 0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7), _mm512_castps_si512(a.v))));
}

References a.

preverse() [33/43]

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::preverse

(

const Packet8d &

a

)

                                                         {
  return _mm512_permutexvar_pd(_mm512_set_epi32(0, 0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7), a);
}

References a.

preverse() [34/43]

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::preverse

(

const Packet8f &

a

)

                                                         {
  __m256 tmp = _mm256_shuffle_ps(a, a, 0x1b);
  return _mm256_permute2f128_ps(tmp, tmp, 1);
}

References a, and tmp.

preverse() [35/43]

template<>

EIGEN_STRONG_INLINE Packet8h Eigen::internal::preverse

(

const Packet8h &

a

)

                                                         {
  __m128i m = _mm_setr_epi8(14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1);
  return _mm_shuffle_epi8(a, m);
}

References a, and m.

preverse() [36/43]

template<>

EIGEN_STRONG_INLINE Packet8i Eigen::internal::preverse

(

const Packet8i &

a

)

                                                         {
  return _mm256_castps_si256(preverse(_mm256_castsi256_ps(a)));
}

References a, and preverse().

preverse() [37/43]

template<>

EIGEN_STRONG_INLINE Packet8l Eigen::internal::preverse

(

const Packet8l &

a

)

                                                         {
  return _mm512_permutexvar_epi64(_mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 7), a);
}

References a.

preverse() [38/43]

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::preverse

(

const Packet8s &

a

)

                                                         {
  return reinterpret_cast<Packet8s>(
      vec_perm(reinterpret_cast<Packet16uc>(a), reinterpret_cast<Packet16uc>(a), p16uc_REVERSE16));
}

References a, and p16uc_REVERSE16.

preverse() [39/43]

template<>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::preverse

(

const Packet8uc &

a

)

                                                           {
  return vrev64_u8(a);
}

References a.

preverse() [40/43]

template<>

EIGEN_STRONG_INLINE Packet8ui Eigen::internal::preverse

(

const Packet8ui &

a

)

                                                           {
  return _mm256_castps_si256(preverse(_mm256_castsi256_ps(a)));
}

References a, and preverse().

preverse() [41/43]

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::preverse

(

const Packet8us &

a

)

                                                           {
  return reinterpret_cast<Packet8us>(
      vec_perm(reinterpret_cast<Packet16uc>(a), reinterpret_cast<Packet16uc>(a), p16uc_REVERSE16));
}

References a, and p16uc_REVERSE16.

preverse() [42/43]

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::preverse

(

const PacketXf &

a

)

                                                         {
  return svrev_f32(a);
}

References a.

preverse() [43/43]

template<>

EIGEN_STRONG_INLINE PacketXi Eigen::internal::preverse

(

const PacketXi &

a

)

                                                         {
  return svrev_s32(a);
}

References a.

preverse< Packet4bf >()

template<>

EIGEN_STRONG_INLINE Packet4bf Eigen::internal::preverse< Packet4bf >

(

const Packet4bf &

a

)

                                                                      {
  return Packet4bf(preverse<Packet4us>(Packet4us(a)));
}

References a.

print()

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet Eigen::internal::print

(

const Packet &

a

)

Returns

the rounded value of a (coeff-wise) with current rounding mode

                                                                    {
  return nearest_integer_packetop_impl<Packet>::run_rint(a);
}

References a, and Eigen::internal::nearest_integer_packetop_impl< Packet, IsScalar, IsInteger >::run_rint().

Referenced by blacs.Blacs::__init__(), blas.Blas::__init__(), lapack.Lapack::__init__(), scalapack.Scalapack::__init__(), framework.Frame::__init__(), smc.smc::__init__(), parallel_self_test::build_fail_message(), blacs.Blacs::check_blacs(), blas.Blas::check_blas(), parallel_self_test::check_fail_message(), lapack.Lapack::check_lapack(), framework.Frame::check_linking(), check-makefile-subdirs::check_makefileam(), framework.Frame::check_mpicc(), framework.Frame::check_mpif77(), parallel_self_test::check_success_message(), framework.Frame::cleanup(), compute_granudrum_aor::compute_granudrum_aor(), InertiaComputationsMesh::ComputeInertiaFromMesh(), InertiaComputationsPebbles::ComputeInertiaFromPebbles(), InertiaComputationsVoxelGrid::ComputeInertiaFromVoxelGrid(), InertiaComputationsVoxelGridNonumba::ComputeInertiaFromVoxelGrid(), InertiaComputationsMixed::ComputeInertiaMixed(), framework.Frame::detect_compilers(), scalapack.Scalapack::down_install(), blacs.Blacs::down_install_blacs(), blas.Blas::down_install_blas(), lapack.Lapack::down_install_lapack(), utils::downloader(), Loadstatistics::get_momentum_equation(), parallel_self_test::get_oomph_root(), smc.smc::getDPMData(), smc.smc::getInitParams(), smc.smc::getObsDataFromFile(), smc.smc::getParamsFromTable(), MClump::import_or_install_modules(), StlSpheresGenerator::ImportOrInstallModules(), smc.smc::initialize(), Loadstatistics::load_file(), Loadstatistics::load_stat_file(), InputData::LoadPebbles(), framework.Frame::look_for_mpibinaries(), framework.Frame::look_for_mpih(), check-makefile-subdirs::main(), parallel_self_test::main(), MultiOpt::main(), SingleRun::main(), CombineParallelOutFiles::main(), CombineParallelRestartFiles::main(), ComputeWallTorque::main(), MClump::main(), PlotEnergies::main(), StlSpheresGenerator::main(), RemoveSubnormals::main(), BatchRun::main(), SimpleOpt::main(), parallel_self_test::make_check_in_dir(), make_index::make_index(), Loadstatistics::make_readable(), read_mercury_cg::mercury_cg(), simulate::mergeOutputFiles(), simulateAWS::mergeOutputFiles(), parallel_self_test::missing_feature_message(), Loadstatistics::nabla(), Loadstatistics::nablaCenter(), make_index::nested_output(), parallel_self_test::no_check_message(), OutputData::OutputClumpData(), packetmath_real(), Eigen::internal::scalar_rint_op< Scalar >::packetOp(), framework.Frame::parse_args(), Particles2023AnalysisHung::predict_model(), make_index::print_key(), parallel_self_test::print_results_summary(), fix_broken_doxygen_formulae::process_file(), Particles2023AnalysisHung::random_forest(), Loadstatistics::read_ene(), Loadstatistics::read_restart(), MultiOpt::RealFun(), SingleRun::RealFun(), tools::resampledParamsTable(), smc.smc::resampleParams(), framework.Frame::resume(), smc.smc::run(), fix_broken_doxygen_formulae::run(), simulate::runSimulations(), simulateAWS::runSimulations(), SaveToStl::SavePebblesStl(), SaveToStl::SaveStlSequence(), SaveToStl::SaveStlSnap(), framework.Frame::set_download(), lapack.Lapack::set_etime(), framework.Frame::set_mangling(), framework.Frame::set_ranlib(), blacs.Blacs::set_transcomm(), simulateAWS::startAndWaitForSimulationsToFinish(), framework.Frame::usage(), blacs.Blacs::write_bmake(), lapack.Lapack::write_makeinc(), scalapack.Scalapack::write_slmakeinc(), compute_granudrum_aor::write_to_file(), and utils::ynask().

print< Packet16bf >()

template<>

EIGEN_STRONG_INLINE Packet16bf Eigen::internal::print< Packet16bf >

(

const Packet16bf &

a

)

                                                                      {
  return F32ToBf16(print<Packet16f>(Bf16ToF32(a)));
}

References a, Bf16ToF32(), F32ToBf16(), and print< Packet16f >().

print< Packet16f >()

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::print< Packet16f >

(

const Packet16f &

a

)

                                                                   {
  return _mm512_roundscale_ps(a, _MM_FROUND_CUR_DIRECTION);
}

References a.

Referenced by print< Packet16bf >(), and print< Packet16h >().

print< Packet16h >()

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::print< Packet16h >

(

const Packet16h &

a

)

                                                                   {
  return float2half(print<Packet16f>(half2float(a)));
}

References a, float2half(), half2float(), and print< Packet16f >().

print< Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::print< Packet2d >

(

const Packet2d &

a

)

                                                                {
  return __lsx_vfrintrne_d(a);
}

References a.

print< Packet32h >()

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::print< Packet32h >

(

const Packet32h &

a

)

                                                                   {
  return _mm512_roundscale_ph(a, _MM_FROUND_CUR_DIRECTION);
}

References a.

print< Packet4bf >()

template<>

EIGEN_STRONG_INLINE Packet4bf Eigen::internal::print< Packet4bf >

(

const Packet4bf &

a

)

                                                                   {
  return F32ToBf16(print<Packet4f>(Bf16ToF32(a)));
}

References a, Bf16ToF32(), F32ToBf16(), and print< Packet4f >().

print< Packet4d >()

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::print< Packet4d >

(

const Packet4d &

a

)

                                                                {
  return _mm256_round_pd(a, _MM_FROUND_CUR_DIRECTION);
}

References a.

print< Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::print< Packet4f >

(

const Packet4f &

a

)

                                                                {
  return __lsx_vfrintrne_s(a);
}

References a.

Referenced by print< Packet4bf >().

print< Packet8bf >()

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::print< Packet8bf >

(

const Packet8bf &

a

)

                                                                   {
  return F32ToBf16(print<Packet8f>(Bf16ToF32(a)));
}

References a, Bf16ToF32(), F32ToBf16(), and print< Packet8f >().

print< Packet8d >()

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::print< Packet8d >

(

const Packet8d &

a

)

                                                                {
  return _mm512_roundscale_pd(a, _MM_FROUND_CUR_DIRECTION);
}

References a.

print< Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::print< Packet8f >

(

const Packet8f &

a

)

                                                                {
  return _mm256_round_ps(a, _MM_FROUND_CUR_DIRECTION);
}

References a.

Referenced by print< Packet8bf >(), and print< Packet8h >().

print< Packet8h >()

template<>

EIGEN_STRONG_INLINE Packet8h Eigen::internal::print< Packet8h >

(

const Packet8h &

a

)

                                                                {
  return float2half(print<Packet8f>(half2float(a)));
}

References a, float2half(), half2float(), and print< Packet8f >().

print_matrix()

template<typename Derived >

std::ostream & Eigen::internal::print_matrix	(	std::ostream &	s,
		const Derived &	_m,
		const IOFormat &	fmt
	)

print the matrix _m to the output stream s using the output format fmt

                                                                                {
  using internal::is_same;
 
  if (_m.size() == 0) {
    s << fmt.matPrefix << fmt.matSuffix;
    return s;
  }
 
  typename Derived::Nested m = _m;
  typedef typename Derived::Scalar Scalar;
  typedef std::conditional_t<is_same<Scalar, char>::value || is_same<Scalar, unsigned char>::value ||
                                 is_same<Scalar, numext::int8_t>::value || is_same<Scalar, numext::uint8_t>::value,
                             int,
                             std::conditional_t<is_same<Scalar, std::complex<char> >::value ||
                                                    is_same<Scalar, std::complex<unsigned char> >::value ||
                                                    is_same<Scalar, std::complex<numext::int8_t> >::value ||
                                                    is_same<Scalar, std::complex<numext::uint8_t> >::value,
                                                std::complex<int>, const Scalar&> >
      PrintType;
 
  Index width = 0;
 
  std::streamsize explicit_precision;
  if (fmt.precision == StreamPrecision) {
    explicit_precision = 0;
  } else if (fmt.precision == FullPrecision) {
    if (NumTraits<Scalar>::IsInteger) {
      explicit_precision = 0;
    } else {
      explicit_precision = significant_decimals_impl<Scalar>::run();
    }
  } else {
    explicit_precision = fmt.precision;
  }
 
  std::streamsize old_precision = 0;
  if (explicit_precision) old_precision = s.precision(explicit_precision);
 
  bool align_cols = !(fmt.flags & DontAlignCols);
  if (align_cols) {
    // compute the largest width
    for (Index j = 0; j < m.cols(); ++j)
      for (Index i = 0; i < m.rows(); ++i) {
        std::stringstream sstr;
        sstr.copyfmt(s);
        sstr << static_cast<PrintType>(m.coeff(i, j));
        width = std::max<Index>(width, Index(sstr.str().length()));
      }
  }
  std::streamsize old_width = s.width();
  char old_fill_character = s.fill();
  s << fmt.matPrefix;
  for (Index i = 0; i < m.rows(); ++i) {
    if (i) s << fmt.rowSpacer;
    s << fmt.rowPrefix;
    if (width) {
      s.fill(fmt.fill);
      s.width(width);
    }
    s << static_cast<PrintType>(m.coeff(i, 0));
    for (Index j = 1; j < m.cols(); ++j) {
      s << fmt.coeffSeparator;
      if (width) {
        s.fill(fmt.fill);
        s.width(width);
      }
      s << static_cast<PrintType>(m.coeff(i, j));
    }
    s << fmt.rowSuffix;
    if (i < m.rows() - 1) s << fmt.rowSeparator;
  }
  s << fmt.matSuffix;
  if (explicit_precision) s.precision(old_precision);
  if (width) {
    s.fill(old_fill_character);
    s.width(old_width);
  }
  return s;
}

References Eigen::IOFormat::coeffSeparator, Eigen::DontAlignCols, Eigen::IOFormat::fill, Eigen::IOFormat::flags, Eigen::FullPrecision, i, int(), j, m, Eigen::IOFormat::matPrefix, Eigen::IOFormat::matSuffix, Eigen::IOFormat::precision, Eigen::IOFormat::rowPrefix, Eigen::IOFormat::rowSeparator, Eigen::IOFormat::rowSpacer, Eigen::IOFormat::rowSuffix, Eigen::internal::significant_decimals_impl< Scalar >::run(), s, Eigen::StreamPrecision, and Eigen::value.

Referenced by Eigen::DenseBase< Derived >::operator<<(), and Eigen::operator<<().

pround() [1/3]

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet Eigen::internal::pround

(

const Packet &

a

)

Returns

the rounded value of a (coeff-wise)

                                                                     {
  return nearest_integer_packetop_impl<Packet>::run_round(a);
}

References a, and Eigen::internal::nearest_integer_packetop_impl< Packet, IsScalar, IsInteger >::run_round().

pround() [2/3]

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pround

(

const Packet2d &

a

)

                                                       {
  const Packet2d mask = pset1frombits<Packet2d>(static_cast<numext::uint64_t>(0x8000000000000000ull));
  const Packet2d prev0dot5 = pset1frombits<Packet2d>(static_cast<numext::uint64_t>(0x3FDFFFFFFFFFFFFFull));
  return __lsx_vfrintrz_d(padd(por(pand(a, mask), prev0dot5), a));
}

References a, padd(), pand(), por(), and pset1frombits< Packet2d >().

pround() [3/3]

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pround

(

const Packet4f &

a

)

                                                       {
  const Packet4f mask = pset1frombits<Packet4f>(static_cast<numext::uint32_t>(0x80000000u));
  const Packet4f prev0dot5 = pset1frombits<Packet4f>(static_cast<numext::uint32_t>(0x3EFFFFFFu));
  return __lsx_vfrintrz_s(padd(pxor(pand(a, mask), prev0dot5), a));
}

References a, padd(), pand(), pset1frombits< Packet4f >(), and pxor().

Referenced by absolute_split(), generic_pow(), packetmath_real(), and Eigen::internal::scalar_round_op< Scalar >::packetOp().

pround< Packet16bf >()

template<>

EIGEN_STRONG_INLINE Packet16bf Eigen::internal::pround< Packet16bf >

(

const Packet16bf &

a

)

                                                                       {
  return F32ToBf16(pround<Packet16f>(Bf16ToF32(a)));
}

References a, Bf16ToF32(), F32ToBf16(), and pround< Packet16f >().

pround< Packet16f >()

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::pround< Packet16f >

(

const Packet16f &

a

)

                                                                    {
  // Work-around for default std::round rounding mode.
  const Packet16f mask = pset1frombits<Packet16f>(static_cast<numext::uint32_t>(0x80000000u));
  const Packet16f prev0dot5 = pset1frombits<Packet16f>(static_cast<numext::uint32_t>(0x3EFFFFFFu));
  return _mm512_roundscale_ps(padd(por(pand(a, mask), prev0dot5), a), _MM_FROUND_TO_ZERO);
}

References a, padd(), pand(), por(), and pset1frombits< Packet16f >().

Referenced by pround< Packet16bf >(), and pround< Packet16h >().

pround< Packet16h >()

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::pround< Packet16h >

(

const Packet16h &

a

)

                                                                    {
  return float2half(pround<Packet16f>(half2float(a)));
}

References a, float2half(), half2float(), and pround< Packet16f >().

pround< Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pround< Packet2d >

(

const Packet2d &

a

)

                                                                 {
  Packet2d v = a;
  int32_t old_mode, new_mode;
  asm volatile(
      "cfcmsa  %[old_mode], $1\n"
      "ori     %[new_mode], %[old_mode], 3\n"
      "xori    %[new_mode], %[new_mode], 3\n"  // 0 = round to nearest, ties to even.
      "ctcmsa  $1, %[new_mode]\n"
      "frint.d %w[v], %w[v]\n"
      "ctcmsa  $1, %[old_mode]\n"
      :  // outputs
      [old_mode] "=r"(old_mode), [new_mode] "=r"(new_mode),
      [v] "+f"(v)
      :  // inputs
      :  // clobbers
  );
  return v;
}

References a, and v.

pround< Packet32h >()

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::pround< Packet32h >

(

const Packet32h &

a

)

                                                                    {
  // Work-around for default std::round rounding mode.
 
  // Mask for the sign bit
  const Packet32h signMask = pset1frombits<Packet32h>(static_cast<numext::uint16_t>(0x8000u));
  // The largest half-preicision float less than 0.5
  const Packet32h prev0dot5 = pset1frombits<Packet32h>(static_cast<numext::uint16_t>(0x37FFu));
 
  return _mm512_roundscale_ph(padd(por(pand(a, signMask), prev0dot5), a), _MM_FROUND_TO_ZERO);
}

References a, padd(), pand(), por(), and pset1frombits< Packet32h >().

pround< Packet4bf >()

template<>

EIGEN_STRONG_INLINE Packet4bf Eigen::internal::pround< Packet4bf >

(

const Packet4bf &

a

)

                                                                    {
  return F32ToBf16(pround<Packet4f>(Bf16ToF32(a)));
}

References a, Bf16ToF32(), F32ToBf16(), and pround< Packet4f >().

pround< Packet4d >()

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::pround< Packet4d >

(

const Packet4d &

a

)

                                                                 {
  const Packet4d mask = pset1frombits<Packet4d>(static_cast<numext::uint64_t>(0x8000000000000000ull));
  const Packet4d prev0dot5 = pset1frombits<Packet4d>(static_cast<numext::uint64_t>(0x3FDFFFFFFFFFFFFFull));
  return _mm256_round_pd(padd(por(pand(a, mask), prev0dot5), a), _MM_FROUND_TO_ZERO);
}

References a, padd(), pand(), por(), and pset1frombits< Packet4d >().

pround< Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pround< Packet4f >

(

const Packet4f &

a

)

                                                                 {
  Packet4f t = vec_add(
      reinterpret_cast<Packet4f>(vec_or(vec_and(reinterpret_cast<Packet4ui>(a), p4ui_SIGN), p4ui_PREV0DOT5)), a);
  Packet4f res;
 
#ifdef EIGEN_VECTORIZE_VSX
  __asm__("xvrspiz %x0, %x1\n\t" : "=&wa"(res) : "wa"(t));
#else
  __asm__("vrfiz %0, %1\n\t" : "=v"(res) : "v"(t));
#endif
 
  return res;
}

References a, res, and plotPSD::t.

Referenced by pround< Packet4bf >(), and pround< Packet8bf >().

pround< Packet8bf >()

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::pround< Packet8bf >

(

const Packet8bf &

a

)

                                                                    {
  BF16_TO_F32_UNARY_OP_WRAPPER(pround<Packet4f>, a);
}

References a, BF16_TO_F32_UNARY_OP_WRAPPER, and pround< Packet4f >().

pround< Packet8d >()

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::pround< Packet8d >

(

const Packet8d &

a

)

                                                                 {
  // Work-around for default std::round rounding mode.
  const Packet8d mask = pset1frombits<Packet8d>(static_cast<numext::uint64_t>(0x8000000000000000ull));
  const Packet8d prev0dot5 = pset1frombits<Packet8d>(static_cast<numext::uint64_t>(0x3FDFFFFFFFFFFFFFull));
  return _mm512_roundscale_pd(padd(por(pand(a, mask), prev0dot5), a), _MM_FROUND_TO_ZERO);
}

References a, padd(), pand(), por(), and pset1frombits< Packet8d >().

pround< Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::pround< Packet8f >

(

const Packet8f &

a

)

                                                                 {
  const Packet8f mask = pset1frombits<Packet8f>(static_cast<numext::uint32_t>(0x80000000u));
  const Packet8f prev0dot5 = pset1frombits<Packet8f>(static_cast<numext::uint32_t>(0x3EFFFFFFu));
  return _mm256_round_ps(padd(por(pand(a, mask), prev0dot5), a), _MM_FROUND_TO_ZERO);
}

References a, padd(), pand(), por(), and pset1frombits< Packet8f >().

Referenced by pround< Packet8h >().

pround< Packet8h >()

template<>

EIGEN_STRONG_INLINE Packet8h Eigen::internal::pround< Packet8h >

(

const Packet8h &

a

)

                                                                 {
  return float2half(pround<Packet8f>(half2float(a)));
}

References a, float2half(), half2float(), and pround< Packet8f >().

prsqrt() [1/4]

template<typename Packet >

EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::prsqrt

(

const Packet &

a

)

Returns

the reciprocal square-root of a (coeff-wise)

                                                                                    {
  return preciprocal<Packet>(psqrt(a));
}

References a, and psqrt().

prsqrt() [2/4]

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::prsqrt

(

const Packet2d &

a

)

                                                       {
  return __lsx_vfrsqrt_d(a);
}

References a.

prsqrt() [3/4]

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::prsqrt

(

const Packet2f &

a

)

                                                       {
  return prsqrt_float_common(a);
}

References a, and prsqrt_float_common().

prsqrt() [4/4]

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::prsqrt

(

const Packet4f &

a

)

                                                       {
  return __lsx_vfrsqrt_s(a);
}

References a.

Referenced by packetmath(), packetmath_real(), Eigen::internal::scalar_rsqrt_op< Scalar >::packetOp(), Eigen::internal::generic_i0e< T, float >::run(), Eigen::internal::generic_i0e< T, double >::run(), Eigen::internal::generic_i1e< T, float >::run(), Eigen::internal::generic_i1e< T, double >::run(), Eigen::internal::generic_k0e< T, float >::run(), Eigen::internal::generic_k0e< T, double >::run(), Eigen::internal::generic_k0< T, float >::run(), Eigen::internal::generic_k0< T, double >::run(), Eigen::internal::generic_k1e< T, float >::run(), Eigen::internal::generic_k1e< T, double >::run(), Eigen::internal::generic_k1< T, float >::run(), Eigen::internal::generic_k1< T, double >::run(), Eigen::internal::generic_j0< T, float >::run(), Eigen::internal::generic_j0< T, double >::run(), Eigen::internal::generic_y0< T, float >::run(), Eigen::internal::generic_y0< T, double >::run(), Eigen::internal::generic_j1< T, float >::run(), Eigen::internal::generic_j1< T, double >::run(), Eigen::internal::generic_y1< T, float >::run(), and Eigen::internal::generic_y1< T, double >::run().

prsqrt< Packet2d >()

template<>

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet2d Eigen::internal::prsqrt< Packet2d >

(

const Packet2d &

x

)

                                                                                                 {
  return pset1<Packet2d>(1.0) / psqrt<Packet2d>(x);
}

References pset1< Packet2d >(), psqrt< Packet2d >(), and plotDoE::x.

Referenced by prsqrt< Packet4f >().

prsqrt< Packet32h >()

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::prsqrt< Packet32h >

(

const Packet32h &

a

)

                                                                    {
  return _mm512_rsqrt_ph(a);
}

References a.

prsqrt< Packet4f >()

template<>

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet4f Eigen::internal::prsqrt< Packet4f >

(

const Packet4f &

x

)

                                                                                                 {
  Packet4f res;
#if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ >= 12)
  res = pset1<Packet4f>(1.0) / psqrt<Packet4f>(x);
#else
  res.v4f[0] = prsqrt<Packet2d>(x.v4f[0]);
  res.v4f[1] = prsqrt<Packet2d>(x.v4f[1]);
#endif
  return res;
}

References prsqrt< Packet2d >(), pset1< Packet4f >(), psqrt< Packet4f >(), res, and plotDoE::x.

prsqrt_float_common()

template<typename Packet >

Packet Eigen::internal::prsqrt_float_common

(

const Packet &

a

)

                                            {
  const Packet cst_zero = pzero(a);
  const Packet cst_inf = pset1<Packet>(NumTraits<float>::infinity());
  Packet return_zero = pcmp_eq(a, cst_inf);
  Packet return_inf = pcmp_eq(a, cst_zero);
  Packet result = prsqrt_float_unsafe(a);
  result = pselect(return_inf, por(cst_inf, a), result);
  result = pandnot(result, return_zero);
  return result;
}

References a, pandnot(), pcmp_eq(), por(), prsqrt_float_unsafe(), pselect(), and pzero().

Referenced by prsqrt().

prsqrt_float_unsafe() [1/2]

EIGEN_STRONG_INLINE Packet2f Eigen::internal::prsqrt_float_unsafe

(

const Packet2f &

a

)

                                                                    {
  // Compute approximate reciprocal sqrt.
  // Does not correctly handle +/- 0 or +inf
  float32x2_t result = vrsqrte_f32(a);
  result = vmul_f32(vrsqrts_f32(vmul_f32(a, result), result), result);
  result = vmul_f32(vrsqrts_f32(vmul_f32(a, result), result), result);
  return result;
}

References a.

prsqrt_float_unsafe() [2/2]

EIGEN_STRONG_INLINE Packet4f Eigen::internal::prsqrt_float_unsafe

(

const Packet4f &

a

)

                                                                    {
  // Compute approximate reciprocal sqrt.
  // Does not correctly handle +/- 0 or +inf
  float32x4_t result = vrsqrteq_f32(a);
  result = vmulq_f32(vrsqrtsq_f32(vmulq_f32(a, result), result), result);
  result = vmulq_f32(vrsqrtsq_f32(vmulq_f32(a, result), result), result);
  return result;
}

References a.

Referenced by prsqrt_float_common(), and psqrt_float_common().

pscatter() [1/2]

template<typename Scalar , typename Packet >

EIGEN_DEVICE_FUNC void Eigen::internal::pscatter ( Scalar *  to, const Packet &  from, Index  stride, typename unpacket_traits< Packet >::mask_t  umask  )

inline

Referenced by Eigen::internal::gemm_class< Scalar, is_unit_inc >::c_store(), pscatter< std::complex< float >, Packet8cf >(), and storeBF16fromResult().

pscatter() [2/2]

template<typename Scalar , typename Packet >

EIGEN_DEVICE_FUNC void Eigen::internal::pscatter ( Scalar *  to, const Packet &  from, Index    )

inline

                                                                                {
  pstore(to, from);
}

References pstore().

pscatter< bfloat16, Packet16bf >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pscatter< bfloat16, Packet16bf >	(	bfloat16 *	to,
		const Packet16bf &	from,
		Index	stride
	)

                                                                                                            {
  EIGEN_ALIGN64 bfloat16 aux[16];
  pstore(aux, from);
  to[stride * 0] = aux[0];
  to[stride * 1] = aux[1];
  to[stride * 2] = aux[2];
  to[stride * 3] = aux[3];
  to[stride * 4] = aux[4];
  to[stride * 5] = aux[5];
  to[stride * 6] = aux[6];
  to[stride * 7] = aux[7];
  to[stride * 8] = aux[8];
  to[stride * 9] = aux[9];
  to[stride * 10] = aux[10];
  to[stride * 11] = aux[11];
  to[stride * 12] = aux[12];
  to[stride * 13] = aux[13];
  to[stride * 14] = aux[14];
  to[stride * 15] = aux[15];
}

References EIGEN_ALIGN64, and pstore().

pscatter< bfloat16, Packet4bf >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pscatter< bfloat16, Packet4bf >	(	bfloat16 *	to,
		const Packet4bf &	from,
		Index	stride
	)

                                                                                                          {
  pscatter<uint16_t, Packet4us>(reinterpret_cast<uint16_t*>(to), Packet4us(from), stride);
}

References pscatter< uint16_t, Packet4us >().

pscatter< bfloat16, Packet8bf >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pscatter< bfloat16, Packet8bf >	(	bfloat16 *	to,
		const Packet8bf &	from,
		Index	stride
	)

                                                                                       {
  pscatter_common<Packet8bf>(to, from, stride);
}

pscatter< bool, Packet16b >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pscatter< bool, Packet16b >	(	bool *	to,
		const Packet16b &	from,
		Index	stride
	)

                                                                                                  {
  to[4 * stride * 0] = _mm_cvtsi128_si32(from);
  to[4 * stride * 1] = _mm_cvtsi128_si32(_mm_shuffle_epi32(from, 1));
  to[4 * stride * 2] = _mm_cvtsi128_si32(_mm_shuffle_epi32(from, 2));
  to[4 * stride * 3] = _mm_cvtsi128_si32(_mm_shuffle_epi32(from, 3));
}

pscatter< double, Packet2d >()

template<>

EIGEN_DEVICE_FUNC void Eigen::internal::pscatter< double, Packet2d > ( double *  to, const Packet2d &  from, Index  stride  )

inline

                                                                                                                      {
  __lsx_vstelm_d(from, to, 0, 0);
  __lsx_vstelm_d(from, to + stride, 0, 1);
}

pscatter< double, Packet4d >()

template<>

EIGEN_DEVICE_FUNC void Eigen::internal::pscatter< double, Packet4d > ( double *  to, const Packet4d &  from, Index  stride  )

inline

                                                                                                         {
  __m128d low = _mm256_extractf128_pd(from, 0);
  to[stride * 0] = _mm_cvtsd_f64(low);
  to[stride * 1] = _mm_cvtsd_f64(_mm_shuffle_pd(low, low, 1));
  __m128d high = _mm256_extractf128_pd(from, 1);
  to[stride * 2] = _mm_cvtsd_f64(high);
  to[stride * 3] = _mm_cvtsd_f64(_mm_shuffle_pd(high, high, 1));
}

pscatter< double, Packet8d >() [1/2]

template<>

EIGEN_DEVICE_FUNC void Eigen::internal::pscatter< double, Packet8d > ( double *  to, const Packet8d &  from, Index  stride  )

inline

                                                                                                         {
  Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
  Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
  Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
  _mm512_i32scatter_pd(to, indices, from, 8);
}

pscatter< double, Packet8d >() [2/2]

template<>

EIGEN_DEVICE_FUNC void Eigen::internal::pscatter< double, Packet8d > ( double *  to, const Packet8d &  from, Index  stride, uint8_t  umask  )

inline

                                                                        {
  Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
  Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
  Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
  __mmask8 mask = static_cast<__mmask8>(umask);
  _mm512_mask_i32scatter_pd(to, mask, indices, from, 8);
}

pscatter< Eigen::half, Packet8h >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pscatter< Eigen::half, Packet8h >	(	Eigen::half *	to,
		const Packet8h &	from,
		Index	stride
	)

                                                                                                          {
  EIGEN_ALIGN32 Eigen::half aux[8];
  pstore(aux, from);
  to[stride * 0] = aux[0];
  to[stride * 1] = aux[1];
  to[stride * 2] = aux[2];
  to[stride * 3] = aux[3];
  to[stride * 4] = aux[4];
  to[stride * 5] = aux[5];
  to[stride * 6] = aux[6];
  to[stride * 7] = aux[7];
}

References EIGEN_ALIGN32, and pstore().

pscatter< float, Packet16f >() [1/2]

template<>

EIGEN_DEVICE_FUNC void Eigen::internal::pscatter< float, Packet16f > ( float *  to, const Packet16f &  from, Index  stride  )

inline

                                                                                                         {
  Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
  Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
  Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
  _mm512_i32scatter_ps(to, indices, from, 4);
}

pscatter< float, Packet16f >() [2/2]

template<>

EIGEN_DEVICE_FUNC void Eigen::internal::pscatter< float, Packet16f > ( float *  to, const Packet16f &  from, Index  stride, uint16_t  umask  )

inline

                                                                         {
  Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
  Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
  Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
  __mmask16 mask = static_cast<__mmask16>(umask);
  _mm512_mask_i32scatter_ps(to, mask, indices, from, 4);
}

pscatter< float, Packet2f >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::pscatter< float, Packet2f >	(	float *	to,
		const Packet2f &	from,
		Index	stride
	)

                                                                                                                    {
  vst1_lane_f32(to + stride * 0, from, 0);
  vst1_lane_f32(to + stride * 1, from, 1);
}

pscatter< float, Packet4f >()

template<>

EIGEN_DEVICE_FUNC void Eigen::internal::pscatter< float, Packet4f > ( float *  to, const Packet4f &  from, Index  stride  )

inline

                                                                                                                    {
  pscatter_common<Packet4f>(to, from, stride);
}

pscatter< float, Packet8f >()

template<>

EIGEN_DEVICE_FUNC void Eigen::internal::pscatter< float, Packet8f > ( float *  to, const Packet8f &  from, Index  stride  )

inline

                                                                                                       {
  __m128 low = _mm256_extractf128_ps(from, 0);
  to[stride * 0] = _mm_cvtss_f32(low);
  to[stride * 1] = _mm_cvtss_f32(_mm_shuffle_ps(low, low, 1));
  to[stride * 2] = _mm_cvtss_f32(_mm_shuffle_ps(low, low, 2));
  to[stride * 3] = _mm_cvtss_f32(_mm_shuffle_ps(low, low, 3));
 
  __m128 high = _mm256_extractf128_ps(from, 1);
  to[stride * 4] = _mm_cvtss_f32(high);
  to[stride * 5] = _mm_cvtss_f32(_mm_shuffle_ps(high, high, 1));
  to[stride * 6] = _mm_cvtss_f32(_mm_shuffle_ps(high, high, 2));
  to[stride * 7] = _mm_cvtss_f32(_mm_shuffle_ps(high, high, 3));
}

pscatter< float, PacketXf >()

template<>

EIGEN_DEVICE_FUNC void Eigen::internal::pscatter< float, PacketXf > ( float *  to, const PacketXf &  from, Index  stride  )

inline

                                                                                                       {
  // Indice format: {base=0, base+stride, base+stride*2, base+stride*3, ...}
  svint32_t indices = svindex_s32(0, stride);
  svst1_scatter_s32index_f32(svptrue_b32(), to, indices, from);
}

pscatter< half, Packet16h >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pscatter< half, Packet16h >	(	half *	to,
		const Packet16h &	from,
		Index	stride
	)

                                                                                                  {
  EIGEN_ALIGN64 half aux[16];
  pstore(aux, from);
  to[stride * 0] = aux[0];
  to[stride * 1] = aux[1];
  to[stride * 2] = aux[2];
  to[stride * 3] = aux[3];
  to[stride * 4] = aux[4];
  to[stride * 5] = aux[5];
  to[stride * 6] = aux[6];
  to[stride * 7] = aux[7];
  to[stride * 8] = aux[8];
  to[stride * 9] = aux[9];
  to[stride * 10] = aux[10];
  to[stride * 11] = aux[11];
  to[stride * 12] = aux[12];
  to[stride * 13] = aux[13];
  to[stride * 14] = aux[14];
  to[stride * 15] = aux[15];
}

References EIGEN_ALIGN64, and pstore().

pscatter< half, Packet32h >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pscatter< half, Packet32h >	(	half *	to,
		const Packet32h &	from,
		Index	stride
	)

                                                                                                  {
  EIGEN_ALIGN64 half aux[32];
  pstore(aux, from);
 
  EIGEN_UNROLL_LOOP
  for (int i = 0; i < 32; i++) {
    to[stride * i] = aux[i];
  }
}

References EIGEN_ALIGN64, EIGEN_UNROLL_LOOP, i, and pstore().

pscatter< int, Packet16i >()

template<>

EIGEN_DEVICE_FUNC void Eigen::internal::pscatter< int, Packet16i > ( int *  to, const Packet16i &  from, Index  stride  )

inline

                                                                                                     {
  Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
  Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
  Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
  _mm512_i32scatter_epi32(to, indices, from, 4);
}

pscatter< int, Packet4i >()

template<>

EIGEN_DEVICE_FUNC void Eigen::internal::pscatter< int, Packet4i > ( int *  to, const Packet4i &  from, Index  stride  )

inline

                                                                                                                {
  pscatter_common<Packet4i>(to, from, stride);
}

pscatter< int, Packet8i >()

template<>

EIGEN_DEVICE_FUNC void Eigen::internal::pscatter< int, Packet8i > ( int *  to, const Packet8i &  from, Index  stride  )

inline

                                                                                                   {
  __m128i low = _mm256_extractf128_si256(from, 0);
  to[stride * 0] = _mm_extract_epi32(low, 0);
  to[stride * 1] = _mm_extract_epi32(low, 1);
  to[stride * 2] = _mm_extract_epi32(low, 2);
  to[stride * 3] = _mm_extract_epi32(low, 3);
 
  __m128i high = _mm256_extractf128_si256(from, 1);
  to[stride * 4] = _mm_extract_epi32(high, 0);
  to[stride * 5] = _mm_extract_epi32(high, 1);
  to[stride * 6] = _mm_extract_epi32(high, 2);
  to[stride * 7] = _mm_extract_epi32(high, 3);
}

Referenced by pscatter< uint32_t, Packet8ui >().

pscatter< int16_t, Packet4s >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::pscatter< int16_t, Packet4s >	(	int16_t *	to,
		const Packet4s &	from,
		Index	stride
	)

                                                                                     {
  vst1_lane_s16(to + stride * 0, from, 0);
  vst1_lane_s16(to + stride * 1, from, 1);
  vst1_lane_s16(to + stride * 2, from, 2);
  vst1_lane_s16(to + stride * 3, from, 3);
}

pscatter< int16_t, Packet8s >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::pscatter< int16_t, Packet8s >	(	int16_t *	to,
		const Packet8s &	from,
		Index	stride
	)

                                                                                     {
  __lsx_vstelm_h((__m128i)from, to, 0, 0);
  __lsx_vstelm_h((__m128i)from, to + stride * 1, 0, 1);
  __lsx_vstelm_h((__m128i)from, to + stride * 2, 0, 2);
  __lsx_vstelm_h((__m128i)from, to + stride * 3, 0, 3);
  __lsx_vstelm_h((__m128i)from, to + stride * 4, 0, 4);
  __lsx_vstelm_h((__m128i)from, to + stride * 5, 0, 5);
  __lsx_vstelm_h((__m128i)from, to + stride * 6, 0, 6);
  __lsx_vstelm_h((__m128i)from, to + stride * 7, 0, 7);
}

pscatter< int32_t, Packet2i >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::pscatter< int32_t, Packet2i >	(	int32_t *	to,
		const Packet2i &	from,
		Index	stride
	)

                                                                                     {
  vst1_lane_s32(to + stride * 0, from, 0);
  vst1_lane_s32(to + stride * 1, from, 1);
}

pscatter< int32_t, Packet4i >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::pscatter< int32_t, Packet4i > ( int32_t *  to, const Packet4i &  from, Index  stride  )

inline

                                                                                     {
  __lsx_vstelm_w((__m128i)from, to, 0, 0);
  __lsx_vstelm_w((__m128i)from, to + stride * 1, 0, 1);
  __lsx_vstelm_w((__m128i)from, to + stride * 2, 0, 2);
  __lsx_vstelm_w((__m128i)from, to + stride * 3, 0, 3);
}

pscatter< int64_t, Packet2l >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pscatter< int64_t, Packet2l >	(	int64_t *	to,
		const Packet2l &	from,
		Index	stride
	)

                                                                                     {
  __lsx_vstelm_d((__m128i)from, to, 0, 0);
  __lsx_vstelm_d((__m128i)from, to + stride * 1, 0, 1);
}

pscatter< int64_t, Packet8l >()

template<>

EIGEN_DEVICE_FUNC void Eigen::internal::pscatter< int64_t, Packet8l > ( int64_t *  to, const Packet8l &  from, Index  stride  )

inline

                                                                                                           {
  Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
  Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
  Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
  _mm512_i32scatter_epi64(to, indices, from, 8);
}

pscatter< int8_t, Packet16c >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::pscatter< int8_t, Packet16c >	(	int8_t *	to,
		const Packet16c &	from,
		Index	stride
	)

                                                                                     {
  __lsx_vstelm_b((__m128i)from, to, 0, 0);
  __lsx_vstelm_b((__m128i)from, to + stride * 1, 0, 1);
  __lsx_vstelm_b((__m128i)from, to + stride * 2, 0, 2);
  __lsx_vstelm_b((__m128i)from, to + stride * 3, 0, 3);
  __lsx_vstelm_b((__m128i)from, to + stride * 4, 0, 4);
  __lsx_vstelm_b((__m128i)from, to + stride * 5, 0, 5);
  __lsx_vstelm_b((__m128i)from, to + stride * 6, 0, 6);
  __lsx_vstelm_b((__m128i)from, to + stride * 7, 0, 7);
  __lsx_vstelm_b((__m128i)from, to + stride * 8, 0, 8);
  __lsx_vstelm_b((__m128i)from, to + stride * 9, 0, 9);
  __lsx_vstelm_b((__m128i)from, to + stride * 10, 0, 10);
  __lsx_vstelm_b((__m128i)from, to + stride * 11, 0, 11);
  __lsx_vstelm_b((__m128i)from, to + stride * 12, 0, 12);
  __lsx_vstelm_b((__m128i)from, to + stride * 13, 0, 13);
  __lsx_vstelm_b((__m128i)from, to + stride * 14, 0, 14);
  __lsx_vstelm_b((__m128i)from, to + stride * 15, 0, 15);
}

pscatter< int8_t, Packet4c >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::pscatter< int8_t, Packet4c >	(	int8_t *	to,
		const Packet4c &	from,
		Index	stride
	)

                                                                                                                      {
  for (int i = 0; i != 4; i++) *(to + i * stride) = reinterpret_cast<const int8_t*>(&from)[i];
}

References i.

pscatter< int8_t, Packet8c >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::pscatter< int8_t, Packet8c >	(	int8_t *	to,
		const Packet8c &	from,
		Index	stride
	)

                                                                                                                      {
  vst1_lane_s8(to + stride * 0, from, 0);
  vst1_lane_s8(to + stride * 1, from, 1);
  vst1_lane_s8(to + stride * 2, from, 2);
  vst1_lane_s8(to + stride * 3, from, 3);
  vst1_lane_s8(to + stride * 4, from, 4);
  vst1_lane_s8(to + stride * 5, from, 5);
  vst1_lane_s8(to + stride * 6, from, 6);
  vst1_lane_s8(to + stride * 7, from, 7);
}

pscatter< numext::int32_t, PacketXi >()

template<>

EIGEN_DEVICE_FUNC void Eigen::internal::pscatter< numext::int32_t, PacketXi > ( numext::int32_t *  to, const PacketXi &  from, Index  stride  )

inline

                                                                                {
  // Indice format: {base=0, base+stride, base+stride*2, base+stride*3, ...}
  svint32_t indices = svindex_s32(0, stride);
  svst1_scatter_s32index_s32(svptrue_b32(), to, indices, from);
}

pscatter< short int, Packet8s >()

template<>

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void Eigen::internal::pscatter< short int, Packet8s >	(	short int *	to,
		const Packet8s &	from,
		Index	stride
	)

                                                                                       {
  pscatter_common<Packet8s>(to, from, stride);
}

pscatter< signed char, Packet16c >()

template<>

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void Eigen::internal::pscatter< signed char, Packet16c >	(	signed char *	to,
		const Packet16c &	from,
		Index	stride
	)

                                                                                          {
  pscatter_common<Packet16c>(to, from, stride);
}

pscatter< std::complex< double >, Packet1cd >() [1/3]

template<>

EIGEN_DEVICE_FUNC void Eigen::internal::pscatter< std::complex< double >, Packet1cd > ( std::complex< double > *  to, const Packet1cd &  from, Index stride   __attribute__(unused)  )

inline

                                                                                                              {
  EIGEN_MSA_DEBUG;
 
  pstore(to, from);
}

References EIGEN_MSA_DEBUG, and pstore().

pscatter< std::complex< double >, Packet1cd >() [2/3]

template<>

EIGEN_DEVICE_FUNC void Eigen::internal::pscatter< std::complex< double >, Packet1cd > ( std::complex< double > *  to, const Packet1cd &  from, Index stride  EIGEN_UNUSED  )

inline

                                                                                                   {
  pstore<std::complex<double> >(to, from);
}

pscatter< std::complex< double >, Packet1cd >() [3/3]

template<>

EIGEN_DEVICE_FUNC void Eigen::internal::pscatter< std::complex< double >, Packet1cd > ( std::complex< double > *  to, const Packet1cd &  from, Index     )

inline

                                                                                 {
  __lsx_vst((__m128i)from.v, (void*)to, 0);
}

pscatter< std::complex< double >, Packet2cd >()

template<>

EIGEN_DEVICE_FUNC void Eigen::internal::pscatter< std::complex< double >, Packet2cd > ( std::complex< double > *  to, const Packet2cd &  from, Index  stride  )

inline

                                                                                      {
  __m128d low = _mm256_extractf128_pd(from.v, 0);
  to[stride * 0] = std::complex<double>(_mm_cvtsd_f64(low), _mm_cvtsd_f64(_mm_shuffle_pd(low, low, 1)));
  __m128d high = _mm256_extractf128_pd(from.v, 1);
  to[stride * 1] = std::complex<double>(_mm_cvtsd_f64(high), _mm_cvtsd_f64(_mm_shuffle_pd(high, high, 1)));
}

pscatter< std::complex< double >, Packet4cd >()

template<>

EIGEN_DEVICE_FUNC void Eigen::internal::pscatter< std::complex< double >, Packet4cd > ( std::complex< double > *  to, const Packet4cd &  from, Index  stride  )

inline

                                                                                      {
  __m512i fromi = _mm512_castpd_si512(from.v);
  double* tod = (double*)(void*)to;
  _mm_storeu_pd(tod + 0 * stride, _mm_castsi128_pd(_mm512_extracti32x4_epi32(fromi, 0)));
  _mm_storeu_pd(tod + 2 * stride, _mm_castsi128_pd(_mm512_extracti32x4_epi32(fromi, 1)));
  _mm_storeu_pd(tod + 4 * stride, _mm_castsi128_pd(_mm512_extracti32x4_epi32(fromi, 2)));
  _mm_storeu_pd(tod + 6 * stride, _mm_castsi128_pd(_mm512_extracti32x4_epi32(fromi, 3)));
}

pscatter< std::complex< float >, Packet1cf >()

template<>

EIGEN_DEVICE_FUNC void Eigen::internal::pscatter< std::complex< float >, Packet1cf > ( std::complex< float > *  to, const Packet1cf &  from, Index  stride  )

inline

                                                                                     {
  to[stride * 0] = std::complex<float>(vget_lane_f32(from.v, 0), vget_lane_f32(from.v, 1));
}

pscatter< std::complex< float >, Packet2cf >()

template<>

EIGEN_DEVICE_FUNC void Eigen::internal::pscatter< std::complex< float >, Packet2cf > ( std::complex< float > *  to, const Packet2cf &  from, Index  stride  )

inline

                                                                                                  {
  pscatter_complex_size2<std::complex<float>, Packet2cf>(to, from, stride);
}

pscatter< std::complex< float >, Packet4cf >()

template<>

EIGEN_DEVICE_FUNC void Eigen::internal::pscatter< std::complex< float >, Packet4cf > ( std::complex< float > *  to, const Packet4cf &  from, Index  stride  )

inline

                                                                                     {
  __m128 low = _mm256_extractf128_ps(from.v, 0);
  to[stride * 0] =
      std::complex<float>(_mm_cvtss_f32(_mm_shuffle_ps(low, low, 0)), _mm_cvtss_f32(_mm_shuffle_ps(low, low, 1)));
  to[stride * 1] =
      std::complex<float>(_mm_cvtss_f32(_mm_shuffle_ps(low, low, 2)), _mm_cvtss_f32(_mm_shuffle_ps(low, low, 3)));
 
  __m128 high = _mm256_extractf128_ps(from.v, 1);
  to[stride * 2] =
      std::complex<float>(_mm_cvtss_f32(_mm_shuffle_ps(high, high, 0)), _mm_cvtss_f32(_mm_shuffle_ps(high, high, 1)));
  to[stride * 3] =
      std::complex<float>(_mm_cvtss_f32(_mm_shuffle_ps(high, high, 2)), _mm_cvtss_f32(_mm_shuffle_ps(high, high, 3)));
}

pscatter< std::complex< float >, Packet8cf >()

template<>

EIGEN_DEVICE_FUNC void Eigen::internal::pscatter< std::complex< float >, Packet8cf > ( std::complex< float > *  to, const Packet8cf &  from, Index  stride  )

inline

                                                                                     {
  pscatter((double*)(void*)to, _mm512_castps_pd(from.v), stride);
}

References pscatter().

pscatter< uint16_t, Packet4us >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::pscatter< uint16_t, Packet4us >	(	uint16_t *	to,
		const Packet4us &	from,
		Index	stride
	)

                                                                                       {
  vst1_lane_u16(to + stride * 0, from, 0);
  vst1_lane_u16(to + stride * 1, from, 1);
  vst1_lane_u16(to + stride * 2, from, 2);
  vst1_lane_u16(to + stride * 3, from, 3);
}

Referenced by pscatter< bfloat16, Packet4bf >().

pscatter< uint16_t, Packet8us >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::pscatter< uint16_t, Packet8us >	(	uint16_t *	to,
		const Packet8us &	from,
		Index	stride
	)

                                                                                       {
  __lsx_vstelm_h((__m128i)from, to, 0, 0);
  __lsx_vstelm_h((__m128i)from, to + stride * 1, 0, 1);
  __lsx_vstelm_h((__m128i)from, to + stride * 2, 0, 2);
  __lsx_vstelm_h((__m128i)from, to + stride * 3, 0, 3);
  __lsx_vstelm_h((__m128i)from, to + stride * 4, 0, 4);
  __lsx_vstelm_h((__m128i)from, to + stride * 5, 0, 5);
  __lsx_vstelm_h((__m128i)from, to + stride * 6, 0, 6);
  __lsx_vstelm_h((__m128i)from, to + stride * 7, 0, 7);
}

pscatter< uint32_t, Packet2ui >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::pscatter< uint32_t, Packet2ui >	(	uint32_t *	to,
		const Packet2ui &	from,
		Index	stride
	)

                                                                                       {
  vst1_lane_u32(to + stride * 0, from, 0);
  vst1_lane_u32(to + stride * 1, from, 1);
}

pscatter< uint32_t, Packet4ui >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pscatter< uint32_t, Packet4ui >	(	uint32_t *	to,
		const Packet4ui &	from,
		Index	stride
	)

                                                                                       {
  __lsx_vstelm_w((__m128i)from, to, 0, 0);
  __lsx_vstelm_w((__m128i)from, to + stride * 1, 0, 1);
  __lsx_vstelm_w((__m128i)from, to + stride * 2, 0, 2);
  __lsx_vstelm_w((__m128i)from, to + stride * 3, 0, 3);
}

pscatter< uint32_t, Packet8ui >()

template<>

EIGEN_DEVICE_FUNC void Eigen::internal::pscatter< uint32_t, Packet8ui > ( uint32_t *  to, const Packet8ui &  from, Index  stride  )

inline

                                                                                                               {
  pscatter<int, Packet8i>((int*)to, (Packet8i)from, stride);
}

References pscatter< int, Packet8i >().

pscatter< uint64_t, Packet2ul >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::pscatter< uint64_t, Packet2ul >	(	uint64_t *	to,
		const Packet2ul &	from,
		Index	stride
	)

                                                                                       {
  __lsx_vstelm_d((__m128i)from, to, 0, 0);
  __lsx_vstelm_d((__m128i)from, to + stride * 1, 0, 1);
}

pscatter< uint8_t, Packet16uc >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::pscatter< uint8_t, Packet16uc >	(	uint8_t *	to,
		const Packet16uc &	from,
		Index	stride
	)

                                                                                       {
  __lsx_vstelm_b((__m128i)from, to, 0, 0);
  __lsx_vstelm_b((__m128i)from, to + stride * 1, 0, 1);
  __lsx_vstelm_b((__m128i)from, to + stride * 2, 0, 2);
  __lsx_vstelm_b((__m128i)from, to + stride * 3, 0, 3);
  __lsx_vstelm_b((__m128i)from, to + stride * 4, 0, 4);
  __lsx_vstelm_b((__m128i)from, to + stride * 5, 0, 5);
  __lsx_vstelm_b((__m128i)from, to + stride * 6, 0, 6);
  __lsx_vstelm_b((__m128i)from, to + stride * 7, 0, 7);
  __lsx_vstelm_b((__m128i)from, to + stride * 8, 0, 8);
  __lsx_vstelm_b((__m128i)from, to + stride * 9, 0, 9);
  __lsx_vstelm_b((__m128i)from, to + stride * 10, 0, 10);
  __lsx_vstelm_b((__m128i)from, to + stride * 11, 0, 11);
  __lsx_vstelm_b((__m128i)from, to + stride * 12, 0, 12);
  __lsx_vstelm_b((__m128i)from, to + stride * 13, 0, 13);
  __lsx_vstelm_b((__m128i)from, to + stride * 14, 0, 14);
  __lsx_vstelm_b((__m128i)from, to + stride * 15, 0, 15);
}

pscatter< uint8_t, Packet4uc >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::pscatter< uint8_t, Packet4uc >	(	uint8_t *	to,
		const Packet4uc &	from,
		Index	stride
	)

                                                                                      {
  for (int i = 0; i != 4; i++) *(to + i * stride) = reinterpret_cast<const uint8_t*>(&from)[i];
}

References i.

pscatter< uint8_t, Packet8uc >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::pscatter< uint8_t, Packet8uc >	(	uint8_t *	to,
		const Packet8uc &	from,
		Index	stride
	)

                                                                                      {
  vst1_lane_u8(to + stride * 0, from, 0);
  vst1_lane_u8(to + stride * 1, from, 1);
  vst1_lane_u8(to + stride * 2, from, 2);
  vst1_lane_u8(to + stride * 3, from, 3);
  vst1_lane_u8(to + stride * 4, from, 4);
  vst1_lane_u8(to + stride * 5, from, 5);
  vst1_lane_u8(to + stride * 6, from, 6);
  vst1_lane_u8(to + stride * 7, from, 7);
}

pscatter< unsigned char, Packet16uc >()

template<>

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void Eigen::internal::pscatter< unsigned char, Packet16uc >	(	unsigned char *	to,
		const Packet16uc &	from,
		Index	stride
	)

                                                                                                                     {
  pscatter_common<Packet16uc>(to, from, stride);
}

pscatter< unsigned short int, Packet8us >()

template<>

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void Eigen::internal::pscatter< unsigned short int, Packet8us >	(	unsigned short int *	to,
		const Packet8us &	from,
		Index	stride
	)

                                                                                                 {
  pscatter_common<Packet8us>(to, from, stride);
}

pscatter_common()

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void Eigen::internal::pscatter_common	(	__UNPACK_TYPE__(Packet) *	to,
		const Packet &	from,
		Index	stride,
		const Index	n = unpacket_traits<Packet>::size
	)

                                                                                                        {
  EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) a[unpacket_traits<Packet>::size];
  eigen_internal_assert(n <= unpacket_traits<Packet>::size && "number of elements will scatter past end of packet");
  if (stride == 1) {
    if (n == unpacket_traits<Packet>::size) {
      return pstoreu(to, from);
    } else {
      return pstoreu_partial(to, from, n);
    }
  } else {
    pstore<__UNPACK_TYPE__(Packet)>(a, from);
    LOAD_STORE_UNROLL_16
    for (Index i = 0; i < n; i++) {
      to[i * stride] = a[i];
    }
  }
}

References __UNPACK_TYPE__(), a, EIGEN_ALIGN16, eigen_internal_assert, i, LOAD_STORE_UNROLL_16, n, pstoreu(), and pstoreu_partial().

pscatter_complex_size2()

template<typename Scalar , typename Packet >

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void Eigen::internal::pscatter_complex_size2	(	Scalar *	to,
		const Packet &	from,
		Index	stride,
		const Index	n = 2
	)

                                                                                     {
  eigen_internal_assert(n <= unpacket_traits<Packet>::size && "number of elements will scatter past end of packet");
  EIGEN_ALIGN16 Scalar af[2];
  pstore<Scalar>((Scalar*)af, from);
  for (Index i = 0; i < n; i++) {
    to[i * stride] = af[i];
  }
}

References EIGEN_ALIGN16, eigen_internal_assert, i, and n.

pscatter_partial()

template<typename Scalar , typename Packet >

EIGEN_DEVICE_FUNC void Eigen::internal::pscatter_partial ( Scalar *  to, const Packet &  from, Index  stride, const Index  n  )

inline

                                                                                                            {
  const Index packet_size = unpacket_traits<Packet>::size;
  EIGEN_ALIGN_MAX Scalar elements[packet_size];
  pstore<Scalar>(elements, from);
  for (Index i = 0; i < numext::mini(n, packet_size); i++) {
    to[i * stride] = elements[i];
  }
}

References EIGEN_ALIGN_MAX, i, Eigen::numext::mini(), and n.

Referenced by storeBF16fromResult().

pscatter_partial< bfloat16, Packet8bf >()

template<>

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void Eigen::internal::pscatter_partial< bfloat16, Packet8bf >	(	bfloat16 *	to,
		const Packet8bf &	from,
		Index	stride,
		const Index	n
	)

                                                                                                              {
  pscatter_common<Packet8bf>(to, from, stride, n);
}

References n.

pscatter_partial< float, Packet4f >()

template<>

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void Eigen::internal::pscatter_partial< float, Packet4f >	(	float *	to,
		const Packet4f &	from,
		Index	stride,
		const Index	n
	)

                                                                                                          {
  pscatter_common<Packet4f>(to, from, stride, n);
}

References n.

pscatter_partial< int, Packet4i >()

template<>

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void Eigen::internal::pscatter_partial< int, Packet4i >	(	int *	to,
		const Packet4i &	from,
		Index	stride,
		const Index	n
	)

                                                                                          {
  pscatter_common<Packet4i>(to, from, stride, n);
}

References n.

pscatter_partial< short int, Packet8s >()

template<>

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void Eigen::internal::pscatter_partial< short int, Packet8s >	(	short int *	to,
		const Packet8s &	from,
		Index	stride,
		const Index	n
	)

                                                                                                              {
  pscatter_common<Packet8s>(to, from, stride, n);
}

References n.

pscatter_partial< signed char, Packet16c >()

template<>

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void Eigen::internal::pscatter_partial< signed char, Packet16c >	(	signed char *	to,
		const Packet16c &	from,
		Index	stride,
		const Index	n
	)

                                                                                                   {
  pscatter_common<Packet16c>(to, from, stride, n);
}

References n.

pscatter_partial< std::complex< float >, Packet2cf >()

template<>

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void Eigen::internal::pscatter_partial< std::complex< float >, Packet2cf >	(	std::complex< float > *	to,
		const Packet2cf &	from,
		Index	stride,
		const Index	n
	)

                                                                                                           {
  pscatter_complex_size2<std::complex<float>, Packet2cf>(to, from, stride, n);
}

References n.

pscatter_partial< unsigned char, Packet16uc >()

template<>

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void Eigen::internal::pscatter_partial< unsigned char, Packet16uc >	(	unsigned char *	to,
		const Packet16uc &	from,
		Index	stride,
		const Index	n
	)

                                                                                                                    {
  pscatter_common<Packet16uc>(to, from, stride, n);
}

References n.

pscatter_partial< unsigned short int, Packet8us >()

template<>

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void Eigen::internal::pscatter_partial< unsigned short int, Packet8us >	(	unsigned short int *	to,
		const Packet8us &	from,
		Index	stride,
		const Index	n
	)

                                                                                                          {
  pscatter_common<Packet8us>(to, from, stride, n);
}

References n.

pselect() [1/29]

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::pselect ( const Packet &  mask, const Packet &  a, const Packet &  b  )

inline

Returns

or for each field in packet according to \mask

                                                                                              {
  return pselect_impl<Packet>::run(mask, a, b);
}

References a, b, and Eigen::internal::pselect_impl< Packet, EnableIf >::run().

pselect() [2/29]

template<>

EIGEN_STRONG_INLINE Packet16bf Eigen::internal::pselect	(	const Packet16bf &	mask,
		const Packet16bf &	a,
		const Packet16bf &	b
	)

                                                                                                         {
  // Input mask is expected to be all 0/1, handle it with 8-bit
  // intrinsic for performance.
  return _mm256_blendv_epi8(b, a, mask);
}

References a, and b.

pselect() [3/29]

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16c Eigen::internal::pselect	(	const Packet16c &	mask,
		const Packet16c &	a,
		const Packet16c &	b
	)

                                                                                                                       {
  return (Packet16c)__lsx_vbitsel_v((__m128i)b, (__m128i)a, (__m128i)mask);
}

References a, and b.

pselect() [4/29]

template<>

EIGEN_DEVICE_FUNC Packet16f Eigen::internal::pselect ( const Packet16f &  mask, const Packet16f &  a, const Packet16f &  b  )

inline

                                                                                                          {
  __mmask16 mask16 = _mm512_cmpeq_epi32_mask(_mm512_castps_si512(mask), _mm512_setzero_epi32());
  return _mm512_mask_blend_ps(mask16, a, b);
}

References a, and b.

pselect() [5/29]

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::pselect	(	const Packet16h &	mask,
		const Packet16h &	a,
		const Packet16h &	b
	)

                                                                                                     {
  return _mm256_blendv_epi8(b, a, mask);
}

References a, and b.

pselect() [6/29]

template<>

EIGEN_DEVICE_FUNC Packet16i Eigen::internal::pselect ( const Packet16i &  mask, const Packet16i &  a, const Packet16i &  b  )

inline

                                                                                                          {
  __mmask16 mask16 = _mm512_cmpeq_epi32_mask(mask, _mm512_setzero_epi32());
  return _mm512_mask_blend_epi32(mask16, a, b);
}

References a, and b.

pselect() [7/29]

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16uc Eigen::internal::pselect	(	const Packet16uc &	mask,
		const Packet16uc &	a,
		const Packet16uc &	b
	)

                                                                              {
  return __lsx_vbitsel_v(b, a, mask);
}

References a, and b.

pselect() [8/29]

template<>

EIGEN_DEVICE_FUNC Packet2cf Eigen::internal::pselect ( const Packet2cf &  mask, const Packet2cf &  a, const Packet2cf &  b  )

inline

                                                                                                          {
  Packet2cf res;
  res.v = (Packet4f)__lsx_vbitsel_v((__m128i)b.v, (__m128i)a.v, (__m128i)mask.v);
  return res;
}

References a, b, and res.

pselect() [9/29]

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2d Eigen::internal::pselect	(	const Packet2d &	mask,
		const Packet2d &	a,
		const Packet2d &	b
	)

                                                                                                                   {
  return (Packet2d)__lsx_vbitsel_v((__m128i)b, (__m128i)a, (__m128i)mask);
}

References a, and b.

pselect() [10/29]

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2f Eigen::internal::pselect	(	const Packet2f &	mask,
		const Packet2f &	a,
		const Packet2f &	b
	)

                                                                                                                   {
  return vbsl_f32(vreinterpret_u32_f32(mask), a, b);
}

References a, and b.

pselect() [11/29]

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2i Eigen::internal::pselect	(	const Packet2i &	mask,
		const Packet2i &	a,
		const Packet2i &	b
	)

                                                                                                                   {
  return vbsl_s32(vreinterpret_u32_s32(mask), a, b);
}

References a, and b.

pselect() [12/29]

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2l Eigen::internal::pselect	(	const Packet2l &	mask,
		const Packet2l &	a,
		const Packet2l &	b
	)

                                                                                                                   {
  return __lsx_vbitsel_v(b, a, mask);
}

References a, and b.

pselect() [13/29]

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ui Eigen::internal::pselect	(	const Packet2ui &	mask,
		const Packet2ui &	a,
		const Packet2ui &	b
	)

                                                                                                                       {
  return vbsl_u32(mask, a, b);
}

References a, and b.

pselect() [14/29]

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ul Eigen::internal::pselect	(	const Packet2ul &	mask,
		const Packet2ul &	a,
		const Packet2ul &	b
	)

                                                                                                                       {
  return __lsx_vbitsel_v(b, a, mask);
}

References a, and b.

pselect() [15/29]

template<>

EIGEN_DEVICE_FUNC Packet32h Eigen::internal::pselect ( const Packet32h &  mask, const Packet32h &  a, const Packet32h &  b  )

inline

                                                                                                          {
  __mmask32 mask32 = _mm512_cmp_epi16_mask(_mm512_castph_si512(mask), _mm512_setzero_epi32(), _MM_CMPINT_EQ);
  return _mm512_mask_blend_ph(mask32, a, b);
}

References a, and b.

pselect() [16/29]

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4bf Eigen::internal::pselect	(	const Packet4bf &	mask,
		const Packet4bf &	a,
		const Packet4bf &	b
	)

                                                                                                                       {
  return Packet4bf(pselect<Packet4us>(Packet4us(mask), Packet4us(a), Packet4us(b)));
}

References a, and b.

pselect() [17/29]

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4f Eigen::internal::pselect	(	const Packet4f &	mask,
		const Packet4f &	a,
		const Packet4f &	b
	)

                                                                                                 {
  return vec_sel(b, a, reinterpret_cast<Packet4ui>(mask));
}

References a, and b.

Referenced by generic_atan(), generic_expm1(), generic_log1p(), generic_ndtri(), generic_ndtri_lt_exp_neg_two(), generic_pow(), generic_pow_impl(), generic_rint(), Eigen::internal::unary_pow::handle_negative_exponent(), Eigen::internal::unary_pow::handle_nonint_nonint_errors(), pabsdiff(), packetmath(), Eigen::internal::scalar_logistic_op< float >::packetOp(), Eigen::internal::scalar_boolean_select_op< ThenScalar, ElseScalar, ConditionScalar >::packetOp(), Eigen::internal::scalar_logistic_op_impl< T, EnableIf >::packetOp(), pacos_float(), paddsub(), pasin_float(), patanh_double(), patanh_float(), pdiv_float_common(), pexp_complex(), pexp_double(), pexp_float(), pfrexp_generic(), phypot_complex(), plog_complex(), plog_impl_double(), plog_impl_float(), prsqrt_float_common(), psincos_double(), psincos_float(), psqrt_complex(), psqrt_float_common(), Eigen::internal::psign_impl< Packet, std::enable_if_t<!NumTraits< typename unpacket_traits< Packet >::type >::IsComplex &&!NumTraits< typename unpacket_traits< Packet >::type >::IsInteger > >::run(), Eigen::internal::psign_impl< Packet, std::enable_if_t< NumTraits< typename unpacket_traits< Packet >::type >::IsComplex &&unpacket_traits< Packet >::vectorizable > >::run(), Eigen::internal::generic_reciprocal_newton_step< Packet, Steps >::run(), Eigen::internal::generic_rsqrt_newton_step< Packet, Steps >::run(), Eigen::internal::generic_sqrt_newton_step< Packet, Steps >::run(), Eigen::internal::pminmax_impl< PropagateNaN >::run(), Eigen::internal::pminmax_impl< PropagateNumbers >::run(), Eigen::internal::generic_i0e< T, float >::run(), Eigen::internal::generic_i0e< T, double >::run(), Eigen::internal::generic_i1e< T, float >::run(), Eigen::internal::generic_i1e< T, double >::run(), Eigen::internal::generic_k0e< T, float >::run(), Eigen::internal::generic_k0e< T, double >::run(), Eigen::internal::generic_k0< T, float >::run(), Eigen::internal::generic_k0< T, double >::run(), Eigen::internal::generic_k1e< T, float >::run(), Eigen::internal::generic_k1e< T, double >::run(), Eigen::internal::generic_k1< T, float >::run(), Eigen::internal::generic_k1< T, double >::run(), Eigen::internal::generic_j0< T, float >::run(), Eigen::internal::generic_j0< T, double >::run(), Eigen::internal::generic_y0< T, float >::run(), Eigen::internal::generic_y0< T, double >::run(), Eigen::internal::generic_j1< T, float >::run(), Eigen::internal::generic_j1< T, double >::run(), Eigen::internal::generic_y1< T, float >::run(), Eigen::internal::generic_y1< T, double >::run(), Eigen::internal::generic_fast_erf< Scalar >::run(), Eigen::internal::generic_fast_erfc< Scalar >::run(), and twosum().

pselect() [18/29]

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4i Eigen::internal::pselect	(	const Packet4i &	mask,
		const Packet4i &	a,
		const Packet4i &	b
	)

                                                                                                                   {
  return __lsx_vbitsel_v(b, a, mask);
}

References a, and b.

pselect() [19/29]

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4s Eigen::internal::pselect	(	const Packet4s &	mask,
		const Packet4s &	a,
		const Packet4s &	b
	)

                                                                                                                   {
  return vbsl_s16(vreinterpret_u16_s16(mask), a, b);
}

References a, and b.

pselect() [20/29]

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4ui Eigen::internal::pselect	(	const Packet4ui &	mask,
		const Packet4ui &	a,
		const Packet4ui &	b
	)

                                                                                                                       {
  return __lsx_vbitsel_v(b, a, mask);
}

References a, and b.

pselect() [21/29]

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4us Eigen::internal::pselect	(	const Packet4us &	mask,
		const Packet4us &	a,
		const Packet4us &	b
	)

                                                                                                                       {
  return vbsl_u16(mask, a, b);
}

References a, and b.

pselect() [22/29]

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::pselect	(	const Packet8bf &	mask,
		const Packet8bf &	a,
		const Packet8bf &	b
	)

                                                                                                     {
  return _mm_blendv_epi8(b, a, mask);
}

References a, and b.

pselect() [23/29]

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8c Eigen::internal::pselect	(	const Packet8c &	mask,
		const Packet8c &	a,
		const Packet8c &	b
	)

                                                                                                                   {
  return vbsl_s8(vreinterpret_u8_s8(mask), a, b);
}

References a, and b.

pselect() [24/29]

template<>

EIGEN_DEVICE_FUNC Packet8d Eigen::internal::pselect ( const Packet8d &  mask, const Packet8d &  a, const Packet8d &  b  )

inline

                                                                                                      {
  __mmask8 mask8 = _mm512_cmp_epi64_mask(_mm512_castpd_si512(mask), _mm512_setzero_epi32(), _MM_CMPINT_EQ);
  return _mm512_mask_blend_pd(mask8, a, b);
}

References a, and b.

pselect() [25/29]

template<>

EIGEN_STRONG_INLINE Packet8h Eigen::internal::pselect	(	const Packet8h &	mask,
		const Packet8h &	a,
		const Packet8h &	b
	)

                                                                                                 {
  return _mm_blendv_epi8(b, a, mask);
}

References a, and b.

pselect() [26/29]

template<>

EIGEN_DEVICE_FUNC Packet8l Eigen::internal::pselect ( const Packet8l &  mask, const Packet8l &  a, const Packet8l &  b  )

inline

                                                                                                      {
  __mmask8 mask8 = _mm512_cmpeq_epi64_mask(mask, _mm512_setzero_si512());
  return _mm512_mask_blend_epi64(mask8, a, b);
}

References a, and b.

pselect() [27/29]

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8s Eigen::internal::pselect	(	const Packet8s &	mask,
		const Packet8s &	a,
		const Packet8s &	b
	)

                                                                                                                   {
  return __lsx_vbitsel_v(b, a, mask);
}

References a, and b.

pselect() [28/29]

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8uc Eigen::internal::pselect	(	const Packet8uc &	mask,
		const Packet8uc &	a,
		const Packet8uc &	b
	)

                                                                                                                       {
  return vbsl_u8(mask, a, b);
}

References a, and b.

pselect() [29/29]

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8us Eigen::internal::pselect	(	const Packet8us &	mask,
		const Packet8us &	a,
		const Packet8us &	b
	)

                                                                                                                       {
  return __lsx_vbitsel_v(b, a, mask);
}

References a, and b.

pselect< bool >()

template<>

EIGEN_DEVICE_FUNC bool Eigen::internal::pselect< bool > ( const bool &  cond, const bool &  a, const bool &  b  )

inline

                                                                                            {
  return cond ? a : b;
}

References a, and b.

pselect< Packet4d >()

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::pselect< Packet4d >	(	const Packet4d &	mask,
		const Packet4d &	a,
		const Packet4d &	b
	)

                                                                                                           {
  return _mm256_blendv_pd(b, a, mask);
}

References a, and b.

Referenced by pblend().

pselect< Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::pselect< Packet8f >	(	const Packet8f &	mask,
		const Packet8f &	a,
		const Packet8f &	b
	)

                                                                                                           {
  return _mm256_blendv_ps(b, a, mask);
}

References a, and b.

Referenced by pblend().

pselect< Packet8i >()

template<>

EIGEN_STRONG_INLINE Packet8i Eigen::internal::pselect< Packet8i >	(	const Packet8i &	mask,
		const Packet8i &	a,
		const Packet8i &	b
	)

                                                                                                           {
  return _mm256_castps_si256(
      _mm256_blendv_ps(_mm256_castsi256_ps(b), _mm256_castsi256_ps(a), _mm256_castsi256_ps(mask)));
}

References a, and b.

pselect< Packet8ui >()

template<>

EIGEN_STRONG_INLINE Packet8ui Eigen::internal::pselect< Packet8ui >	(	const Packet8ui &	mask,
		const Packet8ui &	a,
		const Packet8ui &	b
	)

                                                                                                                {
  return _mm256_castps_si256(
      _mm256_blendv_ps(_mm256_castsi256_ps(b), _mm256_castsi256_ps(a), _mm256_castsi256_ps(mask)));
}

References a, and b.

pset1()

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::pset1 ( const typename unpacket_traits< Packet >::type &  a )

inline

Returns

a packet with constant coefficients a, e.g.: (a,a,a,a)

                                                                                     {
  return a;
}

References a.

pset1< Packet16b >()

template<>

EIGEN_STRONG_INLINE Packet16b Eigen::internal::pset1< Packet16b >

(

const bool &

from

)

                                                                 {
  return _mm_set1_epi8(static_cast<char>(from));
}

Referenced by pcmp_eq(), and ptrue< Packet16b >().

pset1< Packet16bf >()

template<>

EIGEN_STRONG_INLINE Packet16bf Eigen::internal::pset1< Packet16bf >

(

const bfloat16 &

from

)

                                                                       {
  return _mm256_set1_epi16(from.value);
}

References Eigen::bfloat16_impl::__bfloat16_raw::value.

pset1< Packet16c >() [1/2]

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::pset1< Packet16c >

(

const int8_t &

from

)

                                                                   {
  return __lsx_vreplgr2vr_b(from);
}

pset1< Packet16c >() [2/2]

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::pset1< Packet16c >

(

const signed char &

from

)

                                                                        {
  return pset1_size16<Packet16c>(from);
}

Referenced by pdiv< Packet16c >(), and plset< Packet16c >().

pset1< Packet16f >()

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::pset1< Packet16f >

(

const float &

from

)

                                                                  {
  return _mm512_set1_ps(from);
}

Referenced by pstore1< Packet16f >().

pset1< Packet16h >()

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::pset1< Packet16h >

(

const Eigen::half &

from

)

                                                                      {
  return _mm256_set1_epi16(from.x);
}

References Eigen::half_impl::__half_raw::x.

pset1< Packet16i >()

template<>

EIGEN_STRONG_INLINE Packet16i Eigen::internal::pset1< Packet16i >

(

const int &

from

)

                                                                {
  return _mm512_set1_epi32(from);
}

Referenced by pstore1< Packet16i >().

pset1< Packet16uc >() [1/2]

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::pset1< Packet16uc >

(

const uint8_t &

from

)

                                                                      {
  return __lsx_vreplgr2vr_b(from);
}

pset1< Packet16uc >() [2/2]

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::pset1< Packet16uc >

(

const unsigned char &

from

)

                                                                            {
  return pset1_size16<Packet16uc>(from);
}

Referenced by loadPacketPartialZero(), pload_partial_common(), ploadu_partial_common(), plset< Packet16uc >(), pstore_partial_common(), and pstoreu_partial_common().

pset1< Packet1cd >()

template<>

EIGEN_STRONG_INLINE Packet1cd Eigen::internal::pset1< Packet1cd >

(

const std::complex< double > &

from

)

                                                 { /* here we really have to use unaligned loads :( */
  return ploadu<Packet1cd>(&from);
}

References ploadu< Packet1cd >().

Referenced by ploaddup< Packet1cd >(), and pset1< Packet4cd >().

pset1< Packet1cf >()

template<>

EIGEN_STRONG_INLINE Packet1cf Eigen::internal::pset1< Packet1cf >

(

const std::complex< float > &

from

)

                                                                              {
  return Packet1cf(vld1_f32(reinterpret_cast<const float*>(&from)));
}

Referenced by ploaddup< Packet1cf >().

pset1< Packet2cd >()

template<>

EIGEN_STRONG_INLINE Packet2cd Eigen::internal::pset1< Packet2cd >

(

const std::complex< double > &

from

)

                                                                               {
  // in case casting to a __m128d* is really not safe, then we can still fallback to this version: (much slower though)
  //   return Packet2cd(_mm256_loadu2_m128d((const double*)&from,(const double*)&from));
  return Packet2cd(_mm256_broadcast_pd((const __m128d*)(const void*)&from));
}

Referenced by ploaddup< Packet2cd >().

pset1< Packet2cf >()

template<>

EIGEN_STRONG_INLINE Packet2cf Eigen::internal::pset1< Packet2cf >

(

const std::complex< float > &

from

)

                                                                              {
  Packet2cf res;
#ifdef EIGEN_VECTORIZE_VSX
  // Load a single std::complex<float> from memory and duplicate
  //
  // Using pload would read past the end of the reference in this case
  // Using vec_xl_len + vec_splat, generates poor assembly
  __asm__("lxvdsx %x0,%y1" : "=wa"(res.v) : "Z"(from));
#else
  if ((std::ptrdiff_t(&from) % 16) == 0)
    res.v = pload<Packet4f>((const float*)&from);
  else
    res.v = ploadu<Packet4f>((const float*)&from);
  res.v = vec_perm(res.v, res.v, p16uc_PSET64_HI);
#endif
  return res;
}

References p16uc_PSET64_HI, pload< Packet4f >(), ploadu< Packet4f >(), and res.

Referenced by ploaddup< Packet2cf >().

pset1< Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pset1< Packet2d >

(

const double &

from

)

                                                                 {
  Packet2d v = {from, from};
  return v;
}

References v.

Referenced by pldexp_fast< Packet2d >(), pload_real(), ploaddup< Packet2d >(), plset< Packet2d >(), prsqrt< Packet2d >(), pset_zero< Packet1cd >(), and Eigen::internal::compute_inverse_size4< Architecture::Target, double, MatrixType, ResultType >::run().

pset1< Packet2f >()

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::pset1< Packet2f >

(

const float &

from

)

                                                                {
  return vdup_n_f32(from);
}

Referenced by plset< Packet2f >().

pset1< Packet2i >()

template<>

EIGEN_STRONG_INLINE Packet2i Eigen::internal::pset1< Packet2i >

(

const int32_t &

from

)

                                                                  {
  return vdup_n_s32(from);
}

Referenced by pdiv< Packet2i >(), and plset< Packet2i >().

pset1< Packet2l >()

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::pset1< Packet2l >

(

const int64_t &

from

)

                                                                  {
  return __lsx_vreplgr2vr_d(from);
}

Referenced by ploaddup< Packet2l >(), and plset< Packet2l >().

pset1< Packet2ui >()

template<>

EIGEN_STRONG_INLINE Packet2ui Eigen::internal::pset1< Packet2ui >

(

const uint32_t &

from

)

                                                                     {
  return vdup_n_u32(from);
}

Referenced by pdiv< Packet2ui >(), and plset< Packet2ui >().

pset1< Packet2ul >()

template<>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::pset1< Packet2ul >

(

const uint64_t &

from

)

                                                                     {
  return __lsx_vreplgr2vr_d(from);
}

Referenced by ploaddup< Packet2ul >(), plset< Packet2ul >(), and pset1frombits< Packet2d >().

pset1< Packet32h >()

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::pset1< Packet32h >

(

const Eigen::half &

from

)

                                                                      {
  // half/half_raw is bit compatible
  return _mm512_set1_ph(numext::bit_cast<_Float16>(from));
}

Referenced by plset< Packet32h >().

pset1< Packet4bf >()

template<>

EIGEN_STRONG_INLINE Packet4bf Eigen::internal::pset1< Packet4bf >

(

const bfloat16 &

from

)

                                                                     {
  return Packet4bf(pset1<Packet4us>(from.value));
}

References pset1< Packet4us >(), and Eigen::bfloat16_impl::__bfloat16_raw::value.

pset1< Packet4c >()

template<>

EIGEN_STRONG_INLINE Packet4c Eigen::internal::pset1< Packet4c >

(

const int8_t &

from

)

                                                                 {
  return vget_lane_s32(vreinterpret_s32_s8(vdup_n_s8(from)), 0);
}

Referenced by pdiv< Packet4c >().

pset1< Packet4cd >()

template<>

EIGEN_STRONG_INLINE Packet4cd Eigen::internal::pset1< Packet4cd >

(

const std::complex< double > &

from

)

                                                                               {
  return Packet4cd(_mm512_castps_pd(_mm512_broadcast_f32x4(_mm_castpd_ps(pset1<Packet1cd>(from).v))));
}

References pset1< Packet1cd >(), and v.

pset1< Packet4cf >()

template<>

EIGEN_STRONG_INLINE Packet4cf Eigen::internal::pset1< Packet4cf >

(

const std::complex< float > &

from

)

                                                                              {
  const float re = std::real(from);
  const float im = std::imag(from);
  return Packet4cf(_mm256_set_ps(im, re, im, re, im, re, im, re));
}

References imag().

pset1< Packet4d >()

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::pset1< Packet4d >

(

const double &

from

)

                                                                 {
  return _mm256_set1_pd(from);
}

Referenced by pldexp< Packet4d >(), pldexp_fast< Packet4d >(), plset< Packet4d >(), and pstore1< Packet4d >().

pset1< Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pset1< Packet4f >

(

const float &

from

)

                                                                {
  return pset1_size4<Packet4f>(from);
}

Referenced by convertArrayPointerBF16toF32DupOne(), gemmbfloat16(), gemmMMAbfloat16(), gemv_bfloat16_col(), gemv_bfloat16_row(), gemvMMA_bfloat16_col(), gemvMMA_bfloat16_row(), loadTwoRhsFloat32(), pload_real(), plset< Packet4f >(), prsqrt< Packet4f >(), pset_zero< Packet2cf >(), and zeroAccumulators().

pset1< Packet4i >() [1/2]

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pset1< Packet4i >

(

const int &

from

)

                                                              {
  return pset1_size4<Packet4i>(from);
}

Referenced by parithmetic_shift_right(), pdiv< Packet4i >(), pldexp< Packet4d >(), pldexp_fast< Packet4d >(), plogical_shift_left(), plogical_shift_right(), plset< Packet4i >(), and pset1frombits< Packet4f >().

pset1< Packet4i >() [2/2]

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pset1< Packet4i >

(

const int32_t &

from

)

                                                                  {
  return __lsx_vreplgr2vr_w(from);
}

pset1< Packet4s >()

template<>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::pset1< Packet4s >

(

const int16_t &

from

)

                                                                  {
  return vdup_n_s16(from);
}

Referenced by pdiv< Packet4s >(), and plset< Packet4s >().

pset1< Packet4uc >()

template<>

EIGEN_STRONG_INLINE Packet4uc Eigen::internal::pset1< Packet4uc >

(

const uint8_t &

from

)

                                                                    {
  return vget_lane_u32(vreinterpret_u32_u8(vdup_n_u8(from)), 0);
}

Referenced by pdiv< Packet4uc >().

pset1< Packet4ui >()

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::pset1< Packet4ui >

(

const uint32_t &

from

)

                                                                     {
  return __lsx_vreplgr2vr_w(from);
}

Referenced by pcmp_le(), pcmp_lt(), plset< Packet4ui >(), and pset1frombits< Packet4f >().

pset1< Packet4us >()

template<>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::pset1< Packet4us >

(

const uint16_t &

from

)

                                                                     {
  return vdup_n_u16(from);
}

Referenced by pdiv< Packet4us >(), plset< Packet4us >(), pnegate< Packet4bf >(), and pset1< Packet4bf >().

pset1< Packet8bf >()

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::pset1< Packet8bf >

(

const bfloat16 &

from

)

                                                                     {
  return pset1_size8<Packet8us>(reinterpret_cast<const unsigned short int&>(from));
}

Referenced by KLoop(), loadBfloat16(), plset< Packet8bf >(), and vecColLoop().

pset1< Packet8c >()

template<>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::pset1< Packet8c >

(

const int8_t &

from

)

                                                                 {
  return vdup_n_s8(from);
}

Referenced by pdiv< Packet8c >(), and plset< Packet8c >().

pset1< Packet8cf >()

template<>

EIGEN_STRONG_INLINE Packet8cf Eigen::internal::pset1< Packet8cf >

(

const std::complex< float > &

from

)

                                                                              {
  const float re = std::real(from);
  const float im = std::imag(from);
  return Packet8cf(_mm512_set_ps(im, re, im, re, im, re, im, re, im, re, im, re, im, re, im, re));
}

References imag().

pset1< Packet8d >()

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::pset1< Packet8d >

(

const double &

from

)

                                                                 {
  return _mm512_set1_pd(from);
}

Referenced by pldexp< Packet8d >(), and pstore1< Packet8d >().

pset1< Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::pset1< Packet8f >

(

const float &

from

)

                                                                {
  return _mm256_set1_ps(from);
}

Referenced by plset< Packet8f >(), and pstore1< Packet8f >().

pset1< Packet8h >()

template<>

EIGEN_STRONG_INLINE Packet8h Eigen::internal::pset1< Packet8h >

(

const Eigen::half &

from

)

                                                                    {
  return _mm_set1_epi16(numext::bit_cast<numext::uint16_t>(from));
}

pset1< Packet8i >()

template<>

EIGEN_STRONG_INLINE Packet8i Eigen::internal::pset1< Packet8i >

(

const int &

from

)

                                                              {
  return _mm256_set1_epi32(from);
}

Referenced by pldexp< Packet8d >(), plset< Packet8i >(), pset1frombits< Packet8f >(), and pstore1< Packet8i >().

pset1< Packet8l >()

template<>

EIGEN_STRONG_INLINE Packet8l Eigen::internal::pset1< Packet8l >

(

const int64_t &

from

)

                                                                  {
  return _mm512_set1_epi64(from);
}

Referenced by pstore1< Packet8l >().

pset1< Packet8s >() [1/2]

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::pset1< Packet8s >

(

const int16_t &

from

)

                                                                  {
  return __lsx_vreplgr2vr_h(from);
}

pset1< Packet8s >() [2/2]

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::pset1< Packet8s >

(

const short int &

from

)

                                                                    {
  return pset1_size8<Packet8s>(from);
}

Referenced by plset< Packet8s >().

pset1< Packet8uc >()

template<>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::pset1< Packet8uc >

(

const uint8_t &

from

)

                                                                    {
  return vdup_n_u8(from);
}

Referenced by pdiv< Packet8uc >(), and plset< Packet8uc >().

pset1< Packet8ui >()

template<>

EIGEN_STRONG_INLINE Packet8ui Eigen::internal::pset1< Packet8ui >

(

const uint32_t &

from

)

                                                                     {
  return _mm256_set1_epi32(from);
}

Referenced by plset< Packet8ui >().

pset1< Packet8us >() [1/2]

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::pset1< Packet8us >

(

const uint16_t &

from

)

                                                                     {
  return __lsx_vreplgr2vr_h(from);
}

pset1< Packet8us >() [2/2]

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::pset1< Packet8us >

(

const unsigned short int &

from

)

                                                                               {
  return pset1_size8<Packet8us>(from);
}

Referenced by oneConvertBF16Hi(), oneConvertBF16Lo(), oneConvertBF16Perm(), pcast< Packet8bf, Packet4f >(), and plset< Packet8us >().

pset1< PacketXf >()

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::pset1< PacketXf >

(

const float &

from

)

                                                                {
  return svdup_n_f32(from);
}

Referenced by plset< PacketXf >().

pset1< PacketXi >()

template<>

EIGEN_STRONG_INLINE PacketXi Eigen::internal::pset1< PacketXi >

(

const numext::int32_t &

from

)

                                                                        {
  return svdup_n_s32(from);
}

Referenced by plset< PacketXi >().

pset1_size16()

template<typename Packet >

EIGEN_STRONG_INLINE Packet Eigen::internal::pset1_size16

(

const __UNPACK_TYPE__(Packet) &

from

)

                                                                              {
  Packet v = {from, from, from, from, from, from, from, from, from, from, from, from, from, from, from, from};
  return v;
}

References v.

pset1_size4()

template<typename Packet >

EIGEN_STRONG_INLINE Packet Eigen::internal::pset1_size4

(

const __UNPACK_TYPE__(Packet) &

from

)

                                                                             {
  Packet v = {from, from, from, from};
  return v;
}

References v.

pset1_size8()

template<typename Packet >

EIGEN_STRONG_INLINE Packet Eigen::internal::pset1_size8

(

const __UNPACK_TYPE__(Packet) &

from

)

                                                                             {
  Packet v = {from, from, from, from, from, from, from, from};
  return v;
}

References v.

pset1frombits()

template<typename Packet , typename BitsType >

EIGEN_DEVICE_FUNC Packet Eigen::internal::pset1frombits ( BitsType  a )

inline

Returns

a packet with constant coefficients set from bits

pset1frombits< Packet16f >()

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::pset1frombits< Packet16f >

(

unsigned int

from

)

                                                                          {
  return _mm512_castsi512_ps(_mm512_set1_epi32(from));
}

Referenced by pround< Packet16f >().

pset1frombits< Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pset1frombits< Packet2d >

(

uint64_t

from

)

                                                                    {
  return reinterpret_cast<__m128d>((__m128i)pset1<Packet2ul>(from));
}

References pset1< Packet2ul >().

Referenced by pfrexp_generic_get_biased_exponent(), and pround().

pset1frombits< Packet2f >()

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::pset1frombits< Packet2f >

(

uint32_t

from

)

                                                                    {
  return vreinterpret_f32_u32(vdup_n_u32(from));
}

pset1frombits< Packet32h >()

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::pset1frombits< Packet32h >

(

unsigned short

from

)

                                                                            {
  return _mm512_castsi512_ph(_mm512_set1_epi16(from));
}

Referenced by pround< Packet32h >().

pset1frombits< Packet4d >()

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::pset1frombits< Packet4d >

(

uint64_t

from

)

                                                                    {
  return _mm256_castsi256_pd(_mm256_set1_epi64x(from));
}

Referenced by pfrexp_generic_get_biased_exponent(), and pround< Packet4d >().

pset1frombits< Packet4f >() [1/2]

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pset1frombits< Packet4f >

(

uint32_t

from

)

                                                                    {
  return reinterpret_cast<__m128>((__m128i)pset1<Packet4ui>(from));
}

References pset1< Packet4ui >().

pset1frombits< Packet4f >() [2/2]

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pset1frombits< Packet4f >

(

unsigned int

from

)

                                                                        {
  return reinterpret_cast<Packet4f>(pset1<Packet4i>(from));
}

References pset1< Packet4i >().

Referenced by pround().

pset1frombits< Packet8d >()

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::pset1frombits< Packet8d >

(

const numext::uint64_t

from

)

                                                                                {
  return _mm512_castsi512_pd(_mm512_set1_epi64(from));
}

Referenced by pfrexp_generic_get_biased_exponent(), and pround< Packet8d >().

pset1frombits< Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::pset1frombits< Packet8f >

(

unsigned int

from

)

                                                                        {
  return _mm256_castsi256_ps(pset1<Packet8i>(from));
}

References pset1< Packet8i >().

Referenced by pround< Packet8f >().

pset1frombits< PacketXf >()

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::pset1frombits< PacketXf >

(

numext::uint32_t

from

)

                                                                          {
  return svreinterpret_f32_u32(svdup_n_u32_x(svptrue_b32(), from));
}

psign() [1/2]

template<>

EIGEN_DEVICE_FUNC bool Eigen::internal::psign ( const bool &  a )

inline

                                                   {
  return a;
}

References a.

psign() [2/2]

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::psign ( const Packet &  a )

inline

Returns

the sign of a (coeff-wise)

                                                       {
  return psign_impl<Packet>::run(a);
}

References a, and Eigen::internal::psign_impl< Packet, EnableIf >::run().

Referenced by packetmath(), packetmath_complex(), packetmath_real(), and Eigen::internal::scalar_sign_op< Scalar >::packetOp().

psignbit() [1/15]

template<typename Packet >

EIGEN_DEVICE_FUNC constexpr EIGEN_ALWAYS_INLINE Packet Eigen::internal::psignbit ( const Packet &  a )

constexpr

Returns

the sign bit of a as a bitmask

                                                                                 {
  return psignbit_impl<Packet>::run(a);
}

References a, and run().

psignbit() [2/15]

template<>

EIGEN_STRONG_INLINE Packet16bf Eigen::internal::psignbit

(

const Packet16bf &

a

)

                                                             {
  return _mm256_srai_epi16(a, 15);
}

References a.

psignbit() [3/15]

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::psignbit

(

const Packet16f &

a

)

                                                           {
  return _mm512_castsi512_ps(_mm512_srai_epi32(_mm512_castps_si512(a), 31));
}

References a.

psignbit() [4/15]

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::psignbit

(

const Packet16h &

a

)

                                                           {
  return _mm256_srai_epi16(a, 15);
}

References a.

psignbit() [5/15]

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::psignbit

(

const Packet2d &

a

)

                                                         {
  return (__m128d)(__lsx_vsrai_d((__m128i)a, 63));
}

References a.

psignbit() [6/15]

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::psignbit

(

const Packet2f &

a

)

                                                         {
  return vreinterpret_f32_s32(vshr_n_s32(vreinterpret_s32_f32(a), 31));
}

References a.

psignbit() [7/15]

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::psignbit

(

const Packet2l &

a

)

                                                         {
  Packet4i tmp = psignbit<Packet4i>(Packet4i(a));
  return Packet2l(_mm_shuffle_epi32(tmp, (shuffle_mask<1, 1, 3, 3>::mask)));
}

References a, and tmp.

psignbit() [8/15]

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::psignbit

(

const Packet4f &

a

)

                                                         {
  return (Packet4f)vec_sra((Packet4i)a, vec_splats((unsigned int)(31)));
}

References a.

psignbit() [9/15]

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::psignbit

(

const Packet4i &

a

)

                                                         {
  return _mm_srai_epi32(a, 31);
}

References a.

psignbit() [10/15]

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::psignbit

(

const Packet4ui &

a

)

                                                           {
  return pzero(a);
}

References a, and pzero().

psignbit() [11/15]

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::psignbit

(

const Packet8bf &

a

)

                                                           {
  return vec_sra(a.m_val, vec_splat_u16(15));
}

References a.

Referenced by generic_pow(), Eigen::internal::test_signbit_op< Scalar >::packetOp(), and pacos_float().

psignbit() [12/15]

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::psignbit

(

const Packet8d &

a

)

                                                         {
  return _mm512_castsi512_pd(_mm512_srai_epi64(_mm512_castpd_si512(a), 63));
}

References a.

psignbit() [13/15]

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::psignbit

(

const Packet8f &

a

)

                                                         {
#ifdef EIGEN_VECTORIZE_AVX2
  return _mm256_castsi256_ps(_mm256_cmpgt_epi32(_mm256_setzero_si256(), _mm256_castps_si256(a)));
#else
  return _mm256_castsi256_ps(parithmetic_shift_right<31>(Packet8i(_mm256_castps_si256(a))));
#endif
}

References a.

psignbit() [14/15]

template<>

EIGEN_STRONG_INLINE Packet8h Eigen::internal::psignbit

(

const Packet8h &

a

)

                                                         {
  return _mm_cmpgt_epi16(_mm_setzero_si128(), a);
}

References a.

psignbit() [15/15]

template<>

EIGEN_STRONG_INLINE Packet8ui Eigen::internal::psignbit

(

const Packet8ui &

)

                                                           {
  return _mm256_setzero_si256();
}

psignbit< Packet32h >()

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::psignbit< Packet32h >

(

const Packet32h &

a

)

                                                                      {
  return _mm512_castsi512_ph(_mm512_srai_epi16(_mm512_castph_si512(a), 15));
}

References a.

psin()

template<typename Packet >

EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::psin

(

const Packet &

a

)

Returns

the sine of a (coeff-wise)

                                                                                  {
  EIGEN_USING_STD(sin);
  return sin(a);
}

References a, EIGEN_USING_STD, and sin().

Referenced by packetmath_real(), Eigen::internal::scalar_sin_op< Scalar >::packetOp(), psin< Packet32h >(), Eigen::internal::generic_j0< T, double >::run(), Eigen::internal::generic_y0< T, float >::run(), Eigen::internal::generic_y0< T, double >::run(), Eigen::internal::generic_j1< T, double >::run(), Eigen::internal::generic_y1< T, float >::run(), and Eigen::internal::generic_y1< T, double >::run().

psin< Packet16h >()

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::psin< Packet16h >

(

const Packet16h &

)

psin< Packet32h >()

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::psin< Packet32h >

(

const Packet32h &

a

)

                                                                  {
  Packet16h low;
  Packet16h high;
  extract2Packet16h(a, low, high);
 
  Packet16h lowOut = psin(low);
  Packet16h highOut = psin(high);
 
  return combine2Packet16h(lowOut, highOut);
}

References a, combine2Packet16h(), extract2Packet16h(), and psin().

psin< Packet4f >()

template<>

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet4f Eigen::internal::psin< Packet4f >

(

const Packet4f &

x

)

                                                                                               {
  return psincos_inner_msa_float</* sine */ true>(x);
}

References psincos_inner_msa_float(), and plotDoE::x.

psin< Packet8bf >()

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::psin< Packet8bf >

(

const Packet8bf &

a

)

                                                                  {
  BF16_TO_F32_UNARY_OP_WRAPPER(psin_float, a);
}

References a, BF16_TO_F32_UNARY_OP_WRAPPER, and psin_float().

psin< PacketXf >()

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::psin< PacketXf >

(

const PacketXf &

x

)

                                                               {
  return psin_float(x);
}

References psin_float(), and plotDoE::x.

psin_double()

template<typename Packet >

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::psin_double

(

const Packet &

x

)

Returns

sin(x) for double precision float

                                                                                        {
  return psincos_double<true>(x);
}

References plotDoE::x.

psin_float()

template<typename Packet >

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::psin_float

(

const Packet &

x

)

Returns

sin(x) for single precision float

                                                                                       {
  return psincos_float<true>(x);
}

References plotDoE::x.

Referenced by psin< Packet8bf >(), and psin< PacketXf >().

psincos_double()

template<bool ComputeSine, typename Packet , bool ComputeBoth = false>

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::psincos_double

(

const Packet &

x

)

                                    {
  typedef typename unpacket_traits<Packet>::integer_packet PacketI;
  typedef typename unpacket_traits<PacketI>::type ScalarI;
 
  const Packet cst_sign_mask = pset1frombits<Packet>(static_cast<Eigen::numext::uint64_t>(0x8000000000000000u));
 
  // If the argument is smaller than this value, use a simpler argument reduction
  const double small_th = 15;
  // If the argument is bigger than this value, use the non-vectorized std version
  const double huge_th = 1e14;
 
  const Packet cst_2oPI = pset1<Packet>(0.63661977236758134307553505349006);  // 2/PI
  // Integer Packet constants
  const PacketI cst_one = pset1<PacketI>(ScalarI(1));
  // Constant for splitting
  const Packet cst_split = pset1<Packet>(1 << 24);
 
  Packet x_abs = pabs(x);
 
  // Scale x by 2/Pi
  PacketI q_int;
  Packet s;
 
  // TODO Implement huge angle argument reduction
  if (EIGEN_PREDICT_FALSE(predux_any(pcmp_le(pset1<Packet>(small_th), x_abs)))) {
    Packet q_high = pmul(pfloor(pmul(x_abs, pdiv(cst_2oPI, cst_split))), cst_split);
    Packet q_low_noround = psub(pmul(x_abs, cst_2oPI), q_high);
    q_int = pcast<Packet, PacketI>(padd(q_low_noround, pset1<Packet>(0.5)));
    Packet q_low = pcast<PacketI, Packet>(q_int);
    s = trig_reduce_medium_double(x_abs, q_high, q_low);
  } else {
    Packet qval_noround = pmul(x_abs, cst_2oPI);
    q_int = pcast<Packet, PacketI>(padd(qval_noround, pset1<Packet>(0.5)));
    Packet q = pcast<PacketI, Packet>(q_int);
    s = trig_reduce_small_double(x_abs, q);
  }
 
  // All the upcoming approximating polynomials have even exponents
  Packet ss = pmul(s, s);
 
  // Padé approximant of cos(x)
  // Assuring < 1 ULP error on the interval [-pi/4, pi/4]
  // cos(x) ~= (80737373*x^8 - 13853547000*x^6 + 727718024880*x^4 - 11275015752000*x^2 + 23594700729600)/(147173*x^8 +
  // 39328920*x^6 + 5772800880*x^4 + 522334612800*x^2 + 23594700729600)
  // MATLAB code to compute those coefficients:
  //    syms x;
  //    cosf = @(x) cos(x);
  //    pade_cosf = pade(cosf(x), x, 0, 'Order', 8)
  Packet sc1_num = pmadd(ss, pset1<Packet>(80737373), pset1<Packet>(-13853547000));
  Packet sc2_num = pmadd(sc1_num, ss, pset1<Packet>(727718024880));
  Packet sc3_num = pmadd(sc2_num, ss, pset1<Packet>(-11275015752000));
  Packet sc4_num = pmadd(sc3_num, ss, pset1<Packet>(23594700729600));
  Packet sc1_denum = pmadd(ss, pset1<Packet>(147173), pset1<Packet>(39328920));
  Packet sc2_denum = pmadd(sc1_denum, ss, pset1<Packet>(5772800880));
  Packet sc3_denum = pmadd(sc2_denum, ss, pset1<Packet>(522334612800));
  Packet sc4_denum = pmadd(sc3_denum, ss, pset1<Packet>(23594700729600));
  Packet scos = pdiv(sc4_num, sc4_denum);
 
  // Padé approximant of sin(x)
  // Assuring < 1 ULP error on the interval [-pi/4, pi/4]
  // sin(x) ~= (x*(4585922449*x^8 - 1066023933480*x^6 + 83284044283440*x^4 - 2303682236856000*x^2 +
  // 15605159573203200))/(45*(1029037*x^8 + 345207016*x^6 + 61570292784*x^4 + 6603948711360*x^2 + 346781323848960))
  // MATLAB code to compute those coefficients:
  //    syms x;
  //    sinf = @(x) sin(x);
  //    pade_sinf = pade(sinf(x), x, 0, 'Order', 8, 'OrderMode', 'relative')
  Packet ss1_num = pmadd(ss, pset1<Packet>(4585922449), pset1<Packet>(-1066023933480));
  Packet ss2_num = pmadd(ss1_num, ss, pset1<Packet>(83284044283440));
  Packet ss3_num = pmadd(ss2_num, ss, pset1<Packet>(-2303682236856000));
  Packet ss4_num = pmadd(ss3_num, ss, pset1<Packet>(15605159573203200));
  Packet ss1_denum = pmadd(ss, pset1<Packet>(1029037), pset1<Packet>(345207016));
  Packet ss2_denum = pmadd(ss1_denum, ss, pset1<Packet>(61570292784));
  Packet ss3_denum = pmadd(ss2_denum, ss, pset1<Packet>(6603948711360));
  Packet ss4_denum = pmadd(ss3_denum, ss, pset1<Packet>(346781323848960));
  Packet ssin = pdiv(pmul(s, ss4_num), pmul(pset1<Packet>(45), ss4_denum));
 
  Packet poly_mask = preinterpret<Packet>(pcmp_eq(pand(q_int, cst_one), pzero(q_int)));
 
  Packet sign_sin = pxor(x, preinterpret<Packet>(plogical_shift_left<62>(q_int)));
  Packet sign_cos = preinterpret<Packet>(plogical_shift_left<62>(padd(q_int, cst_one)));
  Packet sign_bit, sFinalRes;
  if (ComputeBoth) {
    Packet peven = peven_mask(x);
    sign_bit = pselect((s), sign_sin, sign_cos);
    sFinalRes = pselect(pxor(peven, poly_mask), ssin, scos);
  } else {
    sign_bit = ComputeSine ? sign_sin : sign_cos;
    sFinalRes = ComputeSine ? pselect(poly_mask, ssin, scos) : pselect(poly_mask, scos, ssin);
  }
  sign_bit = pand(sign_bit, cst_sign_mask);  // clear all but left most bit
  sFinalRes = pxor(sFinalRes, sign_bit);
 
  // If the inputs values are higher than that a value that the argument reduction can currently address, compute them
  // using std::sin and std::cos
  // TODO Remove it when huge angle argument reduction is implemented
  if (EIGEN_PREDICT_FALSE(predux_any(pcmp_le(pset1<Packet>(huge_th), x_abs)))) {
    const int PacketSize = unpacket_traits<Packet>::size;
    EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) double sincos_vals[PacketSize];
    EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) double x_cpy[PacketSize];
    pstoreu(x_cpy, x);
    pstoreu(sincos_vals, sFinalRes);
    for (int k = 0; k < PacketSize; ++k) {
      double val = x_cpy[k];
      if (std::abs(val) > huge_th && (numext::isfinite)(val)) {
        if (ComputeBoth)
          sincos_vals[k] = k % 2 == 0 ? std::sin(val) : std::cos(val);
        else
          sincos_vals[k] = ComputeSine ? std::sin(val) : std::cos(val);
      }
    }
    sFinalRes = ploadu<Packet>(sincos_vals);
  }
  return sFinalRes;
}

References abs(), cos(), EIGEN_ALIGN_TO_BOUNDARY, EIGEN_PREDICT_FALSE, Eigen::numext::isfinite(), k, pabs(), padd(), pand(), pcmp_eq(), pcmp_le(), pdiv(), peven_mask(), pfloor(), pmadd(), pmul(), predux_any(), pselect(), pstoreu(), psub(), pxor(), pzero(), Eigen::numext::q, s, sin(), trig_reduce_medium_double(), trig_reduce_small_double(), calibrate::val, plotDoE::x, and x_cpy.

psincos_float()

template<bool ComputeSine, typename Packet , bool ComputeBoth = false>

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::psincos_float

(

const Packet &

_x

)

                                    {
  typedef typename unpacket_traits<Packet>::integer_packet PacketI;
 
  const Packet cst_2oPI = pset1<Packet>(0.636619746685028076171875f);  // 2/PI
  const Packet cst_rounding_magic = pset1<Packet>(12582912);           // 2^23 for rounding
  const PacketI csti_1 = pset1<PacketI>(1);
  const Packet cst_sign_mask = pset1frombits<Packet>(static_cast<Eigen::numext::uint32_t>(0x80000000u));
 
  Packet x = pabs(_x);
 
  // Scale x by 2/Pi to find x's octant.
  Packet y = pmul(x, cst_2oPI);
 
  // Rounding trick to find nearest integer:
  Packet y_round = padd(y, cst_rounding_magic);
  EIGEN_OPTIMIZATION_BARRIER(y_round)
  PacketI y_int = preinterpret<PacketI>(y_round);  // last 23 digits represent integer (if abs(x)<2^24)
  y = psub(y_round, cst_rounding_magic);           // nearest integer to x * (2/pi)
 
// Subtract y * Pi/2 to reduce x to the interval -Pi/4 <= x <= +Pi/4
// using "Extended precision modular arithmetic"
#if defined(EIGEN_VECTORIZE_FMA)
  // This version requires true FMA for high accuracy.
  // It provides a max error of 1ULP up to (with absolute_error < 5.9605e-08):
  const float huge_th = ComputeSine ? 117435.992f : 71476.0625f;
  x = pmadd(y, pset1<Packet>(-1.57079601287841796875f), x);
  x = pmadd(y, pset1<Packet>(-3.1391647326017846353352069854736328125e-07f), x);
  x = pmadd(y, pset1<Packet>(-5.390302529957764765544681040410068817436695098876953125e-15f), x);
#else
  // Without true FMA, the previous set of coefficients maintain 1ULP accuracy
  // up to x<15.7 (for sin), but accuracy is immediately lost for x>15.7.
  // We thus use one more iteration to maintain 2ULPs up to reasonably large inputs.
 
  // The following set of coefficients maintain 1ULP up to 9.43 and 14.16 for sin and cos respectively.
  // and 2 ULP up to:
  const float huge_th = ComputeSine ? 25966.f : 18838.f;
  x = pmadd(y, pset1<Packet>(-1.5703125), x);  // = 0xbfc90000
  EIGEN_OPTIMIZATION_BARRIER(x)
  x = pmadd(y, pset1<Packet>(-0.000483989715576171875), x);  // = 0xb9fdc000
  EIGEN_OPTIMIZATION_BARRIER(x)
  x = pmadd(y, pset1<Packet>(1.62865035235881805419921875e-07), x);                      // = 0x342ee000
  x = pmadd(y, pset1<Packet>(5.5644315544167710640977020375430583953857421875e-11), x);  // = 0x2e74b9ee
 
// For the record, the following set of coefficients maintain 2ULP up
// to a slightly larger range:
// const float huge_th = ComputeSine ? 51981.f : 39086.125f;
// but it slightly fails to maintain 1ULP for two values of sin below pi.
// x = pmadd(y, pset1<Packet>(-3.140625/2.), x);
// x = pmadd(y, pset1<Packet>(-0.00048351287841796875), x);
// x = pmadd(y, pset1<Packet>(-3.13855707645416259765625e-07), x);
// x = pmadd(y, pset1<Packet>(-6.0771006282767103812147979624569416046142578125e-11), x);
 
// For the record, with only 3 iterations it is possible to maintain
// 1 ULP up to 3PI (maybe more) and 2ULP up to 255.
// The coefficients are: 0xbfc90f80, 0xb7354480, 0x2e74b9ee
#endif
 
  if (predux_any(pcmp_le(pset1<Packet>(huge_th), pabs(_x)))) {
    const int PacketSize = unpacket_traits<Packet>::size;
    EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) float vals[PacketSize];
    EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) float x_cpy[PacketSize];
    EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Eigen::numext::int32_t y_int2[PacketSize];
    pstoreu(vals, pabs(_x));
    pstoreu(x_cpy, x);
    pstoreu(y_int2, y_int);
    for (int k = 0; k < PacketSize; ++k) {
      float val = vals[k];
      if (val >= huge_th && (numext::isfinite)(val)) x_cpy[k] = trig_reduce_huge(val, &y_int2[k]);
    }
    x = ploadu<Packet>(x_cpy);
    y_int = ploadu<PacketI>(y_int2);
  }
 
  // Compute the sign to apply to the polynomial.
  // sin: sign = second_bit(y_int) xor signbit(_x)
  // cos: sign = second_bit(y_int+1)
  Packet sign_bit = ComputeSine ? pxor(_x, preinterpret<Packet>(plogical_shift_left<30>(y_int)))
                                : preinterpret<Packet>(plogical_shift_left<30>(padd(y_int, csti_1)));
  sign_bit = pand(sign_bit, cst_sign_mask);  // clear all but left most bit
 
  // Get the polynomial selection mask from the second bit of y_int
  // We'll calculate both (sin and cos) polynomials and then select from the two.
  Packet poly_mask = preinterpret<Packet>(pcmp_eq(pand(y_int, csti_1), pzero(y_int)));
 
  Packet x2 = pmul(x, x);
 
  // Evaluate the cos(x) polynomial. (-Pi/4 <= x <= Pi/4)
  Packet y1 = pset1<Packet>(2.4372266125283204019069671630859375e-05f);
  y1 = pmadd(y1, x2, pset1<Packet>(-0.00138865201734006404876708984375f));
  y1 = pmadd(y1, x2, pset1<Packet>(0.041666619479656219482421875f));
  y1 = pmadd(y1, x2, pset1<Packet>(-0.5f));
  y1 = pmadd(y1, x2, pset1<Packet>(1.f));
 
  // Evaluate the sin(x) polynomial. (Pi/4 <= x <= Pi/4)
  // octave/matlab code to compute those coefficients:
  //    x = (0:0.0001:pi/4)';
  //    A = [x.^3 x.^5 x.^7];
  //    w = ((1.-(x/(pi/4)).^2).^5)*2000+1;         # weights trading relative accuracy
  //    c = (A'*diag(w)*A)\‍(A'*diag(w)*(sin(x)-x)); # weighted LS, linear coeff forced to 1
  //    printf('%.64f\n %.64f\n%.64f\n', c(3), c(2), c(1))
  //
  Packet y2 = pset1<Packet>(-0.0001959234114083702898469196984621021329076029360294342041015625f);
  y2 = pmadd(y2, x2, pset1<Packet>(0.0083326873655616851693794799871284340042620897293090820312500000f));
  y2 = pmadd(y2, x2, pset1<Packet>(-0.1666666203982298255503735617821803316473960876464843750000000000f));
  y2 = pmul(y2, x2);
  y2 = pmadd(y2, x, x);
 
  // Select the correct result from the two polynomials.
  if (ComputeBoth) {
    Packet peven = peven_mask(x);
    Packet ysin = pselect(poly_mask, y2, y1);
    Packet ycos = pselect(poly_mask, y1, y2);
    Packet sign_bit_sin = pxor(_x, preinterpret<Packet>(plogical_shift_left<30>(y_int)));
    Packet sign_bit_cos = preinterpret<Packet>(plogical_shift_left<30>(padd(y_int, csti_1)));
    sign_bit_sin = pand(sign_bit_sin, cst_sign_mask);  // clear all but left most bit
    sign_bit_cos = pand(sign_bit_cos, cst_sign_mask);  // clear all but left most bit
    y = pselect(peven, pxor(ysin, sign_bit_sin), pxor(ycos, sign_bit_cos));
  } else {
    y = ComputeSine ? pselect(poly_mask, y2, y1) : pselect(poly_mask, y1, y2);
    y = pxor(y, sign_bit);
  }
  // Update the sign and filter huge inputs
  return y;
}

References EIGEN_ALIGN_TO_BOUNDARY, EIGEN_OPTIMIZATION_BARRIER, Eigen::numext::isfinite(), k, pabs(), padd(), pand(), pcmp_eq(), pcmp_le(), peven_mask(), pmadd(), pmul(), predux_any(), pselect(), pstoreu(), psub(), pxor(), pzero(), trig_reduce_huge(), calibrate::val, plotDoE::x, Global_parameters::x2(), x_cpy, and y.

psincos_inner_msa_float()

template<bool sine>

Packet4f Eigen::internal::psincos_inner_msa_float

(

const Packet4f &

_x

)

                                                     {
  static EIGEN_DECLARE_CONST_Packet4f(sincos_max_arg, 13176795.0f);  // Approx. (2**24) / (4/Pi).
  static EIGEN_DECLARE_CONST_Packet4f(minus_cephes_DP1, -0.78515625f);
  static EIGEN_DECLARE_CONST_Packet4f(minus_cephes_DP2, -2.4187564849853515625e-4f);
  static EIGEN_DECLARE_CONST_Packet4f(minus_cephes_DP3, -3.77489497744594108e-8f);
  static EIGEN_DECLARE_CONST_Packet4f(sincof_p0, -1.9515295891e-4f);
  static EIGEN_DECLARE_CONST_Packet4f(sincof_p1, 8.3321608736e-3f);
  static EIGEN_DECLARE_CONST_Packet4f(sincof_p2, -1.6666654611e-1f);
  static EIGEN_DECLARE_CONST_Packet4f(coscof_p0, 2.443315711809948e-5f);
  static EIGEN_DECLARE_CONST_Packet4f(coscof_p1, -1.388731625493765e-3f);
  static EIGEN_DECLARE_CONST_Packet4f(coscof_p2, 4.166664568298827e-2f);
  static EIGEN_DECLARE_CONST_Packet4f(cephes_FOPI, 1.27323954473516f);  // 4/Pi.
  static EIGEN_DECLARE_CONST_Packet4f(half, 0.5f);
  static EIGEN_DECLARE_CONST_Packet4f(1, 1.0f);
 
  Packet4f x = pabs(_x);
 
  // Translate infinite arguments into NANs.
  Packet4f zero_or_nan_if_inf = psub(_x, _x);
  x = padd(x, zero_or_nan_if_inf);
  // Prevent sin/cos from generating values larger than 1.0 in magnitude
  // for very large arguments by setting x to 0.0.
  Packet4i small_or_nan_mask = __builtin_msa_fcult_w(x, p4f_sincos_max_arg);
  x = pand(x, (Packet4f)small_or_nan_mask);
 
  // Scale x by 4/Pi to find x's octant.
  Packet4f y = pmul(x, p4f_cephes_FOPI);
  // Get the octant. We'll reduce x by this number of octants or by one more than it.
  Packet4i y_int = __builtin_msa_ftrunc_s_w(y);
  // x's from even-numbered octants will translate to octant 0: [0, +Pi/4].
  // x's from odd-numbered octants will translate to octant -1: [-Pi/4, 0].
  // Adjustment for odd-numbered octants: octant = (octant + 1) & (~1).
  Packet4i y_int1 = __builtin_msa_addvi_w(y_int, 1);
  Packet4i y_int2 = (Packet4i)__builtin_msa_bclri_w((Packet4ui)y_int1, 0);  // bclri = bit-clear
  y = __builtin_msa_ffint_s_w(y_int2);
 
  // Compute the sign to apply to the polynomial.
  Packet4i sign_mask = sine ? pxor(__builtin_msa_slli_w(y_int1, 29), (Packet4i)_x)
                            : __builtin_msa_slli_w(__builtin_msa_addvi_w(y_int, 3), 29);
 
  // Get the polynomial selection mask.
  // We'll calculate both (sin and cos) polynomials and then select from the two.
  Packet4i poly_mask = __builtin_msa_ceqi_w(__builtin_msa_slli_w(y_int2, 30), 0);
 
  // Reduce x by y octants to get: -Pi/4 <= x <= +Pi/4.
  // The magic pass: "Extended precision modular arithmetic"
  // x = ((x - y * DP1) - y * DP2) - y * DP3
  Packet4f tmp1 = pmul(y, p4f_minus_cephes_DP1);
  Packet4f tmp2 = pmul(y, p4f_minus_cephes_DP2);
  Packet4f tmp3 = pmul(y, p4f_minus_cephes_DP3);
  x = padd(x, tmp1);
  x = padd(x, tmp2);
  x = padd(x, tmp3);
 
  // Evaluate the cos(x) polynomial.
  y = p4f_coscof_p0;
  Packet4f z = pmul(x, x);
  y = pmadd(y, z, p4f_coscof_p1);
  y = pmadd(y, z, p4f_coscof_p2);
  y = pmul(y, z);
  y = pmul(y, z);
  y = __builtin_msa_fmsub_w(y, z, p4f_half);
  y = padd(y, p4f_1);
 
  // Evaluate the sin(x) polynomial.
  Packet4f y2 = p4f_sincof_p0;
  y2 = pmadd(y2, z, p4f_sincof_p1);
  y2 = pmadd(y2, z, p4f_sincof_p2);
  y2 = pmul(y2, z);
  y2 = pmadd(y2, x, x);
 
  // Select the correct result from the two polynomials.
  y = sine ? (Packet4f)__builtin_msa_bsel_v((v16u8)poly_mask, (v16u8)y, (v16u8)y2)
           : (Packet4f)__builtin_msa_bsel_v((v16u8)poly_mask, (v16u8)y2, (v16u8)y);
 
  // Update the sign.
  sign_mask = pxor(sign_mask, (Packet4i)y);
  y = (Packet4f)__builtin_msa_binsli_w((v4u32)y, (v4u32)sign_mask, 0);  // binsli = bit-insert-left
  return y;
}

References EIGEN_DECLARE_CONST_Packet4f(), pabs(), padd(), pand(), pmadd(), pmul(), psub(), pxor(), plotDoE::x, and y.

Referenced by pcos< Packet4f >(), and psin< Packet4f >().

psinh()

template<typename Packet >

EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::psinh

(

const Packet &

a

)

Returns

the hyperbolic sine of a (coeff-wise)

                                                                                   {
  EIGEN_USING_STD(sinh);
  return sinh(a);
}

References a, EIGEN_USING_STD, and Eigen::bfloat16_impl::sinh().

Referenced by Eigen::internal::scalar_sinh_op< Scalar >::packetOp().

psqrt() [1/11]

template<typename Packet >

EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::psqrt

(

const Packet &

a

)

Returns

the square-root of a (coeff-wise)

                                                                                   {
  return numext::sqrt(a);
}

References a, and Eigen::numext::sqrt().

psqrt() [2/11]

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::psqrt

(

const Packet16uc &

a

)

Computes the integer square root

Remarks

The calculation is performed using an algorithm which iterates through each binary digit of the result and tests whether setting that digit to 1 would cause the square of the value to be greater than the argument value. The algorithm is described in detail here: http://ww1.microchip.com/downloads/en/AppNotes/91040a.pdf .

                                                          {
  __m128i res = {0, 0};
  __m128i add = {0x0808080808080808, 0x0808080808080808};
  for (int i = 0; i < 4; i++) {
    const __m128i temp = __lsx_vor_v(res, add);
    const __m128i tmul = __lsx_vpackev_b(__lsx_vmulwod_h_bu(temp, temp), __lsx_vmulwev_h_bu(temp, temp));
    res = __lsx_vbitsel_v(res, temp, __lsx_vsle_bu(tmul, a));
    add = __lsx_vsrli_b(add, 1);
  }
  return res;
}

References a, i, and res.

psqrt() [3/11]

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::psqrt

(

const Packet2d &

a

)

                                                      {
  return __lsx_vfsqrt_d(a);
}

References a.

psqrt() [4/11]

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::psqrt

(

const Packet2f &

a

)

                                                      {
  return psqrt_float_common(a);
}

References a, and psqrt_float_common().

psqrt() [5/11]

template<>

EIGEN_STRONG_INLINE Packet2ui Eigen::internal::psqrt

(

const Packet2ui &

a

)

Computes the integer square root

Remarks

The calculation is performed using an algorithm which iterates through each binary digit of the result and tests whether setting that digit to 1 would cause the square of the value to be greater than the argument value. The algorithm is described in detail here: http://ww1.microchip.com/downloads/en/AppNotes/91040a.pdf .

                                                        {
  uint32x2_t res = vdup_n_u32(0);
  uint32x2_t add = vdup_n_u32(0x8000);
  for (int i = 0; i < 16; i++) {
    const uint32x2_t temp = vorr_u32(res, add);
    res = vbsl_u32(vcge_u32(a, vmul_u32(temp, temp)), temp, res);
    add = vshr_n_u32(add, 1);
  }
  return res;
}

References a, i, and res.

psqrt() [6/11]

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::psqrt

(

const Packet4f &

a

)

                                                      {
  return __lsx_vfsqrt_s(a);
}

References a.

Referenced by generic_ndtri_lt_exp_neg_two(), packetmath(), packetmath_complex(), Eigen::internal::scalar_sqrt_op< Scalar >::packetOp(), pacos_float(), pasin_float(), phypot_complex(), prsqrt(), psqrt_complex(), and Eigen::internal::psign_impl< Packet, std::enable_if_t< NumTraits< typename unpacket_traits< Packet >::type >::IsComplex &&unpacket_traits< Packet >::vectorizable > >::run().

psqrt() [7/11]

template<>

EIGEN_STRONG_INLINE Packet4uc Eigen::internal::psqrt

(

const Packet4uc &

a

)

Computes the integer square root

Remarks

The calculation is performed using an algorithm which iterates through each binary digit of the result and tests whether setting that digit to 1 would cause the square of the value to be greater than the argument value. The algorithm is described in detail here: http://ww1.microchip.com/downloads/en/AppNotes/91040a.pdf .

                                                        {
  uint8x8_t x = vreinterpret_u8_u32(vdup_n_u32(a));
  uint8x8_t res = vdup_n_u8(0);
  uint8x8_t add = vdup_n_u8(0x8);
  for (int i = 0; i < 4; i++) {
    const uint8x8_t temp = vorr_u8(res, add);
    res = vbsl_u8(vcge_u8(x, vmul_u8(temp, temp)), temp, res);
    add = vshr_n_u8(add, 1);
  }
  return vget_lane_u32(vreinterpret_u32_u8(res), 0);
}

References a, i, res, and plotDoE::x.

psqrt() [8/11]

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::psqrt

(

const Packet4ui &

a

)

Computes the integer square root

Remarks

The calculation is performed using an algorithm which iterates through each binary digit of the result and tests whether setting that digit to 1 would cause the square of the value to be greater than the argument value. The algorithm is described in detail here: http://ww1.microchip.com/downloads/en/AppNotes/91040a.pdf .

                                                        {
  __m128i res = {0, 0};
  __m128i add = {0x0000800000008000, 0x0000800000008000};
  for (int i = 0; i < 4; i++) {
    const __m128i temp = __lsx_vor_v(res, add);
    const __m128i tmul = __lsx_vpackev_w(__lsx_vmulwod_d_wu(temp, temp), __lsx_vmulwev_d_wu(temp, temp));
    res = __lsx_vbitsel_v(res, temp, __lsx_vsle_wu(tmul, a));
    add = __lsx_vsrli_w(add, 1);
  }
  return res;
}

References a, i, and res.

psqrt() [9/11]

template<>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::psqrt

(

const Packet4us &

a

)

Computes the integer square root

Remarks

The calculation is performed using an algorithm which iterates through each binary digit of the result and tests whether setting that digit to 1 would cause the square of the value to be greater than the argument value. The algorithm is described in detail here: http://ww1.microchip.com/downloads/en/AppNotes/91040a.pdf .

                                                        {
  uint16x4_t res = vdup_n_u16(0);
  uint16x4_t add = vdup_n_u16(0x80);
  for (int i = 0; i < 8; i++) {
    const uint16x4_t temp = vorr_u16(res, add);
    res = vbsl_u16(vcge_u16(a, vmul_u16(temp, temp)), temp, res);
    add = vshr_n_u16(add, 1);
  }
  return res;
}

References a, i, and res.

psqrt() [10/11]

template<>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::psqrt

(

const Packet8uc &

a

)

Computes the integer square root

Remarks

The calculation is performed using an algorithm which iterates through each binary digit of the result and tests whether setting that digit to 1 would cause the square of the value to be greater than the argument value. The algorithm is described in detail here: http://ww1.microchip.com/downloads/en/AppNotes/91040a.pdf .

                                                        {
  uint8x8_t res = vdup_n_u8(0);
  uint8x8_t add = vdup_n_u8(0x8);
  for (int i = 0; i < 4; i++) {
    const uint8x8_t temp = vorr_u8(res, add);
    res = vbsl_u8(vcge_u8(a, vmul_u8(temp, temp)), temp, res);
    add = vshr_n_u8(add, 1);
  }
  return res;
}

References a, i, and res.

psqrt() [11/11]

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::psqrt

(

const Packet8us &

a

)

Computes the integer square root

Remarks

The calculation is performed using an algorithm which iterates through each binary digit of the result and tests whether setting that digit to 1 would cause the square of the value to be greater than the argument value. The algorithm is described in detail here: http://ww1.microchip.com/downloads/en/AppNotes/91040a.pdf .

                                                        {
  __m128i res = {0, 0};
  __m128i add = {0x0080008000800080, 0x0080008000800080};
  for (int i = 0; i < 4; i++) {
    const __m128i temp = __lsx_vor_v(res, add);
    const __m128i tmul = __lsx_vpackev_h(__lsx_vmulwod_w_hu(temp, temp), __lsx_vmulwev_w_hu(temp, temp));
    res = __lsx_vbitsel_v(res, temp, __lsx_vsle_hu(tmul, a));
    add = __lsx_vsrli_h(add, 1);
  }
  return res;
}

References a, i, and res.

psqrt< Packet16b >()

template<>

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet16b Eigen::internal::psqrt< Packet16b >

(

const Packet16b &

x

)

                                                                                                   {
  return x;
}

References plotDoE::x.

psqrt< Packet16f >()

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::psqrt< Packet16f >

(

const Packet16f &

x

)

                                                                   {
  return _mm512_sqrt_ps(x);
}

References plotDoE::x.

psqrt< Packet1cd >()

template<>

EIGEN_STRONG_INLINE Packet1cd Eigen::internal::psqrt< Packet1cd >

(

const Packet1cd &

a

)

                                                                   {
  return psqrt_complex<Packet1cd>(a);
}

References a.

psqrt< Packet1cf >()

template<>

EIGEN_STRONG_INLINE Packet1cf Eigen::internal::psqrt< Packet1cf >

(

const Packet1cf &

a

)

                                                                   {
  return psqrt_complex<Packet1cf>(a);
}

References a.

psqrt< Packet2cd >()

template<>

EIGEN_STRONG_INLINE Packet2cd Eigen::internal::psqrt< Packet2cd >

(

const Packet2cd &

a

)

                                                                   {
  return psqrt_complex<Packet2cd>(a);
}

References a.

psqrt< Packet2cf >()

template<>

EIGEN_STRONG_INLINE Packet2cf Eigen::internal::psqrt< Packet2cf >

(

const Packet2cf &

a

)

                                                                   {
  return psqrt_complex<Packet2cf>(a);
}

References a.

psqrt< Packet2d >()

template<>

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet2d Eigen::internal::psqrt< Packet2d >

(

const Packet2d &

x

)

                                                                                                {
  return _mm_sqrt_pd(x);
}

References plotDoE::x.

Referenced by prsqrt< Packet2d >().

psqrt< Packet32h >()

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::psqrt< Packet32h >

(

const Packet32h &

a

)

                                                                   {
  return _mm512_sqrt_ph(a);
}

References a.

psqrt< Packet4cd >()

template<>

EIGEN_STRONG_INLINE Packet4cd Eigen::internal::psqrt< Packet4cd >

(

const Packet4cd &

a

)

                                                                   {
  return psqrt_complex<Packet4cd>(a);
}

References a.

psqrt< Packet4cf >()

template<>

EIGEN_STRONG_INLINE Packet4cf Eigen::internal::psqrt< Packet4cf >

(

const Packet4cf &

a

)

                                                                   {
  return psqrt_complex<Packet4cf>(a);
}

References a.

psqrt< Packet4d >()

template<>

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet4d Eigen::internal::psqrt< Packet4d >

(

const Packet4d &

_x

)

                                                                                                 {
  return _mm256_sqrt_pd(_x);
}

psqrt< Packet4f >()

template<>

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet4f Eigen::internal::psqrt< Packet4f >

(

const Packet4f &

x

)

                                                                                                {
  Packet4f a;
  for (Index i = 0; i < packet_traits<float>::size; i++) {
    a[i] = numext::sqrt(x[i]);
  }
  return a;
}

References a, i, size, Eigen::numext::sqrt(), and plotDoE::x.

Referenced by prsqrt< Packet4f >().

psqrt< Packet8cf >()

template<>

EIGEN_STRONG_INLINE Packet8cf Eigen::internal::psqrt< Packet8cf >

(

const Packet8cf &

a

)

                                                                   {
  return psqrt_complex<Packet8cf>(a);
}

References a.

psqrt< Packet8d >()

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::psqrt< Packet8d >

(

const Packet8d &

x

)

                                                                {
  return _mm512_sqrt_pd(x);
}

References plotDoE::x.

psqrt< Packet8f >()

template<>

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet8f Eigen::internal::psqrt< Packet8f >

(

const Packet8f &

_x

)

                                                                                                 {
  return _mm256_sqrt_ps(_x);
}

psqrt< PacketXf >()

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::psqrt< PacketXf >

(

const PacketXf &

a

)

                                                                {
  return svsqrt_f32_x(svptrue_b32(), a);
}

References a.

psqrt_complex()

template<typename Packet >

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::psqrt_complex

(

const Packet &

a

)

Returns

sqrt(x) for complex types

                                                                                          {
  typedef typename unpacket_traits<Packet>::type Scalar;
  typedef typename Scalar::value_type RealScalar;
  typedef typename unpacket_traits<Packet>::as_real RealPacket;
 
  // Computes the principal sqrt of the complex numbers in the input.
  //
  // For example, for packets containing 2 complex numbers stored in interleaved format
  //    a = [a0, a1] = [x0, y0, x1, y1],
  // where x0 = real(a0), y0 = imag(a0) etc., this function returns
  //    b = [b0, b1] = [u0, v0, u1, v1],
  // such that b0^2 = a0, b1^2 = a1.
  //
  // To derive the formula for the complex square roots, let's consider the equation for
  // a single complex square root of the number x + i*y. We want to find real numbers
  // u and v such that
  //    (u + i*v)^2 = x + i*y  <=>
  //    u^2 - v^2 + i*2*u*v = x + i*v.
  // By equating the real and imaginary parts we get:
  //    u^2 - v^2 = x
  //    2*u*v = y.
  //
  // For x >= 0, this has the numerically stable solution
  //    u = sqrt(0.5 * (x + sqrt(x^2 + y^2)))
  //    v = 0.5 * (y / u)
  // and for x < 0,
  //    v = sign(y) * sqrt(0.5 * (-x + sqrt(x^2 + y^2)))
  //    u = 0.5 * (y / v)
  //
  //  To avoid unnecessary over- and underflow, we compute sqrt(x^2 + y^2) as
  //     l = max(|x|, |y|) * sqrt(1 + (min(|x|, |y|) / max(|x|, |y|))^2) ,
 
  // In the following, without lack of generality, we have annotated the code, assuming
  // that the input is a packet of 2 complex numbers.
  //
  // Step 1. Compute l = [l0, l0, l1, l1], where
  //    l0 = sqrt(x0^2 + y0^2),  l1 = sqrt(x1^2 + y1^2)
  // To avoid over- and underflow, we use the stable formula for each hypotenuse
  //    l0 = (min0 == 0 ? max0 : max0 * sqrt(1 + (min0/max0)**2)),
  // where max0 = max(|x0|, |y0|), min0 = min(|x0|, |y0|), and similarly for l1.
 
  RealPacket a_abs = pabs(a.v);                        // [|x0|, |y0|, |x1|, |y1|]
  RealPacket a_abs_flip = pcplxflip(Packet(a_abs)).v;  // [|y0|, |x0|, |y1|, |x1|]
  RealPacket a_max = pmax(a_abs, a_abs_flip);
  RealPacket a_min = pmin(a_abs, a_abs_flip);
  RealPacket a_min_zero_mask = pcmp_eq(a_min, pzero(a_min));
  RealPacket a_max_zero_mask = pcmp_eq(a_max, pzero(a_max));
  RealPacket r = pdiv(a_min, a_max);
  const RealPacket cst_one = pset1<RealPacket>(RealScalar(1));
  RealPacket l = pmul(a_max, psqrt(padd(cst_one, pmul(r, r))));  // [l0, l0, l1, l1]
  // Set l to a_max if a_min is zero.
  l = pselect(a_min_zero_mask, a_max, l);
 
  // Step 2. Compute [rho0, *, rho1, *], where
  // rho0 = sqrt(0.5 * (l0 + |x0|)), rho1 =  sqrt(0.5 * (l1 + |x1|))
  // We don't care about the imaginary parts computed here. They will be overwritten later.
  const RealPacket cst_half = pset1<RealPacket>(RealScalar(0.5));
  Packet rho;
  rho.v = psqrt(pmul(cst_half, padd(a_abs, l)));
 
  // Step 3. Compute [rho0, eta0, rho1, eta1], where
  // eta0 = (y0 / l0) / 2, and eta1 = (y1 / l1) / 2.
  // set eta = 0 of input is 0 + i0.
  RealPacket eta = pandnot(pmul(cst_half, pdiv(a.v, pcplxflip(rho).v)), a_max_zero_mask);
  RealPacket real_mask = peven_mask(a.v);
  Packet positive_real_result;
  // Compute result for inputs with positive real part.
  positive_real_result.v = pselect(real_mask, rho.v, eta);
 
  // Step 4. Compute solution for inputs with negative real part:
  //         [|eta0|, sign(y0)*rho0, |eta1|, sign(y1)*rho1]
  const RealPacket cst_imag_sign_mask = pset1<Packet>(Scalar(RealScalar(0.0), RealScalar(-0.0))).v;
  RealPacket imag_signs = pand(a.v, cst_imag_sign_mask);
  Packet negative_real_result;
  // Notice that rho is positive, so taking it's absolute value is a noop.
  negative_real_result.v = por(pabs(pcplxflip(positive_real_result).v), imag_signs);
 
  // Step 5. Select solution branch based on the sign of the real parts.
  Packet negative_real_mask;
  negative_real_mask.v = pcmp_lt(pand(real_mask, a.v), pzero(a.v));
  negative_real_mask.v = por(negative_real_mask.v, pcplxflip(negative_real_mask).v);
  Packet result = pselect(negative_real_mask, negative_real_result, positive_real_result);
 
  // Step 6. Handle special cases for infinities:
  // * If z is (x,+∞), the result is (+∞,+∞) even if x is NaN
  // * If z is (x,-∞), the result is (+∞,-∞) even if x is NaN
  // * If z is (-∞,y), the result is (0*|y|,+∞) for finite or NaN y
  // * If z is (+∞,y), the result is (+∞,0*|y|) for finite or NaN y
  const RealPacket cst_pos_inf = pset1<RealPacket>(NumTraits<RealScalar>::infinity());
  Packet is_inf;
  is_inf.v = pcmp_eq(a_abs, cst_pos_inf);
  Packet is_real_inf;
  is_real_inf.v = pand(is_inf.v, real_mask);
  is_real_inf = por(is_real_inf, pcplxflip(is_real_inf));
  // prepare packet of (+∞,0*|y|) or (0*|y|,+∞), depending on the sign of the infinite real part.
  Packet real_inf_result;
  real_inf_result.v = pmul(a_abs, pset1<Packet>(Scalar(RealScalar(1.0), RealScalar(0.0))).v);
  real_inf_result.v = pselect(negative_real_mask.v, pcplxflip(real_inf_result).v, real_inf_result.v);
  // prepare packet of (+∞,+∞) or (+∞,-∞), depending on the sign of the infinite imaginary part.
  Packet is_imag_inf;
  is_imag_inf.v = pandnot(is_inf.v, real_mask);
  is_imag_inf = por(is_imag_inf, pcplxflip(is_imag_inf));
  Packet imag_inf_result;
  imag_inf_result.v = por(pand(cst_pos_inf, real_mask), pandnot(a.v, real_mask));
  // unless otherwise specified, if either the real or imaginary component is nan, the entire result is nan
  Packet result_is_nan = pisnan(result);
  result = por(result_is_nan, result);
 
  return pselect(is_imag_inf, imag_inf_result, pselect(is_real_inf, real_inf_result, result));
}

References a, TestSoln::eta, pabs(), padd(), pand(), pandnot(), pcmp_eq(), pcmp_lt(), pcplxflip(), pdiv(), peven_mask(), pisnan(), pmax(), pmin(), pmul(), por(), pselect(), psqrt(), pzero(), UniformPSDSelfTest::r, v, and Eigen::internal::Packet2cf::v.

psqrt_float_common()

template<typename Packet >

EIGEN_STRONG_INLINE Packet Eigen::internal::psqrt_float_common

(

const Packet &

a

)

                                                               {
  const Packet cst_zero = pzero(a);
  const Packet cst_inf = pset1<Packet>(NumTraits<float>::infinity());
 
  Packet result = pmul(a, prsqrt_float_unsafe(a));
  Packet a_is_zero = pcmp_eq(a, cst_zero);
  Packet a_is_inf = pcmp_eq(a, cst_inf);
  Packet return_a = por(a_is_zero, a_is_inf);
 
  result = pselect(return_a, a, result);
  return result;
}

References a, pcmp_eq(), pmul(), por(), prsqrt_float_unsafe(), pselect(), and pzero().

Referenced by psqrt().

pstore()

template<typename Scalar , typename Packet >

EIGEN_DEVICE_FUNC void Eigen::internal::pstore ( Scalar *  to, const Packet &  from  )

inline

copy the packet from to *to, to must be properly aligned

                                                                     {
  (*to) = from;
}

Referenced by benchVec(), Eigen::internal::gemm_pack_lhs< Scalar, Index, DataMapper, Pack1, Pack2, Packet, ColMajor, Conjugate, PanelMode >::operator()(), Eigen::internal::gemm_pack_lhs< Scalar, Index, DataMapper, Pack1, Pack2, Packet, RowMajor, Conjugate, PanelMode >::operator()(), packetmath(), packetmath_complex(), packetmath_notcomplex(), packetmath_scatter_gather(), pcast< Packet2l, Packet2f >(), pcast< Packet2ul, Packet2f >(), pcast< Packet8l, Packet8d >(), pfirst< Packet2cf >(), predux_mul< Packet4i >(), pscatter(), pscatter< bfloat16, Packet16bf >(), pscatter< Eigen::half, Packet8h >(), pscatter< half, Packet16h >(), pscatter< half, Packet32h >(), pscatter< std::complex< double >, Packet1cd >(), pstore1(), pstore1< Packet16f >(), pstore1< Packet16i >(), pstore1< Packet2d >(), pstore1< Packet4d >(), pstore1< Packet4f >(), pstore1< Packet8d >(), pstore1< Packet8f >(), pstore1< Packet8i >(), pstore1< Packet8l >(), pstore< std::complex< double > >(), pstore< std::complex< float > >(), pstore_partial_common(), pstoret(), pstoreu_partial_common(), Eigen::internal::selfadjoint_matrix_vector_product< Scalar, Index, StorageOrder, UpLo, ConjugateLhs, ConjugateRhs, Version >::run(), Eigen::internal::apply_rotation_in_the_plane_selector< Scalar, OtherScalar, SizeAtCompileTime, MinAlignment, true >::run(), Eigen::test::packet_helper< Cond, Packet >::store(), and test_conj_helper().

pstore1()

template<typename Packet >

void Eigen::internal::pstore1 ( typename unpacket_traits< Packet >::type *  to, const typename unpacket_traits< Packet >::type &  a  )

inline

copy a packet with constant coefficient a (e.g., [a,a,a,a]) to *to. to must be 16 bytes aligned

                                                                                                             {
  pstore(to, pset1<Packet>(a));
}

References a, and pstore().

pstore1< Packet16f >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore1< Packet16f >	(	float *	to,
		const float &	a
	)

                                                                       {
  Packet16f pa = pset1<Packet16f>(a);
  pstore(to, pa);
}

References a, pa, pset1< Packet16f >(), and pstore().

pstore1< Packet16i >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore1< Packet16i >	(	int *	to,
		const int &	a
	)

                                                                   {
  Packet16i pa = pset1<Packet16i>(a);
  pstore(to, pa);
}

References a, pa, pset1< Packet16i >(), and pstore().

pstore1< Packet2d >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore1< Packet2d >	(	double *	to,
		const double &	a
	)

                                                                        {
  Packet2d pa = _mm_set_sd(a);
  pstore(to, Packet2d(vec2d_swizzle1(pa, 0, 0)));
}

References a, pa, pstore(), and vec2d_swizzle1.

pstore1< Packet4d >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore1< Packet4d >	(	double *	to,
		const double &	a
	)

                                                                        {
  Packet4d pa = pset1<Packet4d>(a);
  pstore(to, pa);
}

References a, pa, pset1< Packet4d >(), and pstore().

pstore1< Packet4f >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore1< Packet4f >	(	float *	to,
		const float &	a
	)

                                                                      {
  Packet4f pa = _mm_set_ss(a);
  pstore(to, Packet4f(vec4f_swizzle1(pa, 0, 0, 0, 0)));
}

References a, pa, pstore(), and vec4f_swizzle1().

pstore1< Packet8d >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore1< Packet8d >	(	double *	to,
		const double &	a
	)

                                                                        {
  Packet8d pa = pset1<Packet8d>(a);
  pstore(to, pa);
}

References a, pa, pset1< Packet8d >(), and pstore().

pstore1< Packet8f >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore1< Packet8f >	(	float *	to,
		const float &	a
	)

                                                                      {
  Packet8f pa = pset1<Packet8f>(a);
  pstore(to, pa);
}

References a, pa, pset1< Packet8f >(), and pstore().

pstore1< Packet8i >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore1< Packet8i >	(	int *	to,
		const int &	a
	)

                                                                  {
  Packet8i pa = pset1<Packet8i>(a);
  pstore(to, pa);
}

References a, pa, pset1< Packet8i >(), and pstore().

pstore1< Packet8l >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore1< Packet8l >	(	int64_t *	to,
		const int64_t &	a
	)

                                                                          {
  Packet8l pa = pset1<Packet8l>(a);
  pstore(to, pa);
}

References a, pa, pset1< Packet8l >(), and pstore().

pstore< bfloat16 >() [1/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< bfloat16 >	(	bfloat16 *	to,
		const Packet16bf &	from
	)

                                                                                {
  _mm256_store_si256(reinterpret_cast<__m256i*>(to), from);
}

pstore< bfloat16 >() [2/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< bfloat16 >	(	bfloat16 *	to,
		const Packet4bf &	from
	)

                                                                               {
  EIGEN_DEBUG_ALIGNED_STORE vst1_u16(reinterpret_cast<uint16_t*>(to), from);
}

References EIGEN_DEBUG_ALIGNED_STORE.

pstore< bfloat16 >() [3/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< bfloat16 >	(	bfloat16 *	to,
		const Packet8bf &	from
	)

                                                                               {
  pstore_common<Packet8us>(reinterpret_cast<unsigned short int*>(to), from.m_val);
}

References Eigen::internal::eigen_packet_wrapper< T, unique_id >::m_val.

Referenced by Eigen::internal::dhs_pack< bfloat16, DataMapper, Packet8bf, StorageOrder, PanelMode, true >::operator()(), and Eigen::internal::dhs_pack< bfloat16, DataMapper, Packet8bf, StorageOrder, PanelMode, false >::operator()().

pstore< bool >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< bool >	(	bool *	to,
		const Packet16b &	from
	)

                                                                       {
  EIGEN_DEBUG_ALIGNED_STORE _mm_store_si128(reinterpret_cast<__m128i*>(to), from);
}

References EIGEN_DEBUG_ALIGNED_STORE.

pstore< double >() [1/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< double >	(	double *	to,
		const Packet2d &	from
	)

                                                                          {
  EIGEN_DEBUG_ALIGNED_STORE __lsx_vst(from, to, 0);
}

References EIGEN_DEBUG_ALIGNED_STORE.

pstore< double >() [2/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< double >	(	double *	to,
		const Packet4d &	from
	)

                                                                          {
  EIGEN_DEBUG_ALIGNED_STORE _mm256_store_pd(to, from);
}

References EIGEN_DEBUG_ALIGNED_STORE.

Referenced by Eigen::internal::dhs_pack< double, DataMapper, Packet2d, StorageOrder, PanelMode, false >::dhs_copy(), Eigen::internal::dhs_pack< double, DataMapper, Packet2d, StorageOrder, PanelMode, true >::operator()(), Eigen::internal::dhs_pack< double, DataMapper, Packet2d, StorageOrder, PanelMode, false >::operator()(), and Eigen::internal::dhs_cpack< double, DataMapper, Packet, PacketC, StorageOrder, Conjugate, PanelMode, true >::operator()().

pstore< double >() [3/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< double >	(	double *	to,
		const Packet8d &	from
	)

                                                                          {
  EIGEN_DEBUG_ALIGNED_STORE _mm512_store_pd(to, from);
}

References EIGEN_DEBUG_ALIGNED_STORE.

pstore< Eigen::half >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< Eigen::half >	(	Eigen::half *	to,
		const Packet8h &	from
	)

                                                                                {
  _mm_store_si128(reinterpret_cast<__m128i*>(to), from);
}

pstore< float >() [1/5]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< float >	(	float *	to,
		const Packet16f &	from
	)

                                                                         {
  EIGEN_DEBUG_ALIGNED_STORE _mm512_store_ps(to, from);
}

References EIGEN_DEBUG_ALIGNED_STORE.

pstore< float >() [2/5]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< float >	(	float *	to,
		const Packet2f &	from
	)

                                                                        {
  EIGEN_DEBUG_ALIGNED_STORE vst1_f32(to, from);
}

References EIGEN_DEBUG_ALIGNED_STORE.

pstore< float >() [3/5]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< float >	(	float *	to,
		const Packet4f &	from
	)

                                                                        {
  pstore_common<Packet4f>(to, from);
}

pstore< float >() [4/5]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< float >	(	float *	to,
		const Packet8f &	from
	)

                                                                        {
  EIGEN_DEBUG_ALIGNED_STORE _mm256_store_ps(to, from);
}

References EIGEN_DEBUG_ALIGNED_STORE.

pstore< float >() [5/5]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< float >	(	float *	to,
		const PacketXf &	from
	)

                                                                        {
  EIGEN_DEBUG_ALIGNED_STORE svst1_f32(svptrue_b32(), to, from);
}

References EIGEN_DEBUG_ALIGNED_STORE.

pstore< half >() [1/2]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< half >	(	Eigen::half *	to,
		const Packet16h &	from
	)

                                                                            {
  // (void*) -> workaround clang warning:
  // cast from 'Eigen::half *' to '__m256i *' increases required alignment from 2 to 32
  _mm256_store_si256((__m256i*)(void*)to, from);
}

Referenced by ptranspose().

pstore< half >() [2/2]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< half >	(	Eigen::half *	to,
		const Packet32h &	from
	)

                                                                            {
  EIGEN_DEBUG_ALIGNED_STORE _mm512_store_ph(to, from);
}

References EIGEN_DEBUG_ALIGNED_STORE.

pstore< int >() [1/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< int >	(	int *	to,
		const Packet16i &	from
	)

                                                                     {
  EIGEN_DEBUG_ALIGNED_STORE _mm512_store_epi32(to, from);
}

References EIGEN_DEBUG_ALIGNED_STORE.

pstore< int >() [2/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< int >	(	int *	to,
		const Packet4i &	from
	)

                                                                    {
  pstore_common<Packet4i>(to, from);
}

pstore< int >() [3/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< int >	(	int *	to,
		const Packet8i &	from
	)

                                                                    {
  EIGEN_DEBUG_ALIGNED_STORE _mm256_store_si256(reinterpret_cast<__m256i*>(to), from);
}

References EIGEN_DEBUG_ALIGNED_STORE.

pstore< int16_t >() [1/2]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< int16_t >	(	int16_t *	to,
		const Packet4s &	from
	)

                                                                            {
  EIGEN_DEBUG_ALIGNED_STORE vst1_s16(to, from);
}

References EIGEN_DEBUG_ALIGNED_STORE.

pstore< int16_t >() [2/2]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< int16_t >	(	int16_t *	to,
		const Packet8s &	from
	)

                                                                            {
  EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
}

References EIGEN_DEBUG_ALIGNED_STORE.

pstore< int32_t >() [1/2]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< int32_t >	(	int32_t *	to,
		const Packet2i &	from
	)

                                                                            {
  EIGEN_DEBUG_ALIGNED_STORE vst1_s32(to, from);
}

References EIGEN_DEBUG_ALIGNED_STORE.

pstore< int32_t >() [2/2]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< int32_t >	(	int32_t *	to,
		const Packet4i &	from
	)

                                                                            {
  EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
}

References EIGEN_DEBUG_ALIGNED_STORE.

pstore< int64_t >() [1/2]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< int64_t >	(	int64_t *	to,
		const Packet2l &	from
	)

                                                                            {
  EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
}

References EIGEN_DEBUG_ALIGNED_STORE.

pstore< int64_t >() [2/2]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< int64_t >	(	int64_t *	to,
		const Packet8l &	from
	)

                                                                            {
  EIGEN_DEBUG_ALIGNED_STORE _mm512_store_epi64(to, from);
}

References EIGEN_DEBUG_ALIGNED_STORE.

pstore< int8_t >() [1/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< int8_t >	(	int8_t *	to,
		const Packet16c &	from
	)

                                                                           {
  EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
}

References EIGEN_DEBUG_ALIGNED_STORE.

pstore< int8_t >() [2/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< int8_t >	(	int8_t *	to,
		const Packet4c &	from
	)

                                                                          {
  memcpy(to, &from, sizeof(from));
}

pstore< int8_t >() [3/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< int8_t >	(	int8_t *	to,
		const Packet8c &	from
	)

                                                                          {
  EIGEN_DEBUG_ALIGNED_STORE vst1_s8(to, from);
}

References EIGEN_DEBUG_ALIGNED_STORE.

pstore< numext::int32_t >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< numext::int32_t >	(	numext::int32_t *	to,
		const PacketXi &	from
	)

                                                                                        {
  EIGEN_DEBUG_ALIGNED_STORE svst1_s32(svptrue_b32(), to, from);
}

References EIGEN_DEBUG_ALIGNED_STORE.

pstore< short int >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< short int >	(	short int *	to,
		const Packet8s &	from
	)

                                                                                {
  pstore_common<Packet8s>(to, from);
}

pstore< signed char >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< signed char >	(	signed char *	to,
		const Packet16c &	from
	)

                                                                                     {
  pstore_common<Packet16c>(to, from);
}

pstore< std::complex< double > >() [1/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< std::complex< double > >	(	std::complex< double > *	to,
		const Packet1cd &	from
	)

                                                                                                    {
  EIGEN_DEBUG_ALIGNED_STORE pstore((double*)to, Packet2d(from.v));
}

References EIGEN_DEBUG_ALIGNED_STORE, and pstore().

pstore< std::complex< double > >() [2/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< std::complex< double > >	(	std::complex< double > *	to,
		const Packet2cd &	from
	)

                                                                                                    {
  EIGEN_DEBUG_ALIGNED_STORE _mm256_store_pd((double*)to, from.v);
}

References EIGEN_DEBUG_ALIGNED_STORE.

pstore< std::complex< double > >() [3/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< std::complex< double > >	(	std::complex< double > *	to,
		const Packet4cd &	from
	)

                                                                                                    {
  EIGEN_DEBUG_ALIGNED_STORE pstore((double*)to, from.v);
}

References EIGEN_DEBUG_ALIGNED_STORE, and pstore().

pstore< std::complex< float > >() [1/4]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< std::complex< float > >	(	std::complex< float > *	to,
		const Packet1cf &	from
	)

                                                                                                  {
  EIGEN_DEBUG_ALIGNED_STORE pstore((float*)to, from.v);
}

References EIGEN_DEBUG_ALIGNED_STORE, and pstore().

pstore< std::complex< float > >() [2/4]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< std::complex< float > >	(	std::complex< float > *	to,
		const Packet2cf &	from
	)

                                                                                                  {
  pstore((float*)to, from.v);
}

References pstore().

pstore< std::complex< float > >() [3/4]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< std::complex< float > >	(	std::complex< float > *	to,
		const Packet4cf &	from
	)

                                                                                                  {
  EIGEN_DEBUG_ALIGNED_STORE _mm256_store_ps(&numext::real_ref(*to), from.v);
}

References EIGEN_DEBUG_ALIGNED_STORE, and Eigen::numext::real_ref().

pstore< std::complex< float > >() [4/4]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< std::complex< float > >	(	std::complex< float > *	to,
		const Packet8cf &	from
	)

                                                                                                  {
  EIGEN_DEBUG_ALIGNED_STORE pstore(&numext::real_ref(*to), from.v);
}

References EIGEN_DEBUG_ALIGNED_STORE, pstore(), and Eigen::numext::real_ref().

pstore< uint16_t >() [1/2]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< uint16_t >	(	uint16_t *	to,
		const Packet4us &	from
	)

                                                                               {
  EIGEN_DEBUG_ALIGNED_STORE vst1_u16(to, from);
}

References EIGEN_DEBUG_ALIGNED_STORE.

pstore< uint16_t >() [2/2]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< uint16_t >	(	uint16_t *	to,
		const Packet8us &	from
	)

                                                                               {
  EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
}

References EIGEN_DEBUG_ALIGNED_STORE.

pstore< uint32_t >() [1/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< uint32_t >	(	uint32_t *	to,
		const Packet2ui &	from
	)

                                                                               {
  EIGEN_DEBUG_ALIGNED_STORE vst1_u32(to, from);
}

References EIGEN_DEBUG_ALIGNED_STORE.

pstore< uint32_t >() [2/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< uint32_t >	(	uint32_t *	to,
		const Packet4ui &	from
	)

                                                                               {
  EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
}

References EIGEN_DEBUG_ALIGNED_STORE.

pstore< uint32_t >() [3/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< uint32_t >	(	uint32_t *	to,
		const Packet8ui &	from
	)

                                                                               {
  EIGEN_DEBUG_ALIGNED_STORE _mm256_store_si256(reinterpret_cast<__m256i*>(to), from);
}

References EIGEN_DEBUG_ALIGNED_STORE.

pstore< uint64_t >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< uint64_t >	(	uint64_t *	to,
		const Packet2ul &	from
	)

                                                                               {
  EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
}

References EIGEN_DEBUG_ALIGNED_STORE.

pstore< uint8_t >() [1/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< uint8_t >	(	uint8_t *	to,
		const Packet16uc &	from
	)

                                                                              {
  EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
}

References EIGEN_DEBUG_ALIGNED_STORE.

pstore< uint8_t >() [2/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< uint8_t >	(	uint8_t *	to,
		const Packet4uc &	from
	)

                                                                             {
  memcpy(to, &from, sizeof(from));
}

pstore< uint8_t >() [3/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< uint8_t >	(	uint8_t *	to,
		const Packet8uc &	from
	)

                                                                             {
  EIGEN_DEBUG_ALIGNED_STORE vst1_u8(to, from);
}

References EIGEN_DEBUG_ALIGNED_STORE.

pstore< unsigned char >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< unsigned char >	(	unsigned char *	to,
		const Packet16uc &	from
	)

                                                                                          {
  pstore_common<Packet16uc>(to, from);
}

pstore< unsigned short int >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstore< unsigned short int >	(	unsigned short int *	to,
		const Packet8us &	from
	)

                                                                                                   {
  pstore_common<Packet8us>(to, from);
}

pstore_common()

template<typename Packet >

EIGEN_STRONG_INLINE void Eigen::internal::pstore_common	(	__UNPACK_TYPE__(Packet) *	to,
		const Packet &	from
	)

                                                                                         {
  // some versions of GCC throw "unused-but-set-parameter" (float *to).
  // ignoring these warnings for now.
  EIGEN_UNUSED_VARIABLE(to);
  EIGEN_DEBUG_ALIGNED_STORE
#ifdef EIGEN_VECTORIZE_VSX
  vec_xst(from, 0, to);
#else
  vec_st(from, 0, to);
#endif
}

References EIGEN_DEBUG_ALIGNED_STORE, and EIGEN_UNUSED_VARIABLE.

pstore_partial()

template<typename Scalar , typename Packet >

EIGEN_DEVICE_FUNC void Eigen::internal::pstore_partial ( Scalar *  to, const Packet &  from, const Index  n, const Index  offset = 0  )

inline

copy n elements of the packet from to *to, to must be properly aligned offset indicates the starting element in which to store and offset + n <= unpacket_traits::size

                                                                                                                    {
  const Index packet_size = unpacket_traits<Packet>::size;
  eigen_assert(n + offset <= packet_size && "number of elements plus offset will write past end of packet");
  EIGEN_ALIGN_MAX Scalar elements[packet_size];
  pstore<Scalar>(elements, from);
  for (Index i = 0; i < numext::mini(n, packet_size - offset); i++) {
    to[i] = elements[i + offset];
  }
}

References EIGEN_ALIGN_MAX, eigen_assert, i, Eigen::numext::mini(), and n.

Referenced by packetmath(), packetmath_scatter_gather(), pstore_partial< std::complex< float > >(), and pstoret_partial().

pstore_partial< bfloat16 >()

template<>

EIGEN_ALWAYS_INLINE void Eigen::internal::pstore_partial< bfloat16 >	(	bfloat16 *	to,
		const Packet8bf &	from,
		const Index	n,
		const Index	offset
	)

                                                                      {
  pstore_partial_common<Packet8us>(reinterpret_cast<unsigned short int*>(to), from.m_val, n, offset);
}

References Eigen::internal::eigen_packet_wrapper< T, unique_id >::m_val, and n.

Referenced by Eigen::internal::dhs_pack< bfloat16, DataMapper, Packet8bf, StorageOrder, PanelMode, true >::operator()().

pstore_partial< float >()

template<>

EIGEN_ALWAYS_INLINE void Eigen::internal::pstore_partial< float >	(	float *	to,
		const Packet4f &	from,
		const Index	n,
		const Index	offset
	)

                                                                                                                   {
  pstore_partial_common<Packet4f>(to, from, n, offset);
}

References n.

pstore_partial< int >()

template<>

EIGEN_ALWAYS_INLINE void Eigen::internal::pstore_partial< int >	(	int *	to,
		const Packet4i &	from,
		const Index	n,
		const Index	offset
	)

                                                                                                               {
  pstore_partial_common<Packet4i>(to, from, n, offset);
}

References n.

pstore_partial< short int >()

template<>

EIGEN_ALWAYS_INLINE void Eigen::internal::pstore_partial< short int >	(	short int *	to,
		const Packet8s &	from,
		const Index	n,
		const Index	offset
	)

                                                                       {
  pstore_partial_common<Packet8s>(to, from, n, offset);
}

References n.

pstore_partial< signed char >()

template<>

EIGEN_ALWAYS_INLINE void Eigen::internal::pstore_partial< signed char >	(	signed char *	to,
		const Packet16c &	from,
		const Index	n,
		const Index	offset
	)

                                                                         {
  pstore_partial_common<Packet16c>(to, from, n, offset);
}

References n.

pstore_partial< std::complex< float > >()

template<>

EIGEN_ALWAYS_INLINE void Eigen::internal::pstore_partial< std::complex< float > >	(	std::complex< float > *	to,
		const Packet2cf &	from,
		const Index	n,
		const Index	offset
	)

                                                                                                 {
  pstore_partial((float*)to, from.v, n * 2, offset * 2);
}

References n, and pstore_partial().

pstore_partial< unsigned char >()

template<>

EIGEN_ALWAYS_INLINE void Eigen::internal::pstore_partial< unsigned char >	(	unsigned char *	to,
		const Packet16uc &	from,
		const Index	n,
		const Index	offset
	)

                                                                           {
  pstore_partial_common<Packet16uc>(to, from, n, offset);
}

References n.

pstore_partial< unsigned short int >()

template<>

EIGEN_ALWAYS_INLINE void Eigen::internal::pstore_partial< unsigned short int >	(	unsigned short int *	to,
		const Packet8us &	from,
		const Index	n,
		const Index	offset
	)

                                                                                               {
  pstore_partial_common<Packet8us>(to, from, n, offset);
}

References n.

pstore_partial_common()

template<typename Packet >

EIGEN_ALWAYS_INLINE void Eigen::internal::pstore_partial_common	(	__UNPACK_TYPE__(Packet) *	to,
		const Packet &	from,
		const Index	n,
		const Index	offset
	)

                                                                   {
  // some versions of GCC throw "unused-but-set-parameter" (float *to).
  // ignoring these warnings for now.
  const Index packet_size = unpacket_traits<Packet>::size;
  eigen_internal_assert(n + offset <= packet_size && "number of elements plus offset will write past end of packet");
  const Index size = sizeof(__UNPACK_TYPE__(Packet));
#ifdef _ARCH_PWR9
  EIGEN_UNUSED_VARIABLE(packet_size);
  EIGEN_UNUSED_VARIABLE(to);
  EIGEN_DEBUG_ALIGNED_STORE
  Packet store = from;
  if (offset) {
    Packet16uc shift = pset1<Packet16uc>(offset * 8 * size);
#ifdef _BIG_ENDIAN
    store = Packet(vec_slo(Packet16uc(store), shift));
#else
    store = Packet(vec_sro(Packet16uc(store), shift));
#endif
  }
  vec_xst_len(store, to, n * size);
#else
  if (n) {
    EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) store[packet_size];
    pstore(store, from);
    unsigned char* store2 = reinterpret_cast<unsigned char*>(store + offset);
    unsigned char* to2 = reinterpret_cast<unsigned char*>(to);
    Index n2 = n * size;
    if (16 <= n2) {
      pstore(to2, ploadu<Packet16uc>(store2));
    } else {
      memcpy((void*)to2, (void*)store2, n2);
    }
  }
#endif
}

References __UNPACK_TYPE__(), EIGEN_ALIGN16, EIGEN_DEBUG_ALIGNED_STORE, eigen_internal_assert, EIGEN_UNUSED_VARIABLE, n, ploadu< Packet16uc >(), pset1< Packet16uc >(), pstore(), and size.

pstorel() [1/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstorel	(	double *	to,
		const Packet2d &	from
	)

                                                                   {
  EIGEN_DEBUG_UNALIGNED_STORE _mm_storel_pd(to, from);
}

References EIGEN_DEBUG_UNALIGNED_STORE.

pstorel() [2/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstorel	(	float *	to,
		const Packet4f &	from
	)

                                                                  {
  EIGEN_DEBUG_UNALIGNED_STORE _mm_storel_pi(reinterpret_cast<__m64*>(to), from);
}

References EIGEN_DEBUG_UNALIGNED_STORE.

pstorel() [3/3]

template<typename Scalar , typename Packet >

EIGEN_STRONG_INLINE void Eigen::internal::pstorel	(	Scalar *	to,
		const Packet &	from
	)

Referenced by Eigen::internal::gemm_class< Scalar, is_unit_inc >::c_store().

pstores() [1/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstores	(	double *	to,
		const Packet2d &	from
	)

                                                                   {
  EIGEN_DEBUG_UNALIGNED_STORE _mm_store_sd(to, from);
}

References EIGEN_DEBUG_UNALIGNED_STORE.

pstores() [2/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstores	(	float *	to,
		const Packet4f &	from
	)

                                                                  {
  EIGEN_DEBUG_UNALIGNED_STORE _mm_store_ss(to, from);
}

References EIGEN_DEBUG_UNALIGNED_STORE.

pstores() [3/3]

template<typename Scalar , typename Packet >

EIGEN_STRONG_INLINE void Eigen::internal::pstores	(	Scalar *	to,
		const Packet &	from
	)

Referenced by Eigen::internal::gemm_class< Scalar, is_unit_inc >::c_store().

pstoret()

template<typename Scalar , typename Packet , int Alignment>

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void Eigen::internal::pstoret	(	Scalar *	to,
		const Packet &	from
	)

copy the packet from to *to. The pointer from must be aligned on a Alignment bytes boundary.

                                                                                   {
  if (Alignment >= unpacket_traits<Packet>::alignment)
    pstore(to, from);
  else
    pstoreu(to, from);
}

References pstore(), and pstoreu().

pstoret_partial()

template<typename Scalar , typename Packet , int Alignment>

EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void Eigen::internal::pstoret_partial	(	Scalar *	to,
		const Packet &	from,
		const Index	n,
		const Index	offset = 0
	)

copy n elements of the packet from to *to. The pointer from must be aligned on a Alignment bytes boundary.

                                                                                   {
  if (Alignment >= unpacket_traits<Packet>::alignment)
    pstore_partial(to, from, n, offset);
  else
    pstoreu_partial(to, from, n, offset);
}

References n, pstore_partial(), and pstoreu_partial().

pstoreu() [1/2]

template<typename Scalar , typename Packet >

EIGEN_DEVICE_FUNC void Eigen::internal::pstoreu ( Scalar *  to, const Packet &  from  )

inline

copy the packet from to *to, (un-aligned store)

                                                                      {
  (*to) = from;
}

Referenced by Eigen::internal::gemm_class< Scalar, is_unit_inc >::c_store(), pcast_array< SrcPacket, TgtPacket, 1, TgtCoeffRatio >::cast(), pcast_array< SrcPacket, TgtPacket, 2, 1 >::cast(), pcast_array< SrcPacket, TgtPacket, 4, 1 >::cast(), pcast_array< SrcPacket, TgtPacket, 8, 1 >::cast(), convertArrayPointerBF16toF32DupOne(), Eigen::internal::gemm_pack_lhs< Scalar, Index, DataMapper, Pack1, Pack2, Packet, ColMajor, Conjugate, PanelMode >::operator()(), Eigen::internal::gemm_pack_rhs< Scalar, Index, DataMapper, 8, ColMajor, Conjugate, PanelMode >::operator()(), Eigen::internal::gemm_pack_rhs< Scalar, Index, DataMapper, 8, RowMajor, Conjugate, PanelMode >::operator()(), Eigen::internal::gemm_pack_rhs< Scalar, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode >::operator()(), Eigen::internal::gemm_pack_rhs< Scalar, Index, DataMapper, nr, RowMajor, Conjugate, PanelMode >::operator()(), operator<<(), outputVecCol(), outputVecResults(), packetmath(), pload_partial_common(), ploadu_partial_common(), pscatter_common(), psincos_double(), psincos_float(), pstoret(), pstoreu< std::complex< double > >(), pstoreu< std::complex< float > >(), pstoreu_partial_common(), pstoreu_pmadd_complex(), Eigen::internal::general_matrix_vector_product< Index, LhsScalar, LhsMapper, ColMajor, ConjugateLhs, RhsScalar, RhsMapper, ConjugateRhs, Version >::run(), Eigen::internal::apply_rotation_in_the_plane_selector< Scalar, OtherScalar, SizeAtCompileTime, MinAlignment, true >::run(), Eigen::test::packet_helper< Cond, Packet >::store(), storeBF16fromResult(), storeConvertTwoBF16(), storeF32(), and storeMaddData().

pstoreu() [2/2]

template<typename Scalar , typename Packet >

EIGEN_DEVICE_FUNC std::enable_if_t<unpacket_traits<Packet>::masked_store_available, void> Eigen::internal::pstoreu ( Scalar *  to, const Packet &  from, typename unpacket_traits< Packet >::mask_t  umask  )

inline

copy the packet from to *to, (un-aligned store with a mask) There is no generic implementation. We only have implementations for specialized cases. Generic case should not be called.

pstoreu< bfloat16 >() [1/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< bfloat16 >	(	bfloat16 *	to,
		const Packet16bf &	from
	)

                                                                                 {
  _mm256_storeu_si256(reinterpret_cast<__m256i*>(to), from);
}

pstoreu< bfloat16 >() [2/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< bfloat16 >	(	bfloat16 *	to,
		const Packet4bf &	from
	)

                                                                                {
  EIGEN_DEBUG_UNALIGNED_STORE vst1_u16(reinterpret_cast<uint16_t*>(to), from);
}

References EIGEN_DEBUG_UNALIGNED_STORE.

pstoreu< bfloat16 >() [3/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< bfloat16 >	(	bfloat16 *	to,
		const Packet8bf &	from
	)

                                                                                {
  pstoreu_common<Packet8us>(reinterpret_cast<unsigned short int*>(to), from.m_val);
}

References Eigen::internal::eigen_packet_wrapper< T, unique_id >::m_val.

pstoreu< bool >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< bool >	(	bool *	to,
		const Packet16b &	from
	)

                                                                        {
  EIGEN_DEBUG_UNALIGNED_STORE _mm_storeu_si128(reinterpret_cast<__m128i*>(to), from);
}

References EIGEN_DEBUG_UNALIGNED_STORE.

pstoreu< double >() [1/4]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< double >	(	double *	to,
		const Packet2d &	from
	)

                                                                           {
  EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst(from, to, 0);
}

References EIGEN_DEBUG_UNALIGNED_STORE.

pstoreu< double >() [2/4]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< double >	(	double *	to,
		const Packet4d &	from
	)

                                                                           {
  EIGEN_DEBUG_UNALIGNED_STORE _mm256_storeu_pd(to, from);
}

References EIGEN_DEBUG_UNALIGNED_STORE.

pstoreu< double >() [3/4]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< double >	(	double *	to,
		const Packet8d &	from
	)

                                                                           {
  EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_pd(to, from);
}

References EIGEN_DEBUG_UNALIGNED_STORE.

pstoreu< double >() [4/4]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< double >	(	double *	to,
		const Packet8d &	from,
		uint8_t	umask
	)

                                                                                          {
  __mmask8 mask = static_cast<__mmask8>(umask);
  EIGEN_DEBUG_UNALIGNED_STORE return _mm512_mask_storeu_pd(to, mask, from);
}

References EIGEN_DEBUG_UNALIGNED_STORE.

pstoreu< Eigen::half >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< Eigen::half >	(	Eigen::half *	to,
		const Packet8h &	from
	)

                                                                                 {
  _mm_storeu_si128(reinterpret_cast<__m128i*>(to), from);
}

pstoreu< float >() [1/7]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< float >	(	float *	to,
		const Packet16f &	from
	)

                                                                          {
  EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_ps(to, from);
}

References EIGEN_DEBUG_UNALIGNED_STORE.

pstoreu< float >() [2/7]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< float >	(	float *	to,
		const Packet16f &	from,
		uint16_t	umask
	)

                                                                                          {
  __mmask16 mask = static_cast<__mmask16>(umask);
  EIGEN_DEBUG_UNALIGNED_STORE return _mm512_mask_storeu_ps(to, mask, from);
}

References EIGEN_DEBUG_UNALIGNED_STORE.

pstoreu< float >() [3/7]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< float >	(	float *	to,
		const Packet2f &	from
	)

                                                                         {
  EIGEN_DEBUG_UNALIGNED_STORE vst1_f32(to, from);
}

References EIGEN_DEBUG_UNALIGNED_STORE.

pstoreu< float >() [4/7]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< float >	(	float *	to,
		const Packet4f &	from
	)

                                                                         {
  pstoreu_common<Packet4f>(to, from);
}

pstoreu< float >() [5/7]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< float >	(	float *	to,
		const Packet8f &	from
	)

                                                                         {
  EIGEN_DEBUG_UNALIGNED_STORE _mm256_storeu_ps(to, from);
}

References EIGEN_DEBUG_UNALIGNED_STORE.

pstoreu< float >() [6/7]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< float >	(	float *	to,
		const Packet8f &	from,
		uint8_t	umask
	)

                                                                                        {
#ifdef EIGEN_VECTORIZE_AVX512
  __mmask16 mask = static_cast<__mmask16>(umask & 0x00FF);
  EIGEN_DEBUG_UNALIGNED_STORE _mm512_mask_storeu_ps(to, mask, _mm512_castps256_ps512(from));
#else
  Packet8i mask = _mm256_set1_epi8(static_cast<char>(umask));
  const Packet8i bit_mask =
      _mm256_set_epi32(0x7f7f7f7f, 0xbfbfbfbf, 0xdfdfdfdf, 0xefefefef, 0xf7f7f7f7, 0xfbfbfbfb, 0xfdfdfdfd, 0xfefefefe);
  mask = por<Packet8i>(mask, bit_mask);
  mask = pcmp_eq<Packet8i>(mask, _mm256_set1_epi32(0xffffffff));
#if EIGEN_COMP_MSVC
  // MSVC sometimes seems to use a bogus mask with maskstore.
  const __m256i ifrom = _mm256_castps_si256(from);
  EIGEN_DEBUG_UNALIGNED_STORE _mm_maskmoveu_si128(_mm256_extractf128_si256(ifrom, 0), _mm256_extractf128_si256(mask, 0),
                                                  reinterpret_cast<char*>(to));
  EIGEN_DEBUG_UNALIGNED_STORE _mm_maskmoveu_si128(_mm256_extractf128_si256(ifrom, 1), _mm256_extractf128_si256(mask, 1),
                                                  reinterpret_cast<char*>(to + 4));
#else
  EIGEN_DEBUG_UNALIGNED_STORE _mm256_maskstore_ps(to, mask, from);
#endif
#endif
}

References EIGEN_DEBUG_UNALIGNED_STORE, and por< Packet8i >().

pstoreu< float >() [7/7]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< float >	(	float *	to,
		const PacketXf &	from
	)

                                                                         {
  EIGEN_DEBUG_UNALIGNED_STORE svst1_f32(svptrue_b32(), to, from);
}

References EIGEN_DEBUG_UNALIGNED_STORE.

pstoreu< half >() [1/2]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< half >	(	Eigen::half *	to,
		const Packet16h &	from
	)

                                                                             {
  // (void*) -> workaround clang warning:
  // cast from 'Eigen::half *' to '__m256i *' increases required alignment from 2 to 32
  _mm256_storeu_si256((__m256i*)(void*)to, from);
}

pstoreu< half >() [2/2]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< half >	(	Eigen::half *	to,
		const Packet32h &	from
	)

                                                                             {
  EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_ph(to, from);
}

References EIGEN_DEBUG_UNALIGNED_STORE.

pstoreu< int >() [1/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< int >	(	int *	to,
		const Packet16i &	from
	)

                                                                      {
  EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_epi32(to, from);
}

References EIGEN_DEBUG_UNALIGNED_STORE.

pstoreu< int >() [2/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< int >	(	int *	to,
		const Packet4i &	from
	)

                                                                     {
  pstoreu_common<Packet4i>(to, from);
}

pstoreu< int >() [3/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< int >	(	int *	to,
		const Packet8i &	from
	)

                                                                     {
  EIGEN_DEBUG_UNALIGNED_STORE _mm256_storeu_si256(reinterpret_cast<__m256i*>(to), from);
}

References EIGEN_DEBUG_UNALIGNED_STORE.

pstoreu< int16_t >() [1/2]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< int16_t >	(	int16_t *	to,
		const Packet4s &	from
	)

                                                                             {
  EIGEN_DEBUG_UNALIGNED_STORE vst1_s16(to, from);
}

References EIGEN_DEBUG_UNALIGNED_STORE.

pstoreu< int16_t >() [2/2]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< int16_t >	(	int16_t *	to,
		const Packet8s &	from
	)

                                                                             {
  EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
}

References EIGEN_DEBUG_UNALIGNED_STORE.

pstoreu< int32_t >() [1/2]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< int32_t >	(	int32_t *	to,
		const Packet2i &	from
	)

                                                                             {
  EIGEN_DEBUG_UNALIGNED_STORE vst1_s32(to, from);
}

References EIGEN_DEBUG_UNALIGNED_STORE.

pstoreu< int32_t >() [2/2]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< int32_t >	(	int32_t *	to,
		const Packet4i &	from
	)

                                                                             {
  EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
}

References EIGEN_DEBUG_UNALIGNED_STORE.

pstoreu< int64_t >() [1/2]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< int64_t >	(	int64_t *	to,
		const Packet2l &	from
	)

                                                                             {
  EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
}

References EIGEN_DEBUG_UNALIGNED_STORE.

pstoreu< int64_t >() [2/2]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< int64_t >	(	int64_t *	to,
		const Packet8l &	from
	)

                                                                             {
  EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_epi64(to, from);
}

References EIGEN_DEBUG_UNALIGNED_STORE.

pstoreu< int8_t >() [1/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< int8_t >	(	int8_t *	to,
		const Packet16c &	from
	)

                                                                            {
  EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
}

References EIGEN_DEBUG_UNALIGNED_STORE.

pstoreu< int8_t >() [2/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< int8_t >	(	int8_t *	to,
		const Packet4c &	from
	)

                                                                           {
  memcpy(to, &from, sizeof(from));
}

pstoreu< int8_t >() [3/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< int8_t >	(	int8_t *	to,
		const Packet8c &	from
	)

                                                                           {
  EIGEN_DEBUG_UNALIGNED_STORE vst1_s8(to, from);
}

References EIGEN_DEBUG_UNALIGNED_STORE.

pstoreu< numext::int32_t >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< numext::int32_t >	(	numext::int32_t *	to,
		const PacketXi &	from
	)

                                                                                         {
  EIGEN_DEBUG_UNALIGNED_STORE svst1_s32(svptrue_b32(), to, from);
}

References EIGEN_DEBUG_UNALIGNED_STORE.

pstoreu< short int >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< short int >	(	short int *	to,
		const Packet8s &	from
	)

                                                                                 {
  pstoreu_common<Packet8s>(to, from);
}

pstoreu< signed char >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< signed char >	(	signed char *	to,
		const Packet16c &	from
	)

                                                                                      {
  pstoreu_common<Packet16c>(to, from);
}

pstoreu< std::complex< double > >() [1/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< std::complex< double > >	(	std::complex< double > *	to,
		const Packet1cd &	from
	)

                                                                                                     {
  EIGEN_DEBUG_UNALIGNED_STORE pstoreu((double*)to, Packet2d(from.v));
}

References EIGEN_DEBUG_UNALIGNED_STORE, and pstoreu().

pstoreu< std::complex< double > >() [2/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< std::complex< double > >	(	std::complex< double > *	to,
		const Packet2cd &	from
	)

                                                                                                     {
  EIGEN_DEBUG_UNALIGNED_STORE _mm256_storeu_pd((double*)to, from.v);
}

References EIGEN_DEBUG_UNALIGNED_STORE.

pstoreu< std::complex< double > >() [3/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< std::complex< double > >	(	std::complex< double > *	to,
		const Packet4cd &	from
	)

                                                                                                     {
  EIGEN_DEBUG_UNALIGNED_STORE pstoreu((double*)to, from.v);
}

References EIGEN_DEBUG_UNALIGNED_STORE, and pstoreu().

pstoreu< std::complex< float > >() [1/4]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< std::complex< float > >	(	std::complex< float > *	to,
		const Packet1cf &	from
	)

                                                                                                   {
  EIGEN_DEBUG_UNALIGNED_STORE pstoreu((float*)to, from.v);
}

References EIGEN_DEBUG_UNALIGNED_STORE, and pstoreu().

pstoreu< std::complex< float > >() [2/4]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< std::complex< float > >	(	std::complex< float > *	to,
		const Packet2cf &	from
	)

                                                                                                   {
  pstoreu((float*)to, from.v);
}

References pstoreu().

pstoreu< std::complex< float > >() [3/4]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< std::complex< float > >	(	std::complex< float > *	to,
		const Packet4cf &	from
	)

                                                                                                   {
  EIGEN_DEBUG_UNALIGNED_STORE _mm256_storeu_ps(&numext::real_ref(*to), from.v);
}

References EIGEN_DEBUG_UNALIGNED_STORE, and Eigen::numext::real_ref().

pstoreu< std::complex< float > >() [4/4]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< std::complex< float > >	(	std::complex< float > *	to,
		const Packet8cf &	from
	)

                                                                                                   {
  EIGEN_DEBUG_UNALIGNED_STORE pstoreu(&numext::real_ref(*to), from.v);
}

References EIGEN_DEBUG_UNALIGNED_STORE, pstoreu(), and Eigen::numext::real_ref().

pstoreu< uint16_t >() [1/2]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< uint16_t >	(	uint16_t *	to,
		const Packet4us &	from
	)

                                                                                {
  EIGEN_DEBUG_UNALIGNED_STORE vst1_u16(to, from);
}

References EIGEN_DEBUG_UNALIGNED_STORE.

pstoreu< uint16_t >() [2/2]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< uint16_t >	(	uint16_t *	to,
		const Packet8us &	from
	)

                                                                                {
  EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
}

References EIGEN_DEBUG_UNALIGNED_STORE.

pstoreu< uint32_t >() [1/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< uint32_t >	(	uint32_t *	to,
		const Packet2ui &	from
	)

                                                                                {
  EIGEN_DEBUG_UNALIGNED_STORE vst1_u32(to, from);
}

References EIGEN_DEBUG_UNALIGNED_STORE.

pstoreu< uint32_t >() [2/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< uint32_t >	(	uint32_t *	to,
		const Packet4ui &	from
	)

                                                                                {
  EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
}

References EIGEN_DEBUG_UNALIGNED_STORE.

pstoreu< uint32_t >() [3/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< uint32_t >	(	uint32_t *	to,
		const Packet8ui &	from
	)

                                                                                {
  EIGEN_DEBUG_UNALIGNED_STORE _mm256_storeu_si256(reinterpret_cast<__m256i*>(to), from);
}

References EIGEN_DEBUG_UNALIGNED_STORE.

pstoreu< uint64_t >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< uint64_t >	(	uint64_t *	to,
		const Packet2ul &	from
	)

                                                                                {
  EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
}

References EIGEN_DEBUG_UNALIGNED_STORE.

pstoreu< uint8_t >() [1/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< uint8_t >	(	uint8_t *	to,
		const Packet16uc &	from
	)

                                                                               {
  EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
}

References EIGEN_DEBUG_UNALIGNED_STORE.

pstoreu< uint8_t >() [2/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< uint8_t >	(	uint8_t *	to,
		const Packet4uc &	from
	)

                                                                              {
  memcpy(to, &from, sizeof(from));
}

pstoreu< uint8_t >() [3/3]

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< uint8_t >	(	uint8_t *	to,
		const Packet8uc &	from
	)

                                                                              {
  EIGEN_DEBUG_UNALIGNED_STORE vst1_u8(to, from);
}

References EIGEN_DEBUG_UNALIGNED_STORE.

pstoreu< unsigned char >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< unsigned char >	(	unsigned char *	to,
		const Packet16uc &	from
	)

                                                                                           {
  pstoreu_common<Packet16uc>(to, from);
}

pstoreu< unsigned short int >()

template<>

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu< unsigned short int >	(	unsigned short int *	to,
		const Packet8us &	from
	)

                                                                                                    {
  pstoreu_common<Packet8us>(to, from);
}

pstoreu_common()

template<typename Packet >

EIGEN_STRONG_INLINE void Eigen::internal::pstoreu_common	(	__UNPACK_TYPE__(Packet) *	to,
		const Packet &	from
	)

                                                                                          {
  EIGEN_DEBUG_UNALIGNED_STORE
#if defined(EIGEN_VECTORIZE_VSX) || !defined(_BIG_ENDIAN)
  vec_xst(from, 0, to);
#else
  // Taken from http://developer.apple.com/hardwaredrivers/ve/alignment.html
  // Warning: not thread safe!
  Packet16uc MSQ, LSQ, edges;
  Packet16uc edgeAlign, align;
 
  MSQ = vec_ld(0, (unsigned char*)to);             // most significant quadword
  LSQ = vec_ld(15, (unsigned char*)to);            // least significant quadword
  edgeAlign = vec_lvsl(0, to);                     // permute map to extract edges
  edges = vec_perm(LSQ, MSQ, edgeAlign);           // extract the edges
  align = vec_lvsr(0, to);                         // permute map to misalign data
  MSQ = vec_perm(edges, (Packet16uc)from, align);  // misalign the data (MSQ)
  LSQ = vec_perm((Packet16uc)from, edges, align);  // misalign the data (LSQ)
  vec_st(LSQ, 15, (unsigned char*)to);             // Store the LSQ part first
  vec_st(MSQ, 0, (unsigned char*)to);              // Store the MSQ part second
#endif
}

References EIGEN_DEBUG_UNALIGNED_STORE.

pstoreu_partial()

template<typename Scalar , typename Packet >

EIGEN_DEVICE_FUNC void Eigen::internal::pstoreu_partial ( Scalar *  to, const Packet &  from, const Index  n, const Index  offset = 0  )

inline

copy n elements of the packet from to *to, (un-aligned store)

                                                                                                                     {
  const Index packet_size = unpacket_traits<Packet>::size;
  eigen_assert(n + offset <= packet_size && "number of elements plus offset will write past end of packet");
  EIGEN_ALIGN_MAX Scalar elements[packet_size];
  pstore<Scalar>(elements, from);
  for (Index i = 0; i < numext::mini(n, packet_size - offset); i++) {
    to[i] = elements[i + offset];
  }
}

References EIGEN_ALIGN_MAX, eigen_assert, i, Eigen::numext::mini(), and n.

Referenced by outputVecCol(), outputVecResults(), packetmath(), pscatter_common(), pstoret_partial(), pstoreu_partial< std::complex< float > >(), storeBF16fromResult(), storeConvertTwoBF16(), and storeF32().

pstoreu_partial< bfloat16 >()

template<>

EIGEN_ALWAYS_INLINE void Eigen::internal::pstoreu_partial< bfloat16 >	(	bfloat16 *	to,
		const Packet8bf &	from,
		const Index	n,
		const Index	offset
	)

                                                                       {
  pstoreu_partial_common<Packet8us>(reinterpret_cast<unsigned short int*>(to), from, n, offset);
}

References n.

pstoreu_partial< float >()

template<>

EIGEN_ALWAYS_INLINE void Eigen::internal::pstoreu_partial< float >	(	float *	to,
		const Packet4f &	from,
		const Index	n,
		const Index	offset
	)

                                                                                                                    {
  pstoreu_partial_common<Packet4f>(to, from, n, offset);
}

References n.

pstoreu_partial< int >()

template<>

EIGEN_ALWAYS_INLINE void Eigen::internal::pstoreu_partial< int >	(	int *	to,
		const Packet4i &	from,
		const Index	n,
		const Index	offset
	)

                                                                                                                {
  pstoreu_partial_common<Packet4i>(to, from, n, offset);
}

References n.

pstoreu_partial< short int >()

template<>

EIGEN_ALWAYS_INLINE void Eigen::internal::pstoreu_partial< short int >	(	short int *	to,
		const Packet8s &	from,
		const Index	n,
		const Index	offset
	)

                                                                        {
  pstoreu_partial_common<Packet8s>(to, from, n, offset);
}

References n.

pstoreu_partial< signed char >()

template<>

EIGEN_ALWAYS_INLINE void Eigen::internal::pstoreu_partial< signed char >	(	signed char *	to,
		const Packet16c &	from,
		const Index	n,
		const Index	offset
	)

                                                                          {
  pstoreu_partial_common<Packet16c>(to, from, n, offset);
}

References n.

pstoreu_partial< std::complex< float > >()

template<>

EIGEN_ALWAYS_INLINE void Eigen::internal::pstoreu_partial< std::complex< float > >	(	std::complex< float > *	to,
		const Packet2cf &	from,
		const Index	n,
		const Index	offset
	)

                                                                                                  {
  pstoreu_partial((float*)to, from.v, n * 2, offset * 2);
}

References n, and pstoreu_partial().

pstoreu_partial< unsigned char >()

template<>

EIGEN_ALWAYS_INLINE void Eigen::internal::pstoreu_partial< unsigned char >	(	unsigned char *	to,
		const Packet16uc &	from,
		const Index	n,
		const Index	offset
	)

                                                                            {
  pstoreu_partial_common<Packet16uc>(to, from, n, offset);
}

References n.

pstoreu_partial< unsigned short int >()

template<>

EIGEN_ALWAYS_INLINE void Eigen::internal::pstoreu_partial< unsigned short int >	(	unsigned short int *	to,
		const Packet8us &	from,
		const Index	n,
		const Index	offset
	)

                                                                                                {
  pstoreu_partial_common<Packet8us>(to, from, n, offset);
}

References n.

pstoreu_partial_common()

template<typename Packet >

EIGEN_ALWAYS_INLINE void Eigen::internal::pstoreu_partial_common	(	__UNPACK_TYPE__(Packet) *	to,
		const Packet &	from,
		const Index	n,
		const Index	offset
	)

                                                                    {
  const Index packet_size = unpacket_traits<Packet>::size;
  eigen_internal_assert(n + offset <= packet_size && "number of elements plus offset will write past end of packet");
  const Index size = sizeof(__UNPACK_TYPE__(Packet));
#ifdef _ARCH_PWR9
  EIGEN_UNUSED_VARIABLE(packet_size);
  EIGEN_DEBUG_UNALIGNED_STORE
  Packet store = from;
  if (offset) {
    Packet16uc shift = pset1<Packet16uc>(offset * 8 * size);
#ifdef _BIG_ENDIAN
    store = Packet(vec_slo(Packet16uc(store), shift));
#else
    store = Packet(vec_sro(Packet16uc(store), shift));
#endif
  }
  vec_xst_len(store, to, n * size);
#else
  if (n) {
    EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) store[packet_size];
    pstore(store, from);
    unsigned char* store2 = reinterpret_cast<unsigned char*>(store + offset);
    unsigned char* to2 = reinterpret_cast<unsigned char*>(to);
    Index n2 = n * size;
    if (16 <= n2) {
      pstoreu(to2, ploadu<Packet16uc>(store2));
    } else {
      memcpy((void*)to2, (void*)store2, n2);
    }
  }
#endif
}

References __UNPACK_TYPE__(), EIGEN_ALIGN16, EIGEN_DEBUG_UNALIGNED_STORE, eigen_internal_assert, EIGEN_UNUSED_VARIABLE, n, ploadu< Packet16uc >(), pset1< Packet16uc >(), pstore(), pstoreu(), and size.

psub()

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::psub ( const Packet &  a, const Packet &  b  )

inline

Returns

a - b (coeff-wise)

                                                                       {
  return a - b;
}

References a, and b.

Referenced by absolute_split(), Eigen::internal::gebp_traits< std::complex< RealScalar >, std::complex< RealScalar >, ConjLhs_, ConjRhs_, Arch, PacketSize_ >::acc(), Eigen::internal::sub_assign_op< DstScalar, SrcScalar >::assignPacket(), doubleword_div_fp(), fast_twosum(), generic_atan(), generic_expm1(), generic_floor(), generic_log1p(), generic_ndtri(), generic_ndtri_gt_exp_neg_two(), generic_ndtri_lt_exp_neg_two(), generic_pow_impl(), generic_rint(), generic_round(), Eigen::internal::accurate_log2< double >::operator()(), Eigen::internal::accurate_log2< float >::operator()(), Eigen::internal::Packet1cd::operator-=(), Eigen::internal::Packet2cf::operator-=(), pabsdiff(), pabsdiff< Packet16c >(), pabsdiff< Packet16uc >(), pabsdiff< Packet4f >(), pabsdiff< Packet4i >(), pabsdiff< Packet4ui >(), pabsdiff< Packet8s >(), pabsdiff< Packet8us >(), packetmath(), Eigen::internal::scalar_difference_op< LhsScalar, RhsScalar >::packetOp(), pacos_float(), paddsub(), patanh_double(), patanh_float(), pcmp_le(), pcmp_lt(), pexp< Packet2d >(), pexp_double(), phypot_complex(), pldexp< Packet4d >(), pldexp< Packet8d >(), plog< Packet4f >(), plog_impl_double(), plog_impl_float(), pmsub(), pnegate(), pnegate< Packet32h >(), pnmadd(), psincos_double(), psincos_float(), psincos_inner_msa_float(), psub< Packet16h >(), psub< Packet8h >(), Eigen::internal::compute_inverse_size4< Architecture::Target, float, MatrixType, ResultType >::run(), Eigen::internal::compute_inverse_size4< Architecture::Target, double, MatrixType, ResultType >::run(), Eigen::internal::quat_product< Architecture::Target, Derived, OtherDerived, float >::run(), Eigen::internal::generic_i0e< T, float >::run(), Eigen::internal::generic_i0e< T, double >::run(), Eigen::internal::generic_i1e< T, float >::run(), Eigen::internal::generic_i1e< T, double >::run(), Eigen::internal::generic_k0e< T, float >::run(), Eigen::internal::generic_k0e< T, double >::run(), Eigen::internal::generic_k0< T, float >::run(), Eigen::internal::generic_k0< T, double >::run(), Eigen::internal::generic_k1e< T, float >::run(), Eigen::internal::generic_k1e< T, double >::run(), Eigen::internal::generic_k1< T, float >::run(), Eigen::internal::generic_k1< T, double >::run(), Eigen::internal::generic_j0< T, float >::run(), Eigen::internal::generic_j0< T, double >::run(), Eigen::internal::generic_y0< T, float >::run(), Eigen::internal::generic_j1< T, float >::run(), Eigen::internal::generic_j1< T, double >::run(), Eigen::internal::generic_y1< T, float >::run(), Eigen::internal::generic_fast_erf< Scalar >::run(), Eigen::internal::cross3_impl< Architecture::Target, VectorLhs, VectorRhs, float, true >::run(), Eigen::internal::pchebevl< Packet, N >::run(), Eigen::internal::apply_rotation_in_the_plane_selector< Scalar, OtherScalar, SizeAtCompileTime, MinAlignment, true >::run(), and veltkamp_splitting().

psub< Packet16b >()

template<>

EIGEN_STRONG_INLINE Packet16b Eigen::internal::psub< Packet16b >	(	const Packet16b &	a,
		const Packet16b &	b
	)

                                                                                      {
  return _mm_xor_si128(a, b);
}

References a, and b.

psub< Packet16bf >()

template<>

EIGEN_STRONG_INLINE Packet16bf Eigen::internal::psub< Packet16bf >	(	const Packet16bf &	a,
		const Packet16bf &	b
	)

                                                                                          {
  return F32ToBf16(psub<Packet16f>(Bf16ToF32(a), Bf16ToF32(b)));
}

References a, b, Bf16ToF32(), F32ToBf16(), and psub< Packet16f >().

psub< Packet16c >()

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::psub< Packet16c >	(	const Packet16c &	a,
		const Packet16c &	b
	)

                                                                                      {
  return a - b;
}

References a, and b.

psub< Packet16f >()

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::psub< Packet16f >	(	const Packet16f &	a,
		const Packet16f &	b
	)

                                                                                      {
  return _mm512_sub_ps(a, b);
}

References a, and b.

Referenced by psub< Packet16bf >().

psub< Packet16h >()

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::psub< Packet16h >	(	const Packet16h &	a,
		const Packet16h &	b
	)

                                                                                      {
  Packet16f af = half2float(a);
  Packet16f bf = half2float(b);
  Packet16f rf = psub(af, bf);
  return float2half(rf);
}

References a, b, float2half(), half2float(), and psub().

psub< Packet16i >()

template<>

EIGEN_STRONG_INLINE Packet16i Eigen::internal::psub< Packet16i >	(	const Packet16i &	a,
		const Packet16i &	b
	)

                                                                                      {
  return _mm512_sub_epi32(a, b);
}

References a, and b.

psub< Packet16uc >()

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::psub< Packet16uc >	(	const Packet16uc &	a,
		const Packet16uc &	b
	)

                                                                                          {
  return a - b;
}

References a, and b.

psub< Packet1cd >()

template<>

EIGEN_STRONG_INLINE Packet1cd Eigen::internal::psub< Packet1cd >	(	const Packet1cd &	a,
		const Packet1cd &	b
	)

                                                                                      {
  return Packet1cd(__lsx_vfsub_d(a.v, b.v));
}

References a, and b.

psub< Packet1cf >()

template<>

EIGEN_STRONG_INLINE Packet1cf Eigen::internal::psub< Packet1cf >	(	const Packet1cf &	a,
		const Packet1cf &	b
	)

                                                                                      {
  return Packet1cf(psub<Packet2f>(a.v, b.v));
}

References a, b, and psub< Packet2f >().

psub< Packet2cd >()

template<>

EIGEN_STRONG_INLINE Packet2cd Eigen::internal::psub< Packet2cd >	(	const Packet2cd &	a,
		const Packet2cd &	b
	)

                                                                                      {
  return Packet2cd(_mm256_sub_pd(a.v, b.v));
}

References a, and b.

psub< Packet2cf >()

template<>

EIGEN_STRONG_INLINE Packet2cf Eigen::internal::psub< Packet2cf >	(	const Packet2cf &	a,
		const Packet2cf &	b
	)

                                                                                      {
  return Packet2cf(a.v - b.v);
}

References a, and b.

psub< Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::psub< Packet2d >	(	const Packet2d &	a,
		const Packet2d &	b
	)

                                                                                  {
  return __lsx_vfsub_d(a, b);
}

References a, and b.

psub< Packet2f >()

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::psub< Packet2f >	(	const Packet2f &	a,
		const Packet2f &	b
	)

                                                                                  {
  return vsub_f32(a, b);
}

References a, and b.

Referenced by psub< Packet1cf >().

psub< Packet2i >()

template<>

EIGEN_STRONG_INLINE Packet2i Eigen::internal::psub< Packet2i >	(	const Packet2i &	a,
		const Packet2i &	b
	)

                                                                                  {
  return vsub_s32(a, b);
}

References a, and b.

psub< Packet2l >()

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::psub< Packet2l >	(	const Packet2l &	a,
		const Packet2l &	b
	)

                                                                                  {
  return __lsx_vsub_d(a, b);
}

References a, and b.

psub< Packet2ui >()

template<>

EIGEN_STRONG_INLINE Packet2ui Eigen::internal::psub< Packet2ui >	(	const Packet2ui &	a,
		const Packet2ui &	b
	)

                                                                                      {
  return vsub_u32(a, b);
}

References a, and b.

psub< Packet2ul >()

template<>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::psub< Packet2ul >	(	const Packet2ul &	a,
		const Packet2ul &	b
	)

                                                                                      {
  return __lsx_vsub_d(a, b);
}

References a, and b.

psub< Packet32h >()

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::psub< Packet32h >	(	const Packet32h &	a,
		const Packet32h &	b
	)

                                                                                      {
  return _mm512_sub_ph(a, b);
}

References a, and b.

psub< Packet4bf >()

template<>

EIGEN_STRONG_INLINE Packet4bf Eigen::internal::psub< Packet4bf >	(	const Packet4bf &	a,
		const Packet4bf &	b
	)

                                                                                      {
  return F32ToBf16(psub<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
}

References a, b, Bf16ToF32(), F32ToBf16(), and psub< Packet4f >().

psub< Packet4c >()

template<>

EIGEN_STRONG_INLINE Packet4c Eigen::internal::psub< Packet4c >	(	const Packet4c &	a,
		const Packet4c &	b
	)

                                                                                  {
  return vget_lane_s32(
      vreinterpret_s32_s8(vsub_s8(vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
}

References a, and b.

psub< Packet4cd >()

template<>

EIGEN_STRONG_INLINE Packet4cd Eigen::internal::psub< Packet4cd >	(	const Packet4cd &	a,
		const Packet4cd &	b
	)

                                                                                      {
  return Packet4cd(_mm512_sub_pd(a.v, b.v));
}

References a, and b.

psub< Packet4cf >()

template<>

EIGEN_STRONG_INLINE Packet4cf Eigen::internal::psub< Packet4cf >	(	const Packet4cf &	a,
		const Packet4cf &	b
	)

                                                                                      {
  return Packet4cf(_mm256_sub_ps(a.v, b.v));
}

References a, and b.

psub< Packet4d >()

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::psub< Packet4d >	(	const Packet4d &	a,
		const Packet4d &	b
	)

                                                                                  {
  return _mm256_sub_pd(a, b);
}

References a, and b.

psub< Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::psub< Packet4f >	(	const Packet4f &	a,
		const Packet4f &	b
	)

                                                                                  {
  return a - b;
}

References a, and b.

Referenced by psub< Packet4bf >(), and psub< Packet8bf >().

psub< Packet4i >()

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::psub< Packet4i >	(	const Packet4i &	a,
		const Packet4i &	b
	)

                                                                                  {
  return a - b;
}

References a, and b.

psub< Packet4s >()

template<>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::psub< Packet4s >	(	const Packet4s &	a,
		const Packet4s &	b
	)

                                                                                  {
  return vsub_s16(a, b);
}

References a, and b.

psub< Packet4uc >()

template<>

EIGEN_STRONG_INLINE Packet4uc Eigen::internal::psub< Packet4uc >	(	const Packet4uc &	a,
		const Packet4uc &	b
	)

                                                                                      {
  return vget_lane_u32(
      vreinterpret_u32_u8(vsub_u8(vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
}

References a, and b.

psub< Packet4ui >()

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::psub< Packet4ui >	(	const Packet4ui &	a,
		const Packet4ui &	b
	)

                                                                                      {
  return __lsx_vsub_w(a, b);
}

References a, and b.

psub< Packet4us >()

template<>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::psub< Packet4us >	(	const Packet4us &	a,
		const Packet4us &	b
	)

                                                                                      {
  return vsub_u16(a, b);
}

References a, and b.

psub< Packet8bf >()

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::psub< Packet8bf >	(	const Packet8bf &	a,
		const Packet8bf &	b
	)

                                                                                      {
  BF16_TO_F32_BINARY_OP_WRAPPER(psub<Packet4f>, a, b);
}

References a, b, BF16_TO_F32_BINARY_OP_WRAPPER, and psub< Packet4f >().

psub< Packet8c >()

template<>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::psub< Packet8c >	(	const Packet8c &	a,
		const Packet8c &	b
	)

                                                                                  {
  return vsub_s8(a, b);
}

References a, and b.

psub< Packet8cf >()

template<>

EIGEN_STRONG_INLINE Packet8cf Eigen::internal::psub< Packet8cf >	(	const Packet8cf &	a,
		const Packet8cf &	b
	)

                                                                                      {
  return Packet8cf(_mm512_sub_ps(a.v, b.v));
}

References a, and b.

psub< Packet8d >()

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::psub< Packet8d >	(	const Packet8d &	a,
		const Packet8d &	b
	)

                                                                                  {
  return _mm512_sub_pd(a, b);
}

References a, and b.

psub< Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::psub< Packet8f >	(	const Packet8f &	a,
		const Packet8f &	b
	)

                                                                                  {
  return _mm256_sub_ps(a, b);
}

References a, and b.

psub< Packet8h >()

template<>

EIGEN_STRONG_INLINE Packet8h Eigen::internal::psub< Packet8h >	(	const Packet8h &	a,
		const Packet8h &	b
	)

                                                                                  {
  Packet8f af = half2float(a);
  Packet8f bf = half2float(b);
  Packet8f rf = psub(af, bf);
  return float2half(rf);
}

References a, b, float2half(), half2float(), and psub().

psub< Packet8i >()

template<>

EIGEN_STRONG_INLINE Packet8i Eigen::internal::psub< Packet8i >	(	const Packet8i &	a,
		const Packet8i &	b
	)

                                                                                  {
#ifdef EIGEN_VECTORIZE_AVX2
  return _mm256_sub_epi32(a, b);
#else
  __m128i lo = _mm_sub_epi32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0));
  __m128i hi = _mm_sub_epi32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1));
  return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1);
#endif
}

References a, and b.

psub< Packet8l >()

template<>

EIGEN_STRONG_INLINE Packet8l Eigen::internal::psub< Packet8l >	(	const Packet8l &	a,
		const Packet8l &	b
	)

                                                                                  {
  return _mm512_sub_epi64(a, b);
}

References a, and b.

psub< Packet8s >()

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::psub< Packet8s >	(	const Packet8s &	a,
		const Packet8s &	b
	)

                                                                                  {
  return a - b;
}

References a, and b.

psub< Packet8uc >()

template<>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::psub< Packet8uc >	(	const Packet8uc &	a,
		const Packet8uc &	b
	)

                                                                                      {
  return vsub_u8(a, b);
}

References a, and b.

psub< Packet8ui >()

template<>

EIGEN_STRONG_INLINE Packet8ui Eigen::internal::psub< Packet8ui >	(	const Packet8ui &	a,
		const Packet8ui &	b
	)

                                                                                      {
#ifdef EIGEN_VECTORIZE_AVX2
  return _mm256_sub_epi32(a, b);
#else
  __m128i lo = _mm_sub_epi32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0));
  __m128i hi = _mm_sub_epi32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1));
  return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1);
#endif
}

References a, and b.

psub< Packet8us >()

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::psub< Packet8us >	(	const Packet8us &	a,
		const Packet8us &	b
	)

                                                                                      {
  return a - b;
}

References a, and b.

Referenced by F32ToBf16Two().

psub< PacketXf >()

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::psub< PacketXf >	(	const PacketXf &	a,
		const PacketXf &	b
	)

                                                                                  {
  return svsub_f32_x(svptrue_b32(), a, b);
}

References a, and b.

psub< PacketXi >()

template<>

EIGEN_STRONG_INLINE PacketXi Eigen::internal::psub< PacketXi >	(	const PacketXi &	a,
		const PacketXi &	b
	)

                                                                                  {
  return svsub_s32_x(svptrue_b32(), a, b);
}

References a, and b.

ptan()

template<typename Packet >

EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::ptan

(

const Packet &

a

)

Returns

the tan of a (coeff-wise)

                                                                                  {
  EIGEN_USING_STD(tan);
  return tan(a);
}

References a, EIGEN_USING_STD, and Eigen::bfloat16_impl::tan().

Referenced by packetmath_real(), and Eigen::internal::scalar_tan_op< Scalar >::packetOp().

ptanh()

template<typename Packet >

EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::ptanh

(

const Packet &

a

)

Returns

the hyperbolic tan of a (coeff-wise)

                                                                                   {
  EIGEN_USING_STD(tanh);
  return tanh(a);
}

References a, EIGEN_USING_STD, and Eigen::bfloat16_impl::tanh().

Referenced by packetmath_real(), Eigen::internal::scalar_tanh_op< Scalar >::packetOp(), and ptanh< Packet32h >().

ptanh< Packet16h >()

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::ptanh< Packet16h >

(

const Packet16h &

)

ptanh< Packet32h >()

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::ptanh< Packet32h >

(

const Packet32h &

a

)

                                                                   {
  Packet16h low;
  Packet16h high;
  extract2Packet16h(a, low, high);
 
  Packet16h lowOut = ptanh(low);
  Packet16h highOut = ptanh(high);
 
  return combine2Packet16h(lowOut, highOut);
}

References a, combine2Packet16h(), extract2Packet16h(), and ptanh().

ptanh< Packet4f >()

template<>

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet4f Eigen::internal::ptanh< Packet4f >

(

const Packet4f &

_x

)

                                                                                                 {
  static EIGEN_DECLARE_CONST_Packet4f(tanh_tiny, 1e-4f);
  static EIGEN_DECLARE_CONST_Packet4f(tanh_hi, 9.0f);
  // The monomial coefficients of the numerator polynomial (odd).
  static EIGEN_DECLARE_CONST_Packet4f(alpha_1, 4.89352455891786e-3f);
  static EIGEN_DECLARE_CONST_Packet4f(alpha_3, 6.37261928875436e-4f);
  static EIGEN_DECLARE_CONST_Packet4f(alpha_5, 1.48572235717979e-5f);
  static EIGEN_DECLARE_CONST_Packet4f(alpha_7, 5.12229709037114e-8f);
  static EIGEN_DECLARE_CONST_Packet4f(alpha_9, -8.60467152213735e-11f);
  static EIGEN_DECLARE_CONST_Packet4f(alpha_11, 2.00018790482477e-13f);
  static EIGEN_DECLARE_CONST_Packet4f(alpha_13, -2.76076847742355e-16f);
  // The monomial coefficients of the denominator polynomial (even).
  static EIGEN_DECLARE_CONST_Packet4f(beta_0, 4.89352518554385e-3f);
  static EIGEN_DECLARE_CONST_Packet4f(beta_2, 2.26843463243900e-3f);
  static EIGEN_DECLARE_CONST_Packet4f(beta_4, 1.18534705686654e-4f);
  static EIGEN_DECLARE_CONST_Packet4f(beta_6, 1.19825839466702e-6f);
 
  Packet4f x = pabs(_x);
  Packet4i tiny_mask = __builtin_msa_fclt_w(x, p4f_tanh_tiny);
 
  // Clamp the inputs to the range [-9, 9] since anything outside
  // this range is -/+1.0f in single-precision.
  x = (Packet4f)__builtin_msa_bsel_v((v16u8)__builtin_msa_fclt_w(p4f_tanh_hi, x), (v16u8)x, (v16u8)p4f_tanh_hi);
 
  // Since the polynomials are odd/even, we need x**2.
  Packet4f x2 = pmul(x, x);
 
  // Evaluate the numerator polynomial p.
  Packet4f p = pmadd(x2, p4f_alpha_13, p4f_alpha_11);
  p = pmadd(x2, p, p4f_alpha_9);
  p = pmadd(x2, p, p4f_alpha_7);
  p = pmadd(x2, p, p4f_alpha_5);
  p = pmadd(x2, p, p4f_alpha_3);
  p = pmadd(x2, p, p4f_alpha_1);
  p = pmul(x, p);
 
  // Evaluate the denominator polynomial q.
  Packet4f q = pmadd(x2, p4f_beta_6, p4f_beta_4);
  q = pmadd(x2, q, p4f_beta_2);
  q = pmadd(x2, q, p4f_beta_0);
 
  // Divide the numerator by the denominator.
  p = pdiv(p, q);
 
  // Reinstate the sign.
  p = (Packet4f)__builtin_msa_binsli_w((v4u32)p, (v4u32)_x, 0);
 
  // When the argument is very small in magnitude it's more accurate to just return it.
  p = (Packet4f)__builtin_msa_bsel_v((v16u8)tiny_mask, (v16u8)p, (v16u8)_x);
 
  return p;
}

References EIGEN_DECLARE_CONST_Packet4f(), p, pabs(), pdiv(), pmadd(), pmul(), Eigen::numext::q, plotDoE::x, and Global_parameters::x2().

ptanh< PacketXf >()

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::ptanh< PacketXf >

(

const PacketXf &

x

)

                                                                {
  return ptanh_float(x);
}

References ptanh_float(), and plotDoE::x.

ptanh_double() [1/2]

template<typename Packet >

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::ptanh_double

(

const Packet &

x

)

Returns

tanh(x) for double precision float

ptanh_double() [2/2]

template<typename T >

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS T Eigen::internal::ptanh_double

(

const T &

a_x

)

Returns

the hyperbolic tan of a (coeff-wise) This uses a 19/18-degree rational interpolant which is accurate up to a couple of ulps in the (approximate) range [-18.7, 18.7], outside of which tanh(x) = +/-1 in single precision. The input is clamped to the range [-c, c]. The value c is chosen as the smallest value where the approximation evaluates to exactly 1.

This implementation works on both scalars and packets.

                                                                                 {
  // Clamp the inputs to the range [-c, c] and set everything
  // outside that range to 1.0. The value c is chosen as the smallest
  // floating point argument such that the approximation is exactly 1.
  // This saves clamping the value at the end.
#ifdef EIGEN_VECTORIZE_FMA
  const T plus_clamp = pset1<T>(17.6610191624600077);
  const T minus_clamp = pset1<T>(-17.6610191624600077);
#else
  const T plus_clamp = pset1<T>(17.714196154005176);
  const T minus_clamp = pset1<T>(-17.714196154005176);
#endif
  const T x = pmax(pmin(a_x, plus_clamp), minus_clamp);
 
  // The following rational approximation was generated by rminimax
  // (https://gitlab.inria.fr/sfilip/rminimax) using the following
  // command:
  // $ ./ratapprox --function="tanh(x)" --dom='[-18.72,18.72]'
  //   --num="odd" --den="even" --type="[19,18]" --numF="[D]"
  //   --denF="[D]" --log --output=tanh.sollya --dispCoeff="dec"
 
  // The monomial coefficients of the numerator polynomial (odd).
  constexpr double alpha[] = {2.6158007860482230e-23, 7.6534862268749319e-19, 3.1309488231386680e-15,
                              4.2303918148209176e-12, 2.4618379131293676e-09, 6.8644367682497074e-07,
                              9.3839087674268880e-05, 5.9809711724441161e-03, 1.5184719640284322e-01};
 
  // The monomial coefficients of the denominator polynomial (even).
  constexpr double beta[] = {6.463747022670968018e-21, 5.782506856739003571e-17,
                             1.293019623712687916e-13, 1.123643448069621992e-10,
                             4.492975677839633985e-08, 8.785185266237658698e-06,
                             8.295161192716231542e-04, 3.437448108450402717e-02,
                             4.851805297361760360e-01, 1.0};
 
  // Since the polynomials are odd/even, we need x^2.
  const T x2 = pmul(x, x);
  const T x3 = pmul(x2, x);
 
  // Interleave the evaluation of the numerator polynomial p and
  // denominator polynomial q.
  T p = ppolevl<T, 8>::run(x2, alpha);
  T q = ppolevl<T, 9>::run(x2, beta);
  // Take advantage of the fact that the constant term in p is 1 to compute
  // x*(x^2*p + 1) = x^3 * p + x.
  p = pmadd(x3, p, x);
 
  // Divide the numerator by the denominator.
  return pdiv(p, q);
}

References alpha, beta, p, pdiv(), pmadd(), pmax(), pmin(), pmul(), Eigen::numext::q, Eigen::internal::ppolevl< Packet, N >::run(), plotDoE::x, and Global_parameters::x2().

ptanh_float() [1/2]

template<typename Packet >

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::ptanh_float

(

const Packet &

x

)

Returns

tanh(x) for single precision float

ptanh_float() [2/2]

template<typename T >

EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS T Eigen::internal::ptanh_float

(

const T &

a_x

)

Returns

the hyperbolic tan of a (coeff-wise) Doesn't do anything fancy, just a 9/8-degree rational interpolant which is accurate up to a couple of ulps in the (approximate) range [-8, 8], outside of which tanh(x) = +/-1 in single precision. The input is clamped to the range [-c, c]. The value c is chosen as the smallest value where the approximation evaluates to exactly 1.

This implementation works on both scalars and packets.

                                                                                {
  // Clamp the inputs to the range [-c, c] and set everything
  // outside that range to 1.0. The value c is chosen as the smallest
  // floating point argument such that the approximation is exactly 1.
  // This saves clamping the value at the end.
#ifdef EIGEN_VECTORIZE_FMA
  const T plus_clamp = pset1<T>(8.01773357391357422f);
  const T minus_clamp = pset1<T>(-8.01773357391357422f);
#else
  const T plus_clamp = pset1<T>(7.90738964080810547f);
  const T minus_clamp = pset1<T>(-7.90738964080810547f);
#endif
  const T x = pmax(pmin(a_x, plus_clamp), minus_clamp);
 
  // The following rational approximation was generated by rminimax
  // (https://gitlab.inria.fr/sfilip/rminimax) using the following
  // command:
  // $ ratapprox --function="tanh(x)" --dom='[-8.67,8.67]' --num="odd"
  //   --den="even" --type="[9,8]" --numF="[SG]" --denF="[SG]" --log
  //   --output=tanhf.sollya --dispCoeff="dec"
 
  // The monomial coefficients of the numerator polynomial (odd).
  constexpr float alpha[] = {1.394553628e-8f, 2.102733560e-5f, 3.520756727e-3f, 1.340216100e-1f};
 
  // The monomial coefficients of the denominator polynomial (even).
  constexpr float beta[] = {8.015776984e-7f, 3.326951409e-4f, 2.597254514e-2f, 4.673548340e-1f, 1.0f};
 
  // Since the polynomials are odd/even, we need x^2.
  const T x2 = pmul(x, x);
  const T x3 = pmul(x2, x);
 
  T p = ppolevl<T, 3>::run(x2, alpha);
  T q = ppolevl<T, 4>::run(x2, beta);
  // Take advantage of the fact that the constant term in p is 1 to compute
  // x*(x^2*p + 1) = x^3 * p + x.
  p = pmadd(x3, p, x);
 
  // Divide the numerator by the denominator.
  return pdiv(p, q);
}

References alpha, beta, p, pdiv(), pmadd(), pmax(), pmin(), pmul(), Eigen::numext::q, Eigen::internal::ppolevl< Packet, N >::run(), plotDoE::x, and Global_parameters::x2().

Referenced by ptanh< PacketXf >().

ptranpose_common()

template<typename T >

EIGEN_DEVICE_FUNC void Eigen::internal::ptranpose_common ( PacketBlock< T, 4 > &  kernel )

inline

                                                                          {
  T t0, t1, t2, t3;
  t0 = vec_mergeh(kernel.packet[0], kernel.packet[2]);
  t1 = vec_mergel(kernel.packet[0], kernel.packet[2]);
  t2 = vec_mergeh(kernel.packet[1], kernel.packet[3]);
  t3 = vec_mergel(kernel.packet[1], kernel.packet[3]);
  kernel.packet[0] = vec_mergeh(t0, t2);
  kernel.packet[1] = vec_mergel(t0, t2);
  kernel.packet[2] = vec_mergeh(t1, t3);
  kernel.packet[3] = vec_mergel(t1, t3);
}

References Eigen::internal::PacketBlock< Packet, N >::packet.

ptranspose() [1/68]

template<typename Packet >

EIGEN_DEVICE_FUNC void Eigen::internal::ptranspose ( PacketBlock< Packet, 1 > &  )

inline

                                                                    {
  // Nothing to do in the scalar case, i.e. a 1x1 matrix.
}

ptranspose() [2/68]

EIGEN_STRONG_INLINE void Eigen::internal::ptranspose

(

PacketBlock< Packet16b, 16 > &

kernel

)

                                                                        {
  // If we number the elements in the input thus:
  // kernel.packet[ 0] = {00, 01, 02, 03, 04, 05, 06, 07, 08, 09, 0a, 0b, 0c, 0d, 0e, 0f}
  // kernel.packet[ 1] = {10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 1a, 1b, 1c, 1d, 1e, 1f}
  // ...
  // kernel.packet[15] = {f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, fa, fb, fc, fd, fe, ff},
  //
  // the desired output is:
  // kernel.packet[ 0] = {00, 10, 20, 30, 40, 50, 60, 70, 80, 90, a0, b0, c0, d0, e0, f0}
  // kernel.packet[ 1] = {01, 11, 21, 31, 41, 51, 61, 71, 81, 91, a1, b1, c1, d1, e1, f1}
  // ...
  // kernel.packet[15] = {0f, 1f, 2f, 3f, 4f, 5f, 6f, 7f, 8f, 9f, af, bf, cf, df, ef, ff},
  __m128i t0 =
      _mm_unpacklo_epi8(kernel.packet[0], kernel.packet[1]);  // 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17
  __m128i t1 =
      _mm_unpackhi_epi8(kernel.packet[0], kernel.packet[1]);  // 08 18 09 19 0a 1a 0b 1b 0c 1c 0d 1d 0e 1e 0f 1f
  __m128i t2 =
      _mm_unpacklo_epi8(kernel.packet[2], kernel.packet[3]);  // 20 30 21 31 22 32 ...                     27 37
  __m128i t3 =
      _mm_unpackhi_epi8(kernel.packet[2], kernel.packet[3]);  // 28 38 29 39 2a 3a ...                     2f 3f
  __m128i t4 =
      _mm_unpacklo_epi8(kernel.packet[4], kernel.packet[5]);  // 40 50 41 51 42 52                         47 57
  __m128i t5 = _mm_unpackhi_epi8(kernel.packet[4], kernel.packet[5]);  // 48 58 49 59 4a 5a
  __m128i t6 = _mm_unpacklo_epi8(kernel.packet[6], kernel.packet[7]);
  __m128i t7 = _mm_unpackhi_epi8(kernel.packet[6], kernel.packet[7]);
  __m128i t8 = _mm_unpacklo_epi8(kernel.packet[8], kernel.packet[9]);
  __m128i t9 = _mm_unpackhi_epi8(kernel.packet[8], kernel.packet[9]);
  __m128i ta = _mm_unpacklo_epi8(kernel.packet[10], kernel.packet[11]);
  __m128i tb = _mm_unpackhi_epi8(kernel.packet[10], kernel.packet[11]);
  __m128i tc = _mm_unpacklo_epi8(kernel.packet[12], kernel.packet[13]);
  __m128i td = _mm_unpackhi_epi8(kernel.packet[12], kernel.packet[13]);
  __m128i te = _mm_unpacklo_epi8(kernel.packet[14], kernel.packet[15]);
  __m128i tf = _mm_unpackhi_epi8(kernel.packet[14], kernel.packet[15]);
 
  __m128i s0 = _mm_unpacklo_epi16(t0, t2);  // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
  __m128i s1 = _mm_unpackhi_epi16(t0, t2);  // 04 14 24 34
  __m128i s2 = _mm_unpacklo_epi16(t1, t3);  // 08 18 28 38 ...
  __m128i s3 = _mm_unpackhi_epi16(t1, t3);  // 0c 1c 2c 3c ...
  __m128i s4 = _mm_unpacklo_epi16(t4, t6);  // 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73
  __m128i s5 = _mm_unpackhi_epi16(t4, t6);  // 44 54 64 74 ...
  __m128i s6 = _mm_unpacklo_epi16(t5, t7);
  __m128i s7 = _mm_unpackhi_epi16(t5, t7);
  __m128i s8 = _mm_unpacklo_epi16(t8, ta);
  __m128i s9 = _mm_unpackhi_epi16(t8, ta);
  __m128i sa = _mm_unpacklo_epi16(t9, tb);
  __m128i sb = _mm_unpackhi_epi16(t9, tb);
  __m128i sc = _mm_unpacklo_epi16(tc, te);
  __m128i sd = _mm_unpackhi_epi16(tc, te);
  __m128i se = _mm_unpacklo_epi16(td, tf);
  __m128i sf = _mm_unpackhi_epi16(td, tf);
 
  __m128i u0 = _mm_unpacklo_epi32(s0, s4);  // 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71
  __m128i u1 = _mm_unpackhi_epi32(s0, s4);  // 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73
  __m128i u2 = _mm_unpacklo_epi32(s1, s5);
  __m128i u3 = _mm_unpackhi_epi32(s1, s5);
  __m128i u4 = _mm_unpacklo_epi32(s2, s6);
  __m128i u5 = _mm_unpackhi_epi32(s2, s6);
  __m128i u6 = _mm_unpacklo_epi32(s3, s7);
  __m128i u7 = _mm_unpackhi_epi32(s3, s7);
  __m128i u8 = _mm_unpacklo_epi32(s8, sc);
  __m128i u9 = _mm_unpackhi_epi32(s8, sc);
  __m128i ua = _mm_unpacklo_epi32(s9, sd);
  __m128i ub = _mm_unpackhi_epi32(s9, sd);
  __m128i uc = _mm_unpacklo_epi32(sa, se);
  __m128i ud = _mm_unpackhi_epi32(sa, se);
  __m128i ue = _mm_unpacklo_epi32(sb, sf);
  __m128i uf = _mm_unpackhi_epi32(sb, sf);
 
  kernel.packet[0] = _mm_unpacklo_epi64(u0, u8);
  kernel.packet[1] = _mm_unpackhi_epi64(u0, u8);
  kernel.packet[2] = _mm_unpacklo_epi64(u1, u9);
  kernel.packet[3] = _mm_unpackhi_epi64(u1, u9);
  kernel.packet[4] = _mm_unpacklo_epi64(u2, ua);
  kernel.packet[5] = _mm_unpackhi_epi64(u2, ua);
  kernel.packet[6] = _mm_unpacklo_epi64(u3, ub);
  kernel.packet[7] = _mm_unpackhi_epi64(u3, ub);
  kernel.packet[8] = _mm_unpacklo_epi64(u4, uc);
  kernel.packet[9] = _mm_unpackhi_epi64(u4, uc);
  kernel.packet[10] = _mm_unpacklo_epi64(u5, ud);
  kernel.packet[11] = _mm_unpackhi_epi64(u5, ud);
  kernel.packet[12] = _mm_unpacklo_epi64(u6, ue);
  kernel.packet[13] = _mm_unpackhi_epi64(u6, ue);
  kernel.packet[14] = _mm_unpacklo_epi64(u7, uf);
  kernel.packet[15] = _mm_unpackhi_epi64(u7, uf);
}

References Eigen::internal::PacketBlock< Packet, N >::packet.

ptranspose() [3/68]

EIGEN_STRONG_INLINE void Eigen::internal::ptranspose

(

PacketBlock< Packet16b, 4 > &

kernel

)

                                                                       {
  __m128i T0 = _mm_unpacklo_epi8(kernel.packet[0], kernel.packet[1]);
  __m128i T1 = _mm_unpackhi_epi8(kernel.packet[0], kernel.packet[1]);
  __m128i T2 = _mm_unpacklo_epi8(kernel.packet[2], kernel.packet[3]);
  __m128i T3 = _mm_unpackhi_epi8(kernel.packet[2], kernel.packet[3]);
  kernel.packet[0] = _mm_unpacklo_epi16(T0, T2);
  kernel.packet[1] = _mm_unpackhi_epi16(T0, T2);
  kernel.packet[2] = _mm_unpacklo_epi16(T1, T3);
  kernel.packet[3] = _mm_unpackhi_epi16(T1, T3);
}

References Eigen::internal::PacketBlock< Packet, N >::packet.

ptranspose() [4/68]

EIGEN_STRONG_INLINE void Eigen::internal::ptranspose

(

PacketBlock< Packet16bf, 16 > &

kernel

)

                                                                         {
  __m256i a = kernel.packet[0];
  __m256i b = kernel.packet[1];
  __m256i c = kernel.packet[2];
  __m256i d = kernel.packet[3];
  __m256i e = kernel.packet[4];
  __m256i f = kernel.packet[5];
  __m256i g = kernel.packet[6];
  __m256i h = kernel.packet[7];
  __m256i i = kernel.packet[8];
  __m256i j = kernel.packet[9];
  __m256i k = kernel.packet[10];
  __m256i l = kernel.packet[11];
  __m256i m = kernel.packet[12];
  __m256i n = kernel.packet[13];
  __m256i o = kernel.packet[14];
  __m256i p = kernel.packet[15];
 
  __m256i ab_07 = _mm256_unpacklo_epi16(a, b);
  __m256i cd_07 = _mm256_unpacklo_epi16(c, d);
  __m256i ef_07 = _mm256_unpacklo_epi16(e, f);
  __m256i gh_07 = _mm256_unpacklo_epi16(g, h);
  __m256i ij_07 = _mm256_unpacklo_epi16(i, j);
  __m256i kl_07 = _mm256_unpacklo_epi16(k, l);
  __m256i mn_07 = _mm256_unpacklo_epi16(m, n);
  __m256i op_07 = _mm256_unpacklo_epi16(o, p);
 
  __m256i ab_8f = _mm256_unpackhi_epi16(a, b);
  __m256i cd_8f = _mm256_unpackhi_epi16(c, d);
  __m256i ef_8f = _mm256_unpackhi_epi16(e, f);
  __m256i gh_8f = _mm256_unpackhi_epi16(g, h);
  __m256i ij_8f = _mm256_unpackhi_epi16(i, j);
  __m256i kl_8f = _mm256_unpackhi_epi16(k, l);
  __m256i mn_8f = _mm256_unpackhi_epi16(m, n);
  __m256i op_8f = _mm256_unpackhi_epi16(o, p);
 
  __m256i abcd_03 = _mm256_unpacklo_epi32(ab_07, cd_07);
  __m256i abcd_47 = _mm256_unpackhi_epi32(ab_07, cd_07);
  __m256i efgh_03 = _mm256_unpacklo_epi32(ef_07, gh_07);
  __m256i efgh_47 = _mm256_unpackhi_epi32(ef_07, gh_07);
  __m256i ijkl_03 = _mm256_unpacklo_epi32(ij_07, kl_07);
  __m256i ijkl_47 = _mm256_unpackhi_epi32(ij_07, kl_07);
  __m256i mnop_03 = _mm256_unpacklo_epi32(mn_07, op_07);
  __m256i mnop_47 = _mm256_unpackhi_epi32(mn_07, op_07);
 
  __m256i abcd_8b = _mm256_unpacklo_epi32(ab_8f, cd_8f);
  __m256i abcd_cf = _mm256_unpackhi_epi32(ab_8f, cd_8f);
  __m256i efgh_8b = _mm256_unpacklo_epi32(ef_8f, gh_8f);
  __m256i efgh_cf = _mm256_unpackhi_epi32(ef_8f, gh_8f);
  __m256i ijkl_8b = _mm256_unpacklo_epi32(ij_8f, kl_8f);
  __m256i ijkl_cf = _mm256_unpackhi_epi32(ij_8f, kl_8f);
  __m256i mnop_8b = _mm256_unpacklo_epi32(mn_8f, op_8f);
  __m256i mnop_cf = _mm256_unpackhi_epi32(mn_8f, op_8f);
 
  __m256i abcdefgh_01 = _mm256_unpacklo_epi64(abcd_03, efgh_03);
  __m256i abcdefgh_23 = _mm256_unpackhi_epi64(abcd_03, efgh_03);
  __m256i ijklmnop_01 = _mm256_unpacklo_epi64(ijkl_03, mnop_03);
  __m256i ijklmnop_23 = _mm256_unpackhi_epi64(ijkl_03, mnop_03);
  __m256i abcdefgh_45 = _mm256_unpacklo_epi64(abcd_47, efgh_47);
  __m256i abcdefgh_67 = _mm256_unpackhi_epi64(abcd_47, efgh_47);
  __m256i ijklmnop_45 = _mm256_unpacklo_epi64(ijkl_47, mnop_47);
  __m256i ijklmnop_67 = _mm256_unpackhi_epi64(ijkl_47, mnop_47);
  __m256i abcdefgh_89 = _mm256_unpacklo_epi64(abcd_8b, efgh_8b);
  __m256i abcdefgh_ab = _mm256_unpackhi_epi64(abcd_8b, efgh_8b);
  __m256i ijklmnop_89 = _mm256_unpacklo_epi64(ijkl_8b, mnop_8b);
  __m256i ijklmnop_ab = _mm256_unpackhi_epi64(ijkl_8b, mnop_8b);
  __m256i abcdefgh_cd = _mm256_unpacklo_epi64(abcd_cf, efgh_cf);
  __m256i abcdefgh_ef = _mm256_unpackhi_epi64(abcd_cf, efgh_cf);
  __m256i ijklmnop_cd = _mm256_unpacklo_epi64(ijkl_cf, mnop_cf);
  __m256i ijklmnop_ef = _mm256_unpackhi_epi64(ijkl_cf, mnop_cf);
 
  // NOTE: no unpacklo/hi instr in this case, so using permute instr.
  kernel.packet[0] = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x20);
  kernel.packet[1] = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x20);
  kernel.packet[2] = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x20);
  kernel.packet[3] = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x20);
  kernel.packet[4] = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x20);
  kernel.packet[5] = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x20);
  kernel.packet[6] = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x20);
  kernel.packet[7] = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x20);
  kernel.packet[8] = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x31);
  kernel.packet[9] = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x31);
  kernel.packet[10] = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x31);
  kernel.packet[11] = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x31);
  kernel.packet[12] = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x31);
  kernel.packet[13] = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x31);
  kernel.packet[14] = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x31);
  kernel.packet[15] = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x31);
}

References a, b, calibrate::c, e(), f(), i, j, k, m, n, p, and Eigen::internal::PacketBlock< Packet, N >::packet.

ptranspose() [5/68]

EIGEN_STRONG_INLINE void Eigen::internal::ptranspose

(

PacketBlock< Packet16bf, 4 > &

kernel

)

                                                                        {
  __m256i a = kernel.packet[0];
  __m256i b = kernel.packet[1];
  __m256i c = kernel.packet[2];
  __m256i d = kernel.packet[3];
 
  __m256i ab_07 = _mm256_unpacklo_epi16(a, b);
  __m256i cd_07 = _mm256_unpacklo_epi16(c, d);
  __m256i ab_8f = _mm256_unpackhi_epi16(a, b);
  __m256i cd_8f = _mm256_unpackhi_epi16(c, d);
 
  __m256i abcd_03 = _mm256_unpacklo_epi32(ab_07, cd_07);
  __m256i abcd_47 = _mm256_unpackhi_epi32(ab_07, cd_07);
  __m256i abcd_8b = _mm256_unpacklo_epi32(ab_8f, cd_8f);
  __m256i abcd_cf = _mm256_unpackhi_epi32(ab_8f, cd_8f);
 
  // NOTE: no unpacklo/hi instr in this case, so using permute instr.
  kernel.packet[0] = _mm256_permute2x128_si256(abcd_03, abcd_47, 0x20);
  kernel.packet[1] = _mm256_permute2x128_si256(abcd_8b, abcd_cf, 0x20);
  kernel.packet[2] = _mm256_permute2x128_si256(abcd_03, abcd_47, 0x31);
  kernel.packet[3] = _mm256_permute2x128_si256(abcd_8b, abcd_cf, 0x31);
}

References a, b, calibrate::c, and Eigen::internal::PacketBlock< Packet, N >::packet.

ptranspose() [6/68]

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::ptranspose ( PacketBlock< Packet16c, 16 > &  kernel )

inline

                                                                             {
  Packet16c step1[16], step2[16], step3[16];
 
  step1[0] = vec_mergeh(kernel.packet[0], kernel.packet[8]);
  step1[1] = vec_mergel(kernel.packet[0], kernel.packet[8]);
  step1[2] = vec_mergeh(kernel.packet[1], kernel.packet[9]);
  step1[3] = vec_mergel(kernel.packet[1], kernel.packet[9]);
  step1[4] = vec_mergeh(kernel.packet[2], kernel.packet[10]);
  step1[5] = vec_mergel(kernel.packet[2], kernel.packet[10]);
  step1[6] = vec_mergeh(kernel.packet[3], kernel.packet[11]);
  step1[7] = vec_mergel(kernel.packet[3], kernel.packet[11]);
  step1[8] = vec_mergeh(kernel.packet[4], kernel.packet[12]);
  step1[9] = vec_mergel(kernel.packet[4], kernel.packet[12]);
  step1[10] = vec_mergeh(kernel.packet[5], kernel.packet[13]);
  step1[11] = vec_mergel(kernel.packet[5], kernel.packet[13]);
  step1[12] = vec_mergeh(kernel.packet[6], kernel.packet[14]);
  step1[13] = vec_mergel(kernel.packet[6], kernel.packet[14]);
  step1[14] = vec_mergeh(kernel.packet[7], kernel.packet[15]);
  step1[15] = vec_mergel(kernel.packet[7], kernel.packet[15]);
 
  step2[0] = vec_mergeh(step1[0], step1[8]);
  step2[1] = vec_mergel(step1[0], step1[8]);
  step2[2] = vec_mergeh(step1[1], step1[9]);
  step2[3] = vec_mergel(step1[1], step1[9]);
  step2[4] = vec_mergeh(step1[2], step1[10]);
  step2[5] = vec_mergel(step1[2], step1[10]);
  step2[6] = vec_mergeh(step1[3], step1[11]);
  step2[7] = vec_mergel(step1[3], step1[11]);
  step2[8] = vec_mergeh(step1[4], step1[12]);
  step2[9] = vec_mergel(step1[4], step1[12]);
  step2[10] = vec_mergeh(step1[5], step1[13]);
  step2[11] = vec_mergel(step1[5], step1[13]);
  step2[12] = vec_mergeh(step1[6], step1[14]);
  step2[13] = vec_mergel(step1[6], step1[14]);
  step2[14] = vec_mergeh(step1[7], step1[15]);
  step2[15] = vec_mergel(step1[7], step1[15]);
 
  step3[0] = vec_mergeh(step2[0], step2[8]);
  step3[1] = vec_mergel(step2[0], step2[8]);
  step3[2] = vec_mergeh(step2[1], step2[9]);
  step3[3] = vec_mergel(step2[1], step2[9]);
  step3[4] = vec_mergeh(step2[2], step2[10]);
  step3[5] = vec_mergel(step2[2], step2[10]);
  step3[6] = vec_mergeh(step2[3], step2[11]);
  step3[7] = vec_mergel(step2[3], step2[11]);
  step3[8] = vec_mergeh(step2[4], step2[12]);
  step3[9] = vec_mergel(step2[4], step2[12]);
  step3[10] = vec_mergeh(step2[5], step2[13]);
  step3[11] = vec_mergel(step2[5], step2[13]);
  step3[12] = vec_mergeh(step2[6], step2[14]);
  step3[13] = vec_mergel(step2[6], step2[14]);
  step3[14] = vec_mergeh(step2[7], step2[15]);
  step3[15] = vec_mergel(step2[7], step2[15]);
 
  kernel.packet[0] = vec_mergeh(step3[0], step3[8]);
  kernel.packet[1] = vec_mergel(step3[0], step3[8]);
  kernel.packet[2] = vec_mergeh(step3[1], step3[9]);
  kernel.packet[3] = vec_mergel(step3[1], step3[9]);
  kernel.packet[4] = vec_mergeh(step3[2], step3[10]);
  kernel.packet[5] = vec_mergel(step3[2], step3[10]);
  kernel.packet[6] = vec_mergeh(step3[3], step3[11]);
  kernel.packet[7] = vec_mergel(step3[3], step3[11]);
  kernel.packet[8] = vec_mergeh(step3[4], step3[12]);
  kernel.packet[9] = vec_mergel(step3[4], step3[12]);
  kernel.packet[10] = vec_mergeh(step3[5], step3[13]);
  kernel.packet[11] = vec_mergel(step3[5], step3[13]);
  kernel.packet[12] = vec_mergeh(step3[6], step3[14]);
  kernel.packet[13] = vec_mergel(step3[6], step3[14]);
  kernel.packet[14] = vec_mergeh(step3[7], step3[15]);
  kernel.packet[15] = vec_mergel(step3[7], step3[15]);
}

References Eigen::internal::PacketBlock< Packet, N >::packet.

ptranspose() [7/68]

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::ptranspose ( PacketBlock< Packet16c, 4 > &  kernel )

inline

                                                                            {
  Packet16c t0, t1, t2, t3;
  t0 = vec_mergeh(kernel.packet[0], kernel.packet[2]);
  t1 = vec_mergel(kernel.packet[0], kernel.packet[2]);
  t2 = vec_mergeh(kernel.packet[1], kernel.packet[3]);
  t3 = vec_mergel(kernel.packet[1], kernel.packet[3]);
  kernel.packet[0] = vec_mergeh(t0, t2);
  kernel.packet[1] = vec_mergel(t0, t2);
  kernel.packet[2] = vec_mergeh(t1, t3);
  kernel.packet[3] = vec_mergel(t1, t3);
}

References Eigen::internal::PacketBlock< Packet, N >::packet.

ptranspose() [8/68]

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::ptranspose

(

PacketBlock< Packet16c, 8 > &

kernel

)

                                                                                         {
  __m128i t0 = __lsx_vilvl_b(kernel.packet[1], kernel.packet[0]);
  __m128i t1 = __lsx_vilvh_b(kernel.packet[1], kernel.packet[0]);
  __m128i t2 = __lsx_vilvl_b(kernel.packet[3], kernel.packet[2]);
  __m128i t3 = __lsx_vilvh_b(kernel.packet[3], kernel.packet[2]);
  __m128i t4 = __lsx_vilvl_b(kernel.packet[5], kernel.packet[4]);
  __m128i t5 = __lsx_vilvh_b(kernel.packet[5], kernel.packet[4]);
  __m128i t6 = __lsx_vilvl_b(kernel.packet[7], kernel.packet[6]);
  __m128i t7 = __lsx_vilvh_b(kernel.packet[7], kernel.packet[6]);
 
  __m128i s0 = __lsx_vilvl_h(t2, t0);
  __m128i s1 = __lsx_vilvh_h(t2, t0);
  __m128i s2 = __lsx_vilvl_h(t3, t1);
  __m128i s3 = __lsx_vilvh_h(t3, t1);
  __m128i s4 = __lsx_vilvl_h(t6, t4);
  __m128i s5 = __lsx_vilvh_h(t6, t4);
  __m128i s6 = __lsx_vilvl_h(t7, t5);
  __m128i s7 = __lsx_vilvh_h(t7, t5);
 
  kernel.packet[0] = __lsx_vilvl_w(s4, s0);
  kernel.packet[1] = __lsx_vilvh_w(s4, s0);
  kernel.packet[2] = __lsx_vilvl_w(s5, s1);
  kernel.packet[3] = __lsx_vilvh_w(s5, s1);
  kernel.packet[4] = __lsx_vilvl_w(s6, s2);
  kernel.packet[5] = __lsx_vilvh_w(s6, s2);
  kernel.packet[6] = __lsx_vilvl_w(s7, s3);
  kernel.packet[7] = __lsx_vilvh_w(s7, s3);
}

References Eigen::internal::PacketBlock< Packet, N >::packet.

ptranspose() [9/68]

EIGEN_DEVICE_FUNC void Eigen::internal::ptranspose ( PacketBlock< Packet16f, 16 > &  kernel )

inline

                                                                             {
  __m512 T0 = _mm512_unpacklo_ps(kernel.packet[0], kernel.packet[1]);
  __m512 T1 = _mm512_unpackhi_ps(kernel.packet[0], kernel.packet[1]);
  __m512 T2 = _mm512_unpacklo_ps(kernel.packet[2], kernel.packet[3]);
  __m512 T3 = _mm512_unpackhi_ps(kernel.packet[2], kernel.packet[3]);
  __m512 T4 = _mm512_unpacklo_ps(kernel.packet[4], kernel.packet[5]);
  __m512 T5 = _mm512_unpackhi_ps(kernel.packet[4], kernel.packet[5]);
  __m512 T6 = _mm512_unpacklo_ps(kernel.packet[6], kernel.packet[7]);
  __m512 T7 = _mm512_unpackhi_ps(kernel.packet[6], kernel.packet[7]);
  __m512 T8 = _mm512_unpacklo_ps(kernel.packet[8], kernel.packet[9]);
  __m512 T9 = _mm512_unpackhi_ps(kernel.packet[8], kernel.packet[9]);
  __m512 T10 = _mm512_unpacklo_ps(kernel.packet[10], kernel.packet[11]);
  __m512 T11 = _mm512_unpackhi_ps(kernel.packet[10], kernel.packet[11]);
  __m512 T12 = _mm512_unpacklo_ps(kernel.packet[12], kernel.packet[13]);
  __m512 T13 = _mm512_unpackhi_ps(kernel.packet[12], kernel.packet[13]);
  __m512 T14 = _mm512_unpacklo_ps(kernel.packet[14], kernel.packet[15]);
  __m512 T15 = _mm512_unpackhi_ps(kernel.packet[14], kernel.packet[15]);
  __m512 S0 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(1, 0, 1, 0));
  __m512 S1 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(3, 2, 3, 2));
  __m512 S2 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(1, 0, 1, 0));
  __m512 S3 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(3, 2, 3, 2));
  __m512 S4 = _mm512_shuffle_ps(T4, T6, _MM_SHUFFLE(1, 0, 1, 0));
  __m512 S5 = _mm512_shuffle_ps(T4, T6, _MM_SHUFFLE(3, 2, 3, 2));
  __m512 S6 = _mm512_shuffle_ps(T5, T7, _MM_SHUFFLE(1, 0, 1, 0));
  __m512 S7 = _mm512_shuffle_ps(T5, T7, _MM_SHUFFLE(3, 2, 3, 2));
  __m512 S8 = _mm512_shuffle_ps(T8, T10, _MM_SHUFFLE(1, 0, 1, 0));
  __m512 S9 = _mm512_shuffle_ps(T8, T10, _MM_SHUFFLE(3, 2, 3, 2));
  __m512 S10 = _mm512_shuffle_ps(T9, T11, _MM_SHUFFLE(1, 0, 1, 0));
  __m512 S11 = _mm512_shuffle_ps(T9, T11, _MM_SHUFFLE(3, 2, 3, 2));
  __m512 S12 = _mm512_shuffle_ps(T12, T14, _MM_SHUFFLE(1, 0, 1, 0));
  __m512 S13 = _mm512_shuffle_ps(T12, T14, _MM_SHUFFLE(3, 2, 3, 2));
  __m512 S14 = _mm512_shuffle_ps(T13, T15, _MM_SHUFFLE(1, 0, 1, 0));
  __m512 S15 = _mm512_shuffle_ps(T13, T15, _MM_SHUFFLE(3, 2, 3, 2));
 
  EIGEN_EXTRACT_8f_FROM_16f(S0, S0);
  EIGEN_EXTRACT_8f_FROM_16f(S1, S1);
  EIGEN_EXTRACT_8f_FROM_16f(S2, S2);
  EIGEN_EXTRACT_8f_FROM_16f(S3, S3);
  EIGEN_EXTRACT_8f_FROM_16f(S4, S4);
  EIGEN_EXTRACT_8f_FROM_16f(S5, S5);
  EIGEN_EXTRACT_8f_FROM_16f(S6, S6);
  EIGEN_EXTRACT_8f_FROM_16f(S7, S7);
  EIGEN_EXTRACT_8f_FROM_16f(S8, S8);
  EIGEN_EXTRACT_8f_FROM_16f(S9, S9);
  EIGEN_EXTRACT_8f_FROM_16f(S10, S10);
  EIGEN_EXTRACT_8f_FROM_16f(S11, S11);
  EIGEN_EXTRACT_8f_FROM_16f(S12, S12);
  EIGEN_EXTRACT_8f_FROM_16f(S13, S13);
  EIGEN_EXTRACT_8f_FROM_16f(S14, S14);
  EIGEN_EXTRACT_8f_FROM_16f(S15, S15);
 
  PacketBlock<Packet8f, 32> tmp;
 
  tmp.packet[0] = _mm256_permute2f128_ps(S0_0, S4_0, 0x20);
  tmp.packet[1] = _mm256_permute2f128_ps(S1_0, S5_0, 0x20);
  tmp.packet[2] = _mm256_permute2f128_ps(S2_0, S6_0, 0x20);
  tmp.packet[3] = _mm256_permute2f128_ps(S3_0, S7_0, 0x20);
  tmp.packet[4] = _mm256_permute2f128_ps(S0_0, S4_0, 0x31);
  tmp.packet[5] = _mm256_permute2f128_ps(S1_0, S5_0, 0x31);
  tmp.packet[6] = _mm256_permute2f128_ps(S2_0, S6_0, 0x31);
  tmp.packet[7] = _mm256_permute2f128_ps(S3_0, S7_0, 0x31);
 
  tmp.packet[8] = _mm256_permute2f128_ps(S0_1, S4_1, 0x20);
  tmp.packet[9] = _mm256_permute2f128_ps(S1_1, S5_1, 0x20);
  tmp.packet[10] = _mm256_permute2f128_ps(S2_1, S6_1, 0x20);
  tmp.packet[11] = _mm256_permute2f128_ps(S3_1, S7_1, 0x20);
  tmp.packet[12] = _mm256_permute2f128_ps(S0_1, S4_1, 0x31);
  tmp.packet[13] = _mm256_permute2f128_ps(S1_1, S5_1, 0x31);
  tmp.packet[14] = _mm256_permute2f128_ps(S2_1, S6_1, 0x31);
  tmp.packet[15] = _mm256_permute2f128_ps(S3_1, S7_1, 0x31);
 
  // Second set of _m256 outputs
  tmp.packet[16] = _mm256_permute2f128_ps(S8_0, S12_0, 0x20);
  tmp.packet[17] = _mm256_permute2f128_ps(S9_0, S13_0, 0x20);
  tmp.packet[18] = _mm256_permute2f128_ps(S10_0, S14_0, 0x20);
  tmp.packet[19] = _mm256_permute2f128_ps(S11_0, S15_0, 0x20);
  tmp.packet[20] = _mm256_permute2f128_ps(S8_0, S12_0, 0x31);
  tmp.packet[21] = _mm256_permute2f128_ps(S9_0, S13_0, 0x31);
  tmp.packet[22] = _mm256_permute2f128_ps(S10_0, S14_0, 0x31);
  tmp.packet[23] = _mm256_permute2f128_ps(S11_0, S15_0, 0x31);
 
  tmp.packet[24] = _mm256_permute2f128_ps(S8_1, S12_1, 0x20);
  tmp.packet[25] = _mm256_permute2f128_ps(S9_1, S13_1, 0x20);
  tmp.packet[26] = _mm256_permute2f128_ps(S10_1, S14_1, 0x20);
  tmp.packet[27] = _mm256_permute2f128_ps(S11_1, S15_1, 0x20);
  tmp.packet[28] = _mm256_permute2f128_ps(S8_1, S12_1, 0x31);
  tmp.packet[29] = _mm256_permute2f128_ps(S9_1, S13_1, 0x31);
  tmp.packet[30] = _mm256_permute2f128_ps(S10_1, S14_1, 0x31);
  tmp.packet[31] = _mm256_permute2f128_ps(S11_1, S15_1, 0x31);
 
  // Pack them into the output
  PACK_OUTPUT(kernel.packet, tmp.packet, 0, 16);
  PACK_OUTPUT(kernel.packet, tmp.packet, 1, 16);
  PACK_OUTPUT(kernel.packet, tmp.packet, 2, 16);
  PACK_OUTPUT(kernel.packet, tmp.packet, 3, 16);
 
  PACK_OUTPUT(kernel.packet, tmp.packet, 4, 16);
  PACK_OUTPUT(kernel.packet, tmp.packet, 5, 16);
  PACK_OUTPUT(kernel.packet, tmp.packet, 6, 16);
  PACK_OUTPUT(kernel.packet, tmp.packet, 7, 16);
 
  PACK_OUTPUT(kernel.packet, tmp.packet, 8, 16);
  PACK_OUTPUT(kernel.packet, tmp.packet, 9, 16);
  PACK_OUTPUT(kernel.packet, tmp.packet, 10, 16);
  PACK_OUTPUT(kernel.packet, tmp.packet, 11, 16);
 
  PACK_OUTPUT(kernel.packet, tmp.packet, 12, 16);
  PACK_OUTPUT(kernel.packet, tmp.packet, 13, 16);
  PACK_OUTPUT(kernel.packet, tmp.packet, 14, 16);
  PACK_OUTPUT(kernel.packet, tmp.packet, 15, 16);
}

References EIGEN_EXTRACT_8f_FROM_16f, PACK_OUTPUT, Eigen::PlainObjectBase< Derived >::packet(), Eigen::internal::PacketBlock< Packet, N >::packet, GlobalParameters::S0, GlobalParameters::S1, and tmp.

ptranspose() [10/68]

EIGEN_DEVICE_FUNC void Eigen::internal::ptranspose ( PacketBlock< Packet16f, 4 > &  kernel )

inline

                                                                            {
  __m512 T0 = _mm512_unpacklo_ps(kernel.packet[0], kernel.packet[1]);
  __m512 T1 = _mm512_unpackhi_ps(kernel.packet[0], kernel.packet[1]);
  __m512 T2 = _mm512_unpacklo_ps(kernel.packet[2], kernel.packet[3]);
  __m512 T3 = _mm512_unpackhi_ps(kernel.packet[2], kernel.packet[3]);
 
  __m512 S0 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(1, 0, 1, 0));
  __m512 S1 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(3, 2, 3, 2));
  __m512 S2 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(1, 0, 1, 0));
  __m512 S3 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(3, 2, 3, 2));
 
  EIGEN_EXTRACT_8f_FROM_16f(S0, S0);
  EIGEN_EXTRACT_8f_FROM_16f(S1, S1);
  EIGEN_EXTRACT_8f_FROM_16f(S2, S2);
  EIGEN_EXTRACT_8f_FROM_16f(S3, S3);
 
  PacketBlock<Packet8f, 8> tmp;
 
  tmp.packet[0] = _mm256_permute2f128_ps(S0_0, S1_0, 0x20);
  tmp.packet[1] = _mm256_permute2f128_ps(S2_0, S3_0, 0x20);
  tmp.packet[2] = _mm256_permute2f128_ps(S0_0, S1_0, 0x31);
  tmp.packet[3] = _mm256_permute2f128_ps(S2_0, S3_0, 0x31);
 
  tmp.packet[4] = _mm256_permute2f128_ps(S0_1, S1_1, 0x20);
  tmp.packet[5] = _mm256_permute2f128_ps(S2_1, S3_1, 0x20);
  tmp.packet[6] = _mm256_permute2f128_ps(S0_1, S1_1, 0x31);
  tmp.packet[7] = _mm256_permute2f128_ps(S2_1, S3_1, 0x31);
 
  PACK_OUTPUT_2(kernel.packet, tmp.packet, 0, 1);
  PACK_OUTPUT_2(kernel.packet, tmp.packet, 1, 1);
  PACK_OUTPUT_2(kernel.packet, tmp.packet, 2, 1);
  PACK_OUTPUT_2(kernel.packet, tmp.packet, 3, 1);
}

References EIGEN_EXTRACT_8f_FROM_16f, PACK_OUTPUT_2, Eigen::PlainObjectBase< Derived >::packet(), Eigen::internal::PacketBlock< Packet, N >::packet, GlobalParameters::S0, GlobalParameters::S1, and tmp.

ptranspose() [11/68]

EIGEN_DEVICE_FUNC void Eigen::internal::ptranspose ( PacketBlock< Packet16f, 8 > &  kernel )

inline

                                                                            {
  __m512 T0 = _mm512_unpacklo_ps(kernel.packet[0], kernel.packet[1]);
  __m512 T1 = _mm512_unpackhi_ps(kernel.packet[0], kernel.packet[1]);
  __m512 T2 = _mm512_unpacklo_ps(kernel.packet[2], kernel.packet[3]);
  __m512 T3 = _mm512_unpackhi_ps(kernel.packet[2], kernel.packet[3]);
  __m512 T4 = _mm512_unpacklo_ps(kernel.packet[4], kernel.packet[5]);
  __m512 T5 = _mm512_unpackhi_ps(kernel.packet[4], kernel.packet[5]);
  __m512 T6 = _mm512_unpacklo_ps(kernel.packet[6], kernel.packet[7]);
  __m512 T7 = _mm512_unpackhi_ps(kernel.packet[6], kernel.packet[7]);
 
  kernel.packet[0] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(T0), _mm512_castps_pd(T2)));
  kernel.packet[1] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(T0), _mm512_castps_pd(T2)));
  kernel.packet[2] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(T1), _mm512_castps_pd(T3)));
  kernel.packet[3] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(T1), _mm512_castps_pd(T3)));
  kernel.packet[4] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(T4), _mm512_castps_pd(T6)));
  kernel.packet[5] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(T4), _mm512_castps_pd(T6)));
  kernel.packet[6] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(T5), _mm512_castps_pd(T7)));
  kernel.packet[7] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(T5), _mm512_castps_pd(T7)));
 
  T0 = _mm512_shuffle_f32x4(kernel.packet[0], kernel.packet[4], 0x44);
  T1 = _mm512_shuffle_f32x4(kernel.packet[0], kernel.packet[4], 0xee);
  T2 = _mm512_shuffle_f32x4(kernel.packet[1], kernel.packet[5], 0x44);
  T3 = _mm512_shuffle_f32x4(kernel.packet[1], kernel.packet[5], 0xee);
  T4 = _mm512_shuffle_f32x4(kernel.packet[2], kernel.packet[6], 0x44);
  T5 = _mm512_shuffle_f32x4(kernel.packet[2], kernel.packet[6], 0xee);
  T6 = _mm512_shuffle_f32x4(kernel.packet[3], kernel.packet[7], 0x44);
  T7 = _mm512_shuffle_f32x4(kernel.packet[3], kernel.packet[7], 0xee);
 
  kernel.packet[0] = _mm512_shuffle_f32x4(T0, T2, 0x88);
  kernel.packet[2] = _mm512_shuffle_f32x4(T0, T2, 0xdd);
  kernel.packet[1] = _mm512_shuffle_f32x4(T4, T6, 0x88);
  kernel.packet[3] = _mm512_shuffle_f32x4(T4, T6, 0xdd);
  kernel.packet[4] = _mm512_shuffle_f32x4(T1, T3, 0x88);
  kernel.packet[6] = _mm512_shuffle_f32x4(T1, T3, 0xdd);
  kernel.packet[5] = _mm512_shuffle_f32x4(T5, T7, 0x88);
  kernel.packet[7] = _mm512_shuffle_f32x4(T5, T7, 0xdd);
}

References Eigen::internal::PacketBlock< Packet, N >::packet.

ptranspose() [12/68]

EIGEN_STRONG_INLINE void Eigen::internal::ptranspose

(

PacketBlock< Packet16h, 16 > &

kernel

)

                                                                        {
  __m256i a = kernel.packet[0];
  __m256i b = kernel.packet[1];
  __m256i c = kernel.packet[2];
  __m256i d = kernel.packet[3];
  __m256i e = kernel.packet[4];
  __m256i f = kernel.packet[5];
  __m256i g = kernel.packet[6];
  __m256i h = kernel.packet[7];
  __m256i i = kernel.packet[8];
  __m256i j = kernel.packet[9];
  __m256i k = kernel.packet[10];
  __m256i l = kernel.packet[11];
  __m256i m = kernel.packet[12];
  __m256i n = kernel.packet[13];
  __m256i o = kernel.packet[14];
  __m256i p = kernel.packet[15];
 
  __m256i ab_07 = _mm256_unpacklo_epi16(a, b);
  __m256i cd_07 = _mm256_unpacklo_epi16(c, d);
  __m256i ef_07 = _mm256_unpacklo_epi16(e, f);
  __m256i gh_07 = _mm256_unpacklo_epi16(g, h);
  __m256i ij_07 = _mm256_unpacklo_epi16(i, j);
  __m256i kl_07 = _mm256_unpacklo_epi16(k, l);
  __m256i mn_07 = _mm256_unpacklo_epi16(m, n);
  __m256i op_07 = _mm256_unpacklo_epi16(o, p);
 
  __m256i ab_8f = _mm256_unpackhi_epi16(a, b);
  __m256i cd_8f = _mm256_unpackhi_epi16(c, d);
  __m256i ef_8f = _mm256_unpackhi_epi16(e, f);
  __m256i gh_8f = _mm256_unpackhi_epi16(g, h);
  __m256i ij_8f = _mm256_unpackhi_epi16(i, j);
  __m256i kl_8f = _mm256_unpackhi_epi16(k, l);
  __m256i mn_8f = _mm256_unpackhi_epi16(m, n);
  __m256i op_8f = _mm256_unpackhi_epi16(o, p);
 
  __m256i abcd_03 = _mm256_unpacklo_epi32(ab_07, cd_07);
  __m256i abcd_47 = _mm256_unpackhi_epi32(ab_07, cd_07);
  __m256i efgh_03 = _mm256_unpacklo_epi32(ef_07, gh_07);
  __m256i efgh_47 = _mm256_unpackhi_epi32(ef_07, gh_07);
  __m256i ijkl_03 = _mm256_unpacklo_epi32(ij_07, kl_07);
  __m256i ijkl_47 = _mm256_unpackhi_epi32(ij_07, kl_07);
  __m256i mnop_03 = _mm256_unpacklo_epi32(mn_07, op_07);
  __m256i mnop_47 = _mm256_unpackhi_epi32(mn_07, op_07);
 
  __m256i abcd_8b = _mm256_unpacklo_epi32(ab_8f, cd_8f);
  __m256i abcd_cf = _mm256_unpackhi_epi32(ab_8f, cd_8f);
  __m256i efgh_8b = _mm256_unpacklo_epi32(ef_8f, gh_8f);
  __m256i efgh_cf = _mm256_unpackhi_epi32(ef_8f, gh_8f);
  __m256i ijkl_8b = _mm256_unpacklo_epi32(ij_8f, kl_8f);
  __m256i ijkl_cf = _mm256_unpackhi_epi32(ij_8f, kl_8f);
  __m256i mnop_8b = _mm256_unpacklo_epi32(mn_8f, op_8f);
  __m256i mnop_cf = _mm256_unpackhi_epi32(mn_8f, op_8f);
 
  __m256i abcdefgh_01 = _mm256_unpacklo_epi64(abcd_03, efgh_03);
  __m256i abcdefgh_23 = _mm256_unpackhi_epi64(abcd_03, efgh_03);
  __m256i ijklmnop_01 = _mm256_unpacklo_epi64(ijkl_03, mnop_03);
  __m256i ijklmnop_23 = _mm256_unpackhi_epi64(ijkl_03, mnop_03);
  __m256i abcdefgh_45 = _mm256_unpacklo_epi64(abcd_47, efgh_47);
  __m256i abcdefgh_67 = _mm256_unpackhi_epi64(abcd_47, efgh_47);
  __m256i ijklmnop_45 = _mm256_unpacklo_epi64(ijkl_47, mnop_47);
  __m256i ijklmnop_67 = _mm256_unpackhi_epi64(ijkl_47, mnop_47);
  __m256i abcdefgh_89 = _mm256_unpacklo_epi64(abcd_8b, efgh_8b);
  __m256i abcdefgh_ab = _mm256_unpackhi_epi64(abcd_8b, efgh_8b);
  __m256i ijklmnop_89 = _mm256_unpacklo_epi64(ijkl_8b, mnop_8b);
  __m256i ijklmnop_ab = _mm256_unpackhi_epi64(ijkl_8b, mnop_8b);
  __m256i abcdefgh_cd = _mm256_unpacklo_epi64(abcd_cf, efgh_cf);
  __m256i abcdefgh_ef = _mm256_unpackhi_epi64(abcd_cf, efgh_cf);
  __m256i ijklmnop_cd = _mm256_unpacklo_epi64(ijkl_cf, mnop_cf);
  __m256i ijklmnop_ef = _mm256_unpackhi_epi64(ijkl_cf, mnop_cf);
 
  // NOTE: no unpacklo/hi instr in this case, so using permute instr.
  __m256i a_p_0 = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x20);
  __m256i a_p_1 = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x20);
  __m256i a_p_2 = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x20);
  __m256i a_p_3 = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x20);
  __m256i a_p_4 = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x20);
  __m256i a_p_5 = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x20);
  __m256i a_p_6 = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x20);
  __m256i a_p_7 = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x20);
  __m256i a_p_8 = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x31);
  __m256i a_p_9 = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x31);
  __m256i a_p_a = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x31);
  __m256i a_p_b = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x31);
  __m256i a_p_c = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x31);
  __m256i a_p_d = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x31);
  __m256i a_p_e = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x31);
  __m256i a_p_f = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x31);
 
  kernel.packet[0] = a_p_0;
  kernel.packet[1] = a_p_1;
  kernel.packet[2] = a_p_2;
  kernel.packet[3] = a_p_3;
  kernel.packet[4] = a_p_4;
  kernel.packet[5] = a_p_5;
  kernel.packet[6] = a_p_6;
  kernel.packet[7] = a_p_7;
  kernel.packet[8] = a_p_8;
  kernel.packet[9] = a_p_9;
  kernel.packet[10] = a_p_a;
  kernel.packet[11] = a_p_b;
  kernel.packet[12] = a_p_c;
  kernel.packet[13] = a_p_d;
  kernel.packet[14] = a_p_e;
  kernel.packet[15] = a_p_f;
}

References a, b, calibrate::c, e(), f(), i, j, k, m, n, p, and Eigen::internal::PacketBlock< Packet, N >::packet.

ptranspose() [13/68]

EIGEN_STRONG_INLINE void Eigen::internal::ptranspose

(

PacketBlock< Packet16h, 4 > &

kernel

)

                                                                       {
  EIGEN_ALIGN64 half in[4][16];
  pstore<half>(in[0], kernel.packet[0]);
  pstore<half>(in[1], kernel.packet[1]);
  pstore<half>(in[2], kernel.packet[2]);
  pstore<half>(in[3], kernel.packet[3]);
 
  EIGEN_ALIGN64 half out[4][16];
 
  for (int i = 0; i < 4; ++i) {
    for (int j = 0; j < 4; ++j) {
      out[i][j] = in[j][4 * i];
    }
    for (int j = 0; j < 4; ++j) {
      out[i][j + 4] = in[j][4 * i + 1];
    }
    for (int j = 0; j < 4; ++j) {
      out[i][j + 8] = in[j][4 * i + 2];
    }
    for (int j = 0; j < 4; ++j) {
      out[i][j + 12] = in[j][4 * i + 3];
    }
  }
 
  kernel.packet[0] = pload<Packet16h>(out[0]);
  kernel.packet[1] = pload<Packet16h>(out[1]);
  kernel.packet[2] = pload<Packet16h>(out[2]);
  kernel.packet[3] = pload<Packet16h>(out[3]);
}

References EIGEN_ALIGN64, i, j, out(), Eigen::internal::PacketBlock< Packet, N >::packet, pload< Packet16h >(), and pstore< half >().

ptranspose() [14/68]

EIGEN_STRONG_INLINE void Eigen::internal::ptranspose

(

PacketBlock< Packet16h, 8 > &

kernel

)

                                                                       {
  EIGEN_ALIGN64 half in[8][16];
  pstore<half>(in[0], kernel.packet[0]);
  pstore<half>(in[1], kernel.packet[1]);
  pstore<half>(in[2], kernel.packet[2]);
  pstore<half>(in[3], kernel.packet[3]);
  pstore<half>(in[4], kernel.packet[4]);
  pstore<half>(in[5], kernel.packet[5]);
  pstore<half>(in[6], kernel.packet[6]);
  pstore<half>(in[7], kernel.packet[7]);
 
  EIGEN_ALIGN64 half out[8][16];
 
  for (int i = 0; i < 8; ++i) {
    for (int j = 0; j < 8; ++j) {
      out[i][j] = in[j][2 * i];
    }
    for (int j = 0; j < 8; ++j) {
      out[i][j + 8] = in[j][2 * i + 1];
    }
  }
 
  kernel.packet[0] = pload<Packet16h>(out[0]);
  kernel.packet[1] = pload<Packet16h>(out[1]);
  kernel.packet[2] = pload<Packet16h>(out[2]);
  kernel.packet[3] = pload<Packet16h>(out[3]);
  kernel.packet[4] = pload<Packet16h>(out[4]);
  kernel.packet[5] = pload<Packet16h>(out[5]);
  kernel.packet[6] = pload<Packet16h>(out[6]);
  kernel.packet[7] = pload<Packet16h>(out[7]);
}

References EIGEN_ALIGN64, i, j, out(), Eigen::internal::PacketBlock< Packet, N >::packet, pload< Packet16h >(), and pstore< half >().

ptranspose() [15/68]

EIGEN_DEVICE_FUNC void Eigen::internal::ptranspose ( PacketBlock< Packet16i, 16 > &  kernel )

inline

                                                                             {
  __m512i T0 = _mm512_unpacklo_epi32(kernel.packet[0], kernel.packet[1]);
  __m512i T1 = _mm512_unpackhi_epi32(kernel.packet[0], kernel.packet[1]);
  __m512i T2 = _mm512_unpacklo_epi32(kernel.packet[2], kernel.packet[3]);
  __m512i T3 = _mm512_unpackhi_epi32(kernel.packet[2], kernel.packet[3]);
  __m512i T4 = _mm512_unpacklo_epi32(kernel.packet[4], kernel.packet[5]);
  __m512i T5 = _mm512_unpackhi_epi32(kernel.packet[4], kernel.packet[5]);
  __m512i T6 = _mm512_unpacklo_epi32(kernel.packet[6], kernel.packet[7]);
  __m512i T7 = _mm512_unpackhi_epi32(kernel.packet[6], kernel.packet[7]);
  __m512i T8 = _mm512_unpacklo_epi32(kernel.packet[8], kernel.packet[9]);
  __m512i T9 = _mm512_unpackhi_epi32(kernel.packet[8], kernel.packet[9]);
  __m512i T10 = _mm512_unpacklo_epi32(kernel.packet[10], kernel.packet[11]);
  __m512i T11 = _mm512_unpackhi_epi32(kernel.packet[10], kernel.packet[11]);
  __m512i T12 = _mm512_unpacklo_epi32(kernel.packet[12], kernel.packet[13]);
  __m512i T13 = _mm512_unpackhi_epi32(kernel.packet[12], kernel.packet[13]);
  __m512i T14 = _mm512_unpacklo_epi32(kernel.packet[14], kernel.packet[15]);
  __m512i T15 = _mm512_unpackhi_epi32(kernel.packet[14], kernel.packet[15]);
  __m512i S0 = SHUFFLE_EPI32(T0, T2, _MM_SHUFFLE(1, 0, 1, 0));
  __m512i S1 = SHUFFLE_EPI32(T0, T2, _MM_SHUFFLE(3, 2, 3, 2));
  __m512i S2 = SHUFFLE_EPI32(T1, T3, _MM_SHUFFLE(1, 0, 1, 0));
  __m512i S3 = SHUFFLE_EPI32(T1, T3, _MM_SHUFFLE(3, 2, 3, 2));
  __m512i S4 = SHUFFLE_EPI32(T4, T6, _MM_SHUFFLE(1, 0, 1, 0));
  __m512i S5 = SHUFFLE_EPI32(T4, T6, _MM_SHUFFLE(3, 2, 3, 2));
  __m512i S6 = SHUFFLE_EPI32(T5, T7, _MM_SHUFFLE(1, 0, 1, 0));
  __m512i S7 = SHUFFLE_EPI32(T5, T7, _MM_SHUFFLE(3, 2, 3, 2));
  __m512i S8 = SHUFFLE_EPI32(T8, T10, _MM_SHUFFLE(1, 0, 1, 0));
  __m512i S9 = SHUFFLE_EPI32(T8, T10, _MM_SHUFFLE(3, 2, 3, 2));
  __m512i S10 = SHUFFLE_EPI32(T9, T11, _MM_SHUFFLE(1, 0, 1, 0));
  __m512i S11 = SHUFFLE_EPI32(T9, T11, _MM_SHUFFLE(3, 2, 3, 2));
  __m512i S12 = SHUFFLE_EPI32(T12, T14, _MM_SHUFFLE(1, 0, 1, 0));
  __m512i S13 = SHUFFLE_EPI32(T12, T14, _MM_SHUFFLE(3, 2, 3, 2));
  __m512i S14 = SHUFFLE_EPI32(T13, T15, _MM_SHUFFLE(1, 0, 1, 0));
  __m512i S15 = SHUFFLE_EPI32(T13, T15, _MM_SHUFFLE(3, 2, 3, 2));
 
  EIGEN_EXTRACT_8i_FROM_16i(S0, S0);
  EIGEN_EXTRACT_8i_FROM_16i(S1, S1);
  EIGEN_EXTRACT_8i_FROM_16i(S2, S2);
  EIGEN_EXTRACT_8i_FROM_16i(S3, S3);
  EIGEN_EXTRACT_8i_FROM_16i(S4, S4);
  EIGEN_EXTRACT_8i_FROM_16i(S5, S5);
  EIGEN_EXTRACT_8i_FROM_16i(S6, S6);
  EIGEN_EXTRACT_8i_FROM_16i(S7, S7);
  EIGEN_EXTRACT_8i_FROM_16i(S8, S8);
  EIGEN_EXTRACT_8i_FROM_16i(S9, S9);
  EIGEN_EXTRACT_8i_FROM_16i(S10, S10);
  EIGEN_EXTRACT_8i_FROM_16i(S11, S11);
  EIGEN_EXTRACT_8i_FROM_16i(S12, S12);
  EIGEN_EXTRACT_8i_FROM_16i(S13, S13);
  EIGEN_EXTRACT_8i_FROM_16i(S14, S14);
  EIGEN_EXTRACT_8i_FROM_16i(S15, S15);
 
  PacketBlock<Packet8i, 32> tmp;
 
  tmp.packet[0] = _mm256_permute2f128_si256(S0_0, S4_0, 0x20);
  tmp.packet[1] = _mm256_permute2f128_si256(S1_0, S5_0, 0x20);
  tmp.packet[2] = _mm256_permute2f128_si256(S2_0, S6_0, 0x20);
  tmp.packet[3] = _mm256_permute2f128_si256(S3_0, S7_0, 0x20);
  tmp.packet[4] = _mm256_permute2f128_si256(S0_0, S4_0, 0x31);
  tmp.packet[5] = _mm256_permute2f128_si256(S1_0, S5_0, 0x31);
  tmp.packet[6] = _mm256_permute2f128_si256(S2_0, S6_0, 0x31);
  tmp.packet[7] = _mm256_permute2f128_si256(S3_0, S7_0, 0x31);
 
  tmp.packet[8] = _mm256_permute2f128_si256(S0_1, S4_1, 0x20);
  tmp.packet[9] = _mm256_permute2f128_si256(S1_1, S5_1, 0x20);
  tmp.packet[10] = _mm256_permute2f128_si256(S2_1, S6_1, 0x20);
  tmp.packet[11] = _mm256_permute2f128_si256(S3_1, S7_1, 0x20);
  tmp.packet[12] = _mm256_permute2f128_si256(S0_1, S4_1, 0x31);
  tmp.packet[13] = _mm256_permute2f128_si256(S1_1, S5_1, 0x31);
  tmp.packet[14] = _mm256_permute2f128_si256(S2_1, S6_1, 0x31);
  tmp.packet[15] = _mm256_permute2f128_si256(S3_1, S7_1, 0x31);
 
  // Second set of _m256 outputs
  tmp.packet[16] = _mm256_permute2f128_si256(S8_0, S12_0, 0x20);
  tmp.packet[17] = _mm256_permute2f128_si256(S9_0, S13_0, 0x20);
  tmp.packet[18] = _mm256_permute2f128_si256(S10_0, S14_0, 0x20);
  tmp.packet[19] = _mm256_permute2f128_si256(S11_0, S15_0, 0x20);
  tmp.packet[20] = _mm256_permute2f128_si256(S8_0, S12_0, 0x31);
  tmp.packet[21] = _mm256_permute2f128_si256(S9_0, S13_0, 0x31);
  tmp.packet[22] = _mm256_permute2f128_si256(S10_0, S14_0, 0x31);
  tmp.packet[23] = _mm256_permute2f128_si256(S11_0, S15_0, 0x31);
 
  tmp.packet[24] = _mm256_permute2f128_si256(S8_1, S12_1, 0x20);
  tmp.packet[25] = _mm256_permute2f128_si256(S9_1, S13_1, 0x20);
  tmp.packet[26] = _mm256_permute2f128_si256(S10_1, S14_1, 0x20);
  tmp.packet[27] = _mm256_permute2f128_si256(S11_1, S15_1, 0x20);
  tmp.packet[28] = _mm256_permute2f128_si256(S8_1, S12_1, 0x31);
  tmp.packet[29] = _mm256_permute2f128_si256(S9_1, S13_1, 0x31);
  tmp.packet[30] = _mm256_permute2f128_si256(S10_1, S14_1, 0x31);
  tmp.packet[31] = _mm256_permute2f128_si256(S11_1, S15_1, 0x31);
 
  // Pack them into the output
  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 0, 16);
  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 1, 16);
  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 2, 16);
  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 3, 16);
 
  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 4, 16);
  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 5, 16);
  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 6, 16);
  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 7, 16);
 
  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 8, 16);
  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 9, 16);
  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 10, 16);
  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 11, 16);
 
  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 12, 16);
  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 13, 16);
  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 14, 16);
  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 15, 16);
}

References EIGEN_EXTRACT_8i_FROM_16i, PACK_OUTPUT_I32, Eigen::PlainObjectBase< Derived >::packet(), Eigen::internal::PacketBlock< Packet, N >::packet, GlobalParameters::S0, GlobalParameters::S1, SHUFFLE_EPI32, and tmp.

ptranspose() [16/68]

EIGEN_DEVICE_FUNC void Eigen::internal::ptranspose ( PacketBlock< Packet16i, 4 > &  kernel )

inline

                                                                            {
  __m512i T0 = _mm512_unpacklo_epi32(kernel.packet[0], kernel.packet[1]);
  __m512i T1 = _mm512_unpackhi_epi32(kernel.packet[0], kernel.packet[1]);
  __m512i T2 = _mm512_unpacklo_epi32(kernel.packet[2], kernel.packet[3]);
  __m512i T3 = _mm512_unpackhi_epi32(kernel.packet[2], kernel.packet[3]);
 
  __m512i S0 = SHUFFLE_EPI32(T0, T2, _MM_SHUFFLE(1, 0, 1, 0));
  __m512i S1 = SHUFFLE_EPI32(T0, T2, _MM_SHUFFLE(3, 2, 3, 2));
  __m512i S2 = SHUFFLE_EPI32(T1, T3, _MM_SHUFFLE(1, 0, 1, 0));
  __m512i S3 = SHUFFLE_EPI32(T1, T3, _MM_SHUFFLE(3, 2, 3, 2));
 
  EIGEN_EXTRACT_8i_FROM_16i(S0, S0);
  EIGEN_EXTRACT_8i_FROM_16i(S1, S1);
  EIGEN_EXTRACT_8i_FROM_16i(S2, S2);
  EIGEN_EXTRACT_8i_FROM_16i(S3, S3);
 
  PacketBlock<Packet8i, 8> tmp;
 
  tmp.packet[0] = _mm256_permute2f128_si256(S0_0, S1_0, 0x20);
  tmp.packet[1] = _mm256_permute2f128_si256(S2_0, S3_0, 0x20);
  tmp.packet[2] = _mm256_permute2f128_si256(S0_0, S1_0, 0x31);
  tmp.packet[3] = _mm256_permute2f128_si256(S2_0, S3_0, 0x31);
 
  tmp.packet[4] = _mm256_permute2f128_si256(S0_1, S1_1, 0x20);
  tmp.packet[5] = _mm256_permute2f128_si256(S2_1, S3_1, 0x20);
  tmp.packet[6] = _mm256_permute2f128_si256(S0_1, S1_1, 0x31);
  tmp.packet[7] = _mm256_permute2f128_si256(S2_1, S3_1, 0x31);
 
  PACK_OUTPUT_I32_2(kernel.packet, tmp.packet, 0, 1);
  PACK_OUTPUT_I32_2(kernel.packet, tmp.packet, 1, 1);
  PACK_OUTPUT_I32_2(kernel.packet, tmp.packet, 2, 1);
  PACK_OUTPUT_I32_2(kernel.packet, tmp.packet, 3, 1);
}

References EIGEN_EXTRACT_8i_FROM_16i, PACK_OUTPUT_I32_2, Eigen::PlainObjectBase< Derived >::packet(), Eigen::internal::PacketBlock< Packet, N >::packet, GlobalParameters::S0, GlobalParameters::S1, SHUFFLE_EPI32, and tmp.

ptranspose() [17/68]

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::ptranspose ( PacketBlock< Packet16uc, 16 > &  kernel )

inline

                                                                              {
  Packet16uc step1[16], step2[16], step3[16];
 
  step1[0] = vec_mergeh(kernel.packet[0], kernel.packet[8]);
  step1[1] = vec_mergel(kernel.packet[0], kernel.packet[8]);
  step1[2] = vec_mergeh(kernel.packet[1], kernel.packet[9]);
  step1[3] = vec_mergel(kernel.packet[1], kernel.packet[9]);
  step1[4] = vec_mergeh(kernel.packet[2], kernel.packet[10]);
  step1[5] = vec_mergel(kernel.packet[2], kernel.packet[10]);
  step1[6] = vec_mergeh(kernel.packet[3], kernel.packet[11]);
  step1[7] = vec_mergel(kernel.packet[3], kernel.packet[11]);
  step1[8] = vec_mergeh(kernel.packet[4], kernel.packet[12]);
  step1[9] = vec_mergel(kernel.packet[4], kernel.packet[12]);
  step1[10] = vec_mergeh(kernel.packet[5], kernel.packet[13]);
  step1[11] = vec_mergel(kernel.packet[5], kernel.packet[13]);
  step1[12] = vec_mergeh(kernel.packet[6], kernel.packet[14]);
  step1[13] = vec_mergel(kernel.packet[6], kernel.packet[14]);
  step1[14] = vec_mergeh(kernel.packet[7], kernel.packet[15]);
  step1[15] = vec_mergel(kernel.packet[7], kernel.packet[15]);
 
  step2[0] = vec_mergeh(step1[0], step1[8]);
  step2[1] = vec_mergel(step1[0], step1[8]);
  step2[2] = vec_mergeh(step1[1], step1[9]);
  step2[3] = vec_mergel(step1[1], step1[9]);
  step2[4] = vec_mergeh(step1[2], step1[10]);
  step2[5] = vec_mergel(step1[2], step1[10]);
  step2[6] = vec_mergeh(step1[3], step1[11]);
  step2[7] = vec_mergel(step1[3], step1[11]);
  step2[8] = vec_mergeh(step1[4], step1[12]);
  step2[9] = vec_mergel(step1[4], step1[12]);
  step2[10] = vec_mergeh(step1[5], step1[13]);
  step2[11] = vec_mergel(step1[5], step1[13]);
  step2[12] = vec_mergeh(step1[6], step1[14]);
  step2[13] = vec_mergel(step1[6], step1[14]);
  step2[14] = vec_mergeh(step1[7], step1[15]);
  step2[15] = vec_mergel(step1[7], step1[15]);
 
  step3[0] = vec_mergeh(step2[0], step2[8]);
  step3[1] = vec_mergel(step2[0], step2[8]);
  step3[2] = vec_mergeh(step2[1], step2[9]);
  step3[3] = vec_mergel(step2[1], step2[9]);
  step3[4] = vec_mergeh(step2[2], step2[10]);
  step3[5] = vec_mergel(step2[2], step2[10]);
  step3[6] = vec_mergeh(step2[3], step2[11]);
  step3[7] = vec_mergel(step2[3], step2[11]);
  step3[8] = vec_mergeh(step2[4], step2[12]);
  step3[9] = vec_mergel(step2[4], step2[12]);
  step3[10] = vec_mergeh(step2[5], step2[13]);
  step3[11] = vec_mergel(step2[5], step2[13]);
  step3[12] = vec_mergeh(step2[6], step2[14]);
  step3[13] = vec_mergel(step2[6], step2[14]);
  step3[14] = vec_mergeh(step2[7], step2[15]);
  step3[15] = vec_mergel(step2[7], step2[15]);
 
  kernel.packet[0] = vec_mergeh(step3[0], step3[8]);
  kernel.packet[1] = vec_mergel(step3[0], step3[8]);
  kernel.packet[2] = vec_mergeh(step3[1], step3[9]);
  kernel.packet[3] = vec_mergel(step3[1], step3[9]);
  kernel.packet[4] = vec_mergeh(step3[2], step3[10]);
  kernel.packet[5] = vec_mergel(step3[2], step3[10]);
  kernel.packet[6] = vec_mergeh(step3[3], step3[11]);
  kernel.packet[7] = vec_mergel(step3[3], step3[11]);
  kernel.packet[8] = vec_mergeh(step3[4], step3[12]);
  kernel.packet[9] = vec_mergel(step3[4], step3[12]);
  kernel.packet[10] = vec_mergeh(step3[5], step3[13]);
  kernel.packet[11] = vec_mergel(step3[5], step3[13]);
  kernel.packet[12] = vec_mergeh(step3[6], step3[14]);
  kernel.packet[13] = vec_mergel(step3[6], step3[14]);
  kernel.packet[14] = vec_mergeh(step3[7], step3[15]);
  kernel.packet[15] = vec_mergel(step3[7], step3[15]);
}

References Eigen::internal::PacketBlock< Packet, N >::packet.

ptranspose() [18/68]

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::ptranspose ( PacketBlock< Packet16uc, 4 > &  kernel )

inline

                                                                             {
  Packet16uc t0, t1, t2, t3;
  t0 = vec_mergeh(kernel.packet[0], kernel.packet[2]);
  t1 = vec_mergel(kernel.packet[0], kernel.packet[2]);
  t2 = vec_mergeh(kernel.packet[1], kernel.packet[3]);
  t3 = vec_mergel(kernel.packet[1], kernel.packet[3]);
  kernel.packet[0] = vec_mergeh(t0, t2);
  kernel.packet[1] = vec_mergel(t0, t2);
  kernel.packet[2] = vec_mergeh(t1, t3);
  kernel.packet[3] = vec_mergel(t1, t3);
}

References Eigen::internal::PacketBlock< Packet, N >::packet.

ptranspose() [19/68]

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::ptranspose

(

PacketBlock< Packet16uc, 8 > &

kernel

)

                                                                                          {
  __m128i t0 = __lsx_vilvl_b(kernel.packet[1], kernel.packet[0]);
  __m128i t1 = __lsx_vilvh_b(kernel.packet[1], kernel.packet[0]);
  __m128i t2 = __lsx_vilvl_b(kernel.packet[3], kernel.packet[2]);
  __m128i t3 = __lsx_vilvh_b(kernel.packet[3], kernel.packet[2]);
  __m128i t4 = __lsx_vilvl_b(kernel.packet[5], kernel.packet[4]);
  __m128i t5 = __lsx_vilvh_b(kernel.packet[5], kernel.packet[4]);
  __m128i t6 = __lsx_vilvl_b(kernel.packet[7], kernel.packet[6]);
  __m128i t7 = __lsx_vilvh_b(kernel.packet[7], kernel.packet[6]);
 
  __m128i s0 = __lsx_vilvl_h(t2, t0);
  __m128i s1 = __lsx_vilvh_h(t2, t0);
  __m128i s2 = __lsx_vilvl_h(t3, t1);
  __m128i s3 = __lsx_vilvh_h(t3, t1);
  __m128i s4 = __lsx_vilvl_h(t6, t4);
  __m128i s5 = __lsx_vilvh_h(t6, t4);
  __m128i s6 = __lsx_vilvl_h(t7, t5);
  __m128i s7 = __lsx_vilvh_h(t7, t5);
 
  kernel.packet[0] = __lsx_vilvl_w(s4, s0);
  kernel.packet[1] = __lsx_vilvh_w(s4, s0);
  kernel.packet[2] = __lsx_vilvl_w(s5, s1);
  kernel.packet[3] = __lsx_vilvh_w(s5, s1);
  kernel.packet[4] = __lsx_vilvl_w(s6, s2);
  kernel.packet[5] = __lsx_vilvh_w(s6, s2);
  kernel.packet[6] = __lsx_vilvl_w(s7, s3);
  kernel.packet[7] = __lsx_vilvh_w(s7, s3);
}

References Eigen::internal::PacketBlock< Packet, N >::packet.

ptranspose() [20/68]

EIGEN_STRONG_INLINE void Eigen::internal::ptranspose

(

PacketBlock< Packet1cd, 2 > &

kernel

)

                                                                       {
  Packet2d tmp = (__m128d)__lsx_vilvl_d((__m128i)kernel.packet[1].v, (__m128i)kernel.packet[0].v);
  kernel.packet[1].v = (__m128d)__lsx_vilvh_d((__m128i)kernel.packet[1].v, (__m128i)kernel.packet[0].v);
  kernel.packet[0].v = tmp;
}

References Eigen::internal::PacketBlock< Packet, N >::packet, and tmp.

ptranspose() [21/68]

EIGEN_DEVICE_FUNC void Eigen::internal::ptranspose ( PacketBlock< Packet1cf, 1 > &  )

inline

{}

ptranspose() [22/68]

EIGEN_DEVICE_FUNC void Eigen::internal::ptranspose ( PacketBlock< Packet2cd, 2 > &  kernel )

inline

                                                                            {
  __m256d tmp = _mm256_permute2f128_pd(kernel.packet[0].v, kernel.packet[1].v, 0 + (2 << 4));
  kernel.packet[1].v = _mm256_permute2f128_pd(kernel.packet[0].v, kernel.packet[1].v, 1 + (3 << 4));
  kernel.packet[0].v = tmp;
}

References Eigen::internal::PacketBlock< Packet, N >::packet, and tmp.

ptranspose() [23/68]

EIGEN_STRONG_INLINE void Eigen::internal::ptranspose ( PacketBlock< Packet2cf, 2 > &  kernel )

inline

                                                                       {
#ifdef EIGEN_VECTORIZE_VSX
  Packet4f tmp = reinterpret_cast<Packet4f>(
      vec_mergeh(reinterpret_cast<Packet2d>(kernel.packet[0].v), reinterpret_cast<Packet2d>(kernel.packet[1].v)));
  kernel.packet[1].v = reinterpret_cast<Packet4f>(
      vec_mergel(reinterpret_cast<Packet2d>(kernel.packet[0].v), reinterpret_cast<Packet2d>(kernel.packet[1].v)));
#else
  Packet4f tmp = vec_perm(kernel.packet[0].v, kernel.packet[1].v, p16uc_TRANSPOSE64_HI);
  kernel.packet[1].v = vec_perm(kernel.packet[0].v, kernel.packet[1].v, p16uc_TRANSPOSE64_LO);
#endif
  kernel.packet[0].v = tmp;
}

References p16uc_TRANSPOSE64_HI, p16uc_TRANSPOSE64_LO, Eigen::internal::PacketBlock< Packet, N >::packet, and tmp.

Referenced by BlockedInPlaceTranspose(), Eigen::internal::dhs_pack< double, DataMapper, Packet2d, StorageOrder, PanelMode, true >::dhs_copy(), Eigen::internal::dhs_pack< double, DataMapper, Packet2d, StorageOrder, PanelMode, false >::dhs_copy(), Eigen::internal::dhs_pack< Scalar, DataMapper, Packet, StorageOrder, PanelMode, UseLhs >::dhs_copy(), Eigen::internal::gemm_pack_lhs< Scalar, Index, DataMapper, Pack1, Pack2, Packet, RowMajor, Conjugate, PanelMode >::operator()(), Eigen::internal::gemm_pack_rhs< Scalar, Index, DataMapper, 8, ColMajor, Conjugate, PanelMode >::operator()(), Eigen::internal::gemm_pack_rhs< Scalar, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode >::operator()(), packetmath(), ptranspose(), and Eigen::internal::inplace_transpose_selector< MatrixType, true, true >::run().

ptranspose() [24/68]

EIGEN_DEVICE_FUNC void Eigen::internal::ptranspose ( PacketBlock< Packet2d, 2 > &  kernel )

inline

                                                                                        {
  Packet2d tmp = (Packet2d)__lsx_vilvh_d((__m128i)kernel.packet[1], (__m128i)kernel.packet[0]);
  kernel.packet[0] = (Packet2d)__lsx_vilvl_d((__m128i)kernel.packet[1], (__m128i)kernel.packet[0]);
  kernel.packet[1] = tmp;
}

References Eigen::internal::PacketBlock< Packet, N >::packet, and tmp.

ptranspose() [25/68]

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::ptranspose

(

PacketBlock< Packet2f, 2 > &

kernel

)

                                                                                        {
  detail::ptranspose_impl(kernel);
}

References Eigen::internal::detail::ptranspose_impl().

ptranspose() [26/68]

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::ptranspose

(

PacketBlock< Packet2i, 2 > &

kernel

)

                                                                                        {
  detail::ptranspose_impl(kernel);
}

References Eigen::internal::detail::ptranspose_impl().

ptranspose() [27/68]

EIGEN_STRONG_INLINE void Eigen::internal::ptranspose

(

PacketBlock< Packet2l, 2 > &

kernel

)

                                                                                        {
  __m128i tmp = __lsx_vilvh_d(kernel.packet[1], kernel.packet[0]);
  kernel.packet[0] = __lsx_vilvl_d(kernel.packet[1], kernel.packet[0]);
  kernel.packet[1] = tmp;
}

References Eigen::internal::PacketBlock< Packet, N >::packet, and tmp.

ptranspose() [28/68]

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::ptranspose

(

PacketBlock< Packet2ui, 2 > &

kernel

)

                                                                                         {
  detail::zip_in_place(kernel.packet[0], kernel.packet[1]);
}

References Eigen::internal::PacketBlock< Packet, N >::packet, and Eigen::internal::detail::zip_in_place().

ptranspose() [29/68]

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::ptranspose

(

PacketBlock< Packet2ul, 2 > &

kernel

)

                                                                                         {
  __m128i tmp = __lsx_vilvh_d(kernel.packet[1], kernel.packet[0]);
  kernel.packet[0] = __lsx_vilvl_d(kernel.packet[1], kernel.packet[0]);
  kernel.packet[1] = tmp;
}

References Eigen::internal::PacketBlock< Packet, N >::packet, and tmp.

ptranspose() [30/68]

EIGEN_DEVICE_FUNC void Eigen::internal::ptranspose ( PacketBlock< Packet32h, 32 > &  a )

inline

                                                                        {
  __m512i t[32];
 
  EIGEN_UNROLL_LOOP
  for (int i = 0; i < 16; i++) {
    t[2 * i] = _mm512_unpacklo_epi16(_mm512_castph_si512(a.packet[2 * i]), _mm512_castph_si512(a.packet[2 * i + 1]));
    t[2 * i + 1] =
        _mm512_unpackhi_epi16(_mm512_castph_si512(a.packet[2 * i]), _mm512_castph_si512(a.packet[2 * i + 1]));
  }
 
  __m512i p[32];
 
  EIGEN_UNROLL_LOOP
  for (int i = 0; i < 8; i++) {
    p[4 * i] = _mm512_unpacklo_epi32(t[4 * i], t[4 * i + 2]);
    p[4 * i + 1] = _mm512_unpackhi_epi32(t[4 * i], t[4 * i + 2]);
    p[4 * i + 2] = _mm512_unpacklo_epi32(t[4 * i + 1], t[4 * i + 3]);
    p[4 * i + 3] = _mm512_unpackhi_epi32(t[4 * i + 1], t[4 * i + 3]);
  }
 
  __m512i q[32];
 
  EIGEN_UNROLL_LOOP
  for (int i = 0; i < 4; i++) {
    q[8 * i] = _mm512_unpacklo_epi64(p[8 * i], p[8 * i + 4]);
    q[8 * i + 1] = _mm512_unpackhi_epi64(p[8 * i], p[8 * i + 4]);
    q[8 * i + 2] = _mm512_unpacklo_epi64(p[8 * i + 1], p[8 * i + 5]);
    q[8 * i + 3] = _mm512_unpackhi_epi64(p[8 * i + 1], p[8 * i + 5]);
    q[8 * i + 4] = _mm512_unpacklo_epi64(p[8 * i + 2], p[8 * i + 6]);
    q[8 * i + 5] = _mm512_unpackhi_epi64(p[8 * i + 2], p[8 * i + 6]);
    q[8 * i + 6] = _mm512_unpacklo_epi64(p[8 * i + 3], p[8 * i + 7]);
    q[8 * i + 7] = _mm512_unpackhi_epi64(p[8 * i + 3], p[8 * i + 7]);
  }
 
  __m512i f[32];
 
#define PACKET32H_TRANSPOSE_HELPER(X, Y)                                                            \
  do {                                                                                              \
    f[Y * 8] = _mm512_inserti32x4(f[Y * 8], _mm512_extracti32x4_epi32(q[X * 8], Y), X);             \
    f[Y * 8 + 1] = _mm512_inserti32x4(f[Y * 8 + 1], _mm512_extracti32x4_epi32(q[X * 8 + 1], Y), X); \
    f[Y * 8 + 2] = _mm512_inserti32x4(f[Y * 8 + 2], _mm512_extracti32x4_epi32(q[X * 8 + 2], Y), X); \
    f[Y * 8 + 3] = _mm512_inserti32x4(f[Y * 8 + 3], _mm512_extracti32x4_epi32(q[X * 8 + 3], Y), X); \
    f[Y * 8 + 4] = _mm512_inserti32x4(f[Y * 8 + 4], _mm512_extracti32x4_epi32(q[X * 8 + 4], Y), X); \
    f[Y * 8 + 5] = _mm512_inserti32x4(f[Y * 8 + 5], _mm512_extracti32x4_epi32(q[X * 8 + 5], Y), X); \
    f[Y * 8 + 6] = _mm512_inserti32x4(f[Y * 8 + 6], _mm512_extracti32x4_epi32(q[X * 8 + 6], Y), X); \
    f[Y * 8 + 7] = _mm512_inserti32x4(f[Y * 8 + 7], _mm512_extracti32x4_epi32(q[X * 8 + 7], Y), X); \
  } while (false);
 
  PACKET32H_TRANSPOSE_HELPER(0, 0);
  PACKET32H_TRANSPOSE_HELPER(1, 1);
  PACKET32H_TRANSPOSE_HELPER(2, 2);
  PACKET32H_TRANSPOSE_HELPER(3, 3);
 
  PACKET32H_TRANSPOSE_HELPER(1, 0);
  PACKET32H_TRANSPOSE_HELPER(2, 0);
  PACKET32H_TRANSPOSE_HELPER(3, 0);
  PACKET32H_TRANSPOSE_HELPER(2, 1);
  PACKET32H_TRANSPOSE_HELPER(3, 1);
  PACKET32H_TRANSPOSE_HELPER(3, 2);
 
  PACKET32H_TRANSPOSE_HELPER(0, 1);
  PACKET32H_TRANSPOSE_HELPER(0, 2);
  PACKET32H_TRANSPOSE_HELPER(0, 3);
  PACKET32H_TRANSPOSE_HELPER(1, 2);
  PACKET32H_TRANSPOSE_HELPER(1, 3);
  PACKET32H_TRANSPOSE_HELPER(2, 3);
 
#undef PACKET32H_TRANSPOSE_HELPER
 
  EIGEN_UNROLL_LOOP
  for (int i = 0; i < 32; i++) {
    a.packet[i] = _mm512_castsi512_ph(f[i]);
  }
}

References a, EIGEN_UNROLL_LOOP, f(), i, p, PACKET32H_TRANSPOSE_HELPER, Eigen::numext::q, and plotPSD::t.

ptranspose() [31/68]

EIGEN_DEVICE_FUNC void Eigen::internal::ptranspose ( PacketBlock< Packet32h, 4 > &  a )

inline

                                                                       {
  __m512i p0, p1, p2, p3, t0, t1, t2, t3, a0, a1, a2, a3;
  t0 = _mm512_unpacklo_epi16(_mm512_castph_si512(a.packet[0]), _mm512_castph_si512(a.packet[1]));
  t1 = _mm512_unpackhi_epi16(_mm512_castph_si512(a.packet[0]), _mm512_castph_si512(a.packet[1]));
  t2 = _mm512_unpacklo_epi16(_mm512_castph_si512(a.packet[2]), _mm512_castph_si512(a.packet[3]));
  t3 = _mm512_unpackhi_epi16(_mm512_castph_si512(a.packet[2]), _mm512_castph_si512(a.packet[3]));
 
  p0 = _mm512_unpacklo_epi32(t0, t2);
  p1 = _mm512_unpackhi_epi32(t0, t2);
  p2 = _mm512_unpacklo_epi32(t1, t3);
  p3 = _mm512_unpackhi_epi32(t1, t3);
 
  a0 = p0;
  a1 = p1;
  a2 = p2;
  a3 = p3;
 
  a0 = _mm512_inserti32x4(a0, _mm512_extracti32x4_epi32(p1, 0), 1);
  a1 = _mm512_inserti32x4(a1, _mm512_extracti32x4_epi32(p0, 1), 0);
 
  a0 = _mm512_inserti32x4(a0, _mm512_extracti32x4_epi32(p2, 0), 2);
  a2 = _mm512_inserti32x4(a2, _mm512_extracti32x4_epi32(p0, 2), 0);
 
  a0 = _mm512_inserti32x4(a0, _mm512_extracti32x4_epi32(p3, 0), 3);
  a3 = _mm512_inserti32x4(a3, _mm512_extracti32x4_epi32(p0, 3), 0);
 
  a1 = _mm512_inserti32x4(a1, _mm512_extracti32x4_epi32(p2, 1), 2);
  a2 = _mm512_inserti32x4(a2, _mm512_extracti32x4_epi32(p1, 2), 1);
 
  a2 = _mm512_inserti32x4(a2, _mm512_extracti32x4_epi32(p3, 2), 3);
  a3 = _mm512_inserti32x4(a3, _mm512_extracti32x4_epi32(p2, 3), 2);
 
  a1 = _mm512_inserti32x4(a1, _mm512_extracti32x4_epi32(p3, 1), 3);
  a3 = _mm512_inserti32x4(a3, _mm512_extracti32x4_epi32(p1, 3), 1);
 
  a.packet[0] = _mm512_castsi512_ph(a0);
  a.packet[1] = _mm512_castsi512_ph(a1);
  a.packet[2] = _mm512_castsi512_ph(a2);
  a.packet[3] = _mm512_castsi512_ph(a3);
}

References a, p0, and p1.

ptranspose() [32/68]

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::ptranspose

(

PacketBlock< Packet4bf, 4 > &

kernel

)

                                                                                         {
  detail::ptranspose_impl(kernel);
}

References Eigen::internal::detail::ptranspose_impl().

ptranspose() [33/68]

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::ptranspose

(

PacketBlock< Packet4c, 4 > &

kernel

)

                                                                                        {
  const int8x8_t a = vreinterpret_s8_s32(vset_lane_s32(kernel.packet[2], vdup_n_s32(kernel.packet[0]), 1));
  const int8x8_t b = vreinterpret_s8_s32(vset_lane_s32(kernel.packet[3], vdup_n_s32(kernel.packet[1]), 1));
 
  const int8x8x2_t zip8 = vzip_s8(a, b);
  const int16x4x2_t zip16 = vzip_s16(vreinterpret_s16_s8(zip8.val[0]), vreinterpret_s16_s8(zip8.val[1]));
 
  kernel.packet[0] = vget_lane_s32(vreinterpret_s32_s16(zip16.val[0]), 0);
  kernel.packet[1] = vget_lane_s32(vreinterpret_s32_s16(zip16.val[0]), 1);
  kernel.packet[2] = vget_lane_s32(vreinterpret_s32_s16(zip16.val[1]), 0);
  kernel.packet[3] = vget_lane_s32(vreinterpret_s32_s16(zip16.val[1]), 1);
}

References a, b, and Eigen::internal::PacketBlock< Packet, N >::packet.

ptranspose() [34/68]

EIGEN_DEVICE_FUNC void Eigen::internal::ptranspose ( PacketBlock< Packet4cd, 4 > &  kernel )

inline

                                                                            {
  __m512d T0 =
      _mm512_shuffle_f64x2(kernel.packet[0].v, kernel.packet[1].v, (shuffle_mask<0, 1, 0, 1>::mask));  // [a0 a1 b0 b1]
  __m512d T1 =
      _mm512_shuffle_f64x2(kernel.packet[0].v, kernel.packet[1].v, (shuffle_mask<2, 3, 2, 3>::mask));  // [a2 a3 b2 b3]
  __m512d T2 =
      _mm512_shuffle_f64x2(kernel.packet[2].v, kernel.packet[3].v, (shuffle_mask<0, 1, 0, 1>::mask));  // [c0 c1 d0 d1]
  __m512d T3 =
      _mm512_shuffle_f64x2(kernel.packet[2].v, kernel.packet[3].v, (shuffle_mask<2, 3, 2, 3>::mask));  // [c2 c3 d2 d3]
 
  kernel.packet[3] = Packet4cd(_mm512_shuffle_f64x2(T1, T3, (shuffle_mask<1, 3, 1, 3>::mask)));  // [a3 b3 c3 d3]
  kernel.packet[2] = Packet4cd(_mm512_shuffle_f64x2(T1, T3, (shuffle_mask<0, 2, 0, 2>::mask)));  // [a2 b2 c2 d2]
  kernel.packet[1] = Packet4cd(_mm512_shuffle_f64x2(T0, T2, (shuffle_mask<1, 3, 1, 3>::mask)));  // [a1 b1 c1 d1]
  kernel.packet[0] = Packet4cd(_mm512_shuffle_f64x2(T0, T2, (shuffle_mask<0, 2, 0, 2>::mask)));  // [a0 b0 c0 d0]
}

References Eigen::internal::PacketBlock< Packet, N >::packet.

ptranspose() [35/68]

EIGEN_DEVICE_FUNC void Eigen::internal::ptranspose ( PacketBlock< Packet4cf, 4 > &  kernel )

inline

                                                                            {
  __m256d P0 = _mm256_castps_pd(kernel.packet[0].v);
  __m256d P1 = _mm256_castps_pd(kernel.packet[1].v);
  __m256d P2 = _mm256_castps_pd(kernel.packet[2].v);
  __m256d P3 = _mm256_castps_pd(kernel.packet[3].v);
 
  __m256d T0 = _mm256_shuffle_pd(P0, P1, 15);
  __m256d T1 = _mm256_shuffle_pd(P0, P1, 0);
  __m256d T2 = _mm256_shuffle_pd(P2, P3, 15);
  __m256d T3 = _mm256_shuffle_pd(P2, P3, 0);
 
  kernel.packet[1].v = _mm256_castpd_ps(_mm256_permute2f128_pd(T0, T2, 32));
  kernel.packet[3].v = _mm256_castpd_ps(_mm256_permute2f128_pd(T0, T2, 49));
  kernel.packet[0].v = _mm256_castpd_ps(_mm256_permute2f128_pd(T1, T3, 32));
  kernel.packet[2].v = _mm256_castpd_ps(_mm256_permute2f128_pd(T1, T3, 49));
}

References Problem_Parameter::P0, and Eigen::internal::PacketBlock< Packet, N >::packet.

ptranspose() [36/68]

EIGEN_DEVICE_FUNC void Eigen::internal::ptranspose ( PacketBlock< Packet4d, 4 > &  kernel )

inline

                                                                           {
  __m256d T0 = _mm256_shuffle_pd(kernel.packet[0], kernel.packet[1], 15);
  __m256d T1 = _mm256_shuffle_pd(kernel.packet[0], kernel.packet[1], 0);
  __m256d T2 = _mm256_shuffle_pd(kernel.packet[2], kernel.packet[3], 15);
  __m256d T3 = _mm256_shuffle_pd(kernel.packet[2], kernel.packet[3], 0);
 
  kernel.packet[1] = _mm256_permute2f128_pd(T0, T2, 32);
  kernel.packet[3] = _mm256_permute2f128_pd(T0, T2, 49);
  kernel.packet[0] = _mm256_permute2f128_pd(T1, T3, 32);
  kernel.packet[2] = _mm256_permute2f128_pd(T1, T3, 49);
}

References Eigen::internal::PacketBlock< Packet, N >::packet.

ptranspose() [37/68]

EIGEN_DEVICE_FUNC void Eigen::internal::ptranspose ( PacketBlock< Packet4f, 4 > &  kernel )

inline

{ ptranpose_common<Packet4f>(kernel); }

ptranspose() [38/68]

EIGEN_DEVICE_FUNC void Eigen::internal::ptranspose ( PacketBlock< Packet4i, 4 > &  kernel )

inline

{ ptranpose_common<Packet4i>(kernel); }

ptranspose() [39/68]

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::ptranspose

(

PacketBlock< Packet4s, 4 > &

kernel

)

                                                                                        {
  detail::ptranspose_impl(kernel);
}

References Eigen::internal::detail::ptranspose_impl().

ptranspose() [40/68]

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::ptranspose

(

PacketBlock< Packet4uc, 4 > &

kernel

)

                                                                                         {
  const uint8x8_t a = vreinterpret_u8_u32(vset_lane_u32(kernel.packet[2], vdup_n_u32(kernel.packet[0]), 1));
  const uint8x8_t b = vreinterpret_u8_u32(vset_lane_u32(kernel.packet[3], vdup_n_u32(kernel.packet[1]), 1));
 
  const uint8x8x2_t zip8 = vzip_u8(a, b);
  const uint16x4x2_t zip16 = vzip_u16(vreinterpret_u16_u8(zip8.val[0]), vreinterpret_u16_u8(zip8.val[1]));
 
  kernel.packet[0] = vget_lane_u32(vreinterpret_u32_u16(zip16.val[0]), 0);
  kernel.packet[1] = vget_lane_u32(vreinterpret_u32_u16(zip16.val[0]), 1);
  kernel.packet[2] = vget_lane_u32(vreinterpret_u32_u16(zip16.val[1]), 0);
  kernel.packet[3] = vget_lane_u32(vreinterpret_u32_u16(zip16.val[1]), 1);
}

References a, b, and Eigen::internal::PacketBlock< Packet, N >::packet.

ptranspose() [41/68]

EIGEN_STRONG_INLINE void Eigen::internal::ptranspose

(

PacketBlock< Packet4ui, 4 > &

kernel

)

                                                                                         {
  __m128i T0 = __lsx_vilvl_w(kernel.packet[1], kernel.packet[0]);
  __m128i T1 = __lsx_vilvh_w(kernel.packet[1], kernel.packet[0]);
  __m128i T2 = __lsx_vilvl_w(kernel.packet[3], kernel.packet[2]);
  __m128i T3 = __lsx_vilvh_w(kernel.packet[3], kernel.packet[2]);
 
  kernel.packet[0] = __lsx_vilvl_d(T2, T0);
  kernel.packet[1] = __lsx_vilvh_d(T2, T0);
  kernel.packet[2] = __lsx_vilvl_d(T3, T1);
  kernel.packet[3] = __lsx_vilvh_d(T3, T1);
}

References Eigen::internal::PacketBlock< Packet, N >::packet.

ptranspose() [42/68]

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::ptranspose

(

PacketBlock< Packet4us, 4 > &

kernel

)

                                                                                         {
  detail::ptranspose_impl(kernel);
}

References Eigen::internal::detail::ptranspose_impl().

ptranspose() [43/68]

EIGEN_STRONG_INLINE void Eigen::internal::ptranspose ( PacketBlock< Packet8bf, 4 > &  kernel )

inline

                                                                            {
  Packet8us t0, t1, t2, t3;
 
  t0 = vec_mergeh(kernel.packet[0].m_val, kernel.packet[2].m_val);
  t1 = vec_mergel(kernel.packet[0].m_val, kernel.packet[2].m_val);
  t2 = vec_mergeh(kernel.packet[1].m_val, kernel.packet[3].m_val);
  t3 = vec_mergel(kernel.packet[1].m_val, kernel.packet[3].m_val);
  kernel.packet[0] = vec_mergeh(t0, t2);
  kernel.packet[1] = vec_mergel(t0, t2);
  kernel.packet[2] = vec_mergeh(t1, t3);
  kernel.packet[3] = vec_mergel(t1, t3);
}

References Eigen::internal::PacketBlock< Packet, N >::packet.

ptranspose() [44/68]

EIGEN_STRONG_INLINE void Eigen::internal::ptranspose ( PacketBlock< Packet8bf, 8 > &  kernel )

inline

                                                                            {
  Packet8bf v[8], sum[8];
 
  v[0] = vec_mergeh(kernel.packet[0].m_val, kernel.packet[4].m_val);
  v[1] = vec_mergel(kernel.packet[0].m_val, kernel.packet[4].m_val);
  v[2] = vec_mergeh(kernel.packet[1].m_val, kernel.packet[5].m_val);
  v[3] = vec_mergel(kernel.packet[1].m_val, kernel.packet[5].m_val);
  v[4] = vec_mergeh(kernel.packet[2].m_val, kernel.packet[6].m_val);
  v[5] = vec_mergel(kernel.packet[2].m_val, kernel.packet[6].m_val);
  v[6] = vec_mergeh(kernel.packet[3].m_val, kernel.packet[7].m_val);
  v[7] = vec_mergel(kernel.packet[3].m_val, kernel.packet[7].m_val);
  sum[0] = vec_mergeh(v[0].m_val, v[4].m_val);
  sum[1] = vec_mergel(v[0].m_val, v[4].m_val);
  sum[2] = vec_mergeh(v[1].m_val, v[5].m_val);
  sum[3] = vec_mergel(v[1].m_val, v[5].m_val);
  sum[4] = vec_mergeh(v[2].m_val, v[6].m_val);
  sum[5] = vec_mergel(v[2].m_val, v[6].m_val);
  sum[6] = vec_mergeh(v[3].m_val, v[7].m_val);
  sum[7] = vec_mergel(v[3].m_val, v[7].m_val);
 
  kernel.packet[0] = vec_mergeh(sum[0].m_val, sum[4].m_val);
  kernel.packet[1] = vec_mergel(sum[0].m_val, sum[4].m_val);
  kernel.packet[2] = vec_mergeh(sum[1].m_val, sum[5].m_val);
  kernel.packet[3] = vec_mergel(sum[1].m_val, sum[5].m_val);
  kernel.packet[4] = vec_mergeh(sum[2].m_val, sum[6].m_val);
  kernel.packet[5] = vec_mergel(sum[2].m_val, sum[6].m_val);
  kernel.packet[6] = vec_mergeh(sum[3].m_val, sum[7].m_val);
  kernel.packet[7] = vec_mergel(sum[3].m_val, sum[7].m_val);
}

References Eigen::internal::PacketBlock< Packet, N >::packet, and v.

ptranspose() [45/68]

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::ptranspose

(

PacketBlock< Packet8c, 4 > &

kernel

)

                                                                                        {
  detail::ptranspose_impl(kernel);
}

References Eigen::internal::detail::ptranspose_impl().

ptranspose() [46/68]

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::ptranspose

(

PacketBlock< Packet8c, 8 > &

kernel

)

                                                                                        {
  detail::ptranspose_impl(kernel);
}

References Eigen::internal::detail::ptranspose_impl().

ptranspose() [47/68]

EIGEN_DEVICE_FUNC void Eigen::internal::ptranspose ( PacketBlock< Packet8cf, 4 > &  kernel )

inline

                                                                            {
  PacketBlock<Packet8d, 4> pb;
 
  pb.packet[0] = _mm512_castps_pd(kernel.packet[0].v);
  pb.packet[1] = _mm512_castps_pd(kernel.packet[1].v);
  pb.packet[2] = _mm512_castps_pd(kernel.packet[2].v);
  pb.packet[3] = _mm512_castps_pd(kernel.packet[3].v);
  ptranspose(pb);
  kernel.packet[0].v = _mm512_castpd_ps(pb.packet[0]);
  kernel.packet[1].v = _mm512_castpd_ps(pb.packet[1]);
  kernel.packet[2].v = _mm512_castpd_ps(pb.packet[2]);
  kernel.packet[3].v = _mm512_castpd_ps(pb.packet[3]);
}

References Eigen::internal::PacketBlock< Packet, N >::packet, pb, and ptranspose().

ptranspose() [48/68]

EIGEN_DEVICE_FUNC void Eigen::internal::ptranspose ( PacketBlock< Packet8cf, 8 > &  kernel )

inline

                                                                            {
  PacketBlock<Packet8d, 8> pb;
 
  pb.packet[0] = _mm512_castps_pd(kernel.packet[0].v);
  pb.packet[1] = _mm512_castps_pd(kernel.packet[1].v);
  pb.packet[2] = _mm512_castps_pd(kernel.packet[2].v);
  pb.packet[3] = _mm512_castps_pd(kernel.packet[3].v);
  pb.packet[4] = _mm512_castps_pd(kernel.packet[4].v);
  pb.packet[5] = _mm512_castps_pd(kernel.packet[5].v);
  pb.packet[6] = _mm512_castps_pd(kernel.packet[6].v);
  pb.packet[7] = _mm512_castps_pd(kernel.packet[7].v);
  ptranspose(pb);
  kernel.packet[0].v = _mm512_castpd_ps(pb.packet[0]);
  kernel.packet[1].v = _mm512_castpd_ps(pb.packet[1]);
  kernel.packet[2].v = _mm512_castpd_ps(pb.packet[2]);
  kernel.packet[3].v = _mm512_castpd_ps(pb.packet[3]);
  kernel.packet[4].v = _mm512_castpd_ps(pb.packet[4]);
  kernel.packet[5].v = _mm512_castpd_ps(pb.packet[5]);
  kernel.packet[6].v = _mm512_castpd_ps(pb.packet[6]);
  kernel.packet[7].v = _mm512_castpd_ps(pb.packet[7]);
}

References Eigen::internal::PacketBlock< Packet, N >::packet, pb, and ptranspose().

ptranspose() [49/68]

EIGEN_DEVICE_FUNC void Eigen::internal::ptranspose ( PacketBlock< Packet8d, 4 > &  kernel )

inline

                                                                           {
  __m512d T0 = _mm512_shuffle_pd(kernel.packet[0], kernel.packet[1], 0);
  __m512d T1 = _mm512_shuffle_pd(kernel.packet[0], kernel.packet[1], 0xff);
  __m512d T2 = _mm512_shuffle_pd(kernel.packet[2], kernel.packet[3], 0);
  __m512d T3 = _mm512_shuffle_pd(kernel.packet[2], kernel.packet[3], 0xff);
 
  PacketBlock<Packet4d, 8> tmp;
 
  tmp.packet[0] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 0), _mm512_extractf64x4_pd(T2, 0), 0x20);
  tmp.packet[1] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 0), _mm512_extractf64x4_pd(T3, 0), 0x20);
  tmp.packet[2] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 0), _mm512_extractf64x4_pd(T2, 0), 0x31);
  tmp.packet[3] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 0), _mm512_extractf64x4_pd(T3, 0), 0x31);
 
  tmp.packet[4] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 1), _mm512_extractf64x4_pd(T2, 1), 0x20);
  tmp.packet[5] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 1), _mm512_extractf64x4_pd(T3, 1), 0x20);
  tmp.packet[6] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 1), _mm512_extractf64x4_pd(T2, 1), 0x31);
  tmp.packet[7] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 1), _mm512_extractf64x4_pd(T3, 1), 0x31);
 
  PACK_OUTPUT_D(kernel.packet, tmp.packet, 0, 1);
  PACK_OUTPUT_D(kernel.packet, tmp.packet, 1, 1);
  PACK_OUTPUT_D(kernel.packet, tmp.packet, 2, 1);
  PACK_OUTPUT_D(kernel.packet, tmp.packet, 3, 1);
}

References PACK_OUTPUT_D, Eigen::PlainObjectBase< Derived >::packet(), Eigen::internal::PacketBlock< Packet, N >::packet, and tmp.

ptranspose() [50/68]

EIGEN_DEVICE_FUNC void Eigen::internal::ptranspose ( PacketBlock< Packet8d, 8 > &  kernel )

inline

                                                                           {
  __m512d T0 = _mm512_unpacklo_pd(kernel.packet[0], kernel.packet[1]);
  __m512d T1 = _mm512_unpackhi_pd(kernel.packet[0], kernel.packet[1]);
  __m512d T2 = _mm512_unpacklo_pd(kernel.packet[2], kernel.packet[3]);
  __m512d T3 = _mm512_unpackhi_pd(kernel.packet[2], kernel.packet[3]);
  __m512d T4 = _mm512_unpacklo_pd(kernel.packet[4], kernel.packet[5]);
  __m512d T5 = _mm512_unpackhi_pd(kernel.packet[4], kernel.packet[5]);
  __m512d T6 = _mm512_unpacklo_pd(kernel.packet[6], kernel.packet[7]);
  __m512d T7 = _mm512_unpackhi_pd(kernel.packet[6], kernel.packet[7]);
 
  kernel.packet[0] = _mm512_permutex_pd(T2, 0x4E);
  kernel.packet[0] = _mm512_mask_blend_pd(0xCC, T0, kernel.packet[0]);
  kernel.packet[2] = _mm512_permutex_pd(T0, 0x4E);
  kernel.packet[2] = _mm512_mask_blend_pd(0xCC, kernel.packet[2], T2);
  kernel.packet[1] = _mm512_permutex_pd(T3, 0x4E);
  kernel.packet[1] = _mm512_mask_blend_pd(0xCC, T1, kernel.packet[1]);
  kernel.packet[3] = _mm512_permutex_pd(T1, 0x4E);
  kernel.packet[3] = _mm512_mask_blend_pd(0xCC, kernel.packet[3], T3);
  kernel.packet[4] = _mm512_permutex_pd(T6, 0x4E);
  kernel.packet[4] = _mm512_mask_blend_pd(0xCC, T4, kernel.packet[4]);
  kernel.packet[6] = _mm512_permutex_pd(T4, 0x4E);
  kernel.packet[6] = _mm512_mask_blend_pd(0xCC, kernel.packet[6], T6);
  kernel.packet[5] = _mm512_permutex_pd(T7, 0x4E);
  kernel.packet[5] = _mm512_mask_blend_pd(0xCC, T5, kernel.packet[5]);
  kernel.packet[7] = _mm512_permutex_pd(T5, 0x4E);
  kernel.packet[7] = _mm512_mask_blend_pd(0xCC, kernel.packet[7], T7);
 
  T0 = _mm512_shuffle_f64x2(kernel.packet[4], kernel.packet[4], 0x4E);
  T0 = _mm512_mask_blend_pd(0xF0, kernel.packet[0], T0);
  T4 = _mm512_shuffle_f64x2(kernel.packet[0], kernel.packet[0], 0x4E);
  T4 = _mm512_mask_blend_pd(0xF0, T4, kernel.packet[4]);
  T1 = _mm512_shuffle_f64x2(kernel.packet[5], kernel.packet[5], 0x4E);
  T1 = _mm512_mask_blend_pd(0xF0, kernel.packet[1], T1);
  T5 = _mm512_shuffle_f64x2(kernel.packet[1], kernel.packet[1], 0x4E);
  T5 = _mm512_mask_blend_pd(0xF0, T5, kernel.packet[5]);
  T2 = _mm512_shuffle_f64x2(kernel.packet[6], kernel.packet[6], 0x4E);
  T2 = _mm512_mask_blend_pd(0xF0, kernel.packet[2], T2);
  T6 = _mm512_shuffle_f64x2(kernel.packet[2], kernel.packet[2], 0x4E);
  T6 = _mm512_mask_blend_pd(0xF0, T6, kernel.packet[6]);
  T3 = _mm512_shuffle_f64x2(kernel.packet[7], kernel.packet[7], 0x4E);
  T3 = _mm512_mask_blend_pd(0xF0, kernel.packet[3], T3);
  T7 = _mm512_shuffle_f64x2(kernel.packet[3], kernel.packet[3], 0x4E);
  T7 = _mm512_mask_blend_pd(0xF0, T7, kernel.packet[7]);
 
  kernel.packet[0] = T0;
  kernel.packet[1] = T1;
  kernel.packet[2] = T2;
  kernel.packet[3] = T3;
  kernel.packet[4] = T4;
  kernel.packet[5] = T5;
  kernel.packet[6] = T6;
  kernel.packet[7] = T7;
}

References Eigen::internal::PacketBlock< Packet, N >::packet.

ptranspose() [51/68]

EIGEN_DEVICE_FUNC void Eigen::internal::ptranspose ( PacketBlock< Packet8f, 4 > &  kernel )

inline

                                                                           {
  __m256 T0 = _mm256_unpacklo_ps(kernel.packet[0], kernel.packet[1]);
  __m256 T1 = _mm256_unpackhi_ps(kernel.packet[0], kernel.packet[1]);
  __m256 T2 = _mm256_unpacklo_ps(kernel.packet[2], kernel.packet[3]);
  __m256 T3 = _mm256_unpackhi_ps(kernel.packet[2], kernel.packet[3]);
 
  __m256 S0 = _mm256_shuffle_ps(T0, T2, _MM_SHUFFLE(1, 0, 1, 0));
  __m256 S1 = _mm256_shuffle_ps(T0, T2, _MM_SHUFFLE(3, 2, 3, 2));
  __m256 S2 = _mm256_shuffle_ps(T1, T3, _MM_SHUFFLE(1, 0, 1, 0));
  __m256 S3 = _mm256_shuffle_ps(T1, T3, _MM_SHUFFLE(3, 2, 3, 2));
 
  kernel.packet[0] = _mm256_permute2f128_ps(S0, S1, 0x20);
  kernel.packet[1] = _mm256_permute2f128_ps(S2, S3, 0x20);
  kernel.packet[2] = _mm256_permute2f128_ps(S0, S1, 0x31);
  kernel.packet[3] = _mm256_permute2f128_ps(S2, S3, 0x31);
}

References Eigen::internal::PacketBlock< Packet, N >::packet, GlobalParameters::S0, and GlobalParameters::S1.

ptranspose() [52/68]

EIGEN_DEVICE_FUNC void Eigen::internal::ptranspose ( PacketBlock< Packet8f, 8 > &  kernel )

inline

                                                                           {
  __m256 T0 = _mm256_unpacklo_ps(kernel.packet[0], kernel.packet[1]);
  __m256 T1 = _mm256_unpackhi_ps(kernel.packet[0], kernel.packet[1]);
  __m256 T2 = _mm256_unpacklo_ps(kernel.packet[2], kernel.packet[3]);
  __m256 T3 = _mm256_unpackhi_ps(kernel.packet[2], kernel.packet[3]);
  __m256 T4 = _mm256_unpacklo_ps(kernel.packet[4], kernel.packet[5]);
  __m256 T5 = _mm256_unpackhi_ps(kernel.packet[4], kernel.packet[5]);
  __m256 T6 = _mm256_unpacklo_ps(kernel.packet[6], kernel.packet[7]);
  __m256 T7 = _mm256_unpackhi_ps(kernel.packet[6], kernel.packet[7]);
  __m256 S0 = _mm256_shuffle_ps(T0, T2, _MM_SHUFFLE(1, 0, 1, 0));
  __m256 S1 = _mm256_shuffle_ps(T0, T2, _MM_SHUFFLE(3, 2, 3, 2));
  __m256 S2 = _mm256_shuffle_ps(T1, T3, _MM_SHUFFLE(1, 0, 1, 0));
  __m256 S3 = _mm256_shuffle_ps(T1, T3, _MM_SHUFFLE(3, 2, 3, 2));
  __m256 S4 = _mm256_shuffle_ps(T4, T6, _MM_SHUFFLE(1, 0, 1, 0));
  __m256 S5 = _mm256_shuffle_ps(T4, T6, _MM_SHUFFLE(3, 2, 3, 2));
  __m256 S6 = _mm256_shuffle_ps(T5, T7, _MM_SHUFFLE(1, 0, 1, 0));
  __m256 S7 = _mm256_shuffle_ps(T5, T7, _MM_SHUFFLE(3, 2, 3, 2));
  kernel.packet[0] = _mm256_permute2f128_ps(S0, S4, 0x20);
  kernel.packet[1] = _mm256_permute2f128_ps(S1, S5, 0x20);
  kernel.packet[2] = _mm256_permute2f128_ps(S2, S6, 0x20);
  kernel.packet[3] = _mm256_permute2f128_ps(S3, S7, 0x20);
  kernel.packet[4] = _mm256_permute2f128_ps(S0, S4, 0x31);
  kernel.packet[5] = _mm256_permute2f128_ps(S1, S5, 0x31);
  kernel.packet[6] = _mm256_permute2f128_ps(S2, S6, 0x31);
  kernel.packet[7] = _mm256_permute2f128_ps(S3, S7, 0x31);
}

References Eigen::internal::PacketBlock< Packet, N >::packet, GlobalParameters::S0, and GlobalParameters::S1.

ptranspose() [53/68]

EIGEN_STRONG_INLINE void Eigen::internal::ptranspose

(

PacketBlock< Packet8h, 4 > &

kernel

)

                                                                      {
  EIGEN_ALIGN32 Eigen::half in[4][8];
  pstore<Eigen::half>(in[0], kernel.packet[0]);
  pstore<Eigen::half>(in[1], kernel.packet[1]);
  pstore<Eigen::half>(in[2], kernel.packet[2]);
  pstore<Eigen::half>(in[3], kernel.packet[3]);
 
  EIGEN_ALIGN32 Eigen::half out[4][8];
 
  for (int i = 0; i < 4; ++i) {
    for (int j = 0; j < 4; ++j) {
      out[i][j] = in[j][2 * i];
    }
    for (int j = 0; j < 4; ++j) {
      out[i][j + 4] = in[j][2 * i + 1];
    }
  }
 
  kernel.packet[0] = pload<Packet8h>(out[0]);
  kernel.packet[1] = pload<Packet8h>(out[1]);
  kernel.packet[2] = pload<Packet8h>(out[2]);
  kernel.packet[3] = pload<Packet8h>(out[3]);
}

References EIGEN_ALIGN32, i, j, out(), Eigen::internal::PacketBlock< Packet, N >::packet, and pload< Packet8h >().

ptranspose() [54/68]

EIGEN_STRONG_INLINE void Eigen::internal::ptranspose

(

PacketBlock< Packet8h, 8 > &

kernel

)

                                                                      {
  __m128i a = kernel.packet[0];
  __m128i b = kernel.packet[1];
  __m128i c = kernel.packet[2];
  __m128i d = kernel.packet[3];
  __m128i e = kernel.packet[4];
  __m128i f = kernel.packet[5];
  __m128i g = kernel.packet[6];
  __m128i h = kernel.packet[7];
 
  __m128i a03b03 = _mm_unpacklo_epi16(a, b);
  __m128i c03d03 = _mm_unpacklo_epi16(c, d);
  __m128i e03f03 = _mm_unpacklo_epi16(e, f);
  __m128i g03h03 = _mm_unpacklo_epi16(g, h);
  __m128i a47b47 = _mm_unpackhi_epi16(a, b);
  __m128i c47d47 = _mm_unpackhi_epi16(c, d);
  __m128i e47f47 = _mm_unpackhi_epi16(e, f);
  __m128i g47h47 = _mm_unpackhi_epi16(g, h);
 
  __m128i a01b01c01d01 = _mm_unpacklo_epi32(a03b03, c03d03);
  __m128i a23b23c23d23 = _mm_unpackhi_epi32(a03b03, c03d03);
  __m128i e01f01g01h01 = _mm_unpacklo_epi32(e03f03, g03h03);
  __m128i e23f23g23h23 = _mm_unpackhi_epi32(e03f03, g03h03);
  __m128i a45b45c45d45 = _mm_unpacklo_epi32(a47b47, c47d47);
  __m128i a67b67c67d67 = _mm_unpackhi_epi32(a47b47, c47d47);
  __m128i e45f45g45h45 = _mm_unpacklo_epi32(e47f47, g47h47);
  __m128i e67f67g67h67 = _mm_unpackhi_epi32(e47f47, g47h47);
 
  __m128i a0b0c0d0e0f0g0h0 = _mm_unpacklo_epi64(a01b01c01d01, e01f01g01h01);
  __m128i a1b1c1d1e1f1g1h1 = _mm_unpackhi_epi64(a01b01c01d01, e01f01g01h01);
  __m128i a2b2c2d2e2f2g2h2 = _mm_unpacklo_epi64(a23b23c23d23, e23f23g23h23);
  __m128i a3b3c3d3e3f3g3h3 = _mm_unpackhi_epi64(a23b23c23d23, e23f23g23h23);
  __m128i a4b4c4d4e4f4g4h4 = _mm_unpacklo_epi64(a45b45c45d45, e45f45g45h45);
  __m128i a5b5c5d5e5f5g5h5 = _mm_unpackhi_epi64(a45b45c45d45, e45f45g45h45);
  __m128i a6b6c6d6e6f6g6h6 = _mm_unpacklo_epi64(a67b67c67d67, e67f67g67h67);
  __m128i a7b7c7d7e7f7g7h7 = _mm_unpackhi_epi64(a67b67c67d67, e67f67g67h67);
 
  kernel.packet[0] = a0b0c0d0e0f0g0h0;
  kernel.packet[1] = a1b1c1d1e1f1g1h1;
  kernel.packet[2] = a2b2c2d2e2f2g2h2;
  kernel.packet[3] = a3b3c3d3e3f3g3h3;
  kernel.packet[4] = a4b4c4d4e4f4g4h4;
  kernel.packet[5] = a5b5c5d5e5f5g5h5;
  kernel.packet[6] = a6b6c6d6e6f6g6h6;
  kernel.packet[7] = a7b7c7d7e7f7g7h7;
}

References a, b, calibrate::c, e(), f(), and Eigen::internal::PacketBlock< Packet, N >::packet.

ptranspose() [55/68]

EIGEN_DEVICE_FUNC void Eigen::internal::ptranspose ( PacketBlock< Packet8i, 4 > &  kernel )

inline

                                                                           {
  __m256i T0 = MM256_UNPACKLO_EPI32(kernel.packet[0], kernel.packet[1]);
  __m256i T1 = MM256_UNPACKHI_EPI32(kernel.packet[0], kernel.packet[1]);
  __m256i T2 = MM256_UNPACKLO_EPI32(kernel.packet[2], kernel.packet[3]);
  __m256i T3 = MM256_UNPACKHI_EPI32(kernel.packet[2], kernel.packet[3]);
 
  __m256i S0 = MM256_SHUFFLE_EPI32(T0, T2, _MM_SHUFFLE(1, 0, 1, 0));
  __m256i S1 = MM256_SHUFFLE_EPI32(T0, T2, _MM_SHUFFLE(3, 2, 3, 2));
  __m256i S2 = MM256_SHUFFLE_EPI32(T1, T3, _MM_SHUFFLE(1, 0, 1, 0));
  __m256i S3 = MM256_SHUFFLE_EPI32(T1, T3, _MM_SHUFFLE(3, 2, 3, 2));
 
  kernel.packet[0] = _mm256_permute2f128_si256(S0, S1, 0x20);
  kernel.packet[1] = _mm256_permute2f128_si256(S2, S3, 0x20);
  kernel.packet[2] = _mm256_permute2f128_si256(S0, S1, 0x31);
  kernel.packet[3] = _mm256_permute2f128_si256(S2, S3, 0x31);
}

References MM256_SHUFFLE_EPI32, MM256_UNPACKHI_EPI32, MM256_UNPACKLO_EPI32, Eigen::internal::PacketBlock< Packet, N >::packet, GlobalParameters::S0, and GlobalParameters::S1.

ptranspose() [56/68]

EIGEN_DEVICE_FUNC void Eigen::internal::ptranspose ( PacketBlock< Packet8i, 8 > &  kernel )

inline

                                                                           {
  __m256i T0 = MM256_UNPACKLO_EPI32(kernel.packet[0], kernel.packet[1]);
  __m256i T1 = MM256_UNPACKHI_EPI32(kernel.packet[0], kernel.packet[1]);
  __m256i T2 = MM256_UNPACKLO_EPI32(kernel.packet[2], kernel.packet[3]);
  __m256i T3 = MM256_UNPACKHI_EPI32(kernel.packet[2], kernel.packet[3]);
  __m256i T4 = MM256_UNPACKLO_EPI32(kernel.packet[4], kernel.packet[5]);
  __m256i T5 = MM256_UNPACKHI_EPI32(kernel.packet[4], kernel.packet[5]);
  __m256i T6 = MM256_UNPACKLO_EPI32(kernel.packet[6], kernel.packet[7]);
  __m256i T7 = MM256_UNPACKHI_EPI32(kernel.packet[6], kernel.packet[7]);
  __m256i S0 = MM256_SHUFFLE_EPI32(T0, T2, _MM_SHUFFLE(1, 0, 1, 0));
  __m256i S1 = MM256_SHUFFLE_EPI32(T0, T2, _MM_SHUFFLE(3, 2, 3, 2));
  __m256i S2 = MM256_SHUFFLE_EPI32(T1, T3, _MM_SHUFFLE(1, 0, 1, 0));
  __m256i S3 = MM256_SHUFFLE_EPI32(T1, T3, _MM_SHUFFLE(3, 2, 3, 2));
  __m256i S4 = MM256_SHUFFLE_EPI32(T4, T6, _MM_SHUFFLE(1, 0, 1, 0));
  __m256i S5 = MM256_SHUFFLE_EPI32(T4, T6, _MM_SHUFFLE(3, 2, 3, 2));
  __m256i S6 = MM256_SHUFFLE_EPI32(T5, T7, _MM_SHUFFLE(1, 0, 1, 0));
  __m256i S7 = MM256_SHUFFLE_EPI32(T5, T7, _MM_SHUFFLE(3, 2, 3, 2));
  kernel.packet[0] = _mm256_permute2f128_si256(S0, S4, 0x20);
  kernel.packet[1] = _mm256_permute2f128_si256(S1, S5, 0x20);
  kernel.packet[2] = _mm256_permute2f128_si256(S2, S6, 0x20);
  kernel.packet[3] = _mm256_permute2f128_si256(S3, S7, 0x20);
  kernel.packet[4] = _mm256_permute2f128_si256(S0, S4, 0x31);
  kernel.packet[5] = _mm256_permute2f128_si256(S1, S5, 0x31);
  kernel.packet[6] = _mm256_permute2f128_si256(S2, S6, 0x31);
  kernel.packet[7] = _mm256_permute2f128_si256(S3, S7, 0x31);
}

References MM256_SHUFFLE_EPI32, MM256_UNPACKHI_EPI32, MM256_UNPACKLO_EPI32, Eigen::internal::PacketBlock< Packet, N >::packet, GlobalParameters::S0, and GlobalParameters::S1.

ptranspose() [57/68]

EIGEN_DEVICE_FUNC void Eigen::internal::ptranspose ( PacketBlock< Packet8l, 4 > &  kernel )

inline

                                                                           {
  __m512i T0 = _mm512_castpd_si512(
      _mm512_shuffle_pd(_mm512_castsi512_pd(kernel.packet[0]), _mm512_castsi512_pd(kernel.packet[1]), 0));
  __m512i T1 = _mm512_castpd_si512(
      _mm512_shuffle_pd(_mm512_castsi512_pd(kernel.packet[0]), _mm512_castsi512_pd(kernel.packet[1]), 0xff));
  __m512i T2 = _mm512_castpd_si512(
      _mm512_shuffle_pd(_mm512_castsi512_pd(kernel.packet[2]), _mm512_castsi512_pd(kernel.packet[3]), 0));
  __m512i T3 = _mm512_castpd_si512(
      _mm512_shuffle_pd(_mm512_castsi512_pd(kernel.packet[2]), _mm512_castsi512_pd(kernel.packet[3]), 0xff));
 
  PacketBlock<Packet4l, 8> tmp;
 
  tmp.packet[0] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T0, 0), _mm512_extracti64x4_epi64(T2, 0), 0x20);
  tmp.packet[1] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T1, 0), _mm512_extracti64x4_epi64(T3, 0), 0x20);
  tmp.packet[2] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T0, 0), _mm512_extracti64x4_epi64(T2, 0), 0x31);
  tmp.packet[3] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T1, 0), _mm512_extracti64x4_epi64(T3, 0), 0x31);
 
  tmp.packet[4] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T0, 1), _mm512_extracti64x4_epi64(T2, 1), 0x20);
  tmp.packet[5] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T1, 1), _mm512_extracti64x4_epi64(T3, 1), 0x20);
  tmp.packet[6] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T0, 1), _mm512_extracti64x4_epi64(T2, 1), 0x31);
  tmp.packet[7] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T1, 1), _mm512_extracti64x4_epi64(T3, 1), 0x31);
 
  PACK_OUTPUT_L(kernel.packet, tmp.packet, 0, 1);
  PACK_OUTPUT_L(kernel.packet, tmp.packet, 1, 1);
  PACK_OUTPUT_L(kernel.packet, tmp.packet, 2, 1);
  PACK_OUTPUT_L(kernel.packet, tmp.packet, 3, 1);
}

References PACK_OUTPUT_L, Eigen::PlainObjectBase< Derived >::packet(), Eigen::internal::PacketBlock< Packet, N >::packet, and tmp.

ptranspose() [58/68]

EIGEN_DEVICE_FUNC void Eigen::internal::ptranspose ( PacketBlock< Packet8l, 8 > &  kernel )

inline

                                                                           {
  __m512i T0 = _mm512_unpacklo_epi64(kernel.packet[0], kernel.packet[1]);
  __m512i T1 = _mm512_unpackhi_epi64(kernel.packet[0], kernel.packet[1]);
  __m512i T2 = _mm512_unpacklo_epi64(kernel.packet[2], kernel.packet[3]);
  __m512i T3 = _mm512_unpackhi_epi64(kernel.packet[2], kernel.packet[3]);
  __m512i T4 = _mm512_unpacklo_epi64(kernel.packet[4], kernel.packet[5]);
  __m512i T5 = _mm512_unpackhi_epi64(kernel.packet[4], kernel.packet[5]);
  __m512i T6 = _mm512_unpacklo_epi64(kernel.packet[6], kernel.packet[7]);
  __m512i T7 = _mm512_unpackhi_epi64(kernel.packet[6], kernel.packet[7]);
 
  kernel.packet[0] = _mm512_permutex_epi64(T2, 0x4E);
  kernel.packet[0] = _mm512_mask_blend_epi64(0xCC, T0, kernel.packet[0]);
  kernel.packet[2] = _mm512_permutex_epi64(T0, 0x4E);
  kernel.packet[2] = _mm512_mask_blend_epi64(0xCC, kernel.packet[2], T2);
  kernel.packet[1] = _mm512_permutex_epi64(T3, 0x4E);
  kernel.packet[1] = _mm512_mask_blend_epi64(0xCC, T1, kernel.packet[1]);
  kernel.packet[3] = _mm512_permutex_epi64(T1, 0x4E);
  kernel.packet[3] = _mm512_mask_blend_epi64(0xCC, kernel.packet[3], T3);
  kernel.packet[4] = _mm512_permutex_epi64(T6, 0x4E);
  kernel.packet[4] = _mm512_mask_blend_epi64(0xCC, T4, kernel.packet[4]);
  kernel.packet[6] = _mm512_permutex_epi64(T4, 0x4E);
  kernel.packet[6] = _mm512_mask_blend_epi64(0xCC, kernel.packet[6], T6);
  kernel.packet[5] = _mm512_permutex_epi64(T7, 0x4E);
  kernel.packet[5] = _mm512_mask_blend_epi64(0xCC, T5, kernel.packet[5]);
  kernel.packet[7] = _mm512_permutex_epi64(T5, 0x4E);
  kernel.packet[7] = _mm512_mask_blend_epi64(0xCC, kernel.packet[7], T7);
 
  T0 = _mm512_shuffle_i64x2(kernel.packet[4], kernel.packet[4], 0x4E);
  T0 = _mm512_mask_blend_epi64(0xF0, kernel.packet[0], T0);
  T4 = _mm512_shuffle_i64x2(kernel.packet[0], kernel.packet[0], 0x4E);
  T4 = _mm512_mask_blend_epi64(0xF0, T4, kernel.packet[4]);
  T1 = _mm512_shuffle_i64x2(kernel.packet[5], kernel.packet[5], 0x4E);
  T1 = _mm512_mask_blend_epi64(0xF0, kernel.packet[1], T1);
  T5 = _mm512_shuffle_i64x2(kernel.packet[1], kernel.packet[1], 0x4E);
  T5 = _mm512_mask_blend_epi64(0xF0, T5, kernel.packet[5]);
  T2 = _mm512_shuffle_i64x2(kernel.packet[6], kernel.packet[6], 0x4E);
  T2 = _mm512_mask_blend_epi64(0xF0, kernel.packet[2], T2);
  T6 = _mm512_shuffle_i64x2(kernel.packet[2], kernel.packet[2], 0x4E);
  T6 = _mm512_mask_blend_epi64(0xF0, T6, kernel.packet[6]);
  T3 = _mm512_shuffle_i64x2(kernel.packet[7], kernel.packet[7], 0x4E);
  T3 = _mm512_mask_blend_epi64(0xF0, kernel.packet[3], T3);
  T7 = _mm512_shuffle_i64x2(kernel.packet[3], kernel.packet[3], 0x4E);
  T7 = _mm512_mask_blend_epi64(0xF0, T7, kernel.packet[7]);
 
  kernel.packet[0] = T0;
  kernel.packet[1] = T1;
  kernel.packet[2] = T2;
  kernel.packet[3] = T3;
  kernel.packet[4] = T4;
  kernel.packet[5] = T5;
  kernel.packet[6] = T6;
  kernel.packet[7] = T7;
}

References Eigen::internal::PacketBlock< Packet, N >::packet.

ptranspose() [59/68]

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::ptranspose ( PacketBlock< Packet8s, 4 > &  kernel )

inline

                                                                           {
  Packet8s t0, t1, t2, t3;
  t0 = vec_mergeh(kernel.packet[0], kernel.packet[2]);
  t1 = vec_mergel(kernel.packet[0], kernel.packet[2]);
  t2 = vec_mergeh(kernel.packet[1], kernel.packet[3]);
  t3 = vec_mergel(kernel.packet[1], kernel.packet[3]);
  kernel.packet[0] = vec_mergeh(t0, t2);
  kernel.packet[1] = vec_mergel(t0, t2);
  kernel.packet[2] = vec_mergeh(t1, t3);
  kernel.packet[3] = vec_mergel(t1, t3);
}

References Eigen::internal::PacketBlock< Packet, N >::packet.

ptranspose() [60/68]

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::ptranspose ( PacketBlock< Packet8s, 8 > &  kernel )

inline

                                                                           {
  Packet8s v[8], sum[8];
 
  v[0] = vec_mergeh(kernel.packet[0], kernel.packet[4]);
  v[1] = vec_mergel(kernel.packet[0], kernel.packet[4]);
  v[2] = vec_mergeh(kernel.packet[1], kernel.packet[5]);
  v[3] = vec_mergel(kernel.packet[1], kernel.packet[5]);
  v[4] = vec_mergeh(kernel.packet[2], kernel.packet[6]);
  v[5] = vec_mergel(kernel.packet[2], kernel.packet[6]);
  v[6] = vec_mergeh(kernel.packet[3], kernel.packet[7]);
  v[7] = vec_mergel(kernel.packet[3], kernel.packet[7]);
  sum[0] = vec_mergeh(v[0], v[4]);
  sum[1] = vec_mergel(v[0], v[4]);
  sum[2] = vec_mergeh(v[1], v[5]);
  sum[3] = vec_mergel(v[1], v[5]);
  sum[4] = vec_mergeh(v[2], v[6]);
  sum[5] = vec_mergel(v[2], v[6]);
  sum[6] = vec_mergeh(v[3], v[7]);
  sum[7] = vec_mergel(v[3], v[7]);
 
  kernel.packet[0] = vec_mergeh(sum[0], sum[4]);
  kernel.packet[1] = vec_mergel(sum[0], sum[4]);
  kernel.packet[2] = vec_mergeh(sum[1], sum[5]);
  kernel.packet[3] = vec_mergel(sum[1], sum[5]);
  kernel.packet[4] = vec_mergeh(sum[2], sum[6]);
  kernel.packet[5] = vec_mergel(sum[2], sum[6]);
  kernel.packet[6] = vec_mergeh(sum[3], sum[7]);
  kernel.packet[7] = vec_mergel(sum[3], sum[7]);
}

References Eigen::internal::PacketBlock< Packet, N >::packet, and v.

ptranspose() [61/68]

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::ptranspose

(

PacketBlock< Packet8uc, 4 > &

kernel

)

                                                                                         {
  detail::ptranspose_impl(kernel);
}

References Eigen::internal::detail::ptranspose_impl().

ptranspose() [62/68]

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::ptranspose

(

PacketBlock< Packet8uc, 8 > &

kernel

)

                                                                                         {
  detail::ptranspose_impl(kernel);
}

References Eigen::internal::detail::ptranspose_impl().

ptranspose() [63/68]

EIGEN_DEVICE_FUNC void Eigen::internal::ptranspose ( PacketBlock< Packet8ui, 4 > &  kernel )

inline

                                                                            {
  ptranspose((PacketBlock<Packet8i, 4>&)kernel);
}

References ptranspose().

ptranspose() [64/68]

EIGEN_DEVICE_FUNC void Eigen::internal::ptranspose ( PacketBlock< Packet8ui, 8 > &  kernel )

inline

                                                                            {
  ptranspose((PacketBlock<Packet8i, 8>&)kernel);
}

References ptranspose().

ptranspose() [65/68]

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::ptranspose ( PacketBlock< Packet8us, 4 > &  kernel )

inline

                                                                            {
  Packet8us t0, t1, t2, t3;
  t0 = vec_mergeh(kernel.packet[0], kernel.packet[2]);
  t1 = vec_mergel(kernel.packet[0], kernel.packet[2]);
  t2 = vec_mergeh(kernel.packet[1], kernel.packet[3]);
  t3 = vec_mergel(kernel.packet[1], kernel.packet[3]);
  kernel.packet[0] = vec_mergeh(t0, t2);
  kernel.packet[1] = vec_mergel(t0, t2);
  kernel.packet[2] = vec_mergeh(t1, t3);
  kernel.packet[3] = vec_mergel(t1, t3);
}

References Eigen::internal::PacketBlock< Packet, N >::packet.

ptranspose() [66/68]

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::ptranspose ( PacketBlock< Packet8us, 8 > &  kernel )

inline

                                                                            {
  Packet8us v[8], sum[8];
 
  v[0] = vec_mergeh(kernel.packet[0], kernel.packet[4]);
  v[1] = vec_mergel(kernel.packet[0], kernel.packet[4]);
  v[2] = vec_mergeh(kernel.packet[1], kernel.packet[5]);
  v[3] = vec_mergel(kernel.packet[1], kernel.packet[5]);
  v[4] = vec_mergeh(kernel.packet[2], kernel.packet[6]);
  v[5] = vec_mergel(kernel.packet[2], kernel.packet[6]);
  v[6] = vec_mergeh(kernel.packet[3], kernel.packet[7]);
  v[7] = vec_mergel(kernel.packet[3], kernel.packet[7]);
  sum[0] = vec_mergeh(v[0], v[4]);
  sum[1] = vec_mergel(v[0], v[4]);
  sum[2] = vec_mergeh(v[1], v[5]);
  sum[3] = vec_mergel(v[1], v[5]);
  sum[4] = vec_mergeh(v[2], v[6]);
  sum[5] = vec_mergel(v[2], v[6]);
  sum[6] = vec_mergeh(v[3], v[7]);
  sum[7] = vec_mergel(v[3], v[7]);
 
  kernel.packet[0] = vec_mergeh(sum[0], sum[4]);
  kernel.packet[1] = vec_mergel(sum[0], sum[4]);
  kernel.packet[2] = vec_mergeh(sum[1], sum[5]);
  kernel.packet[3] = vec_mergel(sum[1], sum[5]);
  kernel.packet[4] = vec_mergeh(sum[2], sum[6]);
  kernel.packet[5] = vec_mergel(sum[2], sum[6]);
  kernel.packet[6] = vec_mergeh(sum[3], sum[7]);
  kernel.packet[7] = vec_mergel(sum[3], sum[7]);
}

References Eigen::internal::PacketBlock< Packet, N >::packet, and v.

ptranspose() [67/68]

template<int N>

EIGEN_DEVICE_FUNC void Eigen::internal::ptranspose ( PacketBlock< PacketXf, N > &  kernel )

inline

                                                                           {
  float buffer[packet_traits<float>::size * N] = {0};
  int i = 0;
 
  PacketXi stride_index = svindex_s32(0, N);
 
  for (i = 0; i < N; i++) {
    svst1_scatter_s32index_f32(svptrue_b32(), buffer + i, stride_index, kernel.packet[i]);
  }
 
  for (i = 0; i < N; i++) {
    kernel.packet[i] = svld1_f32(svptrue_b32(), buffer + i * packet_traits<float>::size);
  }
}

References i, N, and Eigen::internal::PacketBlock< Packet, N >::packet.

ptranspose() [68/68]

template<int N>

EIGEN_DEVICE_FUNC void Eigen::internal::ptranspose ( PacketBlock< PacketXi, N > &  kernel )

inline

                                                                           {
  int buffer[packet_traits<numext::int32_t>::size * N] = {0};
  int i = 0;
 
  PacketXi stride_index = svindex_s32(0, N);
 
  for (i = 0; i < N; i++) {
    svst1_scatter_s32index_s32(svptrue_b32(), buffer + i, stride_index, kernel.packet[i]);
  }
  for (i = 0; i < N; i++) {
    kernel.packet[i] = svld1_s32(svptrue_b32(), buffer + i * packet_traits<numext::int32_t>::size);
  }
}

References i, N, and Eigen::internal::PacketBlock< Packet, N >::packet.

ptrue() [1/5]

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::ptrue ( const Packet &  a )

inline

Returns

one bits.

                                                       {
  return ptrue_impl<Packet>::run(a);
}

References a, and Eigen::internal::ptrue_impl< Packet, EnableIf >::run().

ptrue() [2/5]

template<>

EIGEN_STRONG_INLINE Packet16bf Eigen::internal::ptrue

(

const Packet16bf &

a

)

                                                          {
  return Packet16bf(ptrue<Packet8i>(Packet8i(a)));
}

References a, and ptrue< Packet8i >().

ptrue() [3/5]

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::ptrue

(

const Packet16h &

a

)

                                                        {
  return Packet16h(ptrue(Packet8i(a)));
}

References a, and ptrue().

ptrue() [4/5]

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::ptrue

(

const Packet8bf &

a

)

                                                        {
  return _mm_cmpeq_epi32(a, a);
}

References a.

ptrue() [5/5]

template<>

EIGEN_STRONG_INLINE Packet8h Eigen::internal::ptrue

(

const Packet8h &

a

)

                                                      {
  return _mm_cmpeq_epi32(a, a);
}

References a.

Referenced by Eigen::internal::any_visitor< Scalar >::any_predux(), Eigen::internal::scalar_isnan_op< Scalar, true >::operator()(), Eigen::internal::scalar_isinf_op< Scalar, true >::operator()(), Eigen::internal::scalar_isfinite_op< Scalar, true >::operator()(), packetmath_boolean_mask_ops(), Eigen::internal::scalar_bitwise_not_op< Scalar >::packetOp(), pcmp_eq(), pcmp_le(), pcmp_lt(), pcmp_lt_or_nan(), pisnan(), ptrue(), ptrue< Packet1cd >(), ptrue< Packet2cd >(), ptrue< Packet2cf >(), ptrue< Packet4cd >(), ptrue< Packet4cf >(), and ptrue< Packet8cf >().

ptrue< Packet16b >()

template<>

EIGEN_STRONG_INLINE Packet16b Eigen::internal::ptrue< Packet16b >

(

const Packet16b &

)

                                                                   {
  return pset1<Packet16b>(true);
}

References pset1< Packet16b >().

ptrue< Packet16f >()

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::ptrue< Packet16f >

(

const Packet16f &

a

)

                                                                   {
  return _mm512_castsi512_ps(ptrue<Packet16i>(_mm512_castps_si512(a)));
}

References a, and ptrue< Packet16i >().

ptrue< Packet16i >()

template<>

EIGEN_STRONG_INLINE Packet16i Eigen::internal::ptrue< Packet16i >

(

const Packet16i &

)

                                                                   {
  return _mm512_set1_epi32(int32_t(-1));
}

Referenced by ptrue< Packet16f >(), and ptrue< Packet8d >().

ptrue< Packet1cd >()

template<>

EIGEN_STRONG_INLINE Packet1cd Eigen::internal::ptrue< Packet1cd >

(

const Packet1cd &

a

)

                                                                   {
  return Packet1cd(ptrue(Packet2d(a.v)));
}

References a, and ptrue().

ptrue< Packet2cd >()

template<>

EIGEN_STRONG_INLINE Packet2cd Eigen::internal::ptrue< Packet2cd >

(

const Packet2cd &

a

)

                                                                   {
  return Packet2cd(ptrue(Packet4d(a.v)));
}

References a, and ptrue().

ptrue< Packet2cf >()

template<>

EIGEN_STRONG_INLINE Packet2cf Eigen::internal::ptrue< Packet2cf >

(

const Packet2cf &

a

)

                                                                   {
  return Packet2cf(ptrue(Packet4f(a.v)));
}

References a, and ptrue().

ptrue< Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::ptrue< Packet2d >

(

const Packet2d &

a

)

                                                                {
  Packet4i b = _mm_castpd_si128(a);
  return _mm_castsi128_pd(_mm_cmpeq_epi32(b, b));
}

References a, and b.

ptrue< Packet2l >()

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::ptrue< Packet2l >

(

const Packet2l &

a

)

                                                                {
  return _mm_cmpeq_epi32(a, a);
}

References a.

ptrue< Packet4cd >()

template<>

EIGEN_STRONG_INLINE Packet4cd Eigen::internal::ptrue< Packet4cd >

(

const Packet4cd &

a

)

                                                                   {
  return Packet4cd(ptrue(Packet8d(a.v)));
}

References a, and ptrue().

ptrue< Packet4cf >()

template<>

EIGEN_STRONG_INLINE Packet4cf Eigen::internal::ptrue< Packet4cf >

(

const Packet4cf &

a

)

                                                                   {
  return Packet4cf(ptrue(Packet8f(a.v)));
}

References a, and ptrue().

ptrue< Packet4d >()

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::ptrue< Packet4d >

(

const Packet4d &

a

)

                                                                {
#ifdef EIGEN_VECTORIZE_AVX2
  // vpcmpeqq has lower latency than the more general vcmppd
  const __m256i b = _mm256_castpd_si256(a);
  return _mm256_castsi256_pd(_mm256_cmpeq_epi64(b, b));
#else
  return _mm256_cmp_pd(a, a, _CMP_TRUE_UQ);
#endif
}

References a, and b.

ptrue< Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::ptrue< Packet4f >

(

const Packet4f &

a

)

                                                                {
  Packet4i b = _mm_castps_si128(a);
  return _mm_castsi128_ps(_mm_cmpeq_epi32(b, b));
}

References a, and b.

ptrue< Packet4i >()

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::ptrue< Packet4i >

(

const Packet4i &

a

)

                                                                {
  return _mm_cmpeq_epi32(a, a);
}

References a.

ptrue< Packet8cf >()

template<>

EIGEN_STRONG_INLINE Packet8cf Eigen::internal::ptrue< Packet8cf >

(

const Packet8cf &

a

)

                                                                   {
  return Packet8cf(ptrue(Packet16f(a.v)));
}

References a, and ptrue().

ptrue< Packet8d >()

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::ptrue< Packet8d >

(

const Packet8d &

a

)

                                                                {
  return _mm512_castsi512_pd(ptrue<Packet16i>(_mm512_castpd_si512(a)));
}

References a, and ptrue< Packet16i >().

ptrue< Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::ptrue< Packet8f >

(

const Packet8f &

a

)

                                                                {
#ifdef EIGEN_VECTORIZE_AVX2
  // vpcmpeqd has lower latency than the more general vcmpps
  const __m256i b = _mm256_castps_si256(a);
  return _mm256_castsi256_ps(_mm256_cmpeq_epi32(b, b));
#else
  return _mm256_cmp_ps(a, a, _CMP_TRUE_UQ);
#endif
}

References a, and b.

ptrue< Packet8i >()

template<>

EIGEN_STRONG_INLINE Packet8i Eigen::internal::ptrue< Packet8i >

(

const Packet8i &

a

)

                                                                {
#ifdef EIGEN_VECTORIZE_AVX2
  // vpcmpeqd has lower latency than the more general vcmpps
  return _mm256_cmpeq_epi32(a, a);
#else
  const __m256 b = _mm256_castsi256_ps(a);
  return _mm256_castps_si256(_mm256_cmp_ps(b, b, _CMP_TRUE_UQ));
#endif
}

References a, and b.

Referenced by ptrue().

ptrue< Packet8l >()

template<>

EIGEN_STRONG_INLINE Packet8l Eigen::internal::ptrue< Packet8l >

(

const Packet8l &

)

                                                                {
  return _mm512_set1_epi64(int64_t(-1));
}

ptrue< PacketXf >()

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::ptrue< PacketXf >

(

const PacketXf &

)

                                                                {
  return svreinterpret_f32_u32(svdup_n_u32_x(svptrue_b32(), 0xffffffffu));
}

ptrue< PacketXi >()

template<>

EIGEN_STRONG_INLINE PacketXi Eigen::internal::ptrue< PacketXi >

(

const PacketXi &

)

                                                                {
  return svdup_n_s32_x(svptrue_b32(), 0xffffffffu);
}

ptrunc()

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet Eigen::internal::ptrunc

(

const Packet &

a

)

Returns

the truncation of a (coeff-wise)

                                                                     {
  return nearest_integer_packetop_impl<Packet>::run_trunc(a);
}

References a, and Eigen::internal::nearest_integer_packetop_impl< Packet, IsScalar, IsInteger >::run_trunc().

Referenced by packetmath_real(), and Eigen::internal::scalar_trunc_op< Scalar >::packetOp().

ptrunc< Packet16bf >()

template<>

EIGEN_STRONG_INLINE Packet16bf Eigen::internal::ptrunc< Packet16bf >

(

const Packet16bf &

a

)

                                                                       {
  return F32ToBf16(ptrunc<Packet16f>(Bf16ToF32(a)));
}

References a, Bf16ToF32(), F32ToBf16(), and ptrunc< Packet16f >().

ptrunc< Packet16f >()

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::ptrunc< Packet16f >

(

const Packet16f &

a

)

                                                                    {
  return _mm512_roundscale_ps(a, _MM_FROUND_TO_ZERO);
}

References a.

Referenced by ptrunc< Packet16bf >(), and ptrunc< Packet16h >().

ptrunc< Packet16h >()

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::ptrunc< Packet16h >

(

const Packet16h &

a

)

                                                                    {
  return float2half(ptrunc<Packet16f>(half2float(a)));
}

References a, float2half(), half2float(), and ptrunc< Packet16f >().

ptrunc< Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::ptrunc< Packet2d >

(

const Packet2d &

a

)

                                                                 {
  return __lsx_vfrintrz_d(a);
}

References a.

ptrunc< Packet32h >()

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::ptrunc< Packet32h >

(

const Packet32h &

a

)

                                                                    {
  return _mm512_roundscale_ph(a, _MM_FROUND_TO_ZERO);
}

References a.

ptrunc< Packet4bf >()

template<>

EIGEN_STRONG_INLINE Packet4bf Eigen::internal::ptrunc< Packet4bf >

(

const Packet4bf &

a

)

                                                                    {
  return F32ToBf16(ptrunc<Packet4f>(Bf16ToF32(a)));
}

References a, Bf16ToF32(), F32ToBf16(), and ptrunc< Packet4f >().

ptrunc< Packet4d >()

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::ptrunc< Packet4d >

(

const Packet4d &

a

)

                                                                 {
  return _mm256_round_pd(a, _MM_FROUND_TRUNC);
}

References a.

ptrunc< Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::ptrunc< Packet4f >

(

const Packet4f &

a

)

                                                                 {
  return vec_trunc(a);
}

References a.

Referenced by ptrunc< Packet4bf >(), and ptrunc< Packet8bf >().

ptrunc< Packet8bf >()

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::ptrunc< Packet8bf >

(

const Packet8bf &

a

)

                                                                    {
  BF16_TO_F32_UNARY_OP_WRAPPER(ptrunc<Packet4f>, a);
}

References a, BF16_TO_F32_UNARY_OP_WRAPPER, and ptrunc< Packet4f >().

ptrunc< Packet8d >()

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::ptrunc< Packet8d >

(

const Packet8d &

a

)

                                                                 {
  return _mm512_roundscale_pd(a, _MM_FROUND_TO_ZERO);
}

References a.

ptrunc< Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::ptrunc< Packet8f >

(

const Packet8f &

a

)

                                                                 {
  return _mm256_round_ps(a, _MM_FROUND_TRUNC);
}

References a.

Referenced by ptrunc< Packet8h >().

ptrunc< Packet8h >()

template<>

EIGEN_STRONG_INLINE Packet8h Eigen::internal::ptrunc< Packet8h >

(

const Packet8h &

a

)

                                                                 {
  return float2half(ptrunc<Packet8f>(half2float(a)));
}

References a, float2half(), half2float(), and ptrunc< Packet8f >().

punpackp()

EIGEN_STRONG_INLINE void Eigen::internal::punpackp

(

Packet4f *

vecs

)

                                                  {
  vecs[1] = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(vecs[0]), 0x55));
  vecs[2] = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(vecs[0]), 0xAA));
  vecs[3] = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(vecs[0]), 0xFF));
  vecs[0] = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(vecs[0]), 0x00));
}

putDenseElt() [1/2]

template<typename Scalar >

void Eigen::internal::putDenseElt ( Scalar  value, std::ofstream &  out  )

inline

                                                        {
  out << value << "\n";
}

References out(), and Eigen::value.

Referenced by Eigen::saveMarketDense().

putDenseElt() [2/2]

template<typename Scalar >

void Eigen::internal::putDenseElt ( std::complex< Scalar >  value, std::ofstream &  out  )

inline

                                                                    {
  out << value.real() << " " << value.imag() << "\n";
}

References out(), and Eigen::value.

putMarketHeader()

template<typename Scalar >

void Eigen::internal::putMarketHeader ( std::string &  header, int  sym  )

inline

                                                        {
  header = "%%MatrixMarket matrix coordinate ";
  if (internal::is_same<Scalar, std::complex<float> >::value ||
      internal::is_same<Scalar, std::complex<double> >::value) {
    header += " complex";
    if (sym == Symmetric)
      header += " symmetric";
    else if (sym == SelfAdjoint)
      header += " Hermitian";
    else
      header += " general";
  } else {
    header += " real";
    if (sym == Symmetric)
      header += " symmetric";
    else
      header += " general";
  }
}

References Eigen::SelfAdjoint, Eigen::Symmetric, and Eigen::value.

PutMatrixElt() [1/2]

template<typename Scalar , typename StorageIndex >

void Eigen::internal::PutMatrixElt ( Scalar  value, StorageIndex  row, StorageIndex  col, std::ofstream &  out  )

inline

                                                                                             {
  out << row << " " << col << " " << value << "\n";
}

References col(), out(), row(), and Eigen::value.

Referenced by Eigen::saveMarket().

PutMatrixElt() [2/2]

template<typename Scalar , typename StorageIndex >

void Eigen::internal::PutMatrixElt ( std::complex< Scalar >  value, StorageIndex  row, StorageIndex  col, std::ofstream &  out  )

inline

                                                                                                         {
  out << row << " " << col << " " << value.real() << " " << value.imag() << "\n";
}

References col(), out(), row(), and Eigen::value.

pxor() [1/7]

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::pxor ( const Packet &  a, const Packet &  b  )

inline

Returns

the bitwise xor of a and b

                                                                       {
  return bitwise_helper<Packet>::bitwise_xor(a, b);
}

References a, b, and Eigen::internal::bytewise_bitwise_helper< T >::bitwise_xor().

pxor() [2/7]

template<>

EIGEN_STRONG_INLINE Packet16bf Eigen::internal::pxor	(	const Packet16bf &	a,
		const Packet16bf &	b
	)

                                                                              {
  return Packet16bf(pxor<Packet8i>(Packet8i(a), Packet8i(b)));
}

References a, b, and pxor< Packet8i >().

pxor() [3/7]

template<>

EIGEN_STRONG_INLINE Packet16h Eigen::internal::pxor	(	const Packet16h &	a,
		const Packet16h &	b
	)

                                                                           {
  return Packet16h(pxor(Packet8i(a), Packet8i(b)));
}

References a, b, and pxor().

pxor() [4/7]

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::pxor	(	const Packet32h &	a,
		const Packet32h &	b
	)

                                                                           {
  return _mm512_castsi512_ph(_mm512_xor_si512(_mm512_castph_si512(a), _mm512_castph_si512(b)));
}

References a, and b.

pxor() [5/7]

template<>

EIGEN_STRONG_INLINE Packet4bf Eigen::internal::pxor	(	const Packet4bf &	a,
		const Packet4bf &	b
	)

                                                                           {
  return Packet4bf(pxor<Packet4us>(Packet4us(a), Packet4us(b)));
}

References a, b, and pxor< Packet4us >().

pxor() [6/7]

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::pxor	(	const Packet8bf &	a,
		const Packet8bf &	b
	)

                                                                           {
  return _mm_xor_si128(a, b);
}

References a, and b.

pxor() [7/7]

template<>

EIGEN_STRONG_INLINE Packet8h Eigen::internal::pxor	(	const Packet8h &	a,
		const Packet8h &	b
	)

                                                                        {
  return _mm_xor_si128(a, b);
}

References a, and b.

Referenced by Eigen::internal::Packet1cd::conjugate(), generic_atan(), generic_pow(), packetmath(), Eigen::internal::scalar_boolean_xor_op< Scalar >::packetOp(), Eigen::internal::scalar_bitwise_xor_op< Scalar >::packetOp(), paddsub< Packet2d >(), paddsub< Packet2f >(), paddsub< Packet4f >(), pasin_float(), pcmp_lt(), pconj(), pconj2(), pconjinv(), Eigen::internal::Packet2cf::pmul(), pnegate2(), pround(), psincos_double(), psincos_float(), psincos_inner_msa_float(), pxor(), pxor< Packet16f >(), pxor< Packet4cd >(), pxor< Packet8cf >(), pxor< Packet8d >(), Eigen::internal::compute_inverse_size4< Architecture::Target, float, MatrixType, ResultType >::run(), Eigen::internal::compute_inverse_size4< Architecture::Target, double, MatrixType, ResultType >::run(), Eigen::internal::quat_product< Architecture::Target, Derived, OtherDerived, float >::run(), and Eigen::internal::quat_conj< Architecture::Target, Derived, float >::run().

pxor< Packet16b >()

template<>

EIGEN_STRONG_INLINE Packet16b Eigen::internal::pxor< Packet16b >	(	const Packet16b &	a,
		const Packet16b &	b
	)

                                                                                      {
  return _mm_xor_si128(a, b);
}

References a, and b.

pxor< Packet16c >()

template<>

EIGEN_STRONG_INLINE Packet16c Eigen::internal::pxor< Packet16c >	(	const Packet16c &	a,
		const Packet16c &	b
	)

                                                                                      {
  return __lsx_vxor_v(a, b);
}

References a, and b.

pxor< Packet16f >()

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::pxor< Packet16f >	(	const Packet16f &	a,
		const Packet16f &	b
	)

                                                                                      {
#ifdef EIGEN_VECTORIZE_AVX512DQ
  return _mm512_xor_ps(a, b);
#else
  return _mm512_castsi512_ps(pxor(_mm512_castps_si512(a), _mm512_castps_si512(b)));
#endif
}

References a, b, and pxor().

pxor< Packet16i >()

template<>

EIGEN_STRONG_INLINE Packet16i Eigen::internal::pxor< Packet16i >	(	const Packet16i &	a,
		const Packet16i &	b
	)

                                                                                      {
  return _mm512_xor_si512(a, b);
}

References a, and b.

pxor< Packet16uc >()

template<>

EIGEN_STRONG_INLINE Packet16uc Eigen::internal::pxor< Packet16uc >	(	const Packet16uc &	a,
		const Packet16uc &	b
	)

                                                                                          {
  return __lsx_vxor_v(a, b);
}

References a, and b.

pxor< Packet1cd >()

template<>

EIGEN_STRONG_INLINE Packet1cd Eigen::internal::pxor< Packet1cd >	(	const Packet1cd &	a,
		const Packet1cd &	b
	)

                                                                                      {
  Packet1cd res;
  res.v = (Packet2d)__lsx_vxor_v((__m128i)a.v, (__m128i)b.v);
  return res;
}

References a, b, and res.

pxor< Packet1cf >()

template<>

EIGEN_STRONG_INLINE Packet1cf Eigen::internal::pxor< Packet1cf >	(	const Packet1cf &	a,
		const Packet1cf &	b
	)

                                                                                      {
  return Packet1cf(vreinterpret_f32_u32(veor_u32(vreinterpret_u32_f32(a.v), vreinterpret_u32_f32(b.v))));
}

References a, and b.

pxor< Packet2cd >()

template<>

EIGEN_STRONG_INLINE Packet2cd Eigen::internal::pxor< Packet2cd >	(	const Packet2cd &	a,
		const Packet2cd &	b
	)

                                                                                      {
  return Packet2cd(_mm256_xor_pd(a.v, b.v));
}

References a, and b.

pxor< Packet2cf >()

template<>

EIGEN_STRONG_INLINE Packet2cf Eigen::internal::pxor< Packet2cf >	(	const Packet2cf &	a,
		const Packet2cf &	b
	)

                                                                                      {
  return Packet2cf(pxor<Packet4f>(a.v, b.v));
}

References a, b, and pxor< Packet4f >().

pxor< Packet2d >()

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pxor< Packet2d >	(	const Packet2d &	a,
		const Packet2d &	b
	)

                                                                                  {
  return (Packet2d)__lsx_vxor_v((__m128i)a, (__m128i)b);
}

References a, and b.

pxor< Packet2f >()

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::pxor< Packet2f >	(	const Packet2f &	a,
		const Packet2f &	b
	)

                                                                                  {
  return vreinterpret_f32_u32(veor_u32(vreinterpret_u32_f32(a), vreinterpret_u32_f32(b)));
}

References a, and b.

pxor< Packet2i >()

template<>

EIGEN_STRONG_INLINE Packet2i Eigen::internal::pxor< Packet2i >	(	const Packet2i &	a,
		const Packet2i &	b
	)

                                                                                  {
  return veor_s32(a, b);
}

References a, and b.

pxor< Packet2l >()

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::pxor< Packet2l >	(	const Packet2l &	a,
		const Packet2l &	b
	)

                                                                                  {
  return __lsx_vxor_v(a, b);
}

References a, and b.

pxor< Packet2ui >()

template<>

EIGEN_STRONG_INLINE Packet2ui Eigen::internal::pxor< Packet2ui >	(	const Packet2ui &	a,
		const Packet2ui &	b
	)

                                                                                      {
  return veor_u32(a, b);
}

References a, and b.

pxor< Packet2ul >()

template<>

EIGEN_STRONG_INLINE Packet2ul Eigen::internal::pxor< Packet2ul >	(	const Packet2ul &	a,
		const Packet2ul &	b
	)

                                                                                      {
  return __lsx_vxor_v(a, b);
}

References a, and b.

pxor< Packet4c >()

template<>

EIGEN_STRONG_INLINE Packet4c Eigen::internal::pxor< Packet4c >	(	const Packet4c &	a,
		const Packet4c &	b
	)

                                                                                  {
  return a ^ b;
}

References a, and b.

pxor< Packet4cd >()

template<>

EIGEN_STRONG_INLINE Packet4cd Eigen::internal::pxor< Packet4cd >	(	const Packet4cd &	a,
		const Packet4cd &	b
	)

                                                                                      {
  return Packet4cd(pxor(a.v, b.v));
}

References a, b, and pxor().

pxor< Packet4cf >()

template<>

EIGEN_STRONG_INLINE Packet4cf Eigen::internal::pxor< Packet4cf >	(	const Packet4cf &	a,
		const Packet4cf &	b
	)

                                                                                      {
  return Packet4cf(_mm256_xor_ps(a.v, b.v));
}

References a, and b.

pxor< Packet4d >()

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::pxor< Packet4d >	(	const Packet4d &	a,
		const Packet4d &	b
	)

                                                                                  {
  return _mm256_xor_pd(a, b);
}

References a, and b.

pxor< Packet4f >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pxor< Packet4f >	(	const Packet4f &	a,
		const Packet4f &	b
	)

                                                                                  {
  return vec_xor(a, b);
}

References a, and b.

Referenced by pconj(), and pxor< Packet2cf >().

pxor< Packet4i >()

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pxor< Packet4i >	(	const Packet4i &	a,
		const Packet4i &	b
	)

                                                                                  {
  return vec_xor(a, b);
}

References a, and b.

pxor< Packet4s >()

template<>

EIGEN_STRONG_INLINE Packet4s Eigen::internal::pxor< Packet4s >	(	const Packet4s &	a,
		const Packet4s &	b
	)

                                                                                  {
  return veor_s16(a, b);
}

References a, and b.

pxor< Packet4uc >()

template<>

EIGEN_STRONG_INLINE Packet4uc Eigen::internal::pxor< Packet4uc >	(	const Packet4uc &	a,
		const Packet4uc &	b
	)

                                                                                      {
  return a ^ b;
}

References a, and b.

pxor< Packet4ui >()

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::pxor< Packet4ui >	(	const Packet4ui &	a,
		const Packet4ui &	b
	)

                                                                                      {
  return __lsx_vxor_v(a, b);
}

References a, and b.

pxor< Packet4us >()

template<>

EIGEN_STRONG_INLINE Packet4us Eigen::internal::pxor< Packet4us >	(	const Packet4us &	a,
		const Packet4us &	b
	)

                                                                                      {
  return veor_u16(a, b);
}

References a, and b.

Referenced by pnegate< Packet4bf >(), and pxor().

pxor< Packet8bf >()

template<>

EIGEN_STRONG_INLINE Packet8bf Eigen::internal::pxor< Packet8bf >	(	const Packet8bf &	a,
		const Packet8bf &	b
	)

                                                                                      {
  return pxor<Packet8us>(a, b);
}

References a, b, and pxor< Packet8us >().

pxor< Packet8c >()

template<>

EIGEN_STRONG_INLINE Packet8c Eigen::internal::pxor< Packet8c >	(	const Packet8c &	a,
		const Packet8c &	b
	)

                                                                                  {
  return veor_s8(a, b);
}

References a, and b.

pxor< Packet8cf >()

template<>

EIGEN_STRONG_INLINE Packet8cf Eigen::internal::pxor< Packet8cf >	(	const Packet8cf &	a,
		const Packet8cf &	b
	)

                                                                                      {
  return Packet8cf(pxor(a.v, b.v));
}

References a, b, and pxor().

pxor< Packet8d >()

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::pxor< Packet8d >	(	const Packet8d &	a,
		const Packet8d &	b
	)

                                                                                  {
#ifdef EIGEN_VECTORIZE_AVX512DQ
  return _mm512_xor_pd(a, b);
#else
  return _mm512_castsi512_pd(pxor(_mm512_castpd_si512(a), _mm512_castpd_si512(b)));
#endif
}

References a, b, and pxor().

pxor< Packet8f >()

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::pxor< Packet8f >	(	const Packet8f &	a,
		const Packet8f &	b
	)

                                                                                  {
  return _mm256_xor_ps(a, b);
}

References a, and b.

pxor< Packet8i >()

template<>

EIGEN_STRONG_INLINE Packet8i Eigen::internal::pxor< Packet8i >	(	const Packet8i &	a,
		const Packet8i &	b
	)

                                                                                  {
#ifdef EIGEN_VECTORIZE_AVX2
  return _mm256_xor_si256(a, b);
#else
  return _mm256_castps_si256(_mm256_xor_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b)));
#endif
}

References a, and b.

Referenced by pxor().

pxor< Packet8l >()

template<>

EIGEN_STRONG_INLINE Packet8l Eigen::internal::pxor< Packet8l >	(	const Packet8l &	a,
		const Packet8l &	b
	)

                                                                                  {
  return _mm512_xor_si512(a, b);
}

References a, and b.

pxor< Packet8s >()

template<>

EIGEN_STRONG_INLINE Packet8s Eigen::internal::pxor< Packet8s >	(	const Packet8s &	a,
		const Packet8s &	b
	)

                                                                                  {
  return __lsx_vxor_v(a, b);
}

References a, and b.

pxor< Packet8uc >()

template<>

EIGEN_STRONG_INLINE Packet8uc Eigen::internal::pxor< Packet8uc >	(	const Packet8uc &	a,
		const Packet8uc &	b
	)

                                                                                      {
  return veor_u8(a, b);
}

References a, and b.

pxor< Packet8ui >()

template<>

EIGEN_STRONG_INLINE Packet8ui Eigen::internal::pxor< Packet8ui >	(	const Packet8ui &	a,
		const Packet8ui &	b
	)

                                                                                      {
#ifdef EIGEN_VECTORIZE_AVX2
  return _mm256_xor_si256(a, b);
#else
  return _mm256_castps_si256(_mm256_xor_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b)));
#endif
}

References a, and b.

pxor< Packet8us >()

template<>

EIGEN_STRONG_INLINE Packet8us Eigen::internal::pxor< Packet8us >	(	const Packet8us &	a,
		const Packet8us &	b
	)

                                                                                      {
  return vec_xor(a, b);
}

References a, and b.

Referenced by pnegate< Packet8bf >(), and pxor< Packet8bf >().

pxor< PacketXf >()

template<>

EIGEN_STRONG_INLINE PacketXf Eigen::internal::pxor< PacketXf >	(	const PacketXf &	a,
		const PacketXf &	b
	)

                                                                                  {
  return svreinterpret_f32_u32(sveor_u32_x(svptrue_b32(), svreinterpret_u32_f32(a), svreinterpret_u32_f32(b)));
}

References a, and b.

pxor< PacketXi >()

template<>

EIGEN_STRONG_INLINE PacketXi Eigen::internal::pxor< PacketXi >	(	const PacketXi &	a,
		const PacketXi &	b
	)

                                                                                  {
  return sveor_s32_x(svptrue_b32(), a, b);
}

References a, and b.

pzero() [1/18]

template<typename Packet >

EIGEN_DEVICE_FUNC Packet Eigen::internal::pzero ( const Packet &  a )

inline

Returns

packet of zeros

                                                       {
  return pzero_impl<Packet>::run(a);
}

References a, and Eigen::internal::pzero_impl< Packet, EnableIf >::run().

pzero() [2/18]

template<>

EIGEN_STRONG_INLINE Packet16f Eigen::internal::pzero

(

const Packet16f &

)

                                                        {
  return _mm512_setzero_ps();
}

pzero() [3/18]

template<>

EIGEN_STRONG_INLINE Packet16i Eigen::internal::pzero

(

const Packet16i &

)

                                                        {
  return _mm512_setzero_si512();
}

pzero() [4/18]

template<>

EIGEN_STRONG_INLINE Packet1cf Eigen::internal::pzero

(

const Packet1cf &

)

                                                        {
  return Packet1cf(vdup_n_f32(0.0f));
}

pzero() [5/18]

template<>

EIGEN_STRONG_INLINE Packet2cf Eigen::internal::pzero

(

const Packet2cf &

)

                                                        {
  __m128 v = {0.0f, 0.0f, 0.0f, 0.0f};
  return (Packet2cf)v;
}

References v.

pzero() [6/18]

template<>

EIGEN_STRONG_INLINE Packet2d Eigen::internal::pzero

(

const Packet2d &

)

                                                      {
  Packet2d v = {0.0, 0.0};
  return v;
}

References v.

pzero() [7/18]

template<>

EIGEN_STRONG_INLINE Packet2f Eigen::internal::pzero

(

const Packet2f &

)

                                                      {
  return vdup_n_f32(0.0f);
}

pzero() [8/18]

template<>

EIGEN_STRONG_INLINE Packet2l Eigen::internal::pzero

(

const Packet2l &

)

                                                      {
  return _mm_setzero_si128();
}

pzero() [9/18]

template<>

EIGEN_STRONG_INLINE Packet32h Eigen::internal::pzero

(

const Packet32h &

)

                                                        {
  return _mm512_setzero_ph();
}

pzero() [10/18]

template<>

EIGEN_STRONG_INLINE Packet4d Eigen::internal::pzero

(

const Packet4d &

)

                                                      {
  return _mm256_setzero_pd();
}

pzero() [11/18]

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::pzero

(

const Packet4f &

)

                                                      {
  Packet4f v = {0.0f, 0.0f, 0.0f, 0.0f};
  return v;
}

References v.

pzero() [12/18]

template<>

EIGEN_STRONG_INLINE Packet4i Eigen::internal::pzero

(

const Packet4i &

)

                                                      {
  return _mm_setzero_si128();
}

pzero() [13/18]

template<>

EIGEN_STRONG_INLINE Packet4ui Eigen::internal::pzero

(

const Packet4ui &

)

                                                        {
  return _mm_setzero_si128();
}

pzero() [14/18]

template<>

EIGEN_STRONG_INLINE Packet8d Eigen::internal::pzero

(

const Packet8d &

)

                                                      {
  return _mm512_setzero_pd();
}

pzero() [15/18]

template<>

EIGEN_STRONG_INLINE Packet8f Eigen::internal::pzero

(

const Packet8f &

)

                                                      {
  return _mm256_setzero_ps();
}

Referenced by Eigen::internal::all_visitor< Scalar >::all_predux(), Eigen::internal::any_visitor< Scalar >::any_predux(), Eigen::internal::count_visitor< Scalar >::count_redux(), Eigen::internal::gemm_class< Scalar, is_unit_inc >::gemm_class(), Eigen::internal::unary_pow::handle_nonint_nonint_errors(), Eigen::internal::accurate_log2< Scalar >::operator()(), Eigen::internal::scalar_isnan_op< Scalar, true >::operator()(), Eigen::internal::scalar_isinf_op< Scalar, true >::operator()(), Eigen::internal::scalar_isfinite_op< Scalar, true >::operator()(), pabs(), packetmath(), Eigen::internal::scalar_boolean_not_op< Scalar >::packetOp(), Eigen::internal::scalar_boolean_and_op< Scalar >::packetOp(), Eigen::internal::scalar_boolean_or_op< Scalar >::packetOp(), Eigen::internal::scalar_boolean_xor_op< Scalar >::packetOp(), Eigen::internal::scalar_boolean_select_op< ThenScalar, ElseScalar, ConditionScalar >::packetOp(), pcast< Packet16f, Packet16b >(), pcast< Packet4f, Packet16b >(), pcast< Packet8f, Packet16b >(), pcmp_eq(), pcmp_le(), pcmp_lt(), pcmp_lt_or_nan(), pexp_complex(), pfrexp_generic(), pldexp_fast(), plog_impl_double(), plog_impl_float(), pmadd< Packet1cd >(), pmadd< Packet2cf >(), pnegate(), pnegate< Packet32h >(), predux_any(), prsqrt_float_common(), psignbit(), psincos_double(), psincos_float(), psqrt_complex(), psqrt_float_common(), Eigen::internal::psign_impl< Packet, std::enable_if_t<!NumTraits< typename unpacket_traits< Packet >::type >::IsComplex &&!NumTraits< typename unpacket_traits< Packet >::type >::IsInteger > >::run(), Eigen::internal::psign_impl< Packet, std::enable_if_t<!NumTraits< typename unpacket_traits< Packet >::type >::IsComplex &&NumTraits< typename unpacket_traits< Packet >::type >::IsSigned &&NumTraits< typename unpacket_traits< Packet >::type >::IsInteger > >::run(), Eigen::internal::psign_impl< Packet, std::enable_if_t<!NumTraits< typename unpacket_traits< Packet >::type >::IsComplex &&!NumTraits< typename unpacket_traits< Packet >::type >::IsSigned &&NumTraits< typename unpacket_traits< Packet >::type >::IsInteger > >::run(), Eigen::internal::psign_impl< Packet, std::enable_if_t< NumTraits< typename unpacket_traits< Packet >::type >::IsComplex &&unpacket_traits< Packet >::vectorizable > >::run(), Eigen::internal::psignbit_impl< Packet, false, true >::run(), Eigen::internal::generic_sqrt_newton_step< Packet, Steps >::run(), Eigen::internal::maybe_raise_div_by_zero< Packet, true >::run(), Eigen::internal::gemm_class< Scalar, is_unit_inc >::vaddm(), Eigen::internal::gemm_class< Scalar, is_unit_inc >::vfmaddm(), and Eigen::internal::gemm_class< Scalar, is_unit_inc >::write_c().

pzero() [16/18]

template<>

EIGEN_STRONG_INLINE Packet8i Eigen::internal::pzero

(

const Packet8i &

)

                                                      {
  return _mm256_setzero_si256();
}

pzero() [17/18]

template<>

EIGEN_STRONG_INLINE Packet8l Eigen::internal::pzero

(

const Packet8l &

)

                                                      {
  return _mm512_setzero_si512();
}

pzero() [18/18]

template<>

EIGEN_STRONG_INLINE Packet8ui Eigen::internal::pzero

(

const Packet8ui &

)

                                                        {
  return _mm256_setzero_si256();
}

pzero< Packet1cd >()

template<>

EIGEN_STRONG_INLINE Packet1cd Eigen::internal::pzero< Packet1cd >

(

const Packet1cd &

)

                                                                   {
  __m128d v = {0.0, 0.0};
  return (Packet1cd)v;
}

References v.

pzero< PacketXi >()

template<>

EIGEN_STRONG_INLINE PacketXi Eigen::internal::pzero< PacketXi >

(

const PacketXi &

)

                                                                {
  return svdup_n_s32_x(svptrue_b32(), 0);
}

pzeta()

template<typename Packet >

EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet Eigen::internal::pzeta	(	const Packet &	x,
		const Packet &	q
	)

Returns

the zeta function of two arguments (coeff-wise)

                                                                                                    {
  using numext::zeta;
  return zeta(x, q);
}

References Eigen::numext::q, plotDoE::x, and Eigen::zeta().

Referenced by Eigen::internal::scalar_zeta_op< Scalar >::packetOp().

qrsolv()

template<typename Scalar >

void Eigen::internal::qrsolv	(	Matrix< Scalar, Dynamic, Dynamic > &	s,
		const VectorXi &	ipvt,
		const Matrix< Scalar, Dynamic, 1 > &	diag,
		const Matrix< Scalar, Dynamic, 1 > &	qtb,
		Matrix< Scalar, Dynamic, 1 > &	x,
		Matrix< Scalar, Dynamic, 1 > &	sdiag
	)

{
  typedef DenseIndex Index;
 
  /* Local variables */
  Index i, j, k, l;
  Scalar temp;
  Index n = s.cols();
  Matrix<Scalar, Dynamic, 1> wa(n);
  JacobiRotation<Scalar> givens;
 
  /* Function Body */
  // the following will only change the lower triangular part of s, including
  // the diagonal, though the diagonal is restored afterward
 
  /*     copy r and (q transpose)*b to preserve input and initialize s. */
  /*     in particular, save the diagonal elements of r in x. */
  x = s.diagonal();
  wa = qtb;
 
  s.topLeftCorner(n, n).template triangularView<StrictlyLower>() = s.topLeftCorner(n, n).transpose();
 
  /*     eliminate the diagonal matrix d using a givens rotation. */
  for (j = 0; j < n; ++j) {
    /*        prepare the row of d to be eliminated, locating the */
    /*        diagonal element using p from the qr factorization. */
    l = ipvt[j];
    if (diag[l] == 0.) break;
    sdiag.tail(n - j).setZero();
    sdiag[j] = diag[l];
 
    /*        the transformations to eliminate the row of d */
    /*        modify only a single element of (q transpose)*b */
    /*        beyond the first n, which is initially zero. */
    Scalar qtbpj = 0.;
    for (k = j; k < n; ++k) {
      /*           determine a givens rotation which eliminates the */
      /*           appropriate element in the current row of d. */
      givens.makeGivens(-s(k, k), sdiag[k]);
 
      /*           compute the modified diagonal element of r and */
      /*           the modified element of ((q transpose)*b,0). */
      s(k, k) = givens.c() * s(k, k) + givens.s() * sdiag[k];
      temp = givens.c() * wa[k] + givens.s() * qtbpj;
      qtbpj = -givens.s() * wa[k] + givens.c() * qtbpj;
      wa[k] = temp;
 
      /*           accumulate the transformation in the row of s. */
      for (i = k + 1; i < n; ++i) {
        temp = givens.c() * s(i, k) + givens.s() * sdiag[i];
        sdiag[i] = -givens.s() * s(i, k) + givens.c() * sdiag[i];
        s(i, k) = temp;
      }
    }
  }
 
  /*     solve the triangular system for z. if the system is */
  /*     singular, then obtain a least squares solution. */
  Index nsing;
  for (nsing = 0; nsing < n && sdiag[nsing] != 0; nsing++) {
  }
 
  wa.tail(n - nsing).setZero();
  s.topLeftCorner(nsing, nsing).transpose().template triangularView<Upper>().solveInPlace(wa.head(nsing));
 
  // restore
  sdiag = s.diagonal();
  s.diagonal() = x;
 
  /*     permute the components of z back to components of x. */
  for (j = 0; j < n; ++j) x[ipvt[j]] = wa[j];
}

References Eigen::JacobiRotation< Scalar >::c(), diag, i, j, k, Eigen::JacobiRotation< Scalar >::makeGivens(), n, s, Eigen::JacobiRotation< Scalar >::s(), Eigen::PlainObjectBase< Derived >::setZero(), and plotDoE::x.

queryCacheSizes()

void Eigen::internal::queryCacheSizes ( int &  l1, int &  l2, int &  l3  )

inline

Queries and returns the cache sizes in Bytes of the L1, L2, and L3 data caches respectively

                                                       {
#ifdef EIGEN_CPUID
  int abcd[4];
  const int GenuineIntel[] = {0x756e6547, 0x49656e69, 0x6c65746e};
  const int AuthenticAMD[] = {0x68747541, 0x69746e65, 0x444d4163};
  const int AMDisbetter_[] = {0x69444d41, 0x74656273, 0x21726574};  // "AMDisbetter!"
 
  // identify the CPU vendor
  EIGEN_CPUID(abcd, 0x0, 0);
  int max_std_funcs = abcd[0];
  if (cpuid_is_vendor(abcd, GenuineIntel))
    queryCacheSizes_intel(l1, l2, l3, max_std_funcs);
  else if (cpuid_is_vendor(abcd, AuthenticAMD) || cpuid_is_vendor(abcd, AMDisbetter_))
    queryCacheSizes_amd(l1, l2, l3);
  else
    // by default let's use Intel's API
    queryCacheSizes_intel(l1, l2, l3, max_std_funcs);
 
    // here is the list of other vendors:
    //   ||cpuid_is_vendor(abcd,"VIA VIA VIA ")
    //   ||cpuid_is_vendor(abcd,"CyrixInstead")
    //   ||cpuid_is_vendor(abcd,"CentaurHauls")
    //   ||cpuid_is_vendor(abcd,"GenuineTMx86")
    //   ||cpuid_is_vendor(abcd,"TransmetaCPU")
    //   ||cpuid_is_vendor(abcd,"RiseRiseRise")
    //   ||cpuid_is_vendor(abcd,"Geode by NSC")
    //   ||cpuid_is_vendor(abcd,"SiS SiS SiS ")
    //   ||cpuid_is_vendor(abcd,"UMC UMC UMC ")
    //   ||cpuid_is_vendor(abcd,"NexGenDriven")
#else
  l1 = l2 = l3 = -1;
#endif
}

Referenced by Eigen::internal::CacheSizes::CacheSizes(), main(), queryL1CacheSize(), and queryTopLevelCacheSize().

queryL1CacheSize()

int Eigen::internal::queryL1CacheSize ( )

inline

Returns

the size in Bytes of the L1 data cache

                              {
  int l1(-1), l2, l3;
  queryCacheSizes(l1, l2, l3);
  return l1;
}

References queryCacheSizes().

Referenced by main().

queryTopLevelCacheSize()

int Eigen::internal::queryTopLevelCacheSize ( )

inline

Returns

the size in Bytes of the L2 or L3 cache if this later is present

                                    {
  int l1, l2(-1), l3(-1);
  queryCacheSizes(l1, l2, l3);
  return (std::max)(l2, l3);
}

References max, and queryCacheSizes().

Referenced by main().

QuickSplit()

template<typename VectorV , typename VectorI >

Index Eigen::internal::QuickSplit	(	VectorV &	row,
		VectorI &	ind,
		Index	ncut
	)

Compute a quick-sort split of a vector On output, the vector row is permuted such that its elements satisfy abs(row(i)) >= abs(row(ncut)) if i<ncut abs(row(i)) <= abs(row(ncut)) if i>ncut

Parameters

row	The vector of values
ind	The array of index for the elements in row
ncut	The number of largest elements to keep

                                                         {
  typedef typename VectorV::RealScalar RealScalar;
  using std::abs;
  using std::swap;
  Index mid;
  Index n = row.size(); /* length of the vector */
  Index first, last;
 
  ncut--; /* to fit the zero-based indices */
  first = 0;
  last = n - 1;
  if (ncut < first || ncut > last) return 0;
 
  do {
    mid = first;
    RealScalar abskey = abs(row(mid));
    for (Index j = first + 1; j <= last; j++) {
      if (abs(row(j)) > abskey) {
        ++mid;
        swap(row(mid), row(j));
        swap(ind(mid), ind(j));
      }
    }
    /* Interchange for the pivot element */
    swap(row(mid), row(first));
    swap(ind(mid), ind(first));
 
    if (mid > ncut)
      last = mid - 1;
    else if (mid < ncut)
      first = mid + 1;
  } while (mid != ncut);
 
  return 0; /* mid is equal to ncut */
}

References abs(), ind, j, Eigen::placeholders::last, n, row(), swap(), and swap().

Referenced by Eigen::IncompleteLUT< Scalar_, StorageIndex_ >::factorize(), and Eigen::IncompleteCholesky< Scalar, UpLo_, OrderingType_ >::factorize().

r1mpyq()

template<typename Scalar >

void Eigen::internal::r1mpyq	(	DenseIndex	m,
		DenseIndex	n,
		Scalar *	a,
		const std::vector< JacobiRotation< Scalar > > &	v_givens,
		const std::vector< JacobiRotation< Scalar > > &	w_givens
	)

                                                                {
  typedef DenseIndex Index;
 
  /*     apply the first set of givens rotations to a. */
  for (Index j = n - 2; j >= 0; --j)
    for (Index i = 0; i < m; ++i) {
      Scalar temp = v_givens[j].c() * a[i + m * j] - v_givens[j].s() * a[i + m * (n - 1)];
      a[i + m * (n - 1)] = v_givens[j].s() * a[i + m * j] + v_givens[j].c() * a[i + m * (n - 1)];
      a[i + m * j] = temp;
    }
  /*     apply the second set of givens rotations to a. */
  for (Index j = 0; j < n - 1; ++j)
    for (Index i = 0; i < m; ++i) {
      Scalar temp = w_givens[j].c() * a[i + m * j] + w_givens[j].s() * a[i + m * (n - 1)];
      a[i + m * (n - 1)] = -w_givens[j].s() * a[i + m * j] + w_givens[j].c() * a[i + m * (n - 1)];
      a[i + m * j] = temp;
    }
}

References a, i, j, m, n, and s.

r1updt()

template<typename Scalar >

void Eigen::internal::r1updt	(	Matrix< Scalar, Dynamic, Dynamic > &	s,
		const Matrix< Scalar, Dynamic, 1 > &	u,
		std::vector< JacobiRotation< Scalar > > &	v_givens,
		std::vector< JacobiRotation< Scalar > > &	w_givens,
		Matrix< Scalar, Dynamic, 1 > &	v,
		Matrix< Scalar, Dynamic, 1 > &	w,
		bool *	sing
	)

                                                                                      {
  typedef DenseIndex Index;
  const JacobiRotation<Scalar> IdentityRotation = JacobiRotation<Scalar>(1, 0);
 
  /* Local variables */
  const Index m = s.rows();
  const Index n = s.cols();
  Index i, j = 1;
  Scalar temp;
  JacobiRotation<Scalar> givens;
 
  // r1updt had a broader usecase, but we don't use it here. And, more
  // importantly, we can not test it.
  eigen_assert(m == n);
  eigen_assert(u.size() == m);
  eigen_assert(v.size() == n);
  eigen_assert(w.size() == n);
 
  /* move the nontrivial part of the last column of s into w. */
  w[n - 1] = s(n - 1, n - 1);
 
  /* rotate the vector v into a multiple of the n-th unit vector */
  /* in such a way that a spike is introduced into w. */
  for (j = n - 2; j >= 0; --j) {
    w[j] = 0.;
    if (v[j] != 0.) {
      /* determine a givens rotation which eliminates the */
      /* j-th element of v. */
      givens.makeGivens(-v[n - 1], v[j]);
 
      /* apply the transformation to v and store the information */
      /* necessary to recover the givens rotation. */
      v[n - 1] = givens.s() * v[j] + givens.c() * v[n - 1];
      v_givens[j] = givens;
 
      /* apply the transformation to s and extend the spike in w. */
      for (i = j; i < m; ++i) {
        temp = givens.c() * s(j, i) - givens.s() * w[i];
        w[i] = givens.s() * s(j, i) + givens.c() * w[i];
        s(j, i) = temp;
      }
    } else
      v_givens[j] = IdentityRotation;
  }
 
  /* add the spike from the rank 1 update to w. */
  w += v[n - 1] * u;
 
  /* eliminate the spike. */
  *sing = false;
  for (j = 0; j < n - 1; ++j) {
    if (w[j] != 0.) {
      /* determine a givens rotation which eliminates the */
      /* j-th element of the spike. */
      givens.makeGivens(-s(j, j), w[j]);
 
      /* apply the transformation to s and reduce the spike in w. */
      for (i = j; i < m; ++i) {
        temp = givens.c() * s(j, i) + givens.s() * w[i];
        w[i] = -givens.s() * s(j, i) + givens.c() * w[i];
        s(j, i) = temp;
      }
 
      /* store the information necessary to recover the */
      /* givens rotation. */
      w_givens[j] = givens;
    } else
      v_givens[j] = IdentityRotation;
 
    /* test for zero diagonal elements in the output s. */
    if (s(j, j) == 0.) {
      *sing = true;
    }
  }
  /* move w back into the last column of the output s. */
  s(n - 1, n - 1) = w[n - 1];
 
  if (s(j, j) == 0.) {
    *sing = true;
  }
  return;
}

References Eigen::JacobiRotation< Scalar >::c(), eigen_assert, i, j, m, Eigen::JacobiRotation< Scalar >::makeGivens(), n, s, Eigen::JacobiRotation< Scalar >::s(), v, and w.

RandomToTypeNormal()

template<typename T >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T Eigen::internal::RandomToTypeNormal	(	uint64_t *	state,
		uint64_t	stream
	)

                                                                                             {
  // Use the ratio of uniform method to generate numbers following a normal
  // distribution. See for example Numerical Recipes chapter 7.3.9 for the
  // details.
  T u, v, q;
  do {
    u = RandomToTypeUniform<T>(state, stream);
    v = T(1.7156) * (RandomToTypeUniform<T>(state, stream) - T(0.5));
    const T x = u - T(0.449871);
    const T y = numext::abs(v) + T(0.386595);
    q = x * x + y * (T(0.196) * y - T(0.25472) * x);
  } while (q > T(0.27597) && (q > T(0.27846) || v * v > T(-4) * numext::log(u) * u * u));
 
  return v / u;
}

References Eigen::numext::abs(), Eigen::numext::log(), Eigen::numext::q, v, plotDoE::x, and y.

RandomToTypeNormal< std::complex< double > >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex<double> Eigen::internal::RandomToTypeNormal< std::complex< double > >	(	uint64_t *	state,
		uint64_t	stream
	)

                                                                                                                      {
  return std::complex<double>(RandomToTypeNormal<double>(state, stream), RandomToTypeNormal<double>(state, stream));
}

RandomToTypeNormal< std::complex< float > >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex<float> Eigen::internal::RandomToTypeNormal< std::complex< float > >	(	uint64_t *	state,
		uint64_t	stream
	)

                                                                                                                    {
  return std::complex<float>(RandomToTypeNormal<float>(state, stream), RandomToTypeNormal<float>(state, stream));
}

RandomToTypeUniform()

template<typename T >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T Eigen::internal::RandomToTypeUniform	(	uint64_t *	state,
		uint64_t	stream
	)

                                                                                              {
  unsigned rnd = PCG_XSH_RS_generator(state, stream);
  return static_cast<T>(rnd);
}

References PCG_XSH_RS_generator().

RandomToTypeUniform< double >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double Eigen::internal::RandomToTypeUniform< double >	(	uint64_t *	state,
		uint64_t	stream
	)

                                                                                                           {
  typedef union {
    uint64_t raw;
    double dp;
  } internal;
  internal result;
  result.raw = 0;
  // Generate 52 random bits for the mantissa
  // First generate the upper 20 bits
  unsigned rnd1 = PCG_XSH_RS_generator(state, stream) & 0xfffffu;
  // The generate the lower 32 bits
  unsigned rnd2 = PCG_XSH_RS_generator(state, stream);
  result.raw = (static_cast<uint64_t>(rnd1) << 32) | rnd2;
  // Set the exponent
  result.raw |= (static_cast<uint64_t>(1023) << 52);
  // Return the final result
  return result.dp - 1.0;
}

References PCG_XSH_RS_generator().

Referenced by RandomToTypeUniform< std::complex< double > >().

RandomToTypeUniform< Eigen::bfloat16 >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::bfloat16 Eigen::internal::RandomToTypeUniform< Eigen::bfloat16 >	(	uint64_t *	state,
		uint64_t	stream
	)

                                                                                                            {
  // Generate 7 random bits for the mantissa, merge with exponent.
  unsigned rnd = PCG_XSH_RS_generator(state, stream);
  const uint16_t half_bits = static_cast<uint16_t>(rnd & 0x7fu) | (static_cast<uint16_t>(127) << 7);
  Eigen::bfloat16 result = Eigen::numext::bit_cast<Eigen::bfloat16>(half_bits);
  // Return the final result
  return result - Eigen::bfloat16(1.0f);
}

References PCG_XSH_RS_generator().

RandomToTypeUniform< Eigen::half >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half Eigen::internal::RandomToTypeUniform< Eigen::half >	(	uint64_t *	state,
		uint64_t	stream
	)

                                                                                                                 {
  // Generate 10 random bits for the mantissa, merge with exponent.
  unsigned rnd = PCG_XSH_RS_generator(state, stream);
  const uint16_t half_bits = static_cast<uint16_t>(rnd & 0x3ffu) | (static_cast<uint16_t>(15) << 10);
  Eigen::half result = Eigen::numext::bit_cast<Eigen::half>(half_bits);
  // Return the final result
  return result - Eigen::half(1.0f);
}

References PCG_XSH_RS_generator().

RandomToTypeUniform< float >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float Eigen::internal::RandomToTypeUniform< float >	(	uint64_t *	state,
		uint64_t	stream
	)

                                                                                                         {
  typedef union {
    uint32_t raw;
    float fp;
  } internal;
  internal result;
  // Generate 23 random bits for the mantissa mantissa
  const unsigned rnd = PCG_XSH_RS_generator(state, stream);
  result.raw = rnd & 0x7fffffu;
  // Set the exponent
  result.raw |= (static_cast<uint32_t>(127) << 23);
  // Return the final result
  return result.fp - 1.0f;
}

References PCG_XSH_RS_generator().

Referenced by RandomToTypeUniform< std::complex< float > >().

RandomToTypeUniform< std::complex< double > >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex<double> Eigen::internal::RandomToTypeUniform< std::complex< double > >	(	uint64_t *	state,
		uint64_t	stream
	)

                                                                                                                       {
  return std::complex<double>(RandomToTypeUniform<double>(state, stream), RandomToTypeUniform<double>(state, stream));
}

References RandomToTypeUniform< double >().

RandomToTypeUniform< std::complex< float > >()

template<>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex<float> Eigen::internal::RandomToTypeUniform< std::complex< float > >	(	uint64_t *	state,
		uint64_t	stream
	)

                                                                                                                     {
  return std::complex<float>(RandomToTypeUniform<float>(state, stream), RandomToTypeUniform<float>(state, stream));
}

References RandomToTypeUniform< float >().

rcond_estimate_helper()

template<typename Decomposition >

Decomposition::RealScalar Eigen::internal::rcond_estimate_helper	(	typename Decomposition::RealScalar	matrix_norm,
		const Decomposition &	dec
	)

Reciprocal condition number estimator.

Computing a decomposition of a dense matrix takes O(n^3) operations, while this method estimates the condition number quickly and reliably in O(n^2) operations.

Returns

an estimate of the reciprocal condition number (1 / (||matrix||_1 * ||inv(matrix)||_1)) of matrix, given ||matrix||_1 and its decomposition. Supports the following decompositions: FullPivLU, PartialPivLU, LDLT, and LLT.

See also

FullPivLU, PartialPivLU, LDLT, LLT.

                                                                                   {
  typedef typename Decomposition::RealScalar RealScalar;
  eigen_assert(dec.rows() == dec.cols());
  if (dec.rows() == 0) return NumTraits<RealScalar>::infinity();
  if (numext::is_exactly_zero(matrix_norm)) return RealScalar(0);
  if (dec.rows() == 1) return RealScalar(1);
  const RealScalar inverse_matrix_norm = rcond_invmatrix_L1_norm_estimate(dec);
  return (numext::is_exactly_zero(inverse_matrix_norm) ? RealScalar(0)
                                                       : (RealScalar(1) / inverse_matrix_norm) / matrix_norm);
}

References eigen_assert, Eigen::numext::is_exactly_zero(), and rcond_invmatrix_L1_norm_estimate().

Referenced by Eigen::LDLT< MatrixType_, UpLo_ >::rcond(), Eigen::LLT< MatrixType_, UpLo_ >::rcond(), Eigen::FullPivLU< MatrixType_, PermutationIndex_ >::rcond(), and Eigen::PartialPivLU< MatrixType_, PermutationIndex_ >::rcond().

rcond_invmatrix_L1_norm_estimate()

template<typename Decomposition >

Decomposition::RealScalar Eigen::internal::rcond_invmatrix_L1_norm_estimate

(

const Decomposition &

dec

)

Returns

an estimate of ||inv(matrix)||_1 given a decomposition of matrix that implements .solve() and .adjoint().solve() methods.

This function implements Algorithms 4.1 and 5.1 from http://www.maths.manchester.ac.uk/~higham/narep/narep135.pdf which also forms the basis for the condition number estimators in LAPACK. Since at most 10 calls to the solve method of dec are performed, the total cost is O(dims^2), as opposed to O(dims^3) needed to compute the inverse matrix explicitly.

The most common usage is in estimating the condition number ||matrix||_1 * ||inv(matrix)||_1. The first term ||matrix||_1 can be computed directly in O(n^2) operations.

Supports the following decompositions: FullPivLU, PartialPivLU, LDLT, and LLT.

See also

FullPivLU, PartialPivLU, LDLT, LLT.

                                                                                            {
  typedef typename Decomposition::MatrixType MatrixType;
  typedef typename Decomposition::Scalar Scalar;
  typedef typename Decomposition::RealScalar RealScalar;
  typedef typename internal::plain_col_type<MatrixType>::type Vector;
  typedef typename internal::plain_col_type<MatrixType, RealScalar>::type RealVector;
  const bool is_complex = (NumTraits<Scalar>::IsComplex != 0);
 
  eigen_assert(dec.rows() == dec.cols());
  const Index n = dec.rows();
  if (n == 0) return 0;
 
    // Disable Index to float conversion warning
#ifdef __INTEL_COMPILER
#pragma warning push
#pragma warning(disable : 2259)
#endif
  Vector v = dec.solve(Vector::Ones(n) / Scalar(n));
#ifdef __INTEL_COMPILER
#pragma warning pop
#endif
 
  // lower_bound is a lower bound on
  //   ||inv(matrix)||_1  = sup_v ||inv(matrix) v||_1 / ||v||_1
  // and is the objective maximized by the ("super-") gradient ascent
  // algorithm below.
  RealScalar lower_bound = v.template lpNorm<1>();
  if (n == 1) return lower_bound;
 
  // Gradient ascent algorithm follows: We know that the optimum is achieved at
  // one of the simplices v = e_i, so in each iteration we follow a
  // super-gradient to move towards the optimal one.
  RealScalar old_lower_bound = lower_bound;
  Vector sign_vector(n);
  Vector old_sign_vector;
  Index v_max_abs_index = -1;
  Index old_v_max_abs_index = v_max_abs_index;
  for (int k = 0; k < 4; ++k) {
    sign_vector = internal::rcond_compute_sign<Vector, RealVector, is_complex>::run(v);
    if (k > 0 && !is_complex && sign_vector == old_sign_vector) {
      // Break if the solution stagnated.
      break;
    }
    // v_max_abs_index = argmax |real( inv(matrix)^T * sign_vector )|
    v = dec.adjoint().solve(sign_vector);
    v.real().cwiseAbs().maxCoeff(&v_max_abs_index);
    if (v_max_abs_index == old_v_max_abs_index) {
      // Break if the solution stagnated.
      break;
    }
    // Move to the new simplex e_j, where j = v_max_abs_index.
    v = dec.solve(Vector::Unit(n, v_max_abs_index));  // v = inv(matrix) * e_j.
    lower_bound = v.template lpNorm<1>();
    if (lower_bound <= old_lower_bound) {
      // Break if the gradient step did not increase the lower_bound.
      break;
    }
    if (!is_complex) {
      old_sign_vector = sign_vector;
    }
    old_v_max_abs_index = v_max_abs_index;
    old_lower_bound = lower_bound;
  }
  // The following calculates an independent estimate of ||matrix||_1 by
  // multiplying matrix by a vector with entries of slowly increasing
  // magnitude and alternating sign:
  //   v_i = (-1)^{i} (1 + (i / (dim-1))), i = 0,...,dim-1.
  // This improvement to Hager's algorithm above is due to Higham. It was
  // added to make the algorithm more robust in certain corner cases where
  // large elements in the matrix might otherwise escape detection due to
  // exact cancellation (especially when op and op_adjoint correspond to a
  // sequence of backsubstitutions and permutations), which could cause
  // Hager's algorithm to vastly underestimate ||matrix||_1.
  Scalar alternating_sign(RealScalar(1));
  for (Index i = 0; i < n; ++i) {
    // The static_cast is needed when Scalar is a complex and RealScalar implements expression templates
    v[i] = alternating_sign * static_cast<RealScalar>(RealScalar(1) + (RealScalar(i) / (RealScalar(n - 1))));
    alternating_sign = -alternating_sign;
  }
  v = dec.solve(v);
  const RealScalar alternate_lower_bound = (2 * v.template lpNorm<1>()) / (3 * RealScalar(n));
  return numext::maxi(lower_bound, alternate_lower_bound);
}

References eigen_assert, i, k, Eigen::numext::maxi(), n, Eigen::internal::rcond_compute_sign< Vector, RealVector, IsComplex >::run(), and v.

Referenced by rcond_estimate_helper().

real_2x2_jacobi_svd()

template<typename MatrixType , typename RealScalar , typename Index >

void Eigen::internal::real_2x2_jacobi_svd	(	const MatrixType &	matrix,
		Index	p,
		Index	q,
		JacobiRotation< RealScalar > *	j_left,
		JacobiRotation< RealScalar > *	j_right
	)

                                                              {
  using std::abs;
  using std::sqrt;
  Matrix<RealScalar, 2, 2> m;
  m << numext::real(matrix.coeff(p, p)), numext::real(matrix.coeff(p, q)), numext::real(matrix.coeff(q, p)),
      numext::real(matrix.coeff(q, q));
  JacobiRotation<RealScalar> rot1;
  RealScalar t = m.coeff(0, 0) + m.coeff(1, 1);
  RealScalar d = m.coeff(1, 0) - m.coeff(0, 1);
 
  if (abs(d) < (std::numeric_limits<RealScalar>::min)()) {
    rot1.s() = RealScalar(0);
    rot1.c() = RealScalar(1);
  } else {
    // If d!=0, then t/d cannot overflow because the magnitude of the
    // entries forming d are not too small compared to the ones forming t.
    RealScalar u = t / d;
    RealScalar tmp = sqrt(RealScalar(1) + numext::abs2(u));
    rot1.s() = RealScalar(1) / tmp;
    rot1.c() = u / tmp;
  }
  m.applyOnTheLeft(0, 1, rot1);
  j_right->makeJacobi(m, 0, 1);
  *j_left = rot1 * j_right->transpose();
}

References abs(), Eigen::numext::abs2(), Eigen::JacobiRotation< Scalar >::c(), m, Eigen::JacobiRotation< Scalar >::makeJacobi(), matrix(), min, p, Eigen::numext::q, Eigen::JacobiRotation< Scalar >::s(), sqrt(), plotPSD::t, tmp, and Eigen::JacobiRotation< Scalar >::transpose().

Referenced by Eigen::RealQZ< MatrixType_ >::compute(), and Eigen::JacobiSVD< MatrixType_, Options_ >::compute_impl().

ReducePacket()

template<typename Self >

EIGEN_STRONG_INLINE void Eigen::internal::ReducePacket	(	Self &	self,
		Index	offset,
		typename Self::CoeffReturnType *	data
	)

                                                                                                    {
  using Scalar = typename Self::CoeffReturnType;
  using Packet = typename Self::PacketReturnType;
  // Compute the scan along the axis, starting at the calculated offset
  Packet accum = self.accumulator().template initializePacket<Packet>();
  if (self.stride() == 1) {
    if (self.exclusive()) {
      for (Index curr = offset; curr < offset + self.size(); ++curr) {
        internal::pstoreu<Scalar, Packet>(data + curr, self.accumulator().finalizePacket(accum));
        self.accumulator().reducePacket(self.inner().template packet<Unaligned>(curr), &accum);
      }
    } else {
      for (Index curr = offset; curr < offset + self.size(); ++curr) {
        self.accumulator().reducePacket(self.inner().template packet<Unaligned>(curr), &accum);
        internal::pstoreu<Scalar, Packet>(data + curr, self.accumulator().finalizePacket(accum));
      }
    }
  } else {
    if (self.exclusive()) {
      for (Index idx3 = 0; idx3 < self.size(); idx3++) {
        const Index curr = offset + idx3 * self.stride();
        internal::pstoreu<Scalar, Packet>(data + curr, self.accumulator().finalizePacket(accum));
        self.accumulator().reducePacket(self.inner().template packet<Unaligned>(curr), &accum);
      }
    } else {
      for (Index idx3 = 0; idx3 < self.size(); idx3++) {
        const Index curr = offset + idx3 * self.stride();
        self.accumulator().reducePacket(self.inner().template packet<Unaligned>(curr), &accum);
        internal::pstoreu<Scalar, Packet>(data + curr, self.accumulator().finalizePacket(accum));
      }
    }
  }
}

References data.

Referenced by Eigen::internal::ReduceBlock< Self, true, false >::operator()().

ReduceScalar()

template<typename Self >

EIGEN_STRONG_INLINE void Eigen::internal::ReduceScalar	(	Self &	self,
		Index	offset,
		typename Self::CoeffReturnType *	data
	)

                                                                                                    {
  // Compute the scan along the axis, starting at the given offset
  typename Self::CoeffReturnType accum = self.accumulator().initialize();
  if (self.stride() == 1) {
    if (self.exclusive()) {
      for (Index curr = offset; curr < offset + self.size(); ++curr) {
        data[curr] = self.accumulator().finalize(accum);
        self.accumulator().reduce(self.inner().coeff(curr), &accum);
      }
    } else {
      for (Index curr = offset; curr < offset + self.size(); ++curr) {
        self.accumulator().reduce(self.inner().coeff(curr), &accum);
        data[curr] = self.accumulator().finalize(accum);
      }
    }
  } else {
    if (self.exclusive()) {
      for (Index idx3 = 0; idx3 < self.size(); idx3++) {
        Index curr = offset + idx3 * self.stride();
        data[curr] = self.accumulator().finalize(accum);
        self.accumulator().reduce(self.inner().coeff(curr), &accum);
      }
    } else {
      for (Index idx3 = 0; idx3 < self.size(); idx3++) {
        Index curr = offset + idx3 * self.stride();
        self.accumulator().reduce(self.inner().coeff(curr), &accum);
        data[curr] = self.accumulator().finalize(accum);
      }
    }
  }
}

References data.

Referenced by Eigen::internal::ReduceBlock< Self, Vectorize, Parallel >::operator()(), and Eigen::internal::ReduceBlock< Self, true, false >::operator()().

repeat()

template<int n, typename t >

constexpr array<t, n> Eigen::internal::repeat ( t  v )

constexpr

                                  {
  return h_repeat<n>::run(v, typename gen_numeric_list<int, n>::type());
}

References Eigen::internal::h_repeat< n >::run(), and v.

Referenced by check_all_in_range(), check_histogram(), check_histogram< bool >(), inverse_general_4x4(), cast_test_impl< SrcType, DstType, RowsAtCompileTime, ColsAtCompileTime >::run(), partition_action_t::run(), and special_value_pairs().

resize_if_allowed() [1/2]

template<typename DstXprType , typename SrcXprType , typename Functor >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::resize_if_allowed	(	DstXprType &	dst,
		const SrcXprType &	src,
		const Functor &
	)

                                                                               {
  EIGEN_ONLY_USED_FOR_DEBUG(dst);
  EIGEN_ONLY_USED_FOR_DEBUG(src);
  eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols());
}

References eigen_assert, and EIGEN_ONLY_USED_FOR_DEBUG.

Referenced by call_dense_assignment_loop(), call_restricted_packet_assignment_no_alias(), Eigen::internal::Assignment< DstXprType, SrcXprType, Functor, Sparse2Dense, Weak >::run(), Eigen::internal::eigen_fill_impl< Xpr, true >::run(), and Eigen::internal::eigen_zero_impl< Xpr, true >::run().

resize_if_allowed() [2/2]

template<typename DstXprType , typename SrcXprType , typename T1 , typename T2 >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::resize_if_allowed	(	DstXprType &	dst,
		const SrcXprType &	src,
		const internal::assign_op< T1, T2 > &
	)

                                                                                                 {
  Index dstRows = src.rows();
  Index dstCols = src.cols();
  if (((dst.rows() != dstRows) || (dst.cols() != dstCols))) dst.resize(dstRows, dstCols);
  eigen_assert(dst.rows() == dstRows && dst.cols() == dstCols);
}

References eigen_assert.

return_ptr()

template<typename T >

const T* Eigen::internal::return_ptr

(

)

returnUnsignedIndexValue()

template<typename IndexType >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE make_unsigned<IndexType>::type Eigen::internal::returnUnsignedIndexValue

(

const IndexType &

idx

)

                          {
  EIGEN_STATIC_ASSERT((NumTraits<IndexType>::IsInteger), THIS FUNCTION IS FOR INTEGER TYPES)
  eigen_internal_assert(idx >= 0 && "Index value is negative and target type is unsigned");
  using UnsignedType = typename make_unsigned<IndexType>::type;
  return static_cast<UnsignedType>(idx);
}

References eigen_internal_assert, EIGEN_STATIC_ASSERT, and compute_granudrum_aor::type.

Referenced by Eigen::internal::convert_index_impl< IndexDest, IndexSrc, true, true, true, false >::run(), and Eigen::internal::convert_index_impl< IndexDest, IndexSrc, true, false, true, true >::run().

run_serialized() [1/2]

template<typename Kernel , typename... Args>

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::run_serialized	(	Kernel	kernel,
		uint8_t *	buffer,
		size_t	capacity
	)

                                                                                                           {
  run_serialized<Kernel, Args...>(std::make_index_sequence<sizeof...(Args)>{}, extract_output_indices<Args...>{},
                                  kernel, buffer, capacity);
}

run_serialized() [2/2]

template<typename Kernel , typename... Args, size_t... Indices, size_t... OutputIndices>

EIGEN_DEVICE_FUNC void Eigen::internal::run_serialized	(	std::index_sequence< Indices... >	,
		std::index_sequence< OutputIndices... >	,
		Kernel	kernel,
		uint8_t *	buffer,
		size_t	capacity
	)

                                                                                       {
  using test_detail::get;
  using test_detail::make_tuple;
  using test_detail::tuple;
  // Deserialize input size and inputs.
  size_t input_size;
  const uint8_t* read_ptr = buffer;
  const uint8_t* read_end = buffer + capacity;
  read_ptr = Eigen::deserialize(read_ptr, read_end, input_size);
  // Create value-type instances to populate.
  auto args = make_tuple(typename std::decay<Args>::type{}...);
  EIGEN_UNUSED_VARIABLE(args)  // Avoid NVCC compile warning.
  // NVCC 9.1 requires us to spell out the template parameters explicitly.
  read_ptr = Eigen::deserialize(read_ptr, read_end, get<Indices, typename std::decay<Args>::type...>(args)...);
 
  // Call function, with void->Void conversion so we are guaranteed a complete
  // output type.
  auto result = void_helper::call(kernel, get<Indices, typename std::decay<Args>::type...>(args)...);
 
  // Determine required output size.
  size_t output_size = Eigen::serialize_size(capacity);
  output_size += Eigen::serialize_size(get<OutputIndices, typename std::decay<Args>::type...>(args)...);
  output_size += Eigen::serialize_size(result);
 
  // Always serialize required buffer size.
  uint8_t* write_ptr = buffer;
  uint8_t* write_end = buffer + capacity;
  write_ptr = Eigen::serialize(write_ptr, write_end, output_size);
  // Null `write_ptr` can be safely passed along.
  // Serialize outputs if they fit in the buffer.
  if (output_size <= capacity) {
    // Collect outputs and result.
    write_ptr = Eigen::serialize(write_ptr, write_end, get<OutputIndices, typename std::decay<Args>::type...>(args)...);
    write_ptr = Eigen::serialize(write_ptr, write_end, result);
  }
}

References compute_granudrum_aor::args, Eigen::deserialize(), EIGEN_UNUSED_VARIABLE, get(), Eigen::internal::tuple_impl::make_tuple(), Eigen::serialize(), Eigen::serialize_size(), and compute_granudrum_aor::type.

rwupdt()

template<typename Scalar >

void Eigen::internal::rwupdt	(	Matrix< Scalar, Dynamic, Dynamic > &	r,
		const Matrix< Scalar, Dynamic, 1 > &	w,
		Matrix< Scalar, Dynamic, 1 > &	b,
		Scalar	alpha
	)

                          {
  typedef DenseIndex Index;
 
  const Index n = r.cols();
  eigen_assert(r.rows() >= n);
  std::vector<JacobiRotation<Scalar> > givens(n);
 
  /* Local variables */
  Scalar temp, rowj;
 
  /* Function Body */
  for (Index j = 0; j < n; ++j) {
    rowj = w[j];
 
    /* apply the previous transformations to */
    /* r(i,j), i=0,1,...,j-1, and to w(j). */
    for (Index i = 0; i < j; ++i) {
      temp = givens[i].c() * r(i, j) + givens[i].s() * rowj;
      rowj = -givens[i].s() * r(i, j) + givens[i].c() * rowj;
      r(i, j) = temp;
    }
 
    /* determine a givens rotation which eliminates w(j). */
    givens[j].makeGivens(-r(j, j), rowj);
 
    if (rowj == 0.) continue;  // givens[j] is identity
 
    /* apply the current transformation to r(j,j), b(j), and alpha. */
    r(j, j) = givens[j].c() * r(j, j) + givens[j].s() * rowj;
    temp = givens[j].c() * b[j] + givens[j].s() * alpha;
    alpha = -givens[j].s() * b[j] + givens[j].c() * alpha;
    b[j] = temp;
  }
}

References alpha, b, eigen_assert, i, j, n, UniformPSDSelfTest::r, and w.

set_from_triplets()

template<typename InputIterator , typename SparseMatrixType , typename DupFunctor >

void Eigen::internal::set_from_triplets	(	const InputIterator &	begin,
		const InputIterator &	end,
		SparseMatrixType &	mat,
		DupFunctor	dup_func
	)

                                            {
  constexpr bool IsRowMajor = SparseMatrixType::IsRowMajor;
  using StorageIndex = typename SparseMatrixType::StorageIndex;
  using IndexMap = typename VectorX<StorageIndex>::AlignedMapType;
  using TransposedSparseMatrix =
      SparseMatrix<typename SparseMatrixType::Scalar, IsRowMajor ? ColMajor : RowMajor, StorageIndex>;
 
  if (begin == end) return;
 
  // There are two strategies to consider for constructing a matrix from unordered triplets:
  // A) construct the 'mat' in its native storage order and sort in-place (less memory); or,
  // B) construct the transposed matrix and use an implicit sort upon assignment to `mat` (less time).
  // This routine uses B) for faster execution time.
  TransposedSparseMatrix trmat(mat.rows(), mat.cols());
 
  // scan triplets to determine allocation size before constructing matrix
  Index nonZeros = 0;
  for (InputIterator it(begin); it != end; ++it) {
    eigen_assert(it->row() >= 0 && it->row() < mat.rows() && it->col() >= 0 && it->col() < mat.cols());
    StorageIndex j = convert_index<StorageIndex>(IsRowMajor ? it->col() : it->row());
    if (nonZeros == NumTraits<StorageIndex>::highest()) internal::throw_std_bad_alloc();
    trmat.outerIndexPtr()[j + 1]++;
    nonZeros++;
  }
 
  std::partial_sum(trmat.outerIndexPtr(), trmat.outerIndexPtr() + trmat.outerSize() + 1, trmat.outerIndexPtr());
  eigen_assert(nonZeros == trmat.outerIndexPtr()[trmat.outerSize()]);
  trmat.resizeNonZeros(nonZeros);
 
  // construct temporary array to track insertions (outersize) and collapse duplicates (innersize)
  ei_declare_aligned_stack_constructed_variable(StorageIndex, tmp, numext::maxi(mat.innerSize(), mat.outerSize()), 0);
  smart_copy(trmat.outerIndexPtr(), trmat.outerIndexPtr() + trmat.outerSize(), tmp);
 
  // push triplets to back of each vector
  for (InputIterator it(begin); it != end; ++it) {
    StorageIndex j = convert_index<StorageIndex>(IsRowMajor ? it->col() : it->row());
    StorageIndex i = convert_index<StorageIndex>(IsRowMajor ? it->row() : it->col());
    StorageIndex k = tmp[j];
    trmat.data().index(k) = i;
    trmat.data().value(k) = it->value();
    tmp[j]++;
  }
 
  IndexMap wi(tmp, trmat.innerSize());
  trmat.collapseDuplicates(wi, dup_func);
  // implicit sorting
  mat = trmat;
}

References Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::cols(), Eigen::PlainObjectBase< Derived >::data(), ei_declare_aligned_stack_constructed_variable, eigen_assert, Eigen::placeholders::end, i, Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::innerSize(), j, k, Eigen::numext::maxi(), Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::outerSize(), Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::rows(), smart_copy(), throw_std_bad_alloc(), and tmp.

Referenced by insert_from_triplets().

set_from_triplets_sorted()

template<typename InputIterator , typename SparseMatrixType , typename DupFunctor >

void Eigen::internal::set_from_triplets_sorted	(	const InputIterator &	begin,
		const InputIterator &	end,
		SparseMatrixType &	mat,
		DupFunctor	dup_func
	)

                                                   {
  constexpr bool IsRowMajor = SparseMatrixType::IsRowMajor;
  using StorageIndex = typename SparseMatrixType::StorageIndex;
 
  if (begin == end) return;
 
  constexpr StorageIndex kEmptyIndexValue(-1);
  // deallocate inner nonzeros if present and zero outerIndexPtr
  mat.resize(mat.rows(), mat.cols());
  // use outer indices to count non zero entries (excluding duplicate entries)
  StorageIndex previous_j = kEmptyIndexValue;
  StorageIndex previous_i = kEmptyIndexValue;
  // scan triplets to determine allocation size before constructing matrix
  Index nonZeros = 0;
  for (InputIterator it(begin); it != end; ++it) {
    eigen_assert(it->row() >= 0 && it->row() < mat.rows() && it->col() >= 0 && it->col() < mat.cols());
    StorageIndex j = convert_index<StorageIndex>(IsRowMajor ? it->row() : it->col());
    StorageIndex i = convert_index<StorageIndex>(IsRowMajor ? it->col() : it->row());
    eigen_assert(j > previous_j || (j == previous_j && i >= previous_i));
    // identify duplicates by examining previous location
    bool duplicate = (previous_j == j) && (previous_i == i);
    if (!duplicate) {
      if (nonZeros == NumTraits<StorageIndex>::highest()) internal::throw_std_bad_alloc();
      nonZeros++;
      mat.outerIndexPtr()[j + 1]++;
      previous_j = j;
      previous_i = i;
    }
  }
 
  // finalize outer indices and allocate memory
  std::partial_sum(mat.outerIndexPtr(), mat.outerIndexPtr() + mat.outerSize() + 1, mat.outerIndexPtr());
  eigen_assert(nonZeros == mat.outerIndexPtr()[mat.outerSize()]);
  mat.resizeNonZeros(nonZeros);
 
  previous_i = kEmptyIndexValue;
  previous_j = kEmptyIndexValue;
  Index back = 0;
  for (InputIterator it(begin); it != end; ++it) {
    StorageIndex j = convert_index<StorageIndex>(IsRowMajor ? it->row() : it->col());
    StorageIndex i = convert_index<StorageIndex>(IsRowMajor ? it->col() : it->row());
    bool duplicate = (previous_j == j) && (previous_i == i);
    if (duplicate) {
      mat.data().value(back - 1) = dup_func(mat.data().value(back - 1), it->value());
    } else {
      // push triplets to back
      mat.data().index(back) = i;
      mat.data().value(back) = it->value();
      previous_j = j;
      previous_i = i;
      back++;
    }
  }
  eigen_assert(back == nonZeros);
  // matrix is finalized
}

References Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::cols(), Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::data(), eigen_assert, Eigen::placeholders::end, i, Eigen::internal::CompressedStorage< Scalar_, StorageIndex_ >::index(), j, Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::outerIndexPtr(), Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::outerSize(), Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::resize(), Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::resizeNonZeros(), Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::rows(), throw_std_bad_alloc(), and Eigen::internal::CompressedStorage< Scalar_, StorageIndex_ >::value().

Referenced by insert_from_triplets_sorted().

should_svd_compute_full_u()

constexpr bool Eigen::internal::should_svd_compute_full_u ( int  options )

constexpr

{ return (options & ComputeFullU) != 0; }

References Eigen::ComputeFullU.

Referenced by Eigen::SVDBase< Derived >::allocate().

should_svd_compute_full_v()

constexpr bool Eigen::internal::should_svd_compute_full_v ( int  options )

constexpr

{ return (options & ComputeFullV) != 0; }

References Eigen::ComputeFullV.

Referenced by Eigen::SVDBase< Derived >::allocate().

should_svd_compute_thin_u()

constexpr bool Eigen::internal::should_svd_compute_thin_u ( int  options )

constexpr

{ return (options & ComputeThinU) != 0; }

References Eigen::ComputeThinU.

Referenced by Eigen::SVDBase< Derived >::allocate(), and check_svd_options_assertions().

should_svd_compute_thin_v()

constexpr bool Eigen::internal::should_svd_compute_thin_v ( int  options )

constexpr

{ return (options & ComputeThinV) != 0; }

References Eigen::ComputeThinV.

Referenced by Eigen::SVDBase< Derived >::allocate(), and check_svd_options_assertions().

shuffle()

EIGEN_STRONG_INLINE Packet2d Eigen::internal::shuffle	(	const Packet2d &	m,
		const Packet2d &	n,
		int	mask
	)

                                                                                     {
  const double* a = reinterpret_cast<const double*>(&m);
  const double* b = reinterpret_cast<const double*>(&n);
  Packet2d res = make_packet2d(*(a + (mask & 1)), *(b + ((mask >> 1) & 1)));
  return res;
}

References a, b, m, make_packet2d(), n, and res.

Referenced by check_indexed_view(), BenchmarkSuite< Device, T >::shuffling(), sparse_basic(), sparse_vector(), Eigen::TensorEvaluator< const TensorShufflingOp< Shuffle, ArgType >, Device >::TensorEvaluator(), test_assign_to_tensor_reshape(), test_assign_to_tensor_shuffle(), test_block_io_copy_using_reordered_dimensions(), test_empty_shuffling(), test_eval_tensor_reshape(), test_eval_tensor_shuffle(), test_execute_reshape(), test_execute_shuffle_lvalue(), test_execute_shuffle_rvalue(), test_multithread_shuffle(), test_shuffle_unshuffle(), test_shuffling_as_value(), test_simple_shuffling(), test_simple_shuffling_sycl(), vec2d_swizzle2(), vec2d_unpackhi(), and vec2d_unpacklo().

shuffle1()

EIGEN_STRONG_INLINE Packet4f Eigen::internal::shuffle1	(	const Packet4f &	m,
		int	mask
	)

                                                                   {
  const float* a = reinterpret_cast<const float*>(&m);
  Packet4f res =
      make_packet4f(*(a + (mask & 3)), *(a + ((mask >> 2) & 3)), *(a + ((mask >> 4) & 3)), *(a + ((mask >> 6) & 3)));
  return res;
}

References a, m, make_packet4f(), and res.

Referenced by vec4f_swizzle1().

shuffle2()

template<bool interleave>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::shuffle2	(	const Packet4f &	m,
		const Packet4f &	n,
		int	mask
	)

                                                                                      {
  const float* a = reinterpret_cast<const float*>(&m);
  const float* b = reinterpret_cast<const float*>(&n);
  Packet4f res =
      make_packet4f(*(a + (mask & 3)), *(a + ((mask >> 2) & 3)), *(b + ((mask >> 4) & 3)), *(b + ((mask >> 6) & 3)));
  return res;
}

References a, b, m, make_packet4f(), n, and res.

Referenced by test_shuffling_as_value().

shuffle2< true >()

template<>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::shuffle2< true >	(	const Packet4f &	m,
		const Packet4f &	n,
		int	mask
	)

                                                                                            {
  const float* a = reinterpret_cast<const float*>(&m);
  const float* b = reinterpret_cast<const float*>(&n);
  Packet4f res =
      make_packet4f(*(a + (mask & 3)), *(b + ((mask >> 2) & 3)), *(a + ((mask >> 4) & 3)), *(b + ((mask >> 6) & 3)));
  return res;
}

References a, b, m, make_packet4f(), n, and res.

Referenced by vec4f_unpackhi(), and vec4f_unpacklo().

size_at_compile_time()

constexpr int Eigen::internal::size_at_compile_time ( int  rows, int  cols  )

inlineconstexpr

                                                              {
  if (rows == 0 || cols == 0) return 0;
  if (rows == Dynamic || cols == Dynamic) return Dynamic;
  return rows * cols;
}

References cols, Eigen::Dynamic, and rows.

smart_copy()

template<typename T >

EIGEN_DEVICE_FUNC void Eigen::internal::smart_copy	(	const T *	start,
		const T *	end,
		T *	target
	)

                                                                           {
  smart_copy_helper<T, !NumTraits<T>::RequireInitialization>::run(start, end, target);
}

References Eigen::placeholders::end, Eigen::run(), and oomph::CumulativeTimings::start().

Referenced by Eigen::internal::DenseStorage_impl< T, Size, Dynamic, Cols, Options >::DenseStorage_impl(), Eigen::internal::DenseStorage_impl< T, Size, Rows, Dynamic, Options >::DenseStorage_impl(), Eigen::internal::DenseStorage_impl< T, Size, Dynamic, Dynamic, Options >::DenseStorage_impl(), Eigen::internal::DenseStorage_impl< T, Dynamic, Dynamic, Cols, Options >::DenseStorage_impl(), Eigen::internal::DenseStorage_impl< T, Dynamic, Rows, Dynamic, Options >::DenseStorage_impl(), Eigen::internal::DenseStorage_impl< T, Dynamic, Dynamic, Dynamic, Options >::DenseStorage_impl(), Eigen::internal::CompressedStorage< Scalar_, StorageIndex_ >::operator=(), Eigen::internal::DenseStorage_impl< T, Size, Dynamic, Cols, Options >::operator=(), Eigen::internal::DenseStorage_impl< T, Size, Rows, Dynamic, Options >::operator=(), Eigen::internal::DenseStorage_impl< T, Size, Dynamic, Dynamic, Options >::operator=(), Eigen::internal::DenseStorage_impl< T, Dynamic, Dynamic, Cols, Options >::operator=(), Eigen::internal::DenseStorage_impl< T, Dynamic, Rows, Dynamic, Options >::operator=(), Eigen::internal::DenseStorage_impl< T, Dynamic, Dynamic, Dynamic, Options >::operator=(), Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::operator=(), Eigen::internal::sparse_matrix_block_impl< SparseMatrixType, BlockRows, BlockCols >::operator=(), Eigen::internal::permutation_matrix_product< ExpressionType, Side, Transposed, SparseShape >::permute_inner(), Eigen::internal::permutation_matrix_product< ExpressionType, Side, Transposed, SparseShape >::permute_outer(), set_from_triplets(), swap_plain_array(), and Eigen::TensorStorage< T, DSizes< IndexType, NumIndices_ >, Options_ >::TensorStorage().

smart_memmove()

template<typename T >

void Eigen::internal::smart_memmove	(	const T *	start,
		const T *	end,
		T *	target
	)

                                                            {
  smart_memmove_helper<T, !NumTraits<T>::RequireInitialization>::run(start, end, target);
}

References Eigen::placeholders::end, Eigen::run(), and oomph::CumulativeTimings::start().

Referenced by Eigen::internal::CompressedStorage< Scalar_, StorageIndex_ >::atWithInsertion(), Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::insertEmptyOuterVectors(), Eigen::internal::CompressedStorage< Scalar_, StorageIndex_ >::moveChunk(), Eigen::internal::sparse_matrix_block_impl< SparseMatrixType, BlockRows, BlockCols >::operator=(), Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::removeOuterVectors(), and Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::reserveInnerVectors().

smart_move()

template<typename T >

EIGEN_DEVICE_FUNC T* Eigen::internal::smart_move	(	T *	start,
		T *	end,
		T *	target
	)

                                                             {
  return std::move(start, end, target);
}

References Eigen::placeholders::end, and oomph::CumulativeTimings::start().

solve_sparse_through_dense_panels() [1/2]

template<typename Decomposition , typename Rhs , typename Dest >

std::enable_if_t<Rhs::ColsAtCompileTime != 1 && Dest::ColsAtCompileTime != 1> Eigen::internal::solve_sparse_through_dense_panels	(	const Decomposition &	dec,
		const Rhs &	rhs,
		Dest &	dest
	)

Helper functions to solve with a sparse right-hand-side and result. The rhs is decomposed into small vertical panels which are solved through dense temporaries.

                                                          {
  EIGEN_STATIC_ASSERT((Dest::Flags & RowMajorBit) == 0, THIS_METHOD_IS_ONLY_FOR_COLUMN_MAJOR_MATRICES);
  typedef typename Dest::Scalar DestScalar;
  // we process the sparse rhs per block of NbColsAtOnce columns temporarily stored into a dense matrix.
  static const Index NbColsAtOnce = 4;
  Index rhsCols = rhs.cols();
  Index size = rhs.rows();
  // the temporary matrices do not need more columns than NbColsAtOnce:
  Index tmpCols = (std::min)(rhsCols, NbColsAtOnce);
  Eigen::Matrix<DestScalar, Dynamic, Dynamic> tmp(size, tmpCols);
  Eigen::Matrix<DestScalar, Dynamic, Dynamic> tmpX(size, tmpCols);
  for (Index k = 0; k < rhsCols; k += NbColsAtOnce) {
    Index actualCols = std::min<Index>(rhsCols - k, NbColsAtOnce);
    tmp.leftCols(actualCols) = rhs.middleCols(k, actualCols);
    tmpX.leftCols(actualCols) = dec.solve(tmp.leftCols(actualCols));
    dest.middleCols(k, actualCols) = tmpX.leftCols(actualCols).sparseView();
  }
}

References EIGEN_STATIC_ASSERT, k, min, Eigen::RowMajorBit, size, and tmp.

Referenced by Eigen::SimplicialCholeskyBase< Derived >::_solve_impl(), Eigen::SimplicialCholesky< MatrixType_, UpLo_, Ordering_ >::_solve_impl(), and Eigen::SparseSolverBase< Derived >::_solve_impl().

solve_sparse_through_dense_panels() [2/2]

template<typename Decomposition , typename Rhs , typename Dest >

std::enable_if_t<Rhs::ColsAtCompileTime == 1 || Dest::ColsAtCompileTime == 1> Eigen::internal::solve_sparse_through_dense_panels	(	const Decomposition &	dec,
		const Rhs &	rhs,
		Dest &	dest
	)

                                                          {
  typedef typename Dest::Scalar DestScalar;
  Index size = rhs.rows();
  Eigen::Matrix<DestScalar, Dynamic, 1> rhs_dense(rhs);
  Eigen::Matrix<DestScalar, Dynamic, 1> dest_dense(size);
  dest_dense = dec.solve(rhs_dense);
  dest = dest_dense.sparseView();
}

References size.

sortWithPermutation()

template<typename VectorType , typename IndexType >

void Eigen::internal::sortWithPermutation	(	VectorType &	vec,
		IndexType &	perm,
		typename IndexType::Scalar &	ncut
	)

Computes a permutation vector to have a sorted sequence.

Parameters

vec	The vector to reorder.
perm	gives the sorted sequence on output. Must be initialized with 0..n-1
ncut	Put the ncut smallest elements at the end of the vector WARNING This is an expensive sort, so should be used only for small size vectors TODO Use modified QuickSplit or std::nth_element to get the smallest values

                                                                                           {
  eigen_assert(vec.size() == perm.size());
  bool flag;
  for (Index k = 0; k < ncut; k++) {
    flag = false;
    for (Index j = 0; j < vec.size() - 1; j++) {
      if (vec(perm(j)) < vec(perm(j + 1))) {
        std::swap(perm(j), perm(j + 1));
        flag = true;
      }
      if (!flag) break;  // The vector is in sorted order
    }
  }
}

References eigen_assert, j, k, and swap().

Referenced by Eigen::DGMRES< MatrixType_, Preconditioner_ >::dgmresComputeDeflationData().

sparse_selfadjoint_time_dense_product()

template<int Mode, typename SparseLhsType , typename DenseRhsType , typename DenseResType , typename AlphaType >

void Eigen::internal::sparse_selfadjoint_time_dense_product ( const SparseLhsType &  lhs, const DenseRhsType &  rhs, DenseResType &  res, const AlphaType &  alpha  )

inline

                                                                          {
  EIGEN_ONLY_USED_FOR_DEBUG(alpha);
 
  typedef typename internal::nested_eval<SparseLhsType, DenseRhsType::MaxColsAtCompileTime>::type SparseLhsTypeNested;
  typedef internal::remove_all_t<SparseLhsTypeNested> SparseLhsTypeNestedCleaned;
  typedef evaluator<SparseLhsTypeNestedCleaned> LhsEval;
  typedef typename LhsEval::InnerIterator LhsIterator;
  typedef typename SparseLhsType::Scalar LhsScalar;
 
  enum {
    LhsIsRowMajor = (LhsEval::Flags & RowMajorBit) == RowMajorBit,
    ProcessFirstHalf = ((Mode & (Upper | Lower)) == (Upper | Lower)) || ((Mode & Upper) && !LhsIsRowMajor) ||
                       ((Mode & Lower) && LhsIsRowMajor),
    ProcessSecondHalf = !ProcessFirstHalf
  };
 
  SparseLhsTypeNested lhs_nested(lhs);
  LhsEval lhsEval(lhs_nested);
 
  // work on one column at once
  for (Index k = 0; k < rhs.cols(); ++k) {
    for (Index j = 0; j < lhs.outerSize(); ++j) {
      LhsIterator i(lhsEval, j);
      // handle diagonal coeff
      if (ProcessSecondHalf) {
        while (i && i.index() < j) ++i;
        if (i && i.index() == j) {
          res.coeffRef(j, k) += alpha * i.value() * rhs.coeff(j, k);
          ++i;
        }
      }
 
      // premultiplied rhs for scatters
      typename ScalarBinaryOpTraits<AlphaType, typename DenseRhsType::Scalar>::ReturnType rhs_j(alpha * rhs(j, k));
      // accumulator for partial scalar product
      typename DenseResType::Scalar res_j(0);
      for (; (ProcessFirstHalf ? i && i.index() < j : i); ++i) {
        LhsScalar lhs_ij = i.value();
        if (!LhsIsRowMajor) lhs_ij = numext::conj(lhs_ij);
        res_j += lhs_ij * rhs.coeff(i.index(), k);
        res(i.index(), k) += numext::conj(lhs_ij) * rhs_j;
      }
      res.coeffRef(j, k) += alpha * res_j;
 
      // handle diagonal coeff
      if (ProcessFirstHalf && i && (i.index() == j)) res.coeffRef(j, k) += alpha * i.value() * rhs.coeff(j, k);
    }
  }
}

References alpha, conj(), EIGEN_ONLY_USED_FOR_DEBUG, i, j, k, Eigen::Lower, res, Eigen::RowMajorBit, and Eigen::Upper.

sparse_sparse_product_with_pruning_impl()

template<typename Lhs , typename Rhs , typename ResultType >

static void Eigen::internal::sparse_sparse_product_with_pruning_impl ( const Lhs &  lhs, const Rhs &  rhs, ResultType &  res, const typename ResultType::RealScalar &  tolerance  )

static

                                                                                                    {
  // return sparse_sparse_product_with_pruning_impl2(lhs,rhs,res);
 
  typedef typename remove_all_t<Rhs>::Scalar RhsScalar;
  typedef typename remove_all_t<ResultType>::Scalar ResScalar;
  typedef typename remove_all_t<Lhs>::StorageIndex StorageIndex;
 
  // make sure to call innerSize/outerSize since we fake the storage order.
  Index rows = lhs.innerSize();
  Index cols = rhs.outerSize();
  // Index size = lhs.outerSize();
  eigen_assert(lhs.outerSize() == rhs.innerSize());
 
  // allocate a temporary buffer
  AmbiVector<ResScalar, StorageIndex> tempVector(rows);
 
  // mimics a resizeByInnerOuter:
  if (ResultType::IsRowMajor)
    res.resize(cols, rows);
  else
    res.resize(rows, cols);
 
  evaluator<Lhs> lhsEval(lhs);
  evaluator<Rhs> rhsEval(rhs);
 
  // estimate the number of non zero entries
  // given a rhs column containing Y non zeros, we assume that the respective Y columns
  // of the lhs differs in average of one non zeros, thus the number of non zeros for
  // the product of a rhs column with the lhs is X+Y where X is the average number of non zero
  // per column of the lhs.
  // Therefore, we have nnz(lhs*rhs) = nnz(lhs) + nnz(rhs)
  Index estimated_nnz_prod = lhsEval.nonZerosEstimate() + rhsEval.nonZerosEstimate();
 
  res.reserve(estimated_nnz_prod);
  double ratioColRes = double(estimated_nnz_prod) / (double(lhs.rows()) * double(rhs.cols()));
  for (Index j = 0; j < cols; ++j) {
    // FIXME:
    // double ratioColRes = (double(rhs.innerVector(j).nonZeros()) +
    // double(lhs.nonZeros())/double(lhs.cols()))/double(lhs.rows());
    // let's do a more accurate determination of the nnz ratio for the current column j of res
    tempVector.init(ratioColRes);
    tempVector.setZero();
    for (typename evaluator<Rhs>::InnerIterator rhsIt(rhsEval, j); rhsIt; ++rhsIt) {
      // FIXME should be written like this: tmp += rhsIt.value() * lhs.col(rhsIt.index())
      tempVector.restart();
      RhsScalar x = rhsIt.value();
      for (typename evaluator<Lhs>::InnerIterator lhsIt(lhsEval, rhsIt.index()); lhsIt; ++lhsIt) {
        tempVector.coeffRef(lhsIt.index()) += lhsIt.value() * x;
      }
    }
    res.startVec(j);
    for (typename AmbiVector<ResScalar, StorageIndex>::Iterator it(tempVector, tolerance); it; ++it)
      res.insertBackByOuterInner(j, it.index()) = it.value();
  }
  res.finalize();
}

References Eigen::internal::AmbiVector< Scalar_, StorageIndex_ >::coeffRef(), cols, eigen_assert, Eigen::internal::AmbiVector< Scalar_, StorageIndex_ >::init(), j, res, Eigen::internal::AmbiVector< Scalar_, StorageIndex_ >::restart(), rows, Eigen::internal::AmbiVector< Scalar_, StorageIndex_ >::setZero(), and plotDoE::x.

sparse_sparse_to_dense_product_impl()

template<typename Lhs , typename Rhs , typename ResultType >

static void Eigen::internal::sparse_sparse_to_dense_product_impl ( const Lhs &  lhs, const Rhs &  rhs, ResultType &  res  )

static

                                                                                                 {
  typedef typename remove_all_t<Lhs>::Scalar LhsScalar;
  typedef typename remove_all_t<Rhs>::Scalar RhsScalar;
  Index cols = rhs.outerSize();
  eigen_assert(lhs.outerSize() == rhs.innerSize());
 
  evaluator<Lhs> lhsEval(lhs);
  evaluator<Rhs> rhsEval(rhs);
 
  for (Index j = 0; j < cols; ++j) {
    for (typename evaluator<Rhs>::InnerIterator rhsIt(rhsEval, j); rhsIt; ++rhsIt) {
      RhsScalar y = rhsIt.value();
      Index k = rhsIt.index();
      for (typename evaluator<Lhs>::InnerIterator lhsIt(lhsEval, k); lhsIt; ++lhsIt) {
        Index i = lhsIt.index();
        LhsScalar x = lhsIt.value();
        res.coeffRef(i, j) += x * y;
      }
    }
  }
}

References cols, eigen_assert, i, j, k, res, plotDoE::x, and y.

sparse_time_dense_product()

template<typename SparseLhsType , typename DenseRhsType , typename DenseResType , typename AlphaType >

void Eigen::internal::sparse_time_dense_product ( const SparseLhsType &  lhs, const DenseRhsType &  rhs, DenseResType &  res, const AlphaType &  alpha  )

inline

                                                              {
  sparse_time_dense_product_impl<SparseLhsType, DenseRhsType, DenseResType, AlphaType>::run(lhs, rhs, res, alpha);
}

References alpha, res, and run().

Referenced by Eigen::BlockSparseTimeDenseProduct< Lhs, Rhs >::scaleAndAddTo(), Eigen::internal::generic_product_impl< Lhs, Rhs, SparseShape, DenseShape, ProductType >::scaleAndAddTo(), and Eigen::internal::generic_product_impl< Lhs, Rhs, DenseShape, SparseShape, ProductType >::scaleAndAddTo().

sse_blend_mask() [1/2]

EIGEN_STRONG_INLINE __m128i Eigen::internal::sse_blend_mask

(

const Selector< 2 > &

ifPacket

)

                                                                        {
  return _mm_set_epi64x(0 - ifPacket.select[1], 0 - ifPacket.select[0]);
}

References Eigen::internal::Selector< N >::select.

Referenced by pblend().

sse_blend_mask() [2/2]

EIGEN_STRONG_INLINE __m128i Eigen::internal::sse_blend_mask

(

const Selector< 4 > &

ifPacket

)

                                                                        {
  return _mm_set_epi32(0 - ifPacket.select[3], 0 - ifPacket.select[2], 0 - ifPacket.select[1], 0 - ifPacket.select[0]);
}

References Eigen::internal::Selector< N >::select.

stable_norm_impl() [1/2]

template<typename MatrixType >

MatrixType::RealScalar Eigen::internal::stable_norm_impl	(	const MatrixType &	mat,
		std::enable_if_t<!MatrixType::IsVectorAtCompileTime > *	= 0
	)

                                                                                                        {
  using std::sqrt;
 
  typedef typename MatrixType::RealScalar RealScalar;
  RealScalar scale(0);
  RealScalar invScale(1);
  RealScalar ssq(0);  // sum of squares
 
  for (Index j = 0; j < mat.outerSize(); ++j) stable_norm_impl_inner_step(mat.innerVector(j), ssq, scale, invScale);
  return scale * sqrt(ssq);
}

References j, Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::outerSize(), sqrt(), and stable_norm_impl_inner_step().

stable_norm_impl() [2/2]

template<typename VectorType >

VectorType::RealScalar Eigen::internal::stable_norm_impl	(	const VectorType &	vec,
		std::enable_if_t< VectorType::IsVectorAtCompileTime > *	= 0
	)

                                                                                                       {
  using std::abs;
  using std::sqrt;
 
  Index n = vec.size();
  if (EIGEN_PREDICT_FALSE(n == 1)) return abs(vec.coeff(0));
 
  typedef typename VectorType::RealScalar RealScalar;
  RealScalar scale(0);
  RealScalar invScale(1);
  RealScalar ssq(0);  // sum of squares
 
  stable_norm_impl_inner_step(vec, ssq, scale, invScale);
 
  return scale * sqrt(ssq);
}

References abs(), EIGEN_PREDICT_FALSE, n, sqrt(), and stable_norm_impl_inner_step().

Referenced by Eigen::MatrixBase< Derived >::stableNorm().

stable_norm_impl_inner_step()

template<typename VectorType , typename RealScalar >

void Eigen::internal::stable_norm_impl_inner_step	(	const VectorType &	vec,
		RealScalar &	ssq,
		RealScalar &	scale,
		RealScalar &	invScale
	)

                                                                                                                  {
  const Index blockSize = 4096;
 
  Index n = vec.size();
  Index blockEnd = numext::round_down(n, blockSize);
  for (Index i = 0; i < blockEnd; i += blockSize) {
    internal::stable_norm_kernel(vec.template segment<blockSize>(i), ssq, scale, invScale);
  }
  if (n > blockEnd) {
    internal::stable_norm_kernel(vec.tail(n - blockEnd), ssq, scale, invScale);
  }
}

References i, n, Eigen::numext::round_down(), and stable_norm_kernel().

Referenced by stable_norm_impl().

stable_norm_kernel()

template<typename ExpressionType , typename Scalar >

void Eigen::internal::stable_norm_kernel ( const ExpressionType &  bl, Scalar &  ssq, Scalar &  scale, Scalar &  invScale  )

inline

                                                                                                       {
  Scalar maxCoeff = bl.cwiseAbs().maxCoeff();
 
  if (maxCoeff > scale) {
    ssq = ssq * numext::abs2(scale / maxCoeff);
    Scalar tmp = Scalar(1) / maxCoeff;
    if (tmp > NumTraits<Scalar>::highest()) {
      invScale = NumTraits<Scalar>::highest();
      scale = Scalar(1) / invScale;
    } else if (maxCoeff > NumTraits<Scalar>::highest())  // we got a INF
    {
      invScale = Scalar(1);
      scale = maxCoeff;
    } else {
      scale = maxCoeff;
      invScale = tmp;
    }
  } else if (maxCoeff != maxCoeff)  // we got a NaN
  {
    scale = maxCoeff;
  }
 
  // TODO if the maxCoeff is much much smaller than the current scale,
  // then we can neglect this sub vector
  if (scale > Scalar(0))  // if scale==0, then bl is 0
    ssq += (bl * invScale).squaredNorm();
}

References Eigen::numext::abs2(), and tmp.

Referenced by stable_norm_impl_inner_step().

stem_function_cos()

template<typename Scalar >

Scalar Eigen::internal::stem_function_cos	(	Scalar	x,
		int	n
	)

Cosine (and its derivatives).

                                          {
  using std::cos;
  using std::sin;
  Scalar res;
 
  switch (n % 4) {
    case 0:
      res = std::cos(x);
      break;
    case 1:
      res = -std::sin(x);
      break;
    case 2:
      res = -std::cos(x);
      break;
    case 3:
      res = std::sin(x);
      break;
  }
  return res;
}

References cos(), n, res, sin(), and plotDoE::x.

stem_function_cosh()

template<typename Scalar >

Scalar Eigen::internal::stem_function_cosh	(	Scalar	x,
		int	n
	)

Hyperbolic cosine (and its derivatives).

                                           {
  using std::cosh;
  using std::sinh;
  Scalar res;
 
  switch (n % 2) {
    case 0:
      res = std::cosh(x);
      break;
    case 1:
      res = std::sinh(x);
      break;
  }
  return res;
}

References Eigen::bfloat16_impl::cosh(), n, res, Eigen::bfloat16_impl::sinh(), and plotDoE::x.

stem_function_exp()

template<typename Scalar >

Scalar Eigen::internal::stem_function_exp	(	Scalar	x,
		int
	)

The exponential function (and its derivatives).

                                        {
  using std::exp;
  return exp(x);
}

References Eigen::bfloat16_impl::exp(), and plotDoE::x.

stem_function_sin()

template<typename Scalar >

Scalar Eigen::internal::stem_function_sin	(	Scalar	x,
		int	n
	)

Sine (and its derivatives).

                                          {
  using std::cos;
  using std::sin;
  Scalar res;
 
  switch (n % 4) {
    case 0:
      res = std::sin(x);
      break;
    case 1:
      res = std::cos(x);
      break;
    case 2:
      res = -std::sin(x);
      break;
    case 3:
      res = -std::cos(x);
      break;
  }
  return res;
}

References cos(), n, res, sin(), and plotDoE::x.

stem_function_sinh()

template<typename Scalar >

Scalar Eigen::internal::stem_function_sinh	(	Scalar	x,
		int	n
	)

Hyperbolic sine (and its derivatives).

                                           {
  using std::cosh;
  using std::sinh;
  Scalar res;
 
  switch (n % 2) {
    case 0:
      res = std::sinh(x);
      break;
    case 1:
      res = std::cosh(x);
      break;
  }
  return res;
}

References Eigen::bfloat16_impl::cosh(), n, res, Eigen::bfloat16_impl::sinh(), and plotDoE::x.

storeAccumulator()

template<typename DataMapper , typename Packet , bool full>

EIGEN_ALWAYS_INLINE void Eigen::internal::storeAccumulator	(	Index	i,
		const DataMapper &	data,
		const Packet &	alpha,
		const Index	elements,
		__vector_quad *	acc
	)

                                                              {
  PacketBlock<Packet, 4> result;
  __builtin_mma_disassemble_acc(&result.packet, acc);
 
  PacketBlock<Packet, 4> tRes;
  if (full) {
    EIGEN_UNUSED_VARIABLE(elements);
    bload<DataMapper, Packet, 0, ColMajor, false, 4>(tRes, data, i, 0);
    bscale<Packet, 4>(tRes, result, alpha);
    bstore<DataMapper, Packet, 4>(tRes, data, i);
  } else {
    bload_partial<DataMapper, Packet, 0, false, 4>(tRes, data, i, elements);
    bscale<Packet, 4>(tRes, result, alpha);
    bstore_partial<DataMapper, Packet, 4>(tRes, data, i, elements);
  }
}

References alpha, data, EIGEN_UNUSED_VARIABLE, i, and Eigen::internal::PacketBlock< Packet, N >::packet.

storeBF16fromResult()

template<const Index size, bool non_unit_stride, Index delta>

EIGEN_ALWAYS_INLINE void Eigen::internal::storeBF16fromResult	(	bfloat16 *	dst,
		Packet8bf	data,
		Index	resInc,
		Index	extra = 0
	)

storeBlock()

template<typename Scalar , typename Packet , int N>

EIGEN_ALWAYS_INLINE void Eigen::internal::storeBlock	(	Scalar *	to,
		PacketBlock< Packet, N > &	block
	)

PanelMode Packing might be called several times before being multiplied by gebp_kernel, this happens because on special occasions it fills part of block with other parts of the matrix. Two variables control how PanelMode should behave: offset and stride. The idea is that those variables represent whatever is going to be the real offset and stride in the future and this is what you should obey. The process is to behave as you would with normal packing but leave the start of each part with the correct offset and the end as well respecting the real stride the block will have. Gebp is aware of both blocks stride and offset and behaves accordingly.

                                                                               {
  const Index size = 16 / sizeof(Scalar);
  pstore<Scalar>(to + (0 * size), block.packet[0]);
  pstore<Scalar>(to + (1 * size), block.packet[1]);
  if (N > 2) {
    pstore<Scalar>(to + (2 * size), block.packet[2]);
  }
  if (N > 3) {
    pstore<Scalar>(to + (3 * size), block.packet[3]);
  }
}

References block(), N, and size.

storeComplexAccumulator()

template<typename DataMapper , typename Packet , typename Packetc , const Index accCols, const Index accCols2>

EIGEN_ALWAYS_INLINE void Eigen::internal::storeComplexAccumulator	(	Index	i,
		const DataMapper &	data,
		const Packet &	alphaReal,
		const Packet &	alphaImag,
		const Packet &	pMask,
		__vector_quad *	accReal,
		__vector_quad *	accImag
	)

                                                                         {
  constexpr bool full = (accCols2 > accColsC);
  PacketBlock<Packet, 4> resultReal, resultImag;
  __builtin_mma_disassemble_acc(&resultReal.packet, accReal);
  __builtin_mma_disassemble_acc(&resultImag.packet, accImag);
 
  PacketBlock<Packetc, 8> tRes;
  bload<DataMapper, Packetc, accColsC, ColMajor, true, 4, full>(tRes, data, i, 0);
 
  PacketBlock<Packet, 4> taccReal, taccImag;
  bscalec<Packet, 4, (accCols != accCols2)>(resultReal, resultImag, alphaReal, alphaImag, taccReal, taccImag, pMask);
 
  PacketBlock<Packetc, 4> acc1, acc2;
  bcouple<Packet, Packetc, 4, full>(taccReal, taccImag, tRes, acc1, acc2);
 
  bstore<DataMapper, Packetc, 4>(acc1, data, i);
  if (full) {
    bstore<DataMapper, Packetc, 4>(acc2, data, i + accColsC);
  }
}

References accColsC, data, i, and Eigen::internal::PacketBlock< Packet, N >::packet.

storeConvertBlockBF16()

template<Index N>

EIGEN_ALWAYS_INLINE void Eigen::internal::storeConvertBlockBF16	(	float *	to,
		PacketBlock< Packet8bf,(N+7)/8 > &	block,
		Index	extra
	)

                                                                                                                   {
  storeConvertTwoBF16<N, 0>(to + 0, block, extra);
  if (N >= 16) {
    storeConvertTwoBF16<N, 1>(to + 8, block);
  }
  if (N >= 32) {
    storeConvertTwoBF16<N, 2>(to + 16, block);
    storeConvertTwoBF16<N, 3>(to + 24, block);
  }
}

References block(), and N.

storeConvertTwoBF16()

template<Index N, Index M>

EIGEN_ALWAYS_INLINE void Eigen::internal::storeConvertTwoBF16	(	float *	to,
		PacketBlock< Packet8bf,(N+7)/8 > &	block,
		Index	extra = 0
	)

                                                                                                                     {
  if (N < 4) {
    pstoreu_partial(to + 0, oneConvertBF16Hi(block.packet[0].m_val), extra);
  } else if (N >= (M * 8 + 4)) {
    pstoreu(to + 0, oneConvertBF16Hi(block.packet[M].m_val));
    if (N >= 8) {
      pstoreu(to + 4, oneConvertBF16Lo(block.packet[M].m_val));
    }
  }
}

References block(), N, oneConvertBF16Hi(), oneConvertBF16Lo(), pstoreu(), and pstoreu_partial().

storeF32()

template<bool lhsExtraRows>

EIGEN_ALWAYS_INLINE void Eigen::internal::storeF32	(	float *&	result,
		Packet4f	result_block,
		Index	rows,
		Index	extra_rows
	)

                                                                                                       {
  if (lhsExtraRows) {
    pstoreu_partial(result, result_block, extra_rows);
  } else {
    pstoreu(result, result_block);
  }
  result += rows;
}

References pstoreu(), pstoreu_partial(), and rows.

storeResults()

template<bool rhsExtraCols, bool lhsExtraRows>

EIGEN_ALWAYS_INLINE void Eigen::internal::storeResults	(	Packet4f(&)	acc[4],
		Index	rows,
		const Packet4f	pAlpha,
		float *	result,
		Index	extra_cols,
		Index	extra_rows
	)

                                                                          {
  Index x = 0;
  if (rhsExtraCols) {
    do {
      Packet4f result_block = loadAndMultiplyF32(acc[x], pAlpha, result);
      storeF32<lhsExtraRows>(result, result_block, rows, extra_rows);
    } while (++x < extra_cols);
  } else {
    Packet4f result_block[4];
    float* result2 = result;
    do {
      result_block[x] = loadAndMultiplyF32(acc[x], pAlpha, result);
      result += rows;
    } while (++x < 4);
    x = 0;
    do {
      storeF32<lhsExtraRows>(result2, result_block[x], rows, extra_rows);
    } while (++x < 4);
  }
}

References loadAndMultiplyF32(), rows, and plotDoE::x.

strides() [1/3]

template<int Layout, typename IndexType , int NumDims>

EIGEN_ALWAYS_INLINE DSizes<IndexType, NumDims> Eigen::internal::strides

(

const DSizes< IndexType, NumDims > &

dimensions

)

                                                                                                     {
  DSizes<IndexType, NumDims> strides;
  if (NumDims == 0) return strides;
 
  // TODO(ezhulenev): Use templates to unroll this loop (similar to
  // h_array_reduce in CXX11meta.h)? Benchmark it.
  if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
    strides[0] = 1;
    for (int i = 1; i < NumDims; ++i) {
      strides[i] = strides[i - 1] * dimensions[i - 1];
    }
  } else {
    strides[NumDims - 1] = 1;
    for (int i = NumDims - 2; i >= 0; --i) {
      strides[i] = strides[i + 1] * dimensions[i + 1];
    }
  }
 
  return strides;
}

References Eigen::ColMajor, and i.

Referenced by ComputeStrides(), InsertZeros< TensorType >::eval(), Eigen::TensorEvaluator< const TensorPairReducerOp< ReduceOp, Dims, ArgType >, Device >::gen_strides(), Eigen::TensorBase< Derived, ReadOnlyAccessors >::inflate(), RandomBlock(), SkewedInnerBlock(), Eigen::TensorBase< Derived, ReadOnlyAccessors >::stride(), Eigen::TensorBase< Derived, AccessLevel >::stride(), Eigen::TensorBase< Derived, ReadOnlyAccessors >::stridedSlice(), Eigen::TensorBase< Derived, AccessLevel >::stridedSlice(), BenchmarkSuite< Device, T >::striding(), test_block_io_squeeze_ones(), test_block_mapper_maps_every_element(), test_simple_inflation(), test_simple_inflation_sycl(), test_simple_striding(), test_strided_slice(), test_strided_slice_as_rhs_sycl(), test_strided_slice_write(), test_strided_slice_write_sycl(), and test_striding_as_lvalue().

strides() [2/3]

template<int Layout, typename IndexType , size_t NumDims>

EIGEN_ALWAYS_INLINE DSizes<IndexType, NumDims> Eigen::internal::strides

(

const Eigen::array< IndexType, NumDims > &

dimensions

)

                                                                                                         {
  return strides<Layout>(DSizes<IndexType, NumDims>(dimensions));
}

strides() [3/3]

template<int Layout, std::ptrdiff_t... Indices>

EIGEN_STRONG_INLINE DSizes<std::ptrdiff_t, sizeof...(Indices)> Eigen::internal::strides

(

const Sizes< Indices... > &

sizes

)

                                                                                                     {
  return strides<Layout>(DSizes<std::ptrdiff_t, sizeof...(Indices)>(sizes));
}

References sizes.

supportsMMA()

EIGEN_ALWAYS_INLINE bool Eigen::internal::supportsMMA

(

)

                                       {
#if defined(EIGEN_ALTIVEC_MMA_ONLY)
  return true;
#elif defined(EIGEN_ALTIVEC_MMA_DYNAMIC_DISPATCH) && defined(__BUILTIN_CPU_SUPPORTS__)
  return __builtin_cpu_supports("arch_3_1") && __builtin_cpu_supports("mma");
#else
  return false;  // No dynamic dispatch for LLVM or older GCC
#endif
}

Referenced by Eigen::internal::gebp_kernel< bfloat16, bfloat16, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >::operator()(), Eigen::internal::gebp_kernel< double, double, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >::operator()(), Eigen::internal::gebp_kernel< double, std::complex< double >, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >::operator()(), Eigen::internal::gebp_kernel< float, float, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >::operator()(), Eigen::internal::gebp_kernel< float, std::complex< float >, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >::operator()(), Eigen::internal::gebp_kernel< std::complex< double >, double, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >::operator()(), Eigen::internal::gebp_kernel< std::complex< double >, std::complex< double >, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >::operator()(), Eigen::internal::gebp_kernel< std::complex< float >, float, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >::operator()(), and Eigen::internal::gebp_kernel< std::complex< float >, std::complex< float >, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >::operator()().

swap()

template<typename T >

void Eigen::internal::swap	(	scoped_array< T > &	a,
		scoped_array< T > &	b
	)

                                                  {
  std::swap(a.ptr(), b.ptr());
}

References a, and b.

Referenced by Eigen::internal::swap_assign_op< Scalar >::assignCoeff(), Eigen::internal::DenseStorage_impl< T, Dynamic, Dynamic, Cols, Options >::operator=(), Eigen::internal::DenseStorage_impl< T, Dynamic, Rows, Dynamic, Options >::operator=(), Eigen::internal::DenseStorage_impl< T, Dynamic, Dynamic, Dynamic, Options >::operator=(), QuickSplit(), Eigen::internal::permutation_matrix_product< ExpressionType, Side, Transposed, DenseShape >::run(), and Eigen::internal::tuple_impl::TupleImpl< N, T1, Ts... >::swap().

swap_plain_array()

template<typename T , int Size, int Options, int Alignment>

EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE void Eigen::internal::swap_plain_array ( plain_array< T, Size, Options, Alignment > &  a, plain_array< T, Size, Options, Alignment > &  b, Index  a_size, Index  b_size  )

constexpr

                                                                                                  {
  Index common_size = numext::mini(a_size, b_size);
  std::swap_ranges(a.array, a.array + common_size, b.array);
  if (a_size > b_size)
    smart_copy(a.array + common_size, a.array + a_size, b.array + common_size);
  else if (b_size > a_size)
    smart_copy(b.array + common_size, b.array + b_size, a.array + common_size);
}

References a, b, Eigen::numext::mini(), and smart_copy().

Referenced by Eigen::internal::DenseStorage_impl< T, Size, Dynamic, Cols, Options >::swap(), Eigen::internal::DenseStorage_impl< T, Size, Rows, Dynamic, Options >::swap(), and Eigen::internal::DenseStorage_impl< T, Size, Dynamic, Dynamic, Options >::swap().

symm_pack_complex_lhs_helper()

template<typename Scalar , int StorageOrder>

EIGEN_STRONG_INLINE void Eigen::internal::symm_pack_complex_lhs_helper	(	std::complex< Scalar > *	blockA,
		const std::complex< Scalar > *	_lhs,
		Index	lhsStride,
		Index	cols,
		Index	rows
	)

                                                                                               {
  const Index depth = cols;
  const_blas_data_mapper<std::complex<Scalar>, Index, StorageOrder> lhs(_lhs, lhsStride);
  const Index vectorSize = quad_traits<Scalar>::vectorsize;
  const Index vectorDelta = vectorSize * depth;
  Scalar* blockAf = reinterpret_cast<Scalar*>(blockA);
 
  Index rir = 0, rii, j = 0;
  for (; j + vectorSize <= rows; j += vectorSize) {
    rii = rir + vectorDelta;
 
    for (Index i = 0; i < depth; i++) {
      for (Index k = 0; k < vectorSize; k++) {
        std::complex<Scalar> v = getAdjointVal<Scalar, StorageOrder>(j + k, i, lhs);
 
        blockAf[rir + k] = v.real();
        blockAf[rii + k] = v.imag();
      }
      rir += vectorSize;
      rii += vectorSize;
    }
 
    rir += vectorDelta;
  }
 
  if (j < rows) {
    rii = rir + ((rows - j) * depth);
 
    for (Index i = 0; i < depth; i++) {
      Index k = j;
      for (; k < rows; k++) {
        std::complex<Scalar> v = getAdjointVal<Scalar, StorageOrder>(k, i, lhs);
 
        blockAf[rir] = v.real();
        blockAf[rii] = v.imag();
 
        rir += 1;
        rii += 1;
      }
    }
  }
}

References cols, i, j, k, rows, and v.

symm_pack_complex_rhs_helper()

template<typename Scalar , int StorageOrder, int N>

EIGEN_STRONG_INLINE void Eigen::internal::symm_pack_complex_rhs_helper	(	std::complex< Scalar > *	blockB,
		const std::complex< Scalar > *	_rhs,
		Index	rhsStride,
		Index	rows,
		Index	cols,
		Index	k2
	)

                                                                                                         {
  const Index depth = k2 + rows;
  const_blas_data_mapper<std::complex<Scalar>, Index, StorageOrder> rhs(_rhs, rhsStride);
  const Index vectorSize = N * quad_traits<Scalar>::vectorsize;
  const Index vectorDelta = vectorSize * rows;
  Scalar* blockBf = reinterpret_cast<Scalar*>(blockB);
 
  Index rir = 0, rii, j = 0;
  for (; j + vectorSize <= cols; j += vectorSize) {
    rii = rir + vectorDelta;
 
    for (Index i = k2; i < depth; i++) {
      for (Index k = 0; k < vectorSize; k++) {
        std::complex<Scalar> v = getAdjointVal<Scalar, StorageOrder>(i, j + k, rhs);
 
        blockBf[rir + k] = v.real();
        blockBf[rii + k] = v.imag();
      }
      rir += vectorSize;
      rii += vectorSize;
    }
 
    rir += vectorDelta;
  }
 
  for (; j < cols; j++) {
    rii = rir + rows;
 
    for (Index i = k2; i < depth; i++) {
      std::complex<Scalar> v = getAdjointVal<Scalar, StorageOrder>(i, j, rhs);
 
      blockBf[rir] = v.real();
      blockBf[rii] = v.imag();
 
      rir += 1;
      rii += 1;
    }
 
    rir += rows;
  }
}

References cols, i, j, k, N, rows, and v.

symm_pack_lhs_helper()

template<typename Scalar , int StorageOrder>

EIGEN_STRONG_INLINE void Eigen::internal::symm_pack_lhs_helper	(	Scalar *	blockA,
		const Scalar *	_lhs,
		Index	lhsStride,
		Index	cols,
		Index	rows
	)

                                                          {
  const Index depth = cols;
  const_blas_data_mapper<Scalar, Index, StorageOrder> lhs(_lhs, lhsStride);
  const Index vectorSize = quad_traits<Scalar>::vectorsize;
 
  Index ri = 0, j = 0;
  for (; j + vectorSize <= rows; j += vectorSize) {
    Index i = 0;
 
    for (; i < depth; i++) {
      for (Index k = 0; k < vectorSize; k++) {
        if (i <= j + k)
          blockA[ri + k] = lhs(j + k, i);
        else
          blockA[ri + k] = lhs(i, j + k);
      }
      ri += vectorSize;
    }
  }
 
  if (j < rows) {
    for (Index i = 0; i < depth; i++) {
      Index k = j;
      for (; k < rows; k++) {
        if (i <= k)
          blockA[ri] = lhs(k, i);
        else
          blockA[ri] = lhs(i, k);
        ri += 1;
      }
    }
  }
}

References cols, i, j, k, and rows.

symm_pack_rhs_helper()

template<typename Scalar , int StorageOrder, int N>

EIGEN_STRONG_INLINE void Eigen::internal::symm_pack_rhs_helper	(	Scalar *	blockB,
		const Scalar *	_rhs,
		Index	rhsStride,
		Index	rows,
		Index	cols,
		Index	k2
	)

                                                                    {
  const Index depth = k2 + rows;
  const_blas_data_mapper<Scalar, Index, StorageOrder> rhs(_rhs, rhsStride);
  const Index vectorSize = quad_traits<Scalar>::vectorsize;
 
  Index ri = 0, j = 0;
  for (; j + N * vectorSize <= cols; j += N * vectorSize) {
    Index i = k2;
    for (; i < depth; i++) {
      for (Index k = 0; k < N * vectorSize; k++) {
        if (i <= j + k)
          blockB[ri + k] = rhs(j + k, i);
        else
          blockB[ri + k] = rhs(i, j + k);
      }
      ri += N * vectorSize;
    }
  }
 
  for (; j < cols; j++) {
    for (Index i = k2; i < depth; i++) {
      if (j <= i)
        blockB[ri] = rhs(i, j);
      else
        blockB[ri] = rhs(j, i);
      ri += 1;
    }
  }
}

References cols, i, j, k, N, and rows.

tensor_static_symgroup_index_permute() [1/2]

template<typename Index , std::size_t N, int... ii, int... jj>

constexpr static std::array<Index, N> Eigen::internal::tensor_static_symgroup_index_permute ( std::array< Index, N >  idx, internal::numeric_list< int, ii... >  , internal::numeric_list< int, jj... >    )

inlinestaticconstexpr

                                                                                                                    {
  return {{idx[ii]..., idx[jj]...}};
}

Referenced by Eigen::internal::tensor_static_symgroup_do_apply< internal::type_list< first, next... > >::run().

tensor_static_symgroup_index_permute() [2/2]

template<typename Index , int... ii>

static std::vector<Index> Eigen::internal::tensor_static_symgroup_index_permute ( std::vector< Index >  idx, internal::numeric_list< int, ii... >    )

inlinestatic

                                                                                                        {
  std::vector<Index> result{{idx[ii]...}};
  std::size_t target_size = idx.size();
  for (std::size_t i = result.size(); i < target_size; i++) result.push_back(idx[i]);
  return result;
}

References i.

throw_std_bad_alloc()

EIGEN_DEVICE_FUNC void Eigen::internal::throw_std_bad_alloc ( )

inline

                                                    {
#ifdef EIGEN_EXCEPTIONS
  throw std::bad_alloc();
#else
  std::size_t huge = static_cast<std::size_t>(-1);
#if defined(EIGEN_HIPCC)
  //
  // calls to "::operator new" are to be treated as opaque function calls (i.e no inlining),
  // and as a consequence the code in the #else block triggers the hipcc warning :
  // "no overloaded function has restriction specifiers that are compatible with the ambient context"
  //
  // "throw_std_bad_alloc" has the EIGEN_DEVICE_FUNC attribute, so it seems that hipcc expects
  // the same on "operator new"
  // Reverting code back to the old version in this #if block for the hipcc compiler
  //
  new int[huge];
#else
  void* unused = ::operator new(huge);
  EIGEN_UNUSED_VARIABLE(unused);
#endif
#endif
}

References EIGEN_UNUSED_VARIABLE.

Referenced by aligned_malloc(), aligned_realloc(), check_size_for_overflow(), conditional_aligned_malloc< false >(), Eigen::internal::CompressedStorage< Scalar_, StorageIndex_ >::resize(), Eigen::internal::check_rows_cols_for_overflow< Dynamic, Dynamic, Dynamic >::run(), Eigen::internal::check_rows_cols_for_overflow< Dynamic, Dynamic, MaxColsAtCompileTime >::run(), Eigen::internal::check_rows_cols_for_overflow< Dynamic, MaxRowsAtCompileTime, Dynamic >::run(), Eigen::internal::eigen_zero_impl< Xpr, true >::run(), set_from_triplets(), and set_from_triplets_sorted().

toRotationMatrix() [1/3]

template<typename Scalar , int Dim, typename OtherDerived >

static EIGEN_DEVICE_FUNC const MatrixBase<OtherDerived>& Eigen::internal::toRotationMatrix ( const MatrixBase< OtherDerived > &  mat )

inlinestatic

                                                                                                                      {
  EIGEN_STATIC_ASSERT(OtherDerived::RowsAtCompileTime == Dim && OtherDerived::ColsAtCompileTime == Dim,
                      YOU_MADE_A_PROGRAMMING_MISTAKE)
  return mat;
}

References Global_Variables::Dim, and EIGEN_STATIC_ASSERT.

toRotationMatrix() [2/3]

template<typename Scalar , int Dim, typename OtherDerived >

static EIGEN_DEVICE_FUNC Matrix<Scalar, Dim, Dim> Eigen::internal::toRotationMatrix ( const RotationBase< OtherDerived, Dim > &  r )

inlinestatic

                                                                                                                    {
  return r.toRotationMatrix();
}

References UniformPSDSelfTest::r.

toRotationMatrix() [3/3]

template<typename Scalar , int Dim>

static EIGEN_DEVICE_FUNC Matrix<Scalar, 2, 2> Eigen::internal::toRotationMatrix ( const Scalar &  s )

inlinestatic

Helper function to return an arbitrary rotation object to a rotation matrix.

Template Parameters

Scalar	the numeric type of the matrix coefficients
Dim	the dimension of the current space

It returns a Dim x Dim fixed size matrix.

Default specializations are provided for:

• any scalar type (2D),
• any matrix expression,
• any type based on RotationBase (e.g., Quaternion, AngleAxis, Rotation2D)

Currently toRotationMatrix is only used by Transform.

See also

class Transform, class Rotation2D, class Quaternion, class AngleAxis

                                                                                       {
  EIGEN_STATIC_ASSERT(Dim == 2, YOU_MADE_A_PROGRAMMING_MISTAKE)
  return Rotation2D<Scalar>(s).toRotationMatrix();
}

References Global_Variables::Dim, EIGEN_STATIC_ASSERT, s, and Eigen::Rotation2D< Scalar_ >::toRotationMatrix().

Referenced by transformations().

tranposeResults()

template<Index num_acc>

EIGEN_ALWAYS_INLINE void Eigen::internal::tranposeResults

(

Packet4f(&)

acc[num_acc][4]

)

                                                                      {
  for (Index i = 0; i < num_acc; i++) {
    Packet4ui t0, t1, t2, t3;
    t0 = vec_mergeh(reinterpret_cast<Packet4ui>(acc[i][0]), reinterpret_cast<Packet4ui>(acc[i][2]));
    t1 = vec_mergel(reinterpret_cast<Packet4ui>(acc[i][0]), reinterpret_cast<Packet4ui>(acc[i][2]));
    t2 = vec_mergeh(reinterpret_cast<Packet4ui>(acc[i][1]), reinterpret_cast<Packet4ui>(acc[i][3]));
    t3 = vec_mergel(reinterpret_cast<Packet4ui>(acc[i][1]), reinterpret_cast<Packet4ui>(acc[i][3]));
    acc[i][0] = reinterpret_cast<Packet4f>(vec_mergeh(t0, t2));
    acc[i][1] = reinterpret_cast<Packet4f>(vec_mergel(t0, t2));
    acc[i][2] = reinterpret_cast<Packet4f>(vec_mergeh(t1, t3));
    acc[i][3] = reinterpret_cast<Packet4f>(vec_mergel(t1, t3));
  }
}

References i.

transStoreC()

template<typename Scalar , typename vec , int64_t unrollM, int64_t unrollN, bool remM, bool remN>

EIGEN_ALWAYS_INLINE void Eigen::internal::transStoreC	(	PacketBlock< vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS > &	zmm,
		Scalar *	C_arr,
		int64_t	LDC,
		int64_t	remM_ = 0,
		int64_t	remN_ = 0
	)

Used by gemmKernel for the case A/B row-major and C col-major.

                                                                                        {
  EIGEN_UNUSED_VARIABLE(remN_);
  EIGEN_UNUSED_VARIABLE(remM_);
  using urolls = unrolls::trans<Scalar>;
 
  constexpr int64_t U3 = urolls::PacketSize * 3;
  constexpr int64_t U2 = urolls::PacketSize * 2;
  constexpr int64_t U1 = urolls::PacketSize * 1;
 
  static_assert(unrollN == U1 || unrollN == U2 || unrollN == U3, "unrollN should be a multiple of PacketSize");
  static_assert(unrollM == EIGEN_AVX_MAX_NUM_ROW, "unrollM should be equal to EIGEN_AVX_MAX_NUM_ROW");
 
  urolls::template transpose<unrollN, 0>(zmm);
  EIGEN_IF_CONSTEXPR(unrollN > U2) urolls::template transpose<unrollN, 2>(zmm);
  EIGEN_IF_CONSTEXPR(unrollN > U1) urolls::template transpose<unrollN, 1>(zmm);
 
  static_assert((remN && unrollN == U1) || !remN, "When handling N remainder set unrollN=U1");
  EIGEN_IF_CONSTEXPR(!remN) {
    urolls::template storeC<std::min(unrollN, U1), unrollN, 0, remM>(C_arr, LDC, zmm, remM_);
    EIGEN_IF_CONSTEXPR(unrollN > U1) {
      constexpr int64_t unrollN_ = std::min(unrollN - U1, U1);
      urolls::template storeC<unrollN_, unrollN, 1, remM>(C_arr + U1 * LDC, LDC, zmm, remM_);
    }
    EIGEN_IF_CONSTEXPR(unrollN > U2) {
      constexpr int64_t unrollN_ = std::min(unrollN - U2, U1);
      urolls::template storeC<unrollN_, unrollN, 2, remM>(C_arr + U2 * LDC, LDC, zmm, remM_);
    }
  }
  else {
    EIGEN_IF_CONSTEXPR((std::is_same<Scalar, float>::value)) {
      // Note: without "if constexpr" this section of code will also be
      // parsed by the compiler so each of the storeC will still be instantiated.
      // We use enable_if in aux_storeC to set it to an empty function for
      // these cases.
      if (remN_ == 15)
        urolls::template storeC<15, unrollN, 0, remM>(C_arr, LDC, zmm, remM_);
      else if (remN_ == 14)
        urolls::template storeC<14, unrollN, 0, remM>(C_arr, LDC, zmm, remM_);
      else if (remN_ == 13)
        urolls::template storeC<13, unrollN, 0, remM>(C_arr, LDC, zmm, remM_);
      else if (remN_ == 12)
        urolls::template storeC<12, unrollN, 0, remM>(C_arr, LDC, zmm, remM_);
      else if (remN_ == 11)
        urolls::template storeC<11, unrollN, 0, remM>(C_arr, LDC, zmm, remM_);
      else if (remN_ == 10)
        urolls::template storeC<10, unrollN, 0, remM>(C_arr, LDC, zmm, remM_);
      else if (remN_ == 9)
        urolls::template storeC<9, unrollN, 0, remM>(C_arr, LDC, zmm, remM_);
      else if (remN_ == 8)
        urolls::template storeC<8, unrollN, 0, remM>(C_arr, LDC, zmm, remM_);
      else if (remN_ == 7)
        urolls::template storeC<7, unrollN, 0, remM>(C_arr, LDC, zmm, remM_);
      else if (remN_ == 6)
        urolls::template storeC<6, unrollN, 0, remM>(C_arr, LDC, zmm, remM_);
      else if (remN_ == 5)
        urolls::template storeC<5, unrollN, 0, remM>(C_arr, LDC, zmm, remM_);
      else if (remN_ == 4)
        urolls::template storeC<4, unrollN, 0, remM>(C_arr, LDC, zmm, remM_);
      else if (remN_ == 3)
        urolls::template storeC<3, unrollN, 0, remM>(C_arr, LDC, zmm, remM_);
      else if (remN_ == 2)
        urolls::template storeC<2, unrollN, 0, remM>(C_arr, LDC, zmm, remM_);
      else if (remN_ == 1)
        urolls::template storeC<1, unrollN, 0, remM>(C_arr, LDC, zmm, remM_);
    }
    else {
      if (remN_ == 7)
        urolls::template storeC<7, unrollN, 0, remM>(C_arr, LDC, zmm, remM_);
      else if (remN_ == 6)
        urolls::template storeC<6, unrollN, 0, remM>(C_arr, LDC, zmm, remM_);
      else if (remN_ == 5)
        urolls::template storeC<5, unrollN, 0, remM>(C_arr, LDC, zmm, remM_);
      else if (remN_ == 4)
        urolls::template storeC<4, unrollN, 0, remM>(C_arr, LDC, zmm, remM_);
      else if (remN_ == 3)
        urolls::template storeC<3, unrollN, 0, remM>(C_arr, LDC, zmm, remM_);
      else if (remN_ == 2)
        urolls::template storeC<2, unrollN, 0, remM>(C_arr, LDC, zmm, remM_);
      else if (remN_ == 1)
        urolls::template storeC<1, unrollN, 0, remM>(C_arr, LDC, zmm, remM_);
    }
  }
}

References EIGEN_AVX_MAX_NUM_ROW, EIGEN_IF_CONSTEXPR, EIGEN_UNUSED_VARIABLE, min, and Eigen::value.

treePostorder()

template<typename IndexVector >

void Eigen::internal::treePostorder	(	typename IndexVector::Scalar	n,
		IndexVector &	parent,
		IndexVector &	post
	)

Post order a tree.

Parameters

n	the number of nodes
parent	Input tree
post	postordered tree

                                                                                         {
  typedef typename IndexVector::Scalar StorageIndex;
  IndexVector first_kid, next_kid;  // Linked list of children
  StorageIndex postnum;
  // Allocate storage for working arrays and results
  first_kid.resize(n + 1);
  next_kid.setZero(n + 1);
  post.setZero(n + 1);
 
  // Set up structure describing children
  first_kid.setConstant(-1);
  for (StorageIndex v = n - 1; v >= 0; v--) {
    StorageIndex dad = parent(v);
    next_kid(v) = first_kid(dad);
    first_kid(dad) = v;
  }
 
  // Depth-first search from dummy root vertex #n
  postnum = 0;
  internal::nr_etdfs(n, parent, first_kid, next_kid, post, postnum);
}

References n, nr_etdfs(), and v.

Referenced by Eigen::SparseLU< MatrixType_, OrderingType_ >::analyzePattern(), and Eigen::internal::SparseLUImpl< Scalar, StorageIndex >::heap_relax_snode().

tridiagonalization_inplace() [1/2]

template<typename MatrixType , typename DiagonalType , typename SubDiagonalType , typename CoeffVectorType , typename WorkSpaceType >

EIGEN_DEVICE_FUNC void Eigen::internal::tridiagonalization_inplace	(	MatrixType &	mat,
		DiagonalType &	diag,
		SubDiagonalType &	subdiag,
		CoeffVectorType &	hcoeffs,
		WorkSpaceType &	workspace,
		bool	extractQ
	)

Performs a full tridiagonalization in place.

Parameters

[in,out]	mat	On input, the selfadjoint matrix whose tridiagonal decomposition is to be computed. Only the lower triangular part referenced. The rest is left unchanged. On output, the orthogonal matrix Q in the decomposition if extractQ is true.
[out]	diag	The diagonal of the tridiagonal matrix T in the decomposition.
[out]	subdiag	The subdiagonal of the tridiagonal matrix T in the decomposition.
[in]	extractQ	If true, the orthogonal matrix Q in the decomposition is computed and stored in mat.

Computes the tridiagonal decomposition of the selfadjoint matrix mat in place such that \( mat = Q T Q^* \) where \( Q \) is unitary and \( T \) a real symmetric tridiagonal matrix.

The tridiagonal matrix T is passed to the output parameters diag and subdiag. If extractQ is true, then the orthogonal matrix Q is passed to mat. Otherwise the lower part of the matrix mat is destroyed.

The vectors diag and subdiag are not resized. The function assumes that they are already of the correct size. The length of the vector diag should equal the number of rows in mat, and the length of the vector subdiag should be one left.

This implementation contains an optimized path for 3-by-3 matrices which is especially useful for plane fitting.

Note

Currently, it requires two temporary vectors to hold the intermediate Householder coefficients, and to reconstruct the matrix Q from the Householder reflectors.

Example (this uses the same matrix as the example in Tridiagonalization::Tridiagonalization(const MatrixType&)):

MatrixXd X = MatrixXd::Random(5, 5);
MatrixXd A = X + X.transpose();
cout << "Here is a random symmetric 5x5 matrix:" << endl << A << endl << endl;
 
VectorXd diag(5);
VectorXd subdiag(4);
VectorXd hcoeffs(4);  // Scratch space for householder reflector.
VectorXd workspace(5);
internal::tridiagonalization_inplace(A, diag, subdiag, hcoeffs, workspace, true);
cout << "The orthogonal matrix Q is:" << endl << A << endl;
cout << "The diagonal of the tridiagonal matrix T is:" << endl << diag << endl;
cout << "The subdiagonal of the tridiagonal matrix T is:" << endl << subdiag << endl;

Output:

See also

class Tridiagonalization

                                                                                                                     {
  eigen_assert(mat.cols() == mat.rows() && diag.size() == mat.rows() && subdiag.size() == mat.rows() - 1);
  tridiagonalization_inplace_selector<MatrixType>::run(mat, diag, subdiag, hcoeffs, workspace, extractQ);
}

References Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::cols(), diag, eigen_assert, Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::rows(), and Eigen::internal::tridiagonalization_inplace_selector< MatrixType, Size, IsComplex >::run().

tridiagonalization_inplace() [2/2]

template<typename MatrixType , typename CoeffVectorType >

EIGEN_DEVICE_FUNC void Eigen::internal::tridiagonalization_inplace	(	MatrixType &	matA,
		CoeffVectorType &	hCoeffs
	)

Performs a tridiagonal decomposition of the selfadjoint matrix matA in-place.

Parameters

[in,out]	matA	On input the selfadjoint matrix. Only the lower triangular part is referenced. On output, the strict upper part is left unchanged, and the lower triangular part represents the T and Q matrices in packed format has detailed below.
[out]	hCoeffs	returned Householder coefficients (see below)

On output, the tridiagonal selfadjoint matrix T is stored in the diagonal and lower sub-diagonal of the matrix matA. The unitary matrix Q is represented in a compact way as a product of Householder reflectors \( H_i \) such that: \( Q = H_{N-1} \ldots H_1 H_0 \). The Householder reflectors are defined as \( H_i = (I - h_i v_i v_i^T) \) where \( h_i = hCoeffs[i]\) is the \( i \)th Householder coefficient and \( v_i \) is the Householder vector defined by \( v_i = [ 0, \ldots, 0, 1, matA(i+2,i), \ldots, matA(N-1,i) ]^T \).

Implemented from Golub's "Matrix Computations", algorithm 8.3.1.

See also

Tridiagonalization::packedMatrix()

                                                                                              {
  using numext::conj;
  typedef typename MatrixType::Scalar Scalar;
  typedef typename MatrixType::RealScalar RealScalar;
  Index n = matA.rows();
  eigen_assert(n == matA.cols());
  eigen_assert(n == hCoeffs.size() + 1 || n == 1);
 
  for (Index i = 0; i < n - 1; ++i) {
    Index remainingSize = n - i - 1;
    RealScalar beta;
    Scalar h;
    matA.col(i).tail(remainingSize).makeHouseholderInPlace(h, beta);
 
    // Apply similarity transformation to remaining columns,
    // i.e., A = H A H' where H = I - h v v' and v = matA.col(i).tail(n-i-1)
    matA.col(i).coeffRef(i + 1) = 1;
 
    hCoeffs.tail(n - i - 1).noalias() =
        (matA.bottomRightCorner(remainingSize, remainingSize).template selfadjointView<Lower>() *
         (conj(h) * matA.col(i).tail(remainingSize)));
 
    hCoeffs.tail(n - i - 1) +=
        (conj(h) * RealScalar(-0.5) * (hCoeffs.tail(remainingSize).dot(matA.col(i).tail(remainingSize)))) *
        matA.col(i).tail(n - i - 1);
 
    matA.bottomRightCorner(remainingSize, remainingSize)
        .template selfadjointView<Lower>()
        .rankUpdate(matA.col(i).tail(remainingSize), hCoeffs.tail(remainingSize), Scalar(-1));
 
    matA.col(i).coeffRef(i + 1) = beta;
    hCoeffs.coeffRef(i) = h;
  }
}

References beta, Eigen::Matrix< Scalar_, Rows_, Cols_, Options_, MaxRows_, MaxCols_ >::coeffRef(), Eigen::PlainObjectBase< Derived >::cols(), conj(), Eigen::conj(), eigen_assert, i, matA(), n, and Eigen::PlainObjectBase< Derived >::rows().

Referenced by Eigen::SelfAdjointEigenSolver< MatrixType_ >::compute(), Eigen::Tridiagonalization< MatrixType_ >::compute(), Eigen::internal::tridiagonalization_inplace_selector< MatrixType, Size, IsComplex >::run(), and Eigen::Tridiagonalization< MatrixType_ >::Tridiagonalization().

trig_reduce_huge()

float Eigen::internal::trig_reduce_huge ( float  xf, Eigen::numext::int32_t *  quadrant  )

inline

                                                                      {
  using Eigen::numext::int32_t;
  using Eigen::numext::int64_t;
  using Eigen::numext::uint32_t;
  using Eigen::numext::uint64_t;
 
  const double pio2_62 = 3.4061215800865545e-19;     // pi/2 * 2^-62
  const uint64_t zero_dot_five = uint64_t(1) << 61;  // 0.5 in 2.62-bit fixed-point format
 
  // 192 bits of 2/pi for Payne-Hanek reduction
  // Bits are introduced by packet of 8 to enable aligned reads.
  static const uint32_t two_over_pi[] = {
      0x00000028, 0x000028be, 0x0028be60, 0x28be60db, 0xbe60db93, 0x60db9391, 0xdb939105, 0x9391054a, 0x91054a7f,
      0x054a7f09, 0x4a7f09d5, 0x7f09d5f4, 0x09d5f47d, 0xd5f47d4d, 0xf47d4d37, 0x7d4d3770, 0x4d377036, 0x377036d8,
      0x7036d8a5, 0x36d8a566, 0xd8a5664f, 0xa5664f10, 0x664f10e4, 0x4f10e410, 0x10e41000, 0xe4100000};
 
  uint32_t xi = numext::bit_cast<uint32_t>(xf);
  // Below, -118 = -126 + 8.
  //   -126 is to get the exponent,
  //   +8 is to enable alignment of 2/pi's bits on 8 bits.
  // This is possible because the fractional part of x as only 24 meaningful bits.
  uint32_t e = (xi >> 23) - 118;
  // Extract the mantissa and shift it to align it wrt the exponent
  xi = ((xi & 0x007fffffu) | 0x00800000u) << (e & 0x7);
 
  uint32_t i = e >> 3;
  uint32_t twoopi_1 = two_over_pi[i - 1];
  uint32_t twoopi_2 = two_over_pi[i + 3];
  uint32_t twoopi_3 = two_over_pi[i + 7];
 
  // Compute x * 2/pi in 2.62-bit fixed-point format.
  uint64_t p;
  p = uint64_t(xi) * twoopi_3;
  p = uint64_t(xi) * twoopi_2 + (p >> 32);
  p = (uint64_t(xi * twoopi_1) << 32) + p;
 
  // Round to nearest: add 0.5 and extract integral part.
  uint64_t q = (p + zero_dot_five) >> 62;
  *quadrant = int(q);
  // Now it remains to compute "r = x - q*pi/2" with high accuracy,
  // since we have p=x/(pi/2) with high accuracy, we can more efficiently compute r as:
  //   r = (p-q)*pi/2,
  // where the product can be be carried out with sufficient accuracy using double precision.
  p -= q << 62;
  return float(double(int64_t(p)) * pio2_62);
}

References e(), i, int(), p, and Eigen::numext::q.

Referenced by psincos_float().

trig_reduce_medium_double()

template<typename Packet >

Packet Eigen::internal::trig_reduce_medium_double	(	const Packet &	x,
		const Packet &	q_high,
		const Packet &	q_low
	)

                                                                                             {
  // Pi/2 split into 4 values
  const Packet cst_pio2_a = pset1<Packet>(-1.570796325802803);
  const Packet cst_pio2_b = pset1<Packet>(-9.920935184482005e-10);
  const Packet cst_pio2_c = pset1<Packet>(-6.123234014771656e-17);
  const Packet cst_pio2_d = pset1<Packet>(1.903488962019325e-25);
 
  Packet t;
  t = pmadd(cst_pio2_a, q_high, x);
  t = pmadd(cst_pio2_a, q_low, t);
  t = pmadd(cst_pio2_b, q_high, t);
  t = pmadd(cst_pio2_b, q_low, t);
  t = pmadd(cst_pio2_c, q_high, t);
  t = pmadd(cst_pio2_c, q_low, t);
  t = pmadd(cst_pio2_d, padd(q_low, q_high), t);
  return t;
}

References padd(), pmadd(), plotPSD::t, and plotDoE::x.

Referenced by psincos_double().

trig_reduce_small_double()

template<typename Packet >

Packet Eigen::internal::trig_reduce_small_double	(	const Packet &	x,
		const Packet &	q
	)

                                                                  {
  // Pi/2 split into 2 values
  const Packet cst_pio2_a = pset1<Packet>(-1.570796325802803);
  const Packet cst_pio2_b = pset1<Packet>(-9.920935184482005e-10);
 
  Packet t;
  t = pmadd(cst_pio2_a, q, x);
  t = pmadd(cst_pio2_b, q, t);
  return t;
}

References pmadd(), Eigen::numext::q, plotPSD::t, and plotDoE::x.

Referenced by psincos_double().

triSolve()

template<typename Scalar , bool isARowMajor = true, bool isBRowMajor = true, bool isFWDSolve = true, bool isUnitDiag = false>

void Eigen::internal::triSolve	(	Scalar *	A_arr,
		Scalar *	B_arr,
		int64_t	M,
		int64_t	numRHS,
		int64_t	LDA,
		int64_t	LDB
	)

Main triangular solve driver

Triangular solve with A on the left. Scalar: Scalar precision, only float/double is supported. isARowMajor: is A row-major? isBRowMajor: is B row-major? isFWDSolve: is this forward solve or backward (true => forward)? isUnitDiag: is diagonal of A unit or nonunit (true => A has unit diagonal)?

M: dimension of A numRHS: number of right hand sides (coincides with K dimension for gemm updates)

Here are the mapping between the different TRSM cases (col-major) and triSolve:

LLN (left , lower, A non-transposed) :: isARowMajor=false, isBRowMajor=false, isFWDSolve=true LUT (left , upper, A transposed) :: isARowMajor=true, isBRowMajor=false, isFWDSolve=true LUN (left , upper, A non-transposed) :: isARowMajor=false, isBRowMajor=false, isFWDSolve=false LLT (left , lower, A transposed) :: isARowMajor=true, isBRowMajor=false, isFWDSolve=false RUN (right, upper, A non-transposed) :: isARowMajor=true, isBRowMajor=true, isFWDSolve=true RLT (right, lower, A transposed) :: isARowMajor=false, isBRowMajor=true, isFWDSolve=true RUT (right, upper, A transposed) :: isARowMajor=false, isBRowMajor=true, isFWDSolve=false RLN (right, lower, A non-transposed) :: isARowMajor=true, isBRowMajor=true, isFWDSolve=false

Note: For RXX cases M,numRHS should be swapped.

The values for kB, numM were determined experimentally. kB: Number of RHS we process at a time. numM: number of rows of B we will store in a temporary array (see below.) This should be a multiple of L.

kB was determined by initially setting kB = numRHS and benchmarking triSolve (TRSM-RUN case) performance with M=numRHS. It was observed that performance started to drop around M=numRHS=240. This is likely machine dependent.

numM was chosen "arbitrarily". It should be relatively small so B_temp is not too large, but it should be large enough to allow GEMM updates to have larger "K"s (see below.) No benchmarking has been done so far to determine optimal values for numM.

If B is col-major, we copy it to a fixed-size temporary array of size at most ~numM*kB and transpose it to row-major. Call the solve routine, and copy+transpose it back to the original array. The updated row-major copy of B is reused in the GEMM updates.

For the GEMM updates, we want "K" (K=i+8 in this case) to be large as soon as possible to reuse the accumulators in GEMM as much as possible. So we only update 8xbK blocks of B as follows:

   A             B
__

|__|__ |__| |__|__|__ |__| |__|__|__|__ |__| |********|__| |**|

Similar idea as mentioned above, but here we are limited by the number of updated values of B that can be stored (row-major) in B_temp.

If there is not enough space to store the next batch of 8xbK of B in B_temp, we call GEMM update and partially update the remaining old values of B which depends on the new values of B stored in B_temp. These values are then no longer needed and can be overwritten.

If there is enough space in B_temp, we only update the next 8xbK values of B.

                                                                                                 {
  constexpr int64_t psize = packet_traits<Scalar>::size;
  constexpr int64_t kB = (3 * psize) * 5;  // 5*U3
  constexpr int64_t numM = 8 * EIGEN_AVX_MAX_NUM_ROW;
 
  int64_t sizeBTemp = 0;
  Scalar *B_temp = NULL;
  EIGEN_IF_CONSTEXPR(!isBRowMajor) {
    sizeBTemp = (((std::min(kB, numRHS) + psize - 1) / psize + 4) * psize) * numM;
  }
 
  EIGEN_IF_CONSTEXPR(!isBRowMajor) B_temp = (Scalar *)handmade_aligned_malloc(sizeof(Scalar) * sizeBTemp, 64);
 
  for (int64_t k = 0; k < numRHS; k += kB) {
    int64_t bK = numRHS - k > kB ? kB : numRHS - k;
    int64_t M_ = (M / EIGEN_AVX_MAX_NUM_ROW) * EIGEN_AVX_MAX_NUM_ROW, gemmOff = 0;
 
    // bK rounded up to next multiple of L=EIGEN_AVX_MAX_NUM_ROW. When B_temp is used, we solve for bkL RHS
    // instead of bK RHS in triSolveKernelLxK.
    int64_t bkL = ((bK + (EIGEN_AVX_MAX_NUM_ROW - 1)) / EIGEN_AVX_MAX_NUM_ROW) * EIGEN_AVX_MAX_NUM_ROW;
    const int64_t numScalarPerCache = 64 / sizeof(Scalar);
    // Leading dimension of B_temp, will be a multiple of the cache line size.
    int64_t LDT = ((bkL + (numScalarPerCache - 1)) / numScalarPerCache) * numScalarPerCache;
    int64_t offsetBTemp = 0;
    for (int64_t i = 0; i < M_; i += EIGEN_AVX_MAX_NUM_ROW) {
      EIGEN_IF_CONSTEXPR(!isBRowMajor) {
        int64_t indA_i = isFWDSolve ? i : M - 1 - i;
        int64_t indB_i = isFWDSolve ? i : M - (i + EIGEN_AVX_MAX_NUM_ROW);
        int64_t offB_1 = isFWDSolve ? offsetBTemp : sizeBTemp - EIGEN_AVX_MAX_NUM_ROW * LDT - offsetBTemp;
        int64_t offB_2 = isFWDSolve ? offsetBTemp : sizeBTemp - LDT - offsetBTemp;
        // Copy values from B to B_temp.
        copyBToRowMajor<Scalar, true, false>(B_arr + indB_i + k * LDB, LDB, bK, B_temp + offB_1, LDT);
        // Triangular solve with a small block of A and long horizontal blocks of B (or B_temp if B col-major)
        triSolveKernelLxK<Scalar, isARowMajor, isFWDSolve, isUnitDiag>(
            &A_arr[idA<isARowMajor>(indA_i, indA_i, LDA)], B_temp + offB_2, EIGEN_AVX_MAX_NUM_ROW, bkL, LDA, LDT);
        // Copy values from B_temp back to B. B_temp will be reused in gemm call below.
        copyBToRowMajor<Scalar, false, false>(B_arr + indB_i + k * LDB, LDB, bK, B_temp + offB_1, LDT);
 
        offsetBTemp += EIGEN_AVX_MAX_NUM_ROW * LDT;
      }
      else {
        int64_t ind = isFWDSolve ? i : M - 1 - i;
        triSolveKernelLxK<Scalar, isARowMajor, isFWDSolve, isUnitDiag>(
            &A_arr[idA<isARowMajor>(ind, ind, LDA)], B_arr + k + ind * LDB, EIGEN_AVX_MAX_NUM_ROW, bK, LDA, LDB);
      }
      if (i + EIGEN_AVX_MAX_NUM_ROW < M_) {
        EIGEN_IF_CONSTEXPR(isBRowMajor) {
          int64_t indA_i = isFWDSolve ? i + EIGEN_AVX_MAX_NUM_ROW : M - (i + 2 * EIGEN_AVX_MAX_NUM_ROW);
          int64_t indA_j = isFWDSolve ? 0 : M - (i + EIGEN_AVX_MAX_NUM_ROW);
          int64_t indB_i = isFWDSolve ? 0 : M - (i + EIGEN_AVX_MAX_NUM_ROW);
          int64_t indB_i2 = isFWDSolve ? i + EIGEN_AVX_MAX_NUM_ROW : M - (i + 2 * EIGEN_AVX_MAX_NUM_ROW);
          gemmKernel<Scalar, isARowMajor, isBRowMajor, false, false>(
              &A_arr[idA<isARowMajor>(indA_i, indA_j, LDA)], B_arr + k + indB_i * LDB, B_arr + k + indB_i2 * LDB,
              EIGEN_AVX_MAX_NUM_ROW, bK, i + EIGEN_AVX_MAX_NUM_ROW, LDA, LDB, LDB);
        }
        else {
          if (offsetBTemp + EIGEN_AVX_MAX_NUM_ROW * LDT > sizeBTemp) {
            int64_t indA_i = isFWDSolve ? i + EIGEN_AVX_MAX_NUM_ROW : 0;
            int64_t indA_j = isFWDSolve ? gemmOff : M - (i + EIGEN_AVX_MAX_NUM_ROW);
            int64_t indB_i = isFWDSolve ? i + EIGEN_AVX_MAX_NUM_ROW : 0;
            int64_t offB_1 = isFWDSolve ? 0 : sizeBTemp - offsetBTemp;
            gemmKernel<Scalar, isARowMajor, isBRowMajor, false, false>(
                &A_arr[idA<isARowMajor>(indA_i, indA_j, LDA)], B_temp + offB_1, B_arr + indB_i + (k)*LDB,
                M - (i + EIGEN_AVX_MAX_NUM_ROW), bK, i + EIGEN_AVX_MAX_NUM_ROW - gemmOff, LDA, LDT, LDB);
            offsetBTemp = 0;
            gemmOff = i + EIGEN_AVX_MAX_NUM_ROW;
          } else {
            int64_t indA_i = isFWDSolve ? i + EIGEN_AVX_MAX_NUM_ROW : M - (i + 2 * EIGEN_AVX_MAX_NUM_ROW);
            int64_t indA_j = isFWDSolve ? gemmOff : M - (i + EIGEN_AVX_MAX_NUM_ROW);
            int64_t indB_i = isFWDSolve ? i + EIGEN_AVX_MAX_NUM_ROW : M - (i + 2 * EIGEN_AVX_MAX_NUM_ROW);
            int64_t offB_1 = isFWDSolve ? 0 : sizeBTemp - offsetBTemp;
            gemmKernel<Scalar, isARowMajor, isBRowMajor, false, false>(
                &A_arr[idA<isARowMajor>(indA_i, indA_j, LDA)], B_temp + offB_1, B_arr + indB_i + (k)*LDB,
                EIGEN_AVX_MAX_NUM_ROW, bK, i + EIGEN_AVX_MAX_NUM_ROW - gemmOff, LDA, LDT, LDB);
          }
        }
      }
    }
    // Handle M remainder..
    int64_t bM = M - M_;
    if (bM > 0) {
      if (M_ > 0) {
        EIGEN_IF_CONSTEXPR(isBRowMajor) {
          int64_t indA_i = isFWDSolve ? M_ : 0;
          int64_t indA_j = isFWDSolve ? 0 : bM;
          int64_t indB_i = isFWDSolve ? 0 : bM;
          int64_t indB_i2 = isFWDSolve ? M_ : 0;
          gemmKernel<Scalar, isARowMajor, isBRowMajor, false, false>(
              &A_arr[idA<isARowMajor>(indA_i, indA_j, LDA)], B_arr + k + indB_i * LDB, B_arr + k + indB_i2 * LDB, bM,
              bK, M_, LDA, LDB, LDB);
        }
        else {
          int64_t indA_i = isFWDSolve ? M_ : 0;
          int64_t indA_j = isFWDSolve ? gemmOff : bM;
          int64_t indB_i = isFWDSolve ? M_ : 0;
          int64_t offB_1 = isFWDSolve ? 0 : sizeBTemp - offsetBTemp;
          gemmKernel<Scalar, isARowMajor, isBRowMajor, false, false>(&A_arr[idA<isARowMajor>(indA_i, indA_j, LDA)],
                                                                     B_temp + offB_1, B_arr + indB_i + (k)*LDB, bM, bK,
                                                                     M_ - gemmOff, LDA, LDT, LDB);
        }
      }
      EIGEN_IF_CONSTEXPR(!isBRowMajor) {
        int64_t indA_i = isFWDSolve ? M_ : M - 1 - M_;
        int64_t indB_i = isFWDSolve ? M_ : 0;
        int64_t offB_1 = isFWDSolve ? 0 : (bM - 1) * bkL;
        copyBToRowMajor<Scalar, true, true>(B_arr + indB_i + k * LDB, LDB, bK, B_temp, bkL, bM);
        triSolveKernelLxK<Scalar, isARowMajor, isFWDSolve, isUnitDiag>(&A_arr[idA<isARowMajor>(indA_i, indA_i, LDA)],
                                                                       B_temp + offB_1, bM, bkL, LDA, bkL);
        copyBToRowMajor<Scalar, false, true>(B_arr + indB_i + k * LDB, LDB, bK, B_temp, bkL, bM);
      }
      else {
        int64_t ind = isFWDSolve ? M_ : M - 1 - M_;
        triSolveKernelLxK<Scalar, isARowMajor, isFWDSolve, isUnitDiag>(&A_arr[idA<isARowMajor>(ind, ind, LDA)],
                                                                       B_arr + k + ind * LDB, bM, bK, LDA, LDB);
      }
    }
  }
 
  EIGEN_IF_CONSTEXPR(!isBRowMajor) handmade_aligned_free(B_temp);
}

References EIGEN_AVX_MAX_NUM_ROW, EIGEN_IF_CONSTEXPR, handmade_aligned_free(), handmade_aligned_malloc(), i, ind, k, oomph::OcTreeNames::LDB, and min.

triSolveKernel()

template<typename Scalar , typename vec , int64_t unrollM, bool isARowMajor, bool isFWDSolve, bool isUnitDiag>

EIGEN_ALWAYS_INLINE void Eigen::internal::triSolveKernel	(	Scalar *	A_arr,
		Scalar *	B_arr,
		int64_t	K,
		int64_t	LDA,
		int64_t	LDB
	)

Triangular solve kernel with A on left with K number of rhs. dim(A) = unrollM

unrollM: dimension of A matrix (triangular matrix). unrollM should be <= EIGEN_AVX_MAX_NUM_ROW isFWDSolve: is forward solve? isUnitDiag: is the diagonal of A all ones? The B matrix (RHS) is assumed to be row-major

                                                                                                           {
  static_assert(unrollM <= EIGEN_AVX_MAX_NUM_ROW, "unrollM should be equal to EIGEN_AVX_MAX_NUM_ROW");
  using urolls = unrolls::trsm<Scalar>;
  constexpr int64_t U3 = urolls::PacketSize * 3;
  constexpr int64_t U2 = urolls::PacketSize * 2;
  constexpr int64_t U1 = urolls::PacketSize * 1;
 
  PacketBlock<vec, EIGEN_AVX_MAX_NUM_ACC> RHSInPacket;
  PacketBlock<vec, EIGEN_AVX_MAX_NUM_ROW> AInPacket;
 
  int64_t k = 0;
  while (K - k >= U3) {
    urolls::template loadRHS<isFWDSolve, unrollM, 3>(B_arr + k, LDB, RHSInPacket);
    urolls::template triSolveMicroKernel<isARowMajor, isFWDSolve, isUnitDiag, unrollM, 3>(A_arr, LDA, RHSInPacket,
                                                                                          AInPacket);
    urolls::template storeRHS<isFWDSolve, unrollM, 3>(B_arr + k, LDB, RHSInPacket);
    k += U3;
  }
  if (K - k >= U2) {
    urolls::template loadRHS<isFWDSolve, unrollM, 2>(B_arr + k, LDB, RHSInPacket);
    urolls::template triSolveMicroKernel<isARowMajor, isFWDSolve, isUnitDiag, unrollM, 2>(A_arr, LDA, RHSInPacket,
                                                                                          AInPacket);
    urolls::template storeRHS<isFWDSolve, unrollM, 2>(B_arr + k, LDB, RHSInPacket);
    k += U2;
  }
  if (K - k >= U1) {
    urolls::template loadRHS<isFWDSolve, unrollM, 1>(B_arr + k, LDB, RHSInPacket);
    urolls::template triSolveMicroKernel<isARowMajor, isFWDSolve, isUnitDiag, unrollM, 1>(A_arr, LDA, RHSInPacket,
                                                                                          AInPacket);
    urolls::template storeRHS<isFWDSolve, unrollM, 1>(B_arr + k, LDB, RHSInPacket);
    k += U1;
  }
  if (K - k > 0) {
    // Handle remaining number of RHS
    urolls::template loadRHS<isFWDSolve, unrollM, 1, true>(B_arr + k, LDB, RHSInPacket, K - k);
    urolls::template triSolveMicroKernel<isARowMajor, isFWDSolve, isUnitDiag, unrollM, 1>(A_arr, LDA, RHSInPacket,
                                                                                          AInPacket);
    urolls::template storeRHS<isFWDSolve, unrollM, 1, true>(B_arr + k, LDB, RHSInPacket, K - k);
  }
}

References EIGEN_AVX_MAX_NUM_ROW, k, PlanarWave::K, and oomph::OcTreeNames::LDB.

triSolveKernelLxK()

template<typename Scalar , bool isARowMajor, bool isFWDSolve, bool isUnitDiag>

void Eigen::internal::triSolveKernelLxK	(	Scalar *	A_arr,
		Scalar *	B_arr,
		int64_t	M,
		int64_t	K,
		int64_t	LDA,
		int64_t	LDB
	)

Triangular solve routine with A on left and dimension of at most L with K number of rhs. This is essentially a wrapper for triSolveMicrokernel for M = {1,2,3,4,5,6,7,8}.

isFWDSolve: is forward solve? isUnitDiag: is the diagonal of A all ones? The B matrix (RHS) is assumed to be row-major

                                                                                                     {
  // Note: this assumes EIGEN_AVX_MAX_NUM_ROW = 8. Unrolls should be adjusted
  // accordingly if EIGEN_AVX_MAX_NUM_ROW is smaller.
  using vec = typename std::conditional<std::is_same<Scalar, float>::value, vecFullFloat, vecFullDouble>::type;
  if (M == 8)
    triSolveKernel<Scalar, vec, 8, isARowMajor, isFWDSolve, isUnitDiag>(A_arr, B_arr, K, LDA, LDB);
  else if (M == 7)
    triSolveKernel<Scalar, vec, 7, isARowMajor, isFWDSolve, isUnitDiag>(A_arr, B_arr, K, LDA, LDB);
  else if (M == 6)
    triSolveKernel<Scalar, vec, 6, isARowMajor, isFWDSolve, isUnitDiag>(A_arr, B_arr, K, LDA, LDB);
  else if (M == 5)
    triSolveKernel<Scalar, vec, 5, isARowMajor, isFWDSolve, isUnitDiag>(A_arr, B_arr, K, LDA, LDB);
  else if (M == 4)
    triSolveKernel<Scalar, vec, 4, isARowMajor, isFWDSolve, isUnitDiag>(A_arr, B_arr, K, LDA, LDB);
  else if (M == 3)
    triSolveKernel<Scalar, vec, 3, isARowMajor, isFWDSolve, isUnitDiag>(A_arr, B_arr, K, LDA, LDB);
  else if (M == 2)
    triSolveKernel<Scalar, vec, 2, isARowMajor, isFWDSolve, isUnitDiag>(A_arr, B_arr, K, LDA, LDB);
  else if (M == 1)
    triSolveKernel<Scalar, vec, 1, isARowMajor, isFWDSolve, isUnitDiag>(A_arr, B_arr, K, LDA, LDB);
  return;
}

References PlanarWave::K, oomph::OcTreeNames::LDB, compute_granudrum_aor::type, and Eigen::value.

twoprod() [1/3]

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::twoprod	(	const Packet &	x,
		const Packet &	y,
		Packet &	p_hi,
		Packet &	p_lo
	)

                                                                                                                 {
  Packet x_hi, x_lo, y_hi, y_lo;
  veltkamp_splitting(x, x_hi, x_lo);
  veltkamp_splitting(y, y_hi, y_lo);
 
  p_hi = pmul(x, y);
  p_lo = pmadd(x_hi, y_hi, pnegate(p_hi));
  p_lo = pmadd(x_hi, y_lo, p_lo);
  p_lo = pmadd(x_lo, y_hi, p_lo);
  p_lo = pmadd(x_lo, y_lo, p_lo);
}

References pmadd(), pmul(), pnegate(), veltkamp_splitting(), plotDoE::x, and y.

Referenced by doubleword_div_fp(), generic_exp2(), generic_pow_impl(), Eigen::internal::accurate_log2< double >::operator()(), Eigen::internal::accurate_log2< float >::operator()(), and twoprod().

twoprod() [2/3]

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::twoprod	(	const Packet &	x_hi,
		const Packet &	x_lo,
		const Packet &	y,
		Packet &	p_hi,
		Packet &	p_lo
	)

                                                                               {
  Packet c_hi, c_lo1;
  twoprod(x_hi, y, c_hi, c_lo1);
  const Packet c_lo2 = pmul(x_lo, y);
  Packet t_hi, t_lo1;
  fast_twosum(c_hi, c_lo2, t_hi, t_lo1);
  const Packet t_lo2 = padd(t_lo1, c_lo1);
  fast_twosum(t_hi, t_lo2, p_hi, p_lo);
}

References fast_twosum(), padd(), pmul(), twoprod(), and y.

twoprod() [3/3]

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::twoprod	(	const Packet &	x_hi,
		const Packet &	x_lo,
		const Packet &	y_hi,
		const Packet &	y_lo,
		Packet &	p_hi,
		Packet &	p_lo
	)

                                                                                                   {
  Packet p_hi_hi, p_hi_lo;
  twoprod(x_hi, x_lo, y_hi, p_hi_hi, p_hi_lo);
  Packet p_lo_hi, p_lo_lo;
  twoprod(x_hi, x_lo, y_lo, p_lo_hi, p_lo_lo);
  fast_twosum(p_hi_hi, p_hi_lo, p_lo_hi, p_lo_lo, p_hi, p_lo);
}

References fast_twosum(), and twoprod().

twoprod_low()

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet Eigen::internal::twoprod_low	(	const Packet &	x,
		const Packet &	y,
		const Packet &	xy
	)

                                                                                                             {
  Packet x_hi, x_lo, y_hi, y_lo;
  veltkamp_splitting(x, x_hi, x_lo);
  veltkamp_splitting(y, y_hi, y_lo);
 
  Packet p_lo = pmadd(x_hi, y_hi, pnegate(xy));
  p_lo = pmadd(x_hi, y_lo, p_lo);
  p_lo = pmadd(x_lo, y_hi, p_lo);
  p_lo = pmadd(x_lo, y_lo, p_lo);
  return p_lo;
}

References pmadd(), pnegate(), veltkamp_splitting(), plotDoE::x, xy, and y.

Referenced by erfc_double_large(), and Eigen::internal::generic_fast_erfc< Scalar >::run().

twosum()

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::twosum	(	const Packet &	x_hi,
		const Packet &	x_lo,
		const Packet &	y_hi,
		const Packet &	y_lo,
		Packet &	s_hi,
		Packet &	s_lo
	)

                                                                                                  {
  const Packet x_greater_mask = pcmp_lt(pabs(y_hi), pabs(x_hi));
  Packet r_hi_1, r_lo_1;
  fast_twosum(x_hi, y_hi, r_hi_1, r_lo_1);
  Packet r_hi_2, r_lo_2;
  fast_twosum(y_hi, x_hi, r_hi_2, r_lo_2);
  const Packet r_hi = pselect(x_greater_mask, r_hi_1, r_hi_2);
 
  const Packet s1 = padd(padd(y_lo, r_lo_1), x_lo);
  const Packet s2 = padd(padd(x_lo, r_lo_2), y_lo);
  const Packet s = pselect(x_greater_mask, s1, s2);
 
  fast_twosum(r_hi, s, s_hi, s_lo);
}

References fast_twosum(), pabs(), padd(), pcmp_lt(), pselect(), and s.

update_value() [1/4]

template<typename T >

EIGEN_DEVICE_FUNC void Eigen::internal::update_value	(	T &	val,
		Index	new_val
	)

                                                           {
  val = internal::convert_index<T>(new_val);
}

References calibrate::val.

Referenced by Eigen::internal::tuple_coeff< Idx, ValueT >::set(), and Eigen::internal::tuple_coeff< 0, ValueT >::set().

update_value() [2/4]

template<typename T >

EIGEN_DEVICE_FUNC void Eigen::internal::update_value	(	T &	val,
		IndexPair< Index >	new_val
	)

                                                                      {
  val = new_val;
}

References calibrate::val.

update_value() [3/4]

template<Index n>

EIGEN_DEVICE_FUNC void Eigen::internal::update_value	(	type2index< n > &	val,
		Index	new_val
	)

                                                                       {
  val.set(new_val);
}

References calibrate::val.

update_value() [4/4]

template<Index f, Index s>

EIGEN_DEVICE_FUNC void Eigen::internal::update_value	(	type2indexpair< f, s > &	val,
		IndexPair< Index >	new_val
	)

                                                                                         {
  val.set(new_val);
}

References calibrate::val.

upperbidiagonalization_blocked_helper()

template<typename MatrixType >

void Eigen::internal::upperbidiagonalization_blocked_helper	(	MatrixType &	A,
		typename MatrixType::RealScalar *	diagonal,
		typename MatrixType::RealScalar *	upper_diagonal,
		Index	bs,
		Ref< Matrix< typename MatrixType::Scalar, Dynamic, Dynamic, traits< MatrixType >::Flags &RowMajorBit > >	X,
		Ref< Matrix< typename MatrixType::Scalar, Dynamic, Dynamic, traits< MatrixType >::Flags &RowMajorBit > >	Y
	)

Helper routine for the block reduction to upper bidiagonal form.

Let's partition the matrix A:

 | A00 A01 |

A = | | | A10 A11 |

This function reduces to bidiagonal form the left rows x blockSize vertical panel [A00/A10] and the blockSize x cols horizontal panel [A00 A01] of the matrix A. The bottom-right block A11 is updated using matrix-matrix products: A22 -= V * Y^T - X * U^T where V and U contains the left and right Householder vectors. U and V are stored in A10, and A01 respectively, and the update matrices X and Y are computed during the reduction.

                                                                                                        {
  typedef typename MatrixType::Scalar Scalar;
  typedef typename MatrixType::RealScalar RealScalar;
  typedef typename NumTraits<RealScalar>::Literal Literal;
  static constexpr int StorageOrder = (traits<MatrixType>::Flags & RowMajorBit) ? RowMajor : ColMajor;
  typedef InnerStride<StorageOrder == ColMajor ? 1 : Dynamic> ColInnerStride;
  typedef InnerStride<StorageOrder == ColMajor ? Dynamic : 1> RowInnerStride;
  typedef Ref<Matrix<Scalar, Dynamic, 1>, 0, ColInnerStride> SubColumnType;
  typedef Ref<Matrix<Scalar, 1, Dynamic>, 0, RowInnerStride> SubRowType;
  typedef Ref<Matrix<Scalar, Dynamic, Dynamic, StorageOrder> > SubMatType;
 
  Index brows = A.rows();
  Index bcols = A.cols();
 
  Scalar tau_u, tau_u_prev(0), tau_v;
 
  for (Index k = 0; k < bs; ++k) {
    Index remainingRows = brows - k;
    Index remainingCols = bcols - k - 1;
 
    SubMatType X_k1(X.block(k, 0, remainingRows, k));
    SubMatType V_k1(A.block(k, 0, remainingRows, k));
 
    // 1 - update the k-th column of A
    SubColumnType v_k = A.col(k).tail(remainingRows);
    v_k -= V_k1 * Y.row(k).head(k).adjoint();
    if (k) v_k -= X_k1 * A.col(k).head(k);
 
    // 2 - construct left Householder transform in-place
    v_k.makeHouseholderInPlace(tau_v, diagonal[k]);
 
    if (k + 1 < bcols) {
      SubMatType Y_k(Y.block(k + 1, 0, remainingCols, k + 1));
      SubMatType U_k1(A.block(0, k + 1, k, remainingCols));
 
      // this eases the application of Householder transforAions
      // A(k,k) will store tau_v later
      A(k, k) = Scalar(1);
 
      // 3 - Compute y_k^T = tau_v * ( A^T*v_k - Y_k-1*V_k-1^T*v_k - U_k-1*X_k-1^T*v_k )
      {
        SubColumnType y_k(Y.col(k).tail(remainingCols));
 
        // let's use the beginning of column k of Y as a temporary vector
        SubColumnType tmp(Y.col(k).head(k));
        y_k.noalias() = A.block(k, k + 1, remainingRows, remainingCols).adjoint() * v_k;  // bottleneck
        tmp.noalias() = V_k1.adjoint() * v_k;
        y_k.noalias() -= Y_k.leftCols(k) * tmp;
        tmp.noalias() = X_k1.adjoint() * v_k;
        y_k.noalias() -= U_k1.adjoint() * tmp;
        y_k *= numext::conj(tau_v);
      }
 
      // 4 - update k-th row of A (it will become u_k)
      SubRowType u_k(A.row(k).tail(remainingCols));
      u_k = u_k.conjugate();
      {
        u_k -= Y_k * A.row(k).head(k + 1).adjoint();
        if (k) u_k -= U_k1.adjoint() * X.row(k).head(k).adjoint();
      }
 
      // 5 - construct right Householder transform in-place
      u_k.makeHouseholderInPlace(tau_u, upper_diagonal[k]);
 
      // this eases the application of Householder transformations
      // A(k,k+1) will store tau_u later
      A(k, k + 1) = Scalar(1);
 
      // 6 - Compute x_k = tau_u * ( A*u_k - X_k-1*U_k-1^T*u_k - V_k*Y_k^T*u_k )
      {
        SubColumnType x_k(X.col(k).tail(remainingRows - 1));
 
        // let's use the beginning of column k of X as a temporary vectors
        // note that tmp0 and tmp1 overlaps
        SubColumnType tmp0(X.col(k).head(k)), tmp1(X.col(k).head(k + 1));
 
        x_k.noalias() = A.block(k + 1, k + 1, remainingRows - 1, remainingCols) * u_k.transpose();  // bottleneck
        tmp0.noalias() = U_k1 * u_k.transpose();
        x_k.noalias() -= X_k1.bottomRows(remainingRows - 1) * tmp0;
        tmp1.noalias() = Y_k.adjoint() * u_k.transpose();
        x_k.noalias() -= A.block(k + 1, 0, remainingRows - 1, k + 1) * tmp1;
        x_k *= numext::conj(tau_u);
        tau_u = numext::conj(tau_u);
        u_k = u_k.conjugate();
      }
 
      if (k > 0) A.coeffRef(k - 1, k) = tau_u_prev;
      tau_u_prev = tau_u;
    } else
      A.coeffRef(k - 1, k) = tau_u_prev;
 
    A.coeffRef(k, k) = tau_v;
  }
 
  if (bs < bcols) A.coeffRef(bs - 1, bs) = tau_u_prev;
 
  // update A22
  if (bcols > bs && brows > bs) {
    SubMatType A11(A.bottomRightCorner(brows - bs, bcols - bs));
    SubMatType A10(A.block(bs, 0, brows - bs, bs));
    SubMatType A01(A.block(0, bs, bs, bcols - bs));
    Scalar tmp = A01(bs - 1, 0);
    A01(bs - 1, 0) = Literal(1);
    A11.noalias() -= A10 * Y.topLeftCorner(bcols, bs).bottomRows(bcols - bs).adjoint();
    A11.noalias() -= X.topLeftCorner(brows, bs).bottomRows(brows - bs) * A01;
    A01(bs - 1, 0) = tmp;
  }
}

References Eigen::Matrix< Scalar_, Rows_, Cols_, Options_, MaxRows_, MaxCols_ >::coeffRef(), Eigen::ColMajor, Eigen::PlainObjectBase< Derived >::cols(), conj(), diagonal(), k, Eigen::RowMajor, Eigen::RowMajorBit, Eigen::PlainObjectBase< Derived >::rows(), tmp, X, and Y.

upperbidiagonalization_inplace_blocked()

template<typename MatrixType , typename BidiagType >

void Eigen::internal::upperbidiagonalization_inplace_blocked	(	MatrixType &	A,
		BidiagType &	bidiagonal,
		Index	maxBlockSize = 32,
		typename MatrixType::Scalar *	= 0
	)

Implementation of a block-bidiagonal reduction. It is based on the following paper: The Design of a Parallel Dense Linear Algebra Software Library: Reduction to Hessenberg, Tridiagonal, and Bidiagonal Form. by Jaeyoung Choi, Jack J. Dongarra, David W. Walker. (1995) section 3.3

                                                                              {
  typedef typename MatrixType::Scalar Scalar;
  typedef Block<MatrixType, Dynamic, Dynamic> BlockType;
 
  Index rows = A.rows();
  Index cols = A.cols();
  Index size = (std::min)(rows, cols);
 
  // X and Y are work space
  static constexpr int StorageOrder = (traits<MatrixType>::Flags & RowMajorBit) ? RowMajor : ColMajor;
  Matrix<Scalar, MatrixType::RowsAtCompileTime, Dynamic, StorageOrder, MatrixType::MaxRowsAtCompileTime> X(
      rows, maxBlockSize);
  Matrix<Scalar, MatrixType::ColsAtCompileTime, Dynamic, StorageOrder, MatrixType::MaxColsAtCompileTime> Y(
      cols, maxBlockSize);
  Index blockSize = (std::min)(maxBlockSize, size);
 
  Index k = 0;
  for (k = 0; k < size; k += blockSize) {
    Index bs = (std::min)(size - k, blockSize);  // actual size of the block
    Index brows = rows - k;                      // rows of the block
    Index bcols = cols - k;                      // columns of the block
 
    // partition the matrix A:
    //
    //      | A00 A01 A02 |
    //      |             |
    // A  = | A10 A11 A12 |
    //      |             |
    //      | A20 A21 A22 |
    //
    // where A11 is a bs x bs diagonal block,
    // and let:
    //      | A11 A12 |
    //  B = |         |
    //      | A21 A22 |
 
    BlockType B = A.block(k, k, brows, bcols);
 
    // This stage performs the bidiagonalization of A11, A21, A12, and updating of A22.
    // Finally, the algorithm continue on the updated A22.
    //
    // However, if B is too small, or A22 empty, then let's use an unblocked strategy
 
    auto upper_diagonal = bidiagonal.template diagonal<1>();
    typename MatrixType::RealScalar* upper_diagonal_ptr =
        upper_diagonal.size() > 0 ? &upper_diagonal.coeffRef(k) : nullptr;
 
    if (k + bs == cols || bcols < 48)  // somewhat arbitrary threshold
    {
      upperbidiagonalization_inplace_unblocked(B, &(bidiagonal.template diagonal<0>().coeffRef(k)), upper_diagonal_ptr,
                                               X.data());
      break;  // We're done
    } else {
      upperbidiagonalization_blocked_helper<BlockType>(B, &(bidiagonal.template diagonal<0>().coeffRef(k)),
                                                       upper_diagonal_ptr, bs, X.topLeftCorner(brows, bs),
                                                       Y.topLeftCorner(bcols, bs));
    }
  }
}

References Eigen::ColMajor, cols, Eigen::PlainObjectBase< Derived >::cols(), k, min, Eigen::RowMajor, Eigen::RowMajorBit, rows, Eigen::PlainObjectBase< Derived >::rows(), size, upperbidiagonalization_inplace_unblocked(), X, and Y.

Referenced by Eigen::internal::UpperBidiagonalization< MatrixType_ >::compute().

upperbidiagonalization_inplace_unblocked()

template<typename MatrixType >

void Eigen::internal::upperbidiagonalization_inplace_unblocked	(	MatrixType &	mat,
		typename MatrixType::RealScalar *	diagonal,
		typename MatrixType::RealScalar *	upper_diagonal,
		typename MatrixType::Scalar *	tempData = 0
	)

                                                                                       {
  typedef typename MatrixType::Scalar Scalar;
 
  Index rows = mat.rows();
  Index cols = mat.cols();
 
  typedef Matrix<Scalar, Dynamic, 1, ColMajor, MatrixType::MaxRowsAtCompileTime, 1> TempType;
  TempType tempVector;
  if (tempData == 0) {
    tempVector.resize(rows);
    tempData = tempVector.data();
  }
 
  for (Index k = 0; /* breaks at k==cols-1 below */; ++k) {
    Index remainingRows = rows - k;
    Index remainingCols = cols - k - 1;
 
    // construct left householder transform in-place in A
    mat.col(k).tail(remainingRows).makeHouseholderInPlace(mat.coeffRef(k, k), diagonal[k]);
    // apply householder transform to remaining part of A on the left
    mat.bottomRightCorner(remainingRows, remainingCols)
        .applyHouseholderOnTheLeft(mat.col(k).tail(remainingRows - 1), mat.coeff(k, k), tempData);
 
    if (k == cols - 1) break;
 
    // construct right householder transform in-place in mat
    mat.row(k).tail(remainingCols).makeHouseholderInPlace(mat.coeffRef(k, k + 1), upper_diagonal[k]);
    // apply householder transform to remaining part of mat on the left
    mat.bottomRightCorner(remainingRows - 1, remainingCols)
        .applyHouseholderOnTheRight(mat.row(k).tail(remainingCols - 1).adjoint(), mat.coeff(k, k + 1), tempData);
  }
}

References Eigen::SparseMatrixBase< Derived >::adjoint(), Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::coeff(), Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::coeffRef(), cols, Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::cols(), diagonal(), k, Eigen::PlainObjectBase< Derived >::resize(), rows, and Eigen::SparseMatrix< Scalar_, Options_, StorageIndex_ >::rows().

Referenced by Eigen::internal::UpperBidiagonalization< MatrixType_ >::computeUnblocked(), and upperbidiagonalization_inplace_blocked().

useSpecificBlockingSizes()

template<typename Index >

bool Eigen::internal::useSpecificBlockingSizes ( Index &  k, Index &  m, Index &  n  )

inline

                                                                   {
#ifdef EIGEN_TEST_SPECIFIC_BLOCKING_SIZES
  if (EIGEN_TEST_SPECIFIC_BLOCKING_SIZES) {
    k = numext::mini<Index>(k, EIGEN_TEST_SPECIFIC_BLOCKING_SIZE_K);
    m = numext::mini<Index>(m, EIGEN_TEST_SPECIFIC_BLOCKING_SIZE_M);
    n = numext::mini<Index>(n, EIGEN_TEST_SPECIFIC_BLOCKING_SIZE_N);
    return true;
  }
#else
  EIGEN_UNUSED_VARIABLE(k)
  EIGEN_UNUSED_VARIABLE(m)
  EIGEN_UNUSED_VARIABLE(n)
#endif
  return false;
}

References EIGEN_TEST_SPECIFIC_BLOCKING_SIZE_K, EIGEN_TEST_SPECIFIC_BLOCKING_SIZE_M, EIGEN_TEST_SPECIFIC_BLOCKING_SIZE_N, EIGEN_TEST_SPECIFIC_BLOCKING_SIZES, EIGEN_UNUSED_VARIABLE, k, m, and n.

Referenced by computeProductBlockingSizes().

vec2d_swizzle2()

EIGEN_STRONG_INLINE Packet2d Eigen::internal::vec2d_swizzle2	(	const Packet2d &	a,
		const Packet2d &	b,
		int	mask
	)

                                                                                            {
  return shuffle(a, b, mask);
}

References a, b, and shuffle().

Referenced by Eigen::internal::compute_inverse_size4< Architecture::Target, double, MatrixType, ResultType >::run().

vec2d_unpackhi()

EIGEN_STRONG_INLINE Packet2d Eigen::internal::vec2d_unpackhi	(	const Packet2d &	a,
		const Packet2d &	b
	)

{ return shuffle(a, b, 3); }

References a, b, and shuffle().

Referenced by Eigen::internal::compute_inverse_size4< Architecture::Target, double, MatrixType, ResultType >::run().

vec2d_unpacklo()

EIGEN_STRONG_INLINE Packet2d Eigen::internal::vec2d_unpacklo	(	const Packet2d &	a,
		const Packet2d &	b
	)

{ return shuffle(a, b, 0); }

References a, b, and shuffle().

Referenced by Eigen::internal::compute_inverse_size4< Architecture::Target, double, MatrixType, ResultType >::run().

vec4f_movehl()

EIGEN_STRONG_INLINE Packet4f Eigen::internal::vec4f_movehl	(	const Packet4f &	a,
		const Packet4f &	b
	)

                                                                                {
  return shuffle2<false>(b, a, eigen_lsx_shuffle_mask(2, 3, 2, 3));
}

References a, b, and eigen_lsx_shuffle_mask().

Referenced by Eigen::internal::compute_inverse_size4< Architecture::Target, float, MatrixType, ResultType >::run().

vec4f_movelh()

EIGEN_STRONG_INLINE Packet4f Eigen::internal::vec4f_movelh	(	const Packet4f &	a,
		const Packet4f &	b
	)

                                                                                {
  return shuffle2<false>(a, b, eigen_lsx_shuffle_mask(0, 1, 0, 1));
}

References a, b, and eigen_lsx_shuffle_mask().

Referenced by Eigen::internal::compute_inverse_size4< Architecture::Target, float, MatrixType, ResultType >::run().

vec4f_swizzle1()

EIGEN_STRONG_INLINE Packet4f Eigen::internal::vec4f_swizzle1	(	const Packet4f &	a,
		int	p,
		int	q,
		int	r,
		int	s
	)

                                                                                           {
  return shuffle1(a, eigen_lsx_shuffle_mask(p, q, r, s));
}

References a, eigen_lsx_shuffle_mask(), p, Eigen::numext::q, UniformPSDSelfTest::r, s, and shuffle1().

Referenced by pcplxflip(), pmul(), pstore1< Packet4f >(), Eigen::internal::quat_product< Architecture::Target, Derived, OtherDerived, float >::run(), and Eigen::internal::cross3_impl< Architecture::Target, VectorLhs, VectorRhs, float, true >::run().

vec4f_swizzle2()

EIGEN_STRONG_INLINE Packet4f Eigen::internal::vec4f_swizzle2	(	const Packet4f &	a,
		const Packet4f &	b,
		int	p,
		int	q,
		int	r,
		int	s
	)

                                                                                                              {
  return shuffle2<false>(a, b, eigen_lsx_shuffle_mask(p, q, r, s));
}

References a, b, eigen_lsx_shuffle_mask(), p, Eigen::numext::q, UniformPSDSelfTest::r, and s.

Referenced by Eigen::internal::compute_inverse_size4< Architecture::Target, float, MatrixType, ResultType >::run().

vec4f_unpackhi()

EIGEN_STRONG_INLINE Packet4f Eigen::internal::vec4f_unpackhi	(	const Packet4f &	a,
		const Packet4f &	b
	)

                                                                                  {
  return shuffle2<true>(a, b, eigen_lsx_shuffle_mask(2, 2, 3, 3));
}

References a, b, eigen_lsx_shuffle_mask(), and shuffle2< true >().

Referenced by Eigen::internal::compute_inverse_size4< Architecture::Target, float, MatrixType, ResultType >::run().

vec4f_unpacklo()

EIGEN_STRONG_INLINE Packet4f Eigen::internal::vec4f_unpacklo	(	const Packet4f &	a,
		const Packet4f &	b
	)

                                                                                  {
  return shuffle2<true>(a, b, eigen_lsx_shuffle_mask(0, 0, 1, 1));
}

References a, b, eigen_lsx_shuffle_mask(), and shuffle2< true >().

Referenced by Eigen::internal::compute_inverse_size4< Architecture::Target, float, MatrixType, ResultType >::run().

vec_splat_packet4f()

template<int element>

EIGEN_STRONG_INLINE Packet4f Eigen::internal::vec_splat_packet4f

(

const Packet4f &

from

)

                                                                      {
  Packet4f splat;
  switch (element) {
    case 0:
      splat.v4f[0] = vec_splat(from.v4f[0], 0);
      splat.v4f[1] = splat.v4f[0];
      break;
    case 1:
      splat.v4f[0] = vec_splat(from.v4f[0], 1);
      splat.v4f[1] = splat.v4f[0];
      break;
    case 2:
      splat.v4f[0] = vec_splat(from.v4f[1], 0);
      splat.v4f[1] = splat.v4f[0];
      break;
    case 3:
      splat.v4f[0] = vec_splat(from.v4f[1], 1);
      splat.v4f[1] = splat.v4f[0];
      break;
  }
  return splat;
}

vecColLoop()

template<Index num_acc, typename LhsMapper , typename RhsMapper , bool zero, bool linear>

EIGEN_ALWAYS_INLINE void Eigen::internal::vecColLoop	(	Index	j,
		LhsMapper &	lhs,
		RhsMapper &	rhs,
		__vector_quad(&)	quad_acc[num_acc]
	)

                                                                                                                 {
  Packet8bf a0[num_acc];
  Packet8bf b1 = pset1<Packet8bf>(Eigen::bfloat16(0));
  Packet8bf b0 = loadColData<RhsMapper, linear>(rhs, j);
 
  if (zero) {
    b0 = vec_mergeh(b0.m_val, b1.m_val);
  }
 
  using LhsSubMapper = typename LhsMapper::SubMapper;
 
  LhsSubMapper lhs2 = lhs.getSubMapper(0, j);
  BFLOAT16_UNROLL
  for (Index k = 0; k < num_acc; k += 2) {
    loadVecLoop<num_acc, LhsSubMapper, zero>(k, lhs2, a0, b1);
  }
 
  multVec<num_acc>(quad_acc, a0, b0);
}

References BFLOAT16_UNROLL, j, k, Eigen::internal::eigen_packet_wrapper< T, unique_id >::m_val, pset1< Packet8bf >(), and zero().

vecLoop()

template<Index num_acc, typename LhsMapper , typename RhsMapper >

EIGEN_ALWAYS_INLINE void Eigen::internal::vecLoop	(	Index	cols,
		const LhsMapper &	lhs,
		RhsMapper &	rhs,
		__vector_quad(&)	quad_acc[num_acc],
		Index	extra_cols
	)

                                                   {
  Index j = 0;
  for (; j + 8 <= cols; j += 8) {
    multVecLoop<num_acc, LhsMapper, RhsMapper, false>(quad_acc, lhs, rhs, j, extra_cols);
  }
 
  if (extra_cols) {
    multVecLoop<num_acc, LhsMapper, RhsMapper, true>(quad_acc, lhs, rhs, j, extra_cols);
  }
}

References cols, and j.

veltkamp_splitting()

template<typename Packet >

EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Eigen::internal::veltkamp_splitting	(	const Packet &	x,
		Packet &	x_hi,
		Packet &	x_lo
	)

                                                                                                           {
  typedef typename unpacket_traits<Packet>::type Scalar;
  EIGEN_CONSTEXPR int shift = (NumTraits<Scalar>::digits() + 1) / 2;
  const Scalar shift_scale = Scalar(uint64_t(1) << shift);  // Scalar constructor not necessarily constexpr.
  const Packet gamma = pmul(pset1<Packet>(shift_scale + Scalar(1)), x);
  Packet rho = psub(x, gamma);
  x_hi = padd(rho, gamma);
  x_lo = psub(x, x_hi);
}

References EIGEN_CONSTEXPR, mathsFunc::gamma(), padd(), pmul(), psub(), and plotDoE::x.

Referenced by twoprod(), and twoprod_low().

zeroAccumulators() [1/2]

template<Index num_acc>

EIGEN_ALWAYS_INLINE void Eigen::internal::zeroAccumulators

(

__vector_quad(&)

quad_acc[num_acc]

)

                                                                              {
  BFLOAT16_UNROLL
  for (Index k = 0; k < num_acc; k++) __builtin_mma_xxsetaccz(&(quad_acc[k]));
}

References BFLOAT16_UNROLL, and k.

zeroAccumulators() [2/2]

template<Index num_acc, Index size = 4>

EIGEN_ALWAYS_INLINE void Eigen::internal::zeroAccumulators

(

Packet4f(&)

acc[num_acc][size]

)

                                                                          {
  Packet4f z = pset1<Packet4f>(float(0));
 
  for (Index k = 0; k < num_acc; k++) {
    for (Index j = 0; j < size; j++) {
      acc[k][j] = z;
    }
  }
}

References j, k, pset1< Packet4f >(), and size.

Variable Documentation

arg_prod

template<typename... Ts>

constexpr EIGEN_DEVICE_FUNCdecltype(reduce<product_op, Ts...>::run((*((Ts*)0))...)) EIGEN_STRONG_INLIN Eigen::internal::arg_prod) (Ts... ts) ( Ts...  ts )

constexpr

              {
  return reduce<product_op, Ts...>::run(ts...);
}

Referenced by test_arg_reductions(), and Eigen::Sizes< Indices >::TotalSize().

arg_sum

template<typename... Ts>

constexpr decltype(reduce<sum_op, Ts...>::run((*((Ts*)0))...)) EIGEN_STRONG_INLIN Eigen::internal::arg_sum) (Ts... ts) ( Ts...  ts )

constexpr

                                                                                                   {
  return reduce<sum_op, Ts...>::run(ts...);
}

Referenced by test_arg_reductions().

defaultL1CacheSize

const std::ptrdiff_t Eigen::internal::defaultL1CacheSize = EIGEN_SET_DEFAULT_L1_CACHE_SIZE(16 * 1024)

Referenced by Eigen::internal::CacheSizes::CacheSizes(), and Eigen::internal::gebp_kernel< LhsScalar, RhsScalar, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs >::operator()().

defaultL2CacheSize

const std::ptrdiff_t Eigen::internal::defaultL2CacheSize = EIGEN_SET_DEFAULT_L2_CACHE_SIZE(512 * 1024)

Referenced by Eigen::internal::CacheSizes::CacheSizes().

defaultL3CacheSize

const std::ptrdiff_t Eigen::internal::defaultL3CacheSize = EIGEN_SET_DEFAULT_L3_CACHE_SIZE(512 * 1024)

Referenced by Eigen::internal::CacheSizes::CacheSizes().

is_int_or_enum_v

template<typename A >

constexpr bool Eigen::internal::is_int_or_enum_v = std::is_enum<A>::value || std::is_integral<A>::value

constexpr

Returns true if its argument is of integer or enum type. FIXME this has the same purpose as is_valid_index_type in XprHelper.h

mask4

const Packet4i Eigen::internal::mask4[4] = {{0, 0, 0, 0}, {-1, 0, 0, 0}, {-1, -1, 0, 0}, {-1, -1, -1, 0}}

static

Referenced by bmask().

matrix_function_separation

const float Eigen::internal::matrix_function_separation = 0.1f

static

Maximum distance allowed between eigenvalues to be considered "close".

Referenced by matrix_function_partition_eigenvalues().

p16c_COUNTDOWN

Packet16c Eigen::internal::p16c_COUNTDOWN = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}

static

Referenced by plset< Packet16c >().

p16uc_COMPLEX32_REV [1/2]

Packet16uc Eigen::internal::p16uc_COMPLEX32_REV

static

Initial value:

=
    vec_sld(p16uc_REVERSE32, p16uc_REVERSE32, 8)

Referenced by pcmp_eq(), pcplxflip< Packet2cf >(), and Eigen::internal::Packet2cf::pmul().

p16uc_COMPLEX32_REV [2/2]

Packet16uc Eigen::internal::p16uc_COMPLEX32_REV

static

Initial value:

=
    vec_sld(p16uc_REVERSE32, p16uc_REVERSE32, 8)

p16uc_COMPLEX32_REV2

Packet16uc Eigen::internal::p16uc_COMPLEX32_REV2

static

Initial value:

=
    vec_sld(p16uc_FORWARD, p16uc_FORWARD, 8)

p16uc_COUNTDOWN

Packet16uc Eigen::internal::p16uc_COUNTDOWN = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}

static

Referenced by plset< Packet16uc >().

p16uc_DUPLICATE16_EVEN

const Packet16uc Eigen::internal::p16uc_DUPLICATE16_EVEN = {0, 1, 0, 1, 4, 5, 4, 5, 8, 9, 8, 9, 12, 13, 12, 13}

static

p16uc_DUPLICATE16_ODD

const Packet16uc Eigen::internal::p16uc_DUPLICATE16_ODD = {2, 3, 2, 3, 6, 7, 6, 7, 10, 11, 10, 11, 14, 15, 14, 15}

static

p16uc_DUPLICATE32_HI

Packet16uc Eigen::internal::p16uc_DUPLICATE32_HI = {0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7, 4, 5, 6, 7}

static

p16uc_ELEMENT_VEC3

Packet16uc Eigen::internal::p16uc_ELEMENT_VEC3

static

Initial value:

= {0x0c, 0x0d, 0x0e, 0x0f, 0x1c, 0x1d, 0x1e, 0x1f,
                                        0x0c, 0x0d, 0x0e, 0x0f, 0x1c, 0x1d, 0x1e, 0x1f}

Referenced by preduxVecResults2().

p16uc_FORWARD [1/2]

Packet16uc Eigen::internal::p16uc_FORWARD = p16uc_REVERSE32

static

p16uc_FORWARD [2/2]

Packet16uc Eigen::internal::p16uc_FORWARD = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}

static

p16uc_GETIMAG32

const Packet16uc Eigen::internal::p16uc_GETIMAG32 = {4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31}

static

Referenced by Eigen::internal::dhs_cpack< Scalar, DataMapper, Packet, PacketC, StorageOrder, Conjugate, PanelMode, UseLhs >::dhs_ccopy(), and Eigen::internal::dhs_cpack< Scalar, DataMapper, Packet, PacketC, StorageOrder, Conjugate, PanelMode, UseLhs >::operator()().

p16uc_GETIMAG32b

const Packet16uc Eigen::internal::p16uc_GETIMAG32b = {4, 5, 6, 7, 20, 21, 22, 23, 12, 13, 14, 15, 28, 29, 30, 31}

static

Referenced by Eigen::internal::dhs_cpack< Scalar, DataMapper, Packet, PacketC, StorageOrder, Conjugate, PanelMode, UseLhs >::dhs_ccopy().

p16uc_GETREAL32

const Packet16uc Eigen::internal::p16uc_GETREAL32 = {0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27}

static

Referenced by Eigen::internal::dhs_cpack< Scalar, DataMapper, Packet, PacketC, StorageOrder, Conjugate, PanelMode, UseLhs >::dhs_ccopy(), and Eigen::internal::dhs_cpack< Scalar, DataMapper, Packet, PacketC, StorageOrder, Conjugate, PanelMode, UseLhs >::operator()().

p16uc_GETREAL32b

const Packet16uc Eigen::internal::p16uc_GETREAL32b = {0, 1, 2, 3, 16, 17, 18, 19, 8, 9, 10, 11, 24, 25, 26, 27}

static

Referenced by Eigen::internal::dhs_cpack< Scalar, DataMapper, Packet, PacketC, StorageOrder, Conjugate, PanelMode, UseLhs >::dhs_ccopy().

p16uc_HALF64_0_16

Packet16uc Eigen::internal::p16uc_HALF64_0_16

static

Initial value:

= vec_sld(vec_splat((Packet16uc)vec_abs(p4i_MINUS16), 0), (Packet16uc)p4i_ZERO,
                                              8)

p16uc_MERGE16_32_1

Packet16uc Eigen::internal::p16uc_MERGE16_32_1 = {0, 1, 16, 17, 2, 3, 18, 19, 0, 1, 16, 17, 2, 3, 18, 19}

static

Referenced by convertArrayPointerBF16toF32DupOne().

p16uc_MERGE16_32_2

Packet16uc Eigen::internal::p16uc_MERGE16_32_2 = {4, 5, 20, 21, 6, 7, 22, 23, 4, 5, 20, 21, 6, 7, 22, 23}

static

Referenced by convertArrayPointerBF16toF32DupOne().

p16uc_MERGE16_32_3

Packet16uc Eigen::internal::p16uc_MERGE16_32_3 = {8, 9, 24, 25, 10, 11, 26, 27, 8, 9, 24, 25, 10, 11, 26, 27}

static

Referenced by convertArrayPointerBF16toF32DupOne().

p16uc_MERGE16_32_4

Packet16uc Eigen::internal::p16uc_MERGE16_32_4 = {12, 13, 28, 29, 14, 15, 30, 31, 12, 13, 28, 29, 14, 15, 30, 31}

static

Referenced by convertArrayPointerBF16toF32DupOne().

p16uc_MERGE16_32_5

Packet16uc Eigen::internal::p16uc_MERGE16_32_5 = {0, 1, 16, 17, 16, 17, 16, 17, 0, 1, 16, 17, 16, 17, 16, 17}

static

Referenced by convertArrayPointerBF16toF32DupOne().

p16uc_MERGE16_32_6

Packet16uc Eigen::internal::p16uc_MERGE16_32_6 = {2, 3, 18, 19, 18, 19, 18, 19, 2, 3, 18, 19, 18, 19, 18, 19}

static

Referenced by convertArrayPointerBF16toF32DupOne().

p16uc_MERGE16_32_7

Packet16uc Eigen::internal::p16uc_MERGE16_32_7 = {4, 5, 20, 21, 20, 21, 20, 21, 4, 5, 20, 21, 20, 21, 20, 21}

static

Referenced by convertArrayPointerBF16toF32DupOne().

p16uc_MERGE16_32_8

Packet16uc Eigen::internal::p16uc_MERGE16_32_8 = {6, 7, 22, 23, 22, 23, 22, 23, 6, 7, 22, 23, 22, 23, 22, 23}

static

Referenced by convertArrayPointerBF16toF32DupOne().

p16uc_MERGEE16

Packet16uc Eigen::internal::p16uc_MERGEE16 = {0, 1, 16, 17, 4, 5, 20, 21, 8, 9, 24, 25, 12, 13, 28, 29}

static

Referenced by Bf16PackLow(), and pmerge().

p16uc_MERGEL16

Packet16uc Eigen::internal::p16uc_MERGEL16 = {2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31}

static

Referenced by Bf16PackHigh().

p16uc_MERGEO16

Packet16uc Eigen::internal::p16uc_MERGEO16 = {2, 3, 18, 19, 6, 7, 22, 23, 10, 11, 26, 27, 14, 15, 30, 31}

static

Referenced by Bf16PackHigh(), and pmerge().

p16uc_PSET32_WEVEN [1/2]

Packet16uc Eigen::internal::p16uc_PSET32_WEVEN

static

Initial value:

=
    vec_sld((Packet16uc)vec_splat((Packet4ui)p16uc_FORWARD, 0), (Packet16uc)vec_splat((Packet4ui)p16uc_FORWARD, 2),
            8)

Referenced by Eigen::internal::Packet2cf::pmul().

p16uc_PSET32_WEVEN [2/2]

Packet16uc Eigen::internal::p16uc_PSET32_WEVEN

static

Initial value:

= vec_sld(p16uc_DUPLICATE32_HI, (Packet16uc)vec_splat((Packet4ui)p16uc_FORWARD, 3),
                                               8)

p16uc_PSET32_WODD [1/2]

Packet16uc Eigen::internal::p16uc_PSET32_WODD

static

Initial value:

=
    vec_sld((Packet16uc)vec_splat((Packet4ui)p16uc_FORWARD, 1), (Packet16uc)vec_splat((Packet4ui)p16uc_FORWARD, 3),
            8)

Referenced by Eigen::internal::Packet2cf::pmul().

p16uc_PSET32_WODD [2/2]

Packet16uc Eigen::internal::p16uc_PSET32_WODD

static

Initial value:

=
    vec_sld((Packet16uc)vec_splat((Packet4ui)p16uc_FORWARD, 0), (Packet16uc)vec_splat((Packet4ui)p16uc_FORWARD, 2),
            8)

p16uc_PSET64_HI [1/2]

Packet16uc Eigen::internal::p16uc_PSET64_HI

static

Initial value:

= (Packet16uc)vec_mergeh(
    (Packet4ui)p16uc_PSET32_WODD, (Packet4ui)p16uc_PSET32_WEVEN)

Referenced by pset1< Packet2cf >().

p16uc_PSET64_HI [2/2]

Packet16uc Eigen::internal::p16uc_PSET64_HI = {0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7}

static

p16uc_PSET64_LO [1/2]

Packet16uc Eigen::internal::p16uc_PSET64_LO

static

Initial value:

= (Packet16uc)vec_mergel(
    (Packet4ui)p16uc_PSET32_WODD, (Packet4ui)p16uc_PSET32_WEVEN)

p16uc_PSET64_LO [2/2]

Packet16uc Eigen::internal::p16uc_PSET64_LO

static

Initial value:

= (Packet16uc)vec_mergel(
    (Packet4ui)p16uc_PSET32_WODD, (Packet4ui)p16uc_PSET32_WEVEN)

p16uc_QUADRUPLICATE16

Packet16uc Eigen::internal::p16uc_QUADRUPLICATE16 = {0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3}

static

Referenced by ploadquad< Packet16c >(), and ploadquad< Packet16uc >().

p16uc_QUADRUPLICATE16_HI

Packet16uc Eigen::internal::p16uc_QUADRUPLICATE16_HI = {0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3}

static

Referenced by ploadquad< Packet8s >(), and ploadquad< Packet8us >().

p16uc_REVERSE16

Packet16uc Eigen::internal::p16uc_REVERSE16 = {14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1}

static

Referenced by preverse().

p16uc_REVERSE32 [1/2]

Packet16uc Eigen::internal::p16uc_REVERSE32 = {12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3}

static

Referenced by preverse().

p16uc_REVERSE32 [2/2]

Packet16uc Eigen::internal::p16uc_REVERSE32 = {12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3}

static

p16uc_REVERSE64

Packet16uc Eigen::internal::p16uc_REVERSE64 = {8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7}

static

p16uc_REVERSE8

Packet16uc Eigen::internal::p16uc_REVERSE8 = {15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0}

static

Referenced by preverse().

p16uc_TRANSPOSE64_HI [1/2]

Packet16uc Eigen::internal::p16uc_TRANSPOSE64_HI

static

Initial value:

=
    p16uc_PSET64_HI + p16uc_HALF64_0_16

Referenced by Eigen::internal::dhs_cpack< Scalar, DataMapper, Packet, PacketC, StorageOrder, Conjugate, PanelMode, UseLhs >::dhs_cblock(), Eigen::internal::dhs_pack< bfloat16, DataMapper, Packet8bf, StorageOrder, PanelMode, true >::operator()(), Eigen::internal::dhs_pack< bfloat16, DataMapper, Packet8bf, StorageOrder, PanelMode, false >::operator()(), pload2(), preduxVecResultsVSX(), and ptranspose().

p16uc_TRANSPOSE64_HI [2/2]

Packet16uc Eigen::internal::p16uc_TRANSPOSE64_HI = {0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23}

static

p16uc_TRANSPOSE64_LO [1/2]

Packet16uc Eigen::internal::p16uc_TRANSPOSE64_LO

static

Initial value:

=
    p16uc_PSET64_LO + p16uc_HALF64_0_16

Referenced by Eigen::internal::dhs_cpack< Scalar, DataMapper, Packet, PacketC, StorageOrder, Conjugate, PanelMode, UseLhs >::dhs_cblock(), Eigen::internal::dhs_pack< bfloat16, DataMapper, Packet8bf, StorageOrder, PanelMode, true >::operator()(), Eigen::internal::dhs_pack< bfloat16, DataMapper, Packet8bf, StorageOrder, PanelMode, false >::operator()(), and ptranspose().

p16uc_TRANSPOSE64_LO [2/2]

Packet16uc Eigen::internal::p16uc_TRANSPOSE64_LO = {8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31}

static

p2d_COUNTDOWN

Packet2d Eigen::internal::p2d_COUNTDOWN

static

Initial value:

= reinterpret_cast<Packet2d>(
    vec_sld(reinterpret_cast<Packet16uc>(p2d_ZERO), reinterpret_cast<Packet16uc>(p2d_ONE), 8))

p2d_ONE

Packet2d Eigen::internal::p2d_ONE = {1.0, 1.0}

static

p2d_ZERO_

Packet2d Eigen::internal::p2d_ZERO_

static

Initial value:

= {numext::bit_cast<double>(0x8000000000000000ull),
                             numext::bit_cast<double>(0x8000000000000000ull)}

p2ul_CONJ_XOR1

Packet2ul Eigen::internal::p2ul_CONJ_XOR1

static

Initial value:

=
    (Packet2ul)vec_sld((Packet4ui)p2d_ZERO_, (Packet4ui)p2l_ZERO, 8)

p2ul_CONJ_XOR2

Packet2ul Eigen::internal::p2ul_CONJ_XOR2

static

Initial value:

=
    (Packet2ul)vec_sld((Packet4ui)p2l_ZERO, (Packet4ui)p2d_ZERO_, 8)

p4f_COUNTDOWN [1/2]

Packet4f Eigen::internal::p4f_COUNTDOWN = {0.0, 1.0, 2.0, 3.0}

static

Referenced by plset< Packet4f >().

p4f_COUNTDOWN [2/2]

Packet4f Eigen::internal::p4f_COUNTDOWN = {0.0, 1.0, 2.0, 3.0}

static

p4f_MZERO [1/2]

Packet4f Eigen::internal::p4f_MZERO

static

Initial value:

=
    (Packet4f)vec_sl((Packet4ui)p4i_MINUS1, (Packet4ui)p4i_MINUS1)

Referenced by pdiv< Packet4f >(), pmul< Packet4f >(), and pnegate().

p4f_MZERO [2/2]

Packet4f Eigen::internal::p4f_MZERO = {0x80000000, 0x80000000, 0x80000000, 0x80000000}

static

p4f_ONE

Packet4f Eigen::internal::p4f_ONE = vec_ctf(p4i_ONE, 0)

static

Referenced by pdiv< Packet4f >().

p4i_COUNTDOWN [1/2]

Packet4i Eigen::internal::p4i_COUNTDOWN = {0, 1, 2, 3}

static

Referenced by plset< Packet4i >().

p4i_COUNTDOWN [2/2]

Packet4i Eigen::internal::p4i_COUNTDOWN = {0, 1, 2, 3}

static

p4ui_CONJ_XOR [1/2]

Packet4ui Eigen::internal::p4ui_CONJ_XOR

static

Initial value:

=
    vec_mergeh((Packet4ui)p4i_ZERO, (Packet4ui)p4f_MZERO)

Referenced by pconj(), and Eigen::internal::Packet2cf::pmul().

p4ui_CONJ_XOR [2/2]

Packet4ui Eigen::internal::p4ui_CONJ_XOR

static

Initial value:

= {0x00000000, 0x80000000, 0x00000000,
                                  0x80000000}

p8s_COUNTDOWN

Packet8s Eigen::internal::p8s_COUNTDOWN = {0, 1, 2, 3, 4, 5, 6, 7}

static

Referenced by plset< Packet8s >().

p8us_COUNTDOWN

Packet8us Eigen::internal::p8us_COUNTDOWN = {0, 1, 2, 3, 4, 5, 6, 7}

static

Referenced by plset< Packet8us >().

y

const Scalar& Eigen::internal::y

Initial value:

{
  return EIGEN_MATHFUNC_IMPL(random, Scalar)::run(x, y)

Referenced by addResults(), apply_rotation_in_the_plane(), Eigen::numext::atan2(), basic_tuple_test(), bicgstab(), Eigen::internal::count_bits_impl< BitsType, EnableIf >::clz(), complex_rsqrt(), complex_sqrt(), conservative_sparse_sparse_product_impl(), Eigen::internal::count_bits_impl< BitsType, EnableIf >::ctz(), doubleword_div_fp(), EIGEN_MATHFUNC_RETVAL(), Eigen::numext::equal_strict(), fast_twosum(), generic_pow(), generic_pow_impl(), gmres(), idrstabl(), Eigen::internal::unary_pow::int_pow(), Eigen::internal::scalar_fuzzy_impl< bool >::isApprox(), Eigen::internal::scalar_fuzzy_default_impl< Scalar, false, true >::isApprox(), Eigen::internal::scalar_fuzzy_default_impl< Scalar, false, false >::isApprox(), Eigen::internal::scalar_fuzzy_default_impl< Scalar, true, false >::isApprox(), isApprox(), Eigen::internal::scalar_fuzzy_impl< bool >::isApproxOrLessThan(), Eigen::internal::scalar_fuzzy_default_impl< Scalar, false, true >::isApproxOrLessThan(), Eigen::internal::scalar_fuzzy_default_impl< Scalar, false, false >::isApproxOrLessThan(), isApproxOrLessThan(), Eigen::internal::scalar_fuzzy_default_impl< Scalar, false, false >::isMuchSmallerThan(), Eigen::internal::scalar_fuzzy_default_impl< Scalar, true, false >::isMuchSmallerThan(), isMuchSmallerThan(), MakeCoherentCwiseBinaryOp(), matrix_log_compute_2x2(), MaybeCoherentPad(), Eigen::numext::not_equal_strict(), Eigen::internal::scalar_hypot_op< Scalar, Scalar >::operator()(), Eigen::internal::scalar_atan2_op< LhsScalar, RhsScalar >::operator()(), Eigen::internal::scalar_atan2_op< LhsScalar, RhsScalar >::packetOp(), Eigen::internal::maybe_coherent_pad_helper< DerivativeType, OtherDerivativeType, EnableIf >::pad(), patan2(), pdiv_complex(), pexp< Packet4f >(), pexp_complex(), pexp_float(), plog< Packet4f >(), plog_impl_double(), Eigen::internal::conj_helper< LhsScalar, RhsScalar, true, true >::pmadd(), Eigen::internal::conj_helper< LhsType, RhsType, ConjLhs, ConjRhs >::pmadd(), Eigen::internal::conj_helper< Packet, Packet, ConjLhs, ConjRhs >::pmadd(), Eigen::internal::conj_helper< Packet, Packet, true, true >::pmadd(), Eigen::internal::conj_helper< LhsScalar, RhsScalar, true, true >::pmul(), Eigen::internal::conj_helper< LhsType, RhsType, ConjLhs, ConjRhs >::pmul(), Eigen::internal::conj_helper< Packet, Packet, ConjLhs, ConjRhs >::pmul(), Eigen::internal::conj_helper< Packet, Packet, true, true >::pmul(), positive_real_hypot(), psincos_float(), psincos_inner_msa_float(), RandomToTypeNormal(), Eigen::internal::random_impl< bfloat16 >::run(), Eigen::internal::random_impl< bool >::run(), Eigen::internal::isApprox_selector< Derived, OtherDerived, true >::run(), Eigen::internal::isApprox_selector< Derived, OtherDerived, is_integer >::run(), Eigen::internal::isMuchSmallerThan_object_selector< Derived, OtherDerived, is_integer >::run(), Eigen::internal::isMuchSmallerThan_scalar_selector< Derived, is_integer >::run(), Eigen::internal::random_impl< half >::run(), Eigen::internal::hypot_impl< Scalar >::run(), Eigen::internal::random_int_impl< Scalar, false, true >::run(), Eigen::internal::random_int_impl< Scalar, true, true >::run(), Eigen::internal::random_default_impl< Scalar, false, false >::run(), Eigen::internal::random_default_impl< Scalar, true, false >::run(), Eigen::internal::pow_impl< ScalarX, ScalarY, IsInteger >::run(), Eigen::internal::generic_i0e< T, float >::run(), Eigen::internal::generic_i0e< T, double >::run(), Eigen::internal::generic_i1e< T, float >::run(), Eigen::internal::generic_i1e< T, double >::run(), Eigen::internal::generic_j0< T, float >::run(), Eigen::internal::generic_j0< T, double >::run(), Eigen::internal::generic_j1< T, float >::run(), Eigen::internal::generic_j1< T, double >::run(), Eigen::numext::equal_strict_impl< X, Y, XIsInteger, XIsSigned, YIsInteger, YIsSigned >::run(), Eigen::numext::equal_strict_impl< X, Y, true, false, true, true >::run(), Eigen::numext::equal_strict_impl< X, Y, true, true, true, false >::run(), Eigen::internal::apply_rotation_in_the_plane_selector< Scalar, OtherScalar, SizeAtCompileTime, MinAlignment, Vectorizable >::run(), Eigen::internal::apply_rotation_in_the_plane_selector< Scalar, OtherScalar, SizeAtCompileTime, MinAlignment, true >::run(), Eigen::internal::igammac_cf_impl< Scalar, mode >::run(), Eigen::internal::digamma_impl< Scalar >::run(), Eigen::internal::pow_impl< ScalarX, ScalarY, true >::run(), sparse_sparse_to_dense_product_impl(), twoprod(), and twoprod_low().