Eigen::EventCount Class Reference

#include <EventCount.h>

Classes
class	Waiter

Public Member Functions
	EventCount (MaxSizeVector< Waiter > &waiters)
	EventCount (const EventCount &)=delete
void	operator= (const EventCount &)=delete
	~EventCount ()
void	Prewait ()
void	CommitWait (Waiter *w)
void	CancelWait ()
void	Notify (bool notifyAll)

Private Member Functions
void	Park (Waiter *w)
void	Unpark (Waiter *w)

Static Private Member Functions
static void	CheckState (uint64_t state, bool waiter=false)

Private Attributes
std::atomic< uint64_t >	state_
MaxSizeVector< Waiter > &	waiters_

Static Private Attributes
static const uint64_t	kWaiterBits = 14
static const uint64_t	kStackMask = (1ull << kWaiterBits) - 1
static const uint64_t	kWaiterShift = kWaiterBits
static const uint64_t	kWaiterMask = ((1ull << kWaiterBits) - 1) << kWaiterShift
static const uint64_t	kWaiterInc = 1ull << kWaiterShift
static const uint64_t	kSignalShift = 2 * kWaiterBits
static const uint64_t	kSignalMask = ((1ull << kWaiterBits) - 1) << kSignalShift
static const uint64_t	kSignalInc = 1ull << kSignalShift
static const uint64_t	kEpochShift = 3 * kWaiterBits
static const uint64_t	kEpochBits = 64 - kEpochShift
static const uint64_t	kEpochMask = ((1ull << kEpochBits) - 1) << kEpochShift
static const uint64_t	kEpochInc = 1ull << kEpochShift

Constructor & Destructor Documentation

EventCount() [1/2]

Eigen::EventCount::EventCount ( MaxSizeVector< Waiter > &  waiters )

inline

                                             : state_(kStackMask), waiters_(waiters) {
    eigen_plain_assert(waiters.size() < (1 << kWaiterBits) - 1);
  }

References eigen_plain_assert, kWaiterBits, and Eigen::MaxSizeVector< T >::size().

EventCount() [2/2]

Eigen::EventCount::EventCount ( const EventCount &  )

delete

~EventCount()

Eigen::EventCount::~EventCount ( )

inline

                {
    // Ensure there are no waiters.
    eigen_plain_assert(state_.load() == kStackMask);
  }

References eigen_plain_assert, kStackMask, and state_.

Member Function Documentation

CancelWait()

void Eigen::EventCount::CancelWait ( )

inline

                    {
    uint64_t state = state_.load(std::memory_order_relaxed);
    for (;;) {
      CheckState(state, true);
      uint64_t newstate = state - kWaiterInc;
      // We don't know if the thread was also notified or not,
      // so we should not consume a signal unconditionally.
      // Only if number of waiters is equal to number of signals,
      // we know that the thread was notified and we must take away the signal.
      if (((state & kWaiterMask) >> kWaiterShift) == ((state & kSignalMask) >> kSignalShift)) newstate -= kSignalInc;
      CheckState(newstate);
      if (state_.compare_exchange_weak(state, newstate, std::memory_order_acq_rel)) return;
    }
  }

References CheckState(), kSignalInc, kSignalMask, kSignalShift, kWaiterInc, kWaiterMask, kWaiterShift, and state_.

Referenced by test_basic_eventcount(), test_stress_eventcount(), and Eigen::ThreadPoolTempl< Environment >::WaitForWork().

CheckState()

static void Eigen::EventCount::CheckState ( uint64_t  state, bool  waiter = false  )

inlinestaticprivate

                                                              {
    static_assert(kEpochBits >= 20, "not enough bits to prevent ABA problem");
    const uint64_t waiters = (state & kWaiterMask) >> kWaiterShift;
    const uint64_t signals = (state & kSignalMask) >> kSignalShift;
    eigen_plain_assert(waiters >= signals);
    eigen_plain_assert(waiters < (1 << kWaiterBits) - 1);
    eigen_plain_assert(!waiter || waiters > 0);
    (void)waiters;
    (void)signals;
  }

References eigen_plain_assert, kEpochBits, kSignalMask, kSignalShift, kWaiterBits, kWaiterMask, and kWaiterShift.

Referenced by CancelWait(), CommitWait(), Notify(), and Prewait().

CommitWait()

void Eigen::EventCount::CommitWait ( Waiter *  w )

inline

                             {
    eigen_plain_assert((w->epoch & ~kEpochMask) == 0);
    w->state = Waiter::kNotSignaled;
    const uint64_t me = (w - &waiters_[0]) | w->epoch;
    uint64_t state = state_.load(std::memory_order_seq_cst);
    for (;;) {
      CheckState(state, true);
      uint64_t newstate;
      if ((state & kSignalMask) != 0) {
        // Consume the signal and return immediately.
        newstate = state - kWaiterInc - kSignalInc;
      } else {
        // Remove this thread from pre-wait counter and add to the waiter stack.
        newstate = ((state & kWaiterMask) - kWaiterInc) | me;
        w->next.store(state & (kStackMask | kEpochMask), std::memory_order_relaxed);
      }
      CheckState(newstate);
      if (state_.compare_exchange_weak(state, newstate, std::memory_order_acq_rel)) {
        if ((state & kSignalMask) == 0) {
          w->epoch += kEpochInc;
          Park(w);
        }
        return;
      }
    }
  }

References CheckState(), eigen_plain_assert, kEpochInc, kEpochMask, Eigen::EventCount::Waiter::kNotSignaled, kSignalInc, kSignalMask, kStackMask, kWaiterInc, kWaiterMask, Park(), state_, w, and waiters_.

Referenced by test_basic_eventcount(), test_stress_eventcount(), and Eigen::ThreadPoolTempl< Environment >::WaitForWork().

Notify()

void Eigen::EventCount::Notify ( bool  notifyAll )

inline

                              {
    std::atomic_thread_fence(std::memory_order_seq_cst);
    uint64_t state = state_.load(std::memory_order_acquire);
    for (;;) {
      CheckState(state);
      const uint64_t waiters = (state & kWaiterMask) >> kWaiterShift;
      const uint64_t signals = (state & kSignalMask) >> kSignalShift;
      // Easy case: no waiters.
      if ((state & kStackMask) == kStackMask && waiters == signals) return;
      uint64_t newstate;
      if (notifyAll) {
        // Empty wait stack and set signal to number of pre-wait threads.
        newstate = (state & kWaiterMask) | (waiters << kSignalShift) | kStackMask;
      } else if (signals < waiters) {
        // There is a thread in pre-wait state, unblock it.
        newstate = state + kSignalInc;
      } else {
        // Pop a waiter from list and unpark it.
        Waiter* w = &waiters_[state & kStackMask];
        uint64_t next = w->next.load(std::memory_order_relaxed);
        newstate = (state & (kWaiterMask | kSignalMask)) | next;
      }
      CheckState(newstate);
      if (state_.compare_exchange_weak(state, newstate, std::memory_order_acq_rel)) {
        if (!notifyAll && (signals < waiters)) return;  // unblocked pre-wait thread
        if ((state & kStackMask) == kStackMask) return;
        Waiter* w = &waiters_[state & kStackMask];
        if (!notifyAll) w->next.store(kStackMask, std::memory_order_relaxed);
        Unpark(w);
        return;
      }
    }
  }

References CheckState(), kSignalInc, kSignalMask, kSignalShift, kStackMask, kWaiterMask, kWaiterShift, state_, Unpark(), w, and waiters_.

Referenced by Eigen::ThreadPoolTempl< Environment >::Cancel(), Eigen::ThreadPoolTempl< Environment >::ScheduleWithHint(), test_basic_eventcount(), test_stress_eventcount(), Eigen::ThreadPoolTempl< Environment >::WaitForWork(), and Eigen::ThreadPoolTempl< Environment >::~ThreadPoolTempl().

operator=()

void Eigen::EventCount::operator= ( const EventCount &  )

delete

Park()

void Eigen::EventCount::Park ( Waiter *  w )

inlineprivate

                       {
    EIGEN_MUTEX_LOCK lock(w->mu);
    while (w->state != Waiter::kSignaled) {
      w->state = Waiter::kWaiting;
      w->cv.wait(lock);
    }
  }

References Eigen::EventCount::Waiter::kSignaled, Eigen::EventCount::Waiter::kWaiting, and w.

Referenced by CommitWait().

Prewait()

void Eigen::EventCount::Prewait ( )

inline

                 {
    uint64_t state = state_.load(std::memory_order_relaxed);
    for (;;) {
      CheckState(state);
      uint64_t newstate = state + kWaiterInc;
      CheckState(newstate);
      if (state_.compare_exchange_weak(state, newstate, std::memory_order_seq_cst)) return;
    }
  }

References CheckState(), kWaiterInc, and state_.

Referenced by test_basic_eventcount(), test_stress_eventcount(), and Eigen::ThreadPoolTempl< Environment >::WaitForWork().

Unpark()

void Eigen::EventCount::Unpark ( Waiter *  w )

inlineprivate

                         {
    for (Waiter* next; w; w = next) {
      uint64_t wnext = w->next.load(std::memory_order_relaxed) & kStackMask;
      next = wnext == kStackMask ? nullptr : &waiters_[internal::convert_index<size_t>(wnext)];
      unsigned state;
      {
        EIGEN_MUTEX_LOCK lock(w->mu);
        state = w->state;
        w->state = Waiter::kSignaled;
      }
      // Avoid notifying if it wasn't waiting.
      if (state == Waiter::kWaiting) w->cv.notify_one();
    }
  }

References Eigen::EventCount::Waiter::kSignaled, kStackMask, Eigen::EventCount::Waiter::kWaiting, w, and waiters_.

Referenced by Notify().

Member Data Documentation

kEpochBits

const uint64_t Eigen::EventCount::kEpochBits = 64 - kEpochShift

staticprivate

Referenced by CheckState().

kEpochInc

const uint64_t Eigen::EventCount::kEpochInc = 1ull << kEpochShift

staticprivate

Referenced by CommitWait().

kEpochMask

const uint64_t Eigen::EventCount::kEpochMask = ((1ull << kEpochBits) - 1) << kEpochShift

staticprivate

Referenced by CommitWait().

kEpochShift

const uint64_t Eigen::EventCount::kEpochShift = 3 * kWaiterBits

staticprivate

kSignalInc

const uint64_t Eigen::EventCount::kSignalInc = 1ull << kSignalShift

staticprivate

Referenced by CancelWait(), CommitWait(), and Notify().

kSignalMask

const uint64_t Eigen::EventCount::kSignalMask = ((1ull << kWaiterBits) - 1) << kSignalShift

staticprivate

Referenced by CancelWait(), CheckState(), CommitWait(), and Notify().

kSignalShift

const uint64_t Eigen::EventCount::kSignalShift = 2 * kWaiterBits

staticprivate

Referenced by CancelWait(), CheckState(), and Notify().

kStackMask

const uint64_t Eigen::EventCount::kStackMask = (1ull << kWaiterBits) - 1

staticprivate

Referenced by CommitWait(), Notify(), Unpark(), and ~EventCount().

kWaiterBits

const uint64_t Eigen::EventCount::kWaiterBits = 14

staticprivate

Referenced by CheckState(), and EventCount().

kWaiterInc

const uint64_t Eigen::EventCount::kWaiterInc = 1ull << kWaiterShift

staticprivate

Referenced by CancelWait(), CommitWait(), and Prewait().

kWaiterMask

const uint64_t Eigen::EventCount::kWaiterMask = ((1ull << kWaiterBits) - 1) << kWaiterShift

staticprivate

Referenced by CancelWait(), CheckState(), CommitWait(), and Notify().

kWaiterShift

const uint64_t Eigen::EventCount::kWaiterShift = kWaiterBits

staticprivate

Referenced by CancelWait(), CheckState(), and Notify().

state_

std::atomic<uint64_t> Eigen::EventCount::state_

private

Referenced by CancelWait(), CommitWait(), Notify(), Prewait(), and ~EventCount().

waiters_

MaxSizeVector<Waiter>& Eigen::EventCount::waiters_

private

Referenced by CommitWait(), Notify(), and Unpark().

---

The documentation for this class was generated from the following file:

• EventCount.h