Pipeline.hpp

Go to the documentation of this file.

// <Pipeline> -*- C++ -*-
 
 
#pragma once
 
#include <cinttypes>
#include <memory>
#include <array>
#include <list>
#include <type_traits>
#include <vector>
 
#include "sparta/simulation/Clock.hpp"
#include "sparta/utils/SpartaAssert.hpp"
#include "sparta/utils/MathUtils.hpp"
#include "sparta/collection/IterableCollector.hpp"
 
#include "sparta/resources/Pipe.hpp"
#include "sparta/events/Scheduleable.hpp"
#include "sparta/events/PhasedUniqueEvent.hpp"
#include "sparta/events/PhasedPayloadEvent.hpp"
#include "sparta/kernel/SpartaHandler.hpp"
#include "sparta/utils/IteratorTraits.hpp"
 
namespace sparta
{
 
    template <typename DataT, typename EventT=PhasedUniqueEvent>
    class Pipeline
    {
    public:
        using size_type = uint32_t;
        using value_type = DataT;
        using EventHandle = std::unique_ptr<EventT>;
        using EventHandleList = std::list<EventHandle>;
        using EventList = std::list<EventT*>;
        using EventMatrix = std::array<EventList, NUM_SCHEDULING_PHASES>;
 
        enum class Precedence {
            NONE,
            FORWARD,
            BACKWARD,
            NUM_OF_PRECEDENCE
        };
 
        template<typename DataT2, typename EventT2>
        friend class Pipeline;
 
        template<bool is_const_iterator = true>
        class PipelineIterator : public utils::IteratorTraits<std::forward_iterator_tag, DataT>
        {
            using DataReferenceType =
                typename std::conditional<is_const_iterator, const DataT &, DataT &>::type;
 
            using DataPointerType =
                typename std::conditional<is_const_iterator, const DataT *, DataT *>::type;
 
            using PipelinePointerType =
                typename std::conditional<is_const_iterator,
                                          const Pipeline<DataT, EventT> *, Pipeline<DataT, EventT> *>::type;
 
        public:
            friend class PipelineIterator<true>;
            friend class Pipeline;
 
            PipelineIterator(PipelinePointerType ptr) : pipelinePtr_(ptr) {}
 
            PipelineIterator(PipelinePointerType ptr, uint32_t i) : pipelinePtr_(ptr), index_(i) {}
 
            PipelineIterator(const PipelineIterator<false> & rhs) :
                pipelinePtr_(rhs.pipelinePtr_),
                index_(rhs.index_)
            {
            }
 
            PipelineIterator(const PipelineIterator<true> & rhs) :
                pipelinePtr_(rhs.pipelinePtr_),
                index_(rhs.index_)
            {
            }
 
 
            PipelineIterator& operator=(const PipelineIterator & rhs) = default;
 
            DataReferenceType operator*()
            {
                sparta_assert(isValid(), "Iterator is not valid for dereferencing!");
                return pipelinePtr_->operator[](index_);
            }
 
            DataPointerType operator->()
            {
                sparta_assert(isValid(), "Iterator is not valid for dereferencing!");
                return &(this->operator*());
            }
 
            PipelineIterator operator++()
            {
                index_++;
                if (index_ > pipelinePtr_->capacity()) {
                    index_ = pipelinePtr_->capacity();
                }
                return *this;
            }
 
            PipelineIterator operator++(int)
            {
                PipelineIterator temp(*this);
                this->operator++();
                return temp;
            }
 
            bool operator==(const PipelineIterator & rhs) const
            {
                return ((pipelinePtr_ == rhs.pipelinePtr_) && (index_ == rhs.index_));
            }
 
            bool operator!=(const PipelineIterator & rhs) const { return !this->operator==(rhs); }
 
            bool isValid() const { return pipelinePtr_->isValid(index_); }
 
        private:
            PipelinePointerType pipelinePtr_;
            uint32_t index_ = 0;
        };
 
        using iterator = PipelineIterator<false>;
        using const_iterator = PipelineIterator<true>;
 
        iterator begin() { return iterator(this); }
        const_iterator begin() const { return const_iterator(this); }
        const_iterator cbegin() const { return const_iterator(this); }
 
        iterator end() { return iterator(this, capacity()); }
        const_iterator end() const { return const_iterator(this, capacity()); }
        const_iterator cend() const { return const_iterator(this, capacity()); }
 
        Pipeline(EventSet * es,
                 const std::string & name,
                 const uint32_t num_stages,
                 const Clock * clk) :
            name_(name),
            clock_(clk),
            pipe_(name, num_stages, clk),
            event_list_at_stage_(num_stages),
            event_matrix_at_stage_(num_stages),
            es_((es == nullptr) ? &dummy_es_ : es),
            ev_pipeline_update_
                (es_, name + "_update_event", CREATE_SPARTA_HANDLER(Pipeline, internalUpdate_), 1),
            num_stages_(num_stages)
        {
            sparta_assert(clk != nullptr, "Pipeline requires a clock");
            dummy_es_.setClock(clk);
 
            // Only support the following Event types:
            // 1. PhasedUniqueEvent
            // 2. PhasedPayloadEvent<DataT>
            static_assert(std::is_same_v<EventT, PhasedUniqueEvent> || std::is_same_v<EventT, PhasedPayloadEvent<DataT>>,
                          "Error: Pipeline is templated on a unsupported Event type. Supported Event types: "
                          "UniqueEvent, PayloaodEvent (where DataT == DataT of the Pipeline).");
            events_valid_at_stage_.resize(num_stages, false);
            advance_into_stage_.resize(num_stages, true);
 
            ev_pipeline_update_.setScheduleableClock(clk);
            ev_pipeline_update_.setScheduler(clk->getScheduler());
            ev_pipeline_update_.setContinuing(false);
        }
 
        Pipeline(const std::string & name,
                 const uint32_t num_stages,
                 const Clock * clk) :
            Pipeline(nullptr, name, num_stages, clk)
        {}
 
        template<SchedulingPhase sched_phase = SchedulingPhase::Tick>
        void registerHandlerAtStage(const uint32_t & id, const SpartaHandler & handler)
        {
            sparta_assert(static_cast<uint32_t>(default_precedence_) == static_cast<uint32_t>(Precedence::NONE),
                          "You have specified a default precedence (" << static_cast<uint32_t>(default_precedence_)
                          << ") between stages. No new handlers can be registered any more!");
            sparta_assert(id < event_list_at_stage_.size(),
                          "Attempt to register handler for invalid pipeline stage[" << id << "]!");
 
            // Create a new stage event handler, and add it to its event list
            auto & event_list = event_list_at_stage_[id];
            if constexpr (std::is_same_v<EventT, PhasedPayloadEvent<DataT>>) {
                sparta_assert(handler.argCount() == 1, "Expecting Sparta Handler with 1 data parameter!");
                event_list.emplace_back(new EventT(es_,
                                            "pev_" + name_ + "_stage_" + std::to_string(id) + "_" + std::to_string(event_list_at_stage_[id].size()),
                                            sched_phase,
                                            handler));
            } else {
                sparta_assert(handler.argCount() == 0, "Expecting Sparta Handler with no data parameter!");
                event_list.emplace_back(new EventT(es_,
                                            "uev_" + name_ + "_stage_" + std::to_string(id) + "_" + std::to_string(event_list_at_stage_[id].size()),
                                            sched_phase,
                                            handler));
            }
            auto new_event = event_list.back().get();
 
            // Set clock for this new event handler
            if constexpr (std::is_same_v<EventT, PhasedPayloadEvent<DataT>>) {
                new_event->getScheduleable().setScheduleableClock(clock_);
                new_event->getScheduleable().setScheduler(clock_->getScheduler());
            } else {
                new_event->setScheduleableClock(clock_);
                new_event->setScheduler(clock_->getScheduler());
            }
 
            // Update event matrix
            auto & event_list_per_phase = event_matrix_at_stage_[id][static_cast<uint32_t>(sched_phase)];
            if (event_list_per_phase.size() > 0) {
                auto & producer_event = event_list_per_phase.back();
 
                // Set the latest registered event (in this 'sched_phase') to precedes this new event
                if constexpr (std::is_same_v<EventT, PhasedPayloadEvent<DataT>>) {
                    producer_event->getScheduleable().precedes(new_event->getScheduleable());
                } else {
                    producer_event->precedes(*new_event);
                }
            } else {
                events_valid_at_stage_[id] = true;
            }
 
            // Add the raw pointer of newly added event to the stage event list on 'sched_phase'
            event_list_per_phase.push_back( new_event );
        }
 
        void setPrecedenceBetweenStage(const uint32_t & pid, const uint32_t & cid)
        {
            sparta_assert(static_cast<uint32_t>(default_precedence_) == static_cast<uint32_t>(Precedence::NONE),
                          "You have specified a default precedence (" << static_cast<uint32_t>(default_precedence_)
                          << "). No more precedence between stages can be set!");
            sparta_assert(pid != cid, "Cannot specify precedence with yourself!");
            sparta_assert((pid < event_list_at_stage_.size()) && (event_list_at_stage_[pid].size() > 0),
                          "Precedence setup fails: No handler for pipeline stage[" << pid << "]!");
            sparta_assert((cid < event_list_at_stage_.size()) && (event_list_at_stage_[cid].size() > 0),
                          "Precedence setup fails: No handler for pipeline stage[" << cid << "]!");
 
            for (uint32_t phase_id = 0; phase_id < NUM_SCHEDULING_PHASES; phase_id++) {
                auto & pstage_event_list = event_matrix_at_stage_[pid][phase_id];
                auto & cstage_event_list = event_matrix_at_stage_[cid][phase_id];
 
                if (pstage_event_list.empty() || cstage_event_list.empty()) {
                    continue;
                }
 
                // The last event in producer pipeline stage(#pid) is
                // scheduled earlier than the first event in consumer
                // pipeline stage(#cid)
                if constexpr (std::is_same_v<EventT, PhasedPayloadEvent<DataT>>) {
                    (pstage_event_list.back())->getScheduleable().precedes((cstage_event_list.front())->getScheduleable());
                } else {
                    (pstage_event_list.back())->precedes(*(cstage_event_list.front()));
                }
            }
        }
 
        template<class DataT2, class EventT2>
        void setPrecedenceBetweenPipeline(const uint32_t & pid, Pipeline<DataT2, EventT2> & c_pipeline, const uint32_t & cid)
        {
            sparta_assert(static_cast<void*>(&c_pipeline) != static_cast<void*>(this),
                          "Cannot use this function to set precedence between stages within the same pipeline instance!");
            sparta_assert((pid < event_list_at_stage_.size()) && (event_list_at_stage_[pid].size() > 0),
                          "Precedence setup fails: No handler for pipeline stage[" << pid << "]!");
            sparta_assert((cid < c_pipeline.event_list_at_stage_.size()) && (c_pipeline.event_list_at_stage_[cid].size() > 0),
                          "Precedence setup fails: No handler for pipeline stage[" << cid << "]!");
 
            for (uint32_t phase_id = 0; phase_id < NUM_SCHEDULING_PHASES; phase_id++) {
                auto & pstage_event_list = event_matrix_at_stage_[pid][phase_id];
                auto & cstage_event_list = c_pipeline.event_matrix_at_stage_[cid][phase_id];
 
                if (pstage_event_list.empty() || cstage_event_list.empty()) {
                    continue;
                }
 
                // The last event in producer pipeline stage(#pid) is
                // scheduled earlier than the first event in consumer
                // pipeline stage(#cid)
                if constexpr (std::is_same_v<EventT, PhasedPayloadEvent<DataT>>) {
                    (pstage_event_list.back())->getScheduleable().precedes((cstage_event_list.front())->getScheduleable());
                } else {
                    (pstage_event_list.back())->precedes(*(cstage_event_list.front()));
                }
            }
        }
 
        void setDefaultStagePrecedence(const Precedence & default_precedence)
        {
            sparta_assert(static_cast<uint32_t>(default_precedence) < static_cast<uint32_t>(Precedence::NUM_OF_PRECEDENCE),
                          "Unknown default precedence is specified for sparta::Pipeline!");
 
            if (static_cast<uint32_t>(default_precedence) == static_cast<uint32_t>(Precedence::NONE)) {
                return;
            }
 
            bool forward = (static_cast<uint32_t>(default_precedence) == static_cast<uint32_t>(Precedence::FORWARD));
 
            uint32_t pid = 0;
            uint32_t cid = pid + 1;
            while (cid < event_list_at_stage_.size()) {
                if (event_list_at_stage_[pid].empty()) {
                    ++pid;
                } else if (event_list_at_stage_[cid].empty()) {
                    ++cid;
                } else {
                    if (forward) {
                        // This function can only be called when 'default_precedence_' is Precedence::NONE
                        setPrecedenceBetweenStage(pid, cid);
                    } else {
                        setPrecedenceBetweenStage(cid, pid);
                    }
 
                    // Skip the empty stages between producer and consumer to avoid duplicated checking
                    pid = cid;
                }
 
                if (pid == cid) {
                    ++cid;
                }
            }
 
            default_precedence_ = default_precedence;
        }
 
        template<typename EventType>
        void setProducerForPipelineUpdate(EventType & ev_handler)
        {
            auto phase = ev_handler.getScheduleable().getSchedulingPhase();
            sparta_assert(phase == SchedulingPhase::Update,
                          "Cannot set producer event for pipeline update event, it's not on the Update phase!");
            ev_handler.getScheduleable().precedes(ev_pipeline_update_.getScheduleable());
        }
 
        template<typename EventType>
        void setConsumerForPipelineUpdate(EventType & ev_handler)
        {
            auto phase = ev_handler.getScheduleable().getSchedulingPhase();
            sparta_assert(phase == SchedulingPhase::Update,
                          "Cannot set consumer event for pipeline update event, it's not on the Update phase!");
            ev_pipeline_update_.getScheduleable().precedes(ev_handler.getScheduleable());
        }
 
        template<typename EventType>
        void setProducerForStage(const uint32_t & id, EventType & ev_handler)
        {
            sparta_assert((id < event_list_at_stage_.size()) && (event_list_at_stage_[id].size() > 0),
                          "Precedence setup fails: No handler for pipeline stage[" << id << "]!");
 
            auto phase_id = static_cast<uint32_t>(ev_handler.getScheduleable().getSchedulingPhase());
            auto & event_list = event_matrix_at_stage_[id][phase_id];
 
            sparta_assert(!event_list.empty(),
                          "Cannot set producer event for pipeline stage[" << id << "]. No registered stage event on the SAME phase!");
            if constexpr (std::is_same_v<EventT, PhasedPayloadEvent<DataT>>) {
                ev_handler.getScheduleable().precedes((event_list.front())->getScheduleable());
            } else {
                ev_handler.getScheduleable().precedes(*(event_list.front()));
            }
        }
 
        template<typename EventType>
        void setConsumerForStage(const uint32_t & id, EventType & ev_handler)
        {
            sparta_assert((id < event_list_at_stage_.size()) && (event_list_at_stage_[id].size() > 0),
                          "Precedence setup fails: No handler for pipeline stage[" << id << "]!");
 
            auto phase_id = static_cast<uint32_t>(ev_handler.getScheduleable().getSchedulingPhase());
            auto & event_list = event_matrix_at_stage_[id][phase_id];
 
            sparta_assert(!event_list.empty(),
                          "Cannot set consumer event for pipeline stage[" << id
                          << "]. No registered stage event on the SAME phase!");
            if constexpr (std::is_same_v<EventT, PhasedPayloadEvent<DataT>>) {
                (event_list.back())->getScheduleable().precedes(ev_handler.getScheduleable());
            } else {
                (event_list.back())->precedes(ev_handler.getScheduleable());
            }
        }
 
        EventList & getEventsAtStage(const uint32_t & id,
                                     const SchedulingPhase phase = SchedulingPhase::Tick)
        {
            sparta_assert(id < event_matrix_at_stage_.size(),
                          "Attempt to get events at an invalid pipeline stage["
                          << id << "]!");
            using SchedUType = std::underlying_type<SchedulingPhase>::type;
            auto & event_list = event_matrix_at_stage_[id][static_cast<SchedUType>(phase)];
            sparta_assert(!event_list.empty(),
                          "No registered events at stage[" << id << "]!");
            return event_list;
        }
 
        bool isEventRegisteredAtStage(const uint32_t & id) const {
            sparta_assert(id < event_list_at_stage_.size(),
                          "Attempt to check event handler for invalid pipeline stage[" << id << "]!");
 
            return (event_list_at_stage_[id].size() > 0);
        }
 
        void activateEventAtStage(const uint32_t & id)
        {
            sparta_assert(id < event_list_at_stage_.size(),
                          "Attempt to activate event handler for invalid pipeline stage[" << id << "]!");
            sparta_assert((event_list_at_stage_[id].size() > 0),
                          "Activation fails: No registered event handler for stage[" << id << "]!");
 
            events_valid_at_stage_[id] = true;
        }
 
        void deactivateEventAtStage(const uint32_t & id)
        {
            sparta_assert(id < event_list_at_stage_.size(),
                          "Attempt to deactivate event handler for invalid pipeline stage[" << id << "]!");
            sparta_assert((event_list_at_stage_[id].size() > 0),
                          "Deactivation fails: No registered event handler for stage[" << id << "]!");
 
            events_valid_at_stage_[id] = false;
        }
 
        void append(const DataT & item)
        {
            appendImpl_(item);
        }
 
        void append(DataT && item)
        {
            return appendImpl_(std::move(item));
        }
 
        bool isAppended() const
        {
            return pipe_.isAppended();
        }
 
        const DataT& readAppendedData() const
        {
            return pipe_.readAppendedData();
        }
 
        void writeStage(const uint32_t & stage_id, const DataT & item)
        {
            writeStageImpl_(stage_id, item);
        }
 
        void writeStage(const uint32_t & stage_id, DataT && item)
        {
            writeStageImpl_(stage_id, std::move(item));
        }
 
        void invalidateStage(const uint32_t & stage_id)
        {
            pipe_.invalidatePS(stage_id);
 
            if (perform_own_update_) {
                ev_pipeline_update_.schedule();
            }
        }
 
        void stall(const uint32_t & stall_stage_id,
                   const uint32_t & stall_cycles,
                   const bool crush_bubbles=false,
                   const bool suppress_events=true)
        {
            sparta_assert(pipe_.isValid(stall_stage_id), "Try to stall an empty pipeline stage!");
            sparta_assert(!isStalledOrStalling(), "Try to stall a pipeline that is stalling or already stalled!");
 
            if (stall_cycles == 0) {
                return;
            }
 
            stall_cycles_ = stall_cycles;
            stall_stage_id_ = stall_stage_id;
            deactivate_(stall_stage_id_, crush_bubbles, suppress_events);
        }
 
        bool isStalledOrStalling() const {
            return (stall_cycles_ > 0) || (stall_stage_id_ < std::numeric_limits<uint32_t>::max());
        }
 
        bool isStalledOrStallingAtStage(const uint32_t & stage_id) const {
            return (isStalledOrStalling() && stage_id <= stall_stage_id_);
        }
 
        void flushStage(const uint32_t & flush_stage_id) {
            cancelEventsAtStage_(flush_stage_id);
 
            if (flush_stage_id == stall_stage_id_) {
                sparta_assert(isStalledOrStalling());
                restart_(stall_stage_id_);
 
                stall_cycles_ = 0;
                stall_stage_id_ = std::numeric_limits<uint32_t>::max();
            }
 
            pipe_.flushPS(flush_stage_id);
        }
 
        void flushStage(const const_iterator & const_iter) {
            flushStage(const_iter.index_);
        }
 
        void flushStage(const iterator & iter) {
            flushStage(iter.index_);
        }
 
        void flushAllStages() {
            for (uint32_t stage_id = 0; stage_id < num_stages_; stage_id++) {
                flushStage(stage_id);
            }
        }
 
        void flushAppend() {
            pipe_.flushAppend();
        }
 
        void setContinuing(const bool value) {
            ev_pipeline_update_.setContinuing(value);
        }
 
        const DataT & operator[](const uint32_t & stage_id) const { return pipe_.read(stage_id); }
 
        DataT & operator[](const uint32_t & stage_id) { return pipe_.access(stage_id); }
 
        const DataT & at(const uint32_t & stage_id) const { return pipe_.read(stage_id); }
 
        DataT & at(const uint32_t & stage_id) { return pipe_.access(stage_id); }
 
        bool isValid(const uint32_t & stage_id) const { return pipe_.isValid(stage_id); }
 
        bool isLastValid() const { return pipe_.isLastValid(); }
 
        bool isAnyValid() const { return pipe_.isAnyValid(); }
 
        uint32_t numValid() const { return pipe_.numValid(); }
 
        bool empty() const { return (numValid() == 0); }
 
        uint32_t size() const { return pipe_.size(); }
 
        uint32_t capacity() const { return num_stages_; }
 
        void performOwnUpdates()
        {
            if (!perform_own_update_) {
                perform_own_update_ = true;
 
                if (pipe_.isAnyValid()) {
                    ev_pipeline_update_.schedule();
                }
            }
        }
 
        void update()
        {
            sparta_assert(perform_own_update_ == false, "You asked me to perform my own update!");
 
            internalUpdate_();
        }
 
        template<SchedulingPhase phase = SchedulingPhase::Collection>
        void enableCollection(TreeNode * parent) { pipe_.template enableCollection<phase>(parent); }
 
        bool isCollected() const { return pipe_.isCollected(); }
 
    private:
        template<typename U>
        void appendImpl_(U && item)
        {
            pipe_.append(std::forward<U>(item));
 
            if (perform_own_update_) {
                ev_pipeline_update_.schedule();
            }
        }
 
        template<typename U>
        void writeStageImpl_(const uint32_t & stage_id, U && item)
        {
            pipe_.writePS(stage_id, std::forward<U>(item));
 
            if (perform_own_update_) {
                ev_pipeline_update_.schedule();
            }
        }
 
        void internalUpdate_()
        {
            if (!isStalledOrStalling()) {
                pipe_.update();
                scheduleEventForEachStage_();
            } else {
                drain_(stall_stage_id_ + 1);
                scheduleEventForEachStage_();
 
                --stall_cycles_;
                if (stall_cycles_ == 0) {
                    restart_(stall_stage_id_);
                    stall_stage_id_ = std::numeric_limits<uint32_t>::max();
                }
            }
 
            if (pipe_.isAnyValid() && perform_own_update_) {
                ev_pipeline_update_.schedule();
            }
        }
 
        void drain_(const uint32_t start_id)
        {
            uint32_t stage_id = num_stages_ - 1;
 
            // remove last item from pipe
            if (start_id <= stage_id && pipe_.isValid(stage_id)) {
                pipe_.invalidatePS(stage_id);
            }
 
            // advance stages after stall point
            while (stage_id > start_id) {
                if (pipe_.isValid(stage_id - 1)) {
                    pipe_.writePS(stage_id, pipe_.access(stage_id - 1));
                    pipe_.invalidatePS(stage_id - 1);
                }
 
                --stage_id;
            }
 
            // check stages up to stall point for bubbles
            // NOTE: stage_id now points to 1 past the stall point
            // The stall point cannot advance because its events are disabled
            // The stage before the stall cannot advance because the stall stage is occupied
            // So, there are two useless loops here
            while (stage_id > 0) {
                if (advance_into_stage_[stage_id - 1] &&
                    !pipe_.isValid(stage_id) && pipe_.isValid(stage_id - 1)) {
                    pipe_.writePS(stage_id, pipe_.access(stage_id - 1));
                    pipe_.invalidatePS(stage_id - 1);
                }
 
                --stage_id;
            }
 
            // try advancement from insertion point
            pipe_.shiftAppend();
        }
 
        void cancelEventsAtStage_(const uint32_t & stage_id)
        {
            sparta_assert(stage_id < num_stages_,
                          "Try to cancel events for invalid pipeline stage[" << stage_id << "]");
            if (pipe_.isValid(stage_id) && events_valid_at_stage_[stage_id]) {
                sparta_assert(event_list_at_stage_[stage_id].size());
                for (const auto & ev_ptr :  event_list_at_stage_[stage_id]) {
                    ev_ptr->cancel(sparta::Clock::Cycle(0));
                }
            }
        }
 
        void scheduleEventForEachStage_()
        {
            sparta_assert(num_stages_ == event_list_at_stage_.size());
 
            for (uint32_t i = 0; i < num_stages_; i++) {
                if (pipe_.isValid(i) && events_valid_at_stage_[i]) {
                    sparta_assert(event_list_at_stage_[i].size());
                    for (const auto & ev_ptr :  event_list_at_stage_[i]) {
                        if constexpr (std::is_same_v<EventT, PhasedPayloadEvent<DataT>>) {
                            ev_ptr->preparePayload(at(i))->schedule(sparta::Clock::Cycle(0));
                        } else {
                            ev_ptr->schedule(sparta::Clock::Cycle(0));
                        }
                    }
                }
            }
        }
 
        void deactivate_(const uint32_t & stall_stage_id,
                         const bool crush_bubbles,
                         const bool suppress_events)
        {
            sparta_assert(stall_stage_id < num_stages_,
                          "Try to deactivate events for invalid pipeline stage[" << stall_stage_id << "]");
 
            for (int32_t stage_id = stall_stage_id; stage_id >= 0; stage_id--) {
 
                if (crush_bubbles && !pipe_.isValid(stage_id)) {
                    return; // bubble, crush it by allowing earlier stages to advance
                }
 
                if (suppress_events && event_list_at_stage_[stage_id].size() > 0) {
                    events_valid_at_stage_[stage_id] = false;
                }
                advance_into_stage_[stage_id] = false;
            }
        }
 
        void restart_(const uint32_t & stall_stage_id)
        {
            for (uint32_t stage_id = 0; stage_id <= stall_stage_id; stage_id++) {
                sparta_assert(stage_id < num_stages_,
                              "Try to restart invalid pipeline stage[" << stage_id << "]");
                if (event_list_at_stage_[stage_id].size() > 0) {
                    events_valid_at_stage_[stage_id] = true;
                }
                advance_into_stage_[stage_id] = true;
            }
        }
 
        const std::string name_;
 
        const Clock * clock_;
 
        sparta::Pipe<DataT> pipe_;
 
        std::vector<EventHandleList> event_list_at_stage_;
        // NOTE:
        // (1) One advantage of using PhasedUniqueEvent instead of UnqiueEvent is:
        //     The pipeline stage events could potentially be scheduled in different phases.
        // (2) Scheduleable also works, but EventNode doesn't work (no scheduleable)
        //     std::vector<std::unique_ptr<Scheduleable>> event_list_at_stage_; // OK
        //     std::vector<std::unique_ptr<EventNode>> event_list_at_stage_; // oops
 
        std::vector<bool> events_valid_at_stage_;
        std::vector<bool> advance_into_stage_;
 
        std::vector<EventMatrix> event_matrix_at_stage_;
 
        sparta::EventSet dummy_es_{nullptr};
        sparta::EventSet * es_;
 
        UniqueEvent<SchedulingPhase::Update> ev_pipeline_update_;
 
        bool perform_own_update_ = false;
 
        const uint32_t num_stages_ = 0;
 
        Precedence default_precedence_ = Precedence::NONE;
 
        uint32_t stall_cycles_ = 0;
 
        uint32_t stall_stage_id_ = std::numeric_limits<uint32_t>::max();
    };
 
}