ReportDescriptor.hpp

Go to the documentation of this file.

 
#pragma once
 
#include <algorithm>
#include <boost/algorithm/string/replace.hpp>
#include <boost/any.hpp>
#include <boost/iterator/iterator_traits.hpp>
#include <boost/type_index/type_index_facade.hpp>
#include <cstddef>
#include <cstdint>
#include <deque>
#include <map>
#include <memory>
#include <ostream>
#include <set>
#include <string>
#include <unordered_map>
#include <unordered_set>
#include <utility>
#include <vector>
 
#include "sparta/simulation/TreeNode.hpp"
#include "sparta/report/Report.hpp"
#include "sparta/utils/Utils.hpp"
#include "sparta/report/format/BaseFormatter.hpp"
#include "sparta/app/FeatureConfiguration.hpp"
#include "simdb_fwd.hpp"
#include "sparta/simulation/Clock.hpp"
#include "sparta/utils/SpartaException.hpp"
#include "sparta/utils/ValidValue.hpp"
 
namespace simdb {
    class AsyncTaskEval;
    class ObjectManager;
}  // namespace simdb
 
namespace sparta::app {
    class SimulationConfiguration;
}  // namespace sparta::app
 
namespace sparta::trigger {
    class SkippedAnnotatorBase;
}  // namespace sparta::trigger
 
namespace sparta {
 
    namespace report {
        namespace format {
            class BaseFormatter;
            class FormatterFactory;
        }
    }
 
    class Report;
    class RootTreeNode;
    class Scheduler;
    class TreeNode;
 
    namespace statistics {
        class ReportStatisticsArchive;
        class StreamNode;
    }
 
    namespace async {
        class AsyncTimeseriesReport;
        class AsyncNonTimeseriesReport;
    }
    namespace db {
        class ReportHeader;
    }
 
    namespace app {
 
        class Simulation;
 
        typedef std::unordered_map<std::string, boost::any> NamedExtensions;
        typedef std::unordered_map<std::string, std::string> TriggerKeyValues;
        typedef std::unordered_map<std::string, std::string> MetaDataKeyValues;
 
        class ReportDescriptor
        {
            typedef std::pair<Report*, report::format::BaseFormatter*> inst_t;
 
            std::set<const Report*> triggered_reports_;
 
            std::set<const Report*> idle_reports_;
 
            std::map<std::string, std::shared_ptr<report::format::BaseFormatter>> formatters_;
 
            std::shared_ptr<statistics::ReportStatisticsArchive> report_archive_;
 
            std::shared_ptr<statistics::StreamNode> streaming_stats_root_;
 
            bool updateReportActiveState_(const Report * r);
 
            std::vector<inst_t> instantiations_;
 
            const sparta::report::format::FormatterFactory* fact_;
 
            uint32_t writes_;
 
            uint32_t updates_;
 
            class DescUpdateTracker {
            public:
                void enable(const Scheduler * scheduler);
                bool checkIfDuplicateUpdate();
            private:
                utils::ValidValue<bool> enabled_;
                utils::ValidValue<uint64_t> last_update_at_tick_;
                const Scheduler * scheduler_ = nullptr;
            };
 
            DescUpdateTracker update_tracker_;
 
            std::shared_ptr<sparta::trigger::SkippedAnnotatorBase> skipped_annotator_;
 
            bool report_stopped_ = false;
 
            bool enabled_ = true;
 
            std::string orig_dest_file_;
 
            std::shared_ptr<async::AsyncTimeseriesReport> db_timeseries_;
 
            std::shared_ptr<async::AsyncNonTimeseriesReport> db_non_timeseries_;
 
            const FeatureConfiguration::FeatureOptions * simdb_feature_opts_ = nullptr;
 
            friend class ReportDescriptorCollection;
 
        public:
 
            static const char* GLOBAL_KEYWORD;
 
            std::string loc_pattern;
 
            std::string def_file;
 
            std::string dest_file;
 
            std::string format;
 
            NamedExtensions extensions_;
 
            MetaDataKeyValues header_metadata_;
 
            void disable() {
                enabled_ = false;
            }
 
            bool isEnabled() const {
                return enabled_;
            }
 
            bool isSingleTimeseriesReport() const;
 
            bool isSingleNonTimeseriesReport() const;
 
            void configureAsyncTimeseriesReport(
                simdb::AsyncTaskEval * task_queue,
                simdb::ObjectManager * sim_db,
                const Clock & root_clk);
 
            void configureAsyncNonTimeseriesReport(
                simdb::AsyncTaskEval * task_queue,
                simdb::ObjectManager * sim_db);
 
            db::ReportHeader * getTimeseriesDatabaseHeader();
 
            void doPostProcessing(
                simdb::AsyncTaskEval * task_queue,
                simdb::ObjectManager * sim_db);
 
            std::map<std::string, std::shared_ptr<
                                      report::format::BaseFormatter>> getFormattersByFilename() const;
 
            static bool isValidFormatName(const std::string& format);
 
            ReportDescriptor(const std::string& _loc_pattern,
                             const std::string& _def_file,
                             const std::string& _dest_file,
                             const std::string& _format="text");
 
            ReportDescriptor(const ReportDescriptor&) = default;
 
            ReportDescriptor& operator=(const ReportDescriptor&) = default;
 
            ~ReportDescriptor();
 
            const std::string & getDescriptorPattern() const {
                return loc_pattern;
            }
 
            const std::string & getDescriptorDefFile() const {
                return def_file;
            }
 
            const std::string & getDescriptorDestFile() const {
                return dest_file;
            }
 
            const std::string & getDescriptorFormat() const {
                return format;
            }
 
            const std::string & getDescriptorOrigDestFile() const {
                return orig_dest_file_;
            }
 
            std::shared_ptr<statistics::ReportStatisticsArchive> logOutputValuesToArchive(
                const std::string & dir);
 
            std::shared_ptr<statistics::StreamNode> createRootStatisticsStream();
 
            void inspectSimulatorFeatureValues(
                const app::FeatureConfiguration * feature_config);
 
            void ignoreFurtherUpdates() {
                report_stopped_ = true;
            }
 
            std::string stringize() const {
                std::stringstream ss;
                ss << "Report def \"" << def_file << "\" on node \"" << loc_pattern
                   << "\" -> \"" << (orig_dest_file_.empty() ? dest_file : orig_dest_file_) << "\"";
                if(format.size() != 0){
                    ss << " (format=" << format << ")";
                }
                return ss.str();
            }
 
            report::format::BaseFormatter* addInstantiation(Report* r,
                                                            Simulation* sim,
                                                            std::ostream* out=nullptr);
 
            std::vector<inst_t> getInstantiations() const {
                return instantiations_;
            }
 
            std::vector<Report*> getPendingInstantiations() const {
                auto iter = extensions_.find("pending-reports");
                if (iter != extensions_.end()) {
                    const std::vector<Report*> & pending_reports =
                        boost::any_cast<const std::vector<Report*>&>(iter->second);
 
                    return pending_reports;
                }
                return {};
            }
 
            std::vector<Report*> getAllInstantiations() const {
                std::vector<inst_t> current = getInstantiations();
                std::vector<Report*> pending = getPendingInstantiations();
 
                std::unordered_set<Report*> all(pending.begin(), pending.end());
                for (const auto & cur : current) {
                    all.insert(cur.first);
                }
                return std::vector<Report*>(all.begin(), all.end());
            }
 
            uint32_t writeOutput(std::ostream* out=nullptr);
 
            uint32_t updateOutput(std::ostream* out=nullptr);
 
            void skipOutput();
 
            void capUpdatesToOncePerTick(const Scheduler * scheduler);
 
            void setSkippedAnnotator(std::shared_ptr<sparta::trigger::SkippedAnnotatorBase> annotator);
 
            void clearDestinationFiles(const Simulation& sim);
 
            uint32_t getUsageCount() const { return instantiations_.size(); }
 
            uint32_t getNumWrites() const { return writes_; }
 
            uint32_t getNumUpdates() const { return updates_; }
 
            std::string computeFilename(const Report* r,
                                        const std::string& sim_name,
                                        uint32_t idx) const;
        };
 
        typedef std::vector<ReportDescriptor> ReportDescVec;
        typedef std::vector<std::pair<std::string, std::string>> ReportYamlReplacements;
 
        class ReportDescriptorCollection
        {
        public:
            ReportDescriptorCollection() = default;
            ~ReportDescriptorCollection() = default;
 
            ReportDescriptorCollection(const ReportDescriptorCollection &) = delete;
            ReportDescriptorCollection & operator=(const ReportDescriptorCollection &) = delete;
 
            //While not copyable, moves are enabled to support std::swap
            ReportDescriptorCollection(ReportDescriptorCollection &&) = default;
            ReportDescriptorCollection & operator=(ReportDescriptorCollection &&) = default;
 
            void push_back(const ReportDescriptor & rd) {
                //Internal book-keeping is only for Python shell workflows.
                //We also do not perform any book-keeping for descriptors
                //whose dest_file is "1" (print to stdout). That is the
                //only use case where duplicate dest_file's is actually
                //okay. We will add the descriptor to the collection, but
                //any attempts to access this descriptor later (i.e. to
                //change its parameters or to disable it from Python) will
                //throw an exception that the descriptor "1" cannot be found.
                if (rd.dest_file != "1") {
                    const std::string desc_name = getDescriptorName_(rd);
                    auto iter = indices_by_descriptor_name_.find(desc_name);
                    if (iter != indices_by_descriptor_name_.end()) {
                        //Descriptor found by this name. Throw if it is already
                        //enabled. Users can only replace disabled descriptors.
                        if (rep_descs_.at(iter->second).isEnabled()) {
                            throw SpartaException("Report descriptor named '") <<
                                desc_name << "' already exists in this configuration";
                        }
                    }
                    indices_by_descriptor_name_[desc_name] = rep_descs_.size();
                }
 
                //This incoming 'rd' variable could be a reference to a
                //previously disabled descriptor:
                //
                //   >>> rd = report_config.descriptors.foo_csv
                //   >>> report_config.removeReport('foo_csv')
                //   >>> report_config.addReport(rd)
                //
                //This would still count as an enabled descriptor.
                rep_descs_.push_back(rd);
                rep_descs_.back().enabled_ = true;
            }
 
            template <class... Args>
            void emplace_back(Args&&... args) {
                ReportDescriptor rd(std::forward<Args>(args)...);
                push_back(rd);
            }
 
            void clear() {
                for (auto &rd : rep_descs_) {
                    rd.disable();
                }
            }
 
            size_t size() const {
                size_t sz = 0;
                for (const auto & rep_desc : rep_descs_) {
                    if (rep_desc.isEnabled()) {
                        ++sz;
                    }
                }
                return sz;
            }
 
            bool empty() const {
                return size() == 0;
            }
 
            bool contains(const std::string & desc_name) const {
                auto iter = indices_by_descriptor_name_.find(desc_name);
                if (iter == indices_by_descriptor_name_.end()) {
                    //The name may have been given to us with a period, not
                    //an underscore, separating the file stem and the extension
                    const auto replaced_desc_name = replaceDotsWithUnderscores_(desc_name);
                    iter = indices_by_descriptor_name_.find(replaced_desc_name);
                }
 
                //If this name is not even found in our map, we do not
                //contain this descriptor
                if (iter == indices_by_descriptor_name_.end()) {
                    return false;
                }
 
                //But even if the descriptor is physcially here, we have
                //to also ask it if is is still enabled
                return rep_descs_.at(iter->second).isEnabled();
            }
 
            ReportDescriptor & getDescriptorByName(const std::string & desc_name) {
                auto idx = getDescriptorIndexByName_(desc_name);
                return rep_descs_.at(idx);
            }
 
            void removeDescriptorByName(const std::string & desc_name) {
                getDescriptorByName(desc_name).disable();
            }
 
            std::vector<std::string> getAllDescriptorNames() const {
                std::vector<std::string> names;
                for (const auto & rd : indices_by_descriptor_name_) {
                    if (rep_descs_.at(rd.second).isEnabled()) {
                        names.push_back(rd.first);
                    }
                }
                return names;
            }
 
            std::deque<ReportDescriptor>::iterator begin() {
                return rep_descs_.begin();
            }
 
            std::deque<ReportDescriptor>::const_iterator begin() const {
                return rep_descs_.cbegin();
            }
 
            std::deque<ReportDescriptor>::iterator end() {
                return rep_descs_.end();
            }
 
            std::deque<ReportDescriptor>::const_iterator end() const {
                return rep_descs_.cend();
            }
 
        private:
            std::string getDescriptorName_(const ReportDescriptor & rd) const {
                return replaceDotsWithUnderscores_(rd.dest_file);
            }
 
            std::string replaceDotsWithUnderscores_(const std::string & str) const {
                std::string replaced = str;
                boost::replace_all(replaced, ".", "_");
                return replaced;
            }
 
            size_t getDescriptorIndexByName_(const std::string & desc_name) const {
                auto iter = indices_by_descriptor_name_.find(desc_name);
                if (iter == indices_by_descriptor_name_.end()) {
                    //The name may have been given to us with a period, not
                    //an underscore, separating the file stem and the extension
                    const auto replaced_desc_name = replaceDotsWithUnderscores_(desc_name);
                    iter = indices_by_descriptor_name_.find(replaced_desc_name);
 
                    //If it still is not found, error out
                    if (iter == indices_by_descriptor_name_.end()) {
                        throw SpartaException(
                            "No descriptor named '") << desc_name << "' exists";
                    }
                }
 
                //The descriptor exists in this collection, but we still need
                //to throw if it has been disabled, since we are not supposed
                //to be using those descriptors for any reason anymore.
                if (!rep_descs_.at(iter->second).isEnabled()) {
                    throw SpartaException("The descriptor named '") << desc_name
                                                                    << "' has already been disabled";
                }
                return iter->second;
            }
 
            std::deque<ReportDescriptor> rep_descs_;
            std::unordered_map<std::string, size_t> indices_by_descriptor_name_;
        };
 
        class ReportConfiguration {
        public:
            ReportConfiguration(SimulationConfiguration * sim_config,
                                ReportDescriptorCollection * collection,
                                RootTreeNode * root);
 
            ReportConfiguration() = delete;
            ~ReportConfiguration() = default;
 
            ReportConfiguration(const ReportConfiguration &) = delete;
            ReportConfiguration & operator=(const ReportConfiguration &) = delete;
 
            void addReport(const ReportDescriptor & rd);
 
            void addReportsFromYaml(const std::string & yaml_file);
 
            void removeReportByName(const std::string & rd_name);
 
            void addMemoryReportsFromYaml(const std::string & yaml_file);
 
            void showAllReportDescriptorInfo();
 
            void serializeAllDescriptorsToYaml();
 
            ReportDescriptorCollection * getDescriptors();
 
            const ReportDescriptorCollection * getDescriptors() const;
 
        private:
            //When using the Python shell, adding/removing reports
            //from the configuration will refresh the appropriate
            //variable(s) in the Python workspace
            void republishReportCollection_();
 
            //Remove report config variable from the Python namespace
            void finishPythonInteraction_();
 
            //Finalize any changes to the descriptors from here on out
            void disallowChangesToDescriptors_();
 
            //Give the Simulation base class access to the above private
            //methods related to Python shell interaction. Note that the
            //sim_config_, collection_, and root_ member variables are
            //const pointers, so even with friend access, the Simulation
            //class still won't be able to change those variables for any
            //reason.
            friend class Simulation;
 
            SimulationConfiguration *const sim_config_;
            ReportDescriptorCollection *const collection_;
            RootTreeNode *const root_;
            bool allow_descriptor_changes_ = true;
        };
 
        ReportYamlReplacements createReplacementsFromYaml(
            const std::string & replacements_yaml);
 
        ReportDescVec createDescriptorsFromFile(
            const std::string & def_file,
            TreeNode * context);
 
        ReportDescVec createDescriptorsFromFileWithPlaceholderReplacements(
            const std::string & def_file,
            TreeNode * context,
            const ReportYamlReplacements & placeholder_key_value_pairs);
 
        ReportDescVec createDescriptorsFromDefinitionString(
            const std::string & def_string,
            TreeNode * context);
 
    } // namespace app
} // namespace sparta