Benchmark.h

/*********************************************************************
* Software License Agreement (BSD License)
*
*  Copyright (c) 2010, Rice University
*  All rights reserved.
*
*  Redistribution and use in source and binary forms, with or without
*  modification, are permitted provided that the following conditions
*  are met:
*
*   * Redistributions of source code must retain the above copyright
*     notice, this list of conditions and the following disclaimer.
*   * Redistributions in binary form must reproduce the above
*     copyright notice, this list of conditions and the following
*     disclaimer in the documentation and/or other materials provided
*     with the distribution.
*   * Neither the name of the Rice University nor the names of its
*     contributors may be used to endorse or promote products derived
*     from this software without specific prior written permission.
*
*  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
*  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
*  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
*  FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
*  COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
*  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
*  BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
*  LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
*  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
*  LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
*  ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
*  POSSIBILITY OF SUCH DAMAGE.
*********************************************************************/

/* Author: Ioan Sucan */

#ifndef OMPL_TOOLS_BENCHMARK_BENCHMARK_
#define OMPL_TOOLS_BENCHMARK_BENCHMARK_

#include "ompl/geometric/SimpleSetup.h"
#include "ompl/control/SimpleSetup.h"

namespace ompl
{
    namespace tools
    {
        class Benchmark
        {
        public:

            struct Status
            {
                Status()
                {
                    running = false;
                    activeRun = 0;
                    progressPercentage = 0.0;
                }

                bool         running;

                std::string  activePlanner;

                unsigned int activeRun;

                double       progressPercentage;
            };

            typedef std::map<std::string, std::string> RunProperties;

            typedef std::vector<std::map<std::string, std::string> > RunProgressData;

            typedef std::function<void(const base::PlannerPtr&)> PreSetupEvent;

            typedef std::function<void(const base::PlannerPtr&, RunProperties&)> PostSetupEvent;

            struct PlannerExperiment
            {
                std::string                name;

                std::vector<RunProperties> runs;

                std::vector<std::string>     progressPropertyNames;

                std::vector<RunProgressData> runsProgressData;

                RunProperties              common;

                bool operator==(const PlannerExperiment& p) const
                {
                    return name==p.name && runs==p.runs && common==p.common;
                }
            };

            struct CompleteExperiment
            {
                std::string                    name;

                std::vector<PlannerExperiment> planners;

                double                         maxTime;

                double                         maxMem;

                unsigned int                   runCount;

                time::point                    startTime;

                double                         totalDuration;

                std::string                    setupInfo;

                boost::uint32_t                seed;

                std::string                    host;

                std::string                    cpuInfo;

                std::map<std::string, std::string> parameters;
            };

            struct Request
            {
                Request(double maxTime = 5.0, double maxMem = 4096.0,
                        unsigned int runCount = 100,
                        double timeBetweenUpdates = 0.05,
                        bool displayProgress = true,
                        bool saveConsoleOutput = true, bool useThreads = true,
                        bool simplify = true)
                    : maxTime(maxTime), maxMem(maxMem), runCount(runCount),
                    timeBetweenUpdates(timeBetweenUpdates),
                    displayProgress(displayProgress), saveConsoleOutput(saveConsoleOutput),
                    useThreads(useThreads),
                    simplify(simplify)
                {
                }

                double       maxTime;

                double       maxMem;

                unsigned int runCount;

                double       timeBetweenUpdates;

                bool         displayProgress;

                bool         saveConsoleOutput;

                bool         useThreads;

                bool         simplify;
            };

            Benchmark(geometric::SimpleSetup &setup, const std::string &name = std::string()) : gsetup_(&setup), csetup_(nullptr)
            {
                exp_.name = name;
            }

            Benchmark(control::SimpleSetup &setup, const std::string &name = std::string()) : gsetup_(nullptr), csetup_(&setup)
            {
                exp_.name = name;
            }

            virtual ~Benchmark()
            {
            }

            void addExperimentParameter(const std::string& name, const std::string& type, const std::string& value)
            {
                exp_.parameters[name + " " + type] = value;
            }

            const std::map<std::string, std::string>& getExperimentParameters() const
            {
                return exp_.parameters;
            }

            std::size_t numExperimentParameters() const
            {
                return exp_.parameters.size();
            }

            void setExperimentName(const std::string &name)
            {
                exp_.name = name;
            }

            const std::string& getExperimentName() const
            {
                return exp_.name;
            }

            void addPlanner(const base::PlannerPtr &planner)
            {
                if (planner && planner->getSpaceInformation().get() !=
                    (gsetup_ ? gsetup_->getSpaceInformation().get() : csetup_->getSpaceInformation().get()))
                    throw Exception("Planner instance does not match space information");
                planners_.push_back(planner);
            }

            void addPlannerAllocator(const base::PlannerAllocator &pa)
            {
                planners_.push_back(pa(gsetup_ ? gsetup_->getSpaceInformation() : csetup_->getSpaceInformation()));
            }

            void clearPlanners()
            {
                planners_.clear();
            }

            void setPlannerSwitchEvent(const PreSetupEvent &event)
            {
                plannerSwitch_ = event;
            }

            void setPreRunEvent(const PreSetupEvent &event)
            {
                preRun_ = event;
            }

            void setPostRunEvent(const PostSetupEvent &event)
            {
                postRun_ = event;
            }

            virtual void benchmark(const Request &req);

            const Status& getStatus() const
            {
                return status_;
            }

            const CompleteExperiment& getRecordedExperimentData() const
            {
                return exp_;
            }

            virtual bool saveResultsToStream(std::ostream &out = std::cout) const;

            bool saveResultsToFile(const char *filename) const;

            bool saveResultsToFile() const;

        protected:

            geometric::SimpleSetup       *gsetup_;

            control::SimpleSetup         *csetup_;

            std::vector<base::PlannerPtr> planners_;

            CompleteExperiment            exp_;

            Status                        status_;

            PreSetupEvent                 plannerSwitch_;

            PreSetupEvent                 preRun_;

            PostSetupEvent                postRun_;

        };
    }
}
#endif