diff --git a/.clang-format-ignore b/.clang-format-ignore new file mode 100644 index 0000000000..7b509f206c --- /dev/null +++ b/.clang-format-ignore @@ -0,0 +1,2 @@ +*.json +Framework/*.json diff --git a/Framework/CMakeLists.txt b/Framework/CMakeLists.txt index e659e344c1..57cc6e66a1 100644 --- a/Framework/CMakeLists.txt +++ b/Framework/CMakeLists.txt @@ -140,6 +140,9 @@ add_library(O2QualityControl src/QCInputsAdapters.cxx src/QCInputsFactory.cxx src/UserInputOutput.cxx + src/LateTaskRunner.cxx + src/LateTaskRunnerFactory.cxx + src/LateTaskInterface.cxx src/Actor.cxx src/ActorHelpers.cxx src/DataProcessorAdapter.cxx @@ -185,6 +188,7 @@ add_root_dictionary(O2QualityControl HEADERS include/QualityControl/CheckInterface.h include/QualityControl/TaskInterface.h + include/QualityControl/LateTaskInterface.h include/QualityControl/UserCodeInterface.h include/QualityControl/AggregatorInterface.h include/QualityControl/PostProcessingInterface.h diff --git a/Framework/basic-late-task.json b/Framework/basic-late-task.json new file mode 100644 index 0000000000..c6b6aef437 --- /dev/null +++ b/Framework/basic-late-task.json @@ -0,0 +1,103 @@ +{ + "qc": { + "config": { + "database": { + "implementation": "CCDB", + "host": "ccdb-test.cern.ch:8080", + "maxObjectSize": "2097152", "": "[Bytes, default=2MB] Maximum size allowed, larger objects are rejected." + }, + "Activity": { + "number": "42", + "type": "NONE", + "periodName": "", "": "Period name - e.g. LHC22c, LHC22c1b_test", + "passName": "", "": "Pass type - e.g. spass, cpass1", + "provenance": "qc", "": "Provenance - qc or qc_mc depending whether it is normal data or monte carlo data" + }, + "monitoring": { + "url": "infologger:///debug?qc" + }, + "consul": { + "url": "" + }, + "conditionDB": { + "url": "ccdb-test.cern.ch:8080" + }, + "infologger": { "": "Configuration of the Infologger (optional).", + "filterDiscardDebug": "false", "": "Set to true to discard debug and trace messages (default: false)", + "filterDiscardLevel": "12", "": "Message at this level or above are discarded (default: 21 - Trace)", + "filterDiscardFile": "/tmp/_ID_.txt", "": ["If set, the messages discarded because of filterDiscardLevel", + "will go to this file (default: ); The keyword _ID_ is replaced by the device id. Discarded Debug ", + "messages won't go there."] + }, + "bookkeeping": { + "url": "" + } + }, + "tasks": { + "QcTask": { + "active": "true", + "className": "o2::quality_control_modules::skeleton::SkeletonTask", + "moduleName": "QcSkeleton", + "detectorName": "TST", + "cycleDurationSeconds": "10", + "resetAfterCycles": "1", + "dataSource": { + "type": "dataSamplingPolicy", + "name": "tst-raw" + }, + "movingWindows": ["example"], + "location": "local" + } + }, + "checks": { + "QcCheck": { + "active": "true", + "className": "o2::quality_control_modules::skeleton::SkeletonCheck", + "moduleName": "QcSkeleton", + "policy": "OnAny", + "detectorName": "TST", + "dataSource": [{ + "type": "Task", + "name": "QcTask", + "MOs": ["example"] + }], + "extendedCheckParameters": { + "physics": { + "pp": { + "myOwnKey1": "myOwnValue1c" + } + } + } + } + }, + "lateTasks": { + "late": { + "active": "true", + "className": "o2::quality_control_modules::skeleton::SkeletonLateTask", + "moduleName": "QcSkeleton", + "detectorName": "TST", + "dataSources": [{ + "type": "Task", + "name": "QcTask", + "MOs": ["example", "example2"] + }], + "outputActivityStrategy": "integrated" + } + } + }, + "dataSamplingPolicies": [ + { + "id": "tst-raw", + "active": "true", + "machines": [], + "query": "data:TST/RAWDATA/0", + "samplingConditions": [ + { + "condition": "random", + "fraction": "0.1", + "seed": "1234" + } + ] + } + ] +} diff --git a/Framework/include/QualityControl/DataProcessorAdapter.h b/Framework/include/QualityControl/DataProcessorAdapter.h index 479d15c2bb..29940c9be4 100644 --- a/Framework/include/QualityControl/DataProcessorAdapter.h +++ b/Framework/include/QualityControl/DataProcessorAdapter.h @@ -114,7 +114,7 @@ struct DataProcessorAdapter { static std::string dataProcessorName(std::string_view userCodeName, std::string_view detectorName) { using traits = ActorTraits; - return dataProcessorName(detectorName, userCodeName, traits::sActorTypeKebabCase); + return dataProcessorName(userCodeName, detectorName, traits::sActorTypeKebabCase); } /// \brief Produces standardized QC Data Processor name for cases were no user code is ran and it's not detector specific. diff --git a/Framework/include/QualityControl/DataSourceType.h b/Framework/include/QualityControl/DataSourceType.h index 8250fdac6b..f10697b1fe 100644 --- a/Framework/include/QualityControl/DataSourceType.h +++ b/Framework/include/QualityControl/DataSourceType.h @@ -28,10 +28,10 @@ enum class DataSourceType { Check, Aggregator, PostProcessingTask, + LateTask, ExternalTask, Invalid }; - } #endif // QUALITYCONTROL_DATASOURCETYPE_H \ No newline at end of file diff --git a/Framework/include/QualityControl/InfrastructureGenerator.h b/Framework/include/QualityControl/InfrastructureGenerator.h index 1f696068e0..62c80456d8 100644 --- a/Framework/include/QualityControl/InfrastructureGenerator.h +++ b/Framework/include/QualityControl/InfrastructureGenerator.h @@ -230,6 +230,7 @@ class InfrastructureGenerator static void generateAggregator(framework::WorkflowSpec& workflow, const InfrastructureSpec& infrastructureSpec); static void generatePostProcessing(framework::WorkflowSpec& workflow, const InfrastructureSpec& infrastructureSpec); static void generateBookkeepingQualitySink(framework::WorkflowSpec& workflow, const InfrastructureSpec& infrastructureSpec); + static void generateLateTasks(framework::WorkflowSpec& workflow, const InfrastructureSpec& infrastructureSpec); }; } // namespace core diff --git a/Framework/include/QualityControl/InfrastructureSpec.h b/Framework/include/QualityControl/InfrastructureSpec.h index d764b80007..c679f56503 100644 --- a/Framework/include/QualityControl/InfrastructureSpec.h +++ b/Framework/include/QualityControl/InfrastructureSpec.h @@ -23,6 +23,7 @@ #include "QualityControl/AggregatorSpec.h" #include "QualityControl/PostProcessingTaskSpec.h" #include "QualityControl/ExternalTaskSpec.h" +#include "QualityControl/LateTaskSpec.h" #include @@ -37,6 +38,7 @@ struct InfrastructureSpec { std::vector aggregators; std::vector postProcessingTasks; std::vector externalTasks; + std::vector lateTasks; }; } // namespace o2::quality_control::core diff --git a/Framework/include/QualityControl/InfrastructureSpecReader.h b/Framework/include/QualityControl/InfrastructureSpecReader.h index 8cd381176d..2f7066568b 100644 --- a/Framework/include/QualityControl/InfrastructureSpecReader.h +++ b/Framework/include/QualityControl/InfrastructureSpecReader.h @@ -22,6 +22,7 @@ #include "QualityControl/DataSourceSpec.h" #include "QualityControl/CheckSpec.h" #include "QualityControl/PostProcessingTaskSpec.h" +#include "QualityControl/LateTaskSpec.h" #include "QualityControl/RecoRequestSpecs.h" #include @@ -52,6 +53,8 @@ checker::AggregatorSpec readSpecEntry(const std::string template <> postprocessing::PostProcessingTaskSpec readSpecEntry(const std::string& entryID, const boost::property_tree::ptree& entryTree, const boost::property_tree::ptree& wholeTree); template <> +LateTaskSpec readSpecEntry(const std::string& entryID, const boost::property_tree::ptree& entryTree, const boost::property_tree::ptree& wholeTree); +template <> ExternalTaskSpec readSpecEntry(const std::string& entryID, const boost::property_tree::ptree& entryTree, const boost::property_tree::ptree& wholeTree); template <> GRPGeomRequestSpec readSpecEntry(const std::string& entryID, const boost::property_tree::ptree& entryTree, const boost::property_tree::ptree& wholeTree); diff --git a/Framework/include/QualityControl/LateTaskConfig.h b/Framework/include/QualityControl/LateTaskConfig.h new file mode 100644 index 0000000000..ef22530089 --- /dev/null +++ b/Framework/include/QualityControl/LateTaskConfig.h @@ -0,0 +1,36 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef QC_CORE_LATETASKRUNNERCONFIG_H +#define QC_CORE_LATETASKRUNNERCONFIG_H + +/// +/// \file LateTaskRunnerConfig.h +/// \author Piotr Konopka +/// + +#include + +#include "QualityControl/UserCodeConfig.h" +#include "QualityControl/DataSourceSpec.h" +#include "QualityControl/OutputActivityStrategy.h" + +namespace o2::quality_control::core +{ + +struct LateTaskConfig : public UserCodeConfig { + bool critical = true; + OutputActivityStrategy outputActivityStrategy = OutputActivityStrategy::Integrated; +}; + +} // namespace o2::quality_control::core + +#endif // QC_CORE_LATETASKRUNNERCONFIG_H diff --git a/Framework/include/QualityControl/LateTaskFactory.h b/Framework/include/QualityControl/LateTaskFactory.h new file mode 100644 index 0000000000..9d6f81600d --- /dev/null +++ b/Framework/include/QualityControl/LateTaskFactory.h @@ -0,0 +1,58 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef LATETASKFACTORY_H +#define LATETASKFACTORY_H + +/// +/// \file LateTaskFactory.h +/// \author Piotr Konopka +/// + +#include + +#include "QualityControl/LateTaskConfig.h" +#include "QualityControl/LateTaskInterface.h" +#include "QualityControl/RootClassFactory.h" + +namespace o2::quality_control::core +{ + +class LateTaskInterface; +class ObjectsManager; + +/// \brief Factory in charge of creating late tasks +/// +/// The factory needs a library name and a class name provided as an object of type LateTaskConfig. +/// The class loaded in the library must inherit from LateTaskInterface. +class LateTaskFactory +{ + public: + LateTaskFactory() = default; + virtual ~LateTaskFactory() = default; + + /// \brief Create a new instance of a LateTaskInterface. + /// The LateTaskInterface actual class is decided based on the parameters passed. + static LateTaskInterface* create(const LateTaskConfig& taskConfig, std::shared_ptr objectsManager) + { + auto* result = root_class_factory::create(taskConfig.moduleName, taskConfig.className); + result->setName(taskConfig.name); + result->setObjectsManager(objectsManager); + result->setCustomParameters(taskConfig.customParameters); + result->setCcdbUrl(taskConfig.ccdbUrl); + + return result; + } +}; + +} // namespace o2::quality_control::core + +#endif // LATETASKFACTORY_H diff --git a/Framework/include/QualityControl/LateTaskInterface.h b/Framework/include/QualityControl/LateTaskInterface.h new file mode 100644 index 0000000000..5fa4f8523c --- /dev/null +++ b/Framework/include/QualityControl/LateTaskInterface.h @@ -0,0 +1,116 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef QC_CORE_LATETASKINTERFACE_H +#define QC_CORE_LATETASKINTERFACE_H + +/// +/// \file LateTaskInterface.h +/// \author Piotr Konopka +/// + +#include + +#include +#include + +#include "QualityControl/Activity.h" +#include "QualityControl/ObjectsManager.h" +#include "QualityControl/UserCodeInterface.h" +#include "QualityControl/MonitorObject.h" +#include "QualityControl/QualityObject.h" +#include "QualityControl/QCInputs.h" + +namespace o2::monitoring +{ +class Monitoring; +} + +// namespace o2::globaltracking +//{ +// struct DataRequest; +// } + +namespace o2::framework +{ +struct ConcreteDataMatcher; +} + +namespace o2::quality_control::core +{ + +/// \brief Skeleton of a late QC task. +/// +/// Purely abstract class defining the skeleton and the common interface of a late QC task. +/// It is therefore the parent class of any late QC task. +/// It is responsible for the instantiation, modification and destruction of the TObjects that are published. +/// +/// Late tasks can process any output of a Task, Check or Aggregator and produce new MonitorObjects. +/// In a multinode setup, they always run on remote (QC) nodes, so they can access merged MonitorObjects and any +/// QualityObjects. Thus, it is not possible to run late Tasks on FLPs or EPNs. +/// In async QC, late Tasks can be used in combination with a QCDB reader (not implemented yet) to perform +/// any trends or correlations on series of objects which are available only in the QCDB. +/// +/// TODO: one could even consider allowing to feed late tasks with output of Reductors. +/// It could be an opportunity to refactor them as well (and rename them to Reducers, which sounds more like English). +/// TODO: think how to allow to produce new plots after each `process()` in sync, while producing just one at the end for async. +/// \author Piotr Konopka +class LateTaskInterface : public UserCodeInterface +{ + public: + /// \brief Constructor + /// Can't be used when dynamically loading the class with ROOT. + /// @param objectsManager + explicit LateTaskInterface(ObjectsManager* objectsManager); + + /// \brief Default constructor + LateTaskInterface() = default; + + /// \brief Destructor + virtual ~LateTaskInterface() noexcept = default; + /// Copy constructor + LateTaskInterface(const LateTaskInterface& other) = default; + /// Move constructor + LateTaskInterface(LateTaskInterface&& other) noexcept = default; + /// Copy assignment operator + LateTaskInterface& operator=(const LateTaskInterface& other) = default; + /// Move assignment operator + LateTaskInterface& operator=(LateTaskInterface&& other) /* noexcept */ = default; // error with gcc if noexcept + + /// Invoked during task initialization + virtual void initialize(o2::framework::InitContext& ctx) = 0; + /// Invoked at the start of run in synchronous mode and before the first `process()` in asynchronous mode + virtual void startOfActivity(const Activity& activity) = 0; + /// Invoked each time new data arrive + virtual void process(const core::QCInputs& data) = 0; + /// Invoked at the end of run in synchronous mode and after the last `process()` in asynchronous mode + virtual void endOfActivity(const Activity& activity) = 0; + /// Invoked at the reset() transition in synchronous mode and during workflow cleanup in asynchronous mode + virtual void reset() = 0; + + /// \brief Called each time mCustomParameters is updated. + virtual void configure() override; + + // Setters and getters + void setObjectsManager(std::shared_ptr objectsManager); + void setMonitoring(const std::shared_ptr& mMonitoring); + + protected: + std::shared_ptr getObjectsManager(); + std::shared_ptr mMonitoring; + + private: + std::shared_ptr mObjectsManager; +}; + +} // namespace o2::quality_control::core + +#endif // QC_CORE_LATETASKINTERFACE_H diff --git a/Framework/include/QualityControl/LateTaskRunner.h b/Framework/include/QualityControl/LateTaskRunner.h new file mode 100644 index 0000000000..01a78481bf --- /dev/null +++ b/Framework/include/QualityControl/LateTaskRunner.h @@ -0,0 +1,66 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef QC_CORE_LATETASKRUNNER_H +#define QC_CORE_LATETASKRUNNER_H + +/// +/// \file LateTaskRunner.h +/// \author Piotr Konopka +/// + +#include +#include +#include + +#include "QualityControl/LateTaskConfig.h" +#include "QualityControl/LateTaskRunnerTraits.h" +#include "QualityControl/ActorTraits.h" +#include "QualityControl/Actor.h" + +namespace o2::quality_control::core +{ + +class LateTaskInterface; +class ObjectsManager; +class QCInputs; + +class LateTaskRunner : public Actor +{ + public: + LateTaskRunner(const ServicesConfig& servicesConfig, const LateTaskConfig& config); + ~LateTaskRunner() = default; + + void onStart(framework::ServiceRegistryRef services, const Activity& activity); + void onStop(framework::ServiceRegistryRef services, const Activity& activity); + void onInit(framework::InitContext& iCtx); + void onProcess(framework::ProcessingContext& pCtx); + + std::string_view getDetectorName() const { return mTaskConfig.detectorName; } + std::string_view getUserCodeName() const { return mTaskConfig.name; } + bool isCritical() const { return mTaskConfig.critical; } + + private: + void startOfActivity(); + void endOfActivity(); + int publish(framework::DataAllocator& outputs); + QCInputs extractInputs(framework::ProcessingContext& pCtx); + Activity extractStrictestMatchingActivity(const QCInputs& inputs); + + private: + LateTaskConfig mTaskConfig; + std::shared_ptr mTask; + std::shared_ptr mObjectsManager; + std::optional mObjectActivity; +}; + +} // namespace o2::quality_control::core +#endif // QC_CORE_LATETASKRUNNER_H diff --git a/Framework/include/QualityControl/LateTaskRunnerFactory.h b/Framework/include/QualityControl/LateTaskRunnerFactory.h new file mode 100644 index 0000000000..f6bf2d9d0c --- /dev/null +++ b/Framework/include/QualityControl/LateTaskRunnerFactory.h @@ -0,0 +1,63 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef QC_CORE_LATETASKRUNNERFACTORY_H +#define QC_CORE_LATETASKRUNNERFACTORY_H + +/// +/// \file LateTaskRunnerFactory.h +/// \author Piotr Konopka +/// + +#include +#include + +#include + +#include "QualityControl/LateTaskConfig.h" + +namespace o2::framework +{ +class CompletionPolicy; +} + +namespace o2::quality_control::core +{ + +struct LateTaskSpec; +struct LateTaskConfig; +struct CommonSpec; +struct ServicesConfig; + +/// \brief Factory in charge of creating DataProcessorSpec of LateTaskRunner +class LateTaskRunnerFactory +{ + public: + LateTaskRunnerFactory() = default; + virtual ~LateTaskRunnerFactory() = default; + + /// \brief Creates a DataProcessorSpec for a LateTaskRunner + static o2::framework::DataProcessorSpec create(const ServicesConfig& ServicesConfig, const LateTaskConfig& taskConfig); + + /// \brief Knows how to create LateTaskConfig from Specs + static LateTaskConfig extractConfig(const CommonSpec&, const LateTaskSpec&); + + /// \brief Provides necessary customization of the LateTaskRunners. + /// + /// Provides necessary customization of the Completion Policies of the LateTaskRunners. This is necessary to make + /// them work. Put it inside customize() function before including . + /// \param policies - completion policies vector + static void customizeInfrastructure(std::vector& policies); +}; + +} // namespace o2::quality_control::core + +#endif // QC_CORE_LATETASKRUNNERFACTORY_H diff --git a/Framework/include/QualityControl/LateTaskRunnerTraits.h b/Framework/include/QualityControl/LateTaskRunnerTraits.h new file mode 100644 index 0000000000..af2d909128 --- /dev/null +++ b/Framework/include/QualityControl/LateTaskRunnerTraits.h @@ -0,0 +1,56 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef QUALITYCONTROL_LATETASKRUNNERTRAITS_H +#define QUALITYCONTROL_LATETASKRUNNERTRAITS_H + +/// +/// \file LateTaskRunnerTraits.h +/// \author Piotr Konopka +/// + +#include +#include + +#include "QualityControl/ActorTraits.h" +#include "QualityControl/DataSourceSpec.h" + +namespace o2::quality_control::core +{ + +class LateTaskRunner; + +template <> +struct ActorTraits { + constexpr static std::string_view sActorTypeShort{ "late" }; + constexpr static std::string_view sActorTypeKebabCase{ "qc-late-task" }; + constexpr static std::string_view sActorTypeUpperCamelCase{ "LateTaskRunner" }; + + constexpr static size_t sDataDescriptionHashLength{ 4 }; + + constexpr static std::array sConsumedDataSources{ + DataSourceType::Task, DataSourceType::TaskMovingWindow, DataSourceType::Check, DataSourceType::Aggregator, DataSourceType::LateTask + }; + constexpr static std::array sPublishedDataSources{ DataSourceType::LateTask }; + + constexpr static std::array sRequiredServices{ ServiceRequest::InfoLogger, ServiceRequest::Monitoring }; + constexpr static bkp::DplProcessType sDplProcessType{ bkp::DplProcessType::QC_POSTPROCESSING }; // todo at some point we should add a new type in BKP + + constexpr static UserCodeCardinality sUserCodeCardinality{ UserCodeCardinality::One }; + constexpr static bool sDetectorSpecific{ true }; + constexpr static Criticality sCriticality{ Criticality::UserDefined }; +}; + +static_assert(ValidActorTraits>); + +} // namespace o2::quality_control::core + +#endif // QUALITYCONTROL_LATETASKRUNNERTRAITS_H \ No newline at end of file diff --git a/Framework/include/QualityControl/LateTaskSpec.h b/Framework/include/QualityControl/LateTaskSpec.h new file mode 100644 index 0000000000..235cce8db7 --- /dev/null +++ b/Framework/include/QualityControl/LateTaskSpec.h @@ -0,0 +1,53 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef QC_CORE_LATETASKSPEC_H +#define QC_CORE_LATETASKSPEC_H + +/// +/// \file LateTaskSpec.h +/// \author Piotr Konopka +/// + +#include +#include + +#include "QualityControl/DataSourceSpec.h" +#include "QualityControl/CustomParameters.h" +#include "QualityControl/UpdatePolicyType.h" +#include "QualityControl/OutputActivityStrategy.h" + +namespace o2::quality_control::core +{ + +/// \brief Specification of a Task, which should map the JSON configuration structure. +struct LateTaskSpec { + // default, invalid spec + LateTaskSpec() = default; + // basic + std::string taskName = "Invalid"; + std::string className = "Invalid"; + std::string moduleName = "Invalid"; + std::string detectorName = "Invalid"; + std::vector dataSources; + + // advanced + bool active = true; + bool critical = true; + OutputActivityStrategy outputActivityStrategy = OutputActivityStrategy::Integrated; + CustomParameters customParameters; + + // reco + // GRPGeomRequestSpec grpGeomRequestSpec; + // GlobalTrackingDataRequestSpec globalTrackingDataRequest; +}; +} // namespace o2::quality_control::core +#endif // QC_CORE_LATETASKSPEC_H diff --git a/Framework/include/QualityControl/LinkDef.h b/Framework/include/QualityControl/LinkDef.h index 08fde33986..541332cbcc 100644 --- a/Framework/include/QualityControl/LinkDef.h +++ b/Framework/include/QualityControl/LinkDef.h @@ -13,6 +13,7 @@ #pragma link C++ class o2::quality_control::checker::AggregatorInterface + ; #pragma link C++ class o2::quality_control::postprocessing::PostProcessingInterface + ; #pragma link C++ class o2::quality_control::postprocessing::TrendingTask + ; +#pragma link C++ class o2::quality_control::core::LateTaskInterface + ; #pragma link C++ class o2::quality_control::core::MonitorObjectCollection + ; #pragma link C++ class o2::quality_control::core::ValidityInterval + ; #pragma link C++ class o2::quality_control::postprocessing::SliceInfo + ; diff --git a/Framework/include/QualityControl/OutputActivityStrategy.h b/Framework/include/QualityControl/OutputActivityStrategy.h new file mode 100644 index 0000000000..2d9c3b9e06 --- /dev/null +++ b/Framework/include/QualityControl/OutputActivityStrategy.h @@ -0,0 +1,28 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef QUALITYCONTROL_OUTPUTOBJECTVALIDITY_H +#define QUALITYCONTROL_OUTPUTOBJECTVALIDITY_H + +namespace o2::quality_control::core +{ + +/// This enum lets user select how Activity (incl. validity) of output objects should be calculated +enum class OutputActivityStrategy { + // Output activity contains all activities of input objects + Integrated, + // Output activity contains only the activity of the last input objects + Last +}; + +} // namespace o2::quality_control::core + +#endif // QUALITYCONTROL_OUTPUTOBJECTVALIDITY_H \ No newline at end of file diff --git a/Framework/src/Check.cxx b/Framework/src/Check.cxx index 416c7b4a17..5620e27605 100644 --- a/Framework/src/Check.cxx +++ b/Framework/src/Check.cxx @@ -216,7 +216,7 @@ CheckConfig Check::extractConfig(const CommonSpec& commonSpec, const CheckSpec& UpdatePolicyType updatePolicy = checkSpec.updatePolicy; bool checkAllObjects = false; for (const auto& dataSource : checkSpec.dataSources) { - if (!dataSource.isOneOf(DataSourceType::Task, DataSourceType::TaskMovingWindow, DataSourceType::ExternalTask, DataSourceType::PostProcessingTask)) { + if (!dataSource.isOneOf(DataSourceType::Task, DataSourceType::TaskMovingWindow, DataSourceType::ExternalTask, DataSourceType::PostProcessingTask, DataSourceType::LateTask)) { throw std::runtime_error( "Unsupported dataSource '" + dataSource.name + "' for a Check '" + checkSpec.checkName + "'"); } diff --git a/Framework/src/DataHeaderHelpers.cxx b/Framework/src/DataHeaderHelpers.cxx index c469bab13f..0117c5b94b 100644 --- a/Framework/src/DataHeaderHelpers.cxx +++ b/Framework/src/DataHeaderHelpers.cxx @@ -33,6 +33,8 @@ constexpr char CharIdFrom(DataSourceType type) return 'C'; case DataSourceType::Aggregator: return 'A'; + case DataSourceType::LateTask: + return 'L'; case DataSourceType::PostProcessingTask: return 'P'; default: @@ -113,6 +115,7 @@ constexpr size_t descriptionHashLengthFor(DataSourceType type) case DataSourceType::Check: case DataSourceType::Aggregator: case DataSourceType::PostProcessingTask: + case DataSourceType::LateTask: default: hashLength = 4; } diff --git a/Framework/src/InfrastructureGenerator.cxx b/Framework/src/InfrastructureGenerator.cxx index b3ec45ee51..089fb4a632 100644 --- a/Framework/src/InfrastructureGenerator.cxx +++ b/Framework/src/InfrastructureGenerator.cxx @@ -30,8 +30,11 @@ #include "QualityControl/RootFileSource.h" #include "QualityControl/TaskRunner.h" #include "QualityControl/TaskRunnerFactory.h" +#include "QualityControl/LateTaskRunner.h" +#include "QualityControl/LateTaskRunnerFactory.h" #include "QualityControl/Version.h" #include "QualityControl/UserInputOutput.h" +#include "QualityControl/ActorHelpers.h" #include #include @@ -107,6 +110,7 @@ framework::WorkflowSpec InfrastructureGenerator::generateStandaloneInfrastructur generateCheckRunners(workflow, infrastructureSpec); generateAggregator(workflow, infrastructureSpec); generatePostProcessing(workflow, infrastructureSpec); + generateLateTasks(workflow, infrastructureSpec); generateBookkeepingQualitySink(workflow, infrastructureSpec); return workflow; @@ -151,6 +155,7 @@ framework::WorkflowSpec InfrastructureGenerator::generateFullChainInfrastructure generateCheckRunners(workflow, infrastructureSpec); generateAggregator(workflow, infrastructureSpec); generatePostProcessing(workflow, infrastructureSpec); + generateLateTasks(workflow, infrastructureSpec); generateBookkeepingQualitySink(workflow, infrastructureSpec); return workflow; @@ -312,6 +317,7 @@ o2::framework::WorkflowSpec InfrastructureGenerator::generateRemoteInfrastructur generateCheckRunners(workflow, infrastructureSpec); generateAggregator(workflow, infrastructureSpec); generatePostProcessing(workflow, infrastructureSpec); + generateLateTasks(workflow, infrastructureSpec); generateBookkeepingQualitySink(workflow, infrastructureSpec); return workflow; @@ -388,6 +394,7 @@ framework::WorkflowSpec InfrastructureGenerator::generateRemoteBatchInfrastructu generateCheckRunners(workflow, infrastructureSpec); generateAggregator(workflow, infrastructureSpec); generatePostProcessing(workflow, infrastructureSpec); + generateLateTasks(workflow, infrastructureSpec); generateBookkeepingQualitySink(workflow, infrastructureSpec); return workflow; @@ -407,6 +414,7 @@ void InfrastructureGenerator::customizeInfrastructure(std::vector @@ -50,6 +52,7 @@ InfrastructureSpec InfrastructureSpecReader::readInfrastructureSpec(const boost: spec.aggregators = readSectionSpec(wholeTree, "aggregators"); spec.postProcessingTasks = readSectionSpec(wholeTree, "postprocessing"); spec.externalTasks = readSectionSpec(wholeTree, "externalTasks"); + spec.lateTasks = readSectionSpec(wholeTree, "lateTasks"); return spec; } @@ -204,6 +207,7 @@ DataSourceSpec InfrastructureSpecReader::readSpecEntry(const std { "Check", DataSourceType::Check }, { "Aggregator", DataSourceType::Aggregator }, { "PostProcessing", DataSourceType::PostProcessingTask }, + { "LateTask", DataSourceType::LateTask }, { "ExternalTask", DataSourceType::ExternalTask } }; @@ -293,6 +297,20 @@ DataSourceSpec InfrastructureSpecReader::readSpecEntry(const std } break; } + case DataSourceType::LateTask: { + dss.id = dataSourceTree.get("name"); + // this allows us to have tasks with the same name for different detectors + dss.name = wholeTree.get("qc.lateTasks." + dss.id + ".taskName", dss.id); + auto detectorName = wholeTree.get("qc.lateTasks." + dss.id + ".detectorName"); + + dss.inputs = { createUserInputSpec(DataSourceType::LateTask, detectorName, dss.name) }; + if (dataSourceTree.count("MOs") > 0) { + for (const auto& moName : dataSourceTree.get_child("MOs")) { + dss.subInputs.push_back(moName.second.get_value()); + } + } + break; + } case DataSourceType::Aggregator: { dss.id = dataSourceTree.get("name"); dss.name = wholeTree.get("qc.aggregators." + dss.id + ".checkName", dss.id); @@ -417,6 +435,48 @@ PostProcessingTaskSpec return ppts; } +template <> +LateTaskSpec + InfrastructureSpecReader::readSpecEntry(const std::string& lateTaskId, const boost::property_tree::ptree& lateTaskTree, const boost::property_tree::ptree& wholeTree) +{ + static std::unordered_map const outputActivityStrategyFromString = { + { "integrated", OutputActivityStrategy::Integrated }, + { "last", OutputActivityStrategy::Last } + }; + + LateTaskSpec lts; + + lts.taskName = lateTaskTree.get("taskName", lateTaskId); + lts.active = lateTaskTree.get("active", lts.active); + lts.critical = lateTaskTree.get("critical", lts.critical); + lts.className = lateTaskTree.get("className"); + lts.moduleName = lateTaskTree.get("moduleName"); + lts.detectorName = lateTaskTree.get("detectorName", lts.detectorName); + + lts.outputActivityStrategy = outputActivityStrategyFromString.at(lateTaskTree.get("outputActivityStrategy", "integrated")); + + const auto& dataSourcesTree = lateTaskTree.get_child("dataSources"); + for (const auto& dataSourceTree : dataSourcesTree | std::views::values) { + lts.dataSources.push_back(readSpecEntry(lateTaskId, dataSourceTree, wholeTree)); + } + if (lateTaskTree.count("extendedTaskParameters") > 0) { + lts.customParameters.populateCustomParameters(lateTaskTree.get_child("extendedTaskParameters")); + } else if (lateTaskTree.count("taskParameters") > 0) { + for (const auto& [key, value] : lateTaskTree.get_child("taskParameters")) { + lts.customParameters.set(key, value.get_value()); + } + } + + // if (lateTaskTree.count("grpGeomRequest") > 0) { + // lts.grpGeomRequestSpec = readSpecEntry(lts.taskName, lateTaskTree.get_child("grpGeomRequest"), wholeTree); + // } + // if (lateTaskTree.count("globalTrackingDataRequest") > 0) { + // lts.globalTrackingDataRequest = readSpecEntry(lts.taskName, lateTaskTree.get_child("globalTrackingDataRequest"), wholeTree); + // } + + return lts; +} + template <> ExternalTaskSpec InfrastructureSpecReader::readSpecEntry(const std::string& externalTaskName, const boost::property_tree::ptree& externalTaskTree, const boost::property_tree::ptree&) diff --git a/Framework/src/LateTaskInterface.cxx b/Framework/src/LateTaskInterface.cxx new file mode 100644 index 0000000000..684a49a7ff --- /dev/null +++ b/Framework/src/LateTaskInterface.cxx @@ -0,0 +1,59 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// +/// \file LateTaskInterface.cxx +/// \author Piotr Konopka +/// \author Barthelemy von Haller +/// + +#include "QualityControl/LateTaskInterface.h" + +namespace o2::quality_control::core +{ + +LateTaskInterface::LateTaskInterface(ObjectsManager* objectsManager) : mObjectsManager(objectsManager) +{ +} + +void LateTaskInterface::setObjectsManager(std::shared_ptr objectsManager) +{ + mObjectsManager = objectsManager; +} + +std::shared_ptr LateTaskInterface::getObjectsManager() { return mObjectsManager; } + +void LateTaskInterface::setMonitoring(const std::shared_ptr& mMonitoring) +{ + LateTaskInterface::mMonitoring = mMonitoring; +} +/* +void LateTaskInterface::setGlobalTrackingDataRequest(std::shared_ptr request) +{ + mGlobalTrackingDataRequest = std::move(request); +} + +const o2::globaltracking::DataRequest* LateTaskInterface::getGlobalTrackingDataRequest() const +{ + return mGlobalTrackingDataRequest.get(); +} +*/ +/* +void LateTaskInterface::finaliseCCDB(framework::ConcreteDataMatcher& matcher, void* obj) +{ +} +*/ +void LateTaskInterface::configure() +{ + // noop, override it if you want. +} + +} // namespace o2::quality_control::core diff --git a/Framework/src/LateTaskRunner.cxx b/Framework/src/LateTaskRunner.cxx new file mode 100644 index 0000000000..2131803938 --- /dev/null +++ b/Framework/src/LateTaskRunner.cxx @@ -0,0 +1,175 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// +/// \file LateTaskRunner.cxx +/// \author Piotr Konopka +/// + +#include "QualityControl/LateTaskRunner.h" + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +#include "QualityControl/ObjectsManager.h" +#include "QualityControl/LateTaskFactory.h" +#include "QualityControl/QCInputs.h" +#include "QualityControl/ActivityHelpers.h" +#include "QualityControl/MonitorObject.h" +#include "QualityControl/QualityObject.h" + +using namespace AliceO2::Common; +using namespace o2::framework; +using namespace o2::quality_control::core; + +namespace o2::quality_control::core +{ +LateTaskRunner::LateTaskRunner(const ServicesConfig& servicesConfig, const LateTaskConfig& config) + : Actor(servicesConfig), mTaskConfig(config) +{ +} + +void LateTaskRunner::onInit(InitContext& iCtx) +{ + // setup publisher + mObjectsManager = std::make_shared(mTaskConfig.name, mTaskConfig.className, mTaskConfig.detectorName, 0); + + // setup user's task + mTask.reset(LateTaskFactory::create(mTaskConfig, mObjectsManager)); + + // init user's task + mTask->setObjectsManager(mObjectsManager); + mTask->initialize(iCtx); +} + +void LateTaskRunner::onStart(framework::ServiceRegistryRef services, const Activity& activity) +{ + mTask->startOfActivity(activity); + mObjectsManager->setActivity(activity); + mObjectActivity = std::nullopt; +} + +void LateTaskRunner::onProcess(ProcessingContext& pCtx) +{ + const QCInputs& taskInputs = extractInputs(pCtx); + if (taskInputs.size() == 0) { + ILOG(Warning, Support) << "Could not prepare inputs for task, will not call process() and publish objects. " + "Known causes: no requested objects found in input; corrupted input" + << ENDM; + return; + } + + // run the task + mTask->process(taskInputs); + + // compute and set correct activity and validity for output objects + const Activity activity = extractStrictestMatchingActivity(taskInputs); + mObjectActivity = activity; + mObjectsManager->setActivity(activity); + + // publish objects + std::unique_ptr array(mObjectsManager->getNonOwningArray()); + pCtx.outputs().snapshot(mTaskConfig.name, *array); + mObjectsManager->stopPublishing(PublicationPolicy::Once); +} + +void LateTaskRunner::onStop(framework::ServiceRegistryRef services, const Activity& activity) +{ + mTask->endOfActivity(activity); + if (mObjectActivity.has_value()) { + mObjectsManager->setActivity(mObjectActivity.value()); + } + mObjectsManager->stopPublishing(PublicationPolicy::ThroughStop); +} + +QCInputs LateTaskRunner::extractInputs(ProcessingContext& pCtx) +{ + QCInputs taskInputs; + + for (const auto& ref : InputRecordWalker(pCtx.inputs())) { + // InputRecordWalker because the output of CheckRunner can be multi-part + const auto* inputSpec = ref.spec; + if (inputSpec == nullptr) { + continue; + } + + auto matchingDataSource = std::ranges::find_if(mTaskConfig.dataSources, [&](const auto& dataSource) { + // all data sources we support are expected to have a single InputSpec + return dataSource.inputs.size() == 1 && dataSource.inputs[0].binding == inputSpec->binding; + }); + + if (matchingDataSource == mTaskConfig.dataSources.end()) { + continue; + } + + if (matchingDataSource->isOneOf(DataSourceType::LateTask, DataSourceType::Task, DataSourceType::TaskMovingWindow)) { + auto moc = DataRefUtils::as(ref); + // makes MOs own their TObjects and MOC own MO (the latter we cancel in the next step) + moc->postDeserialization(); + moc->SetOwner(false); + + for (const auto& obj : *moc) { + auto mo = dynamic_cast(obj); + if (mo != nullptr) { + if (matchingDataSource->subInputs.empty() || std::ranges::find(matchingDataSource->subInputs, mo->getName()) != matchingDataSource->subInputs.end()) { + taskInputs.insert(mo->getName(), std::shared_ptr(mo)); + } else { + delete mo; + } + } else { + delete obj; + } + } + } else if (matchingDataSource->isOneOf(DataSourceType::Check, DataSourceType::Aggregator)) { + auto qo = DataRefUtils::as(ref); + auto key = qo->getName(); + taskInputs.insert(key, std::shared_ptr(std::move(qo))); + } else { + BOOST_THROW_EXCEPTION(FatalException() << errinfo_details("unsupported data source type: " + std::to_string(static_cast(matchingDataSource->type)))); + } + } + return taskInputs; +} + +Activity LateTaskRunner::extractStrictestMatchingActivity(const QCInputs& inputs) +{ + // fixme: c++20 can't join ranges, consider refactoring with std::views::concat or similar, when available + // Also, having a common parent class for MonitorObject and QualityObject would help. + std::vector> activities; + + for (const auto& mo : inputs.iterateByType()) { + activities.emplace_back(mo.getActivity()); + } + + for (const auto& qo : inputs.iterateByType()) { + activities.emplace_back(qo.getActivity()); + } + + if (mTaskConfig.outputActivityStrategy == OutputActivityStrategy::Integrated && mObjectActivity.has_value()) { + activities.emplace_back(mObjectActivity.value()); + } + + return activity_helpers::strictestMatchingActivity( + activities | // + std::views::transform([](const auto& ref) -> const Activity& { + return ref.get(); + })); +} +} // namespace o2::quality_control::core diff --git a/Framework/src/LateTaskRunnerFactory.cxx b/Framework/src/LateTaskRunnerFactory.cxx new file mode 100644 index 0000000000..fa7c5f3e4c --- /dev/null +++ b/Framework/src/LateTaskRunnerFactory.cxx @@ -0,0 +1,70 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// +/// \file LateTaskRunnerFactory.cxx +/// \author Piotr Konopka +/// + +#include "QualityControl/LateTaskRunnerFactory.h" + +#include +#include + +#include "QualityControl/ActorHelpers.h" +#include "QualityControl/LateTaskRunner.h" +#include "QualityControl/LateTaskSpec.h" +#include "QualityControl/CommonSpec.h" +#include "QualityControl/InfrastructureSpecReader.h" +#include "QualityControl/DataProcessorAdapter.h" + +using namespace o2::framework; + +namespace o2::quality_control::core +{ + +o2::framework::DataProcessorSpec LateTaskRunnerFactory::create(const ServicesConfig& ServicesConfig, const LateTaskConfig& taskConfig) +{ + auto dataProcessorName = DataProcessorAdapter::dataProcessorName(taskConfig.name, taskConfig.detectorName); + auto inputs = DataProcessorAdapter::collectUserInputs(taskConfig); + auto outputs = DataProcessorAdapter::collectUserOutputs(taskConfig); + + LateTaskRunner task(ServicesConfig, taskConfig); + return DataProcessorAdapter::adapt(std::move(task), std::move(dataProcessorName), std::move(inputs), std::move(outputs), Options{}); +} + +LateTaskConfig LateTaskRunnerFactory::extractConfig(const CommonSpec& commonSpec, const LateTaskSpec& lateTaskSpec) +{ + + return LateTaskConfig{ + { .name = lateTaskSpec.taskName, + .moduleName = lateTaskSpec.moduleName, + .className = lateTaskSpec.className, + .detectorName = lateTaskSpec.detectorName, + .consulUrl = commonSpec.consulUrl, + .customParameters = lateTaskSpec.customParameters, + .ccdbUrl = commonSpec.conditionDBUrl, + .repository = commonSpec.database, + .dataSources = lateTaskSpec.dataSources }, + lateTaskSpec.critical, + lateTaskSpec.outputActivityStrategy + }; +} + +void LateTaskRunnerFactory::customizeInfrastructure(std::vector& policies) +{ + auto matcher = [label = DataProcessorAdapter::dataProcessorLabel()](auto const& device) { + return std::find(device.labels.begin(), device.labels.end(), label) != device.labels.end(); + }; + policies.emplace_back(CompletionPolicyHelpers::consumeWhenAny("lateTasksCompletionPolicy", matcher)); +} + +} // namespace o2::quality_control::core diff --git a/Modules/Skeleton/CMakeLists.txt b/Modules/Skeleton/CMakeLists.txt index cd5dd32324..152e7f0b77 100644 --- a/Modules/Skeleton/CMakeLists.txt +++ b/Modules/Skeleton/CMakeLists.txt @@ -2,7 +2,7 @@ add_library(O2QcSkeleton) -target_sources(O2QcSkeleton PRIVATE src/SkeletonTask.cxx src/SkeletonCheck.cxx src/SkeletonAggregator.cxx src/SkeletonPostProcessing.cxx) +target_sources(O2QcSkeleton PRIVATE src/SkeletonTask.cxx src/SkeletonCheck.cxx src/SkeletonAggregator.cxx src/SkeletonPostProcessing.cxx src/SkeletonLateTask.cxx) target_include_directories( O2QcSkeleton @@ -28,6 +28,7 @@ add_root_dictionary(O2QcSkeleton include/Skeleton/SkeletonCheck.h include/Skeleton/SkeletonAggregator.h include/Skeleton/SkeletonPostProcessing.h + include/Skeleton/SkeletonLateTask.h LINKDEF "include/Skeleton/LinkDef.h") # ---- Tests ---- diff --git a/Modules/Skeleton/include/Skeleton/LinkDef.h b/Modules/Skeleton/include/Skeleton/LinkDef.h index 56abd1507b..33fd12c993 100644 --- a/Modules/Skeleton/include/Skeleton/LinkDef.h +++ b/Modules/Skeleton/include/Skeleton/LinkDef.h @@ -4,6 +4,7 @@ #pragma link off all functions; #pragma link C++ class o2::quality_control_modules::skeleton::SkeletonTask + ; +#pragma link C++ class o2::quality_control_modules::skeleton::SkeletonLateTask + ; #pragma link C++ class o2::quality_control_modules::skeleton::SkeletonCheck + ; #pragma link C++ class o2::quality_control_modules::skeleton::SkeletonPostProcessing + ; #pragma link C++ class o2::quality_control_modules::skeleton::SkeletonAggregator + ; diff --git a/Modules/Skeleton/include/Skeleton/SkeletonLateTask.h b/Modules/Skeleton/include/Skeleton/SkeletonLateTask.h new file mode 100644 index 0000000000..8f4d71b638 --- /dev/null +++ b/Modules/Skeleton/include/Skeleton/SkeletonLateTask.h @@ -0,0 +1,42 @@ +// +// Created by pkonopka on 24/06/25. +// + +#ifndef SKELETONLATETASK_H +#define SKELETONLATETASK_H + +#include "QualityControl/LateTaskInterface.h" +#include + +class TGraph; +class TH2I; + +using namespace o2::quality_control::core; + +namespace o2::quality_control_modules::skeleton +{ +/// \brief Example Quality Control Task +/// \author My Name +class SkeletonLateTask final : public LateTaskInterface +{ + public: + /// \brief Constructor + SkeletonLateTask() = default; + /// Destructor + ~SkeletonLateTask() override; + + // Definition of the methods for the template method pattern + void initialize(o2::framework::InitContext&) override; + void startOfActivity(const Activity&) override; + void process(const quality_control::core::QCInputs&) override; + void endOfActivity(const Activity&) override; + void reset() override; + + private: + std::shared_ptr mMeanTrend = nullptr; + std::shared_ptr mCorrelation = nullptr; +}; + +} // namespace o2::quality_control_modules::skeleton + +#endif // SKELETONLATETASK_H diff --git a/Modules/Skeleton/src/SkeletonCheck.cxx b/Modules/Skeleton/src/SkeletonCheck.cxx index cc49ec628a..3e4f4a133f 100644 --- a/Modules/Skeleton/src/SkeletonCheck.cxx +++ b/Modules/Skeleton/src/SkeletonCheck.cxx @@ -56,7 +56,7 @@ Quality SkeletonCheck::check(const quality_control::core::QCInputs& data) // get MonitorObject with a given name from generic data object and converts it into requested type (TH1 here) const auto histOpt = getMonitorObject(data, name); if (!histOpt.has_value()) { - ILOG(Warning, Support) << "Data object does not contain any MonitorObject with a name: " << name << ", or it couldn't be transformed into TH1" << ENDM; + ILOG(Warning, Support) << "QCInputs do not contain any MonitorObject with a name: " << name << ", or it couldn't be transformed into TH1" << ENDM; return result; } diff --git a/Modules/Skeleton/src/SkeletonLateTask.cxx b/Modules/Skeleton/src/SkeletonLateTask.cxx new file mode 100644 index 0000000000..d7b2d5d8de --- /dev/null +++ b/Modules/Skeleton/src/SkeletonLateTask.cxx @@ -0,0 +1,119 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// +/// \file SkeletonLateTask.cxx +/// \author My Name +/// + +#include +#include +#include +#include + +#include "QualityControl/QcInfoLogger.h" +#include "QualityControl/QCInputs.h" +#include "QualityControl/QCInputsAdapters.h" +#include "Skeleton/SkeletonLateTask.h" + +#include +#include +#include + +namespace o2::quality_control_modules::skeleton +{ + +SkeletonLateTask::~SkeletonLateTask() +{ +} + +void SkeletonLateTask::initialize(o2::framework::InitContext& /*ctx*/) +{ + // THUS FUNCTION BODY IS AN EXAMPLE. PLEASE REMOVE EVERYTHING YOU DO NOT NEED. + + // This creates and registers a graph for publication, we will track "example" histogram mean here + mMeanTrend = std::make_unique(); + mMeanTrend->SetName("mean_trend"); + mMeanTrend->SetTitle("mean_trend"); + mMeanTrend->GetXaxis()->SetTimeDisplay(kTRUE); + mMeanTrend->SetMarkerStyle(kStar); // star markers + mMeanTrend->SetLineStyle(kSolid); // solid line + getObjectsManager()->startPublishing(mMeanTrend.get(), PublicationPolicy::Forever); + + // This creates and registers a 2D histogram for publication, we will fill it with means of "example" and "example2" histograms + mCorrelation = std::make_unique("correlation", "correlation", 20, 0, 10500, 20, 0, 255); + getObjectsManager()->startPublishing(mCorrelation.get(), PublicationPolicy::Forever); + + // this demonstrates how to get a property tree of custom parameters + auto plots = mCustomParameters.getOptionalPtree("plots"); + if (plots.has_value()) { + ILOG(Info, Support) << "nested param: " << plots.value().get("nested") << ENDM; + } +} + +void SkeletonLateTask::startOfActivity(const Activity& activity) +{ + // THIS FUNCTION BODY IS AN EXAMPLE. PLEASE REMOVE EVERYTHING YOU DO NOT NEED. + ILOG(Debug, Devel) << "startOfActivity " << activity.mId << ENDM; + + // remove all existing data in plots to have them clean in a start->stop->start sequence + reset(); +} + +void SkeletonLateTask::process(const quality_control::core::QCInputs& data) +{ + // THIS FUNCTION BODY IS AN EXAMPLE. PLEASE REMOVE EVERYTHING YOU DO NOT NEED. + + // this is how a MonitorObject can be obtained, incl. the wrapper itself + if (auto moOpt = getMonitorObject(data, "example")) { + const MonitorObject& mo = moOpt.value(); + auto validityEndSeconds = mo.getValidity().getMax() / 1000; + + auto histo = dynamic_cast(mo.getObject()); + if (histo) { + mMeanTrend->AddPoint(validityEndSeconds, histo->GetMean()); + ILOG(Debug, Devel) << "New point in graph" << ENDM; + } + } + + // this is how objects can be retrieved without a MonitorObject wrapper + auto example1Opt = getMonitorObject(data, "example"); + auto example2Opt = getMonitorObject(data, "example2"); + if (example1Opt.has_value() && example2Opt.has_value()) { + const TH1& example1 = example1Opt.value(); + const TH1& example2 = example2Opt.value(); + + mCorrelation->Fill(example1.GetMean(), example2.GetMean()); + ILOG(Debug, Devel) << "New entry in correlation" << ENDM; + } +} + +void SkeletonLateTask::endOfActivity(const Activity& /*activity*/) +{ + // THIS FUNCTION BODY IS AN EXAMPLE. PLEASE REMOVE EVERYTHING YOU DO NOT NEED. + ILOG(Debug, Devel) << "endOfActivity" << ENDM; +} + +void SkeletonLateTask::reset() +{ + // THIS FUNCTION BODY IS AN EXAMPLE. PLEASE REMOVE EVERYTHING YOU DO NOT NEED. + + // Clean all the monitor objects here. + ILOG(Debug, Devel) << "Resetting the plots" << ENDM; + if (mMeanTrend) { + mMeanTrend->Set(0); + } + if (mCorrelation) { + mCorrelation->Reset(); + } +} + +} // namespace o2::quality_control_modules::skeleton diff --git a/Modules/o2-qc-module-configurator.sh b/Modules/o2-qc-module-configurator.sh index 1abd3006b0..ba499c8ae3 100755 --- a/Modules/o2-qc-module-configurator.sh +++ b/Modules/o2-qc-module-configurator.sh @@ -82,8 +82,8 @@ function create_class() { local classname=$2 local typename=$3 - if [ "$typename" != "Task" ] && [ "$typename" != "Check" ] && [ "$typename" != "PostProcessing" ] && [ "$typename" != "Aggregator" ]; then - echo "3rd parameter can only be Task, Check, Aggregator or PostProcessing" + if [ "$typename" != "Task" ] && [ "$typename" != "Check" ] && [ "$typename" != "PostProcessing" ] && [ "$typename" != "Aggregator" ] && [ "$typename" != "LateTask" ]; then + echo "3rd parameter can only be Task, Check, Aggregator, PostProcessing or LateTask" return fi @@ -150,8 +150,8 @@ function create_class() { function print_usage() { echo "Usage: ./o2-qc-module-configurator.sh -m MODULE_NAME [OPTION] -Generate template QC module and/or tasks, checks, aggregators and postprocessing. -If a module with specified name already exists, new tasks, checks, aggregators and postprocessing are inserted to the existing module. +Generate template QC module and/or tasks, checks, aggregators, postprocessing and late tasks. +If a module with specified name already exists, new tasks, checks, aggregators, postprocessing and late tasks are inserted to the existing module. Please follow UpperCamelCase convention for modules', tasks' and checks' names. Example: @@ -167,11 +167,12 @@ Options: -c CHECK_NAME create a check named CHECK_NAME -p PP_NAME create a postprocessing task named PP_NAME -a AGG_NAME create an aggregator named AGG_NAME + -l LT_NAME create a late task named LT_NAME " } MODULE= -while getopts 'hm:t:c:p:a:' option; do +while getopts 'hm:t:c:p:a:l:' option; do case "${option}" in \?) print_usage @@ -214,6 +215,13 @@ while getopts 'hm:t:c:p:a:' option; do fi create_class ${MODULE} ${OPTARG} Aggregator ;; + l) + if [ -z ${MODULE} ]; then + echo 'Cannot add a late task, module name not specified, exiting...' + exit 1 + fi + create_class ${MODULE} ${OPTARG} LateTask + ;; esac done diff --git a/README.md b/README.md index 7c8875615b..a35bfc62d9 100644 --- a/README.md +++ b/README.md @@ -31,13 +31,17 @@ For a general overview of our (O2) software, organization and processes, please * [Test run](doc/ModulesDevelopment.md#test-run) * [Modification of the Task](doc/ModulesDevelopment.md#modification-of-the-task) * [Check](doc/ModulesDevelopment.md#check) - * [Configuration](doc/ModulesDevelopment.md#configuration) - * [Implementation](doc/ModulesDevelopment.md#implementation) - * [Results](doc/ModulesDevelopment.md#results) + * [Configuration](doc/ModulesDevelopment.md#check-configuration) + * [Implementation](doc/ModulesDevelopment.md#check-implementation) + * [Results](doc/ModulesDevelopment.md#check-results) * [Quality Aggregation](doc/ModulesDevelopment.md#quality-aggregation) - * [Quick try](doc/ModulesDevelopment.md#quick-try) - * [Configuration](doc/ModulesDevelopment.md#configuration-1) - * [Implementation](doc/ModulesDevelopment.md#implementation-1) + * [Quick try](doc/ModulesDevelopment.md#qa-quick-try) + * [Configuration](doc/ModulesDevelopment.md#qa-configuration) + * [Implementation](doc/ModulesDevelopment.md#qa-implementation) + * [Late Task](doc/ModulesDevelopment.md#late-task) + * [Quick try](doc/ModulesDevelopment.md#late-quick-try) + * [Configuration](doc/ModulesDevelopment.md#late-configuration) + * [Implementation](doc/ModulesDevelopment.md#late-implementation) * [Naming convention](doc/ModulesDevelopment.md#naming-convention) * [Committing code](doc/ModulesDevelopment.md#committing-code) * [Data sources](doc/ModulesDevelopment.md#data-sources) @@ -77,6 +81,7 @@ For a general overview of our (O2) software, organization and processes, please * [QC Checks configuration](doc/Configuration.md#qc-checks-configuration) * [QC Aggregators configuration](doc/Configuration.md#qc-aggregators-configuration) * [QC Post-processing configuration](doc/Configuration.md#qc-post-processing-configuration) + * [QC Late Tasks configuration](doc/Configuration.md#qc-late-tasks-configuration) * [External tasks configuration](doc/Configuration.md#external-tasks-configuration) * [Merging multiple configuration files into one](doc/Configuration.md#merging-multiple-configuration-files-into-one) * [Templating config files](doc/Configuration.md#templating-config-files) diff --git a/doc/Configuration.md b/doc/Configuration.md index 4baeda8b8b..55e613d891 100644 --- a/doc/Configuration.md +++ b/doc/Configuration.md @@ -11,6 +11,7 @@ Configuration reference * [QC Checks configuration](#qc-checks-configuration) * [QC Aggregators configuration](#qc-aggregators-configuration) * [QC Post-processing configuration](#qc-post-processing-configuration) + * [QC Late Tasks configuration](#qc-late-tasks-configuration) * [External tasks configuration](#external-tasks-configuration) * [Merging multiple configuration files into one](#merging-multiple-configuration-files-into-one) * [Templating config files](#templating-config-files) @@ -46,6 +47,9 @@ This is the global structure of the configuration in QC. }, "postprocessing": { + }, + "lateTasks": { + } }, "dataSamplingPolicies": [ @@ -54,7 +58,7 @@ This is the global structure of the configuration in QC. } ``` -There are six QC-related components: +There are following QC-related components: * "config" - contains global configuration of QC which apply to any component. It is required in any configuration file. * "tasks" - contains declarations of QC Tasks. It is mandatory for running topologies with Tasks and @@ -65,6 +69,7 @@ There are six QC-related components: * "aggregators" - contains declarations of QC Aggregators. It is not mandatory. * "postprocessing" - contains declarations of PostProcessing Tasks. It is only needed only when Post-Processing is run. +* "lateTasks" - contains declarations of Late Tasks. Not mandatory. The configuration file can also include a list of Data Sampling Policies. Please refer to the [Data Sampling documentation](https://github.com/AliceO2Group/AliceO2/tree/dev/Utilities/DataSampling) to find more information. @@ -329,6 +334,7 @@ declared inside in the "postprocessing" path. Please also refer to [the Post-pro "postprocessing": { "ExamplePostprocessingID": { "": "ID of the PP Task.", "active": "true", "": "Activation flag. If not \"true\", the PP Task will not be run.", + "critical": "true", "": "if false the task is allowed to die without stopping the workflow, default: true", "taskName": "MyPPTaskName", "": ["Name of the task, used e.g. in the QCDB. If empty, the ID is used.", "Less than 14 character names are preferred."], "className": "namespace::of::PPTask", "": "Class name of the PP Task with full namespace.", @@ -355,6 +361,38 @@ declared inside in the "postprocessing" path. Please also refer to [the Post-pro } ``` +## QC Late Tasks configuration + +Below the full QC Late Tasks configuration structure is described. Note that more than one Late Task might be +declared inside in the "lateTasks" path. Please also refer to [the Late Tasks documentation](ModulesDevelopment.md#late-task) for more details. + +```json +{ + "qc": { + "lateTasks": { + "ExampleLateTaskID": { + "active": "true", "": "Activation flag. If not \"true\", the task will not be run.", + "critical": "true", "": "if false the task is allowed to die without stopping the workflow, default: true", + "taskName": "MyLateTaskName", "": "Name of the task, used e.g. in the QCDB. If empty, the ID is used.", + "className": "namespace::of::LTask", "": "Class name of the Late Task with full namespace.", + "moduleName": "QcSkeleton", "": "Library name. It can be found in CMakeLists of the detector module.", + "detectorName": "TST", "": "3-letter code of the detector.", + "dataSources": [{ "": "List of data sources.", + "type": "Task", "": "Type of the data source: \"Task\", \"TaskMovingWindow\", \"LateTask\"", + "": "\"Check\" or \"Aggregator\" are valid.", + "name": "Clusters", "": "Name of the user component which will provide the data", + "MOs": ["example_object"], "": "List of reuqested MOs. If empty, all are requested.", + "": " Use \"QOs\" to requested QualityObjects" + }], + "outputActivityStrategy": "integrated", "": "Defines how Activity and Validity of objects should be calculated.", + "": "The default \"integrated\" is a union of all inputs during an activity,", + "": "while \"last\" copies the Activity of the latest inputs." + } + } + } +} +``` + ## External tasks configuration Below the external task configuration structure is described. Note that more than one external task might be declared inside in the "externalTasks" path. diff --git a/doc/Framework.md b/doc/Framework.md index c9db415bfe..3eaa9ee93c 100644 --- a/doc/Framework.md +++ b/doc/Framework.md @@ -548,6 +548,16 @@ Post-processing tasks can be marked as critical or non-critical: By default, they are critical meaning that their failure will stop the run. If they are not critical, they will be `expendable` and will not stop the run if they die. +### QC late tasks + +Late tasks can be marked as critical or non-critical: +```json + "lateTasks": { + "LateTaskExample": { + "active": "true", + "critical": "false", "": "if false the task is allowed to die without stopping the workflow, default: true", +``` + ## QC with DPL Analysis ### Uploading objects to QCDB diff --git a/doc/ModulesDevelopment.md b/doc/ModulesDevelopment.md index dd40a9e194..c421189709 100644 --- a/doc/ModulesDevelopment.md +++ b/doc/ModulesDevelopment.md @@ -15,13 +15,17 @@ * [Test run](#test-run) * [Modification of the Task](#modification-of-the-task) * [Check](#check) - * [Configuration](#configuration) - * [Implementation](#implementation) - * [Results](#results) + * [Configuration](#check-configuration) + * [Implementation](#check-implementation) + * [Results](#check-results) * [Quality Aggregation](#quality-aggregation) - * [Quick try](#quick-try) - * [Configuration](#configuration-1) - * [Implementation](#implementation-1) + * [Quick try](#qa-quick-try) + * [Configuration](#qa-configuration) + * [Implementation](#qa-implementation) +* [Late Task](#late-task) + * [Quick try](#late-quick-try) + * [Configuration](#late-configuration) + * [Implementation](#late-implementation) * [Naming convention](#naming-convention) * [Committing code](#committing-code) * [Data sources](#data-sources) @@ -37,9 +41,10 @@ Before developing a module, one should have a bare idea of what the QualityContr ![alt text](images/Architecture.png) -The main data flow is represented in blue. Data samples are selected by the Data Sampling (not represented) and sent to the QC tasks, either on the same machines or on other machines. The tasks produce TObjects, usually histograms, encapsulated in a MonitorObject that are merged (if needed) and then checked. The checkers output a QualityObject along with the MonitorObjects which might have been modified. The MonitorObjects and the QualityObjects are stored in the repository. The QualityObjects can also be aggregated by the Aggregators to produce additional QualityObjects that are also saved in the database. +The main data flow is represented in blue. Data samples are selected by the Data Sampling (not represented) and sent to the QC tasks, either on the same machines or on other machines. The tasks produce TObjects, usually histograms, encapsulated in a MonitorObject that are merged (if needed) and then checked. The checkers output a QualityObject along with the MonitorObjects which might have been modified. The MonitorObjects and the QualityObjects are stored in the repository. The QualityObjects can also be aggregated by the Aggregators to produce additional QualityObjects that are also saved in the database. Optionally, Late Tasks can be employed to process MonitorObjects or QualityObjects and produce additional TObjects, typically trends or visualizations requiring merged objects. Asynchronously, the Post-processing can retrieve MonitorObjects from the database when certain events happen (new version of an object, new run) and produce new TObjects such as a trending plot. +However, if the objects are available in the same QC workflow, using Late Tasks is recommended. ### DPL @@ -191,6 +196,7 @@ Options: -c CHECK_NAME create a check named CHECK_NAME -p PP_NAME create a postprocessing task named PP_NAME -a AGG_NAME create an aggregator named AGG_NAME + -l LT_NAME create a late task named LT_NAME ``` For example, if your detector 3-letter code is TST you might want to do @@ -273,8 +279,9 @@ Once done, recompile it (see section above, `make -j8 install` in the build dire ## Check -A Check is a function (actually `Check::check()`) that determines the quality of the Monitor Objects produced in the previous step (the Task). It can receive multiple Monitor Objects from several Tasks. Along with the `check()` method, the `beautify()` method is a function that can modify the MO itself. It is typically used to add colors or texts on the object to express the quality. +A Check is a function (actually `Check::check()`) that determines the quality of the Monitor Objects produced in the previous step (the Task). It can receive multiple Monitor Objects from several Tasks. Along with the `check()` method, the `beautify()` method is a function that can modify the MO itself. It is typically used to add colors or texts on the object to express the quality. + ### Configuration ```json @@ -323,6 +330,7 @@ A Check is a function (actually `Check::check()`) that determines the quality of * _MOs_ - list of MonitorObjects names or can be omitted to mean that all objects should be taken. * __exportToBookkeeping__ - allows to propagate the results of this Check to Bookkeeping, where they are visualized as time-based Flags (disabled by default). + ### Implementation After the creation of the module described in the above section, every Check functionality requires a separate implementation. The module might implement several Check classes. ```c++ @@ -340,6 +348,7 @@ For each MO or group of MOs, `beautify()` is invoked after `check()` if 1. the check() did not raise an exception 2. there is a single `dataSource` in the configuration of the check + ### Results Checks return Qualities with associated Flags. @@ -353,6 +362,7 @@ The _Aggregators_ are able to collect the QualityObjects produced by the checks ![alt text](images/Aggregation.png) + ### Quick try One can try it with this simple example: @@ -369,6 +379,7 @@ A more complex example with a producer and the `o2-qc`: o2-qc-run-advanced --no-qc --no-debug-output | o2-qc --config json://${QUALITYCONTROL_ROOT}/etc/advanced-aggregator.json ``` + ### Configuration ```json @@ -408,11 +419,12 @@ o2-qc-run-advanced --no-qc --no-debug-output | o2-qc --config json://${QUALITYCO * _OnAnyNonZero_ - Triggers if ANY of the declared monitor objects changes, but only after all listed objects have been received at least once. Please see the notes on the dataSource `QOs` below. * _OnAll_ - Triggers if ALL the listed quality objects have changed. * In case the list of QualityObject is empty for any of the data sources, the policy is simply ignored for all sources and the `aggregator` will be triggered whenever a new QualityObject is received. -* __dataSource__ - declaration of the `check` input +* __dataSource__ - declaration of the `aggregator` input * _type_ - _Check_ or _Aggregator_ * _names_ - name of the Check or Aggregator * _QOs_ - list of QualityObjects names or can be omitted to mean that all objects should be taken. In case of `OnAnyNonZero` one must list the objects and if the the check produces only 1 then it should list only an empty string. + ### Implementation With `o2-qc-module-configurator.sh` (see [here](#module-creation)), create a new Aggregator that can be then used in the config file. @@ -424,6 +436,99 @@ An aggregator inherits from `AggregatorInterface` and in particular this method: The `aggregate` method is called whenever the _policy_ is satisfied. It gets a map with all the declared QualityObjects. It is expected to return a new Quality based on the inputs. +## Late Task + +Late Tasks are useful to create new MonitorObjects based on existing ones in the QC workflow. +They are called "late", because they are typically executed close to the end of the processing chain. +When running QC workflows distributed over multiple nodes, with Mergers combining the results, they can be used to process the merged results. +The adequate use cases involve: +* creating trends and correlations from other MonitorObjects and QualityObjects (trending a histogram average, trending quality) +* creating plots which can only be constructed from merged objects (ratios, visualizations, ...) +* creating summary canvases of QualityObjects available in the QC workflow + + +### Quick try + +One can try it with this simple example: + +```c++ +o2-qc-run-basic --config-path ${QUALITYCONTROL_ROOT}/etc/basic-late-task.json +``` + +Notice the appearance of `qc-late-task-TEST-late` and an associated CheckRunner. +The object `qc/TST/MO/late/graph_example` will be updated in the QCG test instace. + + +### Configuration + +Below is an example of a late task configuration. + +```json +{ + "qc": { + "config": {...}, + "tasks": {...}, + "lateTasks": { + "myLateTask": { + "active": "true", + "className": "o2::quality_control_modules::skeleton::SkeletonLateTask", + "moduleName": "QcSkeleton", + "detectorName": "TST", + "dataSources": [{ + "type": "Task", + "name": "QcTask", + "MOs": ["example"] + }] + } + } + }, + "dataSamplingPolicies": [...] +} +``` + +* __active__ - Boolean to indicate whether the aggregator is active or not +* __moduleName__ - Name of the module which implements the aggregator class (like in tasks) +* __className__ - Name and namespace of the class, which is part of the module specified above (like in tasks) +* __dataSources__ - declaration of the `lateTask` input + * _type_ - _Task_, _TaskMovingWindow_, _LateTask_, _Check_ or _Aggregator_ + * _names_ - name of the user component + * _MOs_ or _QOs_ - list of requested objects. If empty, all objects are requested. + +Late Tasks outputs can be requested by Checks. + + +### Implementation + +With `o2-qc-module-configurator.sh` (see [here](#module-creation)), create a new Late Task that can be then used in the config file. + +A late task inherits from `LateTaskInterface` and in particular from these methods: +```c++ + /// Invoked during task initialization + virtual void initialize(o2::framework::InitContext& ctx) = 0; + /// Invoked at the start of run in synchronous mode and before the first `process()` in asynchronous mode + virtual void startOfActivity(const Activity& activity) = 0; + /// Invoked each time new data arrive + virtual void process(const core::QCInputs& data) = 0; + /// Invoked at the end of run in synchronous mode and after the last `process()` in asynchronous mode + virtual void endOfActivity(const Activity& activity) = 0; + /// Invoked at the reset() transition in synchronous mode and during workflow cleanup in asynchronous mode + virtual void reset() = 0; +``` +Inside the generated sources, you will find examples how to initialize the task, process input data and publish output objects. + +The `process()` method is invoked with input objects as soon as they arrive. +Please note that when several data sources are requested, the objects might not arrive at the same time. + +### Further development plans + +The Late Tasks were developed to eventually replace post-processing tasks as a less error-prone and simpler to use alternative. +They should reduce the load on the QCDB, as they receive objects directly within the message passing framework instead of sending requests to the database. +As the next steps, well-familiar post-processing tasks, such as TrendingTask, SliceTrendingTask, ReferenceComparatorTask, QualityTask, will be rewritten as Late Tasks and usage migration will be encouraged and coordinated. + +Post-processing tasks might remain as the only way to monitor objects available in CCDB or to get triggered on rare events. + +To support post-processing tasks use-cases in asynchronous mode, we will need to develop QCDB object readers. + ## Naming convention We apply a naming convention for Task, Check and Aggregator names, i.e. how they are named in QCDB, not their class names. Here are the rules: diff --git a/doc/PostProcessing.md b/doc/PostProcessing.md index 8d4271a67a..79deca2c28 100644 --- a/doc/PostProcessing.md +++ b/doc/PostProcessing.md @@ -27,6 +27,8 @@ This framework is intended for planned post-processing of objects generated by Q The users can write their own Post-processing Tasks or use the ones provided by the framework (see [Convenience classes](#convenience-classes)) which are supposed to cover the usual needs. Post-processing Tasks run asynchronously to data-taking, but can be triggered by a set of selected events. +Before implementing a Post-processing Task, consider using Late Tasks, which let you consume any other QC objects directly in the message passing framework, instead of going through the QC repository. + ### Post-processing interface Any Post-processing Task should inherit PostProcessingInterface, which includes four methods: