src/dcxmletlnetwork.cpp

#include "dcxmletlnetwork.h"
#include "log.h"
using namespace osdev::components;
// XPath names
// Network related
static const char* selectNetworks = "selectNetworks";
static const char* selectNetwork = "selectNetwork";
// Functional Objects
static const char* selectObjects = "selectObjects";
static const char* selectObject = "selectObject";
// Inputs
static const char* selectInputs = "selectInputs";
static const char* selectInput = "selectInput";
static const char* selectInputVarNames = "selectInputVarNames";
// Outputs
static const char* selectOutputs = "selectOutputs";
static const char* selectOutput = "selectOutput";
static const char* selectOutputVarNames = "selectOutputVarNames";
// Connections
static const char* selectConnections = "selectConnections";
// QThread priorities
static const char* prio_idle = "prio_idle";
static const char* prio_lowest = "prio_lowest";
static const char* prio_low = "prio_low";
static const char* prio_normal = "prio_normal";
static const char* prio_high = "prio_high";
static const char* prio_highest = "prio_highest";
static const char* prio_time_critical = "prio_time_critical";
static const char* prio_inherit = "prio_inherit";
namespace {
/**
 * @brief Generates DataObject input data from an xml inputs node.
 * The inputs node contains one or more input nodes which describe a database table column
 */
class GenerateInputData
{
public:
    // Non constructable
    GenerateInputData() = delete;
    /**
     * @brief Generate input data from an inputs node.
     * @param inputsNode xml inputs node. The method does nothing when an xml node with a different name is supplied.
     * @param[out] data The input data is added to this instance.
     */
    static void generate(const pugi::xml_node& inputsNode, ObjectData& data);
private:
    /**
     * @brief Internal method that parses an xml input node and calls this method on its child nodes when appropriate.
     * @param inputNode xml input node. The method does nothing when an xml node with a different name is supplied.
     * @param containerPath The context for which the input is generated. This argument is supplied by copy so that
     *                      recursive calls can expand the context.
     * @param[out] data The input data is added to this instance.
     * @param level The recursion level (for debugging purposes)
     */
    static void handleInputNode(const pugi::xml_node& inputNode, QStringList containerPath, ObjectData& data, int level);
};
// static
void GenerateInputData::generate(const pugi::xml_node& inputsNode, ObjectData& data)
{
    if (std::string(inputsNode.name()) != std::string("inputs")) {
        return;
    }
    for (auto inputNode = inputsNode.child("input"); inputNode; inputNode = inputNode.next_sibling("input")) {
        handleInputNode(inputNode, QStringList{}, data, 1);
    }
}
// static
void GenerateInputData::handleInputNode(const pugi::xml_node& inputNode, QStringList containerPath, ObjectData& data, int level)
{
    LogTrace("[DcXmlEtlNetwork GenerateInputData]", QString("%1. current container path : %2").arg(level).arg(containerPath.join("/")));
    const auto containerAttribute = inputNode.attribute("container");
    const auto nameAttribute = inputNode.attribute("name");
    // only expand the container when the container attribute is available and the container path is either empty or the last item in the container path
    // is different from the container attribute value.
    if (containerAttribute && (containerPath.empty() || containerPath.back() != QString(containerAttribute.value()))) {
        containerPath.append(containerAttribute.value());
        LogTrace("[DcXmlEtlNetwork GenerateInputData]", QString("%1. container path becomes %2").arg(level).arg(containerPath.join("/")));
    }
    if (nameAttribute) {
        auto excludeFlagAttr = inputNode.attribute("exclude_from_identity_check");
        bool excludeFlag = false;
        if (!excludeFlagAttr.empty()) {
            excludeFlag = DcXmlBase::getBoolean(excludeFlagAttr.value());
        }
        LogTrace("[DcXmlEtlNetwork GenerateInputData]", QString("%1. Generate data for %2 and put it in %3").arg(level).arg(containerPath.join("/") + QString(".") + nameAttribute.value(), data.getObjectId()));
        data.setInputData(containerPath.join("/") + QString(".") + nameAttribute.value(),
            DcXmlBase::getAttributeValue(inputNode, "type"),
            DcXmlBase::getAttributeValue(inputNode, "id"),
            DcXmlBase::getAttributeValue(inputNode, "default"),
            excludeFlag);
    }
    else {
        for (auto childInputNode = inputNode.child("input"); childInputNode; childInputNode = childInputNode.next_sibling("input")) {
            handleInputNode(childInputNode, containerPath, data, level + 1);
        }
    }
}
} // namespace
DcXmlEtlNetwork::DcXmlEtlNetwork()
    : m_qhPriority()
{
    constructXPathHash();
    constructEnumHash();
}
DcXmlEtlNetwork::DcXmlEtlNetwork(const QString& fileName)
    : m_qhPriority()
{
    if (!DcXmlBase::parseFile(fileName)) {
        LogError("[DcXmlEtlNetwork]",
            QString("There was an error reading configuration : %1").arg(fileName));
        throw std::runtime_error("[DcXmlEtlNetwork] parseFile failed");
    }
    constructXPathHash();
    constructEnumHash();
}
DcXmlEtlNetwork::~DcXmlEtlNetwork() = default;
QStringList DcXmlEtlNetwork::getNetworkNames() const
{
    QStringList qsResult;
    QList<pugi::xpath_node> nodeList = DcXmlBase::selectNodes(DcXmlBase::getXPath(selectNetworks));
    for (const auto& nodeItem : nodeList) {
        qsResult.append(DcXmlBase::getAttributeValue(nodeItem.node(), "name"));
    }
    return qsResult;
}
QList<QSharedPointer<ObjectData>> DcXmlEtlNetwork::getObjectsOfNetwork(const QString& networkName) const
{
    QList<QSharedPointer<ObjectData>> lstResult;
    // Create the correct XPath expression. We have the service already...
    QList<QVariant> varList;
    varList.append(networkName);
    // First we get a list of objects
    QList<pugi::xpath_node> objectList = DcXmlBase::selectNodes(DcXmlBase::evaluateXPath(selectObjects, varList));
    for (const auto& objectItem : objectList) {
        auto pData = QSharedPointer<ObjectData>::create();
        pData->setObjectId(DcXmlBase::getAttributeValue(objectItem.node(), "id"));
        pData->setObjectType(DcXmlBase::getAttributeValue(objectItem.node(), "class"));
        //====================================================================================
        // Get some headerinformation based on the objecttype. Some fields are re-used.
        //====================================================================================
        if (pData->getObjectType() == "output") {
            pData->setTargetName(DcXmlBase::getAttributeValue(objectItem.node(), "target"));
            pData->setTargetAction(DcXmlBase::getAttributeValue(objectItem.node(), "default_action"));
            pData->setKeyField(DcXmlBase::getAttributeValue(objectItem.node(), "key_field"));
            pData->setForeignKeyField(DcXmlBase::getAttributeValue(objectItem.node(), "foreignkey_field"));
            auto targetFieldAttr = objectItem.node().attribute("target_field"); // Available in combination with batch/merge update. If not set then targetField is the same as foreignKeyField
            if (targetFieldAttr.empty()) {
                pData->setTargetField(pData->getForeignKeyField());
            }
            else {
                pData->setTargetField(targetFieldAttr.value());
            }
            // Available in combination with merge update. Default is NULL.
            pData->setTargetResetValue(DcXmlBase::getAttributeValue(objectItem.node(), "target_reset_value"));
            auto nonPersistentTimestampUsageAttr = objectItem.node().attribute("use_non_persistent_timestamp");
            if (!nonPersistentTimestampUsageAttr.empty()) {
                pData->setNonPersistentTimestampUsage(getBoolean(nonPersistentTimestampUsageAttr.value()));
            }
            if (pData->getNonPersistentTimestampUsage()) {
                auto nonPersistentTimestampBufferSize = objectItem.node().attribute("nr_of_timestamps");
                if (!nonPersistentTimestampBufferSize.empty()) {
                    bool conversionOk = false;
                    auto nrOfTimestamps = QVariant(nonPersistentTimestampBufferSize.value()).toUInt(&conversionOk);
                    if (conversionOk && nrOfTimestamps > 0) {
                        pData->setNonPersistentTimestampBufferSize(nrOfTimestamps);
                    }
                    else {
                        LogWarning("[DcXmlEtlNetwork]",
                            QString("Invalid nr_of_timestamps value %1, using default buffersize").arg(nonPersistentTimestampBufferSize.value()));
                    }
                }
            }
        }
        if (pData->getObjectType() == "datafilter") {
            pData->setTargetAction(DcXmlBase::getAttributeValue(objectItem.node(), "default_action"));
            pData->setKeyField(DcXmlBase::getAttributeValue(objectItem.node(), "key_field"));
            pData->setKeyValue(DcXmlBase::getAttributeValue(objectItem.node(), "key_value"));
        }
        //====================================================================================
        // Get all inputs if there are any
        varList.clear();
        varList.append(networkName);
        varList.append(DcXmlBase::getAttributeValue(objectItem.node(), "id"));
        if (pData->getObjectType() == "output") {
            auto outputVarList = varList;
            outputVarList.append("output"); // add the class
            // select all inputs from an output object
            QList<pugi::xpath_node> inputsList = DcXmlBase::selectNodes(DcXmlBase::evaluateXPath(selectInputs, outputVarList));
            // Only one inputs (plural!) node is expected.
            // The for loop is defensive programming
            for (const auto& xpathInputsNode : inputsList) {
                GenerateInputData::generate(xpathInputsNode.node(), *pData);
            }
        }
        else if (pData->getObjectType() == "datafilter") {
            auto datafilterVarList = varList;
            datafilterVarList.append("datafilter"); // add the class
            // select all inputs from an output object
            QList<pugi::xpath_node> inputsList = DcXmlBase::selectNodes(DcXmlBase::evaluateXPath(selectInputs, datafilterVarList));
            // Only one inputs (plural!) node is expected.
            // The for loop is defensive programming
            for (const auto& xpathInputsNode : inputsList) {
                for (auto inputNode = xpathInputsNode.node().child("input"); inputNode; inputNode = inputNode.next_sibling("input")) {
                    pData->setInputData(DcXmlBase::getAttributeValue(inputNode, "name"),
                        DcXmlBase::getAttributeValue(inputNode, "type"),
                        QString{}, // no id
                        DcXmlBase::getAttributeValue(inputNode, "default"),
                        false);
                }
            }
        }
        // And finally all outputs.
        varList.clear();
        varList.append(networkName);
        varList.append(DcXmlBase::getAttributeValue(objectItem.node(), "id"));
        QList<pugi::xpath_node> outputList = DcXmlBase::selectNodes(DcXmlBase::evaluateXPath(selectOutputs, varList));
        for (const auto& outputItem : outputList) {
            pData->setOutputData(DcXmlBase::getAttributeValue(outputItem.node(), "name"),
                DcXmlBase::getAttributeValue(outputItem.node(), "type"),
                DcXmlBase::getAttributeValue(outputItem.node(), "default"));
        }
        // Add the entire objectData to the list...
        lstResult.append(pData);
    }
    return lstResult;
}
Connections DcXmlEtlNetwork::getConnectionsOfNetwork(const QString& networkName) const
{
    Connections cResult;
    // Create the variable list and create the correct XPath expression.
    QList<QVariant> varList;
    varList.append(networkName);
    // Get the list of connections..
    QList<pugi::xpath_node> connectList = DcXmlBase::selectNodes(DcXmlBase::evaluateXPath(selectConnections, varList));
    for (const auto& connectItem : connectList) {
        const auto node = connectItem.node();
        cResult.addConnection(DcXmlBase::getAttributeValue(node, "source"),
            DcXmlBase::getAttributeValue(node, "target"),
            DcXmlBase::getAttributeValue(node, "output"),
            DcXmlBase::getAttributeValue(node, "input"));
    }
    return cResult;
}
QStringList DcXmlEtlNetwork::getInputVariables(const QString& networkName) const
{
    QStringList qlResult;
    // Create the variable list and return the correct XPath expression.
    QList<QVariant> varList;
    varList.append(networkName);
    QList<pugi::xpath_node> inputList = DcXmlBase::selectNodes(DcXmlBase::evaluateXPath(selectInputVarNames, varList));
    for (const auto& inputItem : inputList) {
        qlResult.append(DcXmlBase::getAttributeValue(inputItem.node(), "name"));
    }
    return qlResult;
}
QStringList DcXmlEtlNetwork::getOutputVariables(const QString& networkName) const
{
    QStringList qlResult;
    // Create the variable list and return the correct XPath expression.
    QList<QVariant> varList;
    varList.append(networkName);
    QList<pugi::xpath_node> outputList = DcXmlBase::selectNodes(DcXmlBase::evaluateXPath(selectOutputVarNames, varList));
    for (const auto& outputItem : outputList) {
        qlResult.append(DcXmlBase::getAttributeValue(outputItem.node(), "name"));
    }
    return qlResult;
}
QThread::Priority DcXmlEtlNetwork::getPrioByNetworkName(const QString& networkName) const
{
    QThread::Priority priority = QThread::NormalPriority;
    // Create the variable list and create the correct XPath expression.
    QList<QVariant> varList;
    varList.append(networkName);
    pugi::xpath_node nodeItem = DcXmlBase::selectNode(DcXmlBase::evaluateXPath(selectNetwork, varList));
    if (nodeItem) {
        const auto prio = DcXmlBase::getAttributeValue(nodeItem.node(), "priority");
        priority = m_qhPriority.value(prio, QThread::NormalPriority);
    }
    return priority;
}
void DcXmlEtlNetwork::constructXPathHash()
{
    /* Put all XPath expressions in one hash, this way they are all localized
     * centrally and not scattered around the code.
     */
    // Networks
    DcXmlBase::addXPath(selectNetworks, "//network");
    DcXmlBase::addXPath(selectNetwork, "//network[@name='%1']");
    // Objects
    DcXmlBase::addXPath(selectObjects, "//network[@name='%1']/objects/object");
    DcXmlBase::addXPath(selectObject, "//network[@name='%1']/objects/object[@id='%2']");
    // Inputs
    DcXmlBase::addXPath(selectInputs, "//network[@name='%1']/objects/object[@id='%2' and @class='%3']/inputs");
    DcXmlBase::addXPath(selectInputVarNames, "//network[@name='%1']/objects/object[@class='input']/outputs/output");
    DcXmlBase::addXPath(selectInput, "//network[@name='%1']/objects/object[@id='%2']/inputs/input[@name='%3']");
    // Outputs
    DcXmlBase::addXPath(selectOutputs, "//network[@name='%1']/objects/object[@id='%2']/outputs/output");
    DcXmlBase::addXPath(selectOutputVarNames, "//network[@name='%1']/objects/object[@class='output']/inputs/input");
    DcXmlBase::addXPath(selectOutput, "//network[@name='%1']/objects/object[@id='%2']/outputs/output[@name='%3']");
    // Connections
    DcXmlBase::addXPath(selectConnections, "//network[@name='%1']/connections/connection");
}
void DcXmlEtlNetwork::constructEnumHash()
{
    m_qhPriority.insert(prio_idle, QThread::IdlePriority);
    m_qhPriority.insert(prio_lowest, QThread::LowestPriority);
    m_qhPriority.insert(prio_low, QThread::LowPriority);
    m_qhPriority.insert(prio_normal, QThread::NormalPriority);
    m_qhPriority.insert(prio_high, QThread::HighPriority);
    m_qhPriority.insert(prio_highest, QThread::HighestPriority);
    m_qhPriority.insert(prio_time_critical, QThread::TimeCriticalPriority);
    m_qhPriority.insert(prio_inherit, QThread::InheritPriority);
}