/* **************************************************************************** * Copyright 2019 Open Systems Development BV * * * * Permission is hereby granted, free of charge, to any person obtaining a * * copy of this software and associated documentation files (the "Software"), * * to deal in the Software without restriction, including without limitation * * the rights to use, copy, modify, merge, publish, distribute, sublicense, * * and/or sell copies of the Software, and to permit persons to whom the * * Software is furnished to do so, subject to the following conditions: * * * * The above copyright notice and this permission notice shall be included in * * all copies or substantial portions of the Software. * * * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * * DEALINGS IN THE SOFTWARE. * * ***************************************************************************/ #include "ormdata.h" #include "ormtable.h" #include "ormhandler.h" #include "log.h" #include "multisortfilterproxymodel.h" #include #include #include #include #include #include using namespace osdev::components; OrmTable::OrmTable(const DbConnector &db, QObject *_parent) : QSqlRelationalTableModel( _parent, db.getDatabase() ) , m_className( "OrmTable" ) , m_recTrackField() , m_db( db ) , m_qhRelTables() , m_qhRelations() , m_qhFieldTypes() , m_qhTimelines() { if ( !QSqlRelationalTableModel::database().isOpen() ) { LogWarning( m_className, "Database is closed."); } } OrmTable::~OrmTable() { } void OrmTable::setTable( const QString& _table ) { // Set tablename in the base class QSqlRelationalTableModel::setTable( m_db.quoteTableName( _table ) ); m_className += "::" + this->tableName(); // Make every change being written to the database. QSqlRelationalTableModel::setEditStrategy( QSqlTableModel::OnRowChange ); // Now we know the tablename, we can retrieve the fieldtypes by their name. To prevent a functioncall // For each and every fieldname we're looking for, we store this in an internal Hash. Must be quicker than // then accessing the database layer. m_qhFieldTypes = m_db.getFieldNamesAndTypes( _table ); QSqlRecord _headerData = QSqlRelationalTableModel::database().record( m_db.quoteTableName( _table ) ); for( int nCount = 0; nCount < _headerData.count(); nCount++ ) { QSqlRelationalTableModel::setHeaderData( nCount, Qt::Horizontal, _headerData.fieldName(nCount) ); } // Now the table is set, headerinfo loaded and commit behaviour set : // Open the table and read from the database. QSqlRelationalTableModel::select(); } bool OrmTable::insertRecord( int _row, const QSqlRecord &_record ) { if ( QSqlRelationalTableModel::database().isOpen() ) { LogDebug( m_className, "Database is open." ); } else { LogWarning( m_className, "Database is closed." ); } bool b_result = QSqlRelationalTableModel::insertRecord( _row, _record ); if ( !b_result ) { LogError( QString( "[OrmTable - %1]" ).arg( QSqlRelationalTableModel::tableName() ), QString( "Database gave error : %1") .arg( this->lastError().text() ) ); } return b_result; } bool OrmTable::insertRecords(int row, const QList& _records ) { bool l_bResult = true; QSqlRelationalTableModel::setEditStrategy( QSqlTableModel::OnRowChange ); for( const QSqlRecord& _record : _records ) { l_bResult &= insertRecord( row, _record ); } QSqlRelationalTableModel::submitAll(); // Reset the transaction method. QSqlRelationalTableModel::setEditStrategy( QSqlTableModel::OnRowChange ); return l_bResult; } void OrmTable::writeData( const QSharedPointer& dataObject ) { // Reset the filter for this table before doing anything else this->resetFilter(); // Relations can be set on different fields. We have to take in account those // fields could be given in a list. ORMData* pMainTable = this->solveRelations( dataObject ); if( nullptr == pMainTable || pMainTable->getContainerName().isEmpty() ) { LogDebug( "[OrmTable::writeData]", QString( "No maintable defined. Exiting..." ) ); emit signalRejectedData( dataObject ); return; } bool updateSuccess = false; /// Here we check for the variable list. See if one of the variables is of type : /// QVariant( QListhasLists() ) { /// @todo: This is kind of a special case. For now we assume there is an inclusive /// and exclusive way of updating these records. All these updates should be done /// in a single transaction. The 'not-mentioned records' should be de-coupled. /// If those fail, the entire package should be rejected, including the ones that were /// mentioned. Default behaviour is "Each - change - a - transaction" so we need some /// extra precautions here. // Check if all items in the list exist in the database. If not the package is rejected. if (!m_db.recordsExist(pMainTable->getContainerName(), pMainTable->getKeyField(), pMainTable->getValue( pMainTable->getKeyField() ).toList())) { LogInfo( "[OrmTable::writeData]", "Not all records are present. Package is rejected." ); emit signalRejectedData( dataObject ); return; } // Set the TransactionScopeGuard. If it goes out-of-scope, for whatever reason, // it will do a Rollback if a transaction was started. OrmTransGuard l_scopeGuard; // Check to see if the database supports transactions if( m_db.supportTransactions() ) { if( !m_db.transactionBegin() ) { LogDebug( "[OrmTable::writeData]", "Transaction failed to start." ); emit signalRejectedData( dataObject ); return; } l_scopeGuard = OrmTransGuard( m_db ); } std::unique_ptr mainTableCopy(pMainTable->clone()); updateSuccess = batchUpdate( mainTableCopy.get(), dataObject->getSourceId() ); if( !updateSuccess ) { emit signalRejectedData( dataObject ); return; } if( ( static_cast(TableActions::actionMergeUpdate) == mainTableCopy->getDefaultAction() ) && ( !mainTableCopy->isMergeable() ) ) // If there is more than one key besides the keyfield, we cannot handle this (yet) { LogWarning( "[OrmTable::writeData]", QString( "Dataobject (ORMData) is not fit for mergeUpdate and is discarded : %1" ).arg( mainTableCopy->asString() ) ); return; } if( ( static_cast(TableActions::actionMergeUpdate) == mainTableCopy->getDefaultAction() ) && ( mainTableCopy->isMergeable() ) ) { deCoupleRecords( mainTableCopy.get() ); } else if (static_cast(TableActions::actionUpdate) == mainTableCopy->getDefaultAction()) { LogDebug( "[OrmTable::writeData]", QString( "Action is a \"normal\" update : %1" ).arg( mainTableCopy->asString() ) ); } if( !this->commit() ) { emit signalRejectedData( dataObject ); } auto timeline = m_qhTimelines[dataObject->getSourceId()]; if (timeline) { timeline->commit(); } return; } else { if( static_cast(TableActions::actionUpdate) == pMainTable->getDefaultAction() ) { // =================================================================================== // == S I N G L E R E C O R D U P D A T E == // =================================================================================== // See if we can actually update this record by checking its timestamp validity. // if( m_recTrackField.isEmpty() || m_recTrackField.isNull() ) { updateSuccess = this->updateRecord( pMainTable ); } else { if( this->isPackageValid( pMainTable ) ) { pMainTable->setValue( m_recTrackField, QVariant( pMainTable->timeStamp() ) ); updateSuccess = this->updateRecord( pMainTable ); } else { // Nope. Package is older. Discard. LogInfo( "[OrmTable::writeData]", QString( "Discarding package because it is older than information in the database" ) ); return; } } } if( updateSuccess ) { return; } } // =================================================================================== // == I N S E R T == // =================================================================================== // If a timestamp is used in the database, add it to the table definition. if( !m_recTrackField.isEmpty() || !m_recTrackField.isNull() ) { pMainTable->setValue( m_recTrackField, pMainTable->timeStamp() ); } // If we get to here, we're ready to create the new SQLRecord and add it to the table. QSqlRecord newRecord = this->buildSqlRecord( pMainTable ); if( !newRecord.isEmpty() ) { if( !m_db.createRecord( newRecord, this->tableName() ) ) { // @todo : Change to a more database dependant, generic error checking based on keywords.... if( this->lastError().text().contains( "violates unique constraint" ) || this->lastError().text().contains( "violates not-null constraint" ) ) { // Just drop the package and be done with it. This is a unique constraint violation LogDebug( "[OrmTable::writeData]", QString( "There was and error creating the record : %1" ).arg( this->lastError().text() ) ); return; } else { LogDebug( "[OrmTable::writeData]", QString( "There was and error creating the record : %1" ).arg( this->lastError().text() ) ); LogDebug( "[OrmTable::writeData]", "Deferring the datapackage to the TransQueue." ); emit signalRejectedData( dataObject ); } } else { return; } } else { LogDebug( "[OrmTable::writeData]", QString( "Creating the new record failed.! : %1" ).arg( this->lastError().text() ) ); } } bool OrmTable::updateRecord( ORMData* dataObject ) { auto lNumRecords = tableFilter( dataObject ); if( lNumRecords > 1 ) { LogError( "[OrmTable::updateRecord]", QString( "Multiple records found..." ) ); return false; } else if( lNumRecords == 0 ) { LogWarning( "[OrmTable::updateRecord]", QString( "No record found...Falling back to insert...." ) ); return false; } else { // All criteria are met.. Time to update the record. // Each and and every field will be written, with the exception of the keyfield.. // First get a list of all field names. bool updateSuccess = true; QStringList fieldNames = dataObject->getFieldNames(); for( const QString& fieldName : fieldNames ) { if( fieldName != dataObject->getKeyField() ) { int colIndex = this->fieldIndex( fieldName ); if( -1 == colIndex ) { LogError( "[OrmTable::updateRecord]", QString( "No index found for : %1" ).arg( fieldName ) ); } else { LogDebug( "[OrmTable::updateRecord]", QString( "Updating field : %1 (Index : %2) with value : %3" ) .arg( fieldName ) .arg( colIndex ) .arg( dataObject->getValue( fieldName ).toString() ) ); this->setData( this->index( 0, colIndex ), ConversionUtils::convertToType(dataObject->getValue( fieldName ) ,m_qhFieldTypes.value( fieldName )), Qt::EditRole ); } } else { LogDebug( "[OrmTable::updateRecord]", QString( "Skipping keyfield %1 ..." ).arg( fieldName ) ); } } // Process all changes. updateSuccess = this->submit(); if( !updateSuccess ) { LogError( "[OrmTable::updateRecord]", QString( "Record failed to update, Database gave error : %1").arg( this->lastError().text() ) ); } this->setFilter( "" ); this->select(); return updateSuccess; } } bool OrmTable::batchUpdate( ORMData* dataObject, const QUuid& sourceId ) { // If MergedUpdate = true, we have to do an ex- and inclusive update. // (Inclusive meaning everyting that is mentioned, Exclusive : everything that is not in range..) // Similar to WHERE NOT IN ( , , ) QString fieldName; // Get all fields to update, without the keyfield. // We only support a single field update at this moment. for( const auto& fn : dataObject->getFieldNames() ) { if( fn != dataObject->getKeyField() ) { fieldName = fn; break; } } auto timeline = m_qhTimelines[sourceId]; if (!timeline) { auto pTimeline = QSharedPointer::create(); timeline.swap(pTimeline); m_qhTimelines[sourceId] = timeline; } OrmBatchChange obc(Timestamp(dataObject->timeStamp()), dataObject->getValue( fieldName ), OrmBatchChange::toSet(dataObject->getValue( dataObject->getKeyField() ).toList())); auto proposed = timeline->evaluate(obc, static_cast(TableActions::actionMergeUpdate) == dataObject->getDefaultAction()); if (!proposed.valid()) { LogInfo( "[OrmTable::batchUpdate]", "Proposed update will not be executed" ); return false; } dataObject->setValue( dataObject->getKeyField(), OrmBatchChange::toList(proposed.changeSet())); QString strTable = dataObject->getContainerName(); return m_db.associate(strTable, dataObject->getKeyField(), dataObject->getValue( dataObject->getKeyField() ).toList(), fieldName, dataObject->getValue( fieldName )); } bool OrmTable::deCoupleRecords( ORMData* dataObject ) { QString fieldName; // Get all fields to update, without the keyfield. // We only support a single field update at this moment. for( const auto& fn : dataObject->getFieldNames() ) { if( fn != dataObject->getKeyField() ) { fieldName = fn; break; } } QString strTable = dataObject->getContainerName(); return m_db.disassociate(strTable, dataObject->getKeyField(), dataObject->getValue( dataObject->getKeyField() ).toList(), fieldName, dataObject->getValue( fieldName )); } QList OrmTable::readData( const QString& fieldName, const QString& filterExp ) { int columnIndex = this->fieldIndex( fieldName ); QSortFilterProxyModel filterModel; filterModel.setSourceModel( this ); if( columnIndex > -1 ) { filterModel.setFilterKeyColumn( columnIndex ); filterModel.setFilterRegExp( QRegExp( filterExp, Qt::CaseSensitive, QRegExp::Wildcard ) ); } if( filterModel.rowCount() > 0 ) { // Looks like there was a result. Pack into a structure.. int rows = filterModel.rowCount(); int columns = filterModel.columnCount(); // For each row and column, iterate the data structure.. for( int _rowCount = 0; _rowCount < rows; _rowCount++ ) { qDebug() << "=____"; for( int _colCount = 0; _colCount < columns; _colCount++ ) { qDebug() << filterModel.headerData( _colCount, Qt::Horizontal, Qt::DisplayRole ) << " : " << filterModel.data( filterModel.index( _rowCount, _colCount ), Qt::DisplayRole ); } } } return QList(); } QList OrmTable::getRelationsByTableName( const QString& _tableName ) { QList lstResult; for( auto relName : m_qhRelations.keys() ) { if( m_qhRelations.value( relName )->foreignTable() == _tableName ) { lstResult.append( m_qhRelations.value( relName ) ); } } return lstResult; } QSqlRecord OrmTable::buildSqlRecord( ORMData* tableObject ) { LogDebug( "[OrmTable::buildSqlRecord]", QString( "Building record with : %1 " ).arg( tableObject->asString() ) ); QStringList fieldNames = tableObject->getFieldNames(); QSqlRecord sqlRecord; for( const QString& fieldName : fieldNames ) { if( fieldName != "container" ) { QSqlField sqlField( fieldName, m_qhFieldTypes.value( fieldName ) ); sqlField.setValue( ConversionUtils::convertToType( tableObject->getValue( fieldName ), // The value m_qhFieldTypes.value( fieldName ) ) ); // The Type sqlRecord.append( sqlField ); } } return sqlRecord; } void OrmTable::setRelatedTable(const QString& _tableName, QSortFilterProxyModel* pTable) { m_qhRelTables.insert( _tableName, pTable ); } QSortFilterProxyModel* OrmTable::getRelatedTable(const QString& _tableName) const { return m_qhRelTables.value( _tableName ); } void OrmTable::saveRelation( const QString& relationName, DbRelation* pRelation ) { LogDebug( "[OrmTable::saveRelation]", QString( "Saving Relation : %1" ).arg( relationName ) ); m_qhRelations.insert( relationName, pRelation ); } DbRelation* OrmTable::getRelation( const QString& relationName ) const { return m_qhRelations.value( relationName, nullptr ); } QStringList OrmTable::getRelationNames() const { return QStringList( m_qhRelations.keys() ); } ORMData* OrmTable::solveRelations( const QSharedPointer& dataObject ) { // First we get the Maintable.... This will be our return object. ORMData* pMainTable = dataObject->getMainTableContainer(); if( nullptr == pMainTable ) { return nullptr; } if( pMainTable->hasLists() ) { LogWarning( "OrmTable::solveRelations", QString( "Maintable has Lists.... : %1" ).arg( pMainTable->asString() ) ); } QStringList constraintList = this->getRelationNames(); for( auto constraint : constraintList ) { const DbRelation* l_relation = QPointer( this->getRelation( constraint ) ); QString foreignPrimKey; QString foreignFieldName; QVariant foreignFieldValue; LogDebug("[OrmTable::solveRelations]", QString("Get related table %1").arg(l_relation->foreignTable())); ORMData* pTable = dataObject->getRelatedContainer( l_relation->foreignTable() ); if( nullptr != pTable ) { // We assume there is only 1 relation between two tables int numFields = pTable->getFieldCount(); if( 0 == numFields ) { LogWarning( "[OrmTable::solveRelations]", QString( "No relations found to table : %1 " ).arg( l_relation->foreignTable() ) ); return nullptr; } else if( 1 == numFields ) { foreignPrimKey = l_relation->foreignPrimKey(); foreignFieldName = pTable->getFieldNames().first(); // =========================================================================================== // L I S T O R S I N G L E V A L U E R E L A T I O N // =========================================================================================== foreignFieldValue = pTable->getValue( foreignFieldName ); if( foreignFieldValue.isNull() ) { // Some constraints are allowed to be absent. For now treat this constraint as such. // If it is needed afterall it will prevent the write in a later stage pMainTable->setValue( l_relation->indexColumn(), QVariant() ); continue; } // Looks like we have all data collected. Retrieve the Primary key value needed. // If foreignPrimKey is empty it will be retrieved from the database meta data. QVariant l_primKeyValue = m_db.getRelationKey( l_relation->foreignTable(), foreignFieldName, foreignFieldValue, foreignPrimKey ); if( !l_primKeyValue.isNull() ) { // We have a value of the IndexColumn. Complete the mainTable.... pMainTable->setValue( l_relation->indexColumn(), l_primKeyValue ); } else { LogError( "[OrmTable::solveRelations]", QString( "No primary key found for table : %1 " ).arg( l_relation->foreignTable() ) ); return nullptr; } // =========================================================================================== } else { /// @todo : At this moment we don't handle multiple relations. Lotsplitting is requiring this, so it must be implemented as soon as /// the requirements are defined. LogError( "[OrmTable::solveRelations]", QString( "multiple relations found to table : %1 " ).arg( l_relation->foreignTable() ) ); return nullptr; } } else { LogDebug( "[OrmTable::solveRelations]", QString( "Unresolved constraint %1. We assume all data is provided by the object.." ).arg( constraint ) ); } } return pMainTable; } int OrmTable::tableFilter( ORMData* dataObject ) { int nResult = -1; // Check for a valid pointer. if( nullptr == dataObject ) { LogError( "[OrmTable::tableFilter]", QString( "No data structure given." ) ); return 0; } // Check if update is the default action if( static_cast(TableActions::actionUpdate) != dataObject->getDefaultAction() ) { LogWarning( "[OrmTable::tableFilter]", QString( "Update is not the default action for table : %1" ).arg( dataObject->getContainerName() ) ); return 0; } /// @todo : Create an AND / OR filter. // +----------------------------------+ // | Keyfield | KeyValue | TypeFilter | // +----------+----------+------------+ // | = 1 | = 1 | equals | // | = 1 | > 1 | OR | ( WHERE IN ( a, b, c, d...) ) // | > x | > y | AND | ( x equals y) // | <> KV | <> KF | INVALID | // +----------+----------+------------+ // // Check if there is a record that matches our criteria. const auto keyFields = dataObject->getKeyFieldList(); for (const auto& field : keyFields) { if( this->fieldIndex(field) == -1 ) { LogWarning( "[OrmTable::tableFilter]", QString( "No columnIndex found for : %1" ).arg(field) ); return 0; } } QString strFilter; const auto numberOfKeys = keyFields.length(); if (0 == numberOfKeys) { LogWarning( "[OrmTable::tableFilter]", QString( "No keyField given for table : %1" ).arg( dataObject->getContainerName() ) ); return 0; } else if (1 == numberOfKeys) { const auto& keyField = *keyFields.begin(); if( dataObject->getValue( keyField ).type() == QVariant::List ) // OR { QList tmpList = dataObject->getValue( keyField ).toList(); for( int lstCount = 0; lstCount < tmpList.count(); lstCount++ ) { if( tmpList.at(lstCount).type() == QVariant::Int ) { strFilter += keyField + "=" + tmpList.at( lstCount ).toString(); } else { strFilter += "\"" + keyField + "\"='" + tmpList.at( lstCount ).toString().replace("{","").replace("}","") + "'"; } if( ( lstCount + 1 ) < tmpList.count() ) { strFilter += " OR "; } } } else // equals { if( dataObject->getValue( keyField ).type() == QVariant::Int ) { strFilter = "\"" + keyField + "\"="+ dataObject->getValue( keyField ).toString(); } else { strFilter = "\"" + keyField + "\"='" + dataObject->getValue( keyField ).toString().replace("{","").replace("}","") + "'"; } } } else // numberOfKeys > 1, AND { // ========================================================= // @todo: Setup a multiple fields filter. For now it is just a multiple values value filter. auto iter = keyFields.constBegin(); while (keyFields.constEnd() != iter) { if( dataObject->getValue( *iter ).type() == QVariant::Int ) { strFilter += "\"" + *iter + "\"=" + dataObject->getValue( *iter ).toString(); } else { strFilter += "\"" + *iter + "\"='" + dataObject->getValue( *iter ).toString().replace("{","").replace("}","") + "'"; } ++iter; if( keyFields.end() != iter ) { strFilter += " AND "; } } } this->setFilter( strFilter ); LogDebug( "[OrmTable::tableFilter]", QString( "Filter : %1" ).arg( this->filter() ) ); if( this->select() ) { nResult = this->rowCount(); } else { LogError( "[OrmTable::tableFilter]", QString( "There was a problem executing the query : %1" ).arg( this->selectStatement() ) ); } // ========================================================= return nResult; } int OrmTable::resetFilter() { this->setFilter( "" ); this->select(); return rowCount(); } bool OrmTable::isPackageValid( ORMData* dataObject ) { if( nullptr == dataObject ) { LogError( "[OrmTable::isPackageValid]", "No ORMData dataobject passed." ); return false; } // Determine the field indices for the proxy filter auto keyFields = dataObject->getKeyFieldList(); QList fieldIndices; auto iter = keyFields.begin(); while (keyFields.end() != iter) { auto idx = this->fieldIndex(*iter); if (idx > -1) { fieldIndices.push_back(idx); ++iter; } else { LogWarning("[OrmTable::isPackageValid]", QString("Keyfield \"%1\" is unknown. Not using it in filter.").arg(*iter)); iter = keyFields.erase(iter); } } // Create the filter model and set its filter criteria MultiSortFilterProxyModel oModelFilter; oModelFilter.setFilterKeyColumns( fieldIndices ); // Build the filter expression. for (qint32 i = 0; i < fieldIndices.size(); ++i) { oModelFilter.setFilterFixedString( fieldIndices[i], QString( "%1" ).arg( dataObject->getValue( keyFields[i] ).toString() ) ); } // Attach filter to this table model. oModelFilter.setSourceModel( this ); // If the number of record = 1, we seem to have found our record. if( 0 == oModelFilter.rowCount() ) { // Seems like we are good to go. // This is a first timer... LogDebug("[OrmTable::isPackageValid]", "No record found"); return true; } else if ( 1 == oModelFilter.rowCount() ) { QModelIndex modIndex = oModelFilter.index( 0, this->fieldIndex( m_recTrackField ) ); unsigned long long tmp_timestamp = oModelFilter.data( modIndex, Qt::DisplayRole ).toULongLong(); if( dataObject->timeStamp() > tmp_timestamp ) { return true; } } else { LogError( "[OrmTable::isPackageValid]", QString( "Multiple records found (%1) for %2 with value %3" ) .arg( oModelFilter.rowCount() ) .arg( dataObject->getKeyField() ) .arg( dataObject->getValue( dataObject->getKeyField() ).toString() ) ); } return false; } bool OrmTable::commit() { if( m_db.supportTransactions() ) { if( m_db.transactionCommit() ) { return true; } else { LogWarning( "[OrmTable::commit]", QString( "Transaction failed to commit with error : %1" ) .arg( m_db.getDatabase().driver()->lastError().text() ) ); } } else { if( m_db.getDatabase().driver()->lastError().driverText().isEmpty() ) { return true; } else { LogDebug( "[OrmTable::commit]", QString( "Transaction not supported but last query failed with error : %1" ) .arg( m_db.getDatabase().driver()->lastError().driverText() ) ); } } return false; }