Table.hxx

Go to the documentation of this file.

//-----------------------------------------------------------------------------
// CVS: $Revision: 7503 $, $Date: 2010-09-07 16:34:15 +0100 (Tue, 07 Sep 2010) $, $Author: EckhardSutorius $
//-----------------------------------------------------------------------------
#include <cstdlib>
#include <iostream>
#include <string>
#include <limits>

#include "Options.h"
#include "Constants.h"
#include "Logger.h"
#include "MyException.h"
#include "SchemaException.h"
#include "StringOps.h"
//-----------------------------------------------------------------------------
template<typename DataType>
void Table<DataType>::checkData(bool aThrowExceptions)
{
  using namespace std;
  mChecked = true;
  Options options;
  Logger log;

  // Loop over all columns in the table
  for (unsigned colNo = 0; colNo < mDataPtr->qNumCols(); ++colNo)
  {
    // Is bigint really a number?
    string format = mInfoPtr->getFormat(colNo);
    if (format == "bigint")
    {
      bigintTest(*mDataPtr, colNo);
    }

    // Determine attribute varchar / datetime properties
    unsigned varcharLen = 0;
    if (format.find("varchar") != string::npos)
    {
      int posa = format.find("(") + 1;
      int posb = format.rfind(")");
      varcharLen = atoi(format.substr(posa, posb - posa).c_str());
    }
    bool isVarchar = (varcharLen > 0);
    bool isDatetime = (format == "datetime");

    // Counter for number of entries in this column that are set to default
    int numDefaults = 0;

    // Is there a default value for this column?
    string defValue = mInfoPtr->getDefault(colNo);
    bool existsDefault = !defValue.empty();

    // Check each row for this column
    for (unsigned rowNo = 0; rowNo < mDataPtr->qNumRows(); ++rowNo)
    {
      // Is the value undefined for this column and row?
      if (!mDataPtr->isDefined(colNo, rowNo))
      {
        // Assign default for this column if a default value is given
        // in the schema.
        if (existsDefault)
        {
          mDataPtr->assign(colNo, rowNo, defValue);
          ++numDefaults;
        }
        else
        // If no default is specified, then something is wrong.
        // check flag to not ingest this data set
        // (Throw an exception if requested)
        {
          string mess = mInfoPtr->getTableName();
          mess += ": Undefined value but no default given. Assigning: ";
          mess += mInfoPtr->getName(colNo);
          mess += " for row " + StringOps::NumToString(rowNo);
          log.addError(mess);
          mChecked = false;
          if (aThrowExceptions) throw SchemaException(mess);
        }
      }
      else
      {
        if (isDatetime)
        {
          // As well as testing for default, this will add seconds and ending Z
          datetimeTest(*mDataPtr, colNo, rowNo);
        }
        else if (isVarchar)
        {
          // check if varchar format is correct
          varcharTest(*mDataPtr, colNo, rowNo, varcharLen);
        }
        // check if value is Inf
        if (isInf(mDataPtr->at(colNo, rowNo)))
        {
          string mess = mInfoPtr->getTableName() + " ";
          mess += mInfoPtr->getName(colNo) + " ";
          mess += StringOps::NumToString(colNo) + " ";
          mess += StringOps::NumToString(rowNo) + " : ";
          mess += StringOps::NumToString(mDataPtr->at(colNo, rowNo)) + " : ";

          if (StringOps::ToLower(options.getArchive()) == "vsa" or
              mInfoPtr->getTableName().find("Astrometry") != std::string::npos)
          {
            mess += "inf value found, replaced with dbl_max";
            if (format == "real")
            {
              mDataPtr->assign(colNo, rowNo, numeric_limits<float>::max( )-1);
            }
            else if (format == "float")
            {
              mDataPtr->assign(colNo, rowNo, numeric_limits<double>::max( )-1);
            }
            else
            {
              mHasNaNs = true;
              if (aThrowExceptions) throw MyException(mess);
            }
          } else {
            mess += "NaN value found";
            mHasNaNs = true;
            if (aThrowExceptions) throw MyException(mess);
          }
          log.addError(mess);
        }
        // check if value is not NaN
        else if (isNaN(mDataPtr->at(colNo, rowNo)))
        {
          string mess = mInfoPtr->getTableName() + " ";
          mess += mInfoPtr->getName(colNo) + " ";
          mess += StringOps::NumToString(colNo) + " ";
          mess += StringOps::NumToString(rowNo) + " : ";
          mess += StringOps::NumToString(mDataPtr->at(colNo, rowNo)) + " : ";
          mess += "NaN value found";
          log.addError(mess);
          mHasNaNs = true;
          if (aThrowExceptions) throw MyException(mess);
        }
      }
    } // end of loop over rows

    // If a default was used for data in this column, log a warning
    if (existsDefault && numDefaults > 0)
    {
      string mess = mInfoPtr->getTableName() + " ";
      mess += mInfoPtr->getName(colNo) + " : ";
      mess += " used default value of ";
      mess += defValue;
      mess += " on " + StringOps::NumToString(numDefaults) + " rows";
      log.addWarning(mess);
    }
  } // end of loop over columns
} // checkData
//-----------------------------------------------------------------------------
template<typename DataType>
void Table<DataType>::bigintTest(TableData<Metadata>& aData, int aCol)
{
  if (aData.isDefined(aCol, 0) &&
      aData.at(aCol, 0).find_first_not_of("1234567890.") != std::string::npos)
  {
    aData.setundef(aCol, 0);
  }
}
//-----------------------------------------------------------------------------
template<typename DataType>
void Table<DataType>::varcharTest(TableData<Metadata>& aData,
                                  int aCol,
                                  int aRow,
                                  unsigned aLength)
{
  Metadata value = aData.at(aCol, aRow);

  if (value.size() > aLength)
  {
    aData.assign(aCol, aRow,
                 value.substr(0, aLength));

    std::string mess = mInfoPtr->getTableName() + " ";
    mess += mInfoPtr->getName(aCol) + " ";
    mess += StringOps::NumToString(aCol) + " ";
    mess += StringOps::NumToString(aRow) + " : ";
    mess += "String too long for " + mInfoPtr->getFormat(aCol) + ", ";
    mess += value + " truncated";
    Logger log;
    log.addWarning(mess);
  }
} // varcharTest
//-----------------------------------------------------------------------------
template<typename DataType>
void Table<DataType>::datetimeTest(TableData<Metadata>& aData,
                                   int aCol,
                                   int aRow)
{
  Metadata value = aData.at(aCol, aRow);

  if (value[0] == '0')
  {
    aData.assign(aCol, aRow, DefaultValues::datetimedef);
  }
  else
  {
    std::string::size_type colonIndex = value.find(':');
    if (colonIndex != std::string::npos &&
        value.find(':', colonIndex+1) == std::string::npos)
    {
      aData.assign(aCol, aRow, value.append(":00"));
    }
  }
}
//-----------------------------------------------------------------------------
template<typename DataType>
bool Table<DataType>::checkRange()
{
  using namespace std;
  Logger log;
  bool checked = true;
  bool isunique = true;
  string messUnique;
  string extNum, oldExtNum;
  extNum = "0";
  oldExtNum = "0";
  unsigned counter = 0;

  // Loop over all columns in the table
  for (unsigned colNo = 0; colNo < mDataPtr->qNumCols(); ++colNo)
  {
    // Are there range values for this column?
    string rangeValue = mInfoPtr->getRange(colNo);
    bool isRange = !rangeValue.empty();

    // Are there discrete values for this column?
    string discrValue = mInfoPtr->getDiscrValues(colNo);
    bool isDiscr = !discrValue.empty();

    // Is it the seqNum column in Raw table?
    bool isSeqNum = (mInfoPtr->getName(colNo) == ExpectNames::seqNum);
    //&& mInfoPtr->getTableName().find("Raw") != std::string::npos);

    // Check each row for this column
    unsigned maxRows = mDataPtr -> qNumRows();
    if (mInfoPtr->isDetection())
    {
      maxRows -= mDataPtr -> qNumExts();
    }

    for (unsigned rowNo = 0; rowNo < maxRows; ++rowNo)
    {
      // Check Range of Values
      // in the schema.
      bool isgdValue=true;
      if (isRange && mDataPtr->isDefined(colNo, rowNo))
      {
        StringMap range;
        StringOps::split(rangeValue, ',', range);
        isgdValue = mDataPtr->testRange(colNo, rowNo, range);
      }
      else if (isDiscr && mDataPtr->isDefined(colNo, rowNo))
      {
        StringMap values;
        StringOps::split(discrValue, ',', values);
        isgdValue = mDataPtr->testDiscrValues(colNo, rowNo, values);
      }
      else if (isSeqNum)
      {
        string sqnval;
        mDataPtr->value(colNo, rowNo, sqnval);
        oldExtNum = extNum;
        mDataPtr->value(mInfoPtr->getAttNo(ExpectNames::extNum), rowNo, extNum);
        if (extNum != oldExtNum)
        {
          counter = 0;
        }
        counter += 1;
        if (atoi(sqnval.c_str()) > 0)
        {
          isgdValue = (counter == (unsigned)atoi(sqnval.c_str()));
        }
      }
      else
      {
        isgdValue = true;
      }
      // if value outside range, then something is wrong. Throw
      // an exception
      if (!isgdValue && isRange)  // Bad value in range check
      {
        string actval;
        mDataPtr->value(colNo, rowNo, actval);
        string mess = mInfoPtr->getTableName() + " ";
        mess += mInfoPtr->getName(colNo) + " ";
        mess += "Range = " + rangeValue + " ";
        mess += "Value = "+ actval +" : ";
        mess += "Value outside logical range";
        log.addError(mess);
        checked = false;
      }
      else if (!isgdValue && isDiscr)  // Bad value in value check
      {
        string actval;
        mDataPtr->value(colNo, rowNo, actval);
        string mess = mInfoPtr->getTableName() + " ";
        mess += mInfoPtr->getName(colNo) + " ";
        mess += "Discrete Values : " + discrValue + " ";
        mess += "Value = " + actval+ " : ";
        mess += "Value outside logical range";
        log.addError(mess);
        checked = false;
      }
      else if (!isgdValue && isSeqNum)
      {
        messUnique = mInfoPtr->getTableName() + " ";
        messUnique += extNum + ",";
        messUnique += mInfoPtr->getName(colNo) + " ";
        messUnique += "not linear/unique";
        //log.addError(mess);
        checked = false;
        isunique = false;
      }
    } // end of loop over rows
  } // end of loop over columns
  if (!checked and !isunique)
  {
    log.addError(messUnique);
  }
  // Add message to log to confirm that range checking is complete
  string mess = mInfoPtr->getTableName() + " ";
  mess += " Range checking complete ";
  log.addMessage(mess);
  return checked;
}
//-----------------------------------------------------------------------------
template<typename DataType>
void Table<DataType>::testDump()
{
  using namespace std;
  Logger log;
  log.addMessage("Dumping data for table: " + mInfoPtr->getTableName());

  int numRows = mDataPtr->qNumRows();
  int numCols = mDataPtr->qNumCols();

  if (numRows > 10) numRows = 10;

  for (int colNo = 0; colNo < numCols; ++colNo)
  {
    cout << mInfoPtr->getName(colNo) << " = ";

    for (int rowNo = 0; rowNo < numRows; ++rowNo)
    {
      DataType value = mDataPtr->at(colNo, rowNo);
      cout << value << " ";
    }
    cout << endl;
  }
}
//-----------------------------------------------------------------------------
// Change log:
//
// 11-Mar-2004,  IAB: checkData(): Original version
// 26-Jan-2005,  NJC: checkRange(): Added control to exclude missing data
//                    values as these are dealt with later
//                    (OutputData::checkData);
// 27-Jan-2005,  NJC: checkRange(): Added this header text; changed messaging
//                    to only send a message at the end of a table, not at each
//                    column; added a warning message to the log;
// 28-Apr-2005, ETWS: checkData(): Included NaN check
//  9-Aug-2005, ETWS: checkData(): Included length check for varchars
//  3-Nov-2005,  RSC: checkData(): Exceptions can now be thrown
// 15-Nov-2005,  RSC: Modified i/f with TableData to protect against seg faults
// 13-Mar-2006,  RSC: revamped checkData() to make it faster and more reliable
//                    against false NaNs (previously it was checking whether
//                    uninitialised data was a NaN!)
// 30-Mar-2006, ETWS: fixed bug in DatetimePreTest

---