loader.java

数据仓库工具

/*
  Loader - tool for transfering data from one JDBC source to another and
  doing transformations during copy.
    Copyright (C) 2002-2003  Together
    This library is free software; you can redistribute it and/or
    modify it under the terms of the GNU Lesser General Public
    License as published by the Free Software Foundation; either
    version 2.1 of the License, or (at your option) any later version.
    This library is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    Lesser General Public License for more details.
    You should have received a copy of the GNU Lesser General Public
    License along with this library; if not, write to the Free Software
    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 Loader.java
 Date: 03.03.2003.
 @version 2.1 alpha
 @authors:
 Milosevic Sinisa [e-mail]
 Radoslav Dutina  [e-mail]
 */

package org.webdocwf.util.loader;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.math.BigDecimal;
import java.net.URL;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.StringTokenizer;
import java.util.Vector;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;

import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.webdocwf.util.loader.logging.Logger;
import org.webdocwf.util.loader.logging.StandardLogger;
import org.webdocwf.util.loader.transformation.QueryTransformationSet;
import org.webdocwf.util.loader.transformation.Transformation;
import org.webdocwf.util.loader.transformation.Transformations;
import org.xml.sax.SAXException;

/**
 * <p>
 * Loader class loads data from source database into target database using criteria
 * set in XML file which is used as parameter. Also Loader can execute SQL statements
 * which import from XML file. <br>
 * All loading rolls are set in XML file. Some capabilities of this loader are:
 * there could be more then one importDefinition and number of them is not limited, source
 * and target database could be different, source database could be ASCII,Excel or Access file too,
 * operator (the person or application which starts Loader) have to define.
 * Restart mode with second parameter (optional).
 * </p>
 * <p>
 * There are two possible ways to start Loader. The first is as stand alone application,
 * the sintaxs is: <br>
 * <code><B>java org.webdocwf.util.loader.Loader [-options] urlOfTheXmlFile.xml</B></code> <br>
 * urlOfTheXmlFile.xml is URL of the XML file with loading criteria,<br>
 * </p>
 * <p>
 * The second way is to start Loader as package, the sintax is:<br>
 * <code><B>public Loader(String loadJobFileName, String mode, String userID, String logDirName,
 *	String logFileName, boolean restartIndicator, Map variableValues, String vendorFileName,
 *	boolean onErrorContinue, String additionalPaths, int commitCount, int returnCode);
 * </code><br>
 * ldr.load();</B>
 * <br>
 * If we want to execute Loader class without last for parameters, we must put null as these arguments
 * </p>
 * <p>
 * When the Loader is started the log file is generated. Log file is txt file and its
 * name defines date and time when the Loader is started. There are three Log modes:
 * none - report only begining and finishing the application, and exceptions if there
 * are any, normal - add to log importing blocks and full - with detailed report to
 * operator.
 * </p>
 * <p>
 * Another big adventage of Loader is its possibility to execute any SQL statements in target database.
 * (creating database; creating, modifying and deleting tables...)
 * </p>
 * <p>
 * XML file is the key part for loading criteria. XML tags define how the loading job will
 * be done.
 * </p>
 */
public class Loader {
//-u admin -r -v defaultsprache=DE_AT ObjectLoader\DataCleaningExample.xml
  int iValueColumns;
  Vector vecValueColumnsTargetTables = new Vector();
  //Vector vecTransformColumnsTargetTables = new Vector();
  Vector vecSourceColumnName = new Vector();
  Vector vecTargetColumnName = new Vector();
  Vector vecValueMode = new Vector();
  Vector vecTargetColumnValue = new Vector();
  Vector vecTargetKeyColumnName = new Vector();
  Vector vecVariableUseIDTableName = new Vector();
  Vector vecVariableUseIDTableID = new Vector();
  Vector vecVariableUseIDColumnName = new Vector();
  Vector vecVariableUseIDValueMode = new Vector();
  Vector vecVariableColumnName = new Vector();
  Vector vecVariableColumnTargetTableName = new Vector();
  Vector vecVariableColumnTargetTableID = new Vector();
  Vector vecVariableColumnTargetColumnName = new Vector();
  Vector vecVariableColumnValueMode = new Vector();
  Vector vecVariableColumnTypes = new Vector();
  Vector vecVariableTimesTableName = new Vector();
  Vector vecVariableTimesTableID = new Vector();
  Vector vecVariableTimesColumnName = new Vector();
  Vector vecVariableTimesValueMode = new Vector();
  int iConstantColumns;
  Vector vecConstantTargetColumnName = new Vector();
  Vector vecConstantValueMode = new Vector();
  Vector vecConstantConstantValue = new Vector();
  Vector vecConstantColumnTargetTableName = new Vector();
  Vector vecConstantColumnTargetTableID = new Vector();
  int iRelationColumns;
  Vector vecRelationColumnSourceTableName = new Vector();
  Vector vecRelationColumnSourceTableID = new Vector();
  Vector vecRelationColumnSourceColumnName = new Vector();
  Vector vecRelationColumnTargetTableName = new Vector();
  Vector vecRelationColumnTargetColumnName = new Vector();
  Vector vecRelationColumnTargetTableID = new Vector();
  Vector vecRelationColumnRelationMode = new Vector();
  int iTables;
  Vector vecTableTableName = new Vector();
  Vector vecTableTableID = new Vector();
  Vector vecTableInsert = new Vector();
  Vector vecTableTableMode = new Vector();
  Vector vecTableOidLogic = new Vector();
  Vector vecRelationSourceValue = new Vector();
  Vector vecRelationSourceType = new Vector();
  Vector vecTargetColumnType = new Vector();
  Vector vecConstantColumnType = new Vector();
  BigDecimal bdecOidNumber = new BigDecimal(0);
  BigDecimal bdecOidNumber2000 = new BigDecimal(0);
  String strLoaderJobFileName = "";
  boolean bRestartIndicator;
  Map mapVariableValues = new HashMap();
  boolean bOnErrorContinue;
  String strUserID = "";
  String strLogDirName = "";
  String strLogFileName = "";
  String strVendorFileName = "";
  String strAdditionalPaths = "";
  String confJarStructure = "";

  //interne Sinisa
  boolean hasRestartCounter;
  boolean hasUserID;
  Vector vecRelationKeyColumns = new Vector();
  Vector vecRelationKeyTypes = new Vector();
  Vector vecSortValues = new Vector();

  Vector vecSqlStmt = new Vector();
  String strSourceDriverName = "";
  String strTargetDriverName = "";

  //conf file default
  int iFirstColumnResult = 1;
  int iTargetFirstColumnResult = 1;
  //
  Vector vecVariableName = new Vector();
  Vector vecVariableValue = new Vector();
  Vector vecVariablePrefix = new Vector();
  Vector vecVariableSufix = new Vector();
  Vector vecVariableOverride = new Vector();
  Vector vecReplaceInConstants = new Vector();
  Vector vecReplaceInSQL = new Vector();
  Vector vecReplaceInData = new Vector();
  Vector vecReplaceInJDBC = new Vector();
  public static final String LOGMODE_NONE = "none";
  public static final String LOGMODE_NORMAL = "normal";
  public static final String LOGMODE_FULL = "full";
  ByteArrayOutputStream foStreamTmp = new ByteArrayOutputStream();
  boolean bReplaceInData = false;
  int iColumnsInSourceTable = 0;
  boolean isDefaultJdbc = false;
  private String defaultLogMode = LOGMODE_NORMAL;
  ArrayList logModes = new ArrayList();

  private Cache cacheValues;
//  private EchoElement echo;
  private Logger logger;
  private CounterColumns counterColumns;
  private TimeWatch timeCounter;
  private Transformations transformations;

  //NEW CLASSES
  private JdbcParametersElement jdbcParametersElement = new JdbcParametersElement();
  private SqlElement sqlElement = new SqlElement();
  private ConfigReader configReaderSource = new ConfigReader();
  private ConfigReader configReaderTarget = new ConfigReader();
  private LoaderJobAttrReader loaderJobReader = new LoaderJobAttrReader();
  private ImportDefinitionElement importDefinitionElement = new ImportDefinitionElement();
  private DataCleaning dataCleaning = new DataCleaning(configReaderTarget);

  //rale
  private String strQuerySet = "";
  QueryWhereSet queryWhereSet = null;
  QueryConstantSet queryConstantSet = null;
  QueryTransformationSet queryTransformationSet = null;
  
  Hashtable queryStatement = new Hashtable();
  Hashtable indexDValue = new Hashtable();
  Hashtable indexDRelationValue = new Hashtable();
  Hashtable indexConstantDValue = new Hashtable();
  Hashtable indexDVariableValue = new Hashtable();
  Hashtable indexDTransformationValue = new Hashtable();

  Hashtable updateStatement = new Hashtable();
  Hashtable indexDOverwrite = new Hashtable();
  Hashtable indexDSetNull = new Hashtable();

  Hashtable updateConstantStatement = new Hashtable();
  Hashtable indexDConstantOver = new Hashtable();
  Hashtable indexDConstantNull = new Hashtable();
  //zk added this 
  Hashtable resultVector = new Hashtable();
  Hashtable updateTransformationStatement = new Hashtable();
  Hashtable indexDTransformationOver = new Hashtable();
  Hashtable indexDTransformationNull = new Hashtable();
  Hashtable indexDTransformationUpdate = new Hashtable();
  //insert row
  Hashtable queryInsertRow = new Hashtable();
  Hashtable indexIsNotRelationsColumns = new Hashtable();
  Hashtable pstmtForInsert = new Hashtable();
  Hashtable pstmtColumnNames = new Hashtable();
//  Hashtable pstmtColumnTypes = new Hashtable();
	
  Vector pstmtKeys = new Vector();

  //default value
  Vector vecDefaultValue = new Vector();
  Vector vecDefaultVariableValue = new Vector();
  Vector vecDefaultVariableName = new Vector();
  Vector vecDefaultRelationValue = new Vector();
  Vector vecTempUserID = new Vector();
  boolean userIDExists = false;
  private int currentRow = 0;

  //coy table
  //source data which are not maped in to target data
  Hashtable allSourceColumnNameNoMap = new Hashtable();
  Hashtable allSourceColumnValueNoMap = new Hashtable();
  Hashtable allSourceColumnTypeNoMap = new Hashtable();
  Vector ImportDefinitionSourceColumnName = new Vector();
  Vector ImportDefinitionTargetColumnName = new Vector();
  private String currentJobName = "";
  Vector vecCTAutoMapp = new Vector();
  Vector vecCTDefaultMode = new Vector();
  Vector vecBlobVector = new Vector();
  Vector vecToHex = new Vector();
  Vector vecFromHex = new Vector();

  //caching the relations
  RelationsCache relationsCache = new RelationsCache();
  private Vector includeTables = new Vector();

  //caching the 'row check' query
  CheckRowCache checkRowCache = new CheckRowCache();

  //restart counter
  DataTransmition dataTransmition = new DataTransmition(importDefinitionElement);

  //oid and version column name
  private String currentOidColumnName = "oid";
  private String currentVersionColumnName = "version";

//  private PreparedStatementMethods pstmtMethods=new PreparedStatementMethods();
  private Vector transformationsColumnNames = new Vector();
  private Vector transformationsValueModes = new Vector();
  private Vector transformationsColumnValueMode  = new Vector();
  private Vector transformationsColumnTypes = new Vector();
  private Vector transformationsResultVectorTypes = new Vector();
  private Vector resultVectorTypes = new Vector();
  //ValidateSelectStmt validStmt = new  ValidateSelectStmt();
  
  //number of columns before transformation values from source query
  private int iOffsetForTransColumns = 0;
	
  //values for transformations, used from source ResultSet for all transformation
  //source coulmns
  Vector transformationValues = new Vector();