transformation.java

一个数据挖掘软件ALPHAMINERR的整个过程的JAVA版源代码

/*
 *    This program is free software; you can redistribute it and/or modify
 *    it under the terms of the GNU General Public License as published by
 *    the Free Software Foundation; either version 2 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU General Public License for more details.
 *
 *    You should have received a copy of the GNU General Public License
 *    along with this program; if not, write to the Free Software
 *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */

 /**
  * Title: XELOPES Data Mining Library
  * Description: The XELOPES library is an open platform-independent and data-source-independent library for Embedded Data Mining.
  * Copyright: Copyright (c) 2002 Prudential Systems Software GmbH
  * Company: ZSoft (www.zsoft.ru), Prudsys (www.prudsys.com)
  * @author Carsten Weisse
  * @author Michael Thess
  * @version 1.0
  */
package com.prudsys.pdm.Examples;

import java.io.FileReader;
import java.io.FileWriter;

import com.prudsys.pdm.Core.MiningDataSpecification;
import com.prudsys.pdm.Input.MiningFilterStream;
import com.prudsys.pdm.Input.MiningInputStream;
import com.prudsys.pdm.Input.MiningStoredData;
import com.prudsys.pdm.Input.Records.Arff.MiningArffStream;
import com.prudsys.pdm.Transform.MiningTransformationActivity;
import com.prudsys.pdm.Transform.MiningTransformationFactory;
import com.prudsys.pdm.Transform.MiningTransformationStep;
import com.prudsys.pdm.Transform.MultipleToMultiple.AddAttribute;
import com.prudsys.pdm.Transform.MultipleToMultiple.ChangeAttributeOrder;
import com.prudsys.pdm.Transform.MultipleToMultiple.RemoveAttributes;
import com.prudsys.pdm.Transform.MultipleToMultiple.RotateAttributes;
import com.prudsys.pdm.Transform.MultipleToMultiple.WaveletTransformation;
import com.prudsys.pdm.Transform.OneToOne.CreateVirtualAttribute;
import com.prudsys.pdm.Transform.OneToOne.TreatOutlierAttributeValue;
import com.prudsys.pdm.Transform.Special.CopyMiningStream;
import com.prudsys.pdm.Transform.Special.HorizontalPartitionMiningStream;
import com.prudsys.pdm.Transform.Special.NontransactionalToTransactional;
import com.prudsys.pdm.Transform.Special.ReplaceMissingValueStream;
import com.prudsys.pdm.Transform.Special.TransactionalToNontransactional;
import com.prudsys.pdm.Transform.Special.TreatOutlierValueStream;
import com.prudsys.pdm.Utils.PmmlUtils;

/**
 * Demonstrates Transformation package.
 */
public class Transformation extends BasisExample
{
  /**
   * Empty constructor.
   */
  public Transformation() {

  }

  /**
   * Transformation by replacing missing values.
   *
   * @exception Exception something went wrong
   */
  private void replaceMissingValuesAndOutliers() throws Exception {

    // Open 'iris.arff':
    MiningArffStream arff = new MiningArffStream( "data/arff/labor.arff" );

    // Create mining replacement value stream from 'labor' file:
    ReplaceMissingValueStream rep = new ReplaceMissingValueStream(arff);
    MiningInputStream mis         = rep.createReplaceMissingValueStream();

    // Display transformed data matrix:
    System.out.println("Transformed by 'ReplaceMissingValueStream':");
    System.out.println(mis);


    // Create treatment of outliers transformation from 'labor' file:
    TreatOutlierValueStream tro = new TreatOutlierValueStream(arff);
    tro.setNumOutliers(TreatOutlierAttributeValue.OUTLIER_TREATMENT_METHOD_asExtremeValues );
    tro.createTreatOutlierValueTransformationStep();

    // Display transformed data matrix:
    System.out.println(tro.createTreatOutlierValueStream()   );
    System.out.println("Transformation of 'TreatOutlierValueStream':");
    System.out.println(tro);
    System.out.println(tro.getMts());
  }

  /**
   * Examples of MiningStreamTransformer transformations.
   *
   * @exception Exception something went wrong
   */
  private static void transformStreams() throws Exception {

    // Open 'iris.arff':
    MiningArffStream arff = new MiningArffStream( "data/arff/iris.arff" );

    // Display original stream:
    System.out.println( arff );

    // 1. Transformation -> Copy:
    MiningStoredData msd = new MiningStoredData();
    (new CopyMiningStream()).transform(arff, msd);

    // Display "transformed" stream:
    System.out.println("1. Transformed by 'CopyMiningStream': ");
    msd.reset();
    System.out.println( msd );

    // 2. Transformation -> Transactional Format:
    MiningArffStream arff2 = new MiningArffStream( "data/arff/iris-transact.arff" );
    (new NontransactionalToTransactional()).transform(msd, arff2);
    arff2.close();

    // Display transformed stream:
    System.out.println("2. Transformed by 'NontransactionalToTransactional': ");
    System.out.println( arff2 );

    // 3. Transformation -> Nontransactional Format:
    MiningArffStream arff3 = new MiningArffStream( "data/arff/iris-nontransact.arff" );
    TransactionalToNontransactional ttn = new TransactionalToNontransactional();
//    ttn.setVectorNrIdAttName("transactionId");
//    ttn.setAttributeValueAttName("itemId");
//    ttn.setAttributeIndexAttName("itemIndex");
    ttn.transform(arff2, arff3);
    arff3.close();

    // Display transformed stream:
    System.out.println("3. Transformed by 'TransactionalToNontransactional': ");
    System.out.println( arff3 );

    // 4. Transformation -> partition into two streams:
    MiningStoredData[] msdArr = new MiningStoredData[2];
    msdArr[0] = new MiningStoredData();
    msdArr[1] = new MiningStoredData();
    HorizontalPartitionMiningStream hpms = new HorizontalPartitionMiningStream();
    hpms.transform(arff3, msdArr);
    // close methods if target are file streams

    // Display partitioned streams:
    System.out.println("4. Transformed by 'HorizontalPartitionMiningStream': ");
    System.out.println("--> partition 1:");
    System.out.println( msdArr[0] );
    System.out.println("--> partition 2:");
    System.out.println( msdArr[1] );
  }

  /**
   * Transformation by Wavelets. First direct and then inverse
   * wavelet transformation => original data.
   *
   * @exception Exception something went wrong
   */
  private void transformWavelets() throws Exception {

    // Open 'iris.arff':
    MiningArffStream arff = new MiningArffStream( "data/arff/iris.arff" );

    // Create wavelet transformation:
    WaveletTransformation wt = new WaveletTransformation();
    String sourceNames[] = {"sepallength", "sepalwidth", "petallength", "petalwidth"};
    String targetNames[] = {"w_sepallength", "w_sepalwidth", "w_petallength", "w_petalwidth"};
    wt.setSourceName(sourceNames);
    wt.setTargetName(targetNames);

    // Create first mining transformation step using factory:
    MiningTransformationFactory mtf = new MiningTransformationFactory();
    mtf.addMultipleToMultipleMapping(wt);
    MiningTransformationStep mts = mtf.createMiningTransformationStep();

    // Run transformation:
    MiningStoredData msd = new MiningStoredData();
    mts.transform(arff, msd);

    // Show result:
    System.out.println("Transformed by 'WaveletTransformation':");
    System.out.println(msd);

    // Now define inverse transformation:
    WaveletTransformation wti = new WaveletTransformation();
    String sourceNamesInv[] = {"w_sepallength", "w_sepalwidth", "w_petallength", "w_petalwidth"};
    String targetNamesInv[] = {"iw_sepallength", "iw_sepalwidth", "iw_petallength", "iw_petalwidth"};
    wti.setSourceName(sourceNamesInv);
    wti.setTargetName(targetNamesInv);
    wti.setInverseTransform(true);

    // Create second transformation step:
    mtf.reset();
    mtf.addMultipleToMultipleMapping(wti);
    MiningTransformationStep mts2 = mtf.createMiningTransformationStep();

    // Run inverse transformation:
    msd.reset();
    MiningStoredData msd2 = new MiningStoredData();
    mts2.transform(msd, msd2);

    // Show result:
    System.out.println("Transformed by inverse 'WaveletTransformation':");
    System.out.println(msd2);
  }

  /**
   * Transformations of the multiple-to-multiple type.
   *
   * @exception Exception something went wrong
   */
  private void transformMultiple() throws Exception {

    // Open 'transact.arff':
    MiningArffStream arff = new MiningArffStream( "data/arff/transact.arff" );

    // Create add transformation:
    AddAttribute aa = new AddAttribute();
    String sourceNames[] = {"transactId", "itemId", "itemName", "itemPrice"};
//    aa.setSourceName(sourceNames);

    // Create first mining transformation step using factory:
    MiningTransformationFactory mtf = new MiningTransformationFactory();
    mtf.addMultipleToMultipleMapping(aa);
    MiningTransformationStep mts = mtf.createMiningTransformationStep();

    // Run transformation:
    MiningStoredData msd = new MiningStoredData();
    mts.transform(arff, msd);

    // Show result:
    System.out.println("Transformed by 'AddAttribute':");
    System.out.println(msd);

    // Create remove transformation:
    RemoveAttributes ra = new RemoveAttributes();
    String sourceNames2[] = {"transactId", "itemId", "itemName", "itemPrice", "newAtt4"};
    ra.setSourceName(sourceNames2);
    java.util.Vector rat = new java.util.Vector();
    rat.addElement("newAtt4");
    rat.addElement("itemId");
    ra.setRemoveAttributeNames(rat);

    // Create second transformation step:
    mtf.reset();
    mtf.addMultipleToMultipleMapping(ra);
    MiningTransformationStep mts2 = mtf.createMiningTransformationStep();

    // Run transformation:
    msd.reset();
    MiningStoredData msd2 = new MiningStoredData();
    mts2.transform(msd, msd2);

    // Show result:
    System.out.println("Transformed by inverse 'RemoveAttribute':");
    System.out.println(msd2);

    // Create rotating transformation:
    RotateAttributes ras = new RotateAttributes();
    String sourceNames3[] = {"transactId", "itemName", "itemPrice"};
//    ras.setSourceName(sourceNames3);

    // Create transformation step:
    mtf.reset();
    mtf.addMultipleToMultipleMapping(ras);
    MiningTransformationStep mts3 = mtf.createMiningTransformationStep();

    // Run transformation:
    msd2.reset();
    MiningStoredData msd3 = new MiningStoredData();
    mts3.transform(msd2, msd3);

    // Show result:
    System.out.println("Transformed by 'RotateAttributes':");
    System.out.println(msd3);

    // Create change order transformation:
    ChangeAttributeOrder cao = new ChangeAttributeOrder();
    String[] nOrder = { "itemName", "transactId", "itemPrice" };
    cao.setTargetName(nOrder);

    // Create transformation step:
    mtf.reset();
    mtf.addMultipleToMultipleMapping(cao);
    MiningTransformationStep mts4 = mtf.createMiningTransformationStep();

    // Run transformation:
    msd3.reset();
    MiningStoredData msd4 = new MiningStoredData();
    mts4.transform(msd3, msd4);

    // Show result:
    System.out.println("Transformed by 'ChangeAttributeOrder':");
    System.out.println(msd4);

    // JOIN ALL TRANSFORMATIONS:
    MiningTransformationActivity mta = new MiningTransformationActivity();
    mta.addTransformationStep(mts);
    mta.addTransformationStep(mts2);
    mta.addTransformationStep(mts3);
    mta.addTransformationStep(mts4);

    // Run transformation:
    arff.reset();
    MiningStoredData msd5 = new MiningStoredData();
    mta.transform(arff, msd5);

    // Show result:
    System.out.println("Transformed by all transformation steps:");
    System.out.println(msd5);

  }

  /**
   * Examples of MiningTransformer transformations.
   *
   * @exception Exception something went wrong
   */
  private void transformTest() throws Exception {

      // Open 'iris.arff':
      MiningArffStream arff = new MiningArffStream( "data/arff/iris.arff" );

     // Exponential transformation of 'sepallength':
     CreateVirtualAttribute cva = new CreateVirtualAttribute();
     cva.setSourceName( "class" );
     cva.setTargetName( "virtual" );

     // Create first mining transformation step using factory:
     MiningTransformationFactory mtf = new MiningTransformationFactory();
     mtf.addOneToOneMapping(cva);
     MiningTransformationStep mts = mtf.createMiningTransformationStep();

     // Create mining transformation activity:
     MiningTransformationActivity mta = new MiningTransformationActivity();
     mta.addTransformationStep(mts);

     // Create mining filter stream from iris file and our transformer:
     MiningFilterStream filter = new MiningFilterStream( arff, mta );

     // Display transformed data:
     System.out.println("User-Transformed Iris: ");
     System.out.println( filter );

     // Write transformed data into PMML document:
     FileWriter writer = new FileWriter("data/pmml/Transformation.xml");
     MiningDataSpecification mds = filter.getMetaData();
     mds.writePmml(writer);

     // Show in browser:
     if (debug == 2) PmmlUtils.openPmmlBrowser("Transformation.xml");

     FileReader reader = new FileReader("data/pmml/Transformation.xml");
     mds.readPmml(reader);

     MiningDataSpecification mds2 = new MiningDataSpecification();
     mta = mds.getMiningTransformationActivity();
     mds2 = mta.transform(mds);

     // Display transformed data:
     System.out.println("Read meta data Iris: ");
     System.out.println( mds2 );
  }

  /**
   * Run the example of this class.
   *
   * @throws Exception error while example is running
   */
  public void runExample() throws Exception {

    replaceMissingValuesAndOutliers();
    transformStreams();
    transformWavelets();
    transformMultiple();
//    transformTest();
  }

  /**
   * Example of using XELOPES for transformations.
   *
   * @param args (ignored)
   */
  public static void main(String[] args)
  {
      try {
        new Transformation().runExample();
      }
      catch (Exception ex)
      {
        ex.printStackTrace();
      };
  }
}