matlabica.java

wekaUT是 university texas austin 开发的基于weka的半指导学习(semi supervised learning)的分类器

  public ASEvaluation getEvaluator() {
    return m_eval;
  }


  /**
   * Returns the tip text for this property
   * @return tip text for this property suitable for
   * displaying in the explorer/experimenter gui
   */
  public String normalizeTipText() {
    return "Normalize input data.";
  }

  /**
   * Set whether input data will be normalized.
   * @param n true if input data is to be normalized
   */
  public void setNormalize(boolean n) {
    m_normalize = n;
  }

  /**
   * Gets whether or not input data is to be normalized
   * @return true if input data is to be normalized
   */
  public boolean getNormalize() {
    return m_normalize;
  }

  /**
   * Returns the tip text for this property
   * @return tip text for this property suitable for
   * displaying in the explorer/experimenter gui
   */
  public String transformBackToOriginalTipText() {
    return "Transform through the IC space and back to the original space. "
      +"If only the best n ICs are retained (by setting varianceCovered < 1) "
      +"then this option will give a dataset in the original space but with "
      +"less attribute noise.";
  }

  /**
   * Sets whether the data should be transformed back to the original
   * space
   * @param b true if the data should be transformed back to the
   * original space
   */
  public void setTransformBackToOriginal(boolean b) {
    m_transBackToOriginal = b;
  }
  
  /**
   * Gets whether the data is to be transformed back to the original
   * space.
   * @return true if the data is to be transformed back to the original space
   */
  public boolean getTransformBackToOriginal() {
    return m_transBackToOriginal;
  }

  /**
   * Gets the current settings of MatlabICA
   *
   * @return an array of strings suitable for passing to setOptions()
   */
  public String[] getOptions () {

    String[] options = new String[10];
    int current = 0;

    if (!getNormalize()) {
      options[current++] = " -D";
    }

    if (getTransformBackToOriginal()) {
      options[current++] = " -O";
    }
    options[current++] = " -A";
    options[current++] = "" + getICAapproach().getSelectedTag().getReadable();
    options[current++] = " -F";
    options[current++] = "" + getICAfunction().getSelectedTag().getReadable();
    options[current++] = " -N";
    options[current++] = "" + getNumIndependentComponents();
    
    while (current < options.length) {
      options[current++] = "";
    }
    return  options;
  }

  /**
   * Initializes independent components and performs the analysis
   * @param data the instances to analyse/transform
   * @exception Exception if analysis fails
   */
  public void buildEvaluator(Instances data) throws Exception {
    buildAttributeConstructor(data);
  }

  private void buildAttributeConstructor (Instances data) throws Exception {

    System.out.println("data.numInstances: " + data.numInstances());
    m_independentComponents = null;
    m_outputNumAtts = -1;
    m_attributeFilter = null;

    if (data.checkForStringAttributes()) {
      throw  new UnsupportedAttributeTypeException("Can't handle string attributes!");
    }
    m_trainInstances = data;
    System.out.println("ClassIndex is " + m_trainInstances.classIndex());

    // make a copy of the training data so that we can get the class
    // column to append to the transformed data (if necessary)
    m_trainCopy = new Instances(m_trainInstances);
    System.out.println("Copied instances");
    m_replaceMissingFilter = new ReplaceMissingValues();
    m_replaceMissingFilter.setInputFormat(m_trainInstances);
    m_trainInstances = Filter.useFilter(m_trainInstances, 
					m_replaceMissingFilter);
    System.out.println("Replaced missing values");

    if (m_normalize) {
      m_normalizeFilter = new Normalize();
      m_normalizeFilter.setInputFormat(m_trainInstances);
      m_trainInstances = Filter.useFilter(m_trainInstances, m_normalizeFilter);
      System.out.println("Normalized");
    }

    // delete any attributes with only one distinct value or are all missing
    Vector deleteCols = new Vector();
    for (int i=0;i<m_trainInstances.numAttributes();i++) {
      if (m_trainInstances.numDistinctValues(i) <=1) {
	deleteCols.addElement(new Integer(i));
      }
    }
    System.out.println("Deleted single-value attributes");

    if (m_trainInstances.classIndex() >=0) {
      // get rid of the class column
      m_hasClass = true;
      m_classIndex = m_trainInstances.classIndex();
      deleteCols.addElement(new Integer(m_classIndex));
      System.out.println("Deleted class attributes");
    }

    // remove columns from the data if necessary
    if (deleteCols.size() > 0) {
      m_attributeFilter = new Remove();
      int [] todelete = new int [deleteCols.size()];
      for (int i=0;i<deleteCols.size();i++) {
	todelete[i] = ((Integer)(deleteCols.elementAt(i))).intValue();
      }
      m_attributeFilter.setAttributeIndicesArray(todelete);
      m_attributeFilter.setInvertSelection(false);
      m_attributeFilter.setInputFormat(m_trainInstances);
      m_trainInstances = Filter.useFilter(m_trainInstances, m_attributeFilter);
    }
    System.out.println("Removed attributes filtered above");
    
    m_numInstances = m_trainInstances.numInstances();
    m_numAttribs = m_trainInstances.numAttributes();

    if (m_timestamp == null) { 
      m_timestamp = getLogTimestamp();
      m_icaAttributeFilename = new String(m_icaAttributeFilenameBase + m_timestamp + ".txt");
      m_mixingMatrixFilename = new String(m_mixingMatrixFilenameBase + m_timestamp + ".txt");
      m_independentComponentsFilename = new String(m_independentComponentsFilenameBase + m_timestamp + ".txt");
    }

    MatlabPCA.dumpAttributeNames(m_trainInstances, m_icaAttributeFilename);

    System.out.println("About to run ICA on " + m_numInstances + " instances, each with " + m_numAttribs + " attributes");
    dumpInstances(m_dataFilename);
    prepareMatlab(m_ICAMFile);
    runMatlab(m_ICAMFile, "/var/local/ICAMatlab.output");
    System.out.println("Done training ... now parsing matlab output files");

    m_mixingMatrix = readColumnVectors(m_mixingMatrixFilename);
    m_inverseMixingMatrix = readColumnVectors(m_inverseMixingMatrixFilename);
    m_independentComponents = readColumnVectors(m_independentComponentsFilename);
    if (m_mixingMatrix == null || m_independentComponents == null || m_inverseMixingMatrix == null) {
      System.out.println("WARNING!! Could not parse matlab output files");
      m_originalSpaceFormat = setOutputFormatOriginal();
      m_transformedFormat = m_originalSpaceFormat;
      m_outputNumAtts = m_originalSpaceFormat.numAttributes();
    }
    else {
      System.out.println("Successfully parsed matlab output files");
      System.out.println("MixingMatrix: " + m_mixingMatrix.length + "x" + m_mixingMatrix[0].length);
      System.out.println("InverseMixingMatrix: " + m_inverseMixingMatrix.length + "x" + m_inverseMixingMatrix[0].length);
      m_transformedFormat = setOutputFormat();
      if (m_transBackToOriginal) {
	m_originalSpaceFormat = setOutputFormatOriginal();
      }
    }

    // Build the attribute evaluator
    if (m_trainInstances.classIndex() >= 0) {
      m_eval.buildEvaluator(transformedData());
    }
  }

  /**
   * Read column vectors from a text file
   * @param name file name
   * @return a <code>double[][]</code> value
   * @exception Exception if an error occurs
   * @returns double[][] array corresponding to vectors
   */
  public double[][] readColumnVectors(String name) throws Exception {
    BufferedReader r = new BufferedReader(new FileReader(name));
    int numAttributes = -1, numVectors = -1;
        
    // number of rows
    String s =  r.readLine();
    try {
      numAttributes = (int)Double.parseDouble(s);
    } catch (Exception e) {
      System.err.println("Couldn't parse " + s + " as Double");
    }
    
    // number of columns
    s = r.readLine();
    try { 
      numVectors = (int)Double.parseDouble(s);
    } catch (Exception e) {
      System.err.println("Couldn't parse " + s + " as Double");
    }

    if (numAttributes == 0 || numVectors == 0)
      return null;

    double[][] vectors = new double[numAttributes][numVectors];
    int i = 0, j = 0;
    while ((s = r.readLine()) != null) {
      StringTokenizer tokenizer = new StringTokenizer(s);
      while (tokenizer.hasMoreTokens()) {
	String value = tokenizer.nextToken();
	try { 
	  vectors[i][j] = Double.parseDouble(value);
	} catch (Exception e) {
	  System.err.println("Couldn't parse " + value + " as double");
	}
	j++;
	if (j > numVectors) {
	  System.err.println("Too many vectors(" + j + " instead of " + numVectors + ") in line: " + s);
	}
      }
      if (j != numVectors) {
	System.err.println("Too few vectors(" + j + " instead of " + numVectors + ") in line: " + s);
      }
      j = 0;
      i++;
      if (i > numAttributes) {
	System.err.println("Too many attributes: " + i + " expecting " + numAttributes + " attributes");
      }
    }
    if (i != numAttributes) {
      System.err.println("Too few attributes: " + i + " expecting " + numAttributes + " attributes");
    }
    return vectors;
  }

  /**
   * Returns just the header for the transformed data (ie. an empty
   * set of instances. This is so that AttributeSelection can
   * determine the structure of the transformed data without actually
   * having to get all the transformed data through getTransformedData().
   * @return the header of the transformed data.
   * @exception Exception if the header of the transformed data can't
   * be determined.
   */
  public Instances transformedHeader() throws Exception {
    if (m_independentComponents == null) {
      // throw new Exception("Independent components hasn't been built yet");
      System.out.println("WARNING!! Independent components could not be built, returning original data");
    }
    if (m_transBackToOriginal) {
      return m_originalSpaceFormat;
    } else {
      return m_transformedFormat;
    }
  }

  /**
   * Gets the transformed training data.
   * @return the transformed training data
   * @exception Exception if transformed data can't be returned
   */
  public Instances transformedData() throws Exception {
    if (m_independentComponents == null) {
      //      throw new Exception("Independent components hasn't been built yet");
      System.out.println("WARNING!! Independent components could not be built, returning original data");
      return m_trainCopy;
    }
    Instances output;

    if (m_transBackToOriginal) {
      output = new Instances(m_originalSpaceFormat);
    } else {
      output = new Instances(m_transformedFormat);
    }
    for (int i=0;i<m_trainCopy.numInstances();i++) {
      Instance converted = convertInstance(m_trainCopy.instance(i));
      System.out.println("Converted instance: " + converted);
      output.add(converted);
    }

    return output;
  }

  /**
   * Evaluates the merit of a transformed attribute. This is defined
   * to be 1 minus the cumulative variance explained. Merit can't
   * be meaningfully evaluated if the data is to be transformed back
   * to the original space.
   * @param att the attribute to be evaluated
   * @return the merit of a transformed attribute
   * @exception Exception if attribute can't be evaluated
   */
  public double evaluateAttribute(int att) throws Exception {
    if (m_independentComponents == null) {
      //      throw new Exception("Independent components hasn't been built yet!");
      System.out.println("WARNING!! Independent components could not be built, returning original data");
    }
    if (!(m_eval instanceof AttributeEvaluator)) {
      throw new Exception("Invalid attribute evaluator!");
    }
    if (m_trainInstances.classIndex() < 0) {
      return 1;
    } else {
      return ((AttributeEvaluator)m_eval).evaluateAttribute(att);
    }
  }

  /**
   * Dump data matrix into a file