poseventcollector.java

自然语言处理领域的一个开发包

///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2002 Jason Baldridge and Gann Bierner
// 
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
// 
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Lesser General Public License for more details.
// 
// You should have received a copy of the GNU Lesser General Public
// License along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
//////////////////////////////////////////////////////////////////////////////

package opennlp.tools.postag;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.Reader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.StringTokenizer;

import opennlp.maxent.Counter;
import opennlp.maxent.Event;
import opennlp.maxent.EventCollector;
import opennlp.tools.util.Pair;

/**
 * An event generator for the maxent POS Tagger.
 *
 * @author      Gann Bierner
 * @version     $Revision: 1.7 $, $Date: 2005/11/20 04:52:19 $
 */

public class POSEventCollector implements EventCollector {
  
  private BufferedReader br;
  private POSContextGenerator cg;
  
  public POSEventCollector(Reader data, POSContextGenerator gen) {
    br = new BufferedReader(data);
    cg = gen;
  }
  
  private static Pair split(String s) {
    int split = s.lastIndexOf("_");
    if (split == -1) {
      System.out.println("There is a problem in your training data: "
          + s
          + " does not conform to the format WORD_TAG.");
      return new Pair(s, "UNKNOWN");
    }
    return new Pair(s.substring(0, split), s.substring(split+1));
  }
  
  public static Pair convertAnnotatedString(String s) {
    ArrayList tokens = new ArrayList();
    ArrayList outcomes = new ArrayList();
    StringTokenizer st = new StringTokenizer(s);
    while(st.hasMoreTokens()) {
      Pair p = split(st.nextToken());
      tokens.add(p.a);
      outcomes.add(p.b);
    }
    return new Pair(tokens, outcomes);
  }
  
  public Event[] getEvents() {
    return getEvents(false);
  }
  
  private Set getFrequent(BufferedReader br) {
    HashMap map = new HashMap();
    
    try {
      for(String s = br.readLine(); s!=null; s=br.readLine()) {
        StringTokenizer st = new StringTokenizer(s);
        while(st.hasMoreTokens()) {
          String tok = (String)split(st.nextToken()).a;
          Counter c = (Counter)map.get(tok);
          if(c!=null)
            c.increment();
          else
            map.put(tok, new Counter());
        }
      }
    } catch (IOException e) { e.printStackTrace(); }
    
    HashSet set = new HashSet();
    for(Iterator i=map.entrySet().iterator(); i.hasNext();) {
      Map.Entry entry = (Map.Entry)i.next();
      if(((Counter)entry.getValue()).passesCutoff(5))
        set.add(entry.getKey());
    }
    
    return set;
  }
  
  /** 
   * Builds up the list of features using the Reader as input.  For now, this
   * should only be used to create training data.
   */
  public Event[] getEvents(boolean evalMode) {
    ArrayList elist = new ArrayList();
    //int numMatches;
    //Set frequent;
    /*
     if(!evalMode) {
     //System.out.println("Reading in all the data");
      try {
      StringBuffer sb = new StringBuffer();
      for(String s = br.readLine(); s!=null; s=br.readLine())
      sb.append(s+"\n");
      //System.out.println("Getting most frequent words");
       frequent =
       getFrequent(new BufferedReader(
       new StringReader(sb.toString())));
       br = new BufferedReader(new StringReader(sb.toString()));
       sb=null;
       } catch (IOException e) { e.printStackTrace(); }
       }
       */
    //System.out.println("Collecting events");
    try {
      String s = br.readLine();
      
      while (s != null) {
        Pair p = convertAnnotatedString(s);
        ArrayList tokens = (ArrayList)p.a;
        ArrayList outcomes = (ArrayList)p.b;
        ArrayList tags = new ArrayList();
        
        for (int i=0; i<tokens.size(); i++) {
          String[] context = cg.getContext(i,tokens.toArray(),(String[]) tags.toArray(new String[tags.size()]),null);
          Event e = new Event((String)outcomes.get(i), context);
          tags.add(outcomes.get(i));
          elist.add(e);
        }
        s = br.readLine();
      }
    } catch (Exception e) { e.printStackTrace(); }
    
    Event[] events = new Event[elist.size()];
    for(int i=0; i<events.length; i++)
      events[i] = (Event)elist.get(i);
    
    return events;
  }

  /*
  public static void main(String[] args) {
    String data = "the_DT stories_NNS about_IN well-heeled_JJ communities_NNS and_CC developers_NNS";
    EventCollector ec = new POSEventCollector(new StringReader(data),
        new DefaultPOSContextGenerator());
    Event[] events = ec.getEvents();
    for(int i=0; i<events.length; i++)
      System.out.println(events[i].getOutcome());
  }
  */
  
}