sentencetotree.java

无监督的句法学习系统

package action;

import model.*;
import service.TreatService;
import service.PreTreatService;
import service.BaseService;

import java.io.IOException;
import java.util.*;


/**
 * Created by IntelliJ IDEA.
 * User: guowen
 * Date: 2007-11-2
 * Time: 10:54:53
 * To change this template use File | Settings | File Templates.
 */
public class SentenceToTree {
    private TreatService  treatService=new TreatService();
    private PreTreatService preTreatService=new PreTreatService();
    private BaseService baseService=new BaseService();
    private int sentenceNumber;
    private List<PreTreatSentence> preTreatSenList;
    private List<TreatNode> treatNodeList=new ArrayList<TreatNode>();
    public static Set<RuleFull> ruleSetNow=new HashSet<RuleFull>();
    private int iteration=10;
    private int ruleNumber=5;
   // private double stop=1;
    private List<RuleScore> ruleScoreList=new ArrayList<RuleScore>();
    private List<Long> errorList=new ArrayList<Long>();

    public List<TreatNode> getOneRuleList(List<List<TreatNode>> orderList,int ruleNumber,int spanLength){
        List<TreatNode> oneSpanList=orderList.get(spanLength-1);
        List<TreatNode> treatNodeList=new ArrayList<TreatNode>();
        for(int i=0;i<oneSpanList.size();i++){
            TreatNode aNode=oneSpanList.get(i);
            if(aNode.getRule().getRuleNumber()==ruleNumber)
              treatNodeList.add(aNode);
        }
       return treatNodeList;   
    }
    public  double computeInOut(List<List<TreatNode>> orderList){
          TreatNode rootNode=new TreatNode();
        double rootInproperty=0;
        double size=0;
          for(int i=0;i<orderList.size();i++){
            List<TreatNode> aOrderList=orderList.get(i);
            for(int j=0;j<aOrderList.size();j++){
                TreatNode aNode=aOrderList.get(j);
                Rule rule=aNode.getRule();
                RuleFull aRuleFull=aNode.getRuleFull();
                 if(rule.getRuleNumber()==0&&aNode.getParent()!=null){
                    double inProperbility=aNode.getInProperbility()+aNode.getLeftNode().getInProperbility()+Math.log(aNode.getRuleFull().getProperbility());
                    double outProperbility=aNode.getOutProperbility()+aNode.getParent().getOutProperbility()+Math.log(aNode.getParent().getRuleFull().getProperbility());
                    aNode.setInProperbility(inProperbility);
                    aNode.setOutProperbility(outProperbility);
                }
                else if(rule.getRuleNumber()==1){
                    double inProperbility=aNode.getInProperbility()+aNode.getLeftNode().getInProperbility()+aNode.getRightNode().getInProperbility()+Math.log(aNode.getRuleFull().getProperbility());
                    double outProperbility=aNode.getOutProperbility()+aNode.getParent().getOutProperbility()+Math.log(aNode.getParent().getRuleFull().getProperbility());

                     aNode.setInProperbility(inProperbility);
                    aNode.setOutProperbility(outProperbility);
                }
                 else if(rule.getRuleNumber()==2){
                    double inProperbility=aNode.getInProperbility()+aNode.getLeftNode().getInProperbility()+Math.log(aNode.getRuleFull().getProperbility());
                    double outProperbility=0;
                     if(aNode.getParent().getRule().getRuleNumber()==3)
                        outProperbility=aNode.getOutProperbility()+aNode.getParent().getOutProperbility()+aNode.getParent().getLeftNode().getInProperbility()+Math.log(aNode.getParent().getRuleFull().getProperbility());
                    else if(aNode.getParent().getRule().getRuleNumber()==4)
                        outProperbility=aNode.getOutProperbility()+aNode.getParent().getOutProperbility()+Math.log(aNode.getParent().getRuleFull().getProperbility());

                     aNode.setInProperbility(inProperbility);
                    aNode.setOutProperbility(outProperbility);
                }
                else if(rule.getRuleNumber()==3){
                    double inProperbility=aNode.getInProperbility()+aNode.getLeftNode().getInProperbility()+aNode.getRightNode().getInProperbility()+Math.log(aNode.getRuleFull().getProperbility());
                    double outProperbility=0;
                    if(aNode.getParent().getRule().getRuleNumber()==3)
                        outProperbility=aNode.getOutProperbility()+aNode.getParent().getOutProperbility()+aNode.getParent().getLeftNode().getInProperbility()+Math.log(aNode.getParent().getRuleFull().getProperbility());
                    else if(aNode.getParent().getRule().getRuleNumber()==4)
                        outProperbility=aNode.getOutProperbility()+aNode.getParent().getOutProperbility()+Math.log(aNode.getParent().getRuleFull().getProperbility());

                    aNode.setInProperbility(inProperbility);
                    aNode.setOutProperbility(outProperbility);
                }
                 else if(rule.getRuleNumber()==4){
                    double inProperbility=aNode.getInProperbility()+aNode.getLeftNode().getInProperbility()+Math.log(aNode.getRuleFull().getProperbility());
                    double outProperbility=0;
                    if(aNode.getParent()!=null){
                    if(aNode.getParent().getRule().getRuleNumber()==3)
                        outProperbility=aNode.getOutProperbility()+aNode.getParent().getOutProperbility()+aNode.getParent().getRightNode().getInProperbility()+Math.log(aNode.getParent().getRuleFull().getProperbility());
                    else if(aNode.getParent().getRule().getRuleNumber()==1)
                        outProperbility=aNode.getOutProperbility()+aNode.getParent().getOutProperbility()+aNode.getParent().getLeftNode().getInProperbility()+Math.log(aNode.getParent().getRuleFull().getProperbility());
                    }
                    aNode.setInProperbility(inProperbility);
                   aNode.setOutProperbility(outProperbility);
                   if(aNode.getParent()==null){
                       rootInproperty+=aNode.getInProperbility();
                       size++;
                   }
                }
              //  rootNode=(TreatNode)aNode.clone();
            }
         }
        return rootInproperty/size;
    }
    public double ComputerPall(TreatNode nowNode,Rule nowRule){
       double pNumber=0;
       if(nowRule.getRuleNumber()==0){
          pNumber=nowNode.getLeftNode().getInProperbility()+nowNode.getOutProperbility();
       }
        if(nowRule.getRuleNumber()==1){
          pNumber=nowNode.getLeftNode().getInProperbility()+nowNode.getRightNode().getInProperbility()+nowNode.getOutProperbility();
       }
        if(nowRule.getRuleNumber()==2){
          pNumber=nowNode.getLeftNode().getInProperbility()+nowNode.getOutProperbility();
       }
        if(nowRule.getRuleNumber()==3){
          pNumber=nowNode.getLeftNode().getInProperbility()+nowNode.getRightNode().getInProperbility()+nowNode.getOutProperbility();
       }
        if(nowRule.getRuleNumber()==4){
          pNumber=nowNode.getLeftNode().getInProperbility()+nowNode.getOutProperbility();
       }
        return pNumber;
    }
    public void InOutAlgorithm(List<List<TreatNode>> orderList,double rootInproperbility){
          Rule rule=null;
          for(int i=0;i<orderList.size();i++){
            List<TreatNode> aOrderList=orderList.get(i);
              List<TreatNode> newOrderList=new ArrayList<TreatNode>();
            RuleFull ruleFull=new RuleFull();
           Iterator iter=ruleSetNow.iterator();
           for(int k=0;k<ruleSetNow.size();k++){
                ruleFull=(RuleFull) iter.next();
                List<NodeRule> aList=ruleFull.getAList();
            for(int j=0;j<aOrderList.size();j++){
                TreatNode aNode=aOrderList.get(j);
                rule=aNode.getRule();
                RuleFull nowRullFull=aNode.getRuleFull();
                List<NodeRule> nowList=nowRullFull.getAList();
                  if(aList.size()==nowList.size()&&aList.size()==1){
                      if(aList.get(0).getState().equals(nowList.get(0).getState())&&aList.get(0).getWord().equals(nowList.get(0).getWord()))
                         newOrderList.add(aNode);
                   }
                   if(aList.size()==nowList.size()&&aList.size()==2){
                      if(aList.get(0).getState().equals(nowList.get(0).getState())&&aList.get(0).getWord().equals(nowList.get(0).getWord())&&aList.get(1).getState().equals(nowList.get(1).getState())&&aList.get(1).getWord().equals(nowList.get(1).getWord()))
                         newOrderList.add(aNode);
                   }
                    if(aList.size()==nowList.size()&&aList.size()==3){
                      if(aList.get(0).getState().equals(nowList.get(0).getState())&&aList.get(0).getWord().equals(nowList.get(0).getWord())&&aList.get(1).getState().equals(nowList.get(1).getState())&&aList.get(1).getWord().equals(nowList.get(1).getWord())&&aList.get(2).getState().equals(nowList.get(2).getState())&&aList.get(2).getWord().equals(nowList.get(2).getWord()))
                         newOrderList.add(aNode);
                   }
            }
                double properbility=0;
                for(int m=0;m<newOrderList.size();m++)
                     properbility+=this.ComputerPall(newOrderList.get(m),newOrderList.get(m).getRule());
                double nowSize=ruleFull.getSize()+Math.exp(properbility+Math.log(ruleFull.getProperbility()));
                 ruleFull.setSize(nowSize);
                newOrderList.clear();
             }
           }
    }

    public Set<RuleFull>  computerInproperbility(String fileName,String stopFileName) throws IOException {
        List<NodesCell> allTree=this.getTrainTree(fileName,stopFileName);
        List<List<TreatNode>> orderList=new ArrayList<List<TreatNode>>();
         for(int i=0;i<allTree.size();i++){
          NodesCell oneTree=allTree.get(i);
          int length=oneTree.getCellNumber();
          for(int k=0;k<length;k++){
             for(int n=0;n<oneTree.getNodeList().size();n++){
                 TreatNode aTreatNode=oneTree.getNodeList().get(n);
                 List<TreatNode> nodeList=baseService.orderNodeByOneLength(aTreatNode,k+1);
                List<TreatNode> allList= baseService.cloneTreaNodeList(nodeList);
                 if(orderList.size()==(k+1)){
                   List<TreatNode> tempList= orderList.get(k);
                   for(int m=0;m<allList.size();m++){
                       tempList.add(allList.get(m));
                   }
                   orderList.remove(k);
                   orderList.add(tempList);
                 }else{
                   orderList.add(allList);
                 }
             }
           }
         }
        for(int j=0;j<iteration;j++){
          Iterator iter=ruleSetNow.iterator();
            for(int k=0;k<ruleSetNow.size();k++){
                 RuleFull ruleFull=(RuleFull) iter.next();
                 ruleFull.setSize(0);
            }
            TreatNode TreatNode=new TreatNode();
            double rootInproperbility=this.computeInOut(orderList);
            this.InOutAlgorithm(orderList,rootInproperbility);
        Iterator newIter=ruleSetNow.iterator();
        for(int k=0;k<ruleSetNow.size();k++){
             RuleFull ruleFull=(RuleFull) newIter.next();
            if(ruleFull.getAList().size()>0)
             ruleFull.setProperbility(this.getAllRuleSize(ruleFull));
        }
       }
        return  ruleSetNow;
    }
    public double getAllRuleSize(RuleFull ruleFull){
       List<NodeRule> nodeRuleList=ruleFull.getAList();
       NodeRule head=nodeRuleList.get(0);
       double nowSize=0;
       Iterator newIter=ruleSetNow.iterator();
       for(int k=0;k<ruleSetNow.size();k++){
             RuleFull oneRuleFull=(RuleFull) newIter.next();
             List<NodeRule> oneNodeRuleList=oneRuleFull.getAList();
            if(oneNodeRuleList.size()>0){
             NodeRule oneHead=oneNodeRuleList.get(0);
             if(head.getState().equals(oneHead.getState())&&head.getWord().equals(oneHead.getWord()))
                nowSize+=oneRuleFull.getSize();
            }

        }
        return ruleFull.getSize()/nowSize;
    }
    public List<NodesCell> getTrainTree(String fileName,String stopFileName) throws IOException {
        PreTreatText preTreatText=preTreatService.readAllTrainingText(fileName);
        List<PreTreatSentence> preTreatSentenceList=preTreatText.getPreTreatText();
        List<PreTreatSentence> rightPreTreatSentenceList=preTreatText.getPreTreatText();
        this.setPreTreatSenList(preTreatSentenceList);

        List<NodesCell> allNodesCell=new ArrayList<NodesCell>();
        this.setSentenceNumber(preTreatSentenceList.size());
        int index=0;
        for(int i=0;i<preTreatSentenceList.size();i++){
            PreTreatSentence sentence=preTreatSentenceList.get(i);
            List<PreTreatNode>  nodeList=sentence.getPreTreatNodeList();
            List<TreatNode> allTreatNode=new  ArrayList<TreatNode>();
            List<TreatNode> treatNodeList=getOneTree(nodeList);
            for(int j=0;j<treatNodeList.size();j++){
                TreatNode newTree=baseService.widthSearch(treatNodeList.get(j));
                baseService.widthSetStopSearch(newTree);
                if(newTree.getLeftNode()!=null||newTree.getRightNode()!=null)
                   allTreatNode.add(newTree);
            }
            NodesCell nowNodesCell=new NodesCell();
            nowNodesCell.setCellNumber(nodeList.size());
            nowNodesCell.setNodeList(allTreatNode);
            allNodesCell.add(nowNodesCell);
        }
      try{
        baseService.printStopRuleSet(stopFileName);   //杈撳嚭鏂囨湰stop姒傜巼
      }catch(Exception e){e.getMessage();}
       ruleSetNow= baseService.ruleSet;           
        return allNodesCell;
    }
    /*
    public List<TreatNode> getTree(String fileName) throws IOException {
        PreTreatText preTreatText=preTreatService.readTrainingText(fileName);
        List<PreTreatSentence> preTreatSentenceList=preTreatText.getPreTreatText();
        List<PreTreatSentence> rightPreTreatSentenceList=new ArrayList<PreTreatSentence>();
        List<TreatNode> allTreatNode=new  ArrayList<TreatNode>();
        int index=0;
        for(int i=0;i<preTreatSentenceList.size();i++){
            PreTreatSentence sentence=preTreatSentenceList.get(i);
            List<PreTreatNode>  nodeList=sentence.getPreTreatNodeList();
            SentenceCell sentenceCell=getOneTree(nodeList);
            List<NodesCell> cellList=sentenceCell.getCellList();
            TreatNode aTreatNode=new TreatNode();
            TreatNode newTree=new TreatNode();
            for(int j=0;j<cellList.size();j++){
                NodesCell aNodesCell=cellList.get(j);
                if(aNodesCell.getNodeList().size()>0){
                    aTreatNode=aNodesCell.getNodeList().get(0);
                    if(aTreatNode.getLeftNode()==null&&aTreatNode.getRightNode()==null)
                           System.out.println(index++);
                    newTree=baseService.widthSearch2(aTreatNode);
                    break;
                }
            } 
             if(newTree.getLeftNode()!=null||newTree.getRightNode()!=null){
                rightPreTreatSentenceList.add(sentence); 
             }
        }
         for(int i=0;i<rightPreTreatSentenceList.size();i++){
            PreTreatSentence sentence=rightPreTreatSentenceList.get(i);
            List<PreTreatNode>  nodeList=sentence.getPreTreatNodeList();
            SentenceCell sentenceCell=getOneTree(nodeList);
            List<NodesCell> cellList=sentenceCell.getCellList();
            TreatNode aTreatNode=new TreatNode();
            TreatNode newTree=new TreatNode();
            for(int j=0;j<cellList.size();j++){
                NodesCell aNodesCell=cellList.get(j);
                if(aNodesCell.getNodeList().size()>0){
                    aTreatNode=aNodesCell.getNodeList().get(0);
                    if(aTreatNode.getLeftNode()==null&&aTreatNode.getRightNode()==null)
                           System.out.println(index++);