recodesentences.java

无监督的句法学习系统

package action;

import model.*;

import java.util.*;
import java.io.IOException;

import service.TreatService;
import service.PreTreatService;
import service.BaseService;
import service.RecodeService;

/**
 * Created by IntelliJ IDEA.
 * User: Administrator
 * Date: 2007-11-7
 * Time: 22:21:02
 * To change this template use File | Settings | File Templates.
 */
public class RecodeSentences {
    private TreatService treatService=new TreatService();
    private PreTreatService preTreatService=new PreTreatService();
    private BaseService baseService=new BaseService();
    private RecodeService recodeService =new RecodeService();
    public  Set<RuleFull> ruleSetNow=new HashSet<RuleFull>();
    public  Set<StopRule> stopRuleSet=new HashSet<StopRule>();
    private int nBest=20;
    private List<RuleScore> ruleScoreList=new ArrayList<RuleScore>();
    private SentenceToTree sentenceToTree=new SentenceToTree();

     public List<TreatNode> getTree(Set<RuleFull> ruleSet, List<PreTreatSentence> preTreatSentenceList,Set<StopRule> stopRuleSet) throws IOException {
        this.setRuleSetNow(ruleSet);
         this.setStopRuleSet(stopRuleSet);
       // PreTreatText preTreatText=preTreatService.readTrainingText("data/eng_test.conll");
       // List<PreTreatSentence> preTreatSentenceList=preTreatText.getPreTreatText();
        List<TreatNode> allTreatNode=new  ArrayList<TreatNode>();
        int index=0;
        for(int i=0;i<preTreatSentenceList.size();i++){
            PreTreatSentence sentence=preTreatSentenceList.get(i);
            List<PreTreatNode>  nodeList=sentence.getPreTreatNodeList();
            TreatNode lastTreatNode=getRecodeOneTree(nodeList);
            TreatNode newTree=baseService.widthSearch2(lastTreatNode);
            newTree.setFlags(sentence.getFlags());
//            List<NodesCell> cellList=sentenceCell.getCellList();
            TreatNode aTreatNode=new TreatNode();
           // TreatNode newTree=new TreatNode();
           /* for(int j=0;j<cellList.size();j++){
                NodesCell aNodesCell=cellList.get(j);
                if(aNodesCell.getNodeList().size()>0){
                    aTreatNode=aNodesCell.getNodeList().get(0);
                    if(aTreatNode.getLeftNode()==null&&aTreatNode.getRightNode()==null)
                           System.out.println(index++);
                    newTree=baseService.widthSearch2(aTreatNode);
                    break;
                }
            } */
             if(newTree.getLeftNode()!=null||newTree.getRightNode()!=null){
            allTreatNode.add(newTree);
             }
        }
        return allTreatNode;
    }

    public TreatNode getRecodeOneTree( List<PreTreatNode>  nodeList){
          List<TreatNode> initNodeList=new ArrayList<TreatNode>();
          for(int i=0;i<nodeList.size();i++){
              TreatNode treatNode=new  TreatNode();
              treatNode.setState("^");
              treatNode.setLeftNode(null);
              treatNode.setRightNode(null);
              treatNode.setPreTreatNode(nodeList.get(i));
              treatNode.setTreatNodeLenghth(1);
              treatNode.setInProperbility(1);
              treatNode.setOutProperbility(1);
              initNodeList.add(treatNode);
            }
         List<SentenceCell> sentenceList=new ArrayList<SentenceCell>();
          SentenceCell sentenceCell=new SentenceCell();
          sentenceCell.setWordLong(1);
          List<NodesCell>  nodesCellList=new ArrayList();
          for(int i=0;i<initNodeList.size();i++){
              NodesCell nodesCell=new NodesCell();
              List<TreatNode> oneWordnodeList=new ArrayList<TreatNode>();
              TreatNode newTreatNode= recodeService.makeNodeUseOneNode(initNodeList.get(i),1,ruleSetNow,stopRuleSet);
              oneWordnodeList=this.addList(oneWordnodeList,newTreatNode,1);
              if(oneWordnodeList!=null){
              nodesCell.setNodeList(oneWordnodeList);
              nodesCell.setCellNumber(1);
              NodesCell nBestNodesCell=this.getNbest(nodesCell);
              nodesCellList.add(nBestNodesCell);
              }
          }
         sentenceCell.setCellList(nodesCellList);
         sentenceList.add(sentenceCell);
         SentenceCell oldSentenceCell=new SentenceCell();
        List<NodesCell> allNodesCellList=new ArrayList<NodesCell>();
         for(int i=2;i<=initNodeList.size();i++){
              SentenceCell newSentenceCell=new SentenceCell();
              newSentenceCell.setWordLong(i);
              List<int[]> spiltlength=baseService.split(i);
              List<NodesCell> nowNodesCellList=new ArrayList<NodesCell>();

             for(int j=0;j<spiltlength.size();j++){
                 int[] spilt=spiltlength.get(j);
                 int index=0;
                 while(index<nodeList.size()){
                     List<PreTreatNode> priorListLog=new ArrayList<PreTreatNode>();
                     List<PreTreatNode> rearListLog=new ArrayList<PreTreatNode>();
                     int v=index;
                     if(index+spilt[0]<=nodeList.size()&&index+i<=nodeList.size()){
                     for(;v<index+spilt[0];v++)
                        priorListLog.add(nodeList.get(v));
                     for(;v<index+i;v++)
                        rearListLog.add(nodeList.get(v));
                     NodesCell priorList=this.getListNodeByNeighbour(sentenceList,spilt[0],priorListLog);
                     NodesCell nBestPriorNodesCell=this.getNbest(priorList);
                     NodesCell rearList=this.getListNodeByNeighbour(sentenceList,spilt[1],rearListLog);
                     NodesCell nBestRearPriorNodesCell=this.getNbest(rearList);
                     if(nBestPriorNodesCell!=null&&nBestRearPriorNodesCell!=null){
                         NodesCell makeOne= this.getRecodeMergeNodes(nBestPriorNodesCell,nBestRearPriorNodesCell,i,initNodeList.size(),spilt[0],spilt[1],i,nodeList);
                       makeOne.setCellNumber(i);
                     if(makeOne.getNodeList().size()>0){
                       NodesCell nBestNodesCell=this.getNbest(makeOne);
                        allNodesCellList.add(nBestNodesCell);
                        nowNodesCellList.add(nBestNodesCell);
                     }
                     }
                     }
                     index=index+1;
                 }
             }
             newSentenceCell.setCellList(nowNodesCellList);
             sentenceList.add(newSentenceCell);
         }
        SentenceCell aSten=sentenceList.get(sentenceList.size()-1);
        List<NodesCell> lastCell=aSten.getCellList();
        NodesCell lastNodesCell=new NodesCell();
        List<TreatNode> lastNodeList=new ArrayList<TreatNode>();
        for(int i=0;i<lastCell.size();i++){
            NodesCell aNodesCell=lastCell.get(i);
            List<TreatNode> nowList=aNodesCell.getNodeList();
            for(int j=0;j<nowList.size();j++)
                lastNodeList.add(nowList.get(j));
        }
        lastNodesCell.setCellNumber(initNodeList.size());
        lastNodesCell.setNodeList(lastNodeList);
        NodesCell nBestNodesCell=this.getNbest(lastNodesCell);
        if(nBestNodesCell.getNodeList().size()>0){
        TreatNode LastNode=nBestNodesCell.getNodeList().get(0);
        return LastNode;
        }
        else
         return null;
    }

    public NodesCell getRecodeMergeNodes(NodesCell one,NodesCell two,int nowLength,int length,int prior,int rear,int wordLength, List<PreTreatNode> nodeList){
        List<TreatNode> newNodeslist=new ArrayList<TreatNode>();
        Set<TreatNode> oldNodeslist=new HashSet<TreatNode>();
        List<TreatNode> priortreatNodeList=one.getNodeList();
        List<TreatNode> rearTreatNodeList=two.getNodeList();
        for(int i=0;i<one.getNodeList().size();i++){
            for(int j=0;j<two.getNodeList().size();j++){
                TreatNode aNode=(TreatNode)priortreatNodeList.get(i).clone();
                TreatNode bNode=(TreatNode)rearTreatNodeList.get(j).clone();
                TreatNode newTreatNode=null;
               if(aNode.getState().equals(">")&&bNode.getState().equals("~")){
                      newTreatNode=recodeService.makeNode(aNode,bNode,wordLength,ruleSetNow,stopRuleSet);
                }
                else if(bNode.getState().equals("<>")&&aNode.getState().equals("~")){
                      newTreatNode=recodeService.makeNode(aNode,bNode,wordLength,ruleSetNow,stopRuleSet);
                }
               /* else if(bNode.getState().equals(">")&&aNode.getState().equals("~")){
                      newTreatNode=recodeService.makeNode(bNode,aNode,wordLength,ruleSetNow,stopRuleSet);
                } */
            /*   else if(bNode.getState().equals(">")&&aNode.getState().equals("~")){
                      newTreatNode=recodeService.makeNode(bNode,aNode,wordLength,ruleSetNow);
                }
                else if(aNode.getState().equals("<>")&&bNode.getState().equals("~")){
                      newTreatNode=recodeService.makeNode(bNode,aNode,wordLength,ruleSetNow);
                }    */
                if(newTreatNode!=null){
                    oldNodeslist.add(newTreatNode);
                }
            }
        }
        List<TreatNode> tempTreatNodeList=new  ArrayList<TreatNode>();
        Iterator<TreatNode> iter=oldNodeslist.iterator();
         for(int i=0;i<oldNodeslist.size();i++){
             TreatNode root=iter.next();
             tempTreatNodeList.add(root);
         }
        if(nowLength<length){
        for(int j=0;j<tempTreatNodeList.size();j++)
            newNodeslist=this.addList(newNodeslist,tempTreatNodeList.get(j),wordLength);
        }
        else{
            for(int j=0;j<tempTreatNodeList.size();j++)
             newNodeslist=this.addOneNode(newNodeslist,tempTreatNodeList.get(j),wordLength);
        }
        NodesCell newNodesCell=new NodesCell();
        newNodesCell.setNodeList(newNodeslist);
        return newNodesCell;
    }
    
     public NodesCell getListNodeByNeighbour(List<SentenceCell> sentenceList,int length,List<PreTreatNode> priorListLog){
         List<NodesCell> cellList= this.getListNodes( sentenceList,length);
         NodesCell allCell=new NodesCell();
         List<TreatNode> nodeList=new ArrayList<TreatNode>();
         allCell.setCellNumber(length);
        for(int i=0;i<cellList.size();i++){
            NodesCell cell=cellList.get(i);
            List<TreatNode> treatNodeList=cell.getNodeList();
            for(int j=0;j<treatNodeList.size();j++){
                TreatNode one=treatNodeList.get(j);
                List<PreTreatNode> listNode=one.getContainNodes();
                boolean isOk=true;
                for(int v=0;v<priorListLog.size();v++){
                    if(!listNode.contains(priorListLog.get(v))){
                        isOk=false;
                        break;
                    }
                }
                if(isOk==true)
                  nodeList.add(one);
            }
        }
         allCell.setNodeList(nodeList);
        return allCell;
    }

    public List<NodesCell> getListNodes( List<SentenceCell> sentenceList,int length){
        List<NodesCell>  listAll=new ArrayList<NodesCell>();
        for(int i=0;i<sentenceList.size();i++){
             SentenceCell aSentenceCell=sentenceList.get(i);
             if(aSentenceCell.getWordLong()==length){
               return  aSentenceCell.getCellList();
             }
        }
        return listAll;
    }

    public List<TreatNode> addList(List<TreatNode> nowList,TreatNode nowNode,int length){
         nowList.add(nowNode);
         TreatNode treatNode= recodeService.makeNodeUseOneNode(nowNode,length,ruleSetNow,stopRuleSet);
         if(treatNode!=null){
             nowList.add(treatNode);
             TreatNode treatNode2= recodeService.makeNodeUseOneNode(treatNode,length,ruleSetNow,stopRuleSet);
             if(treatNode2!=null){
                 nowList.add(treatNode2);
                  TreatNode treatNode3= recodeService.makeNodeUseOneNode(treatNode2,length,ruleSetNow,stopRuleSet);
                 if(treatNode3==null)
                   return nowList;
                 else
                   return null;
             }
             else
                 return nowList;
         }
          return nowList;
    }

     public List<TreatNode> addOneNode(List<TreatNode> nowList,TreatNode nowNode,int length){
         TreatNode treatNode= recodeService.makeNodeUseOneNode(nowNode,length,ruleSetNow,stopRuleSet);
         if(treatNode!=null){
             TreatNode treatNode2= recodeService.makeNodeUseOneNode(treatNode,length,ruleSetNow,stopRuleSet);
             if(treatNode2!=null){
                 nowList.add(treatNode2);
                  TreatNode treatNode3= recodeService.makeNodeUseOneNode(treatNode2,length,ruleSetNow,stopRuleSet);
                 if(treatNode3==null)
                   return nowList;
                 else
                   return null;
             }
             else{
                 nowList.add(treatNode);
                 return nowList;
             }
         }
          return nowList;
    }

    public NodesCell getNbest(NodesCell old){
       NodesCell nodesCell=new NodesCell(); 
       List<TreatNode> nodeList=old.getNodeList();
       List<TreatNode> newNodeList=new ArrayList<TreatNode>();
       for(int i=0;i<nodeList.size();i++){
            for(int j=i+1;j<nodeList.size();j++){
                TreatNode aNode=nodeList.get(i);
                TreatNode bNode=nodeList.get(j);
                if(aNode.getInProperbility()>bNode.getInProperbility()){
                    TreatNode tempNode=(TreatNode)bNode.clone();
                    nodeList.remove(bNode);
                    nodeList.add(j,aNode);
                     nodeList.remove(aNode);
                    nodeList.add(i,bNode);
                }
            }
       }
       int index=0;
           for(int i=nodeList.size()-1;i>=0;i--){
              if(index++>nBest){
                 break;
              }
              newNodeList.add(nodeList.get(i));
           }
        nodesCell.setCellNumber(old.getCellNumber());
        nodesCell.setNodeList(newNodeList);
        return nodesCell;
       }


    public Set<RuleFull> getRuleSetNow() {
        return ruleSetNow;
    }

    public void setRuleSetNow(Set<RuleFull> ruleSetNow) {
        this.ruleSetNow = ruleSetNow;
    }

    public Set<StopRule> getStopRuleSet() {
        return stopRuleSet;
    }

    public void setStopRuleSet(Set<StopRule> stopRuleSet) {
        this.stopRuleSet = stopRuleSet;
    }
}