intnode.java

一个自然语言处理的Java开源工具包。LingPipe目前已有很丰富的功能

        }
        IntNode dtr = new IntNode(tokIndices,start+1,end);
        if (tokIndices[start] < mTok)
            return new DtrMap2(tokIndices[start],mTok,
                               dtr,mDtr);
        return new DtrMap2(mTok,tokIndices[start],
                           mDtr,dtr);
    }
    public DtrMap incrementDtrs(int[] tokIndices, int start, int end, 
                                int count) {
        if (start == end) return this;
        if (tokIndices[start] == mTok) {
            mDtr.increment(tokIndices,start+1,end,count);
            return this;
        }
        IntNode dtr = new IntNode(tokIndices,start+1,end,count);
        if (tokIndices[start] < mTok)
            return new DtrMap2(tokIndices[start],mTok,
                               dtr,mDtr);
        return new DtrMap2(mTok,tokIndices[start],
                           mDtr,dtr);
    }
    public DtrMap incrementSequence(int[] tokIndices, int start, int end, 
                                    int count) {
        if (start == end) return this;
        if (tokIndices[start] == mTok) {
            mDtr.incrementSequence(tokIndices,start+1,end,count);
            return this;
        }
        IntNode dtr = new IntNode(tokIndices,start+1,end,count,false);
        if (tokIndices[start] < mTok)
            return new DtrMap2(tokIndices[start],mTok,
                               dtr,mDtr);
        return new DtrMap2(mTok,tokIndices[start],
                           mDtr,dtr);
    }
    public long extensionCount() {
        return mDtr.count();
    }
    public int[] integersFollowing() {
        return new int[] { mTok };
    }
}

class DtrMap2 implements DtrMap {
    final int mTok1;
    final int mTok2;
    IntNode mDtr1;
    IntNode mDtr2;
    public DtrMap2(int tok1,
                   int tok2,
                   IntNode dtr1,
                   IntNode dtr2) {
        mTok1 = tok1;
        mDtr1 = dtr1;
        mTok2 = tok2;
        mDtr2 = dtr2;
    }
    public DtrMap prune(int minCount) {
        if (mDtr1.count() < minCount) {
            if (mDtr2.count() < minCount)
                return DtrMap0.EMPTY_DTR_MAP;
            mDtr2.prune(minCount);
            return new DtrMap1(mTok2,mDtr2);
        }
        mDtr1.prune(minCount);
        if (mDtr2.count() < minCount)
            return new DtrMap1(mTok1,mDtr1);
        mDtr2.prune(minCount);
        return this;
    }
    public DtrMap rescale(double countMultiplier) {
        mDtr1.rescale(countMultiplier);
        mDtr2.rescale(countMultiplier);
        if (mDtr1.count() == 0) {
            if (mDtr2.count() == 0)
                return DtrMap0.EMPTY_DTR_MAP;
            return new DtrMap1(mTok2,mDtr2);
        }
        if (mDtr2.count() == 0)
            return new DtrMap1(mTok1,mDtr1);
        return this;
    }
    public int numExtensions() {
        return 2;
    }
    public void toString(StringBuffer sb, int depth, SymbolTable st) {
        if (st != null)
            sb.append(IntNode.idToSymbol(mTok1,st));
        else 
            sb.append(mTok1);
        sb.append(": ");
        mDtr1.toString(sb,depth+1,st);
        AbstractNode.indent(sb,depth);
        if (st != null) 
            sb.append(IntNode.idToSymbol(mTok2,st));
        else
            sb.append(mTok2);
        sb.append(": ");
        mDtr2.toString(sb,depth+1,st);
    }
    public void addDtrs(List queue) {
        queue.add(mDtr1);
        queue.add(mDtr2);
    }
    public int dtrsTrieSize() {
        return mDtr1.trieSize() + mDtr2.trieSize();
    }
    public IntNode getDtr(int tok) {
        if (tok == mTok1) return mDtr1;
        if (tok == mTok2) return mDtr2;
        return null;
    }
    public DtrMap incrementDtrs(int[] tokIndices, int start, int end) {
        if (start == end) return this;
        int tok = tokIndices[start];
        if (tok == mTok1) {
            mDtr1.increment(tokIndices,start+1,end);
            return this;
        }
        if (tok == mTok2) {
            mDtr2.increment(tokIndices,start+1,end);
            return this;
        }
        IntNode dtr = new IntNode(tokIndices,start+1,end);
        return new DtrMapMap(tok, mTok1, mTok2,
                             dtr, mDtr1, mDtr2); // ordered by map
    }
    public DtrMap incrementDtrs(int[] tokIndices, int start, int end, int count) {
        if (start == end) return this;
        int tok = tokIndices[start];
        if (tok == mTok1) {
            mDtr1.increment(tokIndices,start+1,end,count);
            return this;
        }
        if (tok == mTok2) {
            mDtr2.increment(tokIndices,start+1,end,count);
            return this;
        }
        IntNode dtr = new IntNode(tokIndices,start+1,end,count);
        return new DtrMapMap(tok, mTok1, mTok2,
                             dtr, mDtr1, mDtr2); // ordered by map
    }
    public DtrMap incrementSequence(int[] tokIndices, int start, int end,
                                    int count) {
        if (start == end) return this;
        int tok = tokIndices[start];
        if (tok == mTok1) {
            mDtr1.incrementSequence(tokIndices,start+1,end,count);
            return this;
        }
        if (tok == mTok2) {
            mDtr2.incrementSequence(tokIndices,start+1,end,count);
        }
        IntNode dtr = new IntNode(tokIndices,start+1,end,count,false);
        return new DtrMapMap(tok, mTok1, mTok2,
                             dtr, mDtr1, mDtr2);
    }
    public int[] integersFollowing() {
        return new int[] { mTok1, mTok2 };
    }
    public long extensionCount() {
        return ((long) mDtr1.count())
            + (long) mDtr2.count();
    }
}

class DtrMapMap extends TreeMap implements DtrMap {
	static final long serialVersionUID = -840053413688713070L;
    DtrMapMap(int out1, int out2, int out3,
              IntNode dtr1, IntNode dtr2, IntNode dtr3) {
        add(out1,dtr1);
        add(out2,dtr2);
        add(out3,dtr3);
    }
    public DtrMap prune(int minCount) {
        Iterator entryIt = entrySet().iterator();
        while (entryIt.hasNext()) {
            Map.Entry entry = (Map.Entry) entryIt.next();
            IntNode node = (IntNode) entry.getValue();
            if (node.count() < minCount)
                entryIt.remove();
            node.prune(minCount);
        }
        return reduce();
    }
    public DtrMap rescale(double countMultiplier) {
        Iterator entryIt = entrySet().iterator();
        while (entryIt.hasNext()) {
            Map.Entry entry = (Map.Entry) entryIt.next();
            IntNode node = (IntNode) entry.getValue();
            node.rescale(countMultiplier);
            if (node.count() == 0)
                entryIt.remove();
        }
        return reduce();
    }

    public DtrMap reduce() {
        if (size() == 0) 
            return DtrMap0.EMPTY_DTR_MAP;
        if (size() == 1) {
            Iterator entryIt = entrySet().iterator();
            Map.Entry entry = (Map.Entry) entryIt.next();
            int token = ((Integer)(entry.getKey())).intValue();
            IntNode node = (IntNode) entry.getValue();
            return new DtrMap1(token,node);
        } 
        if (size() == 2) {
            Iterator entryIt = entrySet().iterator();
            Map.Entry entry1 = (Map.Entry) entryIt.next();
            int token1 = ((Integer)(entry1.getKey())).intValue();
            IntNode node1 = (IntNode) entry1.getValue();
            Map.Entry entry2 = (Map.Entry) entryIt.next();
            int token2 = ((Integer)(entry2.getKey())).intValue();
            IntNode node2 = (IntNode) entry2.getValue();
            return new DtrMap2(token1,token2,node1,node2);
        }
        return this;
    }

    public int numExtensions() {
        return size();
    }
    private void add(int out, IntNode dtr) {
        put(new Integer(out),dtr);
    }
    public void toString(StringBuffer sb, int depth, SymbolTable st) {
        Iterator it = entrySet().iterator();
        for (int i = 0; it.hasNext(); ++i) {
            if (i > 0) 
                AbstractNode.indent(sb,depth);
            Map.Entry entry = (Map.Entry) it.next();
            Integer key = (Integer) entry.getKey();
            int tok = key.intValue();
            IntNode node = (IntNode) entry.getValue();
            if (st != null)
                sb.append(IntNode.idToSymbol(tok,st));
            else
                sb.append(tok);
            sb.append(": ");
            node.toString(sb,depth+1,st);
        }
    }
    public void addDtrs(List queue) {
        queue.addAll(values());
    }
    public int dtrsTrieSize() {
        int size = 0;
        Iterator it = values().iterator();
        while (it.hasNext())
            size += ((IntNode)it.next()).trieSize();
        return size;
    }
    public DtrMap incrementDtrs(int[] tokIndices, int start, int end) {
        if (start == end) return this;
        Integer tok = new Integer(tokIndices[start]);
        IntNode dtr = getNode(tok);
        if (dtr == null) 
            put(tok,new IntNode(tokIndices,start+1,end));
        else 
            dtr.increment(tokIndices,start+1,end);
        return this;
    }
    public DtrMap incrementDtrs(int[] tokIndices, int start, int end, 
                                int count) {
        if (start == end) return this;
        Integer tok = new Integer(tokIndices[start]);
        IntNode dtr = getNode(tok);
        if (dtr == null) 
            put(tok,new IntNode(tokIndices,start+1,end,count));
        else 
            dtr.increment(tokIndices,start+1,end,count);
        return this;
    }
    public DtrMap incrementSequence(int[] tokIndices, int start, int end,
                                    int count) {
        if (start == end) return this;
        Integer tok = new Integer(tokIndices[start]);
        IntNode dtr = getNode(tok);
        if (dtr == null)
            put(tok,new IntNode(tokIndices,start+1,end,count,false));
        else
            dtr.incrementSequence(tokIndices,start+1,end,count);
        return this;
    }
    public IntNode getDtr(int tok) {
        return getNode(new Integer(tok));
    }
    IntNode getNode(Integer tok) {
        return (IntNode) get(tok);
    }
    public int[] integersFollowing() {
        int[] result = new int[keySet().size()];
        Iterator it = keySet().iterator();
        for (int i = 0; it.hasNext(); ++i)
            result[i] = ((Integer) it.next()).intValue();
        return result;
    }
    public long extensionCount() {
        long extensionCount = 0l;
        Iterator it = values().iterator();
        while (it.hasNext())
            extensionCount += ((IntNode)it.next()).count();
        return extensionCount;
    }
}