mirrorwriterprocessor.java

Heritrix是一个开源,可扩展的web爬虫项目。Heritrix设计成严格按照robots.txt文件的排除指示和META robots标签。

       consisting of substrings ("lumps") that
       are treated atomically.  If the string is shortened, then an entire
       lump is removed.  The intent is to treat each %XX escape as a lump.
       This class also allows single characters in a source string to be
       re-mapped to a different string, possible containing more than
       one character.
       Each re-mapped character is also treated as a lump.
       <p>
       For example, suppose part of a URI, between two slashes, is
       <code>/VeryLongString...%3A/</code>.
       We want to create a corresponding file system directory, but the string
       is a little longer than the allowed maximum.
       It's better to trim the entire
       <code>%3A</code>
       off the end than part of it.
       This is especially true if, later, we need to append some digits
       to create a unique directory name.
       So we treat the entire
       <code>%3A</code>
       as one lump.
    */
    class LumpyString {
        /**
           Lumps are indicated by an auxiliary array aux[],
           indexed the same as the string.  The LUMP_BEGIN bit is set
           for a position in the string at which a lump begins.
        */
        private static final byte LUMP_BEGIN = 0x1;

        /** Bit set for the end of a lump. */
        private static final byte LUMP_END = 0x2;

        /**
           Bit set for all characters in a lump of length greater than 1,
           except the beginning and ending characters.
        */
        private static final byte LUMP_MID = 0x4;

        /** The auxiliary array. */
        private byte[] aux;

        /** Holds the string. */
        private StringBuffer string;

        /**
           Creates a LumpyString.
           @param str the source string
           @param beginIndex the beginning index, inclusive, of the substring
           of str to be used
           @param endIndex the ending index, exclusive, of the substring
           of str to be used
           @param padding reserve this many additional character positions
           before dynamic growth is needed
           @param maxLen the maximum string length, regardless of the
           values of beginIndex, endIndex, and padding
           @param characterMap maps from characters in the source string
           (represented as length-one String values) to replacement String
           values (length at least 1).
           Each replacement string is treated as one lump.
           This is intended to cope with characters that a file system
           does not allow.
           @param dotBegin if non-null, this replaces a '.' at
           <code>str[beginIndex]</code>
           @throws IllegalArgumentException if
           beginIndex is negative.
           @throws IllegalArgumentException if
           endIndex is less than beginIndex.
           @throws IllegalArgumentException if
           padding is negative.
           @throws IllegalArgumentException if
           maxLen is less than one.
           @throws IllegalArgumentException if
           characterMap is null.
           @throws IllegalArgumentException if
           dotBegin is non-null but empty.
        */
        LumpyString(String str, int beginIndex, int endIndex, int padding,
                    int maxLen, Map characterMap, String dotBegin) {
            if (beginIndex < 0) {
                throw new IllegalArgumentException("beginIndex < 0: "
                                                   + beginIndex);
            }
            if (endIndex < beginIndex) {
                throw new IllegalArgumentException("endIndex < beginIndex "
                    + "beginIndex: " + beginIndex + "endIndex: " + endIndex);
            }
            if (padding < 0) {
                throw new IllegalArgumentException("padding < 0: " + padding);
            }
            if (maxLen < 1) {
                throw new IllegalArgumentException("maxLen < 1: " + maxLen);
            }
            if (null == characterMap) {
                throw new IllegalArgumentException("characterMap null");
            }
            if ((null != dotBegin) && (0 == dotBegin.length())) {
                throw new IllegalArgumentException("dotBegin empty");
            }

            // Initial capacity.  Leave some room for %XX lumps.
            // Guaranteed positive.
            int cap = Math.min(2 * (endIndex - beginIndex) + padding + 1,
                               maxLen);
            string = new StringBuffer(cap);
            aux = new byte[cap];
            for (int i = beginIndex; i != endIndex; ++i) {
                String s = str.substring(i, i + 1);
                String lump; // Next lump.
                if (".".equals(s) && (i == beginIndex) && (null != dotBegin)) {
                    lump = dotBegin;
                } else {
                    lump = (String) characterMap.get(s);
                }
                if (null == lump) {
                    if ("%".equals(s) && ((endIndex - i) > 2)
                            && (-1 != Character.digit(str.charAt(i + 1), 16))
                            && (-1 != Character.digit(str.charAt(i + 2), 16))) {

                        // %XX escape; treat as one lump.
                        lump = str.substring(i, i + 3);
                        i += 2;
                    } else {
                        lump = s;
                    }
                }
                if ((string.length() + lump.length()) > maxLen) {
                    assert checkInvariants();
                    return;
                }
                append(lump);
            }
            assert checkInvariants();
        }

        /**
           Converts this LumpyString to a String.
           @return the current string contents
        */
        public String toString() {
            assert checkInvariants();
            return string.toString();
        }

        /**
           Appends one lump to the end of this string.
           @param lump the lump (substring) to append
           @throws IllegalArgumentException if
           lump is null or empty.
        */
        void append(String lump) {
            if (null == lump) {
                throw new IllegalArgumentException("lump null");
            }
            int lumpLen = lump.length();
            if (0 == lumpLen) {
                throw new IllegalArgumentException("lump empty");
            }
            int pos = string.length(); // Current end of string.
            ensureCapacity(pos + lumpLen);
            if (1 == lumpLen) {
                aux[pos] = LUMP_BEGIN | LUMP_END;
            } else {
                assert lumpLen > 1;
                aux[pos] = LUMP_BEGIN;
                ++pos;
                for (int i = lumpLen - 2; 0 != i; --i) {
                    aux[pos] = LUMP_MID;
                    ++pos;
                }
                aux[pos] = LUMP_END;
            }
            string.append(lump);
            assert checkInvariants();
        }

        /**
           Returns the string as a StringBuffer.
           The caller should <em>not</em> modify the return value.
           @return the string
        */
        StringBuffer asStringBuffer() {
            return string;
        }

        /**
           Tests if this string ends with a character.
           @param ch the character to test for
           @return true if and only if this string ends with ch
        */
        boolean endsWith(char ch) {
            assert checkInvariants();
            int len = string.length();
            return (0 != len) && (string.charAt(len - 1) == ch);
        }

        /**
           Prepends one character, as a lump, to this string.
           @param ch the character to prepend
        */
        void prepend(char ch) {
            assert checkInvariants();
            int oldLen = string.length();
            ensureCapacity(1 + oldLen);
            string.insert(0, ch);
            System.arraycopy(aux, 0, aux, 1, oldLen);
            aux[0] = LUMP_BEGIN | LUMP_END;
            assert checkInvariants();
        }

        /**
           Gets the length of this string.
           @return the number of characters in this string
        */
        int length() {
            assert checkInvariants();
            return string.length();
        }

        /**
           If necessary, trims this string to a maximum length.
           Any trimming is done by removing one or more complete
           lumps from the end of this string.
           @param maxLen the new maximum length.
           After trimming, the actual length of this string will be
           at most maxLen.
           @throws IllegalArgumentException if
           maxLen is negative.
        */
        void trimToMax(int maxLen) {
            if (maxLen < 0) {
                throw new IllegalArgumentException("maxLen < 0: " + maxLen);
            }
            assert checkInvariants();
            int cl = string.length(); // Current length.
            if (cl > maxLen) {
                int nl = maxLen; // New length.
                while ((0 != nl) && (LUMP_END != (aux[nl - 1] & LUMP_END))) {
                    --nl;
                }
                for (int i = nl; i != cl; ++i) {
                    aux[i] = 0;
                }
                string.setLength(nl);
            }
            assert checkInvariants();
        }

        /**
           Checks some assertions on the instance variables.
           The intended usage is
           <code>assert checkInvariants();</code>
           so that if assertions are off, no call is made.
           @return true
        */
        private boolean checkInvariants() {

            // There's an aux[] element for every character in the StringBuffer.
            assert aux.length >= string.length()
                : "aux.length: " + aux.length
                + " string.length(): " + string.length();

            // The first character starts a lump.
            assert (0 == string.length())
                || (LUMP_BEGIN == (aux[0] & LUMP_BEGIN))
                : "aux[0]: " + aux[0];

            // The last character ends a lump.
            assert (0 == string.length())
                || (LUMP_END == (aux[string.length() - 1] & LUMP_END))
                : "aux[end]: " + aux[string.length() - 1];
            return true;
        }

        /**
           Ensures that the capacity is at least equal to the specified minimum.
           @param minCapacity the minimum desired capacity
        */
        private void ensureCapacity(int minCapacity) {
            assert checkInvariants();
            if (minCapacity > aux.length) {
                int nc = 2 * aux.length; // New capacity.
                while (nc < minCapacity) {
                    nc *= 2;
                }
                byte[] oldAux = aux;
                aux = new byte[nc];
                System.arraycopy(oldAux, 0, aux, 0, string.length());
            }
            string.ensureCapacity(minCapacity);
            assert checkInvariants();
        }
    }

    /**
       This class is returned by uriToFile.
       It represents a file system path, both as a File and as
       a path relative to the base directory.
    */
    class URIToFileReturn {
        /** The file system path as a File.*/
        private File filePath;

        /** The relative path from baseDir.*/
        private StringBuffer relativePath = new StringBuffer(255);

        /**
           Creates a URIToFileReturn.
           @param baseDir the path to the starting directory
           @param host the host part of the URI, or null if the host name
           should not be part of the path
           @param port the port part of the URI, or -1 if the port
           should not be part of the path
        */
        URIToFileReturn(String baseDir, String host, int port) {

            // The initial path.
            StringBuffer startPath = new StringBuffer(baseDir.length() + 32);
            startPath.append(baseDir);
            if (baseDir.endsWith(File.separator)) {
                assert 1 != baseDir.length();
                startPath.deleteCharAt(startPath.len