recordingoutputstream.java

这是个爬虫和lucece相结合最好了

        if(this.shouldDigest) {
            assert this.digest != null: "Digest is null.";
            this.digest.update(b, off, len);
        }
        tailRecord(b, off, len);
    }

    /**
     * Record without digesting.
     * 
     * @param b Buffer to record.
     * @param off Offset into buffer at which to start recording.
     * @param len Length of buffer to record.
     *
     * @exception IOException Failed write to backing file.
     */
    private void tailRecord(byte[] b, int off, int len) throws IOException {
        if(this.position >= this.buffer.length){
            // TODO: Its possible to call write w/o having first opened a
            // stream.  Lets protect ourselves against this.
            if (this.diskStream == null) {
                throw new IOException("diskstream is null");
            }
            this.diskStream.write(b, off, len);
            this.position += len;
        } else {
            assert this.buffer != null: "Buffer is null";
            int toCopy = (int)Math.min(this.buffer.length - this.position, len);
            assert b != null: "Passed buffer is null";
            System.arraycopy(b, off, this.buffer, (int)this.position, toCopy);
            this.position += toCopy;
            // TODO verify these are +1 -1 right
            if (toCopy < len) {
                tailRecord(b, off + toCopy, len - toCopy);
            }
        }
    }

    public void close() throws IOException {
        if(contentBeginMark<0) {
            // if unset, consider 0 posn as content-start
            // (so that a -1 never survives to replay step)
            contentBeginMark = 0;
        }
        if (this.out != null) {
            this.out.close();
            this.out = null;
        }
        closeRecorder();
    }
    
    protected synchronized void closeDiskStream()
    throws IOException {
        if (this.diskStream != null) {
            this.diskStream.close();
            this.diskStream = null;
        }
    }

    public void closeRecorder() throws IOException {
        recording = false;
        closeDiskStream(); // if any
        // This setting of size is important.  Its passed to ReplayInputStream
        // on creation.  It uses it to know EOS.
        if (this.size == 0) {
            this.size = this.position;
        }
    }

    /* (non-Javadoc)
     * @see java.io.OutputStream#flush()
     */
    public void flush() throws IOException {
        if (this.out != null) {
            this.out.flush();
        }
        if (this.diskStream != null) {
            this.diskStream.flush();
        }
    }

    public ReplayInputStream getReplayInputStream() throws IOException {
        return getReplayInputStream(0);
    }
    
    public ReplayInputStream getReplayInputStream(long skip) throws IOException {
        // If this method is being called, then assumption must be that the
        // stream is closed. If it ain't, then the stream gotten won't work
        // -- the size will zero so any attempt at a read will get back EOF.
        assert this.out == null: "Stream is still open.";
        ReplayInputStream replay = new ReplayInputStream(this.buffer, 
                this.size, this.contentBeginMark, this.backingFilename);
        replay.skip(skip);
        return replay; 
    }

    /**
     * Return a replay stream, cued up to begining of content
     *
     * @throws IOException
     * @return An RIS.
     */
    public ReplayInputStream getContentReplayInputStream() throws IOException {
        return getReplayInputStream(this.contentBeginMark);
    }

    public long getSize() {
        return this.size;
    }

    /**
     * Remember the current position as the start of the "response
     * body". Useful when recording HTTP traffic as a way to start
     * replays after the headers.
     */
    public void markContentBegin() {
        this.contentBeginMark = this.position;
        startDigest();
    }

    /**
     * Return stored content-begin-mark (which is also end-of-headers)
     */
    public long getContentBegin() {
        return this.contentBeginMark;
    }
    
    /**
     * Starts digesting recorded data, if a MessageDigest has been
     * set.
     */
    public void startDigest() {
        if (this.digest != null) {
            this.digest.reset();
            this.shouldDigest = true;
        }
    }

    /**
     * Convenience method for setting SHA1 digest.
     * @see #setDigest(String)
     */
    public void setSha1Digest() {
        setDigest(SHA1);
    }
    

    /**
     * Sets a digest function which may be applied to recorded data.
     * The difference between calling this method and {@link #setDigest(MessageDigest)}
     * is that this method tries to reuse MethodDigest instance if already allocated
     * and of appropriate algorithm.
     * @param algorithm Message digest algorithm to use.
     * @see #setDigest(MessageDigest)
     */
    public void setDigest(String algorithm) {
        try {
            // Reuse extant digest if its sha1 algorithm.
            if (this.digest == null ||
                    !this.digest.getAlgorithm().equals(algorithm)) {
                setDigest(MessageDigest.getInstance(algorithm));
            }
        } catch (NoSuchAlgorithmException e) {
            e.printStackTrace();
        }
    }

    /**
     * Sets a digest function which may be applied to recorded data.
     *
     * As usually only a subset of the recorded data should
     * be fed to the digest, you must also call startDigest()
     * to begin digesting.
     *
     * @param md Message digest function to use.
     */
    public void setDigest(MessageDigest md) {
        this.digest = md;
    }

    /**
     * Return the digest value for any recorded, digested data. Call
     * only after all data has been recorded; otherwise, the running
     * digest state is ruined.
     *
     * @return the digest final value
     */
    public byte[] getDigestValue() {
        if(this.digest == null) {
            return null;
        }
        return this.digest.digest();
    }

    public ReplayCharSequence getReplayCharSequence() throws IOException {
        return getReplayCharSequence(null);
    }

    public ReplayCharSequence getReplayCharSequence(String characterEncoding) 
    throws IOException {
        return getReplayCharSequence(characterEncoding, this.contentBeginMark);
    }
    
    /**
     * @param characterEncoding Encoding of recorded stream.
     * @return A ReplayCharSequence  Will return null if an IOException.  Call
     * close on returned RCS when done.
     * @throws IOException
     */
    public ReplayCharSequence getReplayCharSequence(String characterEncoding, 
            long startOffset) throws IOException {
        // TODO: handled transfer-encoding: chunked content-bodies properly
        float maxBytesPerChar = IoUtils.encodingMaxBytesPerChar(characterEncoding);
        if(maxBytesPerChar<=1) {
            // single
            // TODO: take into account single-byte encoding may be non-default
            return new ByteReplayCharSequence(
                    this.buffer, 
                    this.size, 
                    startOffset,
                    this.backingFilename);
        } else {
            // multibyte 
            if(this.size <= this.buffer.length) {
                // raw data is all in memory; do in memory
                return new MultiByteReplayCharSequence(
                        this.buffer, 
                        this.size, 
                        startOffset,
                        characterEncoding);
                
            } else {
                // raw data overflows to disk; use temp file
                ReplayInputStream ris = getReplayInputStream(startOffset);
                ReplayCharSequence rcs = new MultiByteReplayCharSequence(
                        ris, 
                        this.backingFilename,
                        characterEncoding);
                ris.close(); 
                return rcs;
            }
            
        }
        
    }

    public long getResponseContentLength() {
        return this.size - this.contentBeginMark;
    }

    /**
     * @return True if this ROS is open.
     */
    public boolean isOpen() {
        return this.out != null;
    }
    
    /**
     * When used alongside a mark-supporting RecordingInputStream, remember
     * a position reachable by a future reset().
     */
    public void mark() {
        // remember this position for subsequent reset()
        this.markPosition = position; 
    }
    
    /**
     * When used alongside a mark-supporting RecordingInputStream, reset 
     * the position to that saved by previous mark(). Until the position 
     * again reached "new" material, none of the bytes pushed to this 
     * stream will be digested or recorded. 
     */
    public void reset() {
        // take note of furthest-position-reached to avoid double-recording
        maxPosition = Math.max(maxPosition, position); 
        // reset to previous position
        position = markPosition;
    }
    
    /**
     * Set limits on length, time, and rate to enforce.
     * 
     * @param length
     * @param milliseconds
     * @param rateKBps
     */
    public void setLimits(long length, long milliseconds, long rateKBps) {
        maxLength = (length>0) ? length : Long.MAX_VALUE;
        timeoutMs = (milliseconds>0) ? milliseconds : Long.MAX_VALUE;
        maxRateBytesPerMs = (rateKBps>0) ? rateKBps*1024/1000 : Long.MAX_VALUE;
    }
    
    /**
     * Reset limits to effectively-unlimited defaults
     */
    public void resetLimits() {
        maxLength = Long.MAX_VALUE;
        timeoutMs = Long.MAX_VALUE;
        maxRateBytesPerMs = Long.MAX_VALUE;
    }
    
    /**
     * Return number of bytes that could be recorded without hitting 
     * length limit
     * 
     * @return long byte count
     */
    public long getRemainingLength() {
        return maxLength - position; 
    }
}