experimentalwarcwriter.html

一个开源的网页爬虫一个开源的网页爬虫一个开源的网页爬虫一个开源的网页爬虫一个开源的网页爬虫一个开源的网页爬虫

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>protected &nbsp;java.lang.String</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../org/archive/io/warc/ExperimentalWARCWriter.html#createFile(java.io.File)">createFile</A></B>(java.io.File&nbsp;file)</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>protected &nbsp;byte[]</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../org/archive/io/warc/ExperimentalWARCWriter.html#createRecordHeaderline(java.lang.String, java.lang.String, java.lang.String, java.lang.String, java.net.URI, int, long)">createRecordHeaderline</A></B>(java.lang.String&nbsp;type,
                       java.lang.String&nbsp;url,
                       java.lang.String&nbsp;create14DigitDate,
                       java.lang.String&nbsp;mimetype,
                       java.net.URI&nbsp;recordId,
                       int&nbsp;namedFieldsLength,
                       long&nbsp;contentLength)</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>protected &nbsp;java.net.URI</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../org/archive/io/warc/ExperimentalWARCWriter.html#generateRecordId(java.util.Map)">generateRecordId</A></B>(java.util.Map&lt;java.lang.String,java.lang.String&gt;&nbsp;qualifiers)</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>protected &nbsp;java.net.URI</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../org/archive/io/warc/ExperimentalWARCWriter.html#generateRecordId(java.lang.String, java.lang.String)">generateRecordId</A></B>(java.lang.String&nbsp;key,
                 java.lang.String&nbsp;value)</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>static&nbsp;java.net.URI</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../org/archive/io/warc/ExperimentalWARCWriter.html#getRecordID()">getRecordID</A></B>()</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Convenience method for getting Record-Ids.</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>&nbsp;void</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../org/archive/io/warc/ExperimentalWARCWriter.html#writeMetadataRecord(java.lang.String, java.lang.String, java.lang.String, java.net.URI, org.archive.util.anvl.ANVLRecord, java.io.InputStream, long)">writeMetadataRecord</A></B>(java.lang.String&nbsp;url,
                    java.lang.String&nbsp;create14DigitDate,
                    java.lang.String&nbsp;mimetype,
                    java.net.URI&nbsp;recordId,
                    <A HREF="../../../../org/archive/util/anvl/ANVLRecord.html" title="class in org.archive.util.anvl">ANVLRecord</A>&nbsp;namedFields,
                    java.io.InputStream&nbsp;metadata,
                    long&nbsp;metadataLength)</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>protected &nbsp;void</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../org/archive/io/warc/ExperimentalWARCWriter.html#writeRecord(java.lang.String, java.lang.String, java.lang.String, java.lang.String, java.net.URI, org.archive.util.anvl.ANVLRecord, java.io.InputStream, long)">writeRecord</A></B>(java.lang.String&nbsp;type,
            java.lang.String&nbsp;url,
            java.lang.String&nbsp;create14DigitDate,
            java.lang.String&nbsp;mimetype,
            java.net.URI&nbsp;recordId,
            <A HREF="../../../../org/archive/util/anvl/ANVLRecord.html" title="class in org.archive.util.anvl">ANVLRecord</A>&nbsp;namedFields,
            java.io.InputStream&nbsp;contentStream,
            long&nbsp;contentLength)</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>&nbsp;void</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../org/archive/io/warc/ExperimentalWARCWriter.html#writeRequestRecord(java.lang.String, java.lang.String, java.lang.String, java.net.URI, org.archive.util.anvl.ANVLRecord, java.io.InputStream, long)">writeRequestRecord</A></B>(java.lang.String&nbsp;url,
                   java.lang.String&nbsp;create14DigitDate,
                   java.lang.String&nbsp;mimetype,
                   java.net.URI&nbsp;recordId,
                   <A HREF="../../../../org/archive/util/anvl/ANVLRecord.html" title="class in org.archive.util.anvl">ANVLRecord</A>&nbsp;namedFields,
                   java.io.InputStream&nbsp;request,
                   long&nbsp;requestLength)</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>&nbsp;void</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../org/archive/io/warc/ExperimentalWARCWriter.html#writeResourceRecord(java.lang.String, java.lang.String, java.lang.String, org.archive.util.anvl.ANVLRecord, java.io.InputStream, long)">writeResourceRecord</A></B>(java.lang.String&nbsp;url,
                    java.lang.String&nbsp;create14DigitDate,
                    java.lang.String&nbsp;mimetype,
                    <A HREF="../../../../org/archive/util/anvl/ANVLRecord.html" title="class in org.archive.util.anvl">ANVLRecord</A>&nbsp;namedFields,
                    java.io.InputStream&nbsp;response,
                    long&nbsp;responseLength)</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>&nbsp;void</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../org/archive/io/warc/ExperimentalWARCWriter.html#writeResourceRecord(java.lang.String, java.lang.String, java.lang.String, java.net.URI, org.archive.util.anvl.ANVLRecord, java.io.InputStream, long)">writeResourceRecord</A></B>(java.lang.String&nbsp;url,
                    java.lang.String&nbsp;create14DigitDate,
                    java.lang.String&nbsp;mimetype,
                    java.net.URI&nbsp;recordId,
                    <A HREF="../../../../org/archive/util/anvl/ANVLRecord.html" title="class in org.archive.util.anvl">ANVLRecord</A>&nbsp;namedFields,
                    java.io.InputStream&nbsp;response,
                    long&nbsp;responseLength)</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>&nbsp;void</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../org/archive/io/warc/ExperimentalWARCWriter.html#writeResponseRecord(java.lang.String, java.lang.String, java.lang.String, java.net.URI, org.archive.util.anvl.ANVLRecord, java.io.InputStream, long)">writeResponseRecord</A></B>(java.lang.String&nbsp;url,
                    java.lang.String&nbsp;create14DigitDate,
                    java.lang.String&nbsp;mimetype,
                    java.net.URI&nbsp;recordId,
                    <A HREF="../../../../org/archive/util/anvl/ANVLRecord.html" title="class in org.archive.util.anvl">ANVLRecord</A>&nbsp;namedFields,
                    java.io.InputStream&nbsp;response,
                    long&nbsp;responseLength)</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>&nbsp;java.net.URI</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../org/archive/io/warc/ExperimentalWARCWriter.html#writeWarcinfoRecord(java.lang.String)">writeWarcinfoRecord</A></B>(java.lang.String&nbsp;filename)</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>&nbsp;java.net.URI</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../org/archive/io/warc/ExperimentalWARCWriter.html#writeWarcinfoRecord(java.lang.String, org.archive.util.anvl.ANVLRecord, java.io.InputStream, long)">writeWarcinfoRecord</A></B>(java.lang.String&nbsp;mimetype,
                    <A HREF="../../../../org/archive/util/anvl/ANVLRecord.html" title="class in org.archive.util.anvl">ANVLRecord</A>&nbsp;namedFields,
                    java.io.InputStream&nbsp;fileMetadata,
                    long&nbsp;fileMetadataLength)</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Write a warcinfo to current file.</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>&nbsp;java.net.URI</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../org/archive/io/warc/ExperimentalWARCWriter.html#writeWarcinfoRecord(java.lang.String, java.lang.String)">writeWarcinfoRecord</A></B>(java.lang.String&nbsp;filename,
                    java.lang.String&nbsp;description)</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</TD>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
<CODE>&nbsp;void</CODE></FONT></TD>
<TD><CODE><B><A HREF="../../../../org/archive/io/warc/ExperimentalWARCWriter.html#writeWarcinfoRecord(java.lang.String, java.lang.String, java.net.URI, org.archive.util.anvl.ANVLRecord, java.io.InputStream, long)">writeWarcinfoRecord</A></B>(java.lang.String&nbsp;create14DigitDate,
                    java.lang.String&nbsp;mimetype,
                    java.net.URI&nbsp;recordId,
                    <A HREF="../../../../org/archive/util/anvl/ANVLRecord.html" title="class in org.archive.util.anvl">ANVLRecord</A>&nbsp;namedFields,
                    java.io.InputStream&nbsp;fileMetadata,
                    long&nbsp;fileMetadataLength)</CODE>

<BR>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Write a <code>warcinfo</code> to current file.</TD>
</TR>
</TABLE>
&nbsp;<A NAME="methods_inherited_from_class_org.archive.io.WriterPoolMember"><!-- --></A>
<TABLE BORDER="1" WIDTH="100%" CELLPADDING="3" CELLSPACING="0" SUMMARY="">
<TR BGCOLOR="#EEEEFF" CLASS="TableSubHeadingColor">
<TH ALIGN="left"><B>Methods inherited from class org.archive.io.<A HREF="../../../../org/archive/io/WriterPoolMember.html" title="class in org.archive.io">WriterPoolMember</A></B></TH>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD><CODE><A HREF="../../../../org/archive/io/WriterPoolMember.html#checkSize()">checkSize</A>, <A HREF="../../../../org/archive/io/WriterPoolMember.html#checkWriteable(java.io.File)">checkWriteable</A>, <A HREF="../../../../org/archive/io/WriterPoolMember.html#close()">close</A>, <A HREF="../../../../org/archive/io/WriterPoolMember.html#createFile()">createFile</A>, <A HREF="../../../../org/archive/io/WriterPoolMember.html#flush()">flush</A>, <A HREF="../../../../org/archive/io/WriterPoolMember.html#getBaseFilename()">getBaseFilename</A>, <A HREF="../../../../org/archive/io/WriterPoolMember.html#getCreateTimestamp()">getCreateTimestamp</A>, <A HREF="../../../../org/archive/io/WriterPoolMember.html#getFile()">getFile</A>, <A HREF="../../../../org/archive/io/WriterPoolMember.html#getNextDirectory(java.util.List)">getNextDirectory</A>, <A HREF="../../../../org/archive/io/WriterPoolMember.html#getOutputStream()">getOutputStream</A>, <A HREF="../../../../org/archive/io/WriterPoolMember.html#getPosition()">getPosition</A>, <A HREF="../../../../org/archive/io/WriterPoolMember.html#getTimestampSerialNo()">getTimestampSerialNo</A>, <A HREF="../../../../org/archive/io/WriterPoolMember.html#getTimestampSerialNo(java.lang.String)">getTimestampSerialNo</A>, <A HREF="../../../../org/archive/io/WriterPoolMember.html#isCompressed()">isCompressed</A>, <A HREF="../../../../org/archive/io/WriterPoolMember.html#postWriteRecordTasks()">postWriteRecordTasks</A>, <A HREF="../../../../org/archive/io/WriterPoolMember.html#preWriteRecordTasks()">preWriteRecordTasks</A>, <A HREF="../../../../org/archive/io/WriterPoolMember.html#readFullyFrom(java.io.InputStream, long, byte[])">readFullyFrom</A>, <A HREF="../../../../org/archive/io/WriterPoolMember.html#write(byte[])">write</A>, <A HREF="../../../../org/archive/io/WriterPoolMember.html#write(byte[], int, int)">write</A>, <A HREF="../../../../org/archive/io/WriterPoolMember.html#write(int)">write</A></CODE></TD>
</TR>
</TABLE>
&nbsp;<A NAME="methods_inherited_from_class_java.lang.Object"><!-- --></A>
<TABLE BORDER="1" WIDTH="100%" CELLPADDING="3" CELLSPACING="0" SUMMARY="">
<TR BGCOLOR="#EEEEFF" CLASS="TableSubHeadingColor">
<TH ALIGN="left"><B>Methods inherited from class java.lang.Object</B></TH>
</TR>
<TR BGCOLOR="white" CLASS="TableRowColor">
<TD><CODE>clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait</CODE></TD>
</TR>
</TABLE>
&nbsp;
<P>

<!-- ============ FIELD DETAIL =========== -->

<A NAME="field_detail"><!-- --></A>
<TABLE BORDER="1" WIDTH="100%" CELLPADDING="3" CELLSPACING="0" SUMMARY="">
<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
<TH ALIGN="left" COLSPAN="1"><FONT SIZE="+2">
<B>Field Detail</B></FONT></TH>
</TR>
</TABLE>

<A NAME="CRLF_BYTES"><!-- --></A><H3>
CRLF_BYTES</H3>
<PRE>
public static byte[] <B>CRLF_BYTES</B></PRE>
<DL>
<DD>NEWLINE as bytes.
<P>
<DL>
</DL>
</DL>

<!-- ========= CONSTRUCTOR DETAIL ======== -->

<A NAME="constructor_detail"><!-- --></A>
<TABLE BORDER="1" WIDTH="100%" CELLPADDING="3" CELLSPACING="0" SUMMARY="">
<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
<TH ALIGN="left" COLSPAN="1"><FONT SIZE="+2">
<B>Constructor Detail</B></FONT></TH>
</TR>
</TABLE>

<A NAME="ExperimentalWARCWriter()"><!-- --></A><H3>
ExperimentalWARCWriter</H3>
<PRE>
<B>ExperimentalWARCWriter</B>()</PRE>
<DL>
<DD>Shutdown Constructor
 Has default access so can make instance to test utility methods.
<P>
</DL>
<HR>

<A NAME="ExperimentalWARCWriter(java.util.concurrent.atomic.AtomicInteger, java.io.OutputStream, java.io.File, boolean, java.lang.String, java.util.List)"><!-- --></A><H3>
ExperimentalWARCWriter</H3>