experimentalwarcwriter.html

一个开源的网页爬虫一个开源的网页爬虫一个开源的网页爬虫一个开源的网页爬虫一个开源的网页爬虫一个开源的网页爬虫

<PRE>
public <B>ExperimentalWARCWriter</B>(java.util.concurrent.atomic.AtomicInteger&nbsp;serialNo,
                              java.io.OutputStream&nbsp;out,
                              java.io.File&nbsp;f,
                              boolean&nbsp;cmprs,
                              java.lang.String&nbsp;a14DigitDate,
                              java.util.List&nbsp;warcinfoData)
                       throws java.io.IOException</PRE>
<DL>
<DD>Constructor.
 Takes a stream. Use with caution. There is no upperbound check on size.
 Will just keep writing.  Only pass Streams that are bounded.
<P>
<DL>
<DT><B>Parameters:</B><DD><CODE>serialNo</CODE> - used to generate unique file name sequences<DD><CODE>out</CODE> - Where to write.<DD><CODE>f</CODE> - File the <code>out</code> is connected to.<DD><CODE>cmprs</CODE> - Compress the content written.<DD><CODE>a14DigitDate</CODE> - If null, we'll write current time.
<DT><B>Throws:</B>
<DD><CODE>java.io.IOException</CODE></DL>
</DL>
<HR>

<A NAME="ExperimentalWARCWriter(java.util.concurrent.atomic.AtomicInteger, java.util.List, java.lang.String, java.lang.String, boolean, int, java.util.List)"><!-- --></A><H3>
ExperimentalWARCWriter</H3>
<PRE>
public <B>ExperimentalWARCWriter</B>(java.util.concurrent.atomic.AtomicInteger&nbsp;serialNo,
                              java.util.List&lt;java.io.File&gt;&nbsp;dirs,
                              java.lang.String&nbsp;prefix,
                              java.lang.String&nbsp;suffix,
                              boolean&nbsp;cmprs,
                              int&nbsp;maxSize,
                              java.util.List&nbsp;warcinfoData)</PRE>
<DL>
<DD>Constructor.
<P>
<DL>
<DT><B>Parameters:</B><DD><CODE>dirs</CODE> - Where to drop files.<DD><CODE>prefix</CODE> - File prefix to use.<DD><CODE>cmprs</CODE> - Compress the records written.<DD><CODE>maxSize</CODE> - Maximum size for ARC files written.<DD><CODE>suffix</CODE> - File tail to use.  If null, unused.<DD><CODE>warcinfoData</CODE> - File metadata for warcinfo record.</DL>
</DL>

<!-- ============ METHOD DETAIL ========== -->

<A NAME="method_detail"><!-- --></A>
<TABLE BORDER="1" WIDTH="100%" CELLPADDING="3" CELLSPACING="0" SUMMARY="">
<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
<TH ALIGN="left" COLSPAN="1"><FONT SIZE="+2">
<B>Method Detail</B></FONT></TH>
</TR>
</TABLE>

<A NAME="createFile(java.io.File)"><!-- --></A><H3>
createFile</H3>
<PRE>
protected java.lang.String <B>createFile</B>(java.io.File&nbsp;file)
                               throws java.io.IOException</PRE>
<DL>
<DD><DL>
<DT><B>Overrides:</B><DD><CODE><A HREF="../../../../org/archive/io/WriterPoolMember.html#createFile(java.io.File)">createFile</A></CODE> in class <CODE><A HREF="../../../../org/archive/io/WriterPoolMember.html" title="class in org.archive.io">WriterPoolMember</A></CODE></DL>
</DD>
<DD><DL>

<DT><B>Throws:</B>
<DD><CODE>java.io.IOException</CODE></DL>
</DD>
</DL>
<HR>

<A NAME="baseCharacterCheck(char, java.lang.String)"><!-- --></A><H3>
baseCharacterCheck</H3>
<PRE>
protected void <B>baseCharacterCheck</B>(char&nbsp;c,
                                  java.lang.String&nbsp;parameter)
                           throws java.io.IOException</PRE>
<DL>
<DD><DL>
</DL>
</DD>
<DD><DL>

<DT><B>Throws:</B>
<DD><CODE>java.io.IOException</CODE></DL>
</DD>
</DL>
<HR>

<A NAME="checkHeaderLineParameters(java.lang.String)"><!-- --></A><H3>
checkHeaderLineParameters</H3>
<PRE>
protected java.lang.String <B>checkHeaderLineParameters</B>(java.lang.String&nbsp;parameter)
                                              throws java.io.IOException</PRE>
<DL>
<DD><DL>
</DL>
</DD>
<DD><DL>

<DT><B>Throws:</B>
<DD><CODE>java.io.IOException</CODE></DL>
</DD>
</DL>
<HR>

<A NAME="checkHeaderLineMimetypeParameter(java.lang.String)"><!-- --></A><H3>
checkHeaderLineMimetypeParameter</H3>
<PRE>
protected java.lang.String <B>checkHeaderLineMimetypeParameter</B>(java.lang.String&nbsp;parameter)
                                                     throws java.io.IOException</PRE>
<DL>
<DD><DL>
</DL>
</DD>
<DD><DL>

<DT><B>Throws:</B>
<DD><CODE>java.io.IOException</CODE></DL>
</DD>
</DL>
<HR>

<A NAME="createRecordHeaderline(java.lang.String, java.lang.String, java.lang.String, java.lang.String, java.net.URI, int, long)"><!-- --></A><H3>
createRecordHeaderline</H3>
<PRE>
protected byte[] <B>createRecordHeaderline</B>(java.lang.String&nbsp;type,
                                        java.lang.String&nbsp;url,
                                        java.lang.String&nbsp;create14DigitDate,
                                        java.lang.String&nbsp;mimetype,
                                        java.net.URI&nbsp;recordId,
                                        int&nbsp;namedFieldsLength,
                                        long&nbsp;contentLength)
                                 throws java.io.IOException</PRE>
<DL>
<DD><DL>
</DL>
</DD>
<DD><DL>

<DT><B>Throws:</B>
<DD><CODE>java.io.IOException</CODE></DL>
</DD>
</DL>
<HR>

<A NAME="writeRecord(java.lang.String, java.lang.String, java.lang.String, java.lang.String, java.net.URI, org.archive.util.anvl.ANVLRecord, java.io.InputStream, long)"><!-- --></A><H3>
writeRecord</H3>
<PRE>
protected void <B>writeRecord</B>(java.lang.String&nbsp;type,
                           java.lang.String&nbsp;url,
                           java.lang.String&nbsp;create14DigitDate,
                           java.lang.String&nbsp;mimetype,
                           java.net.URI&nbsp;recordId,
                           <A HREF="../../../../org/archive/util/anvl/ANVLRecord.html" title="class in org.archive.util.anvl">ANVLRecord</A>&nbsp;namedFields,
                           java.io.InputStream&nbsp;contentStream,
                           long&nbsp;contentLength)
                    throws java.io.IOException</PRE>
<DL>
<DD><DL>
</DL>
</DD>
<DD><DL>

<DT><B>Throws:</B>
<DD><CODE>java.io.IOException</CODE></DL>
</DD>
</DL>
<HR>

<A NAME="generateRecordId(java.util.Map)"><!-- --></A><H3>
generateRecordId</H3>
<PRE>
protected java.net.URI <B>generateRecordId</B>(java.util.Map&lt;java.lang.String,java.lang.String&gt;&nbsp;qualifiers)
                                 throws java.io.IOException</PRE>
<DL>
<DD><DL>
</DL>
</DD>
<DD><DL>

<DT><B>Throws:</B>
<DD><CODE>java.io.IOException</CODE></DL>
</DD>
</DL>
<HR>

<A NAME="generateRecordId(java.lang.String, java.lang.String)"><!-- --></A><H3>
generateRecordId</H3>
<PRE>
protected java.net.URI <B>generateRecordId</B>(java.lang.String&nbsp;key,
                                        java.lang.String&nbsp;value)
                                 throws java.io.IOException</PRE>
<DL>
<DD><DL>
</DL>
</DD>
<DD><DL>

<DT><B>Throws:</B>
<DD><CODE>java.io.IOException</CODE></DL>
</DD>
</DL>
<HR>

<A NAME="writeWarcinfoRecord(java.lang.String)"><!-- --></A><H3>
writeWarcinfoRecord</H3>
<PRE>
public java.net.URI <B>writeWarcinfoRecord</B>(java.lang.String&nbsp;filename)
                                 throws java.io.IOException</PRE>
<DL>
<DD><DL>
</DL>
</DD>
<DD><DL>

<DT><B>Throws:</B>
<DD><CODE>java.io.IOException</CODE></DL>
</DD>
</DL>
<HR>

<A NAME="writeWarcinfoRecord(java.lang.String, java.lang.String)"><!-- --></A><H3>
writeWarcinfoRecord</H3>
<PRE>
public java.net.URI <B>writeWarcinfoRecord</B>(java.lang.String&nbsp;filename,
                                        java.lang.String&nbsp;description)
                                 throws java.io.IOException</PRE>
<DL>
<DD><DL>
</DL>
</DD>
<DD><DL>

<DT><B>Throws:</B>
<DD><CODE>java.io.IOException</CODE></DL>
</DD>
</DL>
<HR>

<A NAME="writeWarcinfoRecord(java.lang.String, org.archive.util.anvl.ANVLRecord, java.io.InputStream, long)"><!-- --></A><H3>
writeWarcinfoRecord</H3>
<PRE>
public java.net.URI <B>writeWarcinfoRecord</B>(java.lang.String&nbsp;mimetype,
                                        <A HREF="../../../../org/archive/util/anvl/ANVLRecord.html" title="class in org.archive.util.anvl">ANVLRecord</A>&nbsp;namedFields,
                                        java.io.InputStream&nbsp;fileMetadata,
                                        long&nbsp;fileMetadataLength)
                                 throws java.io.IOException</PRE>
<DL>