📄 extractreuters.java
字号:
package org.apache.lucene.benchmark.utils;/** * Copyright 2005 The Apache Software Foundation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */import java.io.BufferedReader;import java.io.File;import java.io.FileFilter;import java.io.FileReader;import java.io.FileWriter;import java.io.IOException;import java.util.regex.Matcher;import java.util.regex.Pattern;/** * Split the Reuters SGML documents into Simple Text files containing: Title, Date, Dateline, Body */public class ExtractReuters{ private File reutersDir; private File outputDir; private static final String LINE_SEPARATOR = System.getProperty("line.separator"); public ExtractReuters(File reutersDir, File outputDir) { this.reutersDir = reutersDir; this.outputDir = outputDir; System.out.println("Deleting all files in " + outputDir); File [] files = outputDir.listFiles(); for (int i = 0; i < files.length; i++) { files[i].delete(); } } public void extract() { File [] sgmFiles = reutersDir.listFiles(new FileFilter() { public boolean accept(File file) { return file.getName().endsWith(".sgm"); } }); if (sgmFiles != null && sgmFiles.length > 0) { for (int i = 0; i < sgmFiles.length; i++) { File sgmFile = sgmFiles[i]; extractFile(sgmFile); } } else { System.err.println("No .sgm files in " + reutersDir); } } Pattern EXTRACTION_PATTERN = Pattern.compile("<TITLE>(.*?)</TITLE>|<DATE>(.*?)</DATE>|<BODY>(.*?)</BODY>"); private static String[] META_CHARS = {"&", "<", ">", "\"", "'"}; private static String[] META_CHARS_SERIALIZATIONS = {"&", "<", ">", """, "'"}; /** * Override if you wish to change what is extracted * * @param sgmFile */ protected void extractFile(File sgmFile) { try { BufferedReader reader = new BufferedReader(new FileReader(sgmFile)); StringBuffer buffer = new StringBuffer(1024); StringBuffer outBuffer = new StringBuffer(1024); String line = null; int index = -1; int docNumber = 0; while ((line = reader.readLine()) != null) { //when we see a closing reuters tag, flush the file if ((index = line.indexOf("</REUTERS")) == -1) { //Replace the SGM escape sequences buffer.append(line).append(' ');//accumulate the strings for now, then apply regular expression to get the pieces, } else { //Extract the relevant pieces and write to a file in the output dir Matcher matcher = EXTRACTION_PATTERN.matcher(buffer); while (matcher.find()) { for (int i = 1; i <= matcher.groupCount(); i++) { if (matcher.group(i) != null) { outBuffer.append(matcher.group(i)); } } outBuffer.append(LINE_SEPARATOR).append(LINE_SEPARATOR); } String out = outBuffer.toString(); for (int i = 0; i < META_CHARS_SERIALIZATIONS.length; i++) { out = out.replaceAll(META_CHARS_SERIALIZATIONS[i], META_CHARS[i]); } File outFile = new File(outputDir, sgmFile.getName() + "-" + (docNumber++) + ".txt"); //System.out.println("Writing " + outFile); FileWriter writer = new FileWriter(outFile); writer.write(out); writer.close(); outBuffer.setLength(0); buffer.setLength(0); } } reader.close(); } catch ( IOException e ) { throw new RuntimeException(e); } } public static void main(String[] args) { if (args.length != 2) { printUsage(); } File reutersDir = new File(args[0]); if (reutersDir.exists()) { File outputDir = new File(args[1]); outputDir.mkdirs(); ExtractReuters extractor = new ExtractReuters(reutersDir, outputDir); extractor.extract(); } else { printUsage(); } } private static void printUsage() { System.err.println("Usage: java -cp <...> org.apache.lucene.benchmark.utils.ExtractReuters <Path to Reuters SGM files> <Output Path>"); }}⌨️ 快捷键说明
复制代码
Ctrl + C
搜索代码
Ctrl + F
全屏模式
F11
切换主题
Ctrl + Shift + D
显示快捷键
?
增大字号
Ctrl + =
减小字号
Ctrl + -