jobconfigureutils.java

这是个爬虫和lucece相结合最好了

     * E.g. "/admin/jobs/per/overview.jsp".
     * @return A job else we've redirected if no job or readonly.
     * @throws IOException
     */
    public static CrawlJob checkCrawlJob(CrawlJob job,
        HttpServletResponse response, String redirectBasePath,
        String currDomain)
    throws IOException {
        if (job == null) {
            // Didn't find any job with the given UID or no UID given.
            response.sendRedirect(redirectBasePath +
                "?message=No job selected");
        } else if (job.isReadOnly()) {
            // Can't edit this job.
            response.sendRedirect(redirectBasePath +
                "?job=" + job.getUID() +
                ((currDomain != null && currDomain.length() > 0)?
                    "&currDomain=" + currDomain: "") +
                "&message=Can't edit a read only job");
        }
        return job;
    }

    /**
     * Handle job action.
     * @param handler CrawlJobHandler to operate on.
     * @param request Http request.
     * @param response Http response.
     * @param redirectBasePath Full path for where to go next if an error.
     * E.g. "/admin/jobs/per/overview.jsp".
     * @param currDomain Current domain.  Pass null for global domain.
     * @param reference 
     * @return The crawljob configured.
     * @throws IOException
     * @throws AttributeNotFoundException
     * @throws InvocationTargetException
     * @throws InvalidAttributeValueException
     */
    public static CrawlJob handleJobAction(CrawlJobHandler handler,
            HttpServletRequest request, HttpServletResponse response,
            String redirectBasePath, String currDomain, String reference)
    throws IOException, AttributeNotFoundException, InvocationTargetException,
        InvalidAttributeValueException {

        // Load the job to manipulate
        CrawlJob theJob =
            checkCrawlJob(handler.getJob(request.getParameter("job")),
                response, redirectBasePath, currDomain);

        XMLSettingsHandler settingsHandler = theJob.getSettingsHandler();
        // If currDomain is null, then we're at top-level.
        CrawlerSettings settings = settingsHandler
            .getSettingsObject(currDomain);
        
        if(reference != null) {
            // refinement
            Refinement refinement = settings.getRefinement(reference);
            settings = refinement.getSettings();
        }

        // See if we need to take any action
        if (request.getParameter(ACTION) != null) {
            // Need to take some action.
            String action = request.getParameter(ACTION);
            String subaction = request.getParameter(SUBACTION);
            if (action.equals(FILTERS)) {
                // Doing something with the filters.
                String map = request.getParameter(MAP);
                if (map != null && map.length() > 0) {
                    String filter = request.getParameter(FILTER);
                    MapType filterMap = (MapType) settingsHandler
                        .getComplexTypeByAbsoluteName(settings, map);
                    if (subaction.equals(ADD)) {
                        // Add filter
                        String className = request.getParameter(map + ".class");
                        String typeName = request.getParameter(map + ".name");
                        if (typeName != null && typeName.length() > 0 &&
                                className != null && className.length() > 0) {
                            ModuleType tmp = SettingsHandler
                                .instantiateModuleTypeFromClassName(
                                    typeName, className);
                            filterMap.addElement(settings, tmp);
                        }
                    } else if (subaction.equals(MOVEUP)) {
                        // Move a filter down in a map
                        if (filter != null && filter.length() > 0) {
                            filterMap.moveElementUp(settings, filter);
                        }
                    } else if (subaction.equals(MOVEDOWN)) {
                        // Move a filter up in a map
                        if (filter != null && filter.length() > 0) {
                            filterMap.moveElementDown(settings, filter);
                        }
                    } else if (subaction.equals(REMOVE)) {
                        // Remove a filter from a map
                        if (filter != null && filter.length() > 0) {
                            filterMap.removeElement(settings, filter);
                        }
                    }
                }
                // Finally save the changes to disk
                settingsHandler.writeSettingsObject(settings);
            } else if (action.equals(DONE)) {
                // Ok, done editing.
                if(subaction.equals(CONTINUE)) {
                    // was editting an override/refinement, simply continue
                    if (theJob.isRunning()) {
                        handler.kickUpdate(); //Just to make sure.
                    }
                    String overParam = ((currDomain != null && currDomain
                            .length() > 0) ? "&currDomain=" + currDomain : "");
                    String refParam = 
                        ((reference != null && reference.length() > 0) 
                                ? "&reference=" + reference
                                : "");
                    String messageParam = (refParam.length() > 0) 
                         ? "&message=Refinement changes saved"
                         : "&message=Override changes saved";
                    response.sendRedirect(redirectBasePath +
                        "?job=" + theJob.getUID() +
                         overParam +
                         refParam + 
                         messageParam);
                } else {
                    // on main, truly 'done'
                    if (theJob.isNew()) {
                        handler.addJob(theJob);
                        response.sendRedirect(redirectBasePath
                                + "?message=Job created");
                    } else {
                        if (theJob.isRunning()) {
                            handler.kickUpdate();
                        }
                        if (theJob.isProfile()) {
                            response.sendRedirect(redirectBasePath
                                    + "?message=Profile modified");
                        } else {
                            response.sendRedirect(redirectBasePath
                                    + "?message=Job modified");
                        }
                    }
                }
            } else if (action.equals(GOTO)) {
                // Goto another page of the job/profile settings
                String overParam = ((currDomain != null && currDomain
                        .length() > 0) ? "&currDomain=" + currDomain : "");
                String refParam = 
                    ((reference != null && reference.length() > 0) 
                            ? "&reference=" + reference
                            : "");
                response.sendRedirect(request.getParameter(SUBACTION) +
                    overParam + refParam);
            }
        }
        return theJob;
    }
    
    /**
     * Print complete seeds list on passed in PrintWriter.
     * @param hndlr Current handler.
     * @param payload What to write out.
     * @throws AttributeNotFoundException
     * @throws MBeanException
     * @throws ReflectionException
     * @throws IOException
     * @throws IOException
     */
    public static void printOutSeeds(SettingsHandler hndlr, String payload)
    throws AttributeNotFoundException, MBeanException, ReflectionException,
    IOException {
        File seedfile = getSeedFile(hndlr);
        writeReader(new StringReader(payload),
            new BufferedWriter(new FileWriter(seedfile)));
    }
    
    /**
     * Print complete seeds list on passed in PrintWriter.
     * @param hndlr Current handler.
     * @param out Writer to write out all seeds to.
     * @throws ReflectionException
     * @throws MBeanException
     * @throws AttributeNotFoundException
     * @throws IOException
     */
    public static void printOutSeeds(SettingsHandler hndlr, Writer out)
    throws AttributeNotFoundException, MBeanException, ReflectionException,
            IOException {
        // getSeedStream looks for seeds on disk and on classpath.
        InputStream is = getSeedStream(hndlr);
        writeReader(new BufferedReader(new InputStreamReader(is)), out);
    }
    
    /**
     * Test whether seeds file is of a size that's reasonable
     * to edit in an HTML textarea. 
     * @param h current settingsHandler
     * @return true if seeds size is manageable, false otherwise
     * @throws AttributeNotFoundException 
     * @throws MBeanException 
     * @throws ReflectionException 
     * 
     */
    public static boolean seedsEdittableSize(SettingsHandler h)
            throws AttributeNotFoundException, MBeanException,
            ReflectionException {
        return getSeedFile(h).length() <= (32 * 1024); // 32K
    }
    /**
     * @param hndlr Settings handler.
     * @return Seeds file.
     * @throws ReflectionException
     * @throws MBeanException
     * @throws AttributeNotFoundException
     */
    protected static File getSeedFile(SettingsHandler hndlr)
    throws AttributeNotFoundException, MBeanException, ReflectionException {
        String seedsFileStr = (String)((ComplexType)hndlr.getOrder().
            getAttribute("scope")).getAttribute("seedsfile");
        return hndlr.getPathRelativeToWorkingDirectory(seedsFileStr);
    }
    
    /**
     * Return seeds as a stream.
     * This method will work for case where seeds are on disk or on classpath.
     * @param hndlr SettingsHandler.  Used to find seeds.txt file.
     * @return InputStream on current seeds file.
     * @throws IOException
     * @throws ReflectionException
     * @throws MBeanException
     * @throws AttributeNotFoundException
     */
    protected static InputStream getSeedStream(SettingsHandler hndlr)
    throws IOException, AttributeNotFoundException, MBeanException,
            ReflectionException {
        InputStream is = null;
        File seedFile = getSeedFile(hndlr);
        if (!seedFile.exists()) {
            // Is the file on the CLASSPATH?
            is = SettingsHandler.class.
                getResourceAsStream(IoUtils.getClasspathPath(seedFile));
        } else if(seedFile.canRead()) {
            is = new FileInputStream(seedFile);
        }
        if (is == null) {
            throw new IOException(seedFile + " does not" +
            " exist -- neither on disk nor on CLASSPATH -- or is not" +
            " readable.");
        }
        return is;
    }
    
    /**
     * Print complete seeds list on passed in PrintWriter.
     * @param reader File to read seeds from.
     * @param out Writer to write out all seeds to.
     * @throws IOException
     */
    protected static void writeReader(Reader reader, Writer out)
    throws IOException {
        final int bufferSize = 1024 * 4;
        char [] buffer = new char[bufferSize];
        int read = -1;
        while ((read = reader.read(buffer, 0, bufferSize)) != -1) {
            out.write(buffer, 0, read);
        }
        out.flush();
    }
}