spamfilter.java

jspwiki source code,jspwiki source code

                        compiledpatterns.add( m_compiler.compile( line ) );
                    }
                    catch( MalformedPatternException e )
                    {
                        log.debug( "Malformed spam filter pattern "+line );
                    }
                }
            }
            catch( IOException e )
            {
                log.info("Could not read patterns; returning what I got",e);
            }
        }

        return compiledpatterns;
    }

    /**
     *  Takes a single page change and performs a load of tests on the content change.
     *  An admin can modify anything.
     *
     *  @param context
     *  @param content
     *  @throws RedirectException
     */
    private synchronized void checkSinglePageChange( WikiContext context, String content, Change change )
        throws RedirectException
    {
        HttpServletRequest req = context.getHttpRequest();

        if( req != null )
        {
            String addr = req.getRemoteAddr();
            int hostCounter = 0;
            int changeCounter = 0;

            log.debug("Change is "+change.m_change);

            long time = System.currentTimeMillis()-60*1000L; // 1 minute

            for( Iterator i = m_lastModifications.iterator(); i.hasNext(); )
            {
                Host host = (Host)i.next();

                //
                //  Check if this item is invalid
                //
                if( host.getAddedTime() < time )
                {
                    log.debug("Removed host "+host.getAddress()+" from modification queue (expired)");
                    i.remove();
                    continue;
                }

                //
                // Check if this IP address has been seen before
                //

                if( host.getAddress().equals(addr) )
                {
                    hostCounter++;
                }

                //
                //  Check, if this change has been seen before
                //

                if( host.getChange() != null && host.getChange().equals(change) )
                {
                    changeCounter++;
                }
            }

            //
            //  Now, let's check against the limits.
            //
            if( hostCounter >= m_limitSinglePageChanges )
            {
                Host host = new Host( addr, null );

                m_temporaryBanList.add( host );

                String uid = log( context, REJECT, REASON_TOO_MANY_MODIFICATIONS, change.m_change );
                log.info( "SPAM:TooManyModifications ("+uid+"). Added host "+addr+" to temporary ban list for doing too many modifications/minute" );
                checkStrategy( context, REASON_TOO_MANY_MODIFICATIONS, "Herb says you look like a spammer, and I trust Herb! (Incident code "+uid+")" );
            }

            if( changeCounter >= m_limitSimilarChanges )
            {
                Host host = new Host( addr, null );

                m_temporaryBanList.add( host );

                String uid = log( context, REJECT, REASON_SIMILAR_MODIFICATIONS, change.m_change );

                log.info("SPAM:SimilarModifications ("+uid+"). Added host "+addr+" to temporary ban list for doing too many similar modifications" );
                checkStrategy( context, REASON_SIMILAR_MODIFICATIONS, "Herb says you look like a spammer, and I trust Herb! (Incident code "+uid+")");
            }

            //
            //  Calculate the number of links in the addition.
            //

            String tstChange = change.toString();
            int    urlCounter = 0;

            while( m_matcher.contains(tstChange,m_urlPattern) )
            {
                MatchResult m = m_matcher.getMatch();

                tstChange = tstChange.substring( m.endOffset(0) );

                urlCounter++;
            }

            if( urlCounter > m_maxUrls )
            {
                Host host = new Host( addr, null );

                m_temporaryBanList.add( host );

                String uid = log( context, REJECT, REASON_TOO_MANY_URLS, change.toString() );

                log.info("SPAM:TooManyUrls ("+uid+"). Added host "+addr+" to temporary ban list for adding too many URLs" );
                checkStrategy( context, REASON_TOO_MANY_URLS, "Herb says you look like a spammer, and I trust Herb! (Incident code "+uid+")" );
            }

            //
            //  Check bot trap
            //

            checkBotTrap( context, change );

            //
            //  Check UTF-8 mangling
            //

            checkUTF8( context, change );

            //
            //  Do Akismet check.  This is good to be the last, because this is the most
            //  expensive operation.
            //

            checkAkismet( context, change );

            m_lastModifications.add( new Host( addr, change ) );
        }
    }


    /**
     *  Checks against the akismet system.
     *
     * @param context
     * @param change
     * @throws RedirectException
     */
    private void checkAkismet( WikiContext context, Change change )
        throws RedirectException
    {
        if( m_akismetAPIKey != null )
        {
            if( m_akismet == null )
            {
                log.info("Initializing Akismet spam protection.");

                m_akismet = new Akismet( m_akismetAPIKey, context.getEngine().getBaseURL() );

                if( !m_akismet.verifyAPIKey() )
                {
                    log.error("Akismet API key cannot be verified.  Please check your config.");
                    m_akismetAPIKey = null;
                    m_akismet = null;
                }
            }

            HttpServletRequest req = context.getHttpRequest();

            //
            //  Akismet will mark all empty statements as spam, so we'll just
            //  ignore them.
            //
            if( change.m_adds == 0 && change.m_removals > 0 )
            {
                return;
            }
            
            if( req != null && m_akismet != null )
            {
                log.debug("Calling Akismet to check for spam...");

                StopWatch sw = new StopWatch();
                sw.start();

                String ipAddress     = req.getRemoteAddr();
                String userAgent     = req.getHeader("User-Agent");
                String referrer      = req.getHeader( "Referer");
                String permalink     = context.getViewURL( context.getPage().getName() );
                String commentType   = context.getRequestContext().equals(WikiContext.COMMENT) ? "comment" : "edit";
                String commentAuthor = context.getCurrentUser().getName();
                String commentAuthorEmail = null;
                String commentAuthorURL   = null;

                boolean isSpam = m_akismet.commentCheck( ipAddress,
                                                         userAgent,
                                                         referrer,
                                                         permalink,
                                                         commentType,
                                                         commentAuthor,
                                                         commentAuthorEmail,
                                                         commentAuthorURL,
                                                         change.toString(),
                                                         null );

                sw.stop();

                log.debug("Akismet request done in: "+sw);

                if( isSpam )
                {
                    // Host host = new Host( ipAddress, null );

                    // m_temporaryBanList.add( host );

                    String uid = log( context, REJECT, REASON_AKISMET, change.toString() );

                    log.info("SPAM:Akismet ("+uid+"). Akismet thinks this change is spam; added host to temporary ban list.");

                    checkStrategy( context, REASON_AKISMET, "Akismet tells Herb you're a spammer, Herb trusts Akismet, and I trust Herb! (Incident code "+uid+")");
                }
            }
        }
    }

    /**
     *  Returns a static string which can be used to detect spambots which
     *  just wildly fill in all the fields.
     *
     *  @return A string
     */
    public static String getBotFieldName()
    {
        return "submit_auth";
    }

    /**
     *  This checks whether an invisible field is available in the request, and
     *  whether it's contents are suspected spam.
     *
     *  @param context
     *  @param change
     * @throws RedirectException
     */
    private void checkBotTrap( WikiContext context, Change change ) throws RedirectException
    {
        HttpServletRequest request = context.getHttpRequest();

        if( request != null )
        {
            String unspam = request.getParameter( getBotFieldName() );
            if( unspam != null && unspam.length() > 0 )
            {
                String uid = log( context, REJECT, REASON_BOT_TRAP, change.toString() );

                log.info("SPAM:BotTrap ("+uid+").  Wildly behaving bot detected.");

                checkStrategy( context, REASON_BOT_TRAP, "Spamming attempt detected. (Incident code "+uid+")");

            }
        }
    }

    private void checkUTF8( WikiContext context, Change change ) throws RedirectException
    {
        HttpServletRequest request = context.getHttpRequest();

        if( request != null )
        {
            String utf8field = request.getParameter( "encodingcheck" );

            if( utf8field != null && !utf8field.equals("\u3041") )
            {
                String uid = log( context, REJECT, REASON_UTF8_TRAP, change.toString() );

                log.info("SPAM:UTF8Trap ("+uid+").  Wildly posting dumb bot detected.");

                checkStrategy( context, REASON_UTF8_TRAP, "Spamming attempt detected. (Incident code "+uid+")");
            }
        }
    }

    /**
     *  Goes through the ban list and cleans away any host which has expired from it.
     */
    private synchronized void cleanBanList()
    {
        long now = System.currentTimeMillis();

        for( Iterator i = m_temporaryBanList.iterator(); i.hasNext(); )
        {
            Host host = (Host)i.next();

            if( host.getReleaseTime() < now )
            {
                log.debug("Removed host "+host.getAddress()+" from temporary ban list (expired)");
                i.remove();
            }
        }
    }

    /**
     *  Checks the ban list if the IP address of the changer is already on it.
     *
     *  @param context
     *  @throws RedirectException
     */

    private void checkBanList( WikiContext context, Change change )
        throws RedirectException
    {
        HttpServletRequest req = context.getHttpRequest();

        if( req != null )
        {
            String remote = req.getRemoteAddr();

            long now = System.currentTimeMillis();

            for( Iterator i = m_temporaryBanList.iterator(); i.hasNext(); )
            {
                Host host = (Host)i.next();

                if( host.getAddress().equals(remote) )
                {
                    long timeleft = (host.getReleaseTime() - now) / 1000L;

                    log( context, REJECT, REASON_IP_BANNED_TEMPORARILY, change.m_change );

                    checkStrategy( context, REASON_IP_BANNED_TEMPORARILY, "You have been temporarily banned from modifying this wiki. ("+timeleft+" seconds of ban left)");
                }
            }
        }

    }

    /**
     *  If the spam filter notices changes in the black list page, it will refresh
     *  them automatically.
     *
     *  @param context
     */
    private void refreshBlacklists( WikiContext context )
    {
        try
        {
            WikiPage source = context.getEngine().getPage( m_forbiddenWordsPage );
            Attachment att = context.getEngine().getAttachmentManager().getAttachmentInfo( context, m_blacklist );

            boolean rebuild = false;

            //
            //  Rebuild, if the page or the attachment has changed since.
            //
            if( source != null )
            {
                if( m_spamPatterns == null || m_spamPatterns.isEmpty() || source.getLastModified().after(m_lastRebuild) )
                {
                    rebuild = true;
                }
            }

            if( att != null )
            {
                if( m_spamPatterns == null || m_spamPatterns.isEmpty() || att.getLastModified().after(m_lastRebuild) )
                {
                    rebuild = true;
                }
            }


            //
            //  Do the actual rebuilding.  For simplicity's sake, we always rebuild the complete
            //  filter list regardless of what changed.
            //

            if( rebuild )
            {
                m_lastRebuild = new Date();

                m_spamPatterns = parseWordList( source,
                                                (source != null) ? (String)source.getAttribute( LISTVAR ) : null );

                log.info("Spam filter reloaded - recognizing "+m_spamPatterns.size()+" patterns from page "+m_forbiddenWordsPage);

                if( att != null )
                {
                    InputStream in = context.getEngine().getAttachmentManager().getAttachmentStream(att);

                    StringWriter out = new StringWriter();

                    FileUtil.copyContents( new InputStreamReader(in,"UTF-8"), out );

                    Collection<Pattern> blackList = parseBlacklist( out.toString() );

                    log.info("...recognizing additional "+blackList.size()+" patterns from blacklist "+m_blacklist);

                    m_spamPatterns.addAll( blackList );
                }
            }