italiannewsprovider.java

EclipseTrader is a stock exchange analysis system, featuring shares pricing watch, intraday and hi

/*
 * Copyright (c) 2004-2006 Marco Maccaferri and others.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors:
 *     Marco Maccaferri - initial API and implementation
 */

package net.sourceforge.eclipsetrader.yahoo;

import java.net.URL;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.StringTokenizer;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;

import net.sourceforge.eclipsetrader.core.CorePlugin;
import net.sourceforge.eclipsetrader.core.INewsProvider;
import net.sourceforge.eclipsetrader.core.db.NewsItem;
import net.sourceforge.eclipsetrader.core.db.Security;
import net.sourceforge.eclipsetrader.news.NewsPlugin;

import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.UsernamePasswordCredentials;
import org.apache.commons.httpclient.auth.AuthScope;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.eclipse.core.runtime.FileLocator;
import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.core.runtime.IStatus;
import org.eclipse.core.runtime.Path;
import org.eclipse.core.runtime.Status;
import org.eclipse.core.runtime.jobs.Job;
import org.eclipse.jface.preference.IPreferenceStore;
import org.htmlparser.Node;
import org.htmlparser.Parser;
import org.htmlparser.filters.OrFilter;
import org.htmlparser.filters.TagNameFilter;
import org.htmlparser.nodes.TagNode;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.SimpleNodeIterator;
import org.w3c.dom.Document;

import com.sun.syndication.feed.synd.SyndEntry;
import com.sun.syndication.feed.synd.SyndFeed;
import com.sun.syndication.fetcher.impl.FeedFetcherCache;
import com.sun.syndication.fetcher.impl.HashMapFeedInfoCache;
import com.sun.syndication.fetcher.impl.HttpClientFeedFetcher;

public class ItalianNewsProvider implements Runnable, INewsProvider
{
    private Thread thread;
    private boolean stopping = false;
    static private List oldItems = new ArrayList();
    private FeedFetcherCache feedInfoCache = HashMapFeedInfoCache.getInstance();
    private HttpClientFeedFetcher fetcher = new HttpClientFeedFetcher(feedInfoCache);
    private Log log = LogFactory.getLog(getClass());

    public ItalianNewsProvider()
    {
    }

    /* (non-Javadoc)
     * @see net.sourceforge.eclipsetrader.news.INewsProvider#start()
     */
    public void start()
    {
        if (thread == null)
        {
            stopping = false;
            thread = new Thread(this);
            thread.start();
        }
    }

    /* (non-Javadoc)
     * @see net.sourceforge.eclipsetrader.news.INewsProvider#stop()
     */
    public void stop()
    {
        stopping = true;
        if (thread != null)
        {
            try {
                thread.join();
            } catch (InterruptedException e) {
                log.error(e);
            }
            thread = null;
        }
    }

    /* (non-Javadoc)
     * @see net.sourceforge.eclipsetrader.news.INewsProvider#snapshot()
     */
    public void snapshot()
    {
        update();
    }

    /* (non-Javadoc)
     * @see net.sourceforge.eclipsetrader.news.INewsProvider#snapshot(net.sourceforge.eclipsetrader.core.db.Security)
     */
    public void snapshot(Security security)
    {
        try {
            update(new URL("http://it.finance.yahoo.com/rss/headline?s=" + security.getCode().toUpperCase()), security); //$NON-NLS-1$
        } catch(Exception e) {
            CorePlugin.logException(e);
        }
    }

    /* (non-Javadoc)
     * @see java.lang.Runnable#run()
     */
    public void run()
    {
        long nextRun = System.currentTimeMillis() + 2 * 1000;

        while (!stopping)
        {
            if (System.currentTimeMillis() >= nextRun)
            {
                update();
                int interval = NewsPlugin.getDefault().getPreferenceStore().getInt(NewsPlugin.PREFS_UPDATE_INTERVAL);
                nextRun = System.currentTimeMillis() + interval * 60 * 1000;
            }

            try {
                Thread.sleep(1000);
            } catch (InterruptedException e) {
                log.error(e);
                break;
            }
        }

        thread = null;
    }

    private void update()
    {
        Object[] o = oldItems.toArray();
        for (int i = 0; i < o.length; i++)
        {
            ((NewsItem)o[i]).setRecent(false);
            CorePlugin.getRepository().save((NewsItem)o[i]);
        }
        oldItems.clear();
        
        Job job = new Job(Messages.ItalianNewsProvider_JobName) {
            protected IStatus run(IProgressMonitor monitor)
            {
                IPreferenceStore store = YahooPlugin.getDefault().getPreferenceStore();

                List urls = new ArrayList();
                try
                {
                    DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
                    DocumentBuilder builder = factory.newDocumentBuilder();
                    Document document = builder.parse(FileLocator.openStream(YahooPlugin.getDefault().getBundle(), new Path("categories.it.xml"), false)); //$NON-NLS-1$

                    org.w3c.dom.NodeList childNodes = document.getFirstChild().getChildNodes();
                    for (int i = 0; i < childNodes.getLength(); i++)
                    {
                        org.w3c.dom.Node node = childNodes.item(i);
                        String nodeName = node.getNodeName();
                        if (nodeName.equalsIgnoreCase("category")) //$NON-NLS-1$
                        {
                            String id = ((org.w3c.dom.Node)node).getAttributes().getNamedItem("id").getNodeValue(); //$NON-NLS-1$
                         
                            org.w3c.dom.NodeList list = node.getChildNodes();
                            for (int x = 0; x < list.getLength(); x++)
                            {
                                org.w3c.dom.Node item = list.item(x);
                                nodeName = item.getNodeName();
                                org.w3c.dom.Node value = item.getFirstChild();
                                if (value != null)
                                {
                                    if (nodeName.equalsIgnoreCase("url")) //$NON-NLS-1$
                                    {
                                        if (store.getBoolean(id))
                                            urls.add(value.getNodeValue());
                                    }
                                }
                            }
                        }
                    }
                } catch (Exception e) {
                    log.error(e, e);
                }

                List securities = CorePlugin.getRepository().allSecurities();
                monitor.beginTask(Messages.ItalianNewsProvider_TaskName, urls.size() + securities.size());
                log.info("Start fetching Yahoo! News (Italy)"); //$NON-NLS-1$

                for (Iterator iter = securities.iterator(); iter.hasNext(); )
                {
                    Security security = (Security) iter.next();
                    try {
                        String url = "http://it.finance.yahoo.com/rss/headline?s=" + security.getCode().toUpperCase(); //$NON-NLS-1$
                        monitor.subTask(url);
                        update(new URL(url), security);
                        monitor.worked(1);
                    } catch(Exception e) {
                        log.error(e, e);
                    }
                }

                for (Iterator iter = urls.iterator(); iter.hasNext(); )
                {
                    String url = (String) iter.next();
                    monitor.subTask(url);
                    parseNewsPage(url);
                    monitor.worked(1);
                }

                monitor.done();
                return Status.OK_STATUS;
            }
        };
        job.setUser(false);
        job.schedule();
    }

    private void parseNewsPage(String url)
    {
        Calendar limit = Calendar.getInstance();
        limit.add(Calendar.DATE, - CorePlugin.getDefault().getPreferenceStore().getInt(CorePlugin.PREFS_NEWS_DATE_RANGE));

        int dtCount = 0;
        int liCount = 0;
        
        try {
            log.debug(url);
            Parser parser = new Parser(url);

            NodeList list = parser.extractAllNodesThatMatch(new OrFilter(new TagNameFilter("dt"), new TagNameFilter("li"))); //$NON-NLS-1$ //$NON-NLS-2$
            for (SimpleNodeIterator iter = list.elements(); iter.hasMoreNodes();)
            {
                Node root = iter.nextNode();
                list = root.getChildren();

                if (((TagNode) root).getTagName().equalsIgnoreCase("dt") && list.size() == 12) //$NON-NLS-1$
                {
                    LinkTag link = (LinkTag)list.elementAt(3);

                    NewsItem news = new NewsItem();
                    news.setRecent(true);