italiannewsprovider.java

EclipseTrader is a stock exchange analysis system, featuring shares pricing watch, intraday and hi

                    news.setTitle(decode(link.getLinkText().trim()));
                    news.setUrl(link.getLink());

                    String source = list.elementAt(9).getText();
                    source = source.replaceAll("[\r\n]", " "); //$NON-NLS-1$ //$NON-NLS-2$
                    source = source.replaceAll("[()]", ""); //$NON-NLS-1$ //$NON-NLS-2$
                    source = source.replaceAll("[ ]{2,}", " ").trim(); //$NON-NLS-1$ //$NON-NLS-2$
                    news.setSource(source);

                    news.setDate(parseDateString(root.getNextSibling().getNextSibling().getNextSibling().getChildren().elementAt(1).getText()));

                    if (!news.getDate().before(limit.getTime()))
                    {
                        log.trace(news.getTitle() + " (" + news.getSource() + ")"); //$NON-NLS-1$ //$NON-NLS-2$
                        CorePlugin.getRepository().save(news);
                        oldItems.add(news);
                    }
                    dtCount++;
                }
                else if (((TagNode) root).getTagName().equalsIgnoreCase("li") && list.size() == 14) //$NON-NLS-1$
                {
                    LinkTag link = (LinkTag)list.elementAt(1);

                    NewsItem news = new NewsItem();
                    news.setRecent(true);
                    news.setTitle(decode(link.getLinkText().trim()));
                    news.setUrl(link.getLink());

                    String source = list.elementAt(6).getText();
                    source = source.replaceAll("[\r\n]", " "); //$NON-NLS-1$ //$NON-NLS-2$
                    source = source.replaceAll("[()]", ""); //$NON-NLS-1$ //$NON-NLS-2$
                    source = source.replaceAll("[ ]{2,}", " ").trim(); //$NON-NLS-1$ //$NON-NLS-2$
                    news.setSource(source);

                    news.setDate(parseDateString(list.elementAt(10).getText()));

                    if (!news.getDate().before(limit.getTime()))
                    {
                        log.trace(news.getTitle() + " (" + news.getSource() + ")"); //$NON-NLS-1$ //$NON-NLS-2$
                        CorePlugin.getRepository().save(news);
                        oldItems.add(news);
                    }
                    liCount++;
                }
            }
            
            if (dtCount == 0 && liCount == 0)
                log.warn("No news found on that page (" + url + ")"); //$NON-NLS-1$ //$NON-NLS-2$
            else if (dtCount == 0 || liCount == 0)
                log.warn("Page not completely parsed (" + url + ")"); //$NON-NLS-1$ //$NON-NLS-2$
        }
        catch (Exception e) {
            log.error(e, e);
        }
    }

    private void update(URL feedUrl, Security security)
    {
        Calendar limit = Calendar.getInstance();
        limit.add(Calendar.DATE, - CorePlugin.getDefault().getPreferenceStore().getInt(CorePlugin.PREFS_NEWS_DATE_RANGE));

        boolean subscribersOnly = YahooPlugin.getDefault().getPreferenceStore().getBoolean(YahooPlugin.PREFS_SHOW_SUBSCRIBERS_ONLY);
        log.debug(feedUrl);
        
        try {
            HttpClient client = new HttpClient();
            client.getHttpConnectionManager().getParams().setConnectionTimeout(5000);

            IPreferenceStore store = CorePlugin.getDefault().getPreferenceStore();
            if (store.getBoolean(CorePlugin.PREFS_ENABLE_HTTP_PROXY))
            {
                client.getHostConfiguration().setProxy(store.getString(CorePlugin.PREFS_PROXY_HOST_ADDRESS), store.getInt(CorePlugin.PREFS_PROXY_PORT_ADDRESS));
                if (store.getBoolean(CorePlugin.PREFS_ENABLE_PROXY_AUTHENTICATION))
                    client.getState().setProxyCredentials(AuthScope.ANY, new UsernamePasswordCredentials(store.getString(CorePlugin.PREFS_PROXY_USER), store.getString(CorePlugin.PREFS_PROXY_PASSWORD)));
            }

            SyndFeed feed = fetcher.retrieveFeed(feedUrl, client);
            for (Iterator iter = feed.getEntries().iterator(); iter.hasNext(); )
            {
                SyndEntry entry = (SyndEntry) iter.next();
                
                if (!subscribersOnly && entry.getTitle().indexOf("[$$]") != -1) //$NON-NLS-1$
                    continue;
                
                NewsItem news = new NewsItem();
                news.setRecent(true);

                String title = entry.getTitle();
                if (title.endsWith(")")) //$NON-NLS-1$
                {
                    int s = title.lastIndexOf('(');
                    if (s != -1)
                    {
                        news.setSource(title.substring(s + 1, title.length() - 1));
                        title = title.substring(0, s - 1).trim();
                    }
                }
                news.setTitle(decode(title));

                news.setUrl(entry.getLink());

                Date entryDate = entry.getPublishedDate();
                if (entry.getUpdatedDate() != null)
                    entryDate = entry.getUpdatedDate();
                if (entryDate != null)
                {
                    Calendar date = Calendar.getInstance();
                    date.setTime(entryDate);
                    date.set(Calendar.SECOND, 0);
                    date.set(Calendar.MILLISECOND, 0);
                    news.setDate(date.getTime());
                }
                
                if (security != null)
                    news.addSecurity(security);

                if (!news.getDate().before(limit.getTime()) && !isDuplicated(news))
                {
                    log.trace(news.getTitle() + " (" + news.getSource() + ")"); //$NON-NLS-1$ //$NON-NLS-2$
                    CorePlugin.getRepository().save(news);
                    oldItems.add(news);
                }
            }
        } catch(Exception e) {
            CorePlugin.logException(e);
        }
    }

    private Date parseDateString(String date)
    {
        Calendar gc = Calendar.getInstance(Locale.ITALY);
        StringTokenizer st = new StringTokenizer(date, " ,:"); //$NON-NLS-1$
        st.nextToken();
        Integer vint = new Integer(st.nextToken());
        gc.set(Calendar.DAY_OF_MONTH, vint.intValue());
        gc.set(Calendar.MONTH, getMonth(st.nextToken()) - 1);
        vint = new Integer(st.nextToken());
        gc.set(Calendar.YEAR, vint.intValue());
        vint = new Integer(st.nextToken());
        gc.set(Calendar.HOUR_OF_DAY, vint.intValue());
        vint = new Integer(st.nextToken());
        gc.set(Calendar.MINUTE, vint.intValue());
        gc.set(Calendar.SECOND, 0);
        gc.set(Calendar.MILLISECOND, 0);
        return gc.getTime();
    }

    private int getMonth(String t)
    {
        if (t.equalsIgnoreCase("Gennaio") == true) //$NON-NLS-1$
            return 1;
        if (t.equalsIgnoreCase("Febbraio") == true) //$NON-NLS-1$
            return 2;
        if (t.equalsIgnoreCase("Marzo") == true) //$NON-NLS-1$
            return 3;
        if (t.equalsIgnoreCase("Aprile") == true) //$NON-NLS-1$
            return 4;
        if (t.equalsIgnoreCase("Maggio") == true) //$NON-NLS-1$
            return 5;
        if (t.equalsIgnoreCase("Giugno") == true) //$NON-NLS-1$
            return 6;
        if (t.equalsIgnoreCase("Luglio") == true) //$NON-NLS-1$
            return 7;
        if (t.equalsIgnoreCase("Agosto") == true) //$NON-NLS-1$
            return 8;
        if (t.equalsIgnoreCase("Settembre") == true) //$NON-NLS-1$
            return 9;
        if (t.equalsIgnoreCase("Ottobre") == true) //$NON-NLS-1$
            return 10;
        if (t.equalsIgnoreCase("Novembre") == true) //$NON-NLS-1$
            return 11;
        if (t.equalsIgnoreCase("Dicembre") == true) //$NON-NLS-1$
            return 12;

        return 0;
    }
    
    private String decode(String s)
    {
        if (s.indexOf("&#") == -1) //$NON-NLS-1$
            return s;
        
        int i = 0;
        StringBuffer sb = new StringBuffer();
        byte[] bytes = new byte[0];
        try {
            bytes = s.getBytes();
        } catch(Exception e) {
            log.error(e, e);
            bytes = s.getBytes();
        }
        
        while(i < bytes.length)
        {
            byte c = bytes[i++];
            if (c == '&' && i < bytes.length)
            {
                c = bytes[i++];
                if (c == '#')
                {
                    int data = 0;
                    while(i < bytes.length)
                    {
                        c = bytes[i++];
                        if (c < '0' || c > '9')
                            break;
                        data = (data * 10) + (c - '0');
                    }
                    if (data >= ' ')
                    {
                        try {
                            sb.append(new String(new byte[] { (byte)data }));
                        } catch(Exception e) {
                            log.error(e, e);
                        }
                    }
                }
                else
                {
                    try {
                        sb.append('&');
                        sb.append(new String(new byte[] { c }));
                    } catch(Exception e) {
                        log.error(e, e);
                    }
                }
            }
            else if (c >= ' ')
            {
                try {
                    sb.append(new String(new byte[] { c }));
                } catch(Exception e) {
                    log.error(e, e);
                }
            }
        }
        
        return sb.toString();
    }
    
    boolean isDuplicated(NewsItem news)
    {
        NewsItem[] items = (NewsItem[])CorePlugin.getRepository().allNews().toArray(new NewsItem[0]);
        
        for (int i = 0; i < items.length; i++)
        {
            if (news.getTitle().equals(items[i].getTitle()) && news.getUrl().equals(items[i].getUrl()))
            {
                items[i].addSecurities(news.getSecurities());
                CorePlugin.getRepository().save(items[i]);
                return true;
            }
        }

        return false;
    }
}