yahoostockserver.java

JStock是一个免费股市软件

            
        stringBuffer.append(_symbol).append(YAHOO_STOCK_FORMAT);
                    
        final String location = stringBuffer.toString();

        final HttpClient httpClient = new HttpClient();

        for(int retry=0; retry<NUM_OF_RETRY; retry++) {
            HttpMethod method = new GetMethod(location);                        

            try {
                Utils.setHttpClientProxyFromSystemProperties(httpClient);
                httpClient.executeMethod(method);
                final String responde = method.getResponseBodyAsString();
                
                final List<Stock> tmpStocks = YahooStockFormat.getInstance().parse(responde);
                if(tmpStocks.size() != remainder) {
                    if(retry == (NUM_OF_RETRY-1)) {
                        // throw new StockNotFoundException();
                        
                        final int currSize = tmpStocks.size();
                        final int expectedSize = expectedSymbols.size();

                        if(this.isToleranceAllowed(currSize, expectedSize)) {
                            List<Symbol> currSymbols = new ArrayList<Symbol>();
                            List<Stock> emptyStocks = new ArrayList<Stock>();

                            for(Stock stock : tmpStocks) {
                                currSymbols.add(stock.getSymbol());
                            }

                            for(Symbol symbol : expectedSymbols) {
                                if(currSymbols.contains(symbol) == false) {
                                    emptyStocks.add(org.yccheok.jstock.gui.Utils.getEmptyStock(Code.newInstance(symbol.toString()), symbol));
                                }
                            }

                            tmpStocks.addAll(emptyStocks);
                        }
                        else {
                            throw new StockNotFoundException("Expected stock size=" + expectedSize + ", Current stock size=" + currSize + ", Request=" + location);
                        }                        
                    }   // if(retry == (NUM_OF_RETRY-1))   
                    
                    continue;
                }   // if(tmpStocks.size() != remainder)
                
                stocks.addAll(tmpStocks);
            }
            catch(HttpException exp) {
                log.error("location=" + location, exp);                
                continue;
            }
            catch(IOException exp) {
                log.error("location=" + location, exp);
                continue;
            }
            finally {
                method.releaseConnection();
            }

            break;
        }

       if(stocks.size() != symbols.size())
           throw new StockNotFoundException("Inconsistent stock size (" + stocks.size() + ") and symbol size (" + symbols.size() + ")");
       
        return stocks;
    }

    public List<Stock> getStocksByCodes(List<Code> codes) throws StockNotFoundException {
        List<Symbol> symbols = new ArrayList<Symbol>();
        for(Code code : codes) {
            symbols.add(Symbol.newInstance(code.toString()));
        }
        
        return getStocksBySymbols(symbols);
    }

    private Set<Symbol> getSymbols(String responde) {
        Set<Symbol> symbols = new HashSet<Symbol>();
        
        final Matcher matcher = symbolPattern.matcher(responde);
        
        while(matcher.find()){
            for(int j=1; j<=matcher.groupCount(); j++ ) {
                final String string = matcher.group(j);
                symbols.add(Symbol.newInstance(string));
            }
        }
        
        return symbols;
    }
    
    // The returned URLs, shouldn't have any duplication with visited,
    // and they are unique. Although is more suitable that we use Set,
    // use List is more convinient for us to iterate.
    private List<URL> getURLs(String responde, List<URL> visited) {   
        List<URL> urls = new ArrayList<URL>();
        
        final Pattern pattern = patterns.get(country);
        final Matcher matcher = pattern.matcher(responde);
        
        while(matcher.find()){
            for(int j=1; j<=matcher.groupCount(); j++ ) {
                String string = matcher.group(j);

                try {
                    URL url = new URL(baseURL, string);
                    
                    if((urls.contains(url) == false) && (visited.contains(url) == false)) {
                        urls.add(url);
                    }
                } catch (MalformedURLException ex) {
                    log.error("", ex);
                }                                
            }
        }
        
        return urls;
    }
    
    public List<Stock> getAllStocks() throws StockNotFoundException {
        List<URL> visited = new ArrayList<URL>();
        
        // Use Set, for safety purpose to avoid duplication.
        Set<Symbol> symbols = new HashSet<Symbol>();

        visited.add(baseURL);

        final HttpClient httpClient = new HttpClient();

        for(int i=0; i<visited.size(); i++) {
            final String location = visited.get(i).toString();
            
            for(int retry=0; retry<NUM_OF_RETRY; retry++) {
                HttpMethod method = new GetMethod(location);                        

                try {
                    Utils.setHttpClientProxyFromSystemProperties(httpClient);
                    httpClient.executeMethod(method);
                    final String responde = method.getResponseBodyAsString();
                    List<URL> urls = getURLs(responde, visited);
                    Set<Symbol> tmpSymbols = getSymbols(responde);

                    // getURLs already ensure URLs are unique.
                    visited.addAll(urls);
                    symbols.addAll(tmpSymbols);
                }
                catch(HttpException exp) {
                    log.error("location=" + location, exp);                
                    continue;
                }
                catch(IOException exp) {
                    log.error("location=" + location, exp);                
                    continue;
                }
                finally {
                    method.releaseConnection();
                }
                
                break;
            }   // for(int retry=0; retry<NUM_OF_RETRY; retry++)
            
            this.notify(this, symbols.size());
        }
        
        if(symbols.size() == 0) throw new StockNotFoundException();
        
        final List<Symbol> _symbols = new ArrayList<Symbol>(symbols);
        return getStocksBySymbols(_symbols);
    }

    public static void main(String[] args) throws StockNotFoundException {
        YahooStockServer server = new YahooStockServer(Country.Denmark);
        server.getAllStocks();
    }
    
    private final Country country;
    private final URL baseURL;
    
    private static final Map<Country, URL> servers = new HashMap<Country, URL>();
    private static final Map<Country, Pattern> patterns = new HashMap<Country, Pattern>();
    private static final Log log = LogFactory.getLog(YahooStockServer.class);
    
    private static final Pattern symbolPattern = Pattern.compile("<a\\s+href\\s*=[^>]+s=([^\">]+)\"?>Quote");
    
    // Yahoo server limit is 200. We shorter, to avoid URL from being too long.
    // Yahoo sometimes does complain URL for being too long.
    private static final int MAX_STOCK_PER_ITERATION = 180;
    private static final String YAHOO_CSV_BASED_URL = "http://finance.yahoo.com/d/quotes.csv?s=";
    
    // Yahoo server's result is not stable. If we request for 100 stocks, it may only
    // return 99 stocks to us. We allow stability rate in %. Higher rate means more
    // stable.
    private static final double STABILITY_RATE = 90.0;
    
    private static final int NUM_OF_RETRY = 2;
    
    // s = Symbol
    // n = Name
    // x = Stock Exchange
    // o = Open             <-- We are no longer using this one. It will not tally with change and change percentage
    // p = Previous Close
    // l1 = Last Trade (Price Only)
    // h = Day's high
    // g = Day's low
    // v = Volume           <-- We need to take special care on this, it may give us 1,234. This will
    //                          make us difficult to parse csv file. The only workaround is to make integer
    //                          in between two string literal (which will always contains "). By using regular
    //                          expression, we will manually remove the comma.
    // c1 = Change
    // p2 = Change Percent
    // k3 = Last Trade Size <-- We need to take special care on this, it may give us 1,234...
    // b = Bid
    // b6 = Bid Size        <-- We need to take special care on this, it may give us 1,234...
    // a = Ask
    // a5 = Ask Size        <-- We need to take special care on this, it may give us 1,234...
    // d1 = Last Trade Date
    // t1 = Last Trade Time
    //
    // c6k2c1p2c -> Change (Real-time), Change Percent (Real-time), Change, Change in Percent, Change & Percent Change
    // "+1400.00","N/A - +4.31%",+1400.00,"+4.31%","+1400.00 - +4.31%"
    //
    // "MAERSKB.CO","AP MOELLER-MAERS-","Copenhagen",32500.00,33700.00,34200.00,33400.00,660,"+1200.00","N/A - +3.69%",33,33500.00,54,33700.00,96,"11/10/2008","10:53am"
    private static final String YAHOO_STOCK_FORMAT = "&f=snxpl1hgsvsc1p2sk3sbsb6sasa5sd1t1";
    
    static {
        try {
            servers.put(Country.Denmark, new URL("http://uk.biz.yahoo.com/p/dk/cpi/index.html"));
            servers.put(Country.France, new URL("http://uk.biz.yahoo.com/p/fr/cpi/index.html"));
            servers.put(Country.Germany, new URL("http://uk.biz.yahoo.com/p/de/cpi/index.html"));
            servers.put(Country.Italy, new URL("http://uk.biz.yahoo.com/p/it/cpi/index.html"));
            servers.put(Country.Norway, new URL("http://uk.biz.yahoo.com/p/no/cpi/index.html"));
            servers.put(Country.Spain, new URL("http://uk.biz.yahoo.com/p/es/cpi/index.html"));
            servers.put(Country.Sweden, new URL("http://uk.biz.yahoo.com/p/se/cpi/index.html"));
            servers.put(Country.UnitedKingdom, new URL("http://uk.biz.yahoo.com/p/uk/cpi/index.html"));
            servers.put(Country.UnitedState, new URL("http://uk.biz.yahoo.com/p/us/cpi/index.html"));
        }
        catch(MalformedURLException exp) {
            // Shouldn't happen.
            exp.printStackTrace();
        }
        
        // <a href="/p/us/cpi/cpim0.html">
        patterns.put(Country.Denmark, Pattern.compile("<a\\s+href\\s*=\\s*\"?(\\/p\\/dk\\/cpi\\/[^\\s]+\\.html)"));
        patterns.put(Country.France, Pattern.compile("<a\\s+href\\s*=\\s*\"?(\\/p\\/fr\\/cpi\\/[^\\s]+\\.html)"));
        patterns.put(Country.Germany, Pattern.compile("<a\\s+href\\s*=\\s*\"?(\\/p\\/de\\/cpi\\/[^\\s]+\\.html)"));
        patterns.put(Country.Italy, Pattern.compile("<a\\s+href\\s*=\\s*\"?(\\/p\\/it\\/cpi\\/[^\\s]+\\.html)"));
        patterns.put(Country.Norway, Pattern.compile("<a\\s+href\\s*=\\s*\"?(\\/p\\/no\\/cpi\\/[^\\s]+\\.html)"));
        patterns.put(Country.Spain, Pattern.compile("<a\\s+href\\s*=\\s*\"?(\\/p\\/es\\/cpi\\/[^\\s]+\\.html)"));
        patterns.put(Country.Sweden, Pattern.compile("<a\\s+href\\s*=\\s*\"?(\\/p\\/se\\/cpi\\/[^\\s]+\\.html)"));
        patterns.put(Country.UnitedKingdom, Pattern.compile("<a\\s+href\\s*=\\s*\"?(\\/p\\/uk\\/cpi\\/[^\\s]+\\.html)"));
        patterns.put(Country.UnitedState, Pattern.compile("<a\\s+href\\s*=\\s*\"?(\\/p\\/us\\/cpi\\/[^\\s]+\\.html)"));
    }
}