pyparser.java

Python Development Environment (Python IDE plugin for Eclipse). Features editor, code completion, re

        if(obj.o2 != null && obj.o2 instanceof TokenMgrError){
            fireParserError((TokenMgrError) obj.o2, adaptable, document, argsToReparse);
        }
        
        return obj;
    }
    

    
    //static methods that can be used to get the ast (and error if any) --------------------------------------
    

    public static class ParserInfo{
        public IDocument document;
        public boolean stillTryToChangeCurrentLine=true; 
        
        /**
         * The currently selected line
         */
        public int currentLine=-1;
        
        /**
         * The initial document to be parsed
         */
        public String initial = null;
        
        /**
         * A set with the lines that were changed when trying to make the document parseable
         */
        public Set<Integer> linesChanged = new HashSet<Integer>();
        
        /**
         * The 1st parse error we receive when doing the parsing of a document.
         */
        public ParseException parseErr;
        
        /**
         * Marks whether we should use an heuristic to try to make reparses (if false, it will bail out in the 1st error).
         */
        public boolean tryReparse = TRY_REPARSE;
        
        /**
         * This is the version of the grammar to be used 
         * @see IPythonNature.GRAMMAR_XXX constants
         */
        public int grammarVersion;
        
        /**
         * @param grammarVersion: see IPythonNature.GRAMMAR_XXX constants
         */
        public ParserInfo(IDocument document, boolean changedCurrentLine, int grammarVersion){
            this.document = document;
            this.stillTryToChangeCurrentLine = changedCurrentLine;
            this.grammarVersion = grammarVersion;
        }
        
        public ParserInfo(IDocument document, boolean changedCurrentLine, IPythonNature nature){
            this(document, changedCurrentLine, nature.getGrammarVersion());
        }
        
        public ParserInfo(IDocument document, boolean changedCurrentLine, IPythonNature nature, int currentLine){
            this(document, changedCurrentLine, nature);
            this.currentLine = currentLine;
        }
    }
    
    
    /**
     * Removes comments at the end of the document
     * @param doc this is the document from where the comments must be removed
     * @return a tuple with: StringBuffer with the comments that have been removed,
     * beginLine for the comments beginColumn for the comments
     * (both starting at 1)
     */
    public static List<commentType> removeEndingComments(IDocument doc){
        StringBuffer comments = new StringBuffer();
        int lines = doc.getNumberOfLines();
        String delimiter = PySelection.getDelimiter(doc);
        
        for (int i = lines-1; i >= 0; i--) {
            String line = PySelection.getLine(doc, i);
            String trimmed = line.trim();
            if(trimmed.length() > 0 && trimmed.charAt(0) != '#'){
                return makeListOfComments(comments, line.length()+2, i+1);
            }
            comments.insert(0,line);
            comments.insert(0,delimiter);
            try {
                if(line.length() > 0){
                    PySelection.deleteLine(doc, i);
                }
            } catch (Exception e) {
                //ignore
            }
        }
        
        return makeListOfComments(comments,0,0);
    }

    private static List<commentType> makeListOfComments(StringBuffer comments, int beginCol, int beginLine) {
        ArrayList<commentType> ret = new ArrayList<commentType>();
        int len = comments.length();

        char c;
        StringBuffer buf = null;

        int col=0;
        int line=0;
        int startCol=-1;
        for (int i = 0; i < len; i++) {
            c = comments.charAt(i);

            if(buf == null && c == '#'){
                buf = new StringBuffer();
                startCol = col;
            }
            

            if (c == '\r') {
                if (i < len - 1 && comments.charAt(i + 1) == '\n') {
                    i++;
                }
                addCommentLine(ret, buf, beginCol, beginLine, startCol, line);
                buf = null;
                col = 0;
                line++;
                startCol = -1;
            }
            if (c == '\n') {
                addCommentLine(ret, buf, beginCol, beginLine, startCol, line);
                buf = null;
                col = 0;
                line++;
                startCol = -1;
            }
            
            if(buf != null){
                buf.append(c);
            }
            col++;
        }
        
        if (buf != null && buf.length() != 0) {
            addCommentLine(ret, buf, beginCol, beginLine, startCol, line);
        }
        return ret;
    }

    private static void addCommentLine(ArrayList<commentType> ret, StringBuffer buf, int beginCol, int beginLine, int col, int line) {
        if(buf != null){
            commentType comment = new commentType(buf.toString());
            comment.beginLine = beginLine+line;
            if(line == 0){
                comment.beginColumn = beginCol+col;
            }else{
                comment.beginColumn = col;
            }
            ret.add(comment);
        }
    }
    
    
    /**
     * @return a tuple with the SimpleNode root(if parsed) and the error (if any).
     *         if we are able to recover from a reparse, we have both, the root and the error.
     */
    public static Tuple<SimpleNode, Throwable> reparseDocument(ParserInfo info) {
        // create a stream with document's data
        String startDoc = info.document.get();
        if(info.initial == null){
            info.initial = startDoc;
        }

        IDocument newDoc = new Document(startDoc);
        List<commentType> comments = removeEndingComments(newDoc);
        try {
            //make sure it ends with a new line
            newDoc.replace(newDoc.getLength(), 0, "\n");
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
        
        
        CharStream in = null;
        if(USE_FAST_STREAM){
            //we don't want to keep the string from being released, so, just get the char array from the string
            char[] cs = newDoc.get().toCharArray();
        	in = new FastCharStream(cs);
        }else{
        	//this should be deprecated in the future (it is still here so that we can evaluate
        	//the changes done by the change of the reader).
            String initialDoc = newDoc.get();
	        StringReader inString = new StringReader(initialDoc);
	        in = new ReaderCharStream(inString);
            throw new RuntimeException("This char stream reader was deprecated (was maintained only for testing purposes).");
        }
        
        IParserHost host = new CompilerAPI();
        IGrammar grammar = null;
        if(info.grammarVersion == IPythonNature.GRAMMAR_PYTHON_VERSION_2_4){
            grammar = new PythonGrammar24(in, host);
        }else if(info.grammarVersion == IPythonNature.GRAMMAR_PYTHON_VERSION_2_5){
            grammar = new PythonGrammar25(in, host);
        }else{
            throw new RuntimeException("The grammar specified for parsing is not valid: "+info.grammarVersion);
        }

        try {
        	
        	if(ENABLE_TRACING){
        		//grammar has to be generated with debugging info for this to make a difference
        		grammar.enable_tracing();
        	}
            SimpleNode newRoot = grammar.file_input(); // parses the file
            if(newRoot != null){
                Module m = (Module) newRoot;
                for (commentType comment : comments) {
                    m.addSpecial(comment, true);
                }
            }
            return new Tuple<SimpleNode, Throwable>(newRoot,null);
		

        } catch (Throwable e) {
            //ok, some error happened when trying the parse... let's go and clear the local info before doing
            //another parse.
            startDoc = null;
            newDoc = null;
            comments = null;
            in = null;
            host = null;
            grammar = null;
            
            if(e instanceof ParseException){
                ParseException parseErr = (ParseException) e;
                SimpleNode newRoot = null;
                
                if(info.parseErr == null){
                    info.parseErr = parseErr;
                }
                
                if(info.tryReparse){
                    if (info.stillTryToChangeCurrentLine){
                        newRoot = tryReparseAgain(info, info.parseErr);
                    } else {
                        info.currentLine = -1;
                        info.document = new Document(info.initial);
                        newRoot = tryReparseAgain(info, info.parseErr);
                    }
                }
                return new Tuple<SimpleNode, Throwable>(newRoot, parseErr);
                
            }else if(e instanceof TokenMgrError){
                TokenMgrError tokenErr = (TokenMgrError) e;
                SimpleNode newRoot = null;
                
                if(info.tryReparse){
                    if (info.stillTryToChangeCurrentLine){
                        newRoot = tryReparseAgain(info, tokenErr);
                    }
                }
                
                return new Tuple<SimpleNode, Throwable>(newRoot, tokenErr);
                
            }else if(e.getClass().getName().indexOf("LookaheadSuccess") != -1){
                //don't log this kind of error...
            }else{
                Log.log(e);
            }
            return new Tuple<SimpleNode, Throwable>(null, null);
        
        } 
    }

    /**
     * @param tokenErr
     */
    private static SimpleNode tryReparseAgain(ParserInfo info, TokenMgrError tokenErr) {
        int line = -1;
        
        if(info.currentLine > -1){
            line = info.currentLine;
        }else{
            line = tokenErr.errorLine;
        }
        
        return tryReparseChangingLine(info, line);
    }

    /**
     * This method tries to reparse the code again, changing the current line to
     * a 'pass'
     * 
     * Any new errors are ignored, and the error passed as a parameter is fired
     * anyway, so, the utility of this function is trying to make a real model
     * without any problems, so that we can update the outline and ModelUtils
     * with a good approximation of the code.
     * 
     * @param tokenErr
     */
    private static SimpleNode tryReparseAgain(ParserInfo info, ParseException tokenErr) {
        int line = -1;
        
        if(info.currentLine > -1){
            line = info.currentLine;
        
        }else{
            if(tokenErr.currentToken != null){
                line = tokenErr.currentToken.beginLine-2;
            
	        }else{
	            return null;   
            }
        }
        
        if(line < 0){
            line = 0;
        }
        
        boolean okToGo = false;
        
        while(! okToGo){
            
            if(!lineIn(info.linesChanged, line)){
                info.linesChanged.add(new Integer(line));
                okToGo = true;
            } 
            
            if(!okToGo){
                if(info.linesChanged.size() < 5){
                    line += 1;
                    
                } else{
                    //it can go up to 5 reparses before bailing out and returning null.
                    return null;
                }
            }
        }

        return tryReparseChangingLine(info, line);
    }

    /**
     * @param linesChanged the lines that were already changed in the document
     * @param line the line which we want to check if it was already changed or not
     * @return true if the line passed was already changed and false otherwise.
     */
    private static boolean lineIn(Set<Integer> linesChanged, int line) {
        if(linesChanged.contains(line)){
            return true;
        }
        return false;
    }

    /**
     * Try a reparse changing the offending line.
     * 
     * @param document: this is the document to be changed
     * @param line: offending line to be changed to try a reparse.
     * 
     */
    private static SimpleNode tryReparseChangingLine(ParserInfo info, int line) {
        String docToParse = DocUtils.getDocToParseFromLine(info.document, line);
        if(docToParse != null){
            //System.out.println("\n\n\n\n\nSTART Parsing -------------------------");
            //System.out.println(docToParse);
            //System.out.println("END Parsing -------------------------");
            Document doc = new Document(docToParse);
            info.document = doc;
            info.stillTryToChangeCurrentLine = false;
	        return reparseDocument(info).o1;
        }
        return null;
    }

    public void resetTimeoutPreferences(boolean useAnalysisOnlyOnDocSave) {
        this.useAnalysisOnlyOnDocSave = useAnalysisOnlyOnDocSave;
    }

    public List<IParserObserver> getObservers() {
        return new ArrayList<IParserObserver>(this.parserListeners);
    }


    
}