sourcemodule.java

Python Development Environment (Python IDE plugin for Eclipse). Features editor, code completion, re

/*
 * Created on Nov 9, 2004
 *
 * @author Fabio Zadrozny
 */
package org.python.pydev.editor.codecompletion.revisited.modules;

import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.TreeMap;

import org.python.pydev.core.FindInfo;
import org.python.pydev.core.FullRepIterable;
import org.python.pydev.core.ICodeCompletionASTManager;
import org.python.pydev.core.ICompletionState;
import org.python.pydev.core.IDefinition;
import org.python.pydev.core.ILocalScope;
import org.python.pydev.core.IModule;
import org.python.pydev.core.IPythonNature;
import org.python.pydev.core.ISourceModule;
import org.python.pydev.core.IToken;
import org.python.pydev.core.REF;
import org.python.pydev.core.Tuple;
import org.python.pydev.core.Tuple3;
import org.python.pydev.core.structure.CompletionRecursionException;
import org.python.pydev.core.structure.FastStack;
import org.python.pydev.editor.codecompletion.revisited.AbstractToken;
import org.python.pydev.editor.codecompletion.revisited.CompletionState;
import org.python.pydev.editor.codecompletion.revisited.ConcreteToken;
import org.python.pydev.editor.codecompletion.revisited.visitors.AssignDefinition;
import org.python.pydev.editor.codecompletion.revisited.visitors.Definition;
import org.python.pydev.editor.codecompletion.revisited.visitors.FindDefinitionModelVisitor;
import org.python.pydev.editor.codecompletion.revisited.visitors.FindScopeVisitor;
import org.python.pydev.editor.codecompletion.revisited.visitors.GlobalModelVisitor;
import org.python.pydev.editor.codecompletion.revisited.visitors.LocalScope;
import org.python.pydev.editor.codecompletion.revisited.visitors.StopVisitingException;
import org.python.pydev.parser.jython.SimpleNode;
import org.python.pydev.parser.jython.ast.Assign;
import org.python.pydev.parser.jython.ast.Attribute;
import org.python.pydev.parser.jython.ast.ClassDef;
import org.python.pydev.parser.jython.ast.FunctionDef;
import org.python.pydev.parser.jython.ast.ImportFrom;
import org.python.pydev.parser.jython.ast.Name;
import org.python.pydev.parser.jython.ast.Str;
import org.python.pydev.parser.visitors.NodeUtils;
import org.python.pydev.plugin.PydevPlugin;

/**
 * The module should have all the information we need for code completion, find definition, and refactoring on a module.
 * 
 * Note: A module may be represented by a folder if it has an __init__.py file that represents the module or a python file.
 * 
 * Any of those must be a valid python token to be recognized (from the PYTHONPATH).
 * 
 * We don't reuse the ModelUtils already created as we still have to transport a lot of logic to it to make it workable, so, the attempt
 * here is to use a thin tier.
 * 
 * NOTE: When using it, don't forget to use the superclass abstraction.
 *  
 * @author Fabio Zadrozny
 */
public class SourceModule extends AbstractModule implements ISourceModule {

    private static final IToken[] EMPTY_ITOKEN_ARRAY = new IToken[0];

    private static final boolean DEBUG_INTERNAL_GLOBALS_CACHE = false;

    public static boolean TESTING = false;
    
    /**
     * This is the abstract syntax tree based on the jython parser output.
     */
    private SimpleNode ast;

    /**
     * File that originated the syntax tree.
     */
    private File file;

    /**
     * This is the time when the file was last modified.
     */
    private long lastModified;
    
    /**
     * The object may be a SourceToken or a List<SourceToken>
     */
    private HashMap<Integer, TreeMap<String, Object>> tokensCache = new HashMap<Integer, TreeMap<String,Object>>();

    /**
     * @return a reference to all the modules that are imported from this one in the global context as a from xxx import *
     * 
     * This modules are treated specially, as we don't care which tokens were imported. When this is requested, the module is prompted for
     * its tokens.
     */
    public IToken[] getWildImportedModules() {
        return getTokens(GlobalModelVisitor.WILD_MODULES, null);
    }

    /**
     * Searches for the following import tokens:
     *   import xxx 
     *   import xxx as ... 
     *   from xxx import xxx
     *   from xxx import xxx as ....
     * Note, that imports with wildcards are not collected.
     * @return an array of references to the modules that are imported from this one in the global context.
     */
    public IToken[] getTokenImportedModules() {
        return getTokens(GlobalModelVisitor.ALIAS_MODULES, null);
    }

    /**
     * 
     * @return the file this module corresponds to.
     */
    public File getFile(){
        return this.file;
    }
    
    /**
     * @return the tokens that are present in the global scope.
     * 
     * The tokens can be class definitions, method definitions and attributes.
     */
    public IToken[] getGlobalTokens() {
        return getTokens(GlobalModelVisitor.GLOBAL_TOKENS, null);
    }

    /**
     * @return a string representing the module docstring.
     */
    public String getDocString() {
        IToken[] l = getTokens(GlobalModelVisitor.MODULE_DOCSTRING, null);
        if (l.length > 0) {
            SimpleNode a = ((SourceToken) l[0]).getAst();

            return ((Str) a).s;
        }
        return "";
    }


    /**
     * Checks if it is in the global tokens that were created in this module
     * @param tok the token we are looking for
     * @param nature the nature
     * @return true if it was found and false otherwise
     */
    public boolean isInDirectGlobalTokens(String tok){
        TreeMap<String, Object> tokens = tokensCache.get(GlobalModelVisitor.GLOBAL_TOKENS);
    	if(tokens == null){
    		getGlobalTokens();
    	}
    	
    	boolean ret = false;
    	if(tokens != null){
        	synchronized (tokens) {
        		ret = tokens.containsKey(tok);
        	}
    	}
    	
    	if(ret == false){
    		ret = isInDirectImportTokens(tok);
    	}
    	return ret;
    }
    
	public boolean isInDirectImportTokens(String tok) {
        TreeMap<String, Object> tokens = tokensCache.get(GlobalModelVisitor.ALIAS_MODULES);
        if(tokens != null){
			getTokenImportedModules();
		}
		
		boolean ret = false;
		if(tokens != null){
			synchronized (tokens) {
				ret = tokens.containsKey(tok);
			}
		}
		return ret; 
	}

    
	private IToken[] getTokens(int which, ICompletionState state) {
        return getTokens(which, state, null);
    }
    
    /**
     * @param lookOnlyForNameStartingWith: if not null, well only get from the cache tokens starting with the given representation
     * @return a list of IToken
     */
    private synchronized IToken[] getTokens(int which, ICompletionState state, String lookOnlyForNameStartingWith) {
        if((which & GlobalModelVisitor.INNER_DEFS) != 0){
            throw new RuntimeException("Cannot do this one with caches");
        }
    	//cache
        TreeMap<String, Object> tokens = tokensCache.get(which);
        
        if(tokens != null){
            return createArrayFromCacheValues(tokens, lookOnlyForNameStartingWith);
        }
    	//end cache
    	
    	
        try {
            tokensCache.put(GlobalModelVisitor.ALIAS_MODULES, new TreeMap<String, Object>());
            tokensCache.put(GlobalModelVisitor.GLOBAL_TOKENS, new TreeMap<String, Object>());
            tokensCache.put(GlobalModelVisitor.WILD_MODULES, new TreeMap<String, Object>());
            tokensCache.put(GlobalModelVisitor.MODULE_DOCSTRING, new TreeMap<String, Object>());

            int all = GlobalModelVisitor.ALIAS_MODULES | GlobalModelVisitor.GLOBAL_TOKENS | 
                      GlobalModelVisitor.WILD_MODULES | GlobalModelVisitor.MODULE_DOCSTRING;
            
            //we request all and put it into the cache (partitioned), because that's faster than making multiple runs through it
            List<IToken> ret = GlobalModelVisitor.getTokens(ast, all, name, state);
            
            
            if(DEBUG_INTERNAL_GLOBALS_CACHE){
                System.out.println("\n\nLooking for:"+which);
            }
            //cache
    		for (IToken token : ret) {
                int choice;
        		if(token.isWildImport()){
        		    choice = GlobalModelVisitor.WILD_MODULES;
                }else if(token.isImportFrom() || token.isImport()){
                    choice = GlobalModelVisitor.ALIAS_MODULES;
                }else if(token.isString()){
                    choice = GlobalModelVisitor.MODULE_DOCSTRING;
                }else{
                    choice = GlobalModelVisitor.GLOBAL_TOKENS;
                }
                String rep = token.getRepresentation();
                if(DEBUG_INTERNAL_GLOBALS_CACHE){
            		System.out.println("Adding choice:"+choice+" name:"+rep);
                    if(choice != which){
                        System.out.println("Looking for:"+which+"found:"+choice);
                        System.out.println("here");
                    }
                }
                TreeMap<String, Object> treeMap = tokensCache.get(choice);
                SourceToken newSourceToken = (SourceToken) token;
                Object current = treeMap.get(rep);
                if(current == null){ 
                    treeMap.put(rep, newSourceToken);
                }else{
                    //the new references (later in the module) are always added to the head of the position...
                    if(current instanceof List){
                        ((List)current).add(0, newSourceToken);
                        
                    }else if(current instanceof SourceToken){
                        ArrayList<SourceToken> lst = new ArrayList<SourceToken>();
                        lst.add(newSourceToken);
                        lst.add((SourceToken) current);
                        treeMap.put(rep, lst);
                        
                    }else{
                        throw new RuntimeException("Unexpected class in cache:"+current);
                        
                    }
                }
    		}
            //end cache
            
        } catch (Exception e) {
            e.printStackTrace();
        }
        
        //now, let's get it from the cache... (which should be filled by now)
        tokens = tokensCache.get(which);
        return createArrayFromCacheValues(tokens, lookOnlyForNameStartingWith);
    }

    private IToken[] createArrayFromCacheValues(TreeMap<String, Object> tokens, String lookOnlyForNameStartingWith) {
        List<SourceToken> ret = new ArrayList<SourceToken>();
        
        Collection<Object> lookIn;
        if(lookOnlyForNameStartingWith == null){
            lookIn = tokens.values();
        }else{
            lookIn = tokens.subMap(lookOnlyForNameStartingWith, lookOnlyForNameStartingWith+"z").values();
        }
        
        
        for(Object o:lookIn){
            if(o instanceof SourceToken){
                ret.add((SourceToken) o);
            }else if(o instanceof List){
                ret.addAll((List)o);
            }else{
                throw new RuntimeException("Unexpected class in cache:"+o);
            }
        }
        return ret.toArray(new SourceToken[ret.size()]);
    }

    /**
     * 
     * @param name
     * @param f
     * @param n
     */
    public SourceModule(String name, File f, SimpleNode n) {
        super(name);
        this.ast = n;
        this.file = f;
        if(f != null)
            this.lastModified = f.lastModified();
    }