tokenmodule.java

来自「RESIN 3.2 最新源码」· Java 代码 · 共 1,081 行 · 第 1/2 页

/*
 * Copyright (c) 1998-2008 Caucho Technology -- all rights reserved
 *
 * This file is part of Resin(R) Open Source
 *
 * Each copy or derived work must preserve the copyright notice and this
 * notice unmodified.
 *
 * Resin Open Source is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * Resin Open Source is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, or any warranty
 * of NON-INFRINGEMENT.  See the GNU General Public License for more
 * details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Resin Open Source; if not, write to the
 *
 *   Free Software Foundation, Inc.
 *   59 Temple Place, Suite 330
 *   Boston, MA 02111-1307  USA
 *
 * @author Scott Ferguson
 */

package com.caucho.quercus.lib;

import com.caucho.quercus.QuercusModuleException;
import com.caucho.quercus.annotation.Optional;
import com.caucho.quercus.env.*;
import com.caucho.quercus.lib.file.FileModule;
import com.caucho.quercus.module.AbstractQuercusModule;
import com.caucho.quercus.module.IniDefinitions;
import com.caucho.quercus.module.IniDefinition;
import com.caucho.util.IntMap;
import com.caucho.util.L10N;
import com.caucho.vfs.WriteStream;

import java.io.IOException;
import java.util.logging.Logger;

/**
 * Quercus tokenizer 
 */
public class TokenModule extends AbstractQuercusModule {
  private static final L10N L = new L10N(TokenModule.class);
  private static final Logger log
    = Logger.getLogger(TokenModule.class.getName());

  public static final int T_ABSTRACT = 256;
  public static final int T_AND_EQUAL = 257;
  public static final int T_ARRAY = 258;
  public static final int T_ARRAY_CAST = 259;
  public static final int T_AS = 260;
  public static final int T_BAD_CHARACTER = 261;
  public static final int T_BOOLEAN_AND = 262;
  public static final int T_BOOLEAN_OR = 263;
  public static final int T_BOOL_CAST = 264;
  public static final int T_BREAK = 265;
  public static final int T_CASE = 266;
  public static final int T_CATCH = 267;
  public static final int T_CHARACTER = 268;
  public static final int T_CLASS = 269;
  public static final int T_CLONE = 270;
  public static final int T_CLOSE_TAG = 271;
  public static final int T_COMMENT = 272;
  public static final int T_CONCAT_EQUAL = 273;
  public static final int T_CONST = 274;
  public static final int T_CONSTANT_ENCAPSED_STRING = 275;
  public static final int T_CONTINUE = 276;
  public static final int T_CURLY_OPEN = 277;
  public static final int T_DEC = 278;
  public static final int T_DECLARE = 279;
  public static final int T_DEFAULT = 280;
  public static final int T_DIV_EQUAL = 281;
  public static final int T_DNUMBER = 282;
  public static final int T_DOC_COMMENT = 283;
  public static final int T_DO = 284;
  public static final int T_DOLLAR_OPEN_CURLY_BRACES = 285;
  public static final int T_DOUBLE_ARROW = 286;
  public static final int T_DOUBLE_CAST = 287;
  public static final int T_DOUBLE_COLON = 288;
  public static final int T_ECHO = 289;
  public static final int T_ELSE = 290;
  public static final int T_ELSEIF = 291;
  public static final int T_EMPTY = 292;
  public static final int T_ENCAPSED_AND_WHITESPACE = 293;
  public static final int T_ENDDECLARE = 294;
  public static final int T_ENDFOR = 295;
  public static final int T_ENDFOREACH = 296;
  public static final int T_ENDIF = 297;
  public static final int T_ENDSWITCH = 298;
  public static final int T_ENDWHILE = 299;
  public static final int T_END_HEREDOC = 300;
  public static final int T_EVAL = 301;
  public static final int T_EXIT = 302;
  public static final int T_EXTENDS = 303;
  public static final int T_FILE = 304;
  public static final int T_FINAL = 305;
  public static final int T_FOR = 306;
  public static final int T_FOREACH = 307;
  public static final int T_FUNCTION = 308;
  public static final int T_GLOBAL = 309;
  public static final int T_HALT_COMPILER = 310;
  public static final int T_IF = 311;
  public static final int T_IMPLEMENTS = 312;
  public static final int T_INC = 313;
  public static final int T_INCLUDE = 314;
  public static final int T_INCLUDE_ONCE = 315;
  public static final int T_INLINE_HTML = 316;
  public static final int T_INSTANCEOF = 317;
  public static final int T_INT_CAST = 318;
  public static final int T_INTERFACE = 319;
  public static final int T_ISSET = 320;
  public static final int T_IS_EQUAL = 321;
  public static final int T_IS_GREATER_OR_EQUAL = 322;
  public static final int T_IS_IDENTICAL = 323;
  public static final int T_IS_NOT_EQUAL = 324;
  public static final int T_IS_NOT_IDENTICAL = 325;
  public static final int T_IS_SMALLER_OR_EQUAL = 326;
  public static final int T_LINE = 327;
  public static final int T_LIST = 328;
  public static final int T_LNUMBER = 329;
  public static final int T_LOGICAL_AND = 330;
  public static final int T_LOGICAL_OR = 331;
  public static final int T_LOGICAL_XOR = 332;
  public static final int T_MINUS_EQUAL = 333;
  public static final int T_ML_COMMENT = 334;
  public static final int T_MOD_EQUAL = 335;
  public static final int T_MUL_EQUAL = 336;
  public static final int T_NEW = 337;
  public static final int T_NUM_STRING = 338;
  public static final int T_OBJECT_CAST = 339;
  public static final int T_OBJECT_OPERATOR = 340;
  public static final int T_OLD_FUNCTION = 341;
  public static final int T_OPEN_TAG = 342;
  public static final int T_OPEN_TAG_WITH_ECHO = 343;
  public static final int T_OR_EQUAL = 344;
  public static final int T_PAAMAYIM_NEKUDOTAYIM = T_DOUBLE_COLON;
  public static final int T_PLUS_EQUAL = 345;
  public static final int T_PRINT = 346;
  public static final int T_PRIVATE = 347;
  public static final int T_PUBLIC = 348;
  public static final int T_PROTECTED = 349;
  public static final int T_REQUIRE = 350;
  public static final int T_REQUIRE_ONCE = 351;
  public static final int T_RETURN = 352;
  public static final int T_SL = 353;
  public static final int T_SL_EQUAL = 354;
  public static final int T_SR = 355;
  public static final int T_SR_EQUAL = 356;
  public static final int T_START_HEREDOC = 357;
  public static final int T_STATIC = 358;
  public static final int T_STRING = 359;
  public static final int T_STRING_CAST = 360;
  public static final int T_STRING_VARNAME = 361;
  public static final int T_SWITCH = 362;
  public static final int T_THROW = 363;
  public static final int T_TRY = 364;
  public static final int T_UNSET = 365;
  public static final int T_UNSET_CAST = 366;
  public static final int T_USE = 367;
  public static final int T_VAR = 368;
  public static final int T_VARIABLE = 369;
  public static final int T_WHILE = 370;
  public static final int T_WHITESPACE = 371;
  public static final int T_XOR_EQUAL = 372;
  public static final int T_FUNC_C = 373;
  public static final int T_CLASS_C = 374;

  private static final IntMap _reservedMap = new IntMap();

  private static final IniDefinitions _iniDefinitions = new IniDefinitions();

  public String []getLoadedExtensions()
  {
    return new String[] { "tokenizer" };
  }

  /**
   * Returns the default php.ini values.
   */
  public IniDefinitions getIniDefinitions()
  {
    return _iniDefinitions;
  }

  public static Value highlight_file(Env env,
				     StringValue filename,
				     @Optional boolean isReturn)
  {
    StringValue v = FileModule.file_get_contents(env,
						 filename,
						 false,
						 null,
						 0,
						 Integer.MAX_VALUE);

    if (v == null)
      return BooleanValue.FALSE;
    
    return highlight_string(env, v, isReturn);
  }

  public static Value highlight_string(Env env,
				       StringValue s,
				       @Optional boolean isReturn)
  {
    try {
      StringValue sb = isReturn ? env.createUnicodeBuilder() : null;
      WriteStream out = env.getOut();

      Token lexer = new Token(env, s);
      int token;
      StringValue topColor = env.createString("#000000");
      StringValue lastColor = topColor;

      highlight(sb, out, "<code>");
      highlight(sb, out, "<span style=\"color: #000000\">\n");
      
      while ((token = lexer.nextToken()) >= 0) {
	StringValue color = getColor(env, token);

	if (color != null && ! color.equals(lastColor)) {
	  if (! topColor.equals(lastColor))
	    highlight(sb, out, "</span>");
	  
	  if (! topColor.equals(color))
	    highlight(sb, out, "<span style=\"color: " + color + "\">");
	  
	  lastColor = color;
	}
	
	if (0x20 <= token && token <= 0x7f) {
	  if (sb != null)
	    sb.append((char) token);
	  else
	    out.print((char) token);
	}
	else {
	  StringValue lexeme = lexer.getLexeme();

	  highlight(sb, out, lexeme);
	}
      }
      
      if (! topColor.equals(lastColor))
	highlight(sb, out, "</span>\n");
      highlight(sb, out, "</span>\n");
      highlight(sb, out, "</code>");

      if (sb != null)
	return sb;
      else
	return BooleanValue.TRUE;
    } catch (IOException e) {
      throw new QuercusModuleException(e);
    }
  }

  private static void highlight(StringValue sb,
				WriteStream out,
				String string)
    throws IOException
  {
    if (sb != null) {
      sb.append(string);
    }
    else {
      out.print(string);
    }
  }

  private static void highlight(StringValue sb,
				WriteStream out,
				StringValue string)
    throws IOException
  {
    if (sb != null) {
      int len = string.length();
      for (int i = 0; i < len; i++) {
	char ch = string.charAt(i);

	switch (ch) {
	case '<':
	  sb.append("&lt;");
	  break;
	case '>':
	  sb.append("&gt;");
	  break;
	default:
	  sb.append(ch);
	  break;
	}
      }
    }
    else {
      int len = string.length();
      for (int i = 0; i < len; i++) {
	char ch = string.charAt(i);

	switch (ch) {
	case '<':
	  out.print("&lt;");
	  break;
	case '>':
	  out.print("&gt;");
	  break;
	default:
	  out.print(ch);
	  break;
	}
      }
    }
  }

  private static StringValue getColor(Env env, int token)
  {
    switch (token) {
    case T_INLINE_HTML:
      return env.getIni("highlight.html");

    case T_BAD_CHARACTER:
    case T_CHARACTER:
    case T_CLOSE_TAG:
    case T_DNUMBER:
    case T_END_HEREDOC:
    case T_FILE:
    case T_LINE:
    case T_LNUMBER:
    case T_OPEN_TAG:
    case T_OPEN_TAG_WITH_ECHO:
      return env.getIni("highlight.default");
      
    case T_COMMENT:
    case T_DOC_COMMENT:
    case T_ML_COMMENT:
      return env.getIni("highlight.comment");
      
    case T_CONSTANT_ENCAPSED_STRING:
    case T_STRING:
      return env.getIni("highlight.string");

    case T_ENCAPSED_AND_WHITESPACE:
    case T_WHITESPACE:
      return null;
      
    default:
      return env.getIni("highlight.keyword");
    }
  }

  /**
   * Parses the string.
   */
  public static ArrayValue token_get_all(Env env, StringValue s)
  {
    ArrayValue result = new ArrayValueImpl();

    Token lexer = new Token(env, s);
    int token;

    while ((token = lexer.nextToken()) >= 0) {
      if (0x20 <= token && token <= 0x7f) {
	result.put(env.createString((char) token));
      }
      else {
	result.put(new ArrayValueImpl()
		   .append(LongValue.create(token))
		   .append(lexer.getLexeme()));
      }
    }

    return result;
  }
  
  /**
   * Returns the token name for th egiven token.
   */
  public static String token_name(int token)
  {
    switch (token) {
    case T_ABSTRACT: return "T_ABSTRACT";
    case T_AND_EQUAL: return "T_AND_EQUAL";
    case T_ARRAY: return "T_ARRAY";
    case T_ARRAY_CAST: return "T_ARRAY_CAST";
    case T_AS: return "T_AS";
    case T_BAD_CHARACTER: return "T_BAD_CHARACTER";
    case T_BOOLEAN_AND: return "T_BOOLEAN_AND";
    case T_BOOLEAN_OR: return "T_BOOLEAN_OR";
    case T_BOOL_CAST: return "T_BOOL_CAST";
    case T_BREAK: return "T_BREAK";
    case T_CASE: return "T_CASE";
    case T_CATCH: return "T_CATCH";
    case T_CHARACTER: return "T_CHARACTER";
    case T_CLASS: return "T_CLASS";
    case T_CLONE: return "T_CLONE";
    case T_CLOSE_TAG: return "T_CLOSE_TAG";
    case T_COMMENT: return "T_COMMENT";
    case T_CONCAT_EQUAL: return "T_CONCAT_EQUAL";
    case T_CONST: return "T_CONST";
    case T_CONSTANT_ENCAPSED_STRING: return "T_CONSTANT_ENCAPSED_STRING";
    case T_CONTINUE: return "T_CONTINUE";
    case T_CURLY_OPEN: return "T_CURLY_OPEN";
    case T_DEC: return "T_DEC";
    case T_DECLARE: return "T_DECLARE";
    case T_DEFAULT: return "T_DEFAULT";
    case T_DIV_EQUAL: return "T_DIV_EQUAL";
    case T_DNUMBER: return "T_DNUMBER";
    case T_DOC_COMMENT: return "T_DOC_COMMENT";
    case T_DO: return "T_DO";
    case T_DOLLAR_OPEN_CURLY_BRACES: return "T_DOLLAR_OPEN_CURLY_BRACES";
    case T_DOUBLE_ARROW: return "T_DOUBLE_ARROW";
    case T_DOUBLE_CAST: return "T_DOUBLE_CAST";
    case T_DOUBLE_COLON: return "T_DOUBLE_COLON";
    case T_ECHO: return "T_ECHO";
    case T_ELSE: return "T_ELSE";
    case T_ELSEIF: return "T_ELSEIF";
    case T_EMPTY: return "T_EMPTY";
    case T_ENCAPSED_AND_WHITESPACE: return "T_ENCAPSED_AND_WHITESPACE";
    case T_ENDDECLARE: return "T_ENDDECLARE";
    case T_ENDFOR: return "T_ENDFOR";
    case T_ENDFOREACH: return "T_ENDFOREACH";
    case T_ENDIF: return "T_ENDIF";
    case T_ENDSWITCH: return "T_ENDSWITCH";
    case T_ENDWHILE: return "T_ENDWHILE";
    case T_END_HEREDOC: return "T_END_HEREDOC";
    case T_EVAL: return "T_EVAL";
    case T_EXIT: return "T_EXIT";
    case T_EXTENDS: return "T_EXTENDS";
    case T_FILE: return "T_FILE";
    case T_FINAL: return "T_FINAL";
    case T_FOR: return "T_FOR";
    case T_FOREACH: return "T_FOREACH";
    case T_FUNCTION: return "T_FUNCTION";
    case T_GLOBAL: return "T_GLOBAL";
    case T_HALT_COMPILER: return "T_HALT_COMPILER";
    case T_IF: return "T_IF";
    case T_IMPLEMENTS: return "T_IMPLEMENTS";
    case T_INC: return "T_INC";
    case T_INCLUDE: return "T_INCLUDE";
    case T_INCLUDE_ONCE: return "T_INCLUDE_ONCE";
    case T_INLINE_HTML: return "T_INLINE_HTML";
    case T_INSTANCEOF: return "T_INSTANCEOF";
    case T_INT_CAST: return "T_INT_CAST";
    case T_INTERFACE: return "T_INTERFACE";
    case T_ISSET: return "T_ISSET";
    case T_IS_EQUAL: return "T_IS_EQUAL";
    case T_IS_GREATER_OR_EQUAL: return "T_IS_GREATER_OR_EQUAL";
    case T_IS_IDENTICAL: return "T_IS_IDENTICAL";
    case T_IS_NOT_EQUAL: return "T_IS_NOT_EQUAL";
    case T_IS_NOT_IDENTICAL: return "T_IS_NOT_IDENTICAL";
    case T_IS_SMALLER_OR_EQUAL: return "T_IS_SMALLER_OR_EQUAL";
    case T_LINE: return "T_LINE";
    case T_LIST: return "T_LIST";
    case T_LNUMBER: return "T_LNUMBER";
    case T_LOGICAL_AND: return "T_LOGICAL_AND";
    case T_LOGICAL_OR: return "T_LOGICAL_OR";
    case T_LOGICAL_XOR: return "T_LOGICAL_XOR";
    case T_MINUS_EQUAL: return "T_MINUS_EQUAL";
    case T_ML_COMMENT: return "T_ML_COMMENT";
    case T_MOD_EQUAL: return "T_MOD_EQUAL";
    case T_MUL_EQUAL: return "T_MUL_EQUAL";
    case T_NEW: return "T_NEW";
    case T_NUM_STRING: return "T_NUM_STRING";
    case T_OBJECT_CAST: return "T_OBJECT_CAST";
    case T_OBJECT_OPERATOR: return "T_OBJECT_OPERATOR";
    case T_OLD_FUNCTION: return "T_OLD_FUNCTION";
    case T_OPEN_TAG: return "T_OPEN_TAG";
    case T_OPEN_TAG_WITH_ECHO: return "T_OPEN_TAG_WITH_ECHO";
    case T_OR_EQUAL: return "T_OR_EQUAL";
    case T_PLUS_EQUAL: return "T_PLUS_EQUAL";
    case T_PRINT: return "T_PRINT";
    case T_PRIVATE: return "T_PRIVATE";
    case T_PUBLIC: return "T_PUBLIC";
    case T_PROTECTED: return "T_PROTECTED";
    case T_REQUIRE: return "T_REQUIRE";
    case T_REQUIRE_ONCE: return "T_REQUIRE_ONCE";
    case T_RETURN: return "T_RETURN";
    case T_SL: return "T_SL";
    case T_SL_EQUAL: return "T_SL_EQUAL";
    case T_SR: return "T_SR";
    case T_SR_EQUAL: return "T_SR_EQUAL";
    case T_START_HEREDOC: return "T_START_HEREDOC";
    case T_STATIC: return "T_STATIC";
    case T_STRING: return "T_STRING";
    case T_STRING_CAST: return "T_STRING_CAST";
    case T_STRING_VARNAME: return "T_STRING_VARNAME";
    case T_SWITCH: return "T_SWITCH";
    case T_THROW: return "T_THROW";
    case T_TRY: return "T_TRY";
    case T_UNSET: return "T_UNSET";
    case T_UNSET_CAST: return "T_UNSET_CAST";
    case T_USE: return "T_USE";
    case T_VAR: return "T_VAR";
    case T_VARIABLE: return "T_VARIABLE";
    case T_WHILE: return "T_WHILE";
    case T_WHITESPACE: return "T_WHITESPACE";
    case T_XOR_EQUAL: return "T_XOR_EQUAL";
    case T_FUNC_C: return "T_FUNC_C";
    case T_CLASS_C: return "T_CLASS_C";
      
    default:
      return "UNDEFINED";
    }
  }

  static class Token {
    private Env _env;
    private final StringValue _s;
    private final int _length;
    private int _i;
    private boolean _inPhp;

    private StringValue _lexeme;

    Token(Env env, StringValue s)
    {
      _env = env;
      _s = s;
      _length = s.length();
    }

    int nextToken()
    {
      _lexeme = _env.createUnicodeBuilder();

      if (! _inPhp) {
	_inPhp = true;

	if (parseHtml()) {
	  return T_INLINE_HTML;
	}
      }
      
      int ch = read();