comunic/3rdparty/luminous/languages/python.php

<?php
/*
 * Python scanner - includes Django
 *
 * TODO: Django does not respect {% comment  %} ... {% endcomment %}
 */
class LuminousPythonScanner extends LuminousScanner {

  public $django = false;
  
  public function init() {
    
    $this->remove_filter('comment-to-doc');
    
    // so it turns out this template isn't quite as readable as I hoped, but
    // it's a triple string, e.g:
    //  "{3} (?: [^"\\]+ | ""[^"\\]+ | "[^"\\]+  | \\.)* (?: "{3}|$)
    
    
    $triple_str_template = '%1$s{3} (?> [^%1$s\\\\]+ | %1$s%1$s[^%1$s\\\\]+ | %1$s[^%1$s\\\\]+ | \\\\. )* (?: %1$s{3}|$)';
    $str_template = '%1$s (?> [^%1$s\\\\]+ | \\\\. )* (?: %1$s|$)';
    $triple_dstr = sprintf($triple_str_template, '"');
    $triple_sstr = sprintf($triple_str_template, "'");    
    
    $this->add_pattern('IDENT', '/[a-zA-Z_](?>\w*)(?!["\'])/');
    // I *assume* that Django tags terminate these
    $this->add_pattern('COMMENT', sprintf('/\#.*%s/',
      $this->django? '(?=[%}]\})' : ''));
    
    
    // decorator
    $this->add_pattern('TYPE', '/@(\w+\.?)+/');
    
    // Python strings may be prefixed with r (raw) or u (unicode).
    // This affects how it handles backslashes, but I don't *think* it
    // affects escaping of quotes.... 
    $this->add_pattern('STRING', "/[RUru]?$triple_dstr/xs");
    $this->add_pattern('STRING', "/[RUru]?$triple_sstr/xs");
    $this->add_pattern('STRING', "/[RUru]?" . sprintf($str_template, '"') . '/sx');
    $this->add_pattern('STRING', "/[RUru]?" . sprintf($str_template, "'") . '/xs');
    
    // EPIC.
    $this->add_pattern('NUMERIC', '/
    #hex
    (?:0[xX](?>[0-9A-Fa-f]+)[lL]*)
    |
    # binary
    (?:0[bB][0-1]+)
    |
    #octal
    (?:0[oO0][0-7]+)
    |
    # regular number
    (?:
      (?>[0-9]+)
      (?: 
        # long identifier
        [lL]
        |
        # Or a fractional part, which may be imaginary
        (?:
          (?:\.?(?>[0-9]+)?        
            (?:(?:[eE][\+\-]?)?(?>[0-9]+))?
          )[jJ]?
        )
      )?
    )
    |
    (
      # or only after the point, float x = .1;
      \.(?>[0-9]+)(?:(?:[eE][\+\-]?)?(?>[0-9]+))?[jJ]?
    )
    /x');

    // %} and }} are django terminators
    if ($this->django) {
      $this->add_pattern('TERM', '/[%}]\}/');
    }

    // catch the colon separately so we can use $match === ':' in figuring out
    // where docstrs occur
    $this->add_pattern('OPERATOR', '/\+=|-=|\*=|\/=|>=|<=|!=|==|\*\*|[!%^*\-=+;<>\\\\(){}\[\],\\.:]/');

    if ($this->django) {
      // Django specific keywords
      // https://docs.djangoproject.com/en/1.3/ref/templates/builtins/
      $this->add_identifier_mapping('KEYWORD', array('autoescape',
        'endautoescape', 'cycle', 'filter', 'endfilter', 'include',
        'extends', 'firstof', 'empty', 'ifchanged', 'endifchanged',
        'ifequal', 'endifequal', 'ifnotequal', 'endifnotequal',
        'load',  'now',  'regroup', 'spaceless', 'endspaceless',
        'ssi', 'url',  'widthratio', 'endwith',
        'endfor', 'endif',
        'endwhile'));
    }

    $this->add_identifier_mapping('KEYWORD', array('assert', 'as', 'break',
    'class', 'continue', 'del', 'def', 'elif', 'else', 'except', 'exec',
    'finally', 'for', 'from', 'global', 'if', 'import', 'lambda', 
    'print', 'pass', 'raise', 'return', 'try', 'while', 'yield',
    'with',
    'and', 'not', 'in', 'is', 'or', 'print'));
    
    $this->add_identifier_mapping('FUNCTION', array('all', 'abs', 'any', 
    'basestring', 'bin', 'callable', 'chr', 'classmethod', 'cmp', 'compile',
    'dir', 'divmod', 'enumerate', 'eval', 'execfile', 'file', 'filter', 
    'format',
    'frozenset', 'getattr', 'globals', 'hasattr', 'hash', 'help', 'hex', 
    'id', 'input', 'isinstance', 'issubclass', 'iter', 'len', 'locals', 'map',
    'max', 'min', 'memoryview', 'next', 'object', 'oct', 'open', 'ord', 'pow',
     'property', 'range', 'raw_input', 'reduce', 'reload', 'repr', 'reversed',
     'round', 'setattr', 'slice', 'sorted', 'staticmethod', 'sum', 'super',
     'type', 'unichr', 'vars', 'xrange', 'zip', '__import__',

     'bytearray', 'complex', 'dict', 'float', 'int', 'list', 'long',
     'set', 'str', 'tuple', 'unicode', 'apply', 'buffer', 'coerce', 'intern'
   ));

    // http://docs.python.org/library/exceptions.html
    $this->add_identifier_mapping('TYPE', 
      array('BaseException', 'SystemExit', 
      'KeyboardInterrupt', 'GeneratorExit', 'Exception', 'StopIteration', 
      'StandardError', 'BufferError', 'ArithmeticError',
      'FloatingPointError', 'OverflowError', 'ZeroDivisionError', 
      'AssertionError',
      'AttributeError', 'EnvironmentError', 'IOError', 'OSError',
      'WindowsError(Windows)', 'VMSError(VMS)', 'EOFError', 'ImportError',
      'LookupError', 'IndexError', 'KeyError', 'MemoryError', 'NameError',
      'UnboundLocalError', 'ReferenceError', 'RuntimeError', 
      'NotImplementedError',
      'SyntaxError', 'IndentationError', 'TabError', 'SystemError', 'TypeError',
      'ValueError', 'UnicodeError', 'UnicodeDecodeError', 'UnicodeEncodeError',
      'UnicodeTranslateError', 'Warning', 'DeprecationWarning',
      'PendingDeprecationWarning', 'RuntimeWarning', 'SyntaxWarning', 
      'UserWarning',
      'FutureWarning', 'ImportWarning', 'UnicodeWarning', 'BytesWarning'));

    $this->add_identifier_mapping('VALUE', array('False', 'None', 'self', 
      'True'));
  }
  
  
  // mini-scanner to handle highlighting module names in import lines
  private function import_line() {
    $import = false;
    $from = false;
    while(!$this->eol()) {
      $c = $this->peek();
      $tok = null;
      $m = null;
      
      if ($c === '\\') $m = $this->get(2);
      elseif($this->scan('/[,\\.;\\*]+/')) $tok = 'OPERATOR';
      elseif($this->scan("/[ \t]+/")){}
      elseif(($m = $this->scan('/import\\b|from\\b/'))){
        if ($m === 'import') $import = true;
        elseif($m === 'from') $from = true;
        else assert(0);
        $tok = 'IDENT';
      }
      elseif($this->scan('/[_a-zA-Z]\w*/')) {
        assert($from || $import);
        // from module import *item*, or just import *item*
        if ($import) {
          $tok = 'USER_FUNCTION';
          $this->user_defs[$this->match()] = 'TYPE';
        }
        // from *module* ...[import item], the module is not imported
        else $tok = 'IDENT';
      }
      else break;
      $this->record(($m !== null)? $m : $this->match(), $tok);
    }
  }
  
  
  function main() {
    $definition = false;
    $doccstr = false;
    $expect = '';
    while (!$this->eos()) {
      $tok = null;
      $index = $this->pos();
      
      if (($rule = $this->next_match()) !== null) {
        $tok = $rule[0];
        if ($rule[1] > $index) {
          $this->record(substr($this->string(), $index, $rule[1] - $index), null);
        }
      } else {
        $this->record(substr($this->string(), $index), null);
        $this->terminate();
        break;
      }
      // Django terminator tag - break to superscanner
      if ($tok === 'TERM') {
        $this->unscan();
        break;
      }
      
      $m = $this->match();
      
      /* python doc strs are a pain because they're actually just strings. 
       * Also, I'm pretty sure a string in a non-interesting place just counts
       * as a no-op and is also used as a comment sometimes
       * So we've got something a bit complicated going on here: if we meet
       * a 'class' or a 'def' (function def) then we wait until the next ':' 
       * and say "we expect a doc-str now". If the next token is not a string,
       * we discard that state.
       * 
       * similarly, if we meet a string which isn't a doc-str, we look behind 
       * and expect to see an operator or open bracket, else it's a comment.
       * NOTE: we class ':' as a legal string preceding char because it's used
       * as dictionary key:value separators. This will fail on the case:
       * 
       * while 1: 
       *  "do something" 
       *  break
       * 
       * 
       * NOTE: note we're skipping whitespace.
       * NOTE: we disable the no-op detection for Django because the string
       * might be inside an output tag.
       * 
       */
      
      if ($definition && $doccstr) {
        if($tok === 'STRING')
          $tok = 'COMMENT';
      }
      
      elseif ($tok === 'STRING' && !$this->django) {
        $i = count($this->tokens);
        $tok = 'COMMENT';
        while ($i--) {
          $t = $this->tokens[$i][0];
          $s = $this->tokens[$i][1];
          if ($t === null || $t === 'COMMENT') continue;
          elseif ($t === 'OPERATOR' || $t === 'IDENT' || $t === 'NUMERIC') { 
            $tok = 'STRING';
          }
          break;
        }
        // finally, if we can look ahead to a binary operator, or so,
        // we concede it probably is a string
        if ($tok === 'COMMENT') {
          if ($this->check('/\s*(?: [+:&.,] | (?:and|or|is|not)\\b)/x'))
            $tok = 'STRING';
        }
      }
     
      // reset this; if it didn't catch above then it's not valid now.
      if ($definition && $doccstr) {
        $definition = false;
        $doccstr = false;
      }
      
      if ($tok === 'IDENT') {
        if ($m === 'import' || $m === 'from') {
          $this->unscan();
          $this->import_line();
          continue;
        }
        // these are definition keywords, the next token should be an 
        // identifier, which is a user-defined type or function
        if ($m === 'class' || $m === 'def') {
          $definition = true;
          $expect = 'user_def';
        }
        // this is caught on the next iteration
        elseif($expect === 'user_def') {
          $tok = 'USER_FUNCTION';
          $expect = false;
          $this->user_defs[$m] = 'FUNCTION';
        }
      }
      else { 
        // if this hasn't caught, it's not valid
        $expect = false; 
      }
      
      if ($definition && $m === ':') {
        $doccstr = true;
      }
      
      $this->record($m, $tok);
    }
  }


  public static function guess_language($src, $info) {
    if (strpos($info['shebang'], 'python') !== false) return 1.0;
    if ($info['shebang']) return 0.0;
    $p = 0.0;
    // let's look for some trademark pythonic constructs, although I 
    // have a feeling that recent versions of ECMA also impelment some
    // of this
    if (preg_match('/^\s*+ for \s++ \w++ \s++ in \s++ \w++ \s*+ :/xm', $src))
      $p += 0.05;
    if (preg_match('/True|False|None/', $src)) $p += 0.01;
    if (preg_match('/"{3}|\'{3}/', $src)) $p += 0.05;
    // class something(object)
    //
    if (preg_match('/^\s*+ class \s++ \w++ \s*+ \( \s*+ object \s*+ \)/xm', 
      $src)) $p += 0.1;
    // def __init__ (constructor)
    if (preg_match('/\\bdef \s++ __init__\\b/x', $src)) $p += 0.2;
    // method decorators
    if (preg_match("/^\s*+ @[\w\\.]++ .*+ [\n\r]++ \s*+ def\\b/mx", $src)) 
      $p += 0.1;
    // pmax = 0.41

    // common imports: import os|sys|re
    if (preg_match('/^import\s++(os|sys|re)\\b/m', $src))
      $p += 0.05;
    // from x import y
    if (preg_match('/^\s*+ from \s++ (?:\w++(?:\.\w++)*+) \s++ import \s/xm', 
      $src))
      $p += 0.10;


    return $p;
  }
}


class LuminousDjangoScanner extends LuminousScanner {
  // warning: some copying and pasting with the rails scanner here
  
  // HTML scanner has to be persistent.
  private $html_scanner;

  public function init() {
    $this->html_scanner = new LuminousHTMLScanner();
    $this->html_scanner->string($this->string());
    $this->html_scanner->embedded_server = true;
    $this->html_scanner->server_tags = '/\{[{%#]/';
    $this->html_scanner->init();
  }

  public function scan_html() {
    $this->html_scanner->pos($this->pos());
    $this->html_scanner->main();
    $this->record($this->html_scanner->tagged(), null, true);
    $this->pos($this->html_scanner->pos());
  }




  public function scan_python($short=false) {
    $python_scanner = new LuminousPythonScanner($this->string());
    $python_scanner->django = true;
    $python_scanner->init();
    $python_scanner->pos($this->pos());
    $python_scanner->main();
    $this->record($python_scanner->tagged(), $short? 'INTERPOLATION' : null, true);
    $this->pos($python_scanner->pos());
  }


  public function main() {
    while(!$this->eos()) {
      $p = $this->pos();
      // django's tags are {{ }} and {% %}
      // there's also a {#  #} comment tag but we can probably handle that here
      // more easily
      // same for {% comment %} ... {% endcomment %}
      if ($this->scan('/\{([{%])/')) {
        $match = $this->match();
        $m1 = $this->match_group(1);
        // {% comment %} ... {% endcomment %}
        if ($this->scan('/\s*comment\s*%\}/')) {
          $match .= $this->match();
          $end_pattern = '/\{%\s*endcomment\s*%\}/';
          if ($this->scan_until($end_pattern) !== null) {
            $match .= $this->match();
            $match .= $this->scan($end_pattern);
          }
          else {
            $match .= $this->rest();
            $this->terminate();
          }
          $this->record($match, 'COMMENT');
        }
        // {{ ... }} or {% ... %}
        else {
          $this->record($match, 'DELIMITER');
          $this->scan_python($m1 === '{');
          if ($this->scan('/[}%]\}/')) {
            $this->record($this->match(), 'DELIMITER');
          }
        }
      // {# ... #}
      } elseif($this->scan('/\{\# (?: [^\#]++ | \#(?! \} ) )*+ (?: \#\} | $)/x')) {
        $this->record($this->match(), 'COMMENT');
      }
      else {
        $this->scan_html();
      }
      assert($p < $this->pos());
    }
  }

  public static function guess_language($src, $info) {
    if (($html = LuminousHTMLScanner::guess_language($src, $info)) >= 0.2) {
      if (strpos($src, '{{') !== false || strpos($src, '{%') !== false)
        return $html + 0.01;
    }
    return 0.0;
  }
}