a bunch more stuff

themes, visual tests, etc.

Gemfile

@@ -7,3 +7,6 @@ gem 'sexp_processor', '~> 3.0'
 gem 'wrong', '~> 0.6.2'
 
 gem 'rake'
+
+# for visual tests
+gem 'sinatra'

config.ru

@@ -0,0 +1,6 @@
+require 'pathname'
+
+here = Pathname.new(__FILE__).dirname
+load here.join('spec/visual/app.rb')
+
+run VisualTestApp

lib/rouge.rb

@@ -19,3 +19,6 @@ def highlight(text, lexer_name, formatter)
 
 load load_dir.join('rouge/formatter.rb')
 load load_dir.join('rouge/formatters/html.rb')
+
+load load_dir.join('rouge/theme.rb')
+load load_dir.join('rouge/themes/thankful_eyes.rb')

lib/rouge/lexer.rb

@@ -91,7 +91,7 @@ class Rule
       attr_reader :re
       def initialize(re, callback, next_lexer)
         @orig_re = re
-        @re = Regexp.new %/\\A#{re.source}/
+        @re = Regexp.new %/\\A(?:#{re.source})/
         @callback = callback
         @next_lexer = next_lexer
       end
@@ -115,12 +115,33 @@ def consume(stream, &b)
       end
     end
 
-    def lexer(opts={}, &defn)
-      RegexLexer.new(options.merge(opts), &defn)
+    def initialize(parent=nil, opts={}, &defn)
+      if parent.is_a? Hash
+        opts = parent
+        parent = nil
+      end
+
+      @parent = parent
+      super(opts, &defn)
+    end
+
+    def lexer(name, opts={}, &defn)
+      @scope ||= {}
+      name = name.to_s
+
+      if block_given?
+        l = @scope[name] = RegexLexer.new(self, options.merge(opts), &defn)
+        l.instance_variable_set :@name, name
+        l
+      else
+        @scope[name] || @parent && @parent.lexer(name)
+      end
     end
 
     def mixin(lexer)
+      lexer = get_lexer(lexer)
       lexer.force_load!
+
       rules << lexer
     end
 
@@ -140,7 +161,7 @@ def rule(re, token=nil, next_lexer=nil, &callback)
         callback = proc { |match, &b| b.call token, match }
       end
 
-      rules << Rule.new(re, callback, next_lexer)
+      rules << Rule.new(re, callback, get_lexer(next_lexer))
     end
 
     def stream_tokens(stream, &b)
@@ -154,12 +175,13 @@ def stream_with_stack(stream, stack, &b)
       return true if stream.empty?
 
       until stream.empty?
-        debug { "parsing #{stream.inspect}" }
+        debug { "stack: #{stack.map(&:name).inspect}" }
+        debug { "parsing #{stream.slice(0..20).inspect}" }
         success = stack.last.step(stream, stack, &b)
 
         if !success
           debug { "    no match, yielding Error" }
-          b.call(Token['Error'], stream.slice!(0..1))
+          b.call(Token['Error'], stream.slice!(0..0))
         end
       end
     end
@@ -175,7 +197,7 @@ def step(stream, stack, &b)
   private
     def get_lexer(o)
       case o
-      when RegexLexer
+      when RegexLexer, :pop!
         o
       else
         lexer o
@@ -185,6 +207,8 @@ def get_lexer(o)
     def run_rule(rule, stream, stack, &b)
       case rule
       when String, RegexLexer
+        lexer = get_lexer(rule)
+        debug { "  entering mixin #{lexer.name}" }
         get_lexer(rule).step(stream, stack, &b)
       when Rule
         debug { "  trying #{rule.inspect}" }
@@ -196,13 +220,17 @@ def run_rule(rule, stream, stack, &b)
               tok = Token[tok]
             end
 
+            debug { "    yielding #{tok.name.inspect}, #{res.inspect}" }
             b.call(tok, res)
           end
 
           if rule.next_lexer == :pop!
+            debug { "    popping stack" }
             stack.pop
           elsif rule.next_lexer
-            stack.push get_lexer(rule.next_lexer)
+            lexer = get_lexer(rule.next_lexer)
+            debug { "    entering #{lexer.name}" }
+            stack.push lexer
           end
         end
       end

lib/rouge/lexers/shell.rb

@@ -1,12 +1,11 @@
 module Rouge
   module Lexers
     ShellLexer = RegexLexer.create do
+      option :debug, true
+
       name 'shell'
       aliases 'bash', 'zsh', 'ksh', 'sh'
 
-      # because ruby closures are weird
-      root = nil
-
       KEYWORDS = %w(
         if fi else while do done for then return function case
         select continue until esac elif
@@ -21,8 +20,7 @@ module Lexers
         ulimit umask unalias unset wait
       ).join('|')
 
-      basic = lexer do
-
+      lexer :basic do
         rule /#.*\n/, 'Comment'
 
         rule /\b(#{KEYWORDS})\s*\b/, 'Keyword'
@@ -36,15 +34,23 @@ module Lexers
 
         rule /[\[\]{}()=]/, 'Operator'
         rule /&&|\|\|/, 'Operator'
+        # rule /\|\|/, 'Operator'
 
         rule /<<</, 'Operator' # here-string
         rule /<<-?\s*(\'?)\\?(\w+)[\w\W]+?\2/, 'Literal.String'
       end
 
-      data = lexer do
+      lexer :double_quotes do
+        rule /"/, 'String.Double', :pop!
+        rule /\\./, 'String.Escape'
+        mixin :interp
+        rule /[^"`\\$]+/, 'String.Double'
+      end
+
+      lexer :data do
         # TODO: this should be its own sublexer so we can capture
         # interpolation and such
-        rule /$?"(\\"|[^"])*"/, 'String.Double'
+        rule /$?"/, 'String.Double', :double_quotes
 
         # single quotes are much easier than double quotes - we can
         # literally just scan until the next single quote.
@@ -57,45 +63,49 @@ module Lexers
         rule /\s+/, 'Text'
         rule /[^=\s\[\]{}()$"\'`\\<]+/, 'Text'
         rule /\d+(?= |\Z)/, 'Number'
-        rule /\$#?(\w+|.)/, 'Name.Variable'
         rule /</, 'Text'
+        mixin :interp
       end
 
-      curly = lexer do
+      lexer :curly do
         rule /}/, 'Keyword', :pop!
         rule /:-/, 'Keyword'
         rule /[a-zA-Z0-9_]+/, 'Name.Variable'
         rule /[^}:"'`$]+/, 'Punctuation'
-        mixin root
+        mixin :root
       end
 
-      paren = lexer do
+      lexer :paren do
         rule /\)/, 'Keyword', :pop!
-        mixin root
+        mixin :root
       end
 
-      math = lexer do
+      lexer :math do
         rule /\)\)/, 'Keyword', :pop!
         rule %r([-+*/%^|&]|\*\*|\|\|), 'Operator'
         rule /\d+/, 'Number'
-        mixin root
+        mixin :root
       end
 
-      backticks = lexer do
+      lexer :backticks do
         rule /`/, 'String.Backtick', :pop!
-        mixin root
+        mixin :root
+      end
+
+      lexer :interp do
+        rule /\$\(\(/, 'Keyword', :math
+        rule /\$\(/, 'Keyword', :paren
+        rule /\${#?/, 'Keyword', :curly
+        rule /`/, 'String.Backtick', :backticks
+        rule /\$#?(\w+|.)/, 'Name.Variable'
       end
 
-      root = lexer do
-        mixin basic
-        rule /\$\(\(/, 'Keyword', math
-        rule /\$\(/, 'Keyword', paren
-        rule /\${#?/, 'Keyword', curly
-        rule /`/, 'String.Backtick', backticks
-        mixin data
+      lexer :root do
+        mixin :basic
+        mixin :data
       end
 
-      mixin root
+      mixin :root
     end
   end
 end

lib/rouge/theme.rb

@@ -0,0 +1,107 @@
+module Rouge
+  class Theme
+    class << self
+      def styles
+        @styles ||= {}
+      end
+
+      def main_style
+        @main_style ||= {}
+      end
+
+      def style(*tokens)
+        opts = {}
+        opts = tokens.pop if tokens.last.is_a? Hash
+
+        if tokens.empty?
+          @main_style = opts
+        end
+
+        tokens.each do |tok|
+          styles[tok.to_s] = opts
+        end
+      end
+
+      def name(n=nil)
+        return @name if n.nil?
+
+        @name = n.to_s
+        registry[@name] = self
+      end
+
+      def find(n)
+        registry[n.to_s]
+      end
+
+      def registry
+        @registry ||= {}
+      end
+    end
+  end
+
+  class CSSTheme < Theme
+    def initialize(opts={})
+      @opts = opts
+    end
+
+    def render
+      out = []
+      stream { |line| out << line }
+      out.join("\n")
+    end
+
+    def stream(&b)
+      self.class.styles.each do |tokname, style|
+        stream_single(Token[tokname], style, &b)
+      end
+
+      render_stanza('.highlight', self.class.main_style, &b)
+    end
+
+  private
+    def stream_single(tok, style, &b)
+      render_stanza(css_selector(tok), style, &b)
+    end
+
+    def render_stanza(selector, style, &b)
+      return if style.empty?
+
+      yield "#{selector} {"
+      yield "  color: #{style[:fg]};" if style[:fg]
+      yield "  background-color: #{style[:bg]};" if style[:bg]
+      yield "  font-weight: bold;" if style[:bold]
+      yield "  font-style: italic;" if style[:italic]
+      yield "  text-decoration: underline;" if style[:underline]
+
+      (style[:rules] || []).each do |rule|
+        yield "  #{rule};"
+      end
+
+      yield "}"
+    end
+
+    def css_selector(token)
+      tokens = [token]
+      parent = token.parent
+
+      inflate_token(token).map do |tok|
+        base = ".highlight"
+        base << " .#{tok.shortname}" unless tok.shortname.empty?
+
+        base
+      end.join(', ')
+    end
+
+    def inflate_token(tok)
+      Enumerator.new do |out|
+        out << tok
+        tok.sub_tokens.each_value do |st|
+          next if self.class.styles.include? st.name
+
+          out << st
+          inflate_token(st).each { |t| out << t }
+        end
+      end
+    end
+  end
+end