a bunch of stuff

.gitignore

@@ -0,0 +1,5 @@
+# it's a gem, ignore the lockfile
+Gemfile.lock
+
+# build artifacts
+*.gem

Gemfile

@@ -0,0 +1,9 @@
+source 'http://rubygems.org'
+
+gemspec
+
+gem 'minitest'
+gem 'sexp_processor', '~> 3.0'
+gem 'wrong', '~> 0.6.2'
+
+gem 'rake'

Guardfile

@@ -0,0 +1,3 @@
+guard :shell do
+  watch(/\.rb$/) { `rake` }
+end

Rakefile

@@ -0,0 +1,15 @@
+require 'rake/clean'
+
+task :spec do
+  spec_files = FileList.new('./spec/**/*_spec.rb')
+  switch_spec_files = spec_files.map { |x| "-r#{x}" }.join(' ')
+  sh "ruby -I./lib -r ./spec/spec_helper #{switch_spec_files} -e Minitest::Unit.autorun"
+end
+
+CLEAN.include('*.gem')
+task :build => [:clean, :spec] do
+  puts
+  sh "gem build cacher.gemspec"
+end
+
+task :default => :spec

lib/rouge.rb

@@ -0,0 +1,10 @@
+# stdlib
+require 'pathname'
+
+module Rouge
+end
+
+load_dir = Pathname.new(__FILE__).dirname
+load load_dir.join('rouge/token.rb')
+load load_dir.join('rouge/lexer.rb')
+load load_dir.join('rouge/lexers/shell.rb')

lib/rouge/lexer.rb

@@ -0,0 +1,192 @@
+module Rouge
+  class Lexer
+    def initialize(&b)
+      instance_eval(&b)
+    end
+
+    def default_options
+      {}
+    end
+
+    def options(o={})
+      (@options ||= default_options).merge!(o)
+    end
+
+    def option(k, v=:absent)
+      if v == :absent
+        options[k.to_s]
+      else
+        options({ k.to_s => v })
+      end
+    end
+
+    def debug(&b)
+      puts(b.call) if option :debug
+    end
+
+    def get_tokens(stream)
+      Enumerator.new do |out|
+        stream_tokens(stream) do |token, value|
+          out << [token, value]
+        end
+      end.to_a
+    end
+
+    def stream_tokens(stream)
+      raise 'abstract'
+    end
+  end
+
+  class RegexLexer < Lexer
+    class Rule
+      attr_reader :callback
+      attr_reader :next_lexer
+      attr_reader :re
+      def initialize(re, callback, next_lexer)
+        @re = Regexp.new %/\\A#{re.source}/
+        @callback = callback
+        @next_lexer = next_lexer
+      end
+
+      def consume(stream, &b)
+        # TODO: I'm sure there is a much faster way of doing this.
+        # also, encapsulate the stream in its own class.
+        match = stream.match(@re)
+
+        if match
+          stream.slice!(0...$&.size)
+          yield match
+          return true
+        end
+
+        false
+      end
+    end
+
+    def initialize(parent=nil, &defn)
+      @parent = parent
+      super(&defn)
+    end
+
+    def lexer(name, &defn)
+      name = name.to_s
+
+      if block_given?
+        scope[name] = RegexLexer.new(self, &defn)
+      else
+        scope[name] || parent && parent.lexer(name)
+      end
+    end
+
+    def scope
+      @scope ||= {}
+    end
+
+    def mixin(lexer)
+      rules << lexer
+    end
+
+    def rules
+      @rules ||= []
+    end
+
+    def rule(re, token=nil, next_lexer=nil, &callback)
+      if block_given?
+        next_lexer = token
+      else
+        if token.is_a? String
+          token = Token[token]
+        end
+
+        callback = proc { |match, &b| b.call token, match }
+      end
+
+      rules << Rule.new(re, callback, next_lexer)
+    end
+
+    def step(stream, stack, &b)
+      debug { "parsing #{stream.inspect}" }
+      if stack.empty?
+        raise 'empty stack!'
+      end
+
+      lexer = stack.last
+
+      lexer.rules.each do |rule|
+        rule.consume(stream) do |match|
+
+          return true
+        end
+      end
+
+      return false
+    end
+
+    def stream_tokens(stream, &b)
+      stream = stream.dup
+      stack = [self]
+
+      stream_with_stack(stream.dup, [self], &b)
+    end
+
+    def stream_with_stack(stream, stack, &b)
+      return true if stream.empty?
+
+      until stream.empty?
+        debug { "parsing #{stream.inspect}" }
+        success = stack.last.step(stream, stack, &b)
+
+        if !success
+          debug { "    failed parse, returning text" }
+          b.call(Token['Text'], stream)
+          return false
+        end
+      end
+    end
+
+    def step(stream, stack, &b)
+      rules.each do |rule|
+        debug { "  trying #{rule.re.inspect}" }
+        return true if run_rule(rule, stream, stack, &b)
+      end
+
+      false
+    end
+
+  private
+    def get_lexer(o)
+      case o
+      when RegexLexer
+        o
+      else
+        lexer o
+      end
+    end
+
+    def run_rule(rule, stream, stack, &b)
+      case rule
+      when String, RegexLexer
+        get_lexer(rule).step(stream, stack, &b)
+      when Rule
+        rule.consume(stream) do |match|
+          debug { "    got #{match[0].inspect}" }
+
+          rule.callback.call(*match) do |tok, res|
+            if tok.is_a? String
+              tok = Token[tok]
+            end
+
+            b.call(tok, res)
+          end
+
+          if rule.next_lexer == :pop!
+            stack.pop
+          elsif rule.next_lexer
+            stack.push get_lexer(rule.next_lexer)
+          end
+        end
+      end
+    end
+
+  end
+end

lib/rouge/lexers/shell.rb

@@ -0,0 +1,84 @@
+module Rouge
+  module Lexers
+    ShellLexer = RegexLexer.new do
+      lexer :basic do
+        rule /
+          \b(if|fi|else|while|do|done|for|then|return|function|case
+            |select|continue|until|esac|elif
+          )\s*\b
+        /x, 'Keyword'
+
+        rule /
+          \b(alias|bg|bind|break|builtin|caller|cd|command|compgen
+            |complete|declare|dirs|disown|echo|enable|eval|exec|exit
+            |export|false|fc|fg|getopts|hash|help|history|jobs|kill|let
+            |local|logout|popd|printf|pushd|pwd|read|readonly|set|shift
+            |shopt|source|suspend|test|time|times|trap|true|type|typeset
+            |ulimit|umask|unalias|unset|wait
+          )\s*\b(?!\.)
+        /x, 'Name.Builtin'
+
+        rule /#.*\n/, 'Comment'
+
+        rule /(\b\w+)(\s*)(=)/ do |_, var, ws, eq, &out|
+          out.call 'Name.Variable', var
+          out.call 'Text', ws
+          out.call 'Operator', eq
+        end
+
+        rule /[\[\]{}()=]/, 'Operator'
+        rule /&&|\|\|/, 'Operator'
+
+        rule /<<</, 'Operator' # here-string
+        rule /<<-?\s*(\'?)\\?(\w+)[\w\W]+?\2/, 'Literal.String'
+      end
+
+      lexer :data do
+        rule /(?s)\$?"(\\\\|\\[0-7]+|\\.|[^"\\])*"/, 'String.Double'
+        rule /(?s)\$?'(\\\\|\\[0-7]+|\\.|[^'\\])*'/, 'String.Single'
+        rule /;/, 'Text'
+        rule /\s+/, 'Text'
+        rule /[^=\s\[\]{}()$"\'`\\<]+/, 'Text'
+        rule /\d+(?= |\Z)/, 'Number'
+        rule /\$#?(\w+|.)/, 'Name.Variable'
+        rule /</, 'Text'
+      end
+
+      lexer :curly do
+        rule /}/, 'Keyword', :pop!
+        rule /:-/, 'Keyword'
+        rule /[a-zA-Z0-9_]+/, 'Name.Variable'
+        rule /[^}:"'`$]+/, 'Punctuation'
+        mixin :root
+      end
+
+      lexer :paren do
+        rule /\)/, 'Keyword', :pop!
+        mixin :root
+      end
+
+      lexer :math do
+        rule /\)\)/, 'Keyword', :pop!
+        rule /[-+*/%^|&]|\*\*|\|\|/, 'Operator'
+        rule /\d+/, 'Number'
+        mixin :root
+      end
+
+      lexer :root do
+        mixin :basic
+        rule /\$\(\(/, 'Keyword', :math
+        rule /\$\(/, 'Keyword', :paren
+        rule /\${#?/, 'Keyword', :curly
+        rule /`/, 'String.Backtick', :backticks
+        mixin :data
+      end
+
+      lexer :backticks do
+        rule /`/, 'String.Backtick', :pop!
+        mixin :root
+      end
+
+      mixin :root
+    end
+  end
+end