From be4ead52d7a85893449e6262e35597eb5e78d4b4 Mon Sep 17 00:00:00 2001 From: Jay Adkisson Date: Sat, 1 Sep 2012 00:26:17 +0200 Subject: [PATCH] a bunch more stuff themes, visual tests, etc. --- Gemfile | 3 + config.ru | 6 ++ lib/rouge.rb | 3 + lib/rouge/lexer.rb | 44 +++++++++--- lib/rouge/lexers/shell.rb | 58 ++++++++------- lib/rouge/theme.rb | 107 +++++++++++++++++++++++++++ lib/rouge/themes/thankful_eyes.rb | 65 +++++++++++++++++ spec/lexer_spec.rb | 8 ++- spec/theme_spec.rb | 3 + spec/visual/app.rb | 32 +++++++++ spec/visual/shell/sample | 115 ++++++++++++++++++++++++++++++ spec/visual/templates/lexer.erb | 24 +++++++ 12 files changed, 433 insertions(+), 35 deletions(-) create mode 100644 config.ru create mode 100644 lib/rouge/theme.rb create mode 100644 lib/rouge/themes/thankful_eyes.rb create mode 100644 spec/theme_spec.rb create mode 100644 spec/visual/app.rb create mode 100644 spec/visual/shell/sample create mode 100644 spec/visual/templates/lexer.erb diff --git a/Gemfile b/Gemfile index d62736d70f..c2e0da0e7e 100644 --- a/Gemfile +++ b/Gemfile @@ -7,3 +7,6 @@ gem 'sexp_processor', '~> 3.0' gem 'wrong', '~> 0.6.2' gem 'rake' + +# for visual tests +gem 'sinatra' diff --git a/config.ru b/config.ru new file mode 100644 index 0000000000..7754f04c99 --- /dev/null +++ b/config.ru @@ -0,0 +1,6 @@ +require 'pathname' + +here = Pathname.new(__FILE__).dirname +load here.join('spec/visual/app.rb') + +run VisualTestApp diff --git a/lib/rouge.rb b/lib/rouge.rb index 9fb8849853..f0cfabaeca 100644 --- a/lib/rouge.rb +++ b/lib/rouge.rb @@ -19,3 +19,6 @@ def highlight(text, lexer_name, formatter) load load_dir.join('rouge/formatter.rb') load load_dir.join('rouge/formatters/html.rb') + +load load_dir.join('rouge/theme.rb') +load load_dir.join('rouge/themes/thankful_eyes.rb') diff --git a/lib/rouge/lexer.rb b/lib/rouge/lexer.rb index f3485c1505..563ae63b47 100644 --- a/lib/rouge/lexer.rb +++ b/lib/rouge/lexer.rb @@ -91,7 +91,7 @@ class Rule attr_reader :re def initialize(re, callback, next_lexer) @orig_re = re - @re = Regexp.new %/\\A#{re.source}/ + @re = Regexp.new %/\\A(?:#{re.source})/ @callback = callback @next_lexer = next_lexer end @@ -115,12 +115,33 @@ def consume(stream, &b) end end - def lexer(opts={}, &defn) - RegexLexer.new(options.merge(opts), &defn) + def initialize(parent=nil, opts={}, &defn) + if parent.is_a? Hash + opts = parent + parent = nil + end + + @parent = parent + super(opts, &defn) + end + + def lexer(name, opts={}, &defn) + @scope ||= {} + name = name.to_s + + if block_given? + l = @scope[name] = RegexLexer.new(self, options.merge(opts), &defn) + l.instance_variable_set :@name, name + l + else + @scope[name] || @parent && @parent.lexer(name) + end end def mixin(lexer) + lexer = get_lexer(lexer) lexer.force_load! + rules << lexer end @@ -140,7 +161,7 @@ def rule(re, token=nil, next_lexer=nil, &callback) callback = proc { |match, &b| b.call token, match } end - rules << Rule.new(re, callback, next_lexer) + rules << Rule.new(re, callback, get_lexer(next_lexer)) end def stream_tokens(stream, &b) @@ -154,12 +175,13 @@ def stream_with_stack(stream, stack, &b) return true if stream.empty? until stream.empty? - debug { "parsing #{stream.inspect}" } + debug { "stack: #{stack.map(&:name).inspect}" } + debug { "parsing #{stream.slice(0..20).inspect}" } success = stack.last.step(stream, stack, &b) if !success debug { " no match, yielding Error" } - b.call(Token['Error'], stream.slice!(0..1)) + b.call(Token['Error'], stream.slice!(0..0)) end end end @@ -175,7 +197,7 @@ def step(stream, stack, &b) private def get_lexer(o) case o - when RegexLexer + when RegexLexer, :pop! o else lexer o @@ -185,6 +207,8 @@ def get_lexer(o) def run_rule(rule, stream, stack, &b) case rule when String, RegexLexer + lexer = get_lexer(rule) + debug { " entering mixin #{lexer.name}" } get_lexer(rule).step(stream, stack, &b) when Rule debug { " trying #{rule.inspect}" } @@ -196,13 +220,17 @@ def run_rule(rule, stream, stack, &b) tok = Token[tok] end + debug { " yielding #{tok.name.inspect}, #{res.inspect}" } b.call(tok, res) end if rule.next_lexer == :pop! + debug { " popping stack" } stack.pop elsif rule.next_lexer - stack.push get_lexer(rule.next_lexer) + lexer = get_lexer(rule.next_lexer) + debug { " entering #{lexer.name}" } + stack.push lexer end end end diff --git a/lib/rouge/lexers/shell.rb b/lib/rouge/lexers/shell.rb index 806ddba17a..feba738602 100644 --- a/lib/rouge/lexers/shell.rb +++ b/lib/rouge/lexers/shell.rb @@ -1,12 +1,11 @@ module Rouge module Lexers ShellLexer = RegexLexer.create do + option :debug, true + name 'shell' aliases 'bash', 'zsh', 'ksh', 'sh' - # because ruby closures are weird - root = nil - KEYWORDS = %w( if fi else while do done for then return function case select continue until esac elif @@ -21,8 +20,7 @@ module Lexers ulimit umask unalias unset wait ).join('|') - basic = lexer do - + lexer :basic do rule /#.*\n/, 'Comment' rule /\b(#{KEYWORDS})\s*\b/, 'Keyword' @@ -36,15 +34,23 @@ module Lexers rule /[\[\]{}()=]/, 'Operator' rule /&&|\|\|/, 'Operator' + # rule /\|\|/, 'Operator' rule /<< unicorn, :bg => krasna + + style 'Comment', :fg => cool_as_ice + style 'Error', + 'Generic.Error', :fg => aluminum1, :bg => scarletred2 + style 'Keyword', :fg => sandy, :bold => true + style 'Operator', :fg => backlit, :bold => true + style 'Comment.Preproc', + 'Comment.Multiline', + 'Comment.Single', + 'Comment.Special', :fg => cool_as_ice, :italic => true + style 'Generic.Deleted', :fg => scarletred2 + style 'Generic.Emph', :italic => true + style 'Generic.Subheading', :fg => '#800080', :bold => true + style 'Generic.Traceback', :fg => '#0040D0' + style 'Keyword.Constant', :fg => pink_merengue, :bold => true + style 'Keyword.Namespace', + 'Keyword.Pseudo', + 'Keyword.Reserved', :fg => schrill, :bold => true + style 'Keyword.Type', + 'Name.Constant', + 'Name.Class', + 'Name.Decorator', + 'Name.Namespace', + 'Name.Builtin.Pseudo', + 'Name.Exception', :fg => go_get_it, :bold => true + style 'Literal.Number', :fg => pink_merengue, :bold => true + style 'Literal.String', :fg => dune, :bold => true + style 'Literal.String.Escape', + 'Literal.String.Char', + 'Literal.String.Interpol', + 'Literal.String.Other', + 'Literal.String.Symbol', :fg => backlit, :bold => true + style 'Name.Attribute', :fg => '#7D9029' + style 'Name.Builtin', :fg => sandy + style 'Name.Entity', :fg => '#999999', :bold => true + style 'Name.Label', :fg => '#A0A000' + style 'Name.Tag', :fg => '#008000', :bold => true + style 'Text.Whitespace', :fg => '#BBBBBB' + style 'Name.Variable', + 'Name.Function', :fg => chilly + + end + end +end diff --git a/spec/lexer_spec.rb b/spec/lexer_spec.rb index f59085d20f..4525688bec 100644 --- a/spec/lexer_spec.rb +++ b/spec/lexer_spec.rb @@ -15,12 +15,12 @@ it 'makes sublexers' do a_lexer = Rouge::RegexLexer.create do - brace = lexer do + lexer :brace do rule /b/, 'B' rule /}/, 'Brace', :pop! end - rule /{/, 'Brace', brace + rule /{/, 'Brace', :brace rule /a/, 'A' end @@ -28,9 +28,11 @@ assert { result.size == 5 } # failed parses + + t = Rouge::Token assert { a_lexer.get_tokens('{a}') == - [[Rouge::Token['Brace'], '{'], [Rouge::Token['Error'], 'a}']] + [[t['Brace'], '{'], [t['Error'], 'a'], [t['Brace'], '}']] } assert { a_lexer.get_tokens('b') == [[Rouge::Token['Error'], 'b']] } diff --git a/spec/theme_spec.rb b/spec/theme_spec.rb new file mode 100644 index 0000000000..21d7a1019f --- /dev/null +++ b/spec/theme_spec.rb @@ -0,0 +1,3 @@ +describe Rouge::Theme do + # TODO +end diff --git a/spec/visual/app.rb b/spec/visual/app.rb new file mode 100644 index 0000000000..8ce8c80d2d --- /dev/null +++ b/spec/visual/app.rb @@ -0,0 +1,32 @@ +require 'rubygems' +require 'bundler' +Bundler.require +require 'rouge' + +# stdlib +require 'pathname' + +class VisualTestApp < Sinatra::Application + BASE = Pathname.new(__FILE__).dirname + + configure do + set :root, BASE + set :views, BASE.join('templates') + end + + get '/:lexer' do |lexer_name| + dir = BASE.join(lexer_name) + @lexer = Rouge::Lexer.find(lexer_name) + halt 404 unless @lexer + @sample = File.read(dir.join('sample')) + formatter = Rouge::Formatters::HTML.new + @highlighted = Rouge.highlight(@sample, lexer_name, formatter) + @theme = Rouge::Themes::ThankfulEyes.new + + erb :lexer + end + + get '/' do + 'TODO' + end +end diff --git a/spec/visual/shell/sample b/spec/visual/shell/sample new file mode 100644 index 0000000000..9fd50a4f3e --- /dev/null +++ b/spec/visual/shell/sample @@ -0,0 +1,115 @@ +#!/bin/sh -e +# upstart-job +# +# Symlink target for initscripts that have been converted to Upstart. + +set -e + +INITSCRIPT="$(basename "$0")" +JOB="${INITSCRIPT%.sh}" + +if [ "$JOB" = "upstart-job" ]; then + if [ -z "$1" ]; then + echo "Usage: upstart-job JOB COMMAND" 1>&2 + exit 1 + fi + + JOB="$1" + INITSCRIPT="$1" + shift +else + if [ -z "$1" ]; then + echo "Usage: $0 COMMAND" 1>&2 + exit 1 + fi +fi + +COMMAND="$1" +shift + + +if [ -z "$DPKG_MAINTSCRIPT_PACKAGE" ]; then + ECHO=echo +else + ECHO=: +fi + +$ECHO "Rather than invoking init scripts through /etc/init.d, use the service(8)" +$ECHO "utility, e.g. service $INITSCRIPT $COMMAND" + +# Only check if jobs are disabled if the currently _running_ version of +# Upstart (which may be older than the latest _installed_ version) +# supports such a query. +# +# This check is necessary to handle the scenario when upgrading from a +# release without the 'show-config' command (introduced in +# Upstart for Ubuntu version 0.9.7) since without this check, all +# installed packages with associated Upstart jobs would be considered +# disabled. +# +# Once Upstart can maintain state on re-exec, this change can be +# dropped (since the currently running version of Upstart will always +# match the latest installed version). + +UPSTART_VERSION_RUNNING=$(initctl version|awk '{print $3}'|tr -d ')') + +if dpkg --compare-versions "$UPSTART_VERSION_RUNNING" ge 0.9.7 +then + initctl show-config -e "$JOB"|grep -q '^ start on' || DISABLED=1 +fi + +case $COMMAND in +status) + $ECHO + $ECHO "Since the script you are attempting to invoke has been converted to an" + $ECHO "Upstart job, you may also use the $COMMAND(8) utility, e.g. $COMMAND $JOB" + $COMMAND "$JOB" + ;; +start|stop) + $ECHO + $ECHO "Since the script you are attempting to invoke has been converted to an" + $ECHO "Upstart job, you may also use the $COMMAND(8) utility, e.g. $COMMAND $JOB" + if status "$JOB" 2>/dev/null | grep -q ' start/'; then + RUNNING=1 + fi + if [ -z "$RUNNING" ] && [ "$COMMAND" = "stop" ]; then + exit 0 + elif [ -n "$RUNNING" ] && [ "$COMMAND" = "start" ]; then + exit 0 + elif [ -n "$DISABLED" ] && [ "$COMMAND" = "start" ]; then + exit 0 + fi + $COMMAND "$JOB" + ;; +restart) + $ECHO + $ECHO "Since the script you are attempting to invoke has been converted to an" + $ECHO "Upstart job, you may also use the stop(8) and then start(8) utilities," + $ECHO "e.g. stop $JOB ; start $JOB. The restart(8) utility is also available." + if status "$JOB" 2>/dev/null | grep -q ' start/'; then + RUNNING=1 + fi + if [ -n "$RUNNING" ] ; then + stop "$JOB" + fi + # If the job is disabled and is not currently running, the job is + # not restarted. However, if the job is disabled but has been forced into the + # running state, we *do* stop and restart it since this is expected behaviour + # for the admin who forced the start. + if [ -n "$DISABLED" ] && [ -z "$RUNNING" ]; then + exit 0 + fi + start "$JOB" + ;; +reload|force-reload) + $ECHO + $ECHO "Since the script you are attempting to invoke has been converted to an" + $ECHO "Upstart job, you may also use the reload(8) utility, e.g. reload $JOB" + reload "$JOB" + ;; +*) + $ECHO + $ECHO "The script you are attempting to invoke has been converted to an Upstart" 1>&2 + $ECHO "job, but $COMMAND is not supported for Upstart jobs." 1>&2 + exit 1 +esac diff --git a/spec/visual/templates/lexer.erb b/spec/visual/templates/lexer.erb new file mode 100644 index 0000000000..7c71bb1740 --- /dev/null +++ b/spec/visual/templates/lexer.erb @@ -0,0 +1,24 @@ + + + + <%= @lexer.name %> | Visual Test + + + + + +

Highlighted

+ <%= @highlighted %> + +

Raw

+ 
<%= @sample %>
+ +