Proposal for escaping within lexed content

lib/rouge/demos/escape

@@ -0,0 +1,3 @@
+If Formatter.enable_escape! is called, this allows escaping into html
+or the parent format with a special delimiter. For example:
+<!<span style="text-decoration: underline">!>underlined text!<!</span>!>

lib/rouge/formatter.rb

@@ -21,6 +21,26 @@ def self.find(tag)
       REGISTRY[tag]
     end
 
+    def self.with_escape
+      Thread.current[:'rouge/with-escape'] = true
+      yield
+    ensure
+      Thread.current[:'rouge/with-escape'] = false
+    end
+
+    def self.escape_enabled?
+      !!(@escape_enabled || !!Thread.current[:'rouge/with-escape'])
+    end
+
+    def self.enable_escape!
+      @escape_enabled = true
+    end
+
+    def self.disable_escape!
+      @escape_enabled = false
+      Thread.current[:'rouge/with-escape'] = false
+    end
+
     # Format a token stream.  Delegates to {#format}.
     def self.format(tokens, *a, &b)
       new(*a).format(tokens, &b)
@@ -30,8 +50,24 @@ def initialize(opts={})
       # pass
     end
 
+    def escape?(tok)
+      tok == Token::Tokens::Escape
+    end
+
+    def filter_escapes(tokens)
+      tokens.each do |t, v|
+        if t == Token::Tokens::Escape
+          yield Token::Tokens::Error, v
+        else
+          yield t, v
+        end
+      end
+    end
+
     # Format a token stream.
     def format(tokens, &b)
+      tokens = enum_for(:filter_escapes, tokens) unless Formatter.escape_enabled?
+
       return stream(tokens, &b) if block_given?
 
       out = String.new('')

lib/rouge/formatters/html.rb

@@ -13,6 +13,8 @@ def stream(tokens, &b)
       end
 
       def span(tok, val)
+        return val if escape?(tok)
+
         safe_span(tok, val.gsub(/[&<>]/, TABLE_FOR_ESCAPE_HTML))
       end
 

lib/rouge/lexers/escape.rb

@@ -0,0 +1,53 @@
+module Rouge
+  module Lexers
+    class Escape < Lexer
+      tag 'escape'
+      aliases 'esc'
+
+      desc 'A generic lexer for including escaped content - see Formatter.enable_escape!'
+
+      option :start, 'the beginning of the escaped section, default "<!"'
+      option :end, 'the end of the escaped section, e.g. "!>"'
+      option :lang, 'the language to lex in unescaped sections'
+
+      def initialize(*)
+        super
+        @start = string_option(:start) { '<!' }
+        @end = string_option(:end) { '!>' }
+        @lang = lexer_option(:lang) { PlainText.new }
+      end
+
+      def to_start_regex
+        @to_start_regex ||= /(.*?)(#{Regexp.escape(@start)})/m
+      end
+
+      def to_end_regex
+        @to_end_regex ||= /(.*?)(#{Regexp.escape(@end)})/m
+      end
+
+      def stream_tokens(str, &b)
+        stream = StringScanner.new(str)
+        p :to_start_regex => to_start_regex
+        p :to_end_regex => to_end_regex
+
+        loop do
+          if stream.scan(to_start_regex)
+            puts "pre-escape: #{stream[1].inspect}" if @debug
+            @lang.continue_lex(stream[1], &b)
+          else
+            # no more start delimiters, scan til the end
+            @lang.continue_lex(stream.rest, &b)
+            return
+          end
+
+          if stream.scan(to_end_regex)
+            yield Token::Tokens::Escape, stream[1]
+          else
+            yield Token::Tokens::Escape, stream.rest
+            return
+          end
+        end
+      end
+    end
+  end
+end

spec/formatter_spec.rb

@@ -8,4 +8,28 @@
   it 'is found by Rouge.highlight' do
     assert { Rouge.highlight('puts "Hello"', 'ruby', 'terminal256') }
   end
+
+  it 'does not escape by default' do
+    assert { not Rouge::Formatter.escape_enabled? }
+  end
+
+  it 'escapes in all threads with #enable_escape!' do
+    begin
+      Rouge::Formatter.enable_escape!
+      assert { Rouge::Formatter.escape_enabled? }
+    ensure
+      Rouge::Formatter.disable_escape!
+    end
+  end
+
+  it 'escapes locally with #with_escape' do
+    Rouge::Formatter.with_escape do
+      assert { Rouge::Formatter.escape_enabled? }
+      assert { not Thread.new { Rouge::Formatter.escape_enabled? }.value }
+      Rouge::Formatter.disable_escape!
+      assert { not Rouge::Formatter.escape_enabled? }
+    end
+
+    assert { not Rouge::Formatter.escape_enabled? }
+  end
 end

spec/lexers_spec.rb

@@ -23,6 +23,9 @@
           out_buf << value
         end
 
+        # Escape is allowed to drop characters from its input
+        next if lexer_class == Rouge::Lexers::Escape
+
         if out_buf != sample
           out_file = "tmp/mismatch.#{subject.tag}"
           puts "mismatch with #{samples_dir.join(lexer_class.tag)}! logged to #{out_file}"

spec/visual/app.rb

@@ -35,6 +35,7 @@ def query_string
     reload_source!
 
     Rouge::Lexer.enable_debug!
+    Rouge::Formatter.enable_escape! if params[:escape]
 
     theme_class = Rouge::Theme.find(params[:theme] || 'thankful_eyes')
     halt 404 unless theme_class

spec/visual/samples/escape

@@ -0,0 +1,8 @@
+If Formatter.enable_escape! is called, this allows escaping into html
+or the parent format with a special delimiter. For example, here is
+some <!<span style="text-decoration: underline">!>underlined text<!</span>!> that flows with the rest of the document.
+
+When visually speccing this, you should see the escaped portions rendered as an
+error. This is because for safety reasons we don't allow escaping to html without
+explicit opting in. If you provide the ?escape=1 option in the url, you should
+see underlined text.