Proposal for escaping within lexed content

lib/rouge.rb

@@ -85,4 +85,4 @@ def lexer_dir(path = '')
 load_relative 'rouge/themes/gruvbox'
 load_relative 'rouge/themes/tulip'
 load_relative 'rouge/themes/pastie'
-load_relative 'rouge/themes/bw'
+load_relative 'rouge/themes/bw'

lib/rouge/cli.rb

@@ -186,6 +186,11 @@ def self.doc
         yield %[]
         yield %[--require|-r <filename>     require a filename or library before]
         yield %[                            highlighting]
+        yield %[]
+        yield %[--escape                    allow the use of escapes between <! and !>]
+        yield %[]
+        yield %[--escape-with <l> <r>       allow the use of escapes between custom]
+        yield %[                            delimiters. implies --escape]
       end
 
       def self.parse(argv)
@@ -218,6 +223,10 @@ def self.parse(argv)
             opts[:css_class] = argv.shift
           when '--lexer-opts', '-L'
             opts[:lexer_opts] = parse_cgi(argv.shift)
+          when '--escape'
+            opts[:escape] = ['<!', '!>']
+          when '--escape-with'
+            opts[:escape] = [argv.shift, argv.shift]
           when /^--/
             error! "unknown option #{arg.inspect}"
           else
@@ -244,11 +253,23 @@ def lexer_class
         )
       end
 
+      def raw_lexer
+        lexer_class.new(@lexer_opts)
+      end
+
+      def escape_lexer
+        Rouge::Lexers::Escape.new(
+          start: @escape[0],
+          end: @escape[1],
+          lang: raw_lexer,
+        )
+      end
+
       def lexer
-        @lexer ||= lexer_class.new(@lexer_opts)
+        @lexer ||= @escape ? escape_lexer : raw_lexer
       end
 
-      attr_reader :input_file, :lexer_name, :mimetype, :formatter
+      attr_reader :input_file, :lexer_name, :mimetype, :formatter, :escape
 
       def initialize(opts={})
         Rouge::Lexer.enable_debug!
@@ -281,9 +302,12 @@ def initialize(opts={})
         else
           error! "unknown formatter preset #{opts[:formatter]}"
         end
+
+        @escape = opts[:escape]
       end
 
       def run
+        Formatter.enable_escape! if @escape
         formatter.format(lexer.lex(input), &method(:print))
       end
 

lib/rouge/demos/escape

@@ -0,0 +1,3 @@
+If Formatter.enable_escape! is called, this allows escaping into html
+or the parent format with a special delimiter. For example:
+<!<span style="text-decoration: underline">!>underlined text!<!</span>!>

lib/rouge/formatter.rb

@@ -21,6 +21,26 @@ def self.find(tag)
       REGISTRY[tag]
     end
 
+    def self.with_escape
+      Thread.current[:'rouge/with-escape'] = true
+      yield
+    ensure
+      Thread.current[:'rouge/with-escape'] = false
+    end
+
+    def self.escape_enabled?
+      !!(@escape_enabled || Thread.current[:'rouge/with-escape'])
+    end
+
+    def self.enable_escape!
+      @escape_enabled = true
+    end
+
+    def self.disable_escape!
+      @escape_enabled = false
+      Thread.current[:'rouge/with-escape'] = false
+    end
+
     # Format a token stream.  Delegates to {#format}.
     def self.format(tokens, *a, &b)
       new(*a).format(tokens, &b)
@@ -30,8 +50,24 @@ def initialize(opts={})
       # pass
     end
 
+    def escape?(tok)
+      tok == Token::Tokens::Escape
+    end
+
+    def filter_escapes(tokens)
+      tokens.each do |t, v|
+        if t == Token::Tokens::Escape
+          yield Token::Tokens::Error, v
+        else
+          yield t, v
+        end
+      end
+    end
+
     # Format a token stream.
     def format(tokens, &b)
+      tokens = enum_for(:filter_escapes, tokens) unless Formatter.escape_enabled?
+
       return stream(tokens, &b) if block_given?
 
       out = String.new('')

lib/rouge/formatters/html.rb

@@ -13,6 +13,8 @@ def stream(tokens, &b)
       end
 
       def span(tok, val)
+        return val if escape?(tok)
+
         safe_span(tok, val.gsub(/[&<>]/, TABLE_FOR_ESCAPE_HTML))
       end
 

lib/rouge/formatters/terminal256.rb

@@ -159,6 +159,7 @@ def self.closest_color(r, g, b)
 
     # private
       def escape_sequence(token)
+        return '' if escape?(token)
         @escape_sequences ||= {}
         @escape_sequences[token.qualname] ||=
           EscapeSequence.new(get_style(token))

lib/rouge/lexers/escape.rb

@@ -0,0 +1,55 @@
+module Rouge
+  module Lexers
+    class Escape < Lexer
+      tag 'escape'
+      aliases 'esc'
+
+      desc 'A generic lexer for including escaped content - see Formatter.enable_escape!'
+
+      option :start, 'the beginning of the escaped section, default "<!"'
+      option :end, 'the end of the escaped section, e.g. "!>"'
+      option :lang, 'the language to lex in unescaped sections'
+
+      attr_reader :start
+      attr_reader :end
+      attr_reader :lang
+
+      def initialize(*)
+        super
+        @start = string_option(:start) { '<!' }
+        @end = string_option(:end) { '!>' }
+        @lang = lexer_option(:lang) { PlainText.new }
+      end
+
+      def to_start_regex
+        @to_start_regex ||= /(.*?)(#{Regexp.escape(@start)})/m
+      end
+
+      def to_end_regex
+        @to_end_regex ||= /(.*?)(#{Regexp.escape(@end)})/m
+      end
+
+      def stream_tokens(str, &b)
+        stream = StringScanner.new(str)
+
+        loop do
+          if stream.scan(to_start_regex)
+            puts "pre-escape: #{stream[1].inspect}" if @debug
+            @lang.continue_lex(stream[1], &b)
+          else
+            # no more start delimiters, scan til the end
+            @lang.continue_lex(stream.rest, &b)
+            return
+          end
+
+          if stream.scan(to_end_regex)
+            yield Token::Tokens::Escape, stream[1]
+          else
+            yield Token::Tokens::Escape, stream.rest
+            return
+          end
+        end
+      end
+    end
+  end
+end

spec/cli_spec.rb

@@ -51,6 +51,26 @@
         assert { Rouge::Lexers::Ruby === subject.lexer }
       }
     end
+
+    describe 'escaping by default' do
+      let(:argv) { %w(highlight --escape -l ruby) }
+      it('parses') {
+        assert { Rouge::Lexers::Escape === subject.lexer }
+        assert { Rouge::Lexers::Ruby === subject.lexer.lang }
+        assert { subject.lexer.start == '<!' }
+        assert { subject.lexer.end == '!>' }
+      }
+    end
+
+    describe 'escaping with custom delimiters' do
+      let(:argv) { %w(highlight --escape-with [===[ ]===] -l ruby) }
+      it('parses') {
+        assert { Rouge::Lexers::Escape === subject.lexer }
+        assert { Rouge::Lexers::Ruby === subject.lexer.lang }
+        assert { subject.lexer.start == '[===[' }
+        assert { subject.lexer.end == ']===]' }
+      }
+    end
   end
 
   describe Rouge::CLI::List do

spec/formatter_spec.rb

@@ -8,4 +8,28 @@
   it 'is found by Rouge.highlight' do
     assert { Rouge.highlight('puts "Hello"', 'ruby', 'terminal256') }
   end
+
+  it 'does not escape by default' do
+    assert { not Rouge::Formatter.escape_enabled? }
+  end
+
+  it 'escapes in all threads with #enable_escape!' do
+    begin
+      Rouge::Formatter.enable_escape!
+      assert { Rouge::Formatter.escape_enabled? }
+    ensure
+      Rouge::Formatter.disable_escape!
+    end
+  end
+
+  it 'escapes locally with #with_escape' do
+    Rouge::Formatter.with_escape do
+      assert { Rouge::Formatter.escape_enabled? }
+      assert { not Thread.new { Rouge::Formatter.escape_enabled? }.value }
+      Rouge::Formatter.disable_escape!
+      assert { not Rouge::Formatter.escape_enabled? }
+    end
+
+    assert { not Rouge::Formatter.escape_enabled? }
+  end
 end

spec/lexers_spec.rb

@@ -23,6 +23,9 @@
           out_buf << value
         end
 
+        # Escape is allowed to drop characters from its input
+        next if lexer_class == Rouge::Lexers::Escape
+
         if out_buf != sample
           out_file = "tmp/mismatch.#{subject.tag}"
           puts "mismatch with #{samples_dir.join(lexer_class.tag)}! logged to #{out_file}"

spec/visual/app.rb

@@ -35,6 +35,7 @@ def query_string
     reload_source!
 
     Rouge::Lexer.enable_debug!
+    Rouge::Formatter.enable_escape! if params[:escape]
 
     theme_class = Rouge::Theme.find(params[:theme] || 'thankful_eyes')
     halt 404 unless theme_class

spec/visual/samples/escape

@@ -0,0 +1,8 @@
+If Formatter.enable_escape! is called, this allows escaping into html
+or the parent format with a special delimiter. For example, here is
+some <!<span style="text-decoration: underline">!>underlined text<!</span>!> that flows with the rest of the document.
+
+When visually speccing this, you should see the escaped portions rendered as an
+error. This is because for safety reasons we don't allow escaping to html without
+explicit opting in. If you provide the ?escape=1 option in the url, you should
+see underlined text.