Asciidoctor: Support for "open in widget"

Makefile

@@ -38,12 +38,20 @@ expected_files_check: /tmp/readme_asciidoc
 
 .PHONY: same_files_check
 same_files_check: /tmp/readme_asciidoc /tmp/readme_asciidoctor
-	# The `grep -v snippets` is a known issue to be resolved "soon"
+	# The `grep -v`s are a known issue to be resolved "soon"
 	diff \
 		<(cd /tmp/readme_asciidoc    && find * -type f | sort \
-			| grep -v snippets/blocks \
+			| grep -v snippets/ \
 		) \
-		<(cd /tmp/readme_asciidoctor && find * -type f | sort)
+		<(cd /tmp/readme_asciidoctor && find * -type f | sort \
+			| grep -v snippets/ \
+		)
+	# Temporary assertions about the `grep -v`ed files that we'll
+	# generalize in a followup change. Asciidoc doesn't trim trailing
+	# newlines but AsciiDoctor does.
+	diff \
+		<(sed 's/[[:space:]]*$$//' /tmp/readme_asciidoc/snippets/blocks/1.json) \
+		/tmp/readme_asciidoctor/snippets/1.console
 
 /tmp/readme_asciidoc:
 	./build_docs.pl --in_standard_docker \

README.asciidoc

@@ -1005,12 +1005,26 @@ footnote to a particular line of code:
 === View in Console
 
 Code blocks can be followed by a "View in Console" link which, when clicked,
-will open the code snippet in Console.  The snippet can either be taken directly
-from the code block (`CONSOLE`), or be a link to a custom snippet.
+will open the code snippet in Console. There are two ways to do this, the
+"AsciiDoc" way and the "Asciidoctor" way. The "AsciiDoc" way is preferred in
+the Elaticsearch repository because it can recognize it to make tests. The
+"Asciidoctor" way is preferred in other books, but only if they are built with
+"Asciidoctor". Try it first and if it works then use it. Otherwise, use the
+"AsciiDoct" way.
 
-.Code block with CONSOLE link
+////
+
+The tricks that we pull to make ascidoctor support // CONSOLE are windy
+and force us to add subs=+macros when we render an asciidoc snippet.
+We *don't* require that for normal snippets, just those that contain
+asciidoc.
+
+////
+
+.Code block with CONSOLE link (AsciiDoc way)
 ==================================
-[source,asciidoc]
+ifdef::asciidoctor[[source,asciidoc,subs=+macros]]
+ifndef::asciidoctor[[source,asciidoc]]
 --
 [source,js]
 ----------------------------------
@@ -1026,6 +1040,24 @@ GET /_search
 ==================================
 <1> The `// CONSOLE` line must follow immediately after the code block, before any callouts.
 
+.Code block with CONSOLE link (Asciidoctor way)
+==================================
+[source,asciidoc]
+--
+[source,console]
+----------------------------------
+GET /_search
+{
+    "query": "foo bar" \<1>
+}
+----------------------------------
+
+\<1> Here's the explanation
+--
+==================================
+
+Both render as:
+
 [source,js]
 ----------------------------------
 GET /_search
@@ -1051,38 +1083,6 @@ The local web browser can be stopped with `Ctrl-C`.
 
 ================================
 
-==== Custom Console snippets
-
-Sometimes you will want to show a small amount of code in the code block, but
-to provide a full recreation in the Console snippet.  In this case, you need to:
-
-* Save the snippet file in the `./snippets/` directory in the root docs directory.
-* Under the code block, specify the name of the snippet file with
-+
-    // CONSOLE: path/to/snippet.json
-
-For instance, to add a custom snippet to the file `./one/two/three.asciidoc`, save the snippet
-to `./snippets/one/two/three/example_1.json`, then add the `CONSOLE` link below the code block:
-
-.Code block with custom CONSOLE link
-==================================
-[source,asciidoc]
---
-[source,js]
-----------------------------------
-GET /_search
-{
-    "query": "foo bar" \<1>
-}
-----------------------------------
-// CONSOLE:one/two/three/example_1.json <1>
-
-\<1> Here's the explanation
---
-<1> The path should not contain the initial `snippets` directory
-==================================
-
-
 [[admon-blocks]]
 === Admonition blocks
 

lib/ES/Template.pm

@@ -47,6 +47,7 @@ sub apply {
     my $self = shift;
     my $dir  = shift;
     my $lang = shift || die "No lang specified";
+    my $asciidoctor = shift;
 
     my $map = $self->_map;
 
@@ -61,7 +62,7 @@ sub apply {
         $contents =~ s/\s*$/\n/;
 
         # Extract AUTOSENSE snippets
-        $contents = $self->_autosense_snippets( $file, $contents );
+        $contents = $self->_autosense_snippets( $file, $contents ) unless $asciidoctor;
 
         # Fill in template
         my @parts  = @{ $self->_parts };

lib/ES/Util.pm

@@ -153,7 +153,7 @@ sub build_chunked {
     my ($chunk_dir) = grep { -d and /\.chunked$/ } $dest->children
         or die "Couldn't find chunk dir in <$dest>";
 
-    finish_build( $index->parent, $chunk_dir, $lang );
+    finish_build( $index->parent, $chunk_dir, $lang, $asciidoctor );
     extract_toc_from_index($chunk_dir);
     for ( $chunk_dir->children ) {
         run( 'mv', $_, $dest );
@@ -285,7 +285,7 @@ sub build_single {
             or die "Couldn't rename <$src> to <index.html>: $!";
     }
 
-    finish_build( $index->parent, $dest, $lang );
+    finish_build( $index->parent, $dest, $lang, $asciidoctor );
 }
 
 #===================================
@@ -367,10 +367,10 @@ sub build_pdf {
 #===================================
 sub finish_build {
 #===================================
-    my ( $source, $dest, $lang ) = @_;
+    my ( $source, $dest, $lang, $asciidoctor ) = @_;
 
     # Apply template to HTML files
-    $Opts->{template}->apply( $dest, $lang );
+    $Opts->{template}->apply( $dest, $lang, $asciidoctor );
 
     my $snippets_dest = $dest->subdir('snippets');
     my $snippets_src;

resources/asciidoctor/lib/elastic_compat_preprocessor/extension.rb

@@ -89,6 +89,24 @@
 # Because Asciidoc permits these mismatches but asciidoctor does not. We'll
 # emit a warning because, permitted or not, they are bad style.
 #
+# With the help of ElasticCompatTreeProcessor turns
+#   [source,js]
+#   ----
+#   foo
+#   ----
+#   // CONSOLE
+#
+# Into
+#   [source,console]
+#   ----
+#   foo
+#   ----
+# Because Elastic has thousands of these constructs but Asciidoctor feels
+# strongly that comments should not convey meaning. This is a totally
+# reasonable stance and we should migrate away from these comments in new
+# docs when it is possible. But for now we have to support the comments as
+# well.
+#
 class ElasticCompatPreprocessor < Asciidoctor::Extensions::Preprocessor
   include Asciidoctor::Logging
 
@@ -140,13 +158,21 @@ def reader.process_line(line)
             @code_block_start = line
           end
         end
+
         supported = 'added|coming|deprecated'
         # First convert the "block" version of these macros. We convert them
         # to block macros because they are at the start of the line....
         line&.gsub!(/^(#{supported})\[([^\]]*)\]/, '\1::[\2]')
         # Then convert the "inline" version of these macros. We convert them
         # to inline macros because they are *not* at the start of the line....
         line&.gsub!(/(#{supported})\[([^\]]*)\]/, '\1:[\2]')
+        # Transform Elastic's traditional comment based marking for
+        # AUTOSENSE/KIBANA/CONSOLE snippets into a marker that we can pick
+        # up during tree processing to turn the snippet into a marked up
+        # CONSOLE snippet. Asciidoctor really doesn't recommend this sort of
+        # thing but we have thousands of them and it'll take us some time to
+        # stop doing it.
+        line&.gsub!(%r{//\s*(?:AUTOSENSE|KIBANA|CONSOLE|SENSE:[^\n<]+)}, 'pass:[\0]')
       end
     end
     reader

resources/asciidoctor/lib/elastic_compat_tree_processor/extension.rb

@@ -23,14 +23,62 @@
 #   <1> The count of categories that were matched
 #   <2> The categories retrieved
 #
+# Turns
+#   [source,js]
+#   --------------------------------------------------
+#   GET / <1>
+#   --------------------------------------------------
+#   pass:[// CONSOLE]
+#   <1> The count of categories that were matched
+#   <2> The categories retrieved
+#
+# Into
+#   [source,console]
+#   --------------------------------------------------
+#   GET / <1>
+#   --------------------------------------------------
+#   <1> The count of categories that were matched
+#   <2> The categories retrieved
+#
 class ElasticCompatTreeProcessor < TreeProcessorScaffold
+  include Asciidoctor::Logging
+
   def process_block(block)
-    if block.context == :listing && block.style == "source" &&
-          block.subs.include?(:specialcharacters) == false
-      # callouts have to come *after* special characters
-      had_callouts = block.subs.delete(:callouts)
-      block.subs << :specialcharacters
-      block.subs << :callouts if had_callouts
-    end
+    return unless block.context == :listing && block.style == 'source'
+
+    process_subs block
+    process_lang_override block
+  end
+
+  def process_subs(block)
+    return if block.subs.include? :specialcharacters
+
+    # callouts have to come *after* special characters
+    had_callouts = block.subs.delete(:callouts)
+    block.subs << :specialcharacters
+    block.subs << :callouts if had_callouts
+  end
+
+  LANG_MAPPING = {
+    'AUTOSENSE' => 'sense',
+    'CONSOLE' => 'console',
+    'KIBANA' => 'kibana',
+    'SENSE' => 'sense',
+  }
+
+  def process_lang_override(block)
+    next_block = block.next_adjacent_block
+    return unless next_block && next_block.context == :paragraph
+    return unless next_block.source =~ %r{pass:\[//\s*([^:\]]+)(?::\s*([^\]]+))?\]}
+
+    lang = LANG_MAPPING[$1]
+    snippet = $2
+    return unless lang # Not a language we handle
+
+    block.set_attr 'language', lang
+    block.set_attr 'snippet', snippet
+
+    block.parent.blocks.delete next_block
+    block.parent.reindex_sections
   end
 end

resources/asciidoctor/lib/extensions.rb

@@ -7,6 +7,7 @@
 require_relative 'elastic_compat_tree_processor/extension'
 require_relative 'elastic_compat_preprocessor/extension'
 require_relative 'elastic_include_tagged/extension'
+require_relative 'open_in_widget/extension'
 
 Asciidoctor::Extensions.register ChangeAdmonition
 Asciidoctor::Extensions.register do
@@ -18,5 +19,6 @@
   treeprocessor CopyImages::CopyImages
   treeprocessor EditMe
   treeprocessor ElasticCompatTreeProcessor
+  treeprocessor OpenInWidget
   include_processor ElasticIncludeTagged
 end