Asciidoctor: Support for "open in widget"

README.asciidoc

@@ -1014,12 +1014,26 @@ footnote to a particular line of code:
 === View in Console
 
 Code blocks can be followed by a "View in Console" link which, when clicked,
-will open the code snippet in Console.  The snippet can either be taken directly
-from the code block (`CONSOLE`), or be a link to a custom snippet.
+will open the code snippet in Console. There are two ways to do this, the
+"AsciiDoc" way and the "Asciidoctor" way. The "AsciiDoc" way is preferred in
+the Elaticsearch repository because it can recognize it to make tests. The
+"Asciidoctor" way is preferred in other books, but only if they are built with
+"Asciidoctor". Try it first and if it works then use it. Otherwise, use the
+"AsciiDoc" way.
 
-.Code block with CONSOLE link
+////
+
+The tricks that we pull to make ascidoctor support // CONSOLE are windy
+and force us to add subs=+macros when we render an asciidoc snippet.
+We *don't* require that for normal snippets, just those that contain
+asciidoc.
+
+////
+
+.Code block with CONSOLE link (AsciiDoc way)
 ==================================
-[source,asciidoc]
+ifdef::asciidoctor[[source,asciidoc,subs=+macros]]
+ifndef::asciidoctor[[source,asciidoc]]
 --
 [source,js]
 ----------------------------------
@@ -1035,6 +1049,24 @@ GET /_search
 ==================================
 <1> The `// CONSOLE` line must follow immediately after the code block, before any callouts.
 
+.Code block with CONSOLE link (Asciidoctor way)
+==================================
+[source,asciidoc]
+--
+[source,console]
+----------------------------------
+GET /_search
+{
+    "query": "foo bar" \<1>
+}
+----------------------------------
+
+\<1> Here's the explanation
+--
+==================================
+
+Both render as:
+
 [source,js]
 ----------------------------------
 GET /_search
@@ -1060,38 +1092,6 @@ The local web browser can be stopped with `Ctrl-C`.
 
 ================================
 
-==== Custom Console snippets
-
-Sometimes you will want to show a small amount of code in the code block, but
-to provide a full recreation in the Console snippet.  In this case, you need to:
-
-* Save the snippet file in the `./snippets/` directory in the root docs directory.
-* Under the code block, specify the name of the snippet file with
-+
-    // CONSOLE: path/to/snippet.json
-
-For instance, to add a custom snippet to the file `./one/two/three.asciidoc`, save the snippet
-to `./snippets/one/two/three/example_1.json`, then add the `CONSOLE` link below the code block:
-
-.Code block with custom CONSOLE link
-==================================
-[source,asciidoc]
---
-[source,js]
-----------------------------------
-GET /_search
-{
-    "query": "foo bar" \<1>
-}
-----------------------------------
-// CONSOLE:one/two/three/example_1.json <1>
-
-\<1> Here's the explanation
---
-<1> The path should not contain the initial `snippets` directory
-==================================
-
-
 [[admon-blocks]]
 === Admonition blocks
 

integtest/Makefile

@@ -63,14 +63,15 @@ experimental_expected_files: /tmp/experimental_asciidoc
 %_same_files: /tmp/%_asciidoc /tmp/%_asciidoctor
 	diff \
 		<(cd /tmp/$*_asciidoc    && find * -type f | sort \
-			| grep -v snippets/blocks \
+			| grep -v 'snippets/' \
 		) \
-		<(cd /tmp/$*_asciidoctor && find * -type f | sort)
-	# The grep -v below are for known issues with asciidoctor
-	for file in $$(cd /tmp/$*_asciidoc && find * -type f -name '*.html' \
-			| grep -v 'blocks'); do \
+		<(cd /tmp/$*_asciidoctor && find * -type f | sort \
+			| grep -v 'snippets/' \
+		)
+	for file in $$(cd /tmp/$*_asciidoc && find * -type f -name '*.html'); do \
 		./html_diff /tmp/$*_asciidoc/$$file /tmp/$*_asciidoctor/$$file; \
 	done
+	# TODO validate the snippets have the same contents even if the files aren't the same
 
 # Build the docs into the target
 define BD=

integtest/html_diff

@@ -35,10 +35,13 @@ def normalize_html(html):
     # is between words. They are actively nice to have but asciidoc doesn't
     # make them.
     html = html.replace('\u2014\u200b', '\u2014')
-    # Temporary workaround for known issues
-    html = re.sub(
-        r'(?m)^\s+<div class="console_widget" data-snippet="[^"]+">'
-        r'\s+</div>\n', '', html)
+    # We intentionally changed lang-js to lang-console because in Asciidoctor
+    # because that is more accurate
+    html = html.replace('"programlisting prettyprint lang-js"',
+                        '"programlisting prettyprint lang-console"')
+    # The URL for the console snippets has changed
+    html = re.sub(r'data-snippet="[^"]+"', 'data-snippet="snippet"', html)
+    # Temporary work around for known issue
     html = html.replace('\\&lt;1&gt;', '&lt;1&gt;')
     return html
 

lib/ES/Template.pm

@@ -47,6 +47,7 @@ sub apply {
     my $self = shift;
     my $dir  = shift;
     my $lang = shift || die "No lang specified";
+    my $asciidoctor = shift;
 
     my $map = $self->_map;
 
@@ -61,7 +62,7 @@ sub apply {
         $contents =~ s/\s*$/\n/;
 
         # Extract AUTOSENSE snippets
-        $contents = $self->_autosense_snippets( $file, $contents );
+        $contents = $self->_autosense_snippets( $file, $contents ) unless $asciidoctor;
 
         # Fill in template
         my @parts  = @{ $self->_parts };

lib/ES/Util.pm

@@ -154,7 +154,7 @@ sub build_chunked {
     my ($chunk_dir) = grep { -d and /\.chunked$/ } $dest->children
         or die "Couldn't find chunk dir in <$dest>";
 
-    finish_build( $index->parent, $chunk_dir, $lang );
+    finish_build( $index->parent, $chunk_dir, $lang, $asciidoctor );
     extract_toc_from_index($chunk_dir);
     for ( $chunk_dir->children ) {
         run( 'mv', $_, $dest );
@@ -287,7 +287,7 @@ sub build_single {
             or die "Couldn't rename <$src> to <index.html>: $!";
     }
 
-    finish_build( $index->parent, $dest, $lang );
+    finish_build( $index->parent, $dest, $lang, $asciidoctor );
 }
 
 #===================================
@@ -369,10 +369,10 @@ sub build_pdf {
 #===================================
 sub finish_build {
 #===================================
-    my ( $source, $dest, $lang ) = @_;
+    my ( $source, $dest, $lang, $asciidoctor ) = @_;
 
     # Apply template to HTML files
-    $Opts->{template}->apply( $dest, $lang );
+    $Opts->{template}->apply( $dest, $lang, $asciidoctor );
 
     my $snippets_dest = $dest->subdir('snippets');
     my $snippets_src;

resources/asciidoctor/lib/elastic_compat_preprocessor/extension.rb

@@ -89,12 +89,31 @@
 # Because Asciidoc permits these mismatches but asciidoctor does not. We'll
 # emit a warning because, permitted or not, they are bad style.
 #
+# With the help of ElasticCompatTreeProcessor turns
+#   [source,js]
+#   ----
+#   foo
+#   ----
+#   // CONSOLE
+#
+# Into
+#   [source,console]
+#   ----
+#   foo
+#   ----
+# Because Elastic has thousands of these constructs but Asciidoctor feels
+# strongly that comments should not convey meaning. This is a totally
+# reasonable stance and we should migrate away from these comments in new
+# docs when it is possible. But for now we have to support the comments as
+# well.
+#
 class ElasticCompatPreprocessor < Asciidoctor::Extensions::Preprocessor
   include Asciidoctor::Logging
 
   INCLUDE_TAGGED_DIRECTIVE_RX = /^include-tagged::([^\[][^\[]*)\[(#{Asciidoctor::CC_ANY}+)?\]$/.freeze
   SOURCE_WITH_SUBS_RX = /^\["source", ?"[^"]+", ?subs="(#{Asciidoctor::CC_ANY}+)"\]$/.freeze
   CODE_BLOCK_RX = /^-----*$/.freeze
+  SNIPPET_RX = %r{//\s*(?:AUTOSENSE|KIBANA|CONSOLE|SENSE:[^\n<]+)}.freeze
 
   def process(_document, reader)
     reader.instance_variable_set :@in_attribute_only_block, false
@@ -142,13 +161,22 @@ def reader.process_line(line)
             @code_block_start = line
           end
         end
+
         supported = 'added|beta|coming|deprecated|experimental'
         # First convert the "block" version of these macros. We convert them
         # to block macros because they are at the start of the line....
         line&.gsub!(/^(#{supported})\[([^\]]*)\]/, '\1::[\2]')
         # Then convert the "inline" version of these macros. We convert them
         # to inline macros because they are *not* at the start of the line....
         line&.gsub!(/(#{supported})\[([^\]]*)\]/, '\1:[\2]')
+
+        # Transform Elastic's traditional comment based marking for
+        # AUTOSENSE/KIBANA/CONSOLE snippets into a marker that we can pick
+        # up during tree processing to turn the snippet into a marked up
+        # CONSOLE snippet. Asciidoctor really doesn't recommend this sort of
+        # thing but we have thousands of them and it'll take us some time to
+        # stop doing it.
+        line&.gsub!(SNIPPET_RX, 'pass:[\0]')
       end
     end
     reader

resources/asciidoctor/lib/elastic_compat_tree_processor/extension.rb

@@ -23,14 +23,62 @@
 #   <1> The count of categories that were matched
 #   <2> The categories retrieved
 #
+# Turns
+#   [source,js]
+#   --------------------------------------------------
+#   GET / <1>
+#   --------------------------------------------------
+#   pass:[// CONSOLE]
+#   <1> The count of categories that were matched
+#   <2> The categories retrieved
+#
+# Into
+#   [source,console]
+#   --------------------------------------------------
+#   GET / <1>
+#   --------------------------------------------------
+#   <1> The count of categories that were matched
+#   <2> The categories retrieved
+#
 class ElasticCompatTreeProcessor < TreeProcessorScaffold
+  include Asciidoctor::Logging
+
   def process_block(block)
-    if block.context == :listing && block.style == "source" &&
-          block.subs.include?(:specialcharacters) == false
-      # callouts have to come *after* special characters
-      had_callouts = block.subs.delete(:callouts)
-      block.subs << :specialcharacters
-      block.subs << :callouts if had_callouts
-    end
+    return unless block.context == :listing && block.style == 'source'
+
+    process_subs block
+    process_lang_override block
+  end
+
+  def process_subs(block)
+    return if block.subs.include? :specialcharacters
+
+    # callouts have to come *after* special characters
+    had_callouts = block.subs.delete(:callouts)
+    block.subs << :specialcharacters
+    block.subs << :callouts if had_callouts
+  end
+
+  LANG_MAPPING = {
+    'AUTOSENSE' => 'sense',
+    'CONSOLE' => 'console',
+    'KIBANA' => 'kibana',
+    'SENSE' => 'sense',
+  }.freeze
+
+  def process_lang_override(block)
+    next_block = block.next_adjacent_block
+    return unless next_block && next_block.context == :paragraph
+    return unless next_block.source =~ %r{pass:\[//\s*([^:\]]+)(?::\s*([^\]]+))?\]}
+
+    lang = LANG_MAPPING[$1]
+    snippet = $2
+    return unless lang # Not a language we handle
+
+    block.set_attr 'language', lang
+    block.set_attr 'snippet', snippet
+
+    block.parent.blocks.delete next_block
+    block.parent.reindex_sections
   end
 end

resources/asciidoctor/lib/extensions.rb

@@ -8,6 +8,7 @@
 require_relative 'elastic_compat_tree_processor/extension'
 require_relative 'elastic_compat_preprocessor/extension'
 require_relative 'elastic_include_tagged/extension'
+require_relative 'open_in_widget/extension'
 
 Asciidoctor::Extensions.register CareAdmonition
 Asciidoctor::Extensions.register ChangeAdmonition
@@ -20,5 +21,6 @@
   treeprocessor CopyImages::CopyImages
   treeprocessor EditMe
   treeprocessor ElasticCompatTreeProcessor
+  treeprocessor OpenInWidget
   include_processor ElasticIncludeTagged
 end