diff --git a/Gemfile.devel b/Gemfile.devel new file mode 100644 index 00000000..87d89080 --- /dev/null +++ b/Gemfile.devel @@ -0,0 +1 @@ +gem "isodoc", git: "https://github.com/metanorma/isodoc", branch: "fix/yaml-no-xml" diff --git a/lib/metanorma/standoc/cleanup.rb b/lib/metanorma/standoc/cleanup.rb index 8baa7cd5..c90433b2 100644 --- a/lib/metanorma/standoc/cleanup.rb +++ b/lib/metanorma/standoc/cleanup.rb @@ -54,6 +54,7 @@ def cleanup(xmldoc) normref_cleanup(xmldoc) biblio_cleanup(xmldoc) reference_names(xmldoc) + terms_terms_cleanup(xmldoc) # feeds: boilerplate_cleanup asciimath_cleanup(xmldoc) # feeds: mathml_cleanup, termdef_cleanup, # symbols_cleanup symbols_cleanup(xmldoc) # feeds: termdef_cleanup @@ -77,9 +78,10 @@ def cleanup(xmldoc) docidentifier_cleanup(xmldoc) # feeds: bibdata_cleanup ext_contributor_cleanup(xmldoc) # feeds: bibdata_cleanup ext_dochistory_cleanup(xmldoc) # feeds: bibdata_cleanup - bibdata_cleanup(xmldoc) + bibdata_cleanup(xmldoc) # feeds: boilerplate_cleanup + boilerplate_cleanup(xmldoc) # feeds: xref_cleanup for new <<>> introduced + xref_cleanup(xmldoc) svgmap_cleanup(xmldoc) # feeds: img_cleanup - boilerplate_cleanup(xmldoc) toc_cleanup(xmldoc) smartquotes_cleanup(xmldoc) linebreak_cleanup(xmldoc) diff --git a/lib/metanorma/standoc/cleanup_boilerplate.rb b/lib/metanorma/standoc/cleanup_boilerplate.rb index 00895d78..678af8a0 100644 --- a/lib/metanorma/standoc/cleanup_boilerplate.rb +++ b/lib/metanorma/standoc/cleanup_boilerplate.rb @@ -1,41 +1,8 @@ +require_relative "cleanup_terms_boilerplate" + module Metanorma module Standoc module Cleanup - def external_terms_boilerplate(sources) - e = @i18n.external_terms_boilerplate - @i18n.l10n(e.gsub(/%(?=\p{P}|\p{Z}|$)/, sources || "???"), - @lang, @script, @locale) - end - - def internal_external_terms_boilerplate(sources) - e = @i18n.internal_external_terms_boilerplate - @i18n.l10n(e.gsub(/%(?=\p{P}|\p{Z}|$)/, sources || "??"), - @lang, @script) - end - - def term_defs_boilerplate(div, source, term, _preface, isodoc) - a = @i18n.term_def_boilerplate and div.next = a - source.each do |s| - @anchors[s["bibitemid"]] or - @log.add("Crossreferences", nil, - "term source #{s['bibitemid']} not referenced", severity: 1) - end - a = if source.empty? && term.nil? then @i18n.no_terms_boilerplate - else term_defs_boilerplate_cont(source, term, isodoc) - end and div.next = a - end - - def term_defs_boilerplate_cont(src, term, isodoc) - sources = isodoc.sentence_join(src.map do |s| - %{} - end) - if src.empty? then @i18n.internal_terms_boilerplate - elsif term.nil? then external_terms_boilerplate(sources) - else - internal_external_terms_boilerplate(sources) - end - end - def norm_ref_preface(ref) ins = norm_ref_boilerplate_insert_location(ref) ins2 = norm_ref_process_boilerplate_note(ref) @@ -45,7 +12,7 @@ def norm_ref_preface(ref) %w(references bibitem).include? e.name end pref = refs.empty? ? @i18n.norm_empty_pref : @i18n.norm_with_refs_pref - ins.next = "

#{pref}

" + ins.next = boilerplate_snippet_convert(pref) end def norm_ref_process_boilerplate_note(ref) @@ -69,16 +36,15 @@ def norm_ref_boilerplate_insert_location(ref) ref.at("./title") end - TERM_CLAUSE = - "//sections//terms[not(.//ancestor::clause[@type = 'terms'])] | " \ - "//sections/clause[descendant::terms][@type = 'terms'] | " \ - "//sections/clause[not(@type = 'terms')]//terms".freeze - NORM_REF = "//bibliography/references[@normative = 'true'][not(@hidden)] | " \ "//bibliography/clause[.//references[@normative = 'true']]".freeze def boilerplate_isodoc(xmldoc) + # prevent infinite recursion of asciidoc boilerplate processing + # in termdef_boilerplate_insert and initial_boilerplate + xmldoc.at("//metanorma-extension/semantic-metadata/" \ + "headless[text() = 'true']") and return nil x = xmldoc.dup x.root.add_namespace(nil, self.class::XML_NAMESPACE) xml = Nokogiri::XML(x.to_xml) @@ -87,14 +53,6 @@ def boilerplate_isodoc(xmldoc) @isodoc end - def termdef_boilerplate_cleanup(xmldoc) - # termdef_remove_initial_paras(xmldoc) - end - - def termdef_remove_initial_paras(xmldoc) - xmldoc.xpath("//terms/p | //terms/ul").each(&:remove) - end - def unwrap_boilerplate_clauses(xmldoc, xpath) xmldoc.xpath(xpath).each do |f| f.xpath(".//clause[@type = 'boilerplate'] | " \ @@ -105,51 +63,8 @@ def unwrap_boilerplate_clauses(xmldoc, xpath) end end - def termdef_boilerplate_insert(xmldoc, isodoc, once = false) - if once - f = termdef_boilerplate_insert_location(xmldoc) and - termdef_boilerplate_insert1(f, xmldoc, isodoc) - else - xmldoc.xpath(self.class::TERM_CLAUSE).each do |f| - termdef_boilerplate_insert1(f, xmldoc, isodoc) - end - end - end - - def termdef_boilerplate_insert_location(xmldoc) - f = xmldoc.at(self.class::TERM_CLAUSE) - root = xmldoc.at("//sections/terms | //sections/clause[@type = 'terms']") - if f && root && f["id"] != root["id"] - f = termdef_boilerplate_climb_up(f, root) - elsif !f && root then f = root - end - f - end - - def termdef_boilerplate_climb_up(clause, container) - container.at(".//*[@id = '#{clause['id']}']") or return clause - while (n = clause.parent) - n.at(".//definitions") and break - clause = n - n["id"] == container["id"] and break - end - clause - end - - def termdef_boilerplate_insert1(sect, xmldoc, isodoc) - ins = sect.at("./title") - if (ins2 = sect.at("./clause[@type = 'boilerplate'] | " \ - "./note[@type = 'boilerplate']")) - ins2.text.strip.downcase == "(default)" or return - ins2.children = " " - ins = ins2.children.first - end - term_defs_boilerplate(ins, xmldoc.xpath(".//termdocsource"), - sect.at(".//term"), sect.at(".//p"), isodoc) - end - def boilerplate_cleanup(xmldoc) - isodoc = boilerplate_isodoc(xmldoc) + isodoc = boilerplate_isodoc(xmldoc) or return termdef_boilerplate_cleanup(xmldoc) termdef_boilerplate_insert(xmldoc, isodoc) unwrap_boilerplate_clauses(xmldoc, self.class::TERM_CLAUSE) @@ -245,6 +160,7 @@ def boilerplate_xml_cleanup(xml) /^_\d+$/.match?(n["id"]) and n["id"] = "_#{UUIDTools::UUID.random_create}" end + xml end def boilerplate_top_elements(xml) diff --git a/lib/metanorma/standoc/cleanup_terms.rb b/lib/metanorma/standoc/cleanup_terms.rb index c4874313..b8551702 100644 --- a/lib/metanorma/standoc/cleanup_terms.rb +++ b/lib/metanorma/standoc/cleanup_terms.rb @@ -73,11 +73,14 @@ def termdocsource_cleanup(xmldoc) xmldoc.xpath("//termdocsource").each { |s| f.previous = s.remove } end - def term_children_cleanup(xmldoc) + def terms_terms_cleanup(xmldoc) xmldoc.xpath("//terms[terms][not(term)]").each do |t| t.name = "clause" t["type"] = "terms" end + end + + def term_children_cleanup(xmldoc) xmldoc.xpath("//term").each do |t| %w(termnote termexample termsource term).each do |w| t.xpath("./#{w}").each { |n| t << n.remove } diff --git a/lib/metanorma/standoc/cleanup_terms_boilerplate.rb b/lib/metanorma/standoc/cleanup_terms_boilerplate.rb new file mode 100644 index 00000000..d569eb8a --- /dev/null +++ b/lib/metanorma/standoc/cleanup_terms_boilerplate.rb @@ -0,0 +1,106 @@ +module Metanorma + module Standoc + module Cleanup + def external_terms_boilerplate(sources) + e = @i18n.external_terms_boilerplate + e.gsub(/%(?=\p{P}|\p{Z}|$)/, sources || "???") + end + + def internal_external_terms_boilerplate(sources) + e = @i18n.internal_external_terms_boilerplate + e.gsub(/%(?=\p{P}|\p{Z}|$)/, sources || "??") + end + + def boilerplate_snippet_convert(adoc) + ret = boilerplate_xml_cleanup(adoc2xml(adoc, backend.to_sym)) + @i18n.l10n(ret.children.to_xml, @lang, @script) + end + + def term_defs_boilerplate(div, source, term, _preface, isodoc) + verify_term_defs_source(source) + a = @i18n.term_def_boilerplate and + div.next = boilerplate_snippet_convert(a) + a = if source.empty? && term.nil? then @i18n.no_terms_boilerplate + else term_defs_boilerplate_cont(source, term, isodoc) + end + a and div.next = boilerplate_snippet_convert(a) + end + + def verify_term_defs_source(source) + source.each do |s| + @anchors[s["bibitemid"]] or + @log.add("Crossreferences", nil, + "term source #{s['bibitemid']} not referenced", + severity: 1) + end + end + + def term_defs_boilerplate_cont(src, term, isodoc) + sources = isodoc.sentence_join(src.map do |s| + %{<<#{s['bibitemid']}>>} + end) + if src.empty? then @i18n.internal_terms_boilerplate + elsif term.nil? then external_terms_boilerplate(sources) + else + internal_external_terms_boilerplate(sources) + end + end + + TERM_CLAUSE = + "//sections//terms[not(.//ancestor::clause[@type = 'terms'])] | " \ + "//sections/clause[descendant::terms][@type = 'terms'] | " \ + "//sections/clause[not(@type = 'terms')]//terms".freeze + + def termdef_boilerplate_cleanup(xmldoc) + # termdef_remove_initial_paras(xmldoc) + end + + def termdef_remove_initial_paras(xmldoc) + xmldoc.xpath("//terms/p | //terms/ul").each(&:remove) + end + + def termdef_boilerplate_insert(xmldoc, isodoc, once = false) + if once + f = termdef_boilerplate_insert_location(xmldoc) and + termdef_boilerplate_insert1(f, xmldoc, isodoc) + else + xmldoc.xpath(self.class::TERM_CLAUSE).each do |f| + termdef_boilerplate_insert1(f, xmldoc, isodoc) + end + end + end + + def termdef_boilerplate_insert_location(xmldoc) + f = xmldoc.at(self.class::TERM_CLAUSE) + root = xmldoc.at("//sections/terms | //sections/clause[@type = 'terms']") + if f && root && f["id"] != root["id"] + f = termdef_boilerplate_climb_up(f, root) + elsif !f && root then f = root + end + f + end + + def termdef_boilerplate_climb_up(clause, container) + container.at(".//*[@id = '#{clause['id']}']") or return clause + while (n = clause.parent) + n.at(".//definitions") and break + clause = n + n["id"] == container["id"] and break + end + clause + end + + def termdef_boilerplate_insert1(sect, xmldoc, isodoc) + ins = sect.at("./title") + if (ins2 = sect.at("./clause[@type = 'boilerplate'] | " \ + "./note[@type = 'boilerplate']")) + ins2.text.strip.downcase == "(default)" or return + ins2.children = " " + ins = ins2.children.first + end + term_defs_boilerplate(ins, xmldoc.xpath(".//termdocsource"), + sect.at(".//term"), sect.at(".//p"), isodoc) + end + end + end +end diff --git a/lib/metanorma/standoc/cleanup_terms_designations.rb b/lib/metanorma/standoc/cleanup_terms_designations.rb index e71b5eac..e1a1d659 100644 --- a/lib/metanorma/standoc/cleanup_terms_designations.rb +++ b/lib/metanorma/standoc/cleanup_terms_designations.rb @@ -156,8 +156,7 @@ def term_termsource_to_designation(xmldoc) while %w(domain subject).include? p&.name p = p.previous_element end - DESIGNATOR.include?(p&.name) or - next + DESIGNATOR.include?(p&.name) or next related2pref(p) << t.remove end end diff --git a/lib/metanorma/standoc/cleanup_xref.rb b/lib/metanorma/standoc/cleanup_xref.rb index 32511310..68761e59 100644 --- a/lib/metanorma/standoc/cleanup_xref.rb +++ b/lib/metanorma/standoc/cleanup_xref.rb @@ -176,7 +176,7 @@ def xref_compound_cleanup(xmldoc) def xref_compound_cleanup1(xref, locations) xref.children.empty? and xref.children = "" - xref_parse_compound_locations(locations).reverse.each do |y| + xref_parse_compound_locations(locations).reverse_each do |y| xref.children.first.previous = "" end diff --git a/lib/metanorma/standoc/init.rb b/lib/metanorma/standoc/init.rb index d3d008da..5f115ee0 100644 --- a/lib/metanorma/standoc/init.rb +++ b/lib/metanorma/standoc/init.rb @@ -103,6 +103,7 @@ def init_i18n(node) def init_biblio(node) @no_isobib_cache = node.attr("no-isobib-cache") @no_isobib = node.attr("no-isobib") + @flush_caches = node.attr("flush-caches") init_bib_log @bibdb = nil init_bib_caches(node) diff --git a/lib/metanorma/standoc/ref_queue.rb b/lib/metanorma/standoc/ref_queue.rb index 7fe38b9a..7e7b1e15 100644 --- a/lib/metanorma/standoc/ref_queue.rb +++ b/lib/metanorma/standoc/ref_queue.rb @@ -228,7 +228,7 @@ def init_iev_caches(node) @iev_globalname = global_ievcache_name @iev_localname = local_ievcache_name(node.attr("local-cache") || node.attr("local-cache-only")) - if node.attr("flush-caches") + if @flush_caches FileUtils.rm_f @iev_globalname unless @iev_globalname.nil? FileUtils.rm_f @iev_localname unless @iev_localname.nil? end diff --git a/lib/metanorma/standoc/term_lookup_cleanup.rb b/lib/metanorma/standoc/term_lookup_cleanup.rb index ae42ad02..77b5326a 100644 --- a/lib/metanorma/standoc/term_lookup_cleanup.rb +++ b/lib/metanorma/standoc/term_lookup_cleanup.rb @@ -4,7 +4,7 @@ module Metanorma module Standoc # Intelligent term lookup xml modifier class TermLookupCleanup - AUTO_GEN_ID_REGEXP = /\A_/.freeze + AUTO_GEN_ID_REGEXP = /\A_/ attr_reader :xmldoc, :lookup, :log @@ -130,7 +130,7 @@ def remove_missing_ref_msg(node, target, type) end def remove_missing_ref_msg1(_node, target, ret) - target2 = "_#{target.downcase.gsub('-', '_')}" + target2 = "_#{target.downcase.tr('-', '_')}" if @terms_tags[target] || @terms_tags[target2] ret.strip! ret += ". Did you mean to point to a subterm?" diff --git a/lib/metanorma/standoc/utils.rb b/lib/metanorma/standoc/utils.rb index df518fcd..e069b1d4 100644 --- a/lib/metanorma/standoc/utils.rb +++ b/lib/metanorma/standoc/utils.rb @@ -100,8 +100,9 @@ def xml_encode(text) # wrapped in def adoc2xml(text, flavour) Nokogiri::XML(text).root and return text + f = @flush_caches ? ":flush-caches:\n" : "" c = Asciidoctor.convert("= X\nA\n:semantic-metadata-headless: true\n" \ - ":novalid:\n\n#{text}\n", + ":no-isobib:\n#{f}:novalid:\n\n#{text}\n", backend: flavour, header_footer: true) Nokogiri::XML(c).at("//xmlns:sections") end diff --git a/spec/metanorma/cleanup_boilerplate_spec.rb b/spec/metanorma/cleanup_boilerplate_spec.rb index a072e1e4..d3001536 100644 --- a/spec/metanorma/cleanup_boilerplate_spec.rb +++ b/spec/metanorma/cleanup_boilerplate_spec.rb @@ -902,22 +902,51 @@ === Term1 + [bibliography] + == Bibliography + * [[[iso1234,A]]] + * [[[iso5678,B]]] INPUT output = <<~OUTPUT - #{BLANK_HDR} + #{BLANK_HDR} Foreword

Foreword

-
+ + - Terms and definitions

For the purposes of this document, the terms and definitions - given in and and the following apply.

- - Term1 - -
- + Terms and definitions +

+ For the purposes of this document, the terms and definitions given in + + and + + and the following apply. +

+ + + + Term1 + + + + +
+ + + Bibliography + + [NO INFORMATION AVAILABLE] + A + + + [NO INFORMATION AVAILABLE] + B + + + + OUTPUT expect(Xml::C14n.format(strip_guid(Asciidoctor.convert(input, *OPTIONS)))) .to be_equivalent_to Xml::C14n.format(output) @@ -958,6 +987,10 @@ [source="iso1234,iso5678"] == Terms and Definitions + [bibliography] + == Bibliography + * [[[iso1234,A]]] + * [[[iso5678,B]]] INPUT output = <<~OUTPUT #{BLANK_HDR} @@ -965,13 +998,33 @@ Foreword

Foreword

-
- - Terms and definitions -

For the purposes of this document, - the terms and definitions given in and apply.

-
- + + + + Terms and definitions +

+ For the purposes of this document, the terms and definitions given in + + and + + apply. +

+
+
+ + + Bibliography + + [NO INFORMATION AVAILABLE] + A + + + [NO INFORMATION AVAILABLE] + B + + + + OUTPUT expect(Xml::C14n.format(strip_guid(Asciidoctor.convert(input, *OPTIONS)))) .to be_equivalent_to Xml::C14n.format(output) @@ -992,6 +1045,10 @@ [source="iso1234,iso5678"] == Terms and Definitions + [bibliography] + == Bibliography + * [[[iso1234,A]]] + * [[[iso5678,B]]] INPUT output = <<~OUTPUT #{BLANK_HDR.sub(%r{en}, 'fr')} @@ -999,12 +1056,33 @@ Avant-propos

Foreword

-
- - Termes et définitions -

Pour les besoins du présent document, les termes et définitions de et s’appliquent.

-
- + + + + Termes et définitions +

+ Pour les besoins du présent document, les termes et définitions de + + et + + s’appliquent. +

+
+
+ + + Bibliographie + + [PAS D’INFORMATION DISPONIBLE] + A + + + [PAS D’INFORMATION DISPONIBLE] + B + + + + OUTPUT expect(Xml::C14n.format(strip_guid(Asciidoctor.convert(input, *OPTIONS)))) .to be_equivalent_to Xml::C14n.format(output) @@ -1026,17 +1104,42 @@ [source="iso1234,iso5678"] == Terms and Definitions + [bibliography] + == Bibliography + * [[[iso1234,A]]] + * [[[iso5678,B]]] INPUT output = <<~OUTPUT #{BLANK_HDR.sub(%r{en}, 'zh').sub(%r{}, '')} 前言

Foreword

-
- - 术语和定义

界定的术语和定义适用于本文件。

-
- + + + + 术语和定义 +

+ + 和 + + 界定的术语和定义适用于本文件。 +

+
+
+ + + 参考文献 + + [无资料] + A + + + [无资料] + B + + + + OUTPUT expect(Xml::C14n.format(strip_guid(Asciidoctor.convert(input, *OPTIONS)))) .to be_equivalent_to Xml::C14n.format(output) diff --git a/spec/metanorma/cleanup_spec.rb b/spec/metanorma/cleanup_spec.rb index 92dfab85..dbd4ec9f 100644 --- a/spec/metanorma/cleanup_spec.rb +++ b/spec/metanorma/cleanup_spec.rb @@ -692,8 +692,8 @@

- - http://www.example.com + + http://www.example.com

diff --git a/spec/metanorma/isobib_cache_spec.rb b/spec/metanorma/isobib_cache_spec.rb index e6484977..308eac25 100644 --- a/spec/metanorma/isobib_cache_spec.rb +++ b/spec/metanorma/isobib_cache_spec.rb @@ -364,7 +364,7 @@ FileUtils.mv iev_file, iev_file1 if File.exist? iev_file File.write("#{Dir.home}/.relaton/cache", "XXX") - FileUtils.rm_rf File.expand_path("~/.iev/cache") + FileUtils.rm_rf File.expand_path("#{Dir.home}/.iev/cache") # mock_isobib_get_123 VCR.use_cassette("isobib_get_123_2001_and_iev", diff --git a/spec/metanorma/validate_spec.rb b/spec/metanorma/validate_spec.rb index 4c9b7d11..7f2d9ad1 100644 --- a/spec/metanorma/validate_spec.rb +++ b/spec/metanorma/validate_spec.rb @@ -896,7 +896,7 @@ rescue SystemExit end expect(File.read("test.err.html")) - .to include(%(Term reference to Terms-and-Definitions missing: "Terms-and-Definitions" is not defined in document.​ Did you mean to point to a subterm)) + .to include(%(Term reference to Terms-and-Definitions missing: "Terms-and-Definitions" is not defined in document)) expect(File.read("test.err.html")) .to include("Concept term1 is pointing to jkl, which is not a term or symbol. Did you mean to point to a subterm?") expect(File.read("test.err.html"))