Merge pull request #124 from ruby-i18n/movermeyer/draft_level

Filter data by draft status

CHANGELOG.md

@@ -34,6 +34,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Standardize component names for the `thor cldr:export` command (and internally in the codebase), [#121](https://github.com/ruby-i18n/ruby-cldr/pull/121)
 - Standardize locale names for the `thor cldr:export` command (and internally in the codebase), [#121](https://github.com/ruby-i18n/ruby-cldr/pull/121)
 - Output `plurals.rb` with the `ruby-cldr` style locale codes (only affects `pt-PT` in CLDR v34), [#121](https://github.com/ruby-i18n/ruby-cldr/pull/121)
+- Export data at with a consistent minimum draft status, [#124](https://github.com/ruby-i18n/ruby-cldr/pull/124)
+- Add `--draft-status` flag for specifying the minimum draft status for data to be exported, [#124](https://github.com/ruby-i18n/ruby-cldr/pull/124)
 
 ---
 

lib/cldr.rb

@@ -3,10 +3,11 @@
 require "core_ext/hash/symbolize_keys"
 
 module Cldr
-  autoload :Data,   "cldr/data"
-  autoload :Export, "cldr/export"
-  autoload :Locale, "cldr/locale"
-  autoload :Format, "cldr/format"
+  autoload :Data,         "cldr/data"
+  autoload :DraftStatus,  "cldr/draft_status"
+  autoload :Export,       "cldr/export"
+  autoload :Locale,       "cldr/locale"
+  autoload :Format,       "cldr/format"
 
   class << self
     def fallbacks

lib/cldr/draft_status.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Cldr
+  module DraftStatus
+    class Status
+      include Comparable
+
+      def initialize(name, value)
+        @name = name
+        @value = value
+      end
+
+      def <=>(other)
+        @value <=> other.value
+      end
+
+      def to_s
+        @name.to_s
+      end
+
+      protected
+
+      attr_reader :value
+    end
+
+    ALL = [
+      :unconfirmed,
+      :provisional,
+      :contributed,
+      :approved,
+    ].map.with_index do |name, index|
+      const_set(name.upcase, Status.new(name, index))
+    end.freeze
+
+    ALL_BY_NAME = ALL.to_h { |status| [status.to_s, status] }.freeze
+    private_constant :ALL_BY_NAME
+
+    def self.fetch(name)
+      ALL_BY_NAME.fetch(name.to_s)
+    end
+  end
+end

lib/cldr/export.rb

@@ -12,10 +12,11 @@
 
 module Cldr
   module Export
-    autoload :Code, "cldr/export/code"
-    autoload :Data, "cldr/export/data"
-    autoload :Ruby, "cldr/export/ruby"
-    autoload :Yaml, "cldr/export/yaml"
+    autoload :Code,     "cldr/export/code"
+    autoload :Data,     "cldr/export/data"
+    autoload :DataFile, "cldr/export/data_file"
+    autoload :Ruby,     "cldr/export/ruby"
+    autoload :Yaml,     "cldr/export/yaml"
 
     SHARED_COMPONENTS = [
       :Aliases, :CountryCodes, :CurrencyDigitsAndRounding, :LikelySubtags,
@@ -35,9 +36,19 @@ def base_path=(base_path)
         @@base_path = File.expand_path(base_path)
       end
 
+      def minimum_draft_status
+        raise StandardError, "minimum_draft_status is not yet set." unless defined?(@@minimum_draft_status)
+        @@minimum_draft_status
+      end
+
+      def minimum_draft_status=(draft_status)
+        @@minimum_draft_status = draft_status
+      end
+
       def export(options = {}, &block)
         locales        = options[:locales] || Data.locales
         components     = options[:components] || Data.components
+        self.minimum_draft_status = options[:minimum_draft_status] if options[:minimum_draft_status]
         self.base_path = options[:target] if options[:target]
 
         shared_components, locale_components = components.partition do |component|

lib/cldr/export/data/base.rb

@@ -27,12 +27,7 @@ def []=(key, value)
 
         protected
 
-        def draft?(node)
-          draft = node.attribute("draft")
-          draft && draft.value == "unconfirmed"
-        end
-
-        def alt?(node)
+        def alt?(node) # TODO: Move this into DataFile
           !node.attribute("alt").nil?
         end
 
@@ -68,26 +63,13 @@ def paths
         private
 
         def merge_paths(paths_to_merge)
-          # Some parts (`ldml`, `ldmlBCP47` amd `supplementalData`) of CLDR data require that you merge all the
-          # files with the same root element before doing lookups.
-          # Ref: https://www.unicode.org/reports/tr35/tr35.html#XML_Format
-          #
-          # The return of this method is a merged XML Nokogiri document.
-          # Note that it technically is no longer compliant with the CLDR `ldml.dtd`, since:
-          # * it has repeated elements
-          # * the <identity> elements no longer refer to the filename
-          #
-          # However, this is not an issue, since #select will find all of the matches from each of the repeated elements,
-          # and the <identity> elements are not important to us / make no sense when combined together.
-          return Nokogiri::XML("") if paths_to_merge.empty?
-
-          rest = paths_to_merge[1..paths_to_merge.size - 1]
-          rest.each_with_object(Nokogiri::XML(File.read(paths_to_merge.first))) do |path, result|
-            next_doc = Nokogiri::XML(File.read(path))
-
-            next_doc.root.children.each do |child|
-              result.root.add_child(child)
-            end
+          return Cldr::Export::DataFile.new(Nokogiri::XML("")) if paths_to_merge.empty?
+
+          first = Cldr::Export::DataFile.parse(File.read(paths_to_merge.first))
+          rest = paths_to_merge[1..]
+          rest.reduce(first) do |result, path|
+            parsed = Cldr::Export::DataFile.parse(File.read(path))
+            result.merge(parsed)
           end
         end
       end

lib/cldr/export/data/currencies.rb

@@ -20,13 +20,11 @@ def currencies
 
         def currency(node)
           data = select(node, "displayName").each_with_object({}) do |node, result|
-            unless draft?(node)
-              if node.attribute("count")
-                count = node.attribute("count").value.to_sym
-                result[count] = node.content
-              else
-                result[:name] = node.content
-              end
+            if node.attribute("count")
+              count = node.attribute("count").value.to_sym
+              result[count] = node.content
+            else
+              result[:name] = node.content
             end
           end
 

lib/cldr/export/data/currency_digits_and_rounding.rb

@@ -10,7 +10,7 @@ def initialize
           super
 
           path = "#{Cldr::Export::Data.dir}/supplemental/supplementalData.xml"
-          doc = File.open(path) { |file| Nokogiri::XML(file) }
+          doc = Cldr::Export::DataFile.parse(File.read(path))
 
           doc.xpath("//currencyData/fractions/info").each do |node|
             code = node.attr("iso4217")

lib/cldr/export/data/languages.rb

@@ -13,7 +13,7 @@ def initialize(locale)
 
         def languages
           @languages ||= select("localeDisplayNames/languages/language").each_with_object({}) do |node, result|
-            result[Cldr::Export.to_i18n(node.attribute("type").value)] = node.content unless draft?(node) || alt?(node)
+            result[Cldr::Export.to_i18n(node.attribute("type").value)] = node.content unless alt?(node)
           end
         end
       end

lib/cldr/export/data/metazones.rb

@@ -11,7 +11,7 @@ def initialize
           super
 
           path = "#{Cldr::Export::Data.dir}/supplemental/metaZones.xml"
-          doc = File.open(path) { |file| Nokogiri::XML(file) }
+          doc = Cldr::Export::DataFile.parse(File.read(path))
           self[:timezones] = doc.xpath("//metaZones/metazoneInfo/timezone").each_with_object({}) do |node, result|
             timezone = node.attr("type").to_sym
             result[timezone] = metazone(node.xpath("usesMetazone"))

lib/cldr/export/data/numbers.rb

@@ -42,7 +42,7 @@ def currency
 
         def symbols
           select("numbers/symbols/*").each_with_object({}) do |node, result|
-            result[name(node).to_sym] = node.content unless draft?(node)
+            result[name(node).to_sym] = node.content
           end
         end
 
@@ -60,8 +60,6 @@ def format(type)
 
                   pattern_count_node = pattern_node.attribute("count")
 
-                  next if draft?(pattern_node)
-
                   pattern_key = pattern_key_node ? pattern_key_node.value : :default
 
                   if pattern_count_node

lib/cldr/export/data/parent_locales.rb

@@ -10,7 +10,7 @@ def initialize
           super
 
           path = File.join(Cldr::Export::Data.dir, "supplemental", "supplementalData.xml")
-          doc = File.open(path) { |file| Nokogiri::XML(file) }
+          doc = Cldr::Export::DataFile.parse(File.read(path))
 
           doc.xpath("//parentLocales/parentLocale").each do |node|
             parent = Cldr::Export.to_i18n(node.attr("parent"))

lib/cldr/export/data/plural_rules.rb

@@ -22,9 +22,7 @@ def initialize(locale)
 
         def sources
           @sources ||= ["plurals", "ordinals"].each_with_object({}) do |source_name, ret|
-            ret[source_name] = ::Nokogiri::XML(
-              File.read("#{Cldr::Export::Data.dir}/supplemental/#{source_name}.xml")
-            )
+            ret[source_name] = Cldr::Export::DataFile.parse(File.read("#{Cldr::Export::Data.dir}/supplemental/#{source_name}.xml"))
           end
         end
 

lib/cldr/export/data/plurals.rb

@@ -15,11 +15,7 @@ class Plurals < Hash
 
         class << self
           def rules
-            @@rules ||= Rules.parse(source)
-          end
-
-          def source
-            File.read("#{Cldr::Export::Data.dir}/supplemental/plurals.xml")
+            @@rules ||= Rules.parse(File.read("#{Cldr::Export::Data.dir}/supplemental/plurals.xml"))
           end
         end
 

lib/cldr/export/data/plurals/rules.rb

@@ -10,7 +10,7 @@ class Plurals
         class Rules < Array
           class << self
             def parse(xml)
-              doc = Nokogiri.XML(xml)
+              doc = Cldr::Export::DataFile.parse(xml)
 
               rules = new
               doc.xpath("//pluralRules").each do |node|

lib/cldr/export/data/subdivisions.rb

@@ -13,7 +13,7 @@ def initialize(locale)
 
         def subdivisions
           @subdivisions ||= select("localeDisplayNames/subdivisions/subdivision").each_with_object({}) do |node, result|
-            result[node.attribute("type").value.to_sym] = node.content unless draft?(node) || alt?(node)
+            result[node.attribute("type").value.to_sym] = node.content unless alt?(node)
           end
         end
       end

lib/cldr/export/data/territories.rb

@@ -13,7 +13,7 @@ def initialize(locale)
 
         def territories
           @territories ||= select("localeDisplayNames/territories/territory").each_with_object({}) do |node, result|
-            result[node.attribute("type").value.to_sym] = node.content unless draft?(node) || alt?(node)
+            result[node.attribute("type").value.to_sym] = node.content unless alt?(node)
           end
         end
       end

lib/cldr/export/data/units.rb

@@ -31,7 +31,7 @@ def units(node)
         def unit(node)
           node.xpath("unitPattern").each_with_object({}) do |node, result|
             count = node.attribute("count") ? node.attribute("count").value.to_sym : :one
-            result[count] = node.content unless draft?(node)
+            result[count] = node.content
           end
         end
 

lib/cldr/export/data/windows_zones.rb

@@ -11,7 +11,7 @@ def initialize
           super
 
           path = "#{Cldr::Export::Data.dir}/supplemental/windowsZones.xml"
-          doc = File.open(path) { |file| Nokogiri::XML(file) }
+          doc = Cldr::Export::DataFile.parse(File.read(path))
           doc.xpath("//windowsZones/mapTimezones/mapZone").each_with_object(self) do |node, result|
             zone = node.attr("other").to_s
             territory = node.attr("territory")

lib/cldr/export/data_file.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require "nokogiri"
+
+module Cldr
+  module Export
+    class DataFile
+      class << self
+        def parse(string, minimum_draft_status: nil)
+          doc = Nokogiri::XML(string) do |config|
+            config.strict.noblanks
+          end
+          DataFile.new(doc, minimum_draft_status: minimum_draft_status)
+        end
+
+        def filter_by_draft(doc, minimum_draft_status)
+          doc.traverse do |child|
+            next unless child.text?
+
+            draft_status = child.parent.attribute("draft").nil? ? Cldr::DraftStatus::APPROVED : Cldr::DraftStatus.fetch(child.parent.attribute("draft"))
+            if draft_status < minimum_draft_status
+              ancestors = child.ancestors
+              child.remove
+              # Remove the ancestors that are now empty
+              ancestors.each do |ancestor|
+                ancestor.remove if ancestor.children.empty?
+              end
+            end
+          end
+          doc
+        end
+      end
+
+      attr_reader :doc, :minimum_draft_status
+
+      def initialize(doc, minimum_draft_status: nil)
+        @minimum_draft_status = minimum_draft_status || Cldr::Export.minimum_draft_status
+        @doc = Cldr::Export::DataFile.filter_by_draft(doc, @minimum_draft_status)
+      end
+
+      def traverse(&block)
+        @doc.traverse(&block)
+      end
+
+      def xpath(path)
+        @doc.xpath(path)
+      end
+
+      def /(*args)
+        @doc./(*args)
+      end
+
+      def locale
+        language = @doc.xpath("//ldml/identity/language").first&.attribute("type")&.value
+        territory = @doc.xpath("//ldml/identity/territory").first&.attribute("type")&.value
+        elements = [language, territory].compact
+        elements.empty? ? nil : elements.join("-").to_sym
+      end
+
+      def merge(other)
+        # Some parts (`ldml`, `ldmlBCP47` amd `supplementalData`) of CLDR data require that you merge all the
+        # files with the same root element before doing lookups.
+        # Ref: https://www.unicode.org/reports/tr35/tr35.html#XML_Format
+        #
+        # Note that it technically is no longer compliant with the CLDR `ldml.dtd`, since:
+        # * it has repeated elements
+        # * the <identity> elements no longer refer to the filename
+        #
+        # However, this is not an issue, since #xpath will find all of the matches from each of the repeated elements,
+        # and the <identity> elements are not important to us / make no sense when combined together.
+        raise StandardError, "Cannot merge data file with more permissive draft status" if other.minimum_draft_status < minimum_draft_status
+        raise StandardError, "Cannot merge data file from different locales" if other.locale != locale
+
+        result = @doc.dup
+        other.doc.root.children.each do |child|
+          result.root.add_child(child.dup)
+        end
+
+        Cldr::Export::DataFile.new(result, minimum_draft_status: minimum_draft_status)
+      end
+    end
+  end
+end