Support Bulk Download from GlotPress

lib/fastlane/plugin/wpmreleasetoolkit/helper/android/android_strings_file_writer.rb

@@ -0,0 +1,29 @@
+require 'fastlane_core/ui/ui'
+require 'fileutils'
+
+module Fastlane
+  module Helper
+    module Android
+      module StringsFileWriter
+        # @param [String] dir path to destination directory
+        # @param [Locale] locale the locale to write the file for
+        # @param [File, IO] io The File IO containing the translations downloaded from GlotPress
+        def self.write_app_translations_file(dir:, locale:, io:)
+          # `dir` is typically `src/main/res/` here
+          return unless Locale.valid?(locale, :android)
+
+          dest = File.join(dir, locale.android_path)
+          FileUtils.mkdir_p(File.dirname(dest))
+
+          # TODO: reorder XML nodes alphabetically, for easier diffs
+          #   xml = Nokogiri::XML(io, nil, Encoding::UTF_8.to_s)
+          #   # … reorder nodes …
+          #   File.open(main, 'w:UTF-8') { |f| f.write(xml.to_xml(indent: 4)) }
+          # FIXME: For now, just copy blindly until we get time to implement node reordering
+          UI.message("Writing: #{dest}")
+          IO.copy_stream(io, dest)
+        end
+      end
+    end
+  end
+end

lib/fastlane/plugin/wpmreleasetoolkit/helper/fastlane_metadata_file_writer.rb

@@ -0,0 +1,102 @@
+require 'fastlane_core/ui/ui'
+require 'fileutils'
+
+module Fastlane
+  module Helper
+    module FastlaneMetadataFilesWriter
+
+      # A model/struct defining a rule on how to process and map metadata from GlotPress into txt files
+      #
+      # @param [String] key The key in the GlotPress export for the metadata
+      # @param [Int] max_len The maximum length allowed by the App Store / Play Store for that key.
+      #        Note: If the translation for `key` exceeds the specified `max_len`, we will try to find an alternate key named `#{key}_short` by convention.
+      # @param [String] filename The (relative) path to the `.txt` file to write that metadata to
+      #
+      MetadataRule = Struct.new(:key, :max_len, :filename) do
+        # The common standardized set of Metadata rules for an Android project
+        def self.android_rules(version_name:, version_code:)
+          suffix = version_name.gsub('.', '')
+          [
+            MetadataRule.new("release_note_#{suffix}", 500, File.join('changelogs', "#{version_code}.txt")),
+            MetadataRule.new('play_store_app_title', 30, 'title.txt'),
+            MetadataRule.new('play_store_promo', 80, 'short_description.txt'),
+            MetadataRule.new('play_store_desc', 4000, 'full_description.txt'),
+          ]
+        end
+
+        # The common standardized set of Metadata rules for an Android project
+        def self.ios_rules(version_name:)
+          suffix = version_name.gsub('.', '')
+          [
+            MetadataRule.new("release_note_#{suffix}", 4000, 'release_notes.txt'),
+            MetadataRule.new('app_store_name', 30, 'name.txt'),
+            MetadataRule.new('app_store_subtitle', 30, 'subtitle.txt'),
+            MetadataRule.new('app_store_description', 4000, 'description.txt'),
+            MetadataRule.new('app_store_keywords', 100, 'keywords.txt'),
+          ]
+        end
+      end
+
+      # Visit each key/value pair of a translations Hash, and yield keys and matching translations from it based on the passed `MetadataRules`,
+      # trying any potential fallback key if the translation exceeds the max limit, and yielding each found and valid entry to the caller.
+      #
+      # @param [#read] io
+      # @param [Array<MetadataRule>] rules List of rules for each key
+      # @param [Block] rule_for_unknown_key An optional block called when a key that does not match any of the rules is encountered.
+      #        The block will receive a [String] (key) and must return a `MetadataRule` instance (or nil)
+      #
+      # @yield [String, MetadataRule, String] yield each (key, matching_rule, value) tuple found in the JSON, after resolving alternates for values exceeding max length
+      #        Note that if both translations for the key and its (optional) shorter alternate exceeds the max_len, it will still `yield` but with a `nil` value
+      #
+      def self.visit(translations:, rules:, rule_for_unknown_key:)
+        translations.each do |key, value|
+          next if key.nil? || key.end_with?('_short') # skip if alternate key
+
+          rule = rules.find { |r| r.key == key }
+          rule = rule_for_unknown_key.call(key) if rule.nil? && !rule_for_unknown_key.nil?
+          next if rule.nil?
+
+          if rule.max_len != nil && value.length > rule.max_len
+            UI.warning "Translation for #{key} is too long (#{value.length}), trying shorter alternate #{key}."
+            short_key = "#{key}_short"
+            value = json[short_key]
+            if value.nil?
+              UI.warning "No shorter alternate (#{short_key}) available, skipping entirely."
+              yield key, rule, nil
+              next
+            end
+            if value.length > rule.max_len
+              UI.warning "Translation alternate for #{short_key} was too long too (#{value.length}), skipping entirely."
+              yield short_key, rule, nil
+              next
+            end
+          end
+          yield key, rule, value
+        end
+      end
+
+      # Write the `.txt` files to disk for the given exported translation file (typically a JSON export) based on the `MetadataRules` provided
+      #
+      # @param [String] locale_dir the path to the locale directory (e.g. `fastlane/metadata/android/fr`) to write the `.txt` files to
+      # @param [Hash<String,String>] translations The hash of translations (key => translation) to visit based on `MetadataRules` then write to disk.
+      # @param [Array<MetadaataRule>] rules The list of fixed `MetadataRule` to use to extract the expected metadata from the `translations`
+      # @param [Block] rule_for_unknown_key An optional block called when a key that does not match any of the rules is encountered.
+      #        The block will receive a [String] (key) and must return a `MetadataRule` instance (or nil)
+      #
+      def self.write(locale_dir:, translations:, rules:, &rule_for_unknown_key)
+        self.visit(translations: translations, rules: rules, rule_for_unknown_key: rule_for_unknown_key) do |_key, rule, value|
+          dest = File.join(locale_dir, rule.filename)
+          if value.nil? && File.exist?(dest)
+            # Key found in JSON was rejected for being too long. Delete file
+            UI.verbose("Deleting file #{dest}")
+            FileUtils.rm(dest)
+          elsif value
+            UI.verbose("Writing file #{dest}")
+            FileUtils.mkdir_p(File.dirname(dest))
+            File.write(dest, value.chomp)
+          end
+        end
+      end
+    end
+  end
+end

lib/fastlane/plugin/wpmreleasetoolkit/helper/glotpress_downloader.rb

@@ -0,0 +1,93 @@
+require 'fastlane_core/ui/ui'
+require 'json'
+require 'open-uri'
+require 'zip'
+
+module Fastlane
+  module Helper
+    class GPDownloader
+      REQUEST_HEADERS = { 'User-Agent' => Wpmreleasetoolkit::USER_AGENT }
+
+      module FORMAT
+        ANDROID = 'android'
+        IOS = 'strings'
+        JSON = 'json'
+      end
+
+      # The host of the GlotPress instance. e.g. `'translate.wordpress.org'`
+      attr_accessor :host
+      # The path of the project in GlotPress. e.g. `'apps/ios/release-notes'`
+      attr_accessor :project
+
+      def initialize(host:, project:)
+        @host = host
+        @project = project
+      end
+
+      # @param [String] gp_locale
+      # @param [String] format Typically `'android'`, `'strings'` or `'json'`
+      # @param [Hash<String,String>] filters
+      #
+      # @yield [IO] the corresponding downloaded IO content
+      #
+      # @note For this case, `project_url` is on the form 'https://translate.wordpress.org/projects/apps/ios/release-notes'
+      def download_locale(gp_locale:, format:, filters: { status: 'current'})
+        query_params = filters.transform_keys { |k| "filters[#{k}]" }.merge(format: format)
+        uri = URI::HTTPS.build(host: host, path: File.join('/', 'projects', project, gp_locale, 'default', 'export-translations'), query: URI.encode_www_form(query_params))
+
+        UI.message "Downloading #{uri}"
+        io = begin
+          uri.open(REQUEST_HEADERS)
+        rescue StandardError => e
+          UI.error "Error downloading #{gp_locale} - #{e.message}"
+          return
+        end
+        UI.message "Download done."
+        yield io
+      end
+
+      # @param [String] format Typically `'android'`, `'strings'` or `'json'`
+      # @param [Hash<String,String>] filters
+      #
+      # @yield For each locale, a tuple of [String], [IO] corresponding to the glotpress locale code and IO content
+      #
+      # @note requires the GlotPress instance to have the Bulk Downloader plugin installed
+      # @note For this case, `project_url` is on the form 'https://translate.wordpress.org/exporter/apps/android/dev/'
+      def download_all_locales(format:, filters: { status: 'current'})
+        query_params = filters.transform_keys { |k| "filters[#{k}]" }.merge('export-format': format)
+        uri = URI::HTTPS.build(host: host, path: File.join('/', 'exporter', project, '-do'), query: URI.encode_www_form(query_params))
+        UI.message "Downloading #{uri}"
+        zip_stream = uri.open(REQUEST_HEADERS)
+        UI.message "Download done."
+
+        Zip::File.open_buffer(zip_stream) do |zip_file|
+          zip_file.each do |entry|
+            next if entry.name.end_with?('/') && entry.size.zero?
+
+            prefix = File.dirname(entry.name).gsub(/[0-9-]*$/, '') + '-'
+            locale = File.basename(entry.name, File.extname(entry.name)).delete_prefix(prefix)
+            UI.message "- Found locale in ZIP: #{locale}"
+
+            yield locale, entry.get_input_stream
+          end
+        end
+      end
+
+      # Takes a GlotPress JSON export and transform it to a simple `Hash` of key => value pairs
+      #
+      # Since the JSON format for GlotPress exports is a bit odd, with JSON keys actually being a concatenation of actual
+      # copy key and source copy, and values being an array, this allows us to convert this odd export format to a more
+      # usable structure.
+      #
+      # @param [#read] io The `File` or `IO` to read the JSON data exported from GlotPress
+      def parse_json_export(io:)
+        json = JSON.parse(io.read)
+        json.map do |composite_key, values|
+          key = composite_key.split(/\u0004/).first # composite_key is a concatenation of key + \u0004 + source]
+          value = values.first # Each value in the JSON Hash is an Array of all the translations; but if we provided the right filter, the first one should always be the right one
+          [key, value]
+        end.to_h
+      end
+    end # class
+  end # module
+end # module

lib/fastlane/plugin/wpmreleasetoolkit/helper/ios/ios_strings_file_writer.rb

@@ -0,0 +1,24 @@
+require 'fastlane_core/ui/ui'
+require 'fileutils'
+
+module Fastlane
+  module Helper
+    module Ios
+      module StringsFileWriter
+        # @param [String] dir path to destination directory
+        # @param [Locale] locale the locale to write the file for
+        # @param [File, IO] io The File IO containing the translations downloaded from GlotPress
+        def self.write_app_translations_file(dir:, locale:, io:)
+          # `dir` is typically `WordPress/Resources/` here
+          return unless Locale.valid?(locale, :ios)
+
+          dest = File.join(dir, locale.ios_path)
+          FileUtils.mkdir_p(File.dirname(dest))
+          UI.message("Writing: #{dest}")
+          IO.copy_stream(io, dest)
+        end
+      end
+    end
+  end
+end
+

lib/fastlane/plugin/wpmreleasetoolkit/models/locale.rb

@@ -0,0 +1,29 @@
+require 'fastlane_core/ui/ui'
+
+module Fastlane
+  module Wpmreleasetoolkit
+    Locale = Struct.new(:glotpress, :android, :playstore, :ios, :appstore, keyword_init: true) do
+      def android_path
+        File.join("values-#{self.android}", 'strings.xml')
+      end
+
+      def ios_path
+        File.join("#{self.ios}.lproj", 'Localizable.strings')
+      end
+
+      def self.valid?(locale, *keys)
+        if locale.nil?
+          UI.warning("Locale is unknown")
+          return false
+        end
+        keys.each do |key|
+          if locale[key].nil?
+            UI.warning("Locale #{locale} is missing required key #{key}")
+            return false
+          end
+        end
+        return true
+      end
+    end
+  end
+end