Merge pull request #3466 from huginn/enhance_google_translation_agent

Enhance GoogleTranslationAgent

Gemfile

@@ -47,7 +47,7 @@ gem 'weibo_2', github: 'albertsun/weibo_2', branch: 'master'
 
 # GoogleCalendarPublishAgent and GoogleTranslateAgent
 gem 'google-api-client', '~> 0.53'
-gem 'google-cloud-translate', '~> 2.3', require: 'google/cloud/translate'
+gem 'google-cloud-translate-v2'
 
 # Twitter Agents
 gem 'omniauth-twitter'

Gemfile.lock

@@ -345,36 +345,39 @@ GEM
       google-apis-core (>= 0.11.0, < 2.a)
       google-apis-discovery_v1 (~> 0.5)
       thor (>= 0.20, < 2.a)
-    google-cloud-core (1.6.1)
+    google-cloud-core (1.7.1)
       google-cloud-env (>= 1.0, < 3.a)
       google-cloud-errors (~> 1.0)
-    google-cloud-env (1.6.0)
-      faraday (>= 0.17.3, < 3.0)
-    google-cloud-errors (1.3.1)
-    google-cloud-translate (2.3.0)
-      faraday (>= 0.17.3, < 2.0)
-      google-cloud-core (~> 1.2)
-      google-gax (~> 1.8)
-      googleapis-common-protos (>= 1.3.9, < 2.0)
-      googleapis-common-protos-types (>= 1.0.4, < 2.0)
-    google-gax (1.8.2)
-      google-protobuf (~> 3.9)
-      googleapis-common-protos (>= 1.3.9, < 2.0)
-      googleapis-common-protos-types (>= 1.0.4, < 2.0)
-      googleauth (~> 0.9)
-      grpc (~> 1.24)
-      rly (~> 0.2.3)
-    google-protobuf (3.25.3)
-    google-protobuf (3.25.3-aarch64-linux)
-    google-protobuf (3.25.3-arm64-darwin)
-    google-protobuf (3.25.3-x86_64-darwin)
-    google-protobuf (3.25.3-x86_64-linux)
-    googleapis-common-protos (1.4.0)
-      google-protobuf (~> 3.14)
-      googleapis-common-protos-types (~> 1.2)
-      grpc (~> 1.27)
-    googleapis-common-protos-types (1.6.0)
-      google-protobuf (~> 3.14)
+    google-cloud-env (2.2.1)
+      faraday (>= 1.0, < 3.a)
+    google-cloud-errors (1.4.0)
+    google-cloud-translate-v2 (1.0.0)
+      faraday (>= 1.0, < 3.a)
+      google-cloud-core (~> 1.6)
+      googleapis-common-protos (>= 1.3.10, < 2.a)
+      googleapis-common-protos-types (>= 1.0.5, < 2.a)
+      googleauth (>= 0.16.2, < 2.a)
+    google-protobuf (4.28.3)
+      bigdecimal
+      rake (>= 13)
+    google-protobuf (4.28.3-aarch64-linux)
+      bigdecimal
+      rake (>= 13)
+    google-protobuf (4.28.3-arm64-darwin)
+      bigdecimal
+      rake (>= 13)
+    google-protobuf (4.28.3-x86_64-darwin)
+      bigdecimal
+      rake (>= 13)
+    google-protobuf (4.28.3-x86_64-linux)
+      bigdecimal
+      rake (>= 13)
+    googleapis-common-protos (1.6.0)
+      google-protobuf (>= 3.18, < 5.a)
+      googleapis-common-protos-types (~> 1.7)
+      grpc (~> 1.41)
+    googleapis-common-protos-types (1.16.0)
+      google-protobuf (>= 3.18, < 5.a)
     googleauth (0.17.1)
       faraday (>= 0.17.3, < 2.0)
       jwt (>= 1.4, < 3.0)
@@ -382,20 +385,20 @@ GEM
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
       signet (~> 0.15)
-    grpc (1.62.0)
-      google-protobuf (~> 3.25)
+    grpc (1.67.0)
+      google-protobuf (>= 3.25, < 5.0)
       googleapis-common-protos-types (~> 1.0)
-    grpc (1.62.0-aarch64-linux)
-      google-protobuf (~> 3.25)
+    grpc (1.67.0-aarch64-linux)
+      google-protobuf (>= 3.25, < 5.0)
       googleapis-common-protos-types (~> 1.0)
-    grpc (1.62.0-arm64-darwin)
-      google-protobuf (~> 3.25)
+    grpc (1.67.0-arm64-darwin)
+      google-protobuf (>= 3.25, < 5.0)
       googleapis-common-protos-types (~> 1.0)
-    grpc (1.62.0-x86_64-darwin)
-      google-protobuf (~> 3.25)
+    grpc (1.67.0-x86_64-darwin)
+      google-protobuf (>= 3.25, < 5.0)
       googleapis-common-protos-types (~> 1.0)
-    grpc (1.62.0-x86_64-linux)
-      google-protobuf (~> 3.25)
+    grpc (1.67.0-x86_64-linux)
+      google-protobuf (>= 3.25, < 5.0)
       googleapis-common-protos-types (~> 1.0)
     guard (2.18.0)
       formatador (>= 0.2.4)
@@ -680,7 +683,6 @@ GEM
       netrc (~> 0.8)
     retriable (3.1.2)
     rexml (3.3.9)
-    rly (0.2.3)
     rouge (4.1.1)
     rr (3.1.0)
     rspec (3.12.0)
@@ -888,7 +890,7 @@ DEPENDENCIES
   geokit-rails (~> 2.5)
   gmail_xoauth
   google-api-client (~> 0.53)
-  google-cloud-translate (~> 2.3)
+  google-cloud-translate-v2
   guard
   guard-livereload
   guard-rspec

app/models/agents/google_translation_agent.rb

@@ -1,8 +1,15 @@
 module Agents
   class GoogleTranslationAgent < Agent
     cannot_be_scheduled!
-
-    gem_dependency_check { defined?(Google) && defined?(Google::Cloud::Translate) }
+    can_dry_run!
+
+    gem_dependency_check do
+      require 'google/cloud/translate/v2'
+    rescue LoadError
+      false
+    else
+      true
+    end
 
     description <<~MD
       The Translation Agent will attempt to translate text between natural languages.
@@ -18,7 +25,10 @@ module Agents
 
       `from` is the language translated from. If it's not specified, the API will attempt to detect the source language automatically and return it within the response.
 
-      Specify what you would like to translate in `content` field, you can use [Liquid](https://github.com/huginn/huginn/wiki/Formatting-Events-using-Liquid) specify which part of the payload you want to translate.
+      Specify an object in `content` field using [Liquid](https://github.com/huginn/huginn/wiki/Formatting-Events-using-Liquid) expressions, which will be evaluated for each incoming event, and then translated to become the payload of the new event.
+      You can specify a nested object of any levels containing arrays and objects, and all string values except for object keys will be recursively translated.
+
+      Set `mode` to `merge` if you want to merge each translated content with the original event payload.  The default behavior (`clean`) is to emit events with only translated contents.
 
       `expected_receive_period_in_days` is the maximum number of days you would allow to pass between events.
     MD
@@ -27,13 +37,14 @@ module Agents
 
     def default_options
       {
-        'to' => "sv",
+        'mode' => 'clean',
+        'to' => 'sv',
         'from' => 'en',
         'google_api_key' => '',
         'expected_receive_period_in_days' => 1,
         'content' => {
           'text' => "{{message}}",
-          'moretext' => "{{another message}}"
+          'moretext' => "{{another_message}}"
         }
       }
     end
@@ -46,58 +57,83 @@ module Agents
       unless options['google_api_key'].present? && options['to'].present? && options['content'].present? && options['expected_receive_period_in_days'].present?
         errors.add :base, "google_api_key, to, content and expected_receive_period_in_days are all required"
       end
-    end
 
-    def translate_from
-      interpolated["from"].presence
+      case options['mode'].presence
+      when nil, /\A(?:clean|merge)\z|\{/
+        # ok
+      else
+        errors.add(:base, "mode must be 'clean' or 'merge'")
+      end
     end
 
     def receive(incoming_events)
       incoming_events.each do |event|
-        translated_event = {}
-        opts = interpolated(event)
-        opts['content'].each_pair do |key, value|
-          result = translate(value)
-          translated_event[key] = result.text
+        interpolate_with(event) do
+          translated_content = translate(interpolated['content'])
+
+          case interpolated['mode']
+          when 'merge'
+            create_event payload: event.payload.merge(translated_content)
+          else
+            create_event payload: translated_content
+          end
         end
-        create_event payload: translated_event
       end
     end
 
-    def google_client
-      @google_client ||= Google::APIClient.new(
-        {
-          application_name: "Huginn",
-          application_version: "0.0.1",
-          key: options['google_api_key'],
-          authorization: nil
-        }
-      )
-    end
-
-    def translate_service
-      @translate_service ||= google_client.discovered_api('translate', 'v2')
-    end
+    def translate(content)
+      if !content.is_a?(Hash)
+        error("content must be an object, but it is #{content.class}.")
+        return
+      end
 
-    def cloud_translate_service
-      # https://github.com/GoogleCloudPlatform/google-cloud-ruby/blob/master/google-cloud-translate/lib/google-cloud-translate.rb#L130
-      @google_client ||= Google::Cloud::Translate.new(
-        version: :v2,
+      api = Google::Cloud::Translate::V2.new(
         key: interpolated['google_api_key']
       )
-    end
 
-    def translate(value)
-      # google_client.execute(
-      #   api_method: translate_service.translations.list,
-      #   parameters: {
-      #     format: 'text',
-      #     source: translate_from,
-      #     target: options["to"],
-      #     q: value
-      #   }
-      # )
-      cloud_translate_service.translate(value, to: interpolated["to"], from: translate_from, format: "text")
+      texts = []
+      walker = ->(value) {
+        case value
+        in nil | Numeric | true | false
+        in _ if _.blank?
+        in String
+          texts << value
+        in Array
+          value.each(&walker)
+        in Hash
+          value.each_value(&walker)
+        end
+      }
+      walker.call(content)
+
+      translations =
+        if texts.empty?
+          []
+        else
+          api.translate(
+            *texts,
+            from: interpolated['from'].presence,
+            to: interpolated['to'],
+            format: 'text',
+          )
+        end
+
+      # Hash key order should be constant in Ruby
+      mapper = ->(value) {
+        case value
+        in nil | Numeric | true | false
+          value
+        in _ if _.blank?
+          value
+        in String
+          translations&.shift&.text
+        in Array
+          value.map(&mapper)
+        in Hash
+          value.transform_values(&mapper)
+        end
+      }
+      mapper.call(content)
     end
   end
 end

spec/cassettes/Agents_GoogleTranslationAgent/_receive/checks_if_it_can_handle_multiple_events.yml

@@ -2,10 +2,10 @@
 http_interactions:
 - request:
     method: post
-    uri: https://translation.googleapis.com/language/translate/v2?key=some_api_key
+    uri: https://translate.googleapis.com/language/translate/v2?key=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
     body:
       encoding: UTF-8
-      string: '{"q":["hey what are you doing"],"target":"sv","source":"en","format":"text"}'
+      string: '{"q":["hey what are you doing","do tell more"],"target":"sv","source":"en","format":"text"}'
     headers:
       User-Agent:
       - gcloud-ruby/1.0.0
@@ -14,7 +14,7 @@ http_interactions:
       Content-Type:
       - application/json
       X-Goog-Api-Client:
-      - gl-ruby/2.3.4 gccl/1.0.0
+      - gl-ruby/3.2.6 gccl/1.0.0
       Accept-Encoding:
       - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
       Accept:
@@ -31,35 +31,32 @@ http_interactions:
       - Referer
       - X-Origin
       Date:
-      - Fri, 30 Jun 2017 14:54:18 GMT
+      - Sat, 16 Nov 2024 16:43:19 GMT
       Server:
       - ESF
       Cache-Control:
       - private
       X-Xss-Protection:
-      - 1; mode=block
+      - '0'
       X-Frame-Options:
       - SAMEORIGIN
       X-Content-Type-Options:
       - nosniff
       Alt-Svc:
-      - quic=":443"; ma=2592000; v="39,38,37,36,35"
+      - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000
       Transfer-Encoding:
       - chunked
     body:
       encoding: ASCII-8BIT
       string: !binary |-
-        ewogICJkYXRhIjogewogICAgInRyYW5zbGF0aW9ucyI6IFsKICAgICAgewog
-        ICAgICAgICJ0cmFuc2xhdGVkVGV4dCI6ICJIZWogdmFkIGfDtnIgZHUiCiAg
-        ICAgIH0KICAgIF0KICB9Cn0K
-    http_version: 
-  recorded_at: Fri, 30 Jun 2017 14:54:18 GMT
+        ewogICJkYXRhIjogewogICAgInRyYW5zbGF0aW9ucyI6IFsKICAgICAgewogICAgICAgICJ0cmFuc2xhdGVkVGV4dCI6ICJoZWogdmFkIGfDtnIgZHUiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAidHJhbnNsYXRlZFRleHQiOiAiYmVyw6R0dGEgbWVyIgogICAgICB9CiAgICBdCiAgfQp9Cg==
+  recorded_at: Sat, 16 Nov 2024 16:43:19 GMT
 - request:
     method: post
-    uri: https://translation.googleapis.com/language/translate/v2?key=some_api_key
+    uri: https://translate.googleapis.com/language/translate/v2?key=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
     body:
       encoding: UTF-8
-      string: '{"q":["do tell more"],"target":"sv","source":"en","format":"text"}'
+      string: '{"q":["value2","value1"],"target":"sv","source":"en","format":"text"}'
     headers:
       User-Agent:
       - gcloud-ruby/1.0.0
@@ -68,7 +65,7 @@ http_interactions:
       Content-Type:
       - application/json
       X-Goog-Api-Client:
-      - gl-ruby/2.3.4 gccl/1.0.0
+      - gl-ruby/3.2.6 gccl/1.0.0
       Accept-Encoding:
       - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
       Accept:
@@ -85,141 +82,24 @@ http_interactions:
       - Referer
       - X-Origin
       Date:
-      - Fri, 30 Jun 2017 14:54:19 GMT
+      - Sat, 16 Nov 2024 16:43:20 GMT
       Server:
       - ESF
       Cache-Control:
       - private
       X-Xss-Protection:
-      - 1; mode=block
+      - '0'
       X-Frame-Options:
       - SAMEORIGIN
       X-Content-Type-Options:
       - nosniff
       Alt-Svc:
-      - quic=":443"; ma=2592000; v="39,38,37,36,35"
+      - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000
       Transfer-Encoding:
       - chunked
     body:
       encoding: ASCII-8BIT
       string: !binary |-
-        ewogICJkYXRhIjogewogICAgInRyYW5zbGF0aW9ucyI6IFsKICAgICAgewog
-        ICAgICAgICJ0cmFuc2xhdGVkVGV4dCI6ICJCZXLDpHR0YSBtZXIiCiAgICAg
-        IH0KICAgIF0KICB9Cn0K
-    http_version: 
-  recorded_at: Fri, 30 Jun 2017 14:54:19 GMT
-- request:
-    method: post
-    uri: https://translation.googleapis.com/language/translate/v2?key=some_api_key
-    body:
-      encoding: UTF-8
-      string: '{"q":["value2"],"target":"sv","source":"en","format":"text"}'
-    headers:
-      User-Agent:
-      - gcloud-ruby/1.0.0
-      Google-Cloud-Resource-Prefix:
-      - projects/
-      Content-Type:
-      - application/json
-      X-Goog-Api-Client:
-      - gl-ruby/2.3.4 gccl/1.0.0
-      Accept-Encoding:
-      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
-      Accept:
-      - "*/*"
-  response:
-    status:
-      code: 200
-      message: OK
-    headers:
-      Content-Type:
-      - application/json; charset=UTF-8
-      Vary:
-      - Origin
-      - Referer
-      - X-Origin
-      Date:
-      - Fri, 30 Jun 2017 14:54:19 GMT
-      Server:
-      - ESF
-      Cache-Control:
-      - private
-      X-Xss-Protection:
-      - 1; mode=block
-      X-Frame-Options:
-      - SAMEORIGIN
-      X-Content-Type-Options:
-      - nosniff
-      Alt-Svc:
-      - quic=":443"; ma=2592000; v="39,38,37,36,35"
-      Transfer-Encoding:
-      - chunked
-    body:
-      encoding: ASCII-8BIT
-      string: !binary |-
-        ewogICJkYXRhIjogewogICAgInRyYW5zbGF0aW9ucyI6IFsKICAgICAgewog
-        ICAgICAgICJ0cmFuc2xhdGVkVGV4dCI6ICJ2w6RyZGUyIgogICAgICB9CiAg
-        ICBdCiAgfQp9Cg==
-    http_version: 
-  recorded_at: Fri, 30 Jun 2017 14:54:19 GMT
-- request:
-    method: post
-    uri: https://translation.googleapis.com/language/translate/v2?key=some_api_key
-    body:
-      encoding: UTF-8
-      string: '{"q":["value1"],"target":"sv","source":"en","format":"text"}'
-    headers:
-      User-Agent:
-      - gcloud-ruby/1.0.0
-      Google-Cloud-Resource-Prefix:
-      - projects/
-      Content-Type:
-      - application/json
-      X-Goog-Api-Client:
-      - gl-ruby/2.3.4 gccl/1.0.0
-      Accept-Encoding:
-      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
-      Accept:
-      - "*/*"
-  response:
-    status:
-      code: 200
-      message: OK
-    headers:
-      Content-Type:
-      - application/json; charset=UTF-8
-      Vary:
-      - Origin
-      - Referer
-      - X-Origin
-      Date:
-      - Fri, 30 Jun 2017 14:54:19 GMT
-      Server:
-      - ESF
-      Cache-Control:
-      - private
-      X-Xss-Protection:
-      - 1; mode=block
-      X-Frame-Options:
-      - SAMEORIGIN
-      X-Content-Type-Options:
-      - nosniff
-      Alt-Svc:
-      - quic=":443"; ma=2592000; v="39,38,37,36,35"
-      Transfer-Encoding:
-      - chunked
-    body:
-      encoding: ASCII-8BIT
-      string: |
-        {
-          "data": {
-            "translations": [
-              {
-                "translatedText": "value1"
-              }
-            ]
-          }
-        }
-    http_version: 
-  recorded_at: Fri, 30 Jun 2017 14:54:19 GMT
-recorded_with: VCR 3.0.3
+        ewogICJkYXRhIjogewogICAgInRyYW5zbGF0aW9ucyI6IFsKICAgICAgewogICAgICAgICJ0cmFuc2xhdGVkVGV4dCI6ICJ2w6RyZGUyIgogICAgICB9LAogICAgICB7CiAgICAgICAgInRyYW5zbGF0ZWRUZXh0IjogInbDpHJkZTEiCiAgICAgIH0KICAgIF0KICB9Cn0K
+  recorded_at: Sat, 16 Nov 2024 16:43:20 GMT
+recorded_with: VCR 6.1.0

spec/models/agents/google_translation_agent_spec.rb

@@ -7,7 +7,7 @@ describe Agents::GoogleTranslationAgent, :vcr do
       options: {
         to: "sv",
         from: "en",
-        google_api_key: 'some_api_key',
+        google_api_key: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
         expected_receive_period_in_days: 1,
         content: {
           text: "{{message}}",
@@ -79,5 +79,14 @@ describe Agents::GoogleTranslationAgent, :vcr do
       @checker.options[:to] = ""
       expect(@checker).not_to be_valid
     end
+
+    it "should validate the value of 'mode' key" do
+      @checker.options[:mode] = "clean"
+      expect(@checker).to be_valid
+      @checker.options[:mode] = "merge"
+      expect(@checker).to be_valid
+      @checker.options[:mode] = "clear"
+      expect(@checker).not_to be_valid
+    end
   end
 end