Merge branch 'master' into timestamps_in_twitter_agent

.env.example

@@ -167,6 +167,12 @@ EVENT_EXPIRATION_CHECK=6h
 # enabled.
 #USE_GRAPHVIZ_DOT=dot
 
+# Default layout for agent flow diagrams generated by Graphviz.
+# Choose from `circo`, `dot` (default), `fdp`, `neato`, `osage`,
+# `patchwork`, `sfdp`, or `twopi`.  Note that not all layouts are
+# supported by Graphviz depending on the build options.
+#DIAGRAM_DEFAULT_LAYOUT=dot
+
 # Timezone. Use `rake time:zones:local` or `rake time:zones:all` to get your zone name
 TIMEZONE="Pacific Time (US & Canada)"
 

CHANGES.md

@@ -1,5 +1,17 @@
 # Changes
 
+* Oct 17, 2015   - TwitterSearchAgent added for running period Twitter searches.
+* Oct 17, 2015   - GapDetectorAgent added to alert when no data has been seen in a certain period of time.
+* Oct 12, 2015   - Slack agent supports attachments.
+* Oct 9, 2015    - The TriggerAgent can be asked to match on fewer then all match groups.
+* Oct 4, 2015    - Add DelayAgent for buffering incoming Events
+* Oct 3, 2015    - Add SSL verification options to smtp.yml
+* Oct 3, 2015    - Better handling of 'Back' links in the UI.
+* Sep 22, 2015   - Comprehensive EvernoteAgent added
+* Sep 13, 2015   - JavaScriptAgent can access and set Credentials.
+* Sep 9, 2015    - Add AgentRunner and LongRunnable to support long running agents.
+* Sep 8, 2015    - Allow `url_from_event` in the WebsiteAgent to be an Array
+* Sep 7, 2015    - Enable `strict: false` in database.yml
 * Sep 2, 2015    - WebRequestConcern Agents automatically decode gzip/inflate encodings.
 * Sep 1, 2015    - WebhookAgent can configure allowed verbs (GET, POST, PUT, ...) for incoming requests.
 * Aug 21, 2015   - PostAgent supports "xml" as `content_type`.

Gemfile

@@ -29,7 +29,7 @@ gem 'twitter-stream', github: 'cantino/twitter-stream', branch: 'huginn'
 gem 'omniauth-twitter'
 
 # Tumblr Agents
-gem 'tumblr_client', github: 'knu/tumblr_client', branch: 'patch-1'
+gem 'tumblr_client', github: 'tumblr/tumblr_client', branch: 'master'  # '>= 0.8.5'
 gem 'omniauth-tumblr'
 
 # Dropbox Agents
@@ -67,7 +67,7 @@ gem 'devise', '~> 3.4.0'
 gem 'dotenv-rails', '~> 2.0.1'
 gem 'em-http-request', '~> 1.1.2'
 gem 'faraday', '~> 0.9.0'
-gem 'faraday_middleware', '>= 0.10.0'
+gem 'faraday_middleware', github: 'lostisland/faraday_middleware', branch: 'master'  # '>= 0.10.1'
 gem 'feed-normalizer'
 gem 'font-awesome-sass', '~> 4.3.2'
 gem 'foreman', '~> 0.63.0'

Gemfile.lock

@@ -20,9 +20,17 @@ GIT
       rest-client (~> 1.8)
 
 GIT
-  remote: git://github.com/knu/tumblr_client.git
-  revision: d6f1f64a7cba381345c588e28ebcff28048c3a6c
-  branch: patch-1
+  remote: git://github.com/lostisland/faraday_middleware.git
+  revision: c5836ae55857272732b33eb0e0a98d60e995a376
+  branch: master
+  specs:
+    faraday_middleware (0.10.0)
+      faraday (>= 0.7.4, < 0.10)
+
+GIT
+  remote: git://github.com/tumblr/tumblr_client.git
+  revision: 0c59b04e49f2a8c89860613b18cf4e8f978d8dc7
+  branch: master
   specs:
     tumblr_client (0.8.5)
       faraday (~> 0.9.0)
@@ -187,8 +195,6 @@ GEM
     extlib (0.9.16)
     faraday (0.9.1)
       multipart-post (>= 1.2, < 3)
-    faraday_middleware (0.10.0)
-      faraday (>= 0.7.4, < 0.10)
     feed-normalizer (1.5.2)
       hpricot (>= 0.6)
       simple-rss (>= 1.1)
@@ -293,7 +299,7 @@ GEM
     mime-types (2.6.1)
     mini_magick (4.2.3)
     mini_portile (0.6.2)
-    minitest (5.8.0)
+    minitest (5.8.1)
     mqtt (0.3.1)
     multi_json (1.11.2)
     multi_xml (0.5.5)
@@ -442,8 +448,8 @@ GEM
       tilt (~> 1.1)
     select2-rails (3.5.9.3)
       thor (~> 0.14)
-    shoulda-matchers (2.8.0)
-      activesupport (>= 3.0.0)
+    shoulda-matchers (3.0.0)
+      activesupport (>= 4.0.0)
     signet (0.5.1)
       addressable (>= 2.2.3)
       faraday (>= 0.9.0.rc5)
@@ -553,7 +559,7 @@ DEPENDENCIES
   em-http-request (~> 1.1.2)
   evernote_oauth
   faraday (~> 0.9.0)
-  faraday_middleware (>= 0.10.0)
+  faraday_middleware!
   feed-normalizer
   ffi (>= 1.9.4)
   font-awesome-sass (~> 4.3.2)

app/assets/javascripts/diagram.js.coffee

@@ -3,6 +3,7 @@
 $ ->
   svg = document.querySelector('.agent-diagram svg.diagram')
   overlay = document.querySelector('.agent-diagram .overlay')
+  $(overlay).width($(svg).width()).height($(svg).height())
   getTopLeft = (node) ->
     bbox = node.getBBox()
     point = svg.createSVGPoint()

app/concerns/agent_controller_concern.rb

@@ -7,7 +7,7 @@ module AgentControllerConcern
 
   def default_options
     {
-      'action' => 'run',
+      'action' => 'run'
     }
   end
 
@@ -68,7 +68,7 @@ module AgentControllerConcern
             log "Agent '#{target.name}' is disabled"
           end
         when 'configure'
-          target.update!(options: target.options.merge(interpolated['configure_options']))
+          target.update! options: target.options.deep_merge(interpolated['configure_options'])
           log "Agent '#{target.name}' is configured with #{interpolated['configure_options'].inspect}"
         when ''
           # Do nothing

app/concerns/long_runnable.rb

@@ -51,12 +51,13 @@ module LongRunnable
   end
 
   class Worker
-    attr_reader :thread, :id, :agent, :config, :mutex, :scheduler
+    attr_reader :thread, :id, :agent, :config, :mutex, :scheduler, :restarting
 
     def initialize(options = {})
       @id = options[:id]
       @agent = options[:agent]
       @config = options[:config]
+      @restarting = false
     end
 
     def run
@@ -65,6 +66,7 @@ module LongRunnable
 
     def run!
       @thread = Thread.new do
+        Thread.current[:name] = "#{id}-#{Time.now}"
         begin
           run
         rescue SignalException, SystemExit
@@ -90,14 +92,21 @@ module LongRunnable
       if respond_to?(:stop)
         stop
       else
-        thread.terminate
+        terminate_thread!
       end
     end
 
+    def terminate_thread!
+      thread.terminate
+      thread.wakeup if thread.status == 'sleep'
+    end
+
     def restart!
-      stop!
-      setup!(scheduler, mutex)
-      run!
+      without_alive_check do
+        stop!
+        setup!(scheduler, mutex)
+        run!
+      end
     end
 
     def every(*args, &blk)
@@ -120,5 +129,12 @@ module LongRunnable
     def schedule(method, args, &blk)
       @scheduler.send(method, *args, tag: id, &blk)
     end
+
+    def without_alive_check(&blk)
+      @restarting = true
+      yield
+    ensure
+      @restarting = false
+    end
   end
 end

app/concerns/sortable_events.rb

@@ -11,7 +11,7 @@ module SortableEvents
 
   module ClassMethods
     def can_order_created_events!
-      raise if cannot_create_events?
+      raise 'Cannot order events for agent that cannot create events' if cannot_create_events?
       prepend AutomaticSorter
     end
 

app/concerns/web_request_concern.rb

@@ -39,7 +39,7 @@ module WebRequestConcern
           # detection, so we do that.
           case env[:response_headers][:content_type]
           when /;\s*charset\s*=\s*([^()<>@,;:\\\"\/\[\]?={}\s]+)/i
-            encoding = Encoding.find($1) rescue nil
+            encoding = Encoding.find($1) rescue @default_encoding
           when /\A\s*(?:text\/[^\s;]+|application\/(?:[^\s;]+\+)?(?:xml|json))\s*(?:;|\z)/i
             encoding = @default_encoding
           else
@@ -47,7 +47,7 @@ module WebRequestConcern
             next
           end
         end
-        body.encode!(Encoding::UTF_8, encoding) unless body.encoding == Encoding::UTF_8
+        body.encode!(Encoding::UTF_8, encoding)
       end
     end
   end
@@ -123,11 +123,6 @@ module WebRequestConcern
 
       builder.use FaradayMiddleware::Gzip
 
-      unless builder.headers.any? { |key,| /\Aaccept[-_]encoding\z/i =~ key }
-        # Exclude `deflate` by default.  See #1018.
-        builder.headers[:accept_encoding] = 'gzip,identity'
-      end
-
       case backend = faraday_backend
         when :typhoeus
           require 'typhoeus/adapters/faraday'

app/helpers/dot_helper.rb

@@ -1,8 +1,8 @@
 module DotHelper
-  def render_agents_diagram(agents)
+  def render_agents_diagram(agents, layout: nil)
     if (command = ENV['USE_GRAPHVIZ_DOT']) &&
        (svg = IO.popen([command, *%w[-Tsvg -q1 -o/dev/stdout /dev/stdin]], 'w+') { |dot|
-          dot.print agents_dot(agents, true)
+          dot.print agents_dot(agents, rich: true, layout: layout)
           dot.close_write
           dot.read
         } rescue false)
@@ -125,7 +125,7 @@ module DotHelper
     DotDrawer.draw(vars, &block)
   end
 
-  def agents_dot(agents, rich = false)
+  def agents_dot(agents, rich: false, layout: nil)
     draw(agents: agents,
          agent_id: ->agent { 'a%d' % agent.id },
          agent_label: ->agent {
@@ -158,7 +158,10 @@ module DotHelper
       end
 
       block('digraph', 'Agent Event Flow') {
-        # statement 'graph', rankdir: 'LR'
+        layout ||= ENV['DIAGRAM_DEFAULT_LAYOUT'].presence
+        if rich && /\A[a-z]+\z/ === layout
+          statement 'graph', layout: layout, overlap: 'false'
+        end
         statement 'node',
                   shape: 'box',
                   style: 'rounded',
@@ -197,7 +200,6 @@ module DotHelper
       root << svg
       root << overlay_container = Nokogiri::XML::Node.new('div', doc) { |div|
         div['class'] = 'overlay-container'
-        div['style'] = "width: #{svg['width']}; height: #{svg['height']}"
       }
       overlay_container << overlay = Nokogiri::XML::Node.new('div', doc) { |div|
         div['class'] = 'overlay'

app/models/agents/beeper_agent.rb

@@ -0,0 +1,128 @@
+module Agents
+  class BeeperAgent < Agent
+    cannot_be_scheduled!
+    cannot_create_events!
+
+    description <<-MD
+      Beeper agent sends messages to Beeper app on your mobile device via Push notifications.
+
+      You need a Beeper Application ID (`app_id`), Beeper REST API Key (`api_key`) and Beeper Sender ID (`sender_id`) [https://beeper.io](https://beeper.io)
+
+      You have to provide phone number (`phone`) of the recipient which have a mobile device with Beeper installed, or a `group_id` – Beeper Group ID
+
+      Also you have to provide a message `type` which has to be `message`, `image`, `event`, `location` or `task`.
+
+      Depending on message type you have to provide additional fields:
+
+      ##### Message
+      * `text` – **required**
+
+      ##### Image
+      * `image` – **required** (Image URL or Base64-encoded image)
+      * `text` – optional
+
+      ##### Event
+      * `text` – **required**
+      * `start_time` – **required** (Corresponding to ISO 8601)
+      * `end_time` – optional (Corresponding to ISO 8601)
+
+      ##### Location
+      * `latitude` – **required**
+      * `longitude` – **required**
+      * `text` – optional
+
+      ##### Task
+      * `text` – **required**
+
+      You can see additional documentation at [Beeper website](https://beeper.io/docs)
+    MD
+
+    BASE_URL = 'https://api.beeper.io/api'
+
+    TYPE_ATTRIBUTES = {
+      'message'  => %w(text),
+      'image'    => %w(text image),
+      'event'    => %w(text start_time end_time),
+      'location' => %w(text latitude longitude),
+      'task'     => %w(text)
+    }
+
+    MESSAGE_TYPES = TYPE_ATTRIBUTES.keys
+
+    TYPE_REQUIRED_ATTRIBUTES = {
+      'message'  => %w(text),
+      'image'    => %w(image),
+      'event'    => %w(text start_time),
+      'location' => %w(latitude longitude),
+      'task'     => %w(text)
+    }
+
+    def default_options
+      {
+        'type'      => 'message',
+        'app_id'    => '',
+        'api_key'   => '',
+        'sender_id' => '',
+        'phone'     => '',
+        'text'      => '{{title}}'
+      }
+    end
+
+    def validate_options
+      %w(app_id api_key sender_id type).each do |attr|
+        errors.add(:base, "you need to specify a #{attr}") if options[attr].blank?
+      end
+
+      if options['type'].in?(MESSAGE_TYPES)
+        required_attributes = TYPE_REQUIRED_ATTRIBUTES[options['type']]
+        if required_attributes.any? { |attr| options[attr].blank? }
+          errors.add(:base, "you need to specify a #{required_attributes.join(', ')}")
+        end
+      else
+        errors.add(:base, 'you need to specify a valid message type')
+      end
+
+      unless options['group_id'].blank? ^ options['phone'].blank?
+        errors.add(:base, 'you need to specify a phone or group_id')
+      end
+    end
+
+    def working?
+      received_event_without_error? && !recent_error_logs?
+    end
+
+    def receive(incoming_events)
+      incoming_events.each do |event|
+        send_message(event)
+      end
+    end
+
+    def send_message(event)
+      mo = interpolated(event)
+      begin
+        response = HTTParty.post(endpoint_for(mo['type']), body: payload_for(mo), headers: headers)
+        error(response.body) if response.code != 201
+      rescue HTTParty::Error => e
+        error(e.message)
+      end
+    end
+
+    private
+
+    def headers
+      {
+        'X-Beeper-Application-Id' => options['app_id'],
+        'X-Beeper-REST-API-Key'   => options['api_key'],
+        'Content-Type' => 'application/json'
+      }
+    end
+
+    def payload_for(mo)
+      mo.slice(*TYPE_ATTRIBUTES[mo['type']], 'sender_id', 'phone', 'group_id').to_json
+    end
+
+    def endpoint_for(type)
+      "#{BASE_URL}/#{type}s.json"
+    end
+  end
+end

app/models/agents/commander_agent.rb

@@ -36,7 +36,7 @@ module Agents
       true
     end
 
-    def check!
+    def check
       control!
     end
 

app/models/agents/data_output_agent.rb

@@ -1,5 +1,7 @@
 module Agents
   class DataOutputAgent < Agent
+    include WebRequestConcern
+
     cannot_be_scheduled!
 
     description  do
@@ -19,9 +21,10 @@ module Agents
 
           * `secrets` - An array of tokens that the requestor must provide for light-weight authentication.
           * `expected_receive_period_in_days` - How often you expect data to be received by this Agent from other Agents.
-          * `template` - A JSON object representing a mapping between item output keys and incoming event values.  Use [Liquid](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) to format the values.  Values of the `link`, `title`, `description` and `icon` keys will be put into the \\<channel\\> section of RSS output.  The `item` key will be repeated for every Event.  The `pubDate` key for each item will have the creation time of the Event unless given.
+          * `template` - A JSON object representing a mapping between item output keys and incoming event values.  Use [Liquid](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) to format the values.  Values of the `link`, `title`, `description` and `icon` keys will be put into the \\<channel\\> section of RSS output.  Value of the `self` key will be used as URL for this feed itself, which is useful when you serve it via reverse proxy.  The `item` key will be repeated for every Event.  The `pubDate` key for each item will have the creation time of the Event unless given.
           * `events_to_show` - The number of events to output in RSS or JSON. (default: `40`)
           * `ttl` - A value for the \\<ttl\\> element in RSS output. (default: `60`)
+          * `push_hubs` - Set to a list of PubSubHubbub endpoints you want to publish an update to every time this agent receives an event. (default: none)  Popular hubs include [Superfeedr](https://pubsubhubbub.superfeedr.com/) and [Google](https://pubsubhubbub.appspot.com/).  Note that publishing updates will make your feed URL known to the public, so if you want to keep it secret, set up a reverse proxy to serve your feed via a safe URL and specify it in `template.self`.
 
         If you'd like to output RSS tags with attributes, such as `enclosure`, use something like the following in your `template`:
 
@@ -95,6 +98,29 @@ module Agents
       unless options['template'].present? && options['template']['item'].present? && options['template']['item'].is_a?(Hash)
         errors.add(:base, "Please provide template and template.item")
       end
+
+      case options['push_hubs']
+      when nil
+      when Array
+        options['push_hubs'].each do |hub|
+          case hub
+          when /\{/
+            # Liquid templating
+          when String
+            begin
+              URI.parse(hub)
+            rescue URI::Error
+              errors.add(:base, "invalid URL found in push_hubs")
+              break
+            end
+          else
+            errors.add(:base, "push_hubs must be an array of endpoint URLs")
+            break
+          end
+        end
+      else
+        errors.add(:base, "push_hubs must be an array")
+      end
     end
 
     def events_to_show
@@ -114,11 +140,12 @@ module Agents
     end
 
     def feed_url(options = {})
-      feed_link + Rails.application.routes.url_helpers.
-                  web_requests_path(agent_id: id || ':id',
-                                    user_id: user_id,
-                                    secret: options[:secret],
-                                    format: options[:format])
+      interpolated['template']['self'].presence ||
+        feed_link + Rails.application.routes.url_helpers.
+                    web_requests_path(agent_id: id || ':id',
+                                      user_id: user_id,
+                                      secret: options[:secret],
+                                      format: options[:format])
     end
 
     def feed_icon
@@ -129,6 +156,10 @@ module Agents
       interpolated['template']['description'].presence || "A feed of Events received by the '#{name}' Huginn Agent"
     end
 
+    def push_hubs
+      interpolated['push_hubs'].presence || []
+    end
+
     def receive_web_request(params, method, format)
       unless interpolated['secrets'].include?(params['secret'])
         if format =~ /json/
@@ -159,40 +190,54 @@ module Agents
           interpolated
         end
 
+        now = Time.now
+
         if format =~ /json/
           content = {
             'title' => feed_title,
             'description' => feed_description,
-            'pubDate' => Time.now,
+            'pubDate' => now,
             'items' => simplify_item_for_json(items)
           }
 
           return [content, 200]
         else
-          content = Utils.unindent(<<-XML)
-            <?xml version="1.0" encoding="UTF-8" ?>
-            <rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">
-            <channel>
-             <atom:link href=#{feed_url(secret: params['secret'], format: :xml).encode(xml: :attr)} rel="self" type="application/rss+xml" />
-             <atom:icon>#{feed_icon.encode(xml: :text)}</atom:icon>
-             <title>#{feed_title.encode(xml: :text)}</title>
-             <description>#{feed_description.encode(xml: :text)}</description>
-             <link>#{feed_link.encode(xml: :text)}</link>
-             <lastBuildDate>#{Time.now.rfc2822.to_s.encode(xml: :text)}</lastBuildDate>
-             <pubDate>#{Time.now.rfc2822.to_s.encode(xml: :text)}</pubDate>
-             <ttl>#{feed_ttl}</ttl>
-
+          hub_links = push_hubs.map { |hub|
+            <<-XML
+ <atom:link rel="hub" href=#{hub.encode(xml: :attr)}/>
+            XML
+          }.join
+
+          items = simplify_item_for_xml(items)
+                  .to_xml(skip_types: true, root: "items", skip_instruct: true, indent: 1)
+                  .gsub(%r{^</?items>\n}, '')
+
+          return [<<-XML, 200, 'text/xml']
+<?xml version="1.0" encoding="UTF-8" ?>
+<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom" xmlns:media="http://search.yahoo.com/mrss/">
+<channel>
+ <atom:link href=#{feed_url(secret: params['secret'], format: :xml).encode(xml: :attr)} rel="self" type="application/rss+xml" />
+ <atom:icon>#{feed_icon.encode(xml: :text)}</atom:icon>
+#{hub_links}
+ <title>#{feed_title.encode(xml: :text)}</title>
+ <description>#{feed_description.encode(xml: :text)}</description>
+ <link>#{feed_link.encode(xml: :text)}</link>
+ <lastBuildDate>#{now.rfc2822.to_s.encode(xml: :text)}</lastBuildDate>
+ <pubDate>#{now.rfc2822.to_s.encode(xml: :text)}</pubDate>
+ <ttl>#{feed_ttl}</ttl>
+#{items}
+</channel>
+</rss>
           XML
+        end
+      end
+    end
 
-          content += simplify_item_for_xml(items).to_xml(skip_types: true, root: "items", skip_instruct: true, indent: 1).gsub(/^<\/?items>/, '').strip
-
-          content += Utils.unindent(<<-XML)
-            </channel>
-            </rss>
-          XML
+    def receive(incoming_events)
+      url = feed_url(secret: interpolated['secrets'].first, format: :xml)
 
-          return [content, 200, 'text/xml']
-        end
+      push_hubs.each do |hub|
+        push_to_hub(hub, url)
       end
     end
 
@@ -261,5 +306,32 @@ module Agents
         item
       end
     end
+
+    def push_to_hub(hub, url)
+      hub_uri =
+        begin
+          URI.parse(hub)
+        rescue URI::Error
+          nil
+        end
+
+      if !hub_uri.is_a?(URI::HTTP)
+        error "Invalid push endpoint: #{hub}"
+        return
+      end
+
+      log "Pushing #{url} to #{hub_uri}"
+
+      return if dry_run?
+
+      begin
+        faraday.post hub_uri, {
+          'hub.mode' => 'publish',
+          'hub.url' => url
+        }
+     rescue => e
+       error "Push failed: #{e.message}"
+      end
+    end
   end
 end

app/models/agents/delay_agent.rb

@@ -0,0 +1,73 @@
+module Agents
+  class DelayAgent < Agent
+    default_schedule "every_12h"
+
+    description <<-MD
+      The DelayAgent stores received Events and emits copies of them on a schedule. Use this as a buffer or queue of Events.
+
+      `max_events` should be set to the maximum number of events that you'd like to hold in the buffer. When this number is
+      reached, new events will either be ignored, or will displace the oldest event already in the buffer, depending on
+      whether you set `keep` to `newest` or `oldest`.
+
+      `expected_receive_period_in_days` is used to determine if the Agent is working. Set it to the maximum number of days
+      that you anticipate passing without this Agent receiving an incoming Event.
+
+      `max_emitted_events` is used to limit the number of the maximum events which should be created. If you omit this DelayAgent will create events for every event stored in the memory.
+    MD
+
+    def default_options
+      {
+        'expected_receive_period_in_days' => "10",
+        'max_events' => "100",
+        'keep' => 'newest'
+      }
+    end
+
+    def validate_options
+      unless options['expected_receive_period_in_days'].present? && options['expected_receive_period_in_days'].to_i > 0
+        errors.add(:base, "Please provide 'expected_receive_period_in_days' to indicate how many days can pass before this Agent is considered to be not working")
+      end
+
+      unless options['keep'].present? && options['keep'].in?(%w[newest oldest])
+        errors.add(:base, "The 'keep' option is required and must be set to 'oldest' or 'newest'")
+      end
+
+      unless options['max_events'].present? && options['max_events'].to_i > 0
+        errors.add(:base, "The 'max_events' option is required and must be an integer greater than 0")
+      end
+    end
+
+    def working?
+      last_receive_at && last_receive_at > options['expected_receive_period_in_days'].to_i.days.ago && !recent_error_logs?
+    end
+
+    def receive(incoming_events)
+      incoming_events.each do |event|
+        memory['event_ids'] ||= []
+        memory['event_ids'] << event.id
+        if memory['event_ids'].length > interpolated['max_events'].to_i
+          if interpolated['keep'] == 'newest'
+            memory['event_ids'].shift
+          else
+            memory['event_ids'].pop
+          end
+        end
+      end
+    end
+
+    def check
+      if memory['event_ids'] && memory['event_ids'].length > 0
+        events = received_events.where(id: memory['event_ids']).reorder('events.id asc')
+
+        if options['max_emitted_events'].present?
+          events = events.limit(options['max_emitted_events'].to_i)
+        end
+
+        events.each do |event|
+          create_event payload: event.payload
+          memory['event_ids'].delete(event.id)
+        end
+      end
+    end
+  end
+end

app/models/agents/event_formatting_agent.rb

@@ -1,6 +1,7 @@
 module Agents
   class EventFormattingAgent < Agent
     cannot_be_scheduled!
+    can_dry_run!
 
     description <<-MD
       The Event Formatting Agent allows you to format incoming Events, adding new fields as needed.

app/models/agents/ftpsite_agent.rb

@@ -196,7 +196,7 @@ module Agents
     end
 
     def uri_path_escape(string)
-      str = string.dup.force_encoding(Encoding::ASCII_8BIT)  # string.b in Ruby >=2.0
+      str = string.b
       str.gsub!(/([^A-Za-z0-9\-._~!$&()*+,=@]+)/) { |m|
         '%' + m.unpack('H2' * m.bytesize).join('%').upcase
       }

app/models/agents/gap_detector_agent.rb

@@ -0,0 +1,67 @@
+module Agents
+  class GapDetectorAgent < Agent
+    default_schedule "every_10m"
+
+    description <<-MD
+      The Gap Detector Agent will watch for holes or gaps in a stream of incoming Events and generate "no data alerts".
+
+      The `value_path` value is a [JSONPath](http://goessner.net/articles/JsonPath/) to a value of interest. If either
+      this value is empty, or no Events are received, during `window_duration_in_days`, an Event will be created with
+      a payload of `message`.
+    MD
+
+    event_description <<-MD
+      Events look like:
+
+          {
+            "message": "No data has been received!",
+            "gap_started_at": "1234567890"
+          }
+    MD
+
+    def validate_options
+      unless options['message'].present?
+        errors.add(:base, "message is required")
+      end
+
+      unless options['window_duration_in_days'].present? && options['window_duration_in_days'].to_f > 0
+        errors.add(:base, "window_duration_in_days must be provided as an integer or floating point number")
+      end
+    end
+
+    def default_options
+      {
+        'window_duration_in_days' => "2",
+        'message' => "No data has been received!"
+      }
+    end
+
+    def working?
+      true
+    end
+
+    def receive(incoming_events)
+      incoming_events.sort_by(&:created_at).each do |event|
+        memory['newest_event_created_at'] ||= 0
+
+        if !interpolated['value_path'].present? || Utils.value_at(event.payload, interpolated['value_path']).present?
+          if event.created_at.to_i > memory['newest_event_created_at']
+            memory['newest_event_created_at'] = event.created_at.to_i
+            memory.delete('alerted_at')
+          end
+        end
+      end
+    end
+
+    def check
+      window = interpolated['window_duration_in_days'].to_f.days.ago
+      if memory['newest_event_created_at'].present? && Time.at(memory['newest_event_created_at']) < window
+        unless memory['alerted_at']
+          memory['alerted_at'] = Time.now.to_i
+          create_event payload: { message: interpolated['message'],
+                                  gap_started_at: memory['newest_event_created_at'] }
+        end
+      end
+    end
+  end
+end

app/models/agents/peak_detector_agent.rb

@@ -7,7 +7,7 @@ module Agents
     description <<-MD
       The Peak Detector Agent will watch for peaks in an event stream.  When a peak is detected, the resulting Event will have a payload message of `message`.  You can include extractions in the message, for example: `I saw a bar of: {{foo.bar}}`, have a look at the [Wiki](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) for details.
 
-      The `value_path` value is a [JSONPaths](http://goessner.net/articles/JsonPath/) to the value of interest.  `group_by_path` is a hash path that will be used to group values, if present.
+      The `value_path` value is a [JSONPath](http://goessner.net/articles/JsonPath/) to the value of interest.  `group_by_path` is a JSONPath that will be used to group values, if present.
 
       Set `expected_receive_period_in_days` to the maximum amount of time that you'd expect to pass between Events being received by this Agent.
 

app/models/agents/rss_agent.rb

@@ -87,34 +87,41 @@ module Agents
     end
 
     def check
-      Array(interpolated['url']).each do |url|
-        check_url(url)
-      end
+      check_urls(Array(interpolated['url']))
     end
 
     protected
 
-    def check_url(url)
-      response = faraday.get(url)
-      if response.success?
-        feed = FeedNormalizer::FeedNormalizer.parse(response.body, loose: true)
-        feed.clean! if boolify(interpolated['clean'])
-        max_events = (interpolated['max_events_per_run'].presence || 0).to_i
-        created_event_count = 0
-        sort_events(feed_to_events(feed)).each.with_index do |event, index|
-          break if max_events && max_events > 0 && index >= max_events
-          entry_id = event.payload[:id]
-          if check_and_track(entry_id)
+    def check_urls(urls)
+      new_events = []
+      max_events = (interpolated['max_events_per_run'].presence || 0).to_i
+
+      urls.each do |url|
+        begin
+          response = faraday.get(url)
+          if response.success?
+            feed = FeedNormalizer::FeedNormalizer.parse(response.body, loose: true)
+            feed.clean! if boolify(interpolated['clean'])
+            new_events.concat feed_to_events(feed)
+          else
+            error "Failed to fetch #{url}: #{response.inspect}"
+          end
+        rescue => e
+          error "Failed to fetch #{url} with message '#{e.message}': #{e.backtrace}"
+        end
+      end
+
+      created_event_count = 0
+      sort_events(new_events).each.with_index do |event, index|
+        entry_id = event.payload[:id]
+        if check_and_track(entry_id)
+          unless max_events && max_events > 0 && index >= max_events
             created_event_count += 1
             create_event(event)
           end
         end
-        log "Fetched #{url} and created #{created_event_count} event(s)."
-      else
-        error "Failed to fetch #{url}: #{response.inspect}"
       end
-    rescue => e
-      error "Failed to fetch #{url} with message '#{e.message}': #{e.backtrace}"
+      log "Fetched #{urls.to_sentence} and created #{created_event_count} event(s)."
     end
 
     def get_entry_id(entry)

app/models/agents/scheduler_agent.rb

@@ -87,10 +87,6 @@ module Agents
       true
     end
 
-    def check!
-      control!
-    end
-
     def validate_options
       if (spec = options['schedule']).present?
         begin

app/models/agents/shell_command_agent.rb

@@ -1,9 +1,9 @@
-require 'open3'
-
 module Agents
   class ShellCommandAgent < Agent
     default_schedule "never"
 
+    can_dry_run!
+
     def self.should_run?
       ENV['ENABLE_INSECURE_AGENTS'] == "true"
     end
@@ -11,7 +11,7 @@ module Agents
     description <<-MD
       The Shell Command Agent will execute commands on your local system, returning the output.
 
-      `command` specifies the command to be executed, and `path` will tell ShellCommandAgent in what directory to run this command.
+      `command` specifies the command (either a shell command line string or an array of command line arguments) to be executed, and `path` will tell ShellCommandAgent in what directory to run this command.  The content of `stdin` will be fed to the command via the standard input.
 
       `expected_update_period_in_days` is used to determine if the Agent is working.
 
@@ -20,6 +20,10 @@ module Agents
 
       The resulting event will contain the `command` which was executed, the `path` it was executed under, the `exit_status` of the command, the `errors`, and the actual `output`. ShellCommandAgent will not log an error if the result implies that something went wrong.
 
+      If `suppress_on_failure` is set to true, no event is emitted when `exit_status` is not zero.
+
+      If `suppress_on_empty_output` is set to true, no event is emitted when `output` is empty.
+
       *Warning*: This type of Agent runs arbitrary commands on your system, #{Agents::ShellCommandAgent.should_run? ? "but is **currently enabled**" : "and is **currently disabled**"}.
       Only enable this Agent if you trust everyone using your Huginn installation.
       You can enable this Agent in your .env file by setting `ENABLE_INSECURE_AGENTS` to `true`.
@@ -31,7 +35,7 @@ module Agents
         {
           "command": "pwd",
           "path": "/home/Huginn",
-          "exit_status": "0",
+          "exit_status": 0,
           "errors": "",
           "output": "/home/Huginn"
         }
@@ -41,6 +45,8 @@ module Agents
       {
           'path' => "/",
           'command' => "pwd",
+          'suppress_on_failure' => false,
+          'suppress_on_empty_output' => false,
           'expected_update_period_in_days' => 1
       }
     end
@@ -50,6 +56,16 @@ module Agents
         errors.add(:base, "The path, command, and expected_update_period_in_days fields are all required.")
       end
 
+      case options['stdin']
+      when String, nil
+      else
+        errors.add(:base, "stdin must be a string.")
+      end
+
+      unless Array(options['command']).all? { |o| o.is_a?(String) }
+        errors.add(:base, "command must be a shell command line string or an array of command line arguments.")
+      end
+
       unless File.directory?(options['path'])
         errors.add(:base, "#{options['path']} is not a real directory.")
       end
@@ -75,38 +91,62 @@ module Agents
       if Agents::ShellCommandAgent.should_run?
         command = opts['command']
         path = opts['path']
+        stdin = opts['stdin']
+
+        result, errors, exit_status = run_command(path, command, stdin)
 
-        result, errors, exit_status = run_command(path, command)
+        payload = {
+          'command' => command,
+          'path' => path,
+          'exit_status' => exit_status,
+          'errors' => errors,
+          'output' => result,
+        }
 
-        vals = {"command" => command, "path" => path, "exit_status" => exit_status, "errors" => errors, "output" => result}
-        created_event = create_event :payload => vals
+        unless suppress_event?(payload)
+          created_event = create_event payload: payload
+        end
 
-        log("Ran '#{command}' under '#{path}'", :outbound_event => created_event, :inbound_event => event)
+        log("Ran '#{command}' under '#{path}'", outbound_event: created_event, inbound_event: event)
       else
         log("Unable to run because insecure agents are not enabled.  Edit ENABLE_INSECURE_AGENTS in the Huginn .env configuration.")
       end
     end
 
-    def run_command(path, command)
-      result = nil
-      errors = nil
-      exit_status = nil
-
-      Dir.chdir(path){
-        begin
-          stdin, stdout, stderr, wait_thr = Open3.popen3(command)
-          exit_status = wait_thr.value.to_i
-          result = stdout.gets(nil)
-          errors = stderr.gets(nil)
-        rescue Exception => e
-          errors = e.to_s
+    def run_command(path, command, stdin)
+      begin
+        rout, wout = IO.pipe
+        rerr, werr = IO.pipe
+        rin,  win = IO.pipe
+
+        pid = spawn(*command, chdir: path, out: wout, err: werr, in: rin)
+
+        wout.close
+        werr.close
+        rin.close
+
+        if stdin
+          win.write stdin
+          win.close
         end
-      }
 
-      result = result.to_s.strip
-      errors = errors.to_s.strip
+        (result = rout.read).strip!
+        (errors = rerr.read).strip!
+
+        _, status = Process.wait2(pid)
+        exit_status = status.exitstatus
+      rescue => e
+        errors = e.to_s
+        result = ''.freeze
+        exit_status = nil
+      end
 
       [result, errors, exit_status]
     end
+
+    def suppress_event?(payload)
+      (boolify(interpolated['suppress_on_failure']) && payload['exit_status'].nonzero?) ||
+        (boolify(interpolated['suppress_on_empty_output']) && payload['output'].empty?)
+    end
   end
 end

app/models/agents/slack_agent.rb

@@ -1,6 +1,7 @@
 module Agents
   class SlackAgent < Agent
     DEFAULT_USERNAME = 'Huginn'
+    ALLOWED_PARAMS = ['channel', 'username', 'unfurl_links', 'attachments']
 
     cannot_be_scheduled!
     cannot_create_events!
@@ -13,7 +14,7 @@ module Agents
       #{'## Include `slack-notifier` in your Gemfile to use this Agent!' if dependencies_missing?}
 
       To get started, you will first need to configure an incoming webhook.
-      
+
       - Go to `https://my.slack.com/services/new/incoming-webhook`, choose a default channel and add the integration.
 
       Your webhook URL will look like: `https://hooks.slack.com/services/some/random/characters`
@@ -65,14 +66,22 @@ module Agents
       @slack_notifier ||= Slack::Notifier.new(webhook_url, username: username)
     end
 
+    def filter_options(opts)
+      opts.select { |key, value| ALLOWED_PARAMS.include? key }.symbolize_keys
+    end
+
     def receive(incoming_events)
       incoming_events.each do |event|
         opts = interpolated(event)
-        if /^:/.match(opts[:icon])
-          slack_notifier.ping opts[:message], channel: opts[:channel], username: opts[:username], icon_emoji: opts[:icon], unfurl_links: opts[:unfurl_links]
-        else
-          slack_notifier.ping opts[:message], channel: opts[:channel], username: opts[:username], icon_url: opts[:icon], unfurl_links: opts[:unfurl_links]
+        slack_opts = filter_options(opts)
+        if opts[:icon].present?
+          if /^:/.match(opts[:icon])
+            slack_opts[:icon_emoji] = opts[:icon]
+          else
+            slack_opts[:icon_url] = opts[:icon]
+          end
         end
+        slack_notifier.ping opts[:message], slack_opts
       end
     end
   end

app/models/agents/trigger_agent.rb

@@ -13,7 +13,10 @@ module Agents
 
       The `value` can be a single value or an array of values. In the case of an array, if one or more values match then the rule matches.
 
-      All rules must match for the Agent to match.  The resulting Event will have a payload message of `message`.  You can use liquid templating in the `message, have a look at the [Wiki](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) for details.
+      By default, all rules must match for the Agent to trigger. You can switch this so that only one rule must match by
+      setting `must_match` to `1`.
+
+      The resulting Event will have a payload message of `message`.  You can use liquid templating in the `message, have a look at the [Wiki](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) for details.
 
       Set `keep_event` to `true` if you'd like to re-emit the incoming event, optionally merged with 'message' when provided.
 
@@ -35,6 +38,14 @@ module Agents
       errors.add(:base, "message is required unless 'keep_event' is 'true'") unless options['message'].present? || keep_event?
 
       errors.add(:base, "keep_event, when present, must be 'true' or 'false'") unless options['keep_event'].blank? || %w[true false].include?(options['keep_event'])
+
+      if options['must_match'].present?
+        if options['must_match'].to_i < 1
+          errors.add(:base, "If used, the 'must_match' option must be a positive integer")
+        elsif options['must_match'].to_i > options['rules'].length
+          errors.add(:base, "If used, the 'must_match' option must be equal to or less than the number of rules")
+        end
+      end
     end
 
     def default_options
@@ -59,12 +70,12 @@ module Agents
 
         opts = interpolated(event)
 
-        match = opts['rules'].all? do |rule|
+        match_results = opts['rules'].map do |rule|
           value_at_path = Utils.value_at(event['payload'], rule['path'])
           rule_values = rule['value']
           rule_values = [rule_values] unless rule_values.is_a?(Array)
 
-          match_found = rule_values.any? do |rule_value|
+          rule_values.any? do |rule_value|
             case rule['type']
             when "regex"
               value_at_path.to_s =~ Regexp.new(rule_value, Regexp::IGNORECASE)
@@ -88,7 +99,7 @@ module Agents
           end
         end
 
-        if match
+        if matches?(match_results)
           if keep_event?
             payload = event.payload.dup
             payload['message'] = opts['message'] if opts['message'].present?
@@ -101,6 +112,14 @@ module Agents
       end
     end
 
+    def matches?(matches)
+      if options['must_match'].present?
+        matches.select { |match| match }.length >= options['must_match'].to_i
+      else
+        matches.all?
+      end
+    end
+
     def keep_event?
       boolify(interpolated['keep_event'])
     end

app/models/agents/tumblr_publish_agent.rb

@@ -17,9 +17,9 @@ module Agents
 
       **Required fields:**
 
-      `blog_name` Your Tumblr URL (e.g. "mustardhamsters.tumblr.com") 
+      `blog_name` Your Tumblr URL (e.g. "mustardhamsters.tumblr.com")
 
-      `post_type` One of [text, photo, quote, link, chat, audio, video] 
+      `post_type` One of [text, photo, quote, link, chat, audio, video, reblog]
 
 
       -------------
@@ -35,13 +35,13 @@ module Agents
       * `format` html, markdown
       * `slug` short text summary at end of the post URL
 
-      **Text** `title` `body` 
+      **Text** `title` `body`
 
       **Photo** `caption` `link`  `source`
 
       **Quote** `quote` `source`
 
-      **Link** `title` `url` `description` 
+      **Link** `title` `url` `description`
 
       **Chat** `title` `conversation`
 
@@ -49,6 +49,7 @@ module Agents
 
       **Video** `caption` `embed`
 
+      **Reblog** `id` `reblog_key` `comment`
 
       -------------
 
@@ -90,6 +91,9 @@ module Agents
           'conversation' => "{{conversation}}",
           'external_url' => "{{external_url}}",
           'embed' => "{{embed}}",
+          'id' => "{{id}}",
+          'reblog_key' => "{{reblog_key}}",
+          'comment' => "{{comment}}",
         },
       }
     end
@@ -105,19 +109,25 @@ module Agents
         options = interpolated(event)['options']
         begin
           post = publish_post(blog_name, post_type, options)
+          if !post.has_key?('id')
+            log("Failed to create #{post_type} post on #{blog_name}: #{post.to_json}, options: #{options.to_json}")
+            return
+          end
+          expanded_post = get_post(blog_name, post["id"])
           create_event :payload => {
             'success' => true,
             'published_post' => "["+blog_name+"] "+post_type,
             'post_id' => post["id"],
             'agent_id' => event.agent_id,
-            'event_id' => event.id
+            'event_id' => event.id,
+            'post' => expanded_post
           }
         end
       end
     end
 
-    def publish_post(blog_name, post_type, options)      
-      options_obj = { 
+    def publish_post(blog_name, post_type, options)
+      options_obj = {
           :state => options['state'],
           :tags => options['tags'],
           :tweet => options['tweet'],
@@ -157,7 +167,19 @@ module Agents
         options_obj[:caption] = options['caption']
         options_obj[:embed] = options['embed']
         tumblr.video(blog_name, options_obj)
+      when "reblog"
+        options_obj[:id] = options['id']
+        options_obj[:reblog_key] = options['reblog_key']
+        options_obj[:comment] = options['comment']
+        tumblr.reblog(blog_name, options_obj)
       end
     end
+
+    def get_post(blog_name, id)
+      obj = tumblr.posts(blog_name, {
+        :id => id
+      })
+      obj["posts"].first
+    end
   end
 end

app/models/agents/twitter_search_agent.rb

@@ -0,0 +1,105 @@
+module Agents
+  class TwitterSearchAgent < Agent
+    include TwitterConcern
+
+    cannot_receive_events!
+
+    description <<-MD
+      The Twitter Search Agent performs and emits the results of a specified Twitter search.
+
+      #{twitter_dependencies_missing if dependencies_missing?}
+
+      If you want realtime data from Twitter about frequent terms, you should definitely use the Twitter Stream Agent instead.
+
+      To be able to use this Agent you need to authenticate with Twitter in the [Services](/services) section first.
+
+      You must provide the desired `search`.
+      
+      Set `result_type` to specify which [type of search results](https://dev.twitter.com/rest/reference/get/search/tweets) you would prefer to receive. Options are "mixed", "recent", and "popular". (default: `mixed`)
+
+      Set `max_results` to limit the amount of results to retrieve per run(default: `500`. The API rate limit is ~18,000 per 15 minutes. [Click here to learn more about rate limits](https://dev.twitter.com/rest/public/rate-limiting).
+
+      Set `expected_update_period_in_days` to the maximum amount of time that you'd expect to pass between Events being created by this Agent.
+
+      Set `starting_at` to the date/time (eg. `Mon Jun 02 00:38:12 +0000 2014`) you want to start receiving tweets from (default: agent's `created_at`)
+    MD
+
+    event_description <<-MD
+      Events are the raw JSON provided by the [Twitter API](https://dev.twitter.com/rest/reference/get/search/tweets). Should look something like:
+
+          {
+             ... every Tweet field, including ...
+            "text": "something",
+            "user": {
+              "name": "Mr. Someone",
+              "screen_name": "Someone",
+              "location": "Vancouver BC Canada",
+              "description":  "...",
+              "followers_count": 486,
+              "friends_count": 1983,
+              "created_at": "Mon Aug 29 23:38:14 +0000 2011",
+              "time_zone": "Pacific Time (US & Canada)",
+              "statuses_count": 3807,
+              "lang": "en"
+            },
+            "retweet_count": 0,
+            "entities": ...
+            "lang": "en"
+          }
+    MD
+
+    default_schedule "every_1h"
+
+    def working?
+      event_created_within?(interpolated['expected_update_period_in_days']) && !recent_error_logs?
+    end
+
+    def default_options
+      {
+        'search' => 'freebandnames',
+        'expected_update_period_in_days' => '2'
+      }
+    end
+
+    def validate_options
+      errors.add(:base, "search is required") unless options['search'].present?
+      errors.add(:base, "expected_update_period_in_days is required") unless options['expected_update_period_in_days'].present?
+
+      if options[:starting_at].present?
+        Time.parse(interpolated[:starting_at]) rescue errors.add(:base, "Error parsing starting_at")
+      end
+    end
+
+    def starting_at
+      if interpolated[:starting_at].present?
+        Time.parse(interpolated[:starting_at]) rescue created_at
+      else
+        created_at
+      end
+    end
+
+    def max_results
+      (interpolated['max_results'].presence || 500).to_i
+    end
+
+    def check
+      since_id = memory['since_id'] || nil
+      opts = {include_entities: true}
+      opts.merge! result_type: interpolated[:result_type] if interpolated[:result_type].present?
+      opts.merge! since_id: since_id unless since_id.nil?
+
+      # http://www.rubydoc.info/gems/twitter/Twitter/REST/Search
+      tweets = twitter.search(interpolated['search'], opts).take(max_results)
+
+      tweets.each do |tweet|
+        if (tweet.created_at >= starting_at)
+          memory['since_id'] = tweet.id if !memory['since_id'] || (tweet.id > memory['since_id'])
+
+          create_event payload: tweet.attrs
+        end
+      end
+
+      save!
+    end
+  end
+end

app/models/agents/twitter_stream_agent.rb

@@ -125,13 +125,13 @@ module Agents
     end
 
     def self.setup_worker
-      if Agents::TwitterStreamAgent.dependencies_missing?
-        STDERR.puts Agents::TwitterStreamAgent.twitter_dependencies_missing
-        STDERR.flush
-        return false
-      end
-
       Agents::TwitterStreamAgent.active.group_by { |agent| agent.twitter_oauth_token }.map do |oauth_token, agents|
+        if Agents::TwitterStreamAgent.dependencies_missing?
+          STDERR.puts Agents::TwitterStreamAgent.twitter_dependencies_missing
+          STDERR.flush
+          return false
+        end
+
         filter_to_agent_map = agents.map { |agent| agent.options[:filters] }.flatten.uniq.compact.map(&:strip).inject({}) { |m, f| m[f] = []; m }
 
         agents.each do |agent|
@@ -176,7 +176,7 @@ module Agents
 
       def stop
         EventMachine.stop_event_loop if EventMachine.reactor_running?
-        thread.terminate
+        terminate_thread!
       end
 
       private

app/models/agents/weather_agent.rb

@@ -71,8 +71,16 @@ module Agents
         'expected_update_period_in_days' => '2'
       }
     end
+    
+    def check
+      if key_setup?
+        create_event :payload => model(weather_provider, which_day).merge('location' => location)
+      end
+    end
 
-    def service
+    private
+    
+    def weather_provider
       interpolated["service"].presence || "wunderground"
     end
 
@@ -85,8 +93,7 @@ module Agents
     end
 
     def validate_options
-      errors.add(:base, "service is required") unless service.present?
-      errors.add(:base, "service must be set to 'forecastio' or 'wunderground'") unless ["forecastio", "wunderground"].include?(service)
+      errors.add(:base, "service must be set to 'forecastio' or 'wunderground'") unless ["forecastio", "wunderground"].include?(weather_provider)
       errors.add(:base, "location is required") unless location.present?
       errors.add(:base, "api_key is required") unless key_setup?
       errors.add(:base, "which_day selection is required") unless which_day.present?
@@ -104,10 +111,10 @@ module Agents
       end
     end
 
-    def model(service,which_day)
-      if service == "wunderground"
+    def model(weather_provider,which_day)
+      if weather_provider == "wunderground"
         wunderground[which_day]
-      elsif service == "forecastio"
+      elsif weather_provider == "forecastio"
         forecastio.each do |value|
           timestamp = Time.at(value.time)
           if (timestamp.to_date - Time.now.to_date).to_i == which_day
@@ -174,12 +181,5 @@ module Agents
         end
       end
     end
-
-    def check
-      if key_setup?
-        create_event :payload => model(service, which_day).merge('location' => location)
-      end
-    end
-
   end
 end

app/models/agents/webhook_agent.rb

@@ -18,11 +18,12 @@ module Agents
         * `expected_receive_period_in_days` - How often you expect to receive
           events this way. Used to determine if the agent is working.
         * `payload_path` - JSONPath of the attribute in the POST body to be
-          used as the Event payload.  If `payload_path` points to an array,
-          Events will be created for each element.
+          used as the Event payload.  Set to `.` to return the entire message.
+          If `payload_path` points to an array, Events will be created for each element.
         * `verbs` - Comma-separated list of http verbs your agent will accept.
           For example, "post,get" will enable POST and GET requests. Defaults
           to "post".
+        * `response` - The response message to the request. Defaults to 'Event Created'.
       MD
     end
 
@@ -53,7 +54,7 @@ module Agents
         create_event(payload: payload)
       end
 
-      ['Event Created', 201]
+      [response_message, 201]
     end
 
     def working?
@@ -69,5 +70,9 @@ module Agents
     def payload_for(params)
       Utils.value_at(params, interpolated['payload_path']) || {}
     end
+
+    def response_message
+      interpolated['response'] || 'Event Created'
+    end
   end
 end

app/models/agents/website_agent.rb

@@ -264,8 +264,9 @@ module Agents
         error "Ignoring a non-HTTP url: #{url.inspect}"
         return
       end
-      log "Fetching #{url}"
-      response = faraday.get(url)
+      uri = Utils.normalize_uri(url)
+      log "Fetching #{uri}"
+      response = faraday.get(uri)
       raise "Failed: #{response.inspect}" unless response.success?
 
       interpolation_context.stack {
@@ -303,7 +304,7 @@ module Agents
           interpolated['extract'].keys.each do |name|
             result[name] = output[name][index]
             if name.to_s == 'url'
-              result[name] = (response.env[:url] + result[name]).to_s
+              result[name] = (response.env[:url] + Utils.normalize_uri(result[name])).to_s
             end
           end
 
@@ -439,7 +440,14 @@ module Agents
         case nodes
         when Nokogiri::XML::NodeSet
           result = nodes.map { |node|
-            case value = node.xpath(extraction_details['value'] || '.')
+            value = node.xpath(extraction_details['value'] || '.')
+            if value.is_a?(Nokogiri::XML::NodeSet)
+              child = value.first
+              if child && child.cdata?
+                value = child.text
+              end
+            end
+            case value
             when Float
               # Node#xpath() returns any numeric value as float;
               # convert it to integer as appropriate.

app/views/agents/_form.html.erb

@@ -65,9 +65,8 @@
               <div class="can-control-other-agents">
                 <div class="form-group">
                   <%= f.label :control_targets %>
-                  <% eventControlTargets = current_user.agents.select(&:can_be_scheduled?) %>
                   <%= f.select(:control_target_ids,
-                               options_for_select(eventControlTargets.map {|s| [s.name, s.id] },
+                               options_for_select(current_user.agents.map {|s| [s.name, s.id] },
                                                   @agent.control_target_ids),
                                {}, { multiple: true, size: 5, class: 'select2 form-control' }) %>
                 </div>

app/views/agents/_table.html.erb

@@ -53,7 +53,7 @@
         </td>
         <td class='<%= "agent-unavailable" if agent.unavailable? %>'>
           <% if agent.can_create_events? %>
-            <%= link_to(agent.events_count || 0, agent_events_path(agent)) %>
+            <%= link_to(agent.events_count || 0, agent_events_path(agent, return: (defined?(return_to) && return_to) || request.path)) %>
           <% else %>
             <span class='not-applicable'></span>
           <% end %>

app/views/agents/show.html.erb

@@ -15,7 +15,7 @@
           <li><a href="#logs" data-toggle="tab" data-agent-id="<%= @agent.id %>" class='<%= @agent.recent_error_logs? ? 'recent-errors' : '' %>'><span class='glyphicon glyphicon-list-alt'></span> Logs</a></li>
 
           <% if @agent.can_create_events? && @agent.events.count > 0 %>
-            <li><%= link_to icon_tag('glyphicon-random') + ' Events'.html_safe, agent_events_path(@agent) %></li>
+            <li><%= link_to icon_tag('glyphicon-random') + ' Events'.html_safe, agent_events_path(@agent, return: request.fullpath) %></li>
           <% else %>
             <li class='disabled'><a><span class='glyphicon glyphicon-random'></span> Events</a></li>
           <% end %>

app/views/diagrams/show.html.erb

@@ -20,7 +20,7 @@
       </div>
 
       <div class='digraph'>
-        <%= render_agents_diagram(@agents) %>
+        <%= render_agents_diagram(@agents, layout: params[:layout]) %>
       </div>
     </div>
   </div>

app/views/events/index.html.erb

@@ -40,7 +40,7 @@
 
       <% if @agent %>
         <div class="btn-group">
-          <%= link_to icon_tag('glyphicon-eye-open') + ' View Agent'.html_safe, agent_path(@agent, return: request.fullpath), class: "btn btn-default" %>
+          <%= link_to icon_tag('glyphicon-chevron-left') + ' Back'.html_safe, filtered_agent_return_link || agents_path, class: "btn btn-default" %>
           <%= link_to icon_tag('glyphicon-random') + ' See all events'.html_safe, events_path, class: "btn btn-default" %>
         </div>
       <% end %>

config/initializers/delayed_job.rb

@@ -16,4 +16,4 @@ end
 
 Delayed::Backend::ActiveRecord.configure do |config|
   config.reserve_sql_strategy = :default_sql
-end
+end

config/smtp.yml

@@ -6,6 +6,9 @@ development:
   enable_starttls_auto: <%= ENV['SMTP_ENABLE_STARTTLS_AUTO'] == 'true' ? true : false %>
   user_name: <%= ENV['SMTP_USER_NAME'] || "" %>
   password: <%= ENV['SMTP_PASSWORD'] || "" %>
+  openssl_verify_mode: <%= ENV['SMTP_OPENSSL_VERIFY_MODE'].presence || 'null' %>
+  ca_path: <%= ENV['SMTP_OPENSSL_CA_PATH'].presence || 'null' %>
+  ca_file: <%= ENV['SMTP_OPENSSL_CA_FILE'].presence || 'null' %>
 
 staging:
   address: <%= ENV['SMTP_SERVER'] || "smtp.gmail.com" %>
@@ -15,6 +18,9 @@ staging:
   enable_starttls_auto: <%= ENV['SMTP_ENABLE_STARTTLS_AUTO'] == 'true' ? true : false %>
   user_name: <%= ENV['SMTP_USER_NAME'] || "" %>
   password: <%= ENV['SMTP_PASSWORD'] || "" %>
+  openssl_verify_mode: <%= ENV['SMTP_OPENSSL_VERIFY_MODE'].presence || 'null' %>
+  ca_path: <%= ENV['SMTP_OPENSSL_CA_PATH'].presence || 'null' %>
+  ca_file: <%= ENV['SMTP_OPENSSL_CA_FILE'].presence || 'null' %>
 
 production:
   address: <%= ENV['SMTP_SERVER'] || "smtp.gmail.com" %>
@@ -23,4 +29,7 @@ production:
   authentication: <%= ENV['SMTP_AUTHENTICATION'] || "plain" %>
   enable_starttls_auto: <%= ENV['SMTP_ENABLE_STARTTLS_AUTO'] == 'true' ? true : false %>
   user_name: <%= ENV['SMTP_USER_NAME'] || "" %>
-  password: <%= ENV['SMTP_PASSWORD'] || "" %>
+  password: <%= ENV['SMTP_PASSWORD'] || "" %>
+  openssl_verify_mode: <%= ENV['SMTP_OPENSSL_VERIFY_MODE'].presence || 'null' %>
+  ca_path: <%= ENV['SMTP_OPENSSL_CA_PATH'].presence || 'null' %>
+  ca_file: <%= ENV['SMTP_OPENSSL_CA_FILE'].presence || 'null' %>

doc/heroku/install.md

@@ -10,7 +10,7 @@ If you still wish to use the Heroku free plan (which won't work very well), plea
 
 * Heroku's [free plan](https://www.heroku.com/pricing) limits total runtime per day to 18 hours. This means that Huginn must sleep some of the time, and so recurring tasks will only run if their recurrence frequency fits within the free plan's awake time, which is 30 minutes. Therefore, we recommend that you only use the every 1 minute, every 2 minute, and every 5 minute Agent scheduling options.
 * If you're using the free plan, you need to signup for a free [uptimerobot](https://uptimerobot.com) account and have it ping your Huginn URL on Heroku once every 70 minutes.  If you still receive warnings from Heroku, try a longer interval.
-* Heroku's free Postgres plan limits the number of database rows that you can have to 10,000, so you should be sure to set a low event retention schedule for your agents.
+* Heroku's free Postgres plan limits the number of database rows that you can have to 10,000, so you should be sure to set a low event retention schedule for your agents and set `AGENT_LOG_LENGTH`, the number of log lines kept in the DB per Agent, to something small: `heroku config:set AGENT_LOG_LENGTH=20`.
 
 ## Instructions
 

lib/agent_runner.rb

@@ -100,7 +100,7 @@ class AgentRunner
 
   def restart_dead_workers
     @workers.each_pair do |id, worker|
-      if worker.thread && !worker.thread.alive?
+      if !worker.restarting && worker.thread && !worker.thread.alive?
         puts "Restarting #{id.to_s}"
         @workers[id].run!
       end

lib/huginn_scheduler.rb

@@ -61,7 +61,7 @@ class Rufus::Scheduler
         job.scheduler_agent_id = agent_id
 
         if scheduler_agent = job.scheduler_agent
-          scheduler_agent.check!
+          scheduler_agent.control!
         else
           puts "Unscheduling SchedulerAgent##{job.scheduler_agent_id} (disabled or deleted)"
           job.unschedule

lib/json_with_indifferent_access.rb

@@ -1,6 +1,9 @@
 class JSONWithIndifferentAccess
   def self.load(json)
     ActiveSupport::HashWithIndifferentAccess.new(JSON.parse(json || '{}'))
+  rescue JSON::ParserError
+    Rails.logger.error "Unparsable JSON in JSONWithIndifferentAccess: #{json}"
+    { 'error' => 'unparsable json detected during de-serialization' }
   end
 
   def self.dump(hash)

lib/tasks/production.rake

@@ -38,6 +38,11 @@ namespace :production do
     run_sv('start')
   end
 
+  task :force_stop => :check do
+    puts "Force stopping huginn ..."
+    run_sv('force-stop')
+  end
+
   task :status => :check do
     run_sv('status')
   end
@@ -91,4 +96,4 @@ rescue StandardError => e
   raise e
 else
   puts output
-end
+end

lib/utils.rb

@@ -21,6 +21,18 @@ module Utils
     end
   end
 
+  def self.normalize_uri(uri)
+    begin
+      URI(uri)
+    rescue URI::Error
+      URI(uri.to_s.gsub(/[^\-_.!~*'()a-zA-Z\d;\/?:@&=+$,\[\]]+/) { |unsafe|
+            unsafe.bytes.each_with_object(String.new) { |uc, s|
+              s << sprintf('%%%02X', uc)
+            }
+          }.force_encoding(Encoding::US_ASCII))
+    end
+  end
+
   def self.interpolate_jsonpaths(value, data, options = {})
     if options[:leading_dollarsign_is_jsonpath] && value[0] == '$'
       Utils.values_at(data, value).first.to_s

spec/concerns/dry_runnable_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe DryRunnable do
   class Agents::SandboxedAgent < Agent

spec/concerns/form_configurable_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe FormConfigurable do
   class Agent1

spec/concerns/inheritance_tracking_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 require 'inheritance_tracking'
 
 describe InheritanceTracking do

spec/concerns/liquid_droppable_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe LiquidDroppable do
   before do

spec/concerns/liquid_interpolatable_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 require 'nokogiri'
 
 describe LiquidInterpolatable::Filters do

spec/concerns/long_runnable_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe LongRunnable do
   class LongRunnableAgent < Agent
@@ -76,6 +76,14 @@ describe LongRunnable do
     context "#stop!" do
       it "terminates the thread" do
         mock(@worker.thread).terminate
+        mock(@worker.thread).status { 'run' }
+        @worker.stop!
+      end
+
+      it "wakes up sleeping threads after termination" do
+        mock(@worker.thread).terminate
+        mock(@worker.thread).status { 'sleep' }
+        mock(@worker.thread).wakeup
         @worker.stop!
       end
 

spec/concerns/sortable_events_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe SortableEvents do
   let(:agent_class) {
@@ -152,7 +152,7 @@ describe SortableEvents do
             passive_agent_class.class_eval do
               can_order_created_events!
             end
-          }.to raise_error
+          }.to raise_error('Cannot order events for agent that cannot create events')
         end
 
         it 'should work if called from an Agent that can create events' do
@@ -160,7 +160,7 @@ describe SortableEvents do
             active_agent_class.class_eval do
               can_order_created_events!
             end
-          }.not_to raise_error
+          }.not_to raise_error()
         end
       end
 

spec/controllers/agents_controller_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe AgentsController do
   def valid_attributes(options = {})

spec/controllers/concerns/sortable_table_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe SortableTable do
   class SortableTestController

spec/controllers/events_controller_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe EventsController do
   before do

spec/controllers/jobs_controller_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe JobsController do
   describe "GET index" do

spec/controllers/logs_controller_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe LogsController do
   describe "GET index" do

spec/controllers/omniauth_callbacks_controller_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe OmniauthCallbacksController do
   before do

spec/controllers/scenario_imports_controller_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe ScenarioImportsController do
   before do

spec/controllers/scenarios_controller_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe ScenariosController do
   def valid_attributes(options = {})

spec/controllers/services_controller_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe ServicesController do
   before do

spec/controllers/user_credentials_controller_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe UserCredentialsController do
   def valid_attributes(options = {})

spec/controllers/web_requests_controller_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe WebRequestsController do
   class Agents::WebRequestReceiverAgent < Agent

spec/data_fixtures/urlTest.html

@@ -0,0 +1,17 @@
+<html>
+    <head>
+        <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+        <title>test</title>
+    </head>
+    <body>
+        <ul>
+            <li><a href="http://google.com">google</a></li>
+            <li><a href="https://www.google.ca/search?q=some query">broken</a></li>
+            <li><a href="https://www.google.ca/search?q=some%20query">escaped</a></li>
+            <li><a href="http://ko.wikipedia.org/wiki/위키백과:대문">unicode url</a></li>
+            <li><a href="https://www.google.ca/search?q=위키백과:대문">unicode param</a></li>
+            <li><a href="http://ko.wikipedia.org/wiki/%EC%9C%84%ED%82%A4%EB%B0%B1%EA%B3%BC:%EB%8C%80%EB%AC%B8">percent encoded url</a></li>
+            <li><a href="https://www.google.ca/search?q=%EC%9C%84%ED%82%A4%EB%B0%B1%EA%B3%BC:%EB%8C%80%EB%AC%B8">percent encoded param</a></li>
+        </ul>
+    </body>
+</html>

spec/fixtures/agents.yml

@@ -111,3 +111,24 @@ jane_basecamp_agent:
   user: jane
   service: generic
   guid: <%= SecureRandom.hex %>
+
+
+bob_data_output_agent:
+  type: Agents::DataOutputAgent
+  user: bob
+  name: RSS Feed 
+  guid: <%= SecureRandom.hex %>
+  options: <%= {
+    expected_receive_period_in_days: 3,
+    secrets: ['secret'],
+    template: {
+      title: 'unchanged',
+      description: 'unchanged',
+      item: {
+        title: 'unchanged',
+        description: 'unchanged',
+        author: 'unchanged',
+        link: 'http://example.com'
+        }
+      }
+    }.to_json.inspect %>

spec/helpers/application_helper_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe ApplicationHelper do
   describe '#icon_tag' do

spec/helpers/dot_helper_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe DotHelper do
   describe "with example Agents" do
@@ -72,7 +72,7 @@ describe DotHelper do
       end
 
       it "generates a richer DOT script" do
-        expect(agents_dot(@agents, true)).to match(%r{
+        expect(agents_dot(@agents, rich: true)).to match(%r{
           \A
           digraph \x20 "Agent \x20 Event \x20 Flow" \{
             node \[ [^\]]+ \];

spec/helpers/jobs_helper_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe JobsHelper do
   let(:job) { Delayed::Job.new }

spec/helpers/markdown_helper_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe MarkdownHelper do
 

spec/helpers/scenario_helper_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe ScenarioHelper do
   let(:scenario) { users(:bob).scenarios.build(name: 'Scene', tag_fg_color: '#AAAAAA', tag_bg_color: '#000000') }

spec/lib/agent_runner_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe AgentRunner do
   context "without traps" do

spec/lib/agents_exporter_spec.rb

@@ -1,6 +1,6 @@
 # encoding: utf-8
 
-require 'spec_helper'
+require 'rails_helper'
 
 describe AgentsExporter do
   describe "#as_json" do

spec/lib/delayed_job_worker_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe DelayedJobWorker do
   before do

spec/lib/huginn_scheduler_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 require 'huginn_scheduler'
 
 describe HuginnScheduler do

spec/lib/liquid_migrator_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe LiquidMigrator do
   describe "converting JSONPath strings" do

spec/lib/location_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe Location do
   let(:location) {
@@ -30,14 +30,14 @@ describe Location do
     expect(location['lat']).to eq 2.0
   end
 
-  it "has a convencience accessor for combined latitude and longitude" do
+  it "has a convenience accessor for combined latitude and longitude" do
     expect(location.latlng).to eq "2.0,3.0"
   end
 
   it "does not allow hash-style assignment" do
     expect {
       location[:lat] = 2.0
-    }.to raise_error
+    }.to raise_error(NoMethodError)
   end
 
   it "ignores invalid values" do

spec/lib/utils_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe Utils do
   describe "#unindent" do

spec/models/agent_log_spec.rb

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-require 'spec_helper'
+require 'rails_helper'
 
 describe AgentLog do
   describe "validations" do

spec/models/agent_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe Agent do
   it_behaves_like WorkingHelpers
@@ -223,7 +223,7 @@ describe Agent do
         mock(Agent).find(@checker.id) { @checker }
         expect {
           Agents::SomethingSource.async_check(@checker.id)
-        }.to raise_error
+        }.to raise_error(RuntimeError)
         log = @checker.logs.first
         expect(log.message).to match(/Exception/)
         expect(log.level).to eq(4)
@@ -263,7 +263,7 @@ describe Agent do
         Agent.async_check(agents(:bob_weather_agent).id)
         expect {
           Agent.async_receive(agents(:bob_rain_notifier_agent).id, [agents(:bob_weather_agent).events.last.id])
-        }.to raise_error
+        }.to raise_error(RuntimeError)
         log = agents(:bob_rain_notifier_agent).logs.first
         expect(log.message).to match(/Exception/)
         expect(log.level).to eq(4)

spec/models/agents/adioso_agent_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe Agents::AdiosoAgent do
 	before do

spec/models/agents/basecamp_agent_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 require 'models/concerns/oauthable'
 
 describe Agents::BasecampAgent do

spec/models/agents/beeper_agent_spec.rb

@@ -0,0 +1,145 @@
+require 'rails_helper'
+
+
+describe Agents::BeeperAgent do
+  let(:base_params) {
+    {
+      'type'      => 'message',
+      'app_id'    => 'some-app-id',
+      'api_key'   => 'some-api-key',
+      'sender_id' => 'sender-id',
+      'phone'     => '+111111111111',
+      'text'      => 'Some text'
+    }
+  }
+
+  subject {
+    agent = described_class.new(name: 'beeper-agent', options: base_params)
+    agent.user = users(:jane)
+    agent.save! and return agent
+  }
+
+  context 'validation' do
+    it 'valid' do
+      expect(subject).to be_valid
+    end
+
+    [:type, :app_id, :api_key, :sender_id].each do |attr|
+      it "invalid without #{attr}" do
+        subject.options[attr] = nil
+        expect(subject).not_to be_valid
+      end
+    end
+
+    it 'invalid with group_id and phone' do
+      subject.options['group_id'] ='some-group-id'
+      expect(subject).not_to be_valid
+    end
+
+    context '#message' do
+      it 'requires text' do
+        subject.options[:text] = nil
+        expect(subject).not_to be_valid
+      end
+    end
+
+    context '#image' do
+      before(:each) do
+        subject.options[:type] = 'image'
+      end
+
+      it 'invalid without image' do
+        expect(subject).not_to be_valid
+      end
+
+      it 'valid with image' do
+        subject.options[:image] = 'some-url'
+        expect(subject).to be_valid
+      end
+    end
+
+    context '#event' do
+      before(:each) do
+        subject.options[:type] = 'event'
+      end
+
+      it 'invalid without start_time' do
+        expect(subject).not_to be_valid
+      end
+
+      it 'valid with start_time' do
+        subject.options[:start_time] = Time.now
+        expect(subject).to be_valid
+      end
+    end
+
+    context '#location' do
+      before(:each) do
+        subject.options[:type] = 'location'
+      end
+
+      it 'invalid without latitude and longitude' do
+        expect(subject).not_to be_valid
+      end
+
+      it 'valid with latitude and longitude' do
+        subject.options[:latitude] = 15.0
+        subject.options[:longitude] = 16.0
+        expect(subject).to be_valid
+      end
+    end
+
+    context '#task' do
+      before(:each) do
+        subject.options[:type] = 'task'
+      end
+
+      it 'valid with text' do
+        expect(subject).to be_valid
+      end
+    end
+  end
+
+  context 'payload_for' do
+    it 'removes unwanted attributes' do
+      result = subject.send(:payload_for, {'type' => 'message', 'text' => 'text',
+        'sender_id' => 'sender', 'phone' => '+1', 'random_attribute' => 'unwanted'})
+      expect(result).to eq('{"text":"text","sender_id":"sender","phone":"+1"}')
+    end
+  end
+
+  context 'headers' do
+    it 'sets X-Beeper-Application-Id header with app_id' do
+      expect(subject.send(:headers)['X-Beeper-Application-Id']).to eq(base_params['app_id'])
+    end
+
+    it 'sets X-Beeper-REST-API-Key header with api_key' do
+      expect(subject.send(:headers)['X-Beeper-REST-API-Key']).to eq(base_params['api_key'])
+    end
+
+    it 'sets Content-Type' do
+      expect(subject.send(:headers)['Content-Type']).to eq('application/json')
+    end
+  end
+
+  context 'endpoint_for' do
+    it 'returns valid URL for message' do
+      expect(subject.send(:endpoint_for, 'message')).to eq('https://api.beeper.io/api/messages.json')
+    end
+
+    it 'returns valid URL for image' do
+      expect(subject.send(:endpoint_for, 'image')).to eq('https://api.beeper.io/api/images.json')
+    end
+
+    it 'returns valid URL for event' do
+      expect(subject.send(:endpoint_for, 'event')).to eq('https://api.beeper.io/api/events.json')
+    end
+
+    it 'returns valid URL for location' do
+      expect(subject.send(:endpoint_for, 'location')).to eq('https://api.beeper.io/api/locations.json')
+    end
+    it 'returns valid URL for task' do
+      expect(subject.send(:endpoint_for, 'task')).to eq('https://api.beeper.io/api/tasks.json')
+    end
+  end
+end

spec/models/agents/change_detector_agent_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe Agents::ChangeDetectorAgent do
   def create_event(output=nil)

spec/models/agents/commander_agent_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe Agents::CommanderAgent do
   let(:valid_params) {
@@ -19,10 +19,10 @@ describe Agents::CommanderAgent do
 
   it_behaves_like AgentControllerConcern
 
-  describe "check!" do
+  describe "check" do
     it "should command targets" do
       stub(agent).control!.once { nil }
-      agent.check!
+      agent.check
     end
   end
 

spec/models/agents/data_output_agent_spec.rb

@@ -1,6 +1,6 @@
 # encoding: utf-8
 
-require 'spec_helper'
+require 'rails_helper'
 
 describe Agents::DataOutputAgent do
   let(:agent) do
@@ -73,6 +73,29 @@ describe Agents::DataOutputAgent do
     end
   end
 
+  describe "#receive" do
+    it "should push to hubs when push_hubs is given" do
+      agent.options[:push_hubs] = %w[http://push.example.com]
+      agent.options[:template] = { 'link' => 'http://huginn.example.org' }
+
+      alist = nil
+
+      stub_request(:post, 'http://push.example.com/')
+        .with(headers: { 'Content-Type' => %r{\Aapplication/x-www-form-urlencoded\s*(?:;|\z)} })
+        .to_return { |request|
+        alist = URI.decode_www_form(request.body).sort
+        { status: 200, body: 'ok' }
+      }
+
+      agent.receive(events(:bob_website_agent_event))
+
+      expect(alist).to eq [
+        ["hub.mode", "publish"],
+        ["hub.url", agent.feed_url(secret: agent.options[:secrets].first, format: :xml)]
+      ]
+    end
+  end
+
   describe "#receive_web_request" do
     before do
       current_time = Time.now
@@ -130,7 +153,7 @@ describe Agents::DataOutputAgent do
         expect(content_type).to eq('text/xml')
         expect(content.gsub(/\s+/, '')).to eq Utils.unindent(<<-XML).gsub(/\s+/, '')
           <?xml version="1.0" encoding="UTF-8" ?>
-          <rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">
+          <rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom" xmlns:media="http://search.yahoo.com/mrss/">
           <channel>
            <atom:link href="https://yoursite.com/users/#{agent.user.id}/web_requests/#{agent.id}/secret1.xml" rel="self" type="application/rss+xml"/>
            <atom:icon>https://yoursite.com/favicon.ico</atom:icon>
@@ -170,6 +193,16 @@ describe Agents::DataOutputAgent do
         XML
       end
 
+      it "can output RSS with hub links when push_hubs is specified" do
+        stub(agent).feed_link { "https://yoursite.com" }
+        agent.options[:push_hubs] = %w[https://pubsubhubbub.superfeedr.com/ https://pubsubhubbub.appspot.com/]
+        content, status, content_type = agent.receive_web_request({ 'secret' => 'secret1' }, 'get', 'text/xml')
+        expect(status).to eq(200)
+        expect(content_type).to eq('text/xml')
+        xml = Nokogiri::XML(content)
+        expect(xml.xpath('/rss/channel/atom:link[@rel="hub"]/@href').map(&:text).sort).to eq agent.options[:push_hubs].sort
+      end
+
       it "can output JSON" do
         agent.options['template']['item']['foo'] = "hi"
 
@@ -359,7 +392,7 @@ describe Agents::DataOutputAgent do
         expect(content_type).to eq('text/xml')
         expect(content.gsub(/\s+/, '')).to eq Utils.unindent(<<-XML).gsub(/\s+/, '')
           <?xml version="1.0" encoding="UTF-8" ?>
-          <rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">
+          <rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom" xmlns:media="http://search.yahoo.com/mrss/">
           <channel>
            <atom:link href="https://yoursite.com/users/#{agent.user.id}/web_requests/#{agent.id}/secret1.xml" rel="self" type="application/rss+xml"/>
            <atom:icon>https://yoursite.com/favicon.ico</atom:icon>

spec/models/agents/de_duplication_agent_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe Agents::DeDuplicationAgent do
   def create_event(output=nil)

spec/models/agents/delay_agent_spec.rb

@@ -0,0 +1,127 @@
+require 'rails_helper'
+
+describe Agents::DelayAgent do
+  let(:agent) do
+    _agent = Agents::DelayAgent.new(name: 'My DelayAgent')
+    _agent.options = _agent.default_options.merge('max_events' => 2)
+    _agent.user = users(:bob)
+    _agent.sources << agents(:bob_website_agent)
+    _agent.save!
+    _agent
+  end
+
+  def create_event
+    _event = Event.new(payload: { random: rand })
+    _event.agent = agents(:bob_website_agent)
+    _event.save!
+    _event
+  end
+
+  let(:first_event) { create_event }
+  let(:second_event) { create_event }
+  let(:third_event) { create_event }
+
+  describe "#working?" do
+    it "checks if events have been received within expected receive period" do
+      expect(agent).not_to be_working
+      Agents::DelayAgent.async_receive agent.id, [events(:bob_website_agent_event).id]
+      expect(agent.reload).to be_working
+      the_future = (agent.options[:expected_receive_period_in_days].to_i + 1).days.from_now
+      stub(Time).now { the_future }
+      expect(agent.reload).not_to be_working
+    end
+  end
+
+  describe "validation" do
+    before do
+      expect(agent).to be_valid
+    end
+
+    it "should validate max_events" do
+      agent.options.delete('max_events')
+      expect(agent).not_to be_valid
+      agent.options['max_events'] = ""
+      expect(agent).not_to be_valid
+      agent.options['max_events'] = "0"
+      expect(agent).not_to be_valid
+      agent.options['max_events'] = "10"
+      expect(agent).to be_valid
+    end
+
+    it "should validate presence of expected_receive_period_in_days" do
+      agent.options['expected_receive_period_in_days'] = ""
+      expect(agent).not_to be_valid
+      agent.options['expected_receive_period_in_days'] = 0
+      expect(agent).not_to be_valid
+      agent.options['expected_receive_period_in_days'] = -1
+      expect(agent).not_to be_valid
+    end
+
+    it "should validate keep" do
+      agent.options.delete('keep')
+      expect(agent).not_to be_valid
+      agent.options['keep'] = ""
+      expect(agent).not_to be_valid
+      agent.options['keep'] = 'wrong'
+      expect(agent).not_to be_valid
+      agent.options['keep'] = 'newest'
+      expect(agent).to be_valid
+      agent.options['keep'] = 'oldest'
+      expect(agent).to be_valid
+    end
+  end
+
+  describe "#receive" do
+    it "records Events" do
+      expect(agent.memory).to be_empty
+      agent.receive([first_event])
+      expect(agent.memory).not_to be_empty
+      agent.receive([second_event])
+      expect(agent.memory['event_ids']).to eq [first_event.id, second_event.id]
+    end
+
+    it "keeps the newest when 'keep' is set to 'newest'" do
+      expect(agent.options['keep']).to eq 'newest'
+      agent.receive([first_event, second_event, third_event])
+      expect(agent.memory['event_ids']).to eq [second_event.id, third_event.id]
+    end
+
+    it "keeps the oldest when 'keep' is set to 'oldest'" do
+      agent.options['keep'] = 'oldest'
+      agent.receive([first_event, second_event, third_event])
+      expect(agent.memory['event_ids']).to eq [first_event.id, second_event.id]
+    end
+  end
+
+  describe "#check" do
+    it "re-emits Events and clears the memory" do
+      agent.receive([first_event, second_event, third_event])
+      expect(agent.memory['event_ids']).to eq [second_event.id, third_event.id]
+
+      expect {
+        agent.check
+      }.to change { agent.events.count }.by(2)
+
+      events = agent.events.reorder('events.id desc')
+      expect(events.first.payload).to eq third_event.payload
+      expect(events.second.payload).to eq second_event.payload
+
+      expect(agent.memory['event_ids']).to eq []
+    end
+
+    it "re-emits max_emitted_events and clears just them from the memory" do
+      agent.options['max_emitted_events'] = 1
+      agent.receive([first_event, second_event, third_event])
+      expect(agent.memory['event_ids']).to eq [second_event.id, third_event.id]
+
+      expect {
+        agent.check
+      }.to change { agent.events.count }.by(1)
+
+      events = agent.events.reorder('events.id desc')
+      expect(agent.memory['event_ids']).to eq [third_event.id]
+      expect(events.first.payload).to eq second_event.payload
+
+    end
+  end
+end

spec/models/agents/dropbox_file_url_agent_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe Agents::DropboxFileUrlAgent do
   before(:each) do

spec/models/agents/dropbox_watch_agent_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe Agents::DropboxWatchAgent do
   before(:each) do

spec/models/agents/email_agent_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe Agents::EmailAgent do
   it_behaves_like EmailConcern

spec/models/agents/email_digest_agent_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe Agents::EmailDigestAgent do
   it_behaves_like EmailConcern

spec/models/agents/event_formatting_agent_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe Agents::EventFormattingAgent do
   before do

spec/models/agents/evernote_agent_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe Agents::EvernoteAgent do
   class FakeEvernoteNoteStore

spec/models/agents/ftpsite_agent_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 require 'time'
 
 describe Agents::FtpsiteAgent do
@@ -26,7 +26,7 @@ describe Agents::FtpsiteAgent do
 
       it "should validate the integer fields" do
         @checker.options['expected_update_period_in_days'] = "nonsense"
-        expect { @checker.save! }.to raise_error;
+        expect { @checker.save! }.to raise_error(/Invalid expected_update_period_in_days format/);
         @checker.options = @site
       end
 

spec/models/agents/gap_detector_agent_spec.rb

@@ -0,0 +1,112 @@
+require 'rails_helper'
+
+describe Agents::GapDetectorAgent do
+  let(:valid_params) {
+    {
+      'name' => "my gap detector agent",
+      'options' => {
+        'window_duration_in_days' => "2",
+        'message' => "A gap was found!"
+      }
+    }
+  }
+
+  let(:agent) {
+    _agent = Agents::GapDetectorAgent.new(valid_params)
+    _agent.user = users(:bob)
+    _agent.save!
+    _agent
+  }
+
+  describe 'validation' do
+    before do
+      expect(agent).to be_valid
+    end
+
+    it 'should validate presence of message' do
+      agent.options['message'] = nil
+      expect(agent).not_to be_valid
+    end
+
+    it 'should validate presence of window_duration_in_days' do
+      agent.options['window_duration_in_days'] = ""
+      expect(agent).not_to be_valid
+
+      agent.options['window_duration_in_days'] = "wrong"
+      expect(agent).not_to be_valid
+
+      agent.options['window_duration_in_days'] = "1"
+      expect(agent).to be_valid
+
+      agent.options['window_duration_in_days'] = "0.5"
+      expect(agent).to be_valid
+    end
+  end
+
+  describe '#receive' do
+    it 'records the event if it has a created_at newer than the last seen' do
+      agent.receive([events(:bob_website_agent_event)])
+      expect(agent.memory['newest_event_created_at']).to eq events(:bob_website_agent_event).created_at.to_i
+
+      events(:bob_website_agent_event).created_at = 2.days.ago
+
+      expect {
+        agent.receive([events(:bob_website_agent_event)])
+      }.to_not change { agent.memory['newest_event_created_at'] }
+
+      events(:bob_website_agent_event).created_at = 2.days.from_now
+
+      expect {
+        agent.receive([events(:bob_website_agent_event)])
+      }.to change { agent.memory['newest_event_created_at'] }.to(events(:bob_website_agent_event).created_at.to_i)
+    end
+
+    it 'ignores the event if value_path is present and the value at the path is blank' do
+      agent.options['value_path'] = 'title'
+      agent.receive([events(:bob_website_agent_event)])
+      expect(agent.memory['newest_event_created_at']).to eq events(:bob_website_agent_event).created_at.to_i
+
+      events(:bob_website_agent_event).created_at = 2.days.from_now
+      events(:bob_website_agent_event).payload['title'] = ''
+
+      expect {
+        agent.receive([events(:bob_website_agent_event)])
+      }.to_not change { agent.memory['newest_event_created_at'] }
+
+      events(:bob_website_agent_event).payload['title'] = 'present!'
+
+      expect {
+        agent.receive([events(:bob_website_agent_event)])
+      }.to change { agent.memory['newest_event_created_at'] }.to(events(:bob_website_agent_event).created_at.to_i)
+    end
+
+    it 'clears any previous alert' do
+      agent.memory['alerted_at'] = 2.days.ago.to_i
+      agent.receive([events(:bob_website_agent_event)])
+      expect(agent.memory).to_not have_key('alerted_at')
+    end
+  end
+
+  describe '#check' do
+    it 'alerts once if no data has been received during window_duration_in_days' do
+      agent.memory['newest_event_created_at'] = 1.days.ago.to_i
+
+      expect {
+        agent.check
+      }.to_not change { agent.events.count }
+
+      agent.memory['newest_event_created_at'] = 3.days.ago.to_i
+
+      expect {
+        agent.check
+      }.to change { agent.events.count }.by(1)
+
+      expect(agent.events.last.payload).to eq ({ 'message' => 'A gap was found!',
+                                                 'gap_started_at' => agent.memory['newest_event_created_at'] })
+
+      expect {
+        agent.check
+      }.not_to change { agent.events.count }
+    end
+  end
+end

spec/models/agents/google_calendar_publish_agent_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe Agents::GoogleCalendarPublishAgent, :vcr do
   before do

spec/models/agents/growl_agent_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe Agents::GrowlAgent do
   before do

spec/models/agents/hipchat_agent_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe Agents::HipchatAgent do
   before(:each) do

spec/models/agents/human_task_agent_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe Agents::HumanTaskAgent do
   before do

spec/models/agents/imap_folder_agent_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 require 'time'
 
 describe Agents::ImapFolderAgent do

spec/models/agents/jabber_agent_spec.rb

@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'rails_helper'
 
 describe Agents::JabberAgent do
   let(:sent) { [] }