Merge branch 'master' into dsander-rails41

Conflicts:
	Gemfile

.env.example

@@ -30,8 +30,18 @@ DATABASE_PASSWORD=""
 
 # Configure Rails environment.  This should only be needed in production and may cause errors in development.
 # RAILS_ENV=production
+
+# Should Rails force all requests to use SSL?
 FORCE_SSL=false
 
+############################
+#     Allowing Signups     #
+############################
+
+# This invitation code will be required for users to signup with your Huginn installation.
+# You can see its use in user.rb.  PLEASE CHANGE THIS!
+INVITATION_CODE=try-huginn
+
 #############################
 #    Email Configuration    #
 #############################
@@ -52,14 +62,7 @@ SMTP_ENABLE_STARTTLS_AUTO=true
 
 # The address from which system emails will appear to be sent.
 EMAIL_FROM_ADDRESS=from_address@gmail.com
-
-############################
-#     Allowing Signups     #
-############################
-
-# This invitation code will be required for users to signup with your Huginn installation.
-# You can see its use in user.rb.
-INVITATION_CODE=try-huginn
+dd
 
 ###########################
 #      Agent Logging      #
@@ -83,10 +86,21 @@ AWS_SANDBOX=false
 #   Various Settings   #
 ########################
 
+# Specify the HTTP backend library for Faraday, used in WebsiteAgent.
+# You can change this depending on the performance and stability you
+# need for your service.  Any choice other than "typhoeus",
+# "net_http", or "em_http" should require you to bundle a corresponding
+# gem via Gemfile.
+FARADAY_HTTP_BACKEND=typhoeus
+
 # Allow JSONPath eval expresions. i.e., $..price[?(@ < 20)]
 # You should not allow this on a shared Huginn box because it is not secure.
 ALLOW_JSONPATH_EVAL=false
 
+# Enable this setting to allow insecure Agents like the ShellCommandAgent.  Only do this
+# when you trust everyone using your Huginn installation.
+ENABLE_INSECURE_AGENTS=false
+
 # Use Graphviz for generating diagrams instead of using Google Chart
 # Tools.  Specify a dot(1) command path built with SVG support
 # enabled.

CHANGES.md

@@ -1,6 +1,7 @@
 # Changes
 
-* 0.4 (April 10, 2014) - WebHooksController has been renamed to WebRequestsController and all HTTP verbs are now accepted and passed through to Agents' #receive_web_request method. The new DataOutputAgent returns JSON or RSS feeds of incoming Events via external web request.
+* 0.5 (April 20, 2014) - Tons of new additions! FtpsiteAgent; WebsiteAgent has xpath, multiple URL, and encoding support; regexp extractions in EventFormattingAgent; PostAgent takes default params and headers, and can make GET requests; local Graphviz support; ShellCommandAgent; BasecampAgent; HipchatAgent; and lots of bug fixes!
+* 0.4 (April 10, 2014) - WebHooksController has been renamed to WebRequestsController and all HTTP verbs are now accepted and passed through to Agents' #receive\_web\_request method. The new DataOutputAgent returns JSON or RSS feeds of incoming Events via external web request.  [Documentation is on the wiki.](https://github.com/cantino/huginn/wiki/Creating-a-new-agent#receiving-web-requests).
 * 0.31 (Jan 2, 2014)   - Agents now have an optional keep\_events\_for option that is propagated to created events' expires\_at field, and they update their events' expires\_at fields on change.
 * 0.3 (Jan 1, 2014)    - Remove symbolization of memory, options, and payloads; convert memory, options, and payloads to JSON from YAML.  Migration will perform conversion and adjust tables to be UTF-8.  Recommend making a DB backup before migrating.
 * 0.2 (Nov 6, 2013)    - PeakDetectorAgent now uses `window_duration_in_days` and `min_peak_spacing_in_days`.  Additionally, peaks trigger when the time series rises over the standard deviation multiple, not after it starts to fall.

Gemfile

@@ -35,6 +35,8 @@ gem 'geokit', '~> 1.8.4'
 gem 'geokit-rails', '~> 2.0.1'
 
 gem 'kramdown', '~> 1.3.3'
+gem 'faraday', '~> 0.9.0'
+gem 'faraday_middleware'
 gem 'typhoeus', '~> 0.6.3'
 gem 'nokogiri', '~> 1.6.1'
 

Gemfile.lock

@@ -87,6 +87,7 @@ GEM
     diff-lcs (1.2.5)
     docile (1.1.3)
     dotenv (0.10.0)
+    dotenv-deployment (0.0.2)
     dotenv-rails (0.10.0)
       dotenv (= 0.10.0)
     em-http-request (1.1.2)
@@ -107,6 +108,8 @@ GEM
     execjs (2.0.2)
     faraday (0.9.0)
       multipart-post (>= 1.2, < 3)
+    faraday_middleware (0.9.1)
+      faraday (>= 0.7.4, < 0.10)
     ffi (1.9.3)
     forecast_io (2.0.0)
       faraday
@@ -317,8 +320,11 @@ DEPENDENCIES
   delayed_job_active_record (~> 4.0.0)
   delorean
   devise (~> 3.2.4)
+  dotenv-deployment
   dotenv-rails
   em-http-request (~> 1.1.2)
+  faraday (~> 0.9.0)
+  faraday_middleware
   forecast_io (~> 2.0.0)
   foreman (~> 0.63.0)
   geokit (~> 1.8.4)
@@ -333,6 +339,7 @@ DEPENDENCIES
   nokogiri (~> 1.6.1)
   protected_attributes (~> 1.0.7)
   pry
+  rack
   rails (= 4.1.0)
   rr
   rspec

app/controllers/agents_controller.rb

@@ -1,4 +1,6 @@
 class AgentsController < ApplicationController
+  include DotHelper
+
   def index
     @agents = current_user.agents.page(params[:page])
 

app/helpers/application_helper.rb

@@ -14,19 +14,4 @@ module ApplicationHelper
       link_to '<span class="label label-warning">No</span>'.html_safe, agent_path(agent, :tab => (agent.recent_error_logs? ? 'logs' : 'details'))
     end
   end
-
-  def render_dot(dot_format_string)
-    if (command = ENV['USE_GRAPHVIZ_DOT']) &&
-       (svg = IO.popen([command, *%w[-Tsvg -q1 -o/dev/stdout /dev/stdin]], 'w+') { |dot|
-          dot.print dot_format_string
-          dot.close_write
-          dot.read
-        } rescue false)
-      svg.html_safe
-    else
-      tag('img', src: URI('https://chart.googleapis.com/chart').tap { |uri|
-            uri.query = URI.encode_www_form(cht: 'gv', chl: dot_format_string)
-          })
-    end
-  end
 end

app/helpers/dot_helper.rb

@@ -0,0 +1,40 @@
+module DotHelper
+  def render_agents_diagram(agents)
+    if (command = ENV['USE_GRAPHVIZ_DOT']) &&
+       (svg = IO.popen([command, *%w[-Tsvg -q1 -o/dev/stdout /dev/stdin]], 'w+') { |dot|
+          dot.print agents_dot(agents, true)
+          dot.close_write
+          dot.read
+        } rescue false)
+      svg.html_safe
+    else
+      tag('img', src: URI('https://chart.googleapis.com/chart').tap { |uri|
+            uri.query = URI.encode_www_form(cht: 'gv', chl: agents_dot(agents))
+          })
+    end
+  end
+
+  private
+
+  def dot_id(string)
+    # Backslash escaping seems to work for the backslash itself,
+    # despite the DOT language document.
+    '"%s"' % string.gsub(/\\/, "\\\\\\\\").gsub(/"/, "\\\\\"")
+  end
+
+  def agents_dot(agents, rich = false)
+    "digraph foo {".tap { |dot|
+      agents.each.with_index do |agent, index|
+        if rich
+          dot << '%s[URL=%s];' % [dot_id(agent.name), dot_id(agent_path(agent.id))]
+        else
+          dot << '%s;' % dot_id(agent.name)
+        end
+        agent.receivers.each do |receiver|
+          dot << "%s->%s;" % [dot_id(agent.name), dot_id(receiver.name)]
+        end
+      end
+      dot << "}"
+    }
+  end
+end

app/models/agent.rb

@@ -16,7 +16,7 @@ class Agent < ActiveRecord::Base
 
   load_types_in "Agents"
 
-  SCHEDULES = %w[every_2m every_5m every_10m every_30m every_1h every_2h every_5h every_12h every_1d every_2d every_7d
+  SCHEDULES = %w[every_1m every_2m every_5m every_10m every_30m every_1h every_2h every_5h every_12h every_1d every_2d every_7d
                  midnight 1am 2am 3am 4am 5am 6am 7am 8am 9am 10am 11am noon 1pm 2pm 3pm 4pm 5pm 6pm 7pm 8pm 9pm 10pm 11pm never]
 
   EVENT_RETENTION_SCHEDULES = [["Forever", 0], ["1 day", 1], *([2, 3, 4, 5, 7, 14, 21, 30, 45, 90, 180, 365].map {|n| ["#{n} days", n] })]

app/models/agents/post_agent.rb

@@ -1,10 +1,15 @@
 module Agents
   class PostAgent < Agent
-    cannot_be_scheduled!
     cannot_create_events!
 
+    default_schedule "never"
+
     description <<-MD
-       Post Agent receives events from other agents and send those events as the contents of a post request to a specified url. `post_url` field must specify where you would like to receive post requests and do not forget to include URI scheme (`http` or `https`)
+      A PostAgent receives events from other agents (or runs periodically), merges those events with the contents of `payload`, and sends the results as POST (or GET) requests to a specified url.
+
+      The `post_url` field must specify where you would like to send requests. Please include the URI scheme (`http` or `https`).
+
+      The `headers` field is optional.  When present, it should be a hash of headers to send with the request.
     MD
 
     event_description "Does not produce events."
@@ -12,7 +17,12 @@ module Agents
     def default_options
       {
         'post_url' => "http://www.example.com",
-        'expected_receive_period_in_days' => 1
+        'expected_receive_period_in_days' => 1,
+        'method' => 'post',
+        'payload' => {
+          'key' => 'value'
+        },
+        'headers' => {}
       }
     end
 
@@ -20,23 +30,71 @@ module Agents
       last_receive_at && last_receive_at > options['expected_receive_period_in_days'].to_i.days.ago && !recent_error_logs?
     end
 
+    def method
+      (options['method'].presence || 'post').to_s.downcase
+    end
+
+    def headers
+      options['headers'].presence || {}
+    end
+
     def validate_options
       unless options['post_url'].present? && options['expected_receive_period_in_days'].present?
         errors.add(:base, "post_url and expected_receive_period_in_days are required fields")
       end
-    end
 
-    def post_event(uri, event)
-      req = Net::HTTP::Post.new(uri.request_uri)
-      req.form_data = event
-      Net::HTTP.start(uri.hostname, uri.port, :use_ssl => uri.scheme == "https") { |http| http.request(req) }
+      if options['payload'].present? && !options['payload'].is_a?(Hash)
+        errors.add(:base, "if provided, payload must be a hash")
+      end
+
+      unless %w[post get].include?(method)
+        errors.add(:base, "method must be 'post' or 'get'")
+      end
+
+      unless headers.is_a?(Hash)
+        errors.add(:base, "if provided, headers must be a hash")
+      end
     end
 
     def receive(incoming_events)
       incoming_events.each do |event|
-        uri = URI options[:post_url]
-        post_event uri, event.payload
+        handle (options['payload'].presence || {}).merge(event.payload)
       end
     end
+
+    def check
+      handle options['payload'].presence || {}
+    end
+
+    def generate_uri(params = nil)
+      uri = URI options[:post_url]
+      uri.query = URI.encode_www_form(Hash[URI.decode_www_form(uri.query || '')].merge(params)) if params
+      uri
+    end
+
+    private
+
+    def handle(data)
+      if method == 'post'
+        post_data(data)
+      elsif method == 'get'
+        get_data(data)
+      else
+        error "Invalid method '#{method}'"
+      end
+    end
+
+    def post_data(data)
+      uri = generate_uri
+      req = Net::HTTP::Post.new(uri.request_uri, headers)
+      req.form_data = data
+      Net::HTTP.start(uri.hostname, uri.port, :use_ssl => uri.scheme == "https") { |http| http.request(req) }
+    end
+
+    def get_data(data)
+      uri = generate_uri(data)
+      req = Net::HTTP::Get.new(uri.request_uri, headers)
+      Net::HTTP.start(uri.hostname, uri.port, :use_ssl => uri.scheme == "https") { |http| http.request(req) }
+    end
   end
 end

app/models/agents/shell_command_agent.rb

@@ -0,0 +1,111 @@
+require 'open3'
+
+module Agents
+  class ShellCommandAgent < Agent
+    default_schedule "never"
+
+    def self.should_run?
+      ENV['ENABLE_INSECURE_AGENTS'] == "true"
+    end
+
+    description <<-MD
+      The ShellCommandAgent can execute commands on your local system, returning the output.
+
+      `command` specifies the command to be executed, and `path` will tell ShellCommandAgent in what directory to run this command.
+
+      `expected_update_period_in_days` is used to determine if the Agent is working.
+
+      ShellCommandAgent can also act upon received events. These events may contain their own `path` and `command` values. If they do not, ShellCommandAgent will use the configured options. For this reason, please specify defaults even if you are planning to have this Agent to respond to events.
+
+      The resulting event will contain the `command` which was executed, the `path` it was executed under, the `exit_status` of the command, the `errors`, and the actual `output`. ShellCommandAgent will not log an error if the result implies that something went wrong.
+
+      *Warning*: This type of Agent runs arbitrary commands on your system, #{Agents::ShellCommandAgent.should_run? ? "but is **currently enabled**" : "and is **currently disabled**"}.
+      Only enable this Agent if you trust everyone using your Huginn installation.
+      You can enable this Agent in your .env file by setting `ENABLE_INSECURE_AGENTS` to `true`.
+    MD
+
+    event_description <<-MD
+    Events look like this:
+
+      {
+        'command' => 'pwd',
+        'path' => '/home/Huginn',
+        'exit_status' => '0',
+        'errors' => '',
+        'output' => '/home/Huginn' 
+      }
+    MD
+
+    def default_options
+      {
+          'path' => "/",
+          'command' => "pwd",
+          'expected_update_period_in_days' => 1
+      }
+    end
+
+    def validate_options
+      unless options['path'].present? && options['command'].present? && options['expected_update_period_in_days'].present?
+        errors.add(:base, "The path, command, and expected_update_period_in_days fields are all required.")
+      end
+
+      unless File.directory?(options['path'])
+        errors.add(:base, "#{options['path']} is not a real directory.")
+      end
+    end
+
+    def working?
+      Agents::ShellCommandAgent.should_run? && event_created_within?(options['expected_update_period_in_days']) && !recent_error_logs?
+    end
+
+    def receive(incoming_events)
+      incoming_events.each do |event|
+        handle(event.payload, event)
+      end
+    end
+
+    def check
+      handle(options)
+    end
+
+    private
+
+    def handle(opts = options, event = nil)
+      if Agents::ShellCommandAgent.should_run?
+        command = opts['command'] || options['command']
+        path = opts['path'] || options['path']
+
+        result, errors, exit_status = run_command(path, command)
+
+        vals = {"command" => command, "path" => path, "exit_status" => exit_status, "errors" => errors, "output" => result}
+        created_event = create_event :payload => vals
+
+        log("Ran '#{command}' under '#{path}'", :outbound_event => created_event, :inbound_event => event)
+      else
+        log("Unable to run because insecure agents are not enabled.  Edit ENABLE_INSECURE_AGENTS in the Huginn .env configuration.")
+      end
+    end
+
+    def run_command(path, command)
+      result = nil
+      errors = nil
+      exit_status = nil
+
+      Dir.chdir(path){
+        begin
+          stdin, stdout, stderr, wait_thr = Open3.popen3(command)
+          exit_status = wait_thr.value.to_i
+          result = stdout.gets(nil)
+          errors = stderr.gets(nil)
+        rescue Exception => e
+          errors = e.to_s
+        end
+      }
+
+      result = result.to_s.strip
+      errors = errors.to_s.strip
+
+      [result, errors, exit_status]
+    end
+  end
+end

app/models/agents/trigger_agent.rb

@@ -11,6 +11,8 @@ module Agents
 
       The `type` can be one of #{VALID_COMPARISON_TYPES.map { |t| "`#{t}`" }.to_sentence} and compares with the `value`.
 
+      The `value` can be a single value or an array of values. In the case of an array, if one or more values match then the rule matches. 
+
       All rules must match for the Agent to match.  The resulting Event will have a payload message of `message`.  You can include extractions in the message, for example: `I saw a bar of: <foo.bar>`
 
       Set `expected_receive_period_in_days` to the maximum amount of time that you'd expect to pass between Events being received by this Agent.
@@ -49,25 +51,30 @@ module Agents
       incoming_events.each do |event|
         match = options['rules'].all? do |rule|
           value_at_path = Utils.value_at(event['payload'], rule['path'])
-          case rule['type']
+          rule_values = rule['value']
+          rule_values = [rule_values] unless rule_values.is_a?(Array)
+
+          match_found = rule_values.any? do |rule_value|
+            case rule['type']
             when "regex"
-              value_at_path.to_s =~ Regexp.new(rule['value'], Regexp::IGNORECASE)
+              value_at_path.to_s =~ Regexp.new(rule_value, Regexp::IGNORECASE)
             when "!regex"
-              value_at_path.to_s !~ Regexp.new(rule['value'], Regexp::IGNORECASE)
+              value_at_path.to_s !~ Regexp.new(rule_value, Regexp::IGNORECASE)
             when "field>value"
-              value_at_path.to_f > rule['value'].to_f
+              value_at_path.to_f > rule_value.to_f
             when "field>=value"
-              value_at_path.to_f >= rule['value'].to_f
+              value_at_path.to_f >= rule_value.to_f
             when "field<value"
-              value_at_path.to_f < rule['value'].to_f
+              value_at_path.to_f < rule_value.to_f
             when "field<=value"
-              value_at_path.to_f <= rule['value'].to_f
+              value_at_path.to_f <= rule_value.to_f
             when "field==value"
-              value_at_path.to_s == rule['value'].to_s
+              value_at_path.to_s == rule_value.to_s
             when "field!=value"
-              value_at_path.to_s != rule['value'].to_s
+              value_at_path.to_s != rule_value.to_s
             else
               raise "Invalid type of #{rule['type']} in TriggerAgent##{id}"
+            end
           end
         end
 

app/models/agents/website_agent.rb

@@ -1,10 +1,10 @@
 require 'nokogiri'
-require 'typhoeus'
+require 'faraday'
+require 'faraday_middleware'
 require 'date'
 
 module Agents
   class WebsiteAgent < Agent
-    cannot_receive_events!
 
     default_schedule "every_12h"
 
@@ -22,30 +22,34 @@ module Agents
 
       To tell the Agent how to parse the content, specify `extract` as a hash with keys naming the extractions and values of hashes.
 
-      When parsing HTML or XML, these sub-hashes specify how to extract with either a `css` CSS selector or a `xpath` XPath expression and either `'text': true` or `attr` pointing to an attribute name to grab.  An example:
+      When parsing HTML or XML, these sub-hashes specify how to extract with either a `css` CSS selector or a `xpath` XPath expression and either `"text": true` or `attr` pointing to an attribute name to grab.  An example:
 
-          'extract': {
-            'url': { 'css': "#comic img", 'attr': "src" },
-            'title': { 'css': "#comic img", 'attr': "title" },
-            'body_text': { 'css': "div.main", 'text': true }
+          "extract": {
+            "url": { "css": "#comic img", "attr": "src" },
+            "title": { "css": "#comic img", "attr": "title" },
+            "body_text": { "css": "div.main", "text": true }
           }
 
       When parsing JSON, these sub-hashes specify [JSONPaths](http://goessner.net/articles/JsonPath/) to the values that you care about.  For example:
 
-          'extract': {
-            'title': { 'path': "results.data[*].title" },
-            'description': { 'path': "results.data[*].description" }
+          "extract": {
+            "title": { "path": "results.data[*].title" },
+            "description": { "path": "results.data[*].description" }
           }
 
       Note that for all of the formats, whatever you extract MUST have the same number of matches for each extractor.  E.g., if you're extracting rows, all extractors must match all rows.  For generating CSS selectors, something like [SelectorGadget](http://selectorgadget.com) may be helpful.
 
-      Can be configured to use HTTP basic auth by including the `basic_auth` parameter with `username:password`.
+      Can be configured to use HTTP basic auth by including the `basic_auth` parameter with `"username:password"`, or `["username", "password"]`.
 
       Set `expected_update_period_in_days` to the maximum amount of time that you'd expect to pass between Events being created by this Agent.  This is only used to set the "working" status.
 
       Set `uniqueness_look_back` to limit the number of events checked for uniqueness (typically for performance).  This defaults to the larger of #{UNIQUENESS_LOOK_BACK} or #{UNIQUENESS_FACTOR}x the number of detected received results.
 
       Set `force_encoding` to an encoding name if the website does not return a Content-Type header with a proper charset.
+
+      Set `user_agent` to a custom User-Agent name if the website does not like the default value ("Faraday v#{Faraday::VERSION}").
+
+      The WebsiteAgent can also scrape based on incoming events. It will scrape the url contained in the `url` key of the incoming event payload.
     MD
 
     event_description do
@@ -102,30 +106,29 @@ module Agents
           errors.add(:base, "force_encoding must be a string")
         end
       end
-    end
-
-    def check
-      hydra = Typhoeus::Hydra.new
-      log "Fetching #{options['url']}"
-      request_opts = { :followlocation => true }
-      request_opts[:userpwd] = options['basic_auth'] if options['basic_auth'].present?
 
-      requests = []
+      if options['user_agent'].present?
+        errors.add(:base, "user_agent must be a string") unless options['user_agent'].is_a?(String)
+      end
 
-      if options['url'].kind_of?(Array)
-        options['url'].each do |url|
-           requests.push(Typhoeus::Request.new(url, request_opts))
-        end
-      else
-        requests.push(Typhoeus::Request.new(options['url'], request_opts))
+      begin
+        basic_auth_credentials()
+      rescue => e
+        errors.add(:base, e.message)
       end
+    end
 
-      requests.each do |request|
-        request.on_failure do |response|
-          error "Failed: #{response.inspect}"
-        end
+    def check
+      check_url options['url']
+    end
+
+    def check_url(in_url)
+      return unless in_url.present?
 
-        request.on_success do |response|
+      Array(in_url).each do |url|
+        log "Fetching #{url}"
+        response = faraday.get(url)
+        if response.success?
           body = response.body
           if (encoding = options['force_encoding']).present?
             body = body.encode(Encoding::UTF_8, encoding)
@@ -150,7 +153,7 @@ module Agents
                 when xpath = extraction_details['xpath']
                   nodes = doc.xpath(xpath)
                 else
-                  error "'css' or 'xpath' is required for HTML or XML extraction"
+                  error '"css" or "xpath" is required for HTML or XML extraction'
                   return
                 end
                 unless Nokogiri::XML::NodeSet === nodes
@@ -163,7 +166,7 @@ module Agents
                   elsif extraction_details['text']
                     node.text()
                   else
-                    error "'attr' or 'text' is required on HTML or XML extraction patterns"
+                    error '"attr" or "text" is required on HTML or XML extraction patterns'
                     return
                   end
                 }
@@ -178,14 +181,14 @@ module Agents
               error "Got an uneven number of matches for #{options['name']}: #{options['extract'].inspect}"
               return
             end
-        
+
             old_events = previous_payloads num_unique_lengths.first
             num_unique_lengths.first.times do |index|
               result = {}
               options['extract'].keys.each do |name|
                 result[name] = output[name][index]
                 if name.to_s == 'url'
-                  result[name] = URI.join(options['url'], result[name]).to_s if (result[name] =~ URI::DEFAULT_PARSER.regexp[:ABS_URI]).nil?
+                  result[name] = (response.env[:url] + result[name]).to_s
                 end
               end
 
@@ -195,10 +198,16 @@ module Agents
               end
             end
           end
+        else
+          error "Failed: #{response.inspect}"
         end
+      end
+    end
 
-        hydra.queue request
-        hydra.run
+    def receive(incoming_events)
+      incoming_events.each do |event|
+        url_to_scrape = event.payload['url']
+        check_url(url_to_scrape) if url_to_scrape =~ /^https?:\/\//i
       end
     end
 
@@ -275,5 +284,41 @@ module Agents
         false
       end
     end
+
+    def faraday
+      @faraday ||= Faraday.new { |builder|
+        if (user_agent = options['user_agent']).present?
+          builder.headers[:user_agent] = user_agent
+        end
+
+        builder.use FaradayMiddleware::FollowRedirects
+        builder.request :url_encoded
+        if userinfo = basic_auth_credentials()
+          builder.request :basic_auth, *userinfo
+        end
+
+        case backend = faraday_backend
+        when :typhoeus
+          require 'typhoeus/adapters/faraday'
+        end
+        builder.adapter backend
+      }
+    end
+
+    def faraday_backend
+      ENV.fetch('FARADAY_HTTP_BACKEND', 'typhoeus').to_sym
+    end
+
+    def basic_auth_credentials
+      case value = options['basic_auth']
+      when nil, ''
+        return nil
+      when Array
+        return value if value.size == 2
+      when /:/
+        return value.split(/:/, 2)
+      end
+      raise "bad value for basic_auth: #{value.inspect}"
+    end
   end
 end

app/views/agents/diagram.html.erb

@@ -9,17 +9,7 @@
       </div>
 
       <div class='digraph'>
-        <%
-           dot_format_string = "digraph foo {"
-           @agents.each.with_index do |agent, index|
-             dot_format_string += "\"#{agent.name}\";"
-             agent.receivers.each do |receiver|
-               dot_format_string += "\"#{agent.name}\"->\"#{receiver.name}\";"
-             end
-           end
-           dot_format_string = dot_format_string + "}"
-        %>
-        <%= render_dot(dot_format_string) %>
+        <%= render_agents_diagram(@agents) %>
       </div>
     </div>
   </div>

bin/schedule.rb

@@ -64,7 +64,7 @@ class HuginnScheduler
 
     # Schedule repeating events.
 
-    %w[2m 5m 10m 30m 1h 2h 5h 12h 1d 2d 7d].each do |schedule|
+    %w[1m 2m 5m 10m 30m 1h 2h 5h 12h 1d 2d 7d].each do |schedule|
       rufus_scheduler.every schedule do
         run_schedule "every_#{schedule}"
       end

deployment/Vagrantfile

@@ -3,37 +3,34 @@
 
 Vagrant.configure("2") do |config|
   config.omnibus.chef_version = :latest
-  config.vm.define :vb do |vb|
-    vb.vm.box = "precise32"
-    vb.vm.box_url = "http://files.vagrantup.com/precise32.box"
-    vb.vm.network :forwarded_port, host: 3000, guest: 3000
 
-    vb.vm.provision :chef_solo do |chef|
-      chef.roles_path = "roles"
-      chef.cookbooks_path = ["cookbooks", "site-cookbooks"]
-      chef.add_role("huginn_development")
-    end
+  config.vm.provision :chef_solo do |chef|
+    chef.roles_path = "roles"
+    chef.cookbooks_path = ["cookbooks", "site-cookbooks"]
+    chef.add_role("huginn_development")
+    # chef.add_role("huginn_production")
   end
 
-  config.vm.define :ec2 do |ec2|
-    ec2.vm.box = "dummy"
-    ec2.vm.box_url = "https://github.com/mitchellh/vagrant-aws/raw/master/dummy.box"
+  config.vm.provider :virtualbox do |vb, override|
+    override.vm.box = "hashicorp/precise64"
+    override.vm.network :forwarded_port, host: 3000, guest: 3000
+  end
+
+  config.vm.provider :parallels do |prl, override|
+    override.vm.box = "parallels/ubuntu-12.04"
+  end
+
+  config.vm.provider :aws do |aws, override| 
+    override.vm.box = "dummy"
+    override.vm.box_url = "https://github.com/mitchellh/vagrant-aws/raw/master/dummy.box"
 
-    ec2.vm.provider :aws do |aws, override|
-      aws.access_key_id = ""
-      aws.secret_access_key = ""
-      aws.keypair_name = ""
-      aws.region = "us-east-1"
-      aws.ami = "ami-d0f89fb9"
+    aws.access_key_id = ""
+    aws.secret_access_key = ""
+    aws.keypair_name = ""
+    aws.region = "us-east-1"
+    aws.ami = "ami-d0f89fb9"
 
-      override.ssh.username = "ubuntu"
-      override.ssh.private_key_path = ""
-    end
-    ec2.vm.provision :chef_solo do |chef|
-      chef.roles_path = "roles"
-      chef.cookbooks_path = ["cookbooks", "site-cookbooks"]
-      chef.add_role("huginn_production")
-    
-    end
+    override.ssh.username = "ubuntu"
+    override.ssh.private_key_path = ""
   end
 end

deployment/roles/huginn_development.json

@@ -23,6 +23,7 @@
              "recipe[git]",
              "recipe[apt]",
              "recipe[mysql::server]",
+             "recipe[mysql::client]",
              "recipe[nodejs::install_from_binary]",
              "recipe[huginn_development]"
            ]

deployment/site-cookbooks/huginn_development/recipes/default.rb

@@ -16,12 +16,19 @@ group "huginn" do
   action :create
 end
 
-%w("ruby1.9.1" "ruby1.9.1-dev" "libxslt-dev" "libxml2-dev" "curl" "libmysqlclient-dev").each do |pkg|
+%w("ruby1.9.1" "ruby1.9.1-dev" "libxslt-dev" "libxml2-dev" "curl" "libmysqlclient-dev" "rubygems").each do |pkg|
   package pkg do
     action :install
   end
 end
 
+bash "Setting default ruby version to 1.9" do
+  code <<-EOH
+    update-alternatives --set ruby /usr/bin/ruby1.9.1
+    update-alternatives --set gem /usr/bin/gem1.9.1
+  EOH
+end
+
 git "/home/huginn/huginn" do
   repository 'git://github.com/cantino/huginn.git'
   reference 'master'
@@ -48,7 +55,7 @@ bash "huginn dependencies" do
     export LANG="en_US.UTF-8"
     export LC_ALL="en_US.UTF-8"
     sudo bundle install
-    sed s/REPLACE_ME_NOW\!/$(sudo rake secret)/ .env.example > .env
+    sed s/REPLACE_ME_NOW\!/$(sudo bundle exec rake secret)/ .env.example > .env
     sudo bundle exec rake db:create
     sudo bundle exec rake db:migrate
     sudo bundle exec rake db:seed
@@ -59,6 +66,6 @@ bash "huginn has been installed and will start in a minute" do
   user "huginn"
   cwd "/home/huginn/huginn"
   code <<-EOH
-    sudo foreman start
+    sudo nohup foreman start &
     EOH
 end

spec/helpers/dot_helper_spec.rb

@@ -0,0 +1,48 @@
+require 'spec_helper'
+
+describe DotHelper do
+  describe "#dot_id" do
+    it "properly escapes double quotaion and backslash" do
+      dot_id('hello\\"').should == '"hello\\\\\\""'
+    end
+  end
+
+  describe "with example Agents" do
+    class Agents::DotFoo < Agent
+      default_schedule "2pm"
+
+      def check
+        create_event :payload => {}
+      end
+    end
+
+    class Agents::DotBar < Agent
+      cannot_be_scheduled!
+
+      def check
+        create_event :payload => {}
+      end
+    end
+
+    before do
+      stub(Agents::DotFoo).valid_type?("Agents::DotFoo") { true }
+      stub(Agents::DotBar).valid_type?("Agents::DotBar") { true }
+    end
+
+    describe "#agents_dot" do
+      it "generates a DOT script" do
+        @foo = Agents::DotFoo.new(:name => "foo")
+        @foo.user = users(:bob)
+        @foo.save!
+
+        @bar = Agents::DotBar.new(:name => "bar")
+        @bar.user = users(:bob)
+        @bar.sources << @foo
+        @bar.save!
+
+        agents_dot([@foo, @bar]).should == 'digraph foo {"foo";"foo"->"bar";"bar";}'
+        agents_dot([@foo, @bar], true).should == 'digraph foo {"foo"[URL="/agents/%d"];"foo"->"bar";"bar"[URL="/agents/%d"];}' % [@foo.id, @bar.id]
+      end
+    end
+  end
+end

spec/models/agents/post_agent_spec.rb

@@ -5,8 +5,11 @@ describe Agents::PostAgent do
     @valid_params = {
       :name => "somename",
       :options => {
-        :post_url => "http://www.example.com",
-        :expected_receive_period_in_days => 1
+        'post_url' => "http://www.example.com",
+        'expected_receive_period_in_days' => 1,
+        'payload' => {
+          'default' => 'value'
+        }
       }
     }
 
@@ -17,28 +20,69 @@ describe Agents::PostAgent do
     @event = Event.new
     @event.agent = agents(:jane_weather_agent)
     @event.payload = {
-      :somekey => "somevalue",
-      :someotherkey => {
-        :somekey => "value"
+      'somekey' => 'somevalue',
+      'someotherkey' => {
+        'somekey' => 'value'
       }
     }
 
-    @sent_messages = []
-    stub.any_instance_of(Agents::PostAgent).post_event { |uri, event| @sent_messages << event }
+    @sent_posts = []
+    @sent_gets = []
+    stub.any_instance_of(Agents::PostAgent).post_data { |data| @sent_posts << data }
+    stub.any_instance_of(Agents::PostAgent).get_data { |data| @sent_gets << data }
   end
 
   describe "#receive" do
-    it "checks if it can handle multiple events" do
+    it "can handle multiple events and merge the payloads with options['payload']" do
       event1 = Event.new
       event1.agent = agents(:bob_weather_agent)
       event1.payload = {
-        :xyz => "value1",
-        :message => "value2"
+        'xyz' => 'value1',
+        'message' => 'value2',
+        'default' => 'value2'
       }
 
       lambda {
-        @checker.receive([@event, event1])
-      }.should change { @sent_messages.length }.by(2)
+        lambda {
+          @checker.receive([@event, event1])
+        }.should change { @sent_posts.length }.by(2)
+      }.should_not change { @sent_gets.length }
+
+      @sent_posts[0].should == @event.payload.merge('default' => 'value')
+      @sent_posts[1].should == event1.payload
+    end
+
+    it "can make GET requests" do
+      @checker.options['method'] = 'get'
+
+      lambda {
+        lambda {
+          @checker.receive([@event])
+        }.should change { @sent_gets.length }.by(1)
+      }.should_not change { @sent_posts.length }
+
+      @sent_gets[0].should == @event.payload.merge('default' => 'value')
+    end
+  end
+
+  describe "#check" do
+    it "sends options['payload'] as a POST request" do
+      lambda {
+        @checker.check
+      }.should change { @sent_posts.length }.by(1)
+
+      @sent_posts[0].should == @checker.options['payload']
+    end
+
+    it "sends options['payload'] as a GET request" do
+      @checker.options['method'] = 'get'
+      lambda {
+        lambda {
+          @checker.check
+        }.should change { @sent_gets.length }.by(1)
+      }.should_not change { @sent_posts.length }
+
+      @sent_gets[0].should == @checker.options['payload']
     end
   end
 
@@ -59,13 +103,82 @@ describe Agents::PostAgent do
     end
 
     it "should validate presence of post_url" do
-      @checker.options[:post_url] = ""
+      @checker.options['post_url'] = ""
       @checker.should_not be_valid
     end
 
     it "should validate presence of expected_receive_period_in_days" do
-      @checker.options[:expected_receive_period_in_days] = ""
+      @checker.options['expected_receive_period_in_days'] = ""
       @checker.should_not be_valid
     end
+
+    it "should validate method as post or get, defaulting to post" do
+      @checker.options['method'] = ""
+      @checker.method.should == "post"
+      @checker.should be_valid
+
+      @checker.options['method'] = "POST"
+      @checker.method.should == "post"
+      @checker.should be_valid
+
+      @checker.options['method'] = "get"
+      @checker.method.should == "get"
+      @checker.should be_valid
+
+      @checker.options['method'] = "wut"
+      @checker.method.should == "wut"
+      @checker.should_not be_valid
+    end
+
+    it "should validate payload as a hash, if present" do
+      @checker.options['payload'] = ""
+      @checker.should be_valid
+
+      @checker.options['payload'] = "hello"
+      @checker.should_not be_valid
+
+      @checker.options['payload'] = ["foo", "bar"]
+      @checker.should_not be_valid
+
+      @checker.options['payload'] = { 'this' => 'that' }
+      @checker.should be_valid
+    end
+
+    it "requires headers to be a hash, if present" do
+      @checker.options['headers'] = [1,2,3]
+      @checker.should_not be_valid
+
+      @checker.options['headers'] = "hello world"
+      @checker.should_not be_valid
+
+      @checker.options['headers'] = ""
+      @checker.should be_valid
+
+      @checker.options['headers'] = {}
+      @checker.should be_valid
+
+      @checker.options['headers'] = { "Authorization" => "foo bar" }
+      @checker.should be_valid
+    end
+  end
+
+  describe "#generate_uri" do
+    it "merges params with any in the post_url" do
+      @checker.options['post_url'] = "http://example.com/a/path?existing_param=existing_value"
+      uri = @checker.generate_uri("some_param" => "some_value", "another_param" => "another_value")
+      uri.request_uri.should == "/a/path?existing_param=existing_value&some_param=some_value&another_param=another_value"
+    end
+
+    it "works fine with urls that do not have a query" do
+      @checker.options['post_url'] = "http://example.com/a/path"
+      uri = @checker.generate_uri("some_param" => "some_value", "another_param" => "another_value")
+      uri.request_uri.should == "/a/path?some_param=some_value&another_param=another_value"
+    end
+
+    it "just returns the post_uri when no params are given" do
+      @checker.options['post_url'] = "http://example.com/a/path?existing_param=existing_value"
+      uri = @checker.generate_uri
+      uri.request_uri.should == "/a/path?existing_param=existing_value"
+    end
   end
 end

spec/models/agents/shell_command_agent_spec.rb

@@ -0,0 +1,99 @@
+require 'spec_helper'
+
+describe Agents::ShellCommandAgent do
+  before do
+    @valid_path = Dir.pwd
+
+    @valid_params = {
+        :path  => @valid_path,
+        :command  => "pwd",
+        :expected_update_period_in_days => "1",
+      }
+
+    @checker = Agents::ShellCommandAgent.new(:name => "somename", :options => @valid_params)
+    @checker.user = users(:jane)
+    @checker.save!
+
+    @event = Event.new
+    @event.agent = agents(:jane_weather_agent)
+    @event.payload = {
+      :command => "ls"
+    }
+    @event.save!
+
+    stub(Agents::ShellCommandAgent).should_run? { true }
+  end
+
+  describe "validation" do
+    before do
+      @checker.should be_valid
+    end
+
+    it "should validate presence of necessary fields" do
+      @checker.options[:command] = nil
+      @checker.should_not be_valid
+    end
+
+    it "should validate path" do
+      @checker.options[:path] = 'notarealpath/itreallyisnt'
+      @checker.should_not be_valid
+    end
+
+    it "should validate path" do
+      @checker.options[:path] = '/'
+      @checker.should be_valid
+    end
+  end
+
+  describe "#working?" do
+    it "generating events as scheduled" do
+      stub(@checker).run_command(@valid_path, 'pwd') { ["fake pwd output", "", 0] }
+
+      @checker.should_not be_working
+      @checker.check
+      @checker.reload.should be_working
+      three_days_from_now = 3.days.from_now
+      stub(Time).now { three_days_from_now }
+      @checker.should_not be_working
+    end
+  end
+
+  describe "#check" do
+    before do
+      stub(@checker).run_command(@valid_path, 'pwd') { ["fake pwd output", "", 0] }
+    end
+
+    it "should create an event when checking" do
+      expect { @checker.check }.to change { Event.count }.by(1)
+      Event.last.payload[:path].should == @valid_path
+      Event.last.payload[:command].should == 'pwd'
+      Event.last.payload[:output].should == "fake pwd output"
+    end
+
+    it "does not run when should_run? is false" do
+      stub(Agents::ShellCommandAgent).should_run? { false }
+      expect { @checker.check }.not_to change { Event.count }
+    end
+  end
+
+  describe "#receive" do
+    before do
+      stub(@checker).run_command(@valid_path, @event.payload[:command]) { ["fake ls output", "", 0] }
+    end
+
+    it "creates events" do
+      @checker.receive([@event])
+      Event.last.payload[:path].should == @valid_path
+      Event.last.payload[:command].should == @event.payload[:command]
+      Event.last.payload[:output].should == "fake ls output"
+    end
+
+    it "does not run when should_run? is false" do
+      stub(Agents::ShellCommandAgent).should_run? { false }
+
+      expect {
+        @checker.receive([@event])
+      }.not_to change { Event.count }
+    end
+  end
+end

spec/models/agents/trigger_agent_spec.rb

@@ -71,6 +71,28 @@ describe Agents::TriggerAgent do
       }.should change { Event.count }.by(1)
     end
 
+    it "handles array of regex" do
+      @event.payload['foo']['bar']['baz'] = "a222b"
+      @checker.options['rules'][0] = {
+        'type' => "regex",
+        'value' => ["a\\db", "a\\Wb"],
+        'path' => "foo.bar.baz",
+      }
+      lambda {
+        @checker.receive([@event])
+      }.should_not change { Event.count }
+
+      @event.payload['foo']['bar']['baz'] = "a2b"
+      lambda {
+        @checker.receive([@event])
+      }.should change { Event.count }.by(1)
+
+      @event.payload['foo']['bar']['baz'] = "a b"
+      lambda {
+        @checker.receive([@event])
+      }.should change { Event.count }.by(1)
+    end
+
     it "handles negated regex" do
       @event.payload['foo']['bar']['baz'] = "a2b"
       @checker.options['rules'][0] = {
@@ -89,6 +111,24 @@ describe Agents::TriggerAgent do
       }.should change { Event.count }.by(1)
     end
 
+    it "handles array of negated regex" do
+      @event.payload['foo']['bar']['baz'] = "a2b"
+      @checker.options['rules'][0] = {
+        'type' => "!regex",
+        'value' => ["a\\db", "a2b"],
+        'path' => "foo.bar.baz",
+      }
+
+      lambda {
+        @checker.receive([@event])
+      }.should_not change { Event.count }
+
+      @event.payload['foo']['bar']['baz'] = "a3b"
+      lambda {
+        @checker.receive([@event])
+      }.should change { Event.count }.by(1)
+    end
+
     it "puts can extract values into the message based on paths" do
       @checker.receive([@event])
       Event.last.payload['message'].should == "I saw 'a2b' from Joe"
@@ -109,6 +149,21 @@ describe Agents::TriggerAgent do
       }.should_not change { Event.count }
     end
 
+    it "handles array of numerical comparisons" do
+      @event.payload['foo']['bar']['baz'] = "5"
+      @checker.options['rules'].first['value'] = [6, 3]
+      @checker.options['rules'].first['type'] = "field<value"
+
+      lambda {
+        @checker.receive([@event])
+      }.should change { Event.count }.by(1)
+
+      @checker.options['rules'].first['value'] = [4, 3]
+      lambda {
+        @checker.receive([@event])
+      }.should_not change { Event.count }
+    end
+
     it "handles exact comparisons" do
       @event.payload['foo']['bar']['baz'] = "hello world"
       @checker.options['rules'].first['type'] = "field==value"
@@ -124,6 +179,21 @@ describe Agents::TriggerAgent do
       }.should change { Event.count }.by(1)
     end
 
+    it "handles array of exact comparisons" do
+      @event.payload['foo']['bar']['baz'] = "hello world"
+      @checker.options['rules'].first['type'] = "field==value"
+
+      @checker.options['rules'].first['value'] = ["hello there", "hello universe"]
+      lambda {
+        @checker.receive([@event])
+      }.should_not change { Event.count }
+
+      @checker.options['rules'].first['value'] = ["hello world", "hello universe"]
+      lambda {
+        @checker.receive([@event])
+      }.should change { Event.count }.by(1)
+    end
+
     it "handles negated comparisons" do
       @event.payload['foo']['bar']['baz'] = "hello world"
       @checker.options['rules'].first['type'] = "field!=value"
@@ -140,6 +210,22 @@ describe Agents::TriggerAgent do
       }.should change { Event.count }.by(1)
     end
 
+    it "handles array of negated comparisons" do
+      @event.payload['foo']['bar']['baz'] = "hello world"
+      @checker.options['rules'].first['type'] = "field!=value"
+      @checker.options['rules'].first['value'] = ["hello world", "hello world"]
+
+      lambda {
+        @checker.receive([@event])
+      }.should_not change { Event.count }
+
+      @checker.options['rules'].first['value'] = ["hello there", "hello world"]
+
+      lambda {
+        @checker.receive([@event])
+      }.should change { Event.count }.by(1)
+    end
+
     it "does fine without dots in the path" do
       @event.payload = { 'hello' => "world" }
       @checker.options['rules'].first['type'] = "field==value"

spec/models/agents/website_agent_spec.rb

@@ -331,11 +331,26 @@ describe Agents::WebsiteAgent do
         end
       end
     end
+
+    describe "#receive" do
+      it "should scrape from the url element in incoming event payload" do
+        @event = Event.new
+        @event.agent = agents(:bob_rain_notifier_agent)
+        @event.payload = { 'url' => "http://xkcd.com" }
+
+        lambda {
+          @checker.options = @site
+          @checker.receive([@event])
+        }.should change { Event.count }.by(1)
+      end
+    end
   end
 
   describe "checking with http basic auth" do
     before do
-      stub_request(:any, /user:pass/).to_return(:body => File.read(Rails.root.join("spec/data_fixtures/xkcd.html")), :status => 200)
+      stub_request(:any, /example/).
+        with(headers: { 'Authorization' => "Basic #{['user:pass'].pack('m').chomp}" }).
+        to_return(:body => File.read(Rails.root.join("spec/data_fixtures/xkcd.html")), :status => 200)
       @site = {
         'name' => "XKCD",
         'expected_update_period_in_days' => 2,
@@ -361,4 +376,35 @@ describe Agents::WebsiteAgent do
       end
     end
   end
+
+  describe "checking with User-Agent" do
+    before do
+      stub_request(:any, /example/).
+        with(headers: { 'User-Agent' => 'Sushi' }).
+        to_return(:body => File.read(Rails.root.join("spec/data_fixtures/xkcd.html")), :status => 200)
+      @site = {
+        'name' => "XKCD",
+        'expected_update_period_in_days' => 2,
+        'type' => "html",
+        'url' => "http://www.example.com",
+        'mode' => 'on_change',
+        'extract' => {
+          'url' => { 'css' => "#comic img", 'attr' => "src" },
+          'title' => { 'css' => "#comic img", 'attr' => "alt" },
+          'hovertext' => { 'css' => "#comic img", 'attr' => "title" }
+        },
+        'user_agent' => "Sushi"
+      }
+      @checker = Agents::WebsiteAgent.new(:name => "ua", :options => @site)
+      @checker.user = users(:bob)
+      @checker.save!
+    end
+
+    describe "#check" do
+      it "should check for changes" do
+        lambda { @checker.check }.should change { Event.count }.by(1)
+        lambda { @checker.check }.should_not change { Event.count }
+      end
+    end
+  end
 end