Merge branch 'master' into 196-disable-agents

Conflicts:
	app/views/agents/diagram.html.erb
	app/views/agents/show.html.erb

Manually Edited:
	app/helpers/dot_helper.rb

.env.example

@@ -31,6 +31,17 @@ DATABASE_PASSWORD=""
 # Configure Rails environment.  This should only be needed in production and may cause errors in development.
 # RAILS_ENV=production
 
+# Should Rails force all requests to use SSL?
+FORCE_SSL=false
+
+############################
+#     Allowing Signups     #
+############################
+
+# This invitation code will be required for users to signup with your Huginn installation.
+# You can see its use in user.rb.  PLEASE CHANGE THIS!
+INVITATION_CODE=try-huginn
+
 #############################
 #    Email Configuration    #
 #############################
@@ -52,13 +63,6 @@ SMTP_ENABLE_STARTTLS_AUTO=true
 # The address from which system emails will appear to be sent.
 EMAIL_FROM_ADDRESS=from_address@gmail.com
 
-############################
-#     Allowing Signups     #
-############################
-
-# This invitation code will be required for users to signup with your Huginn installation.
-# You can see its use in user.rb.
-INVITATION_CODE=try-huginn
 
 ###########################
 #      Agent Logging      #
@@ -82,6 +86,13 @@ AWS_SANDBOX=false
 #   Various Settings   #
 ########################
 
+# Specify the HTTP backend library for Faraday, used in WebsiteAgent.
+# You can change this depending on the performance and stability you
+# need for your service.  Any choice other than "typhoeus",
+# "net_http", or "em_http" should require you to bundle a corresponding
+# gem via Gemfile.
+FARADAY_HTTP_BACKEND=typhoeus
+
 # Allow JSONPath eval expresions. i.e., $..price[?(@ < 20)]
 # You should not allow this on a shared Huginn box because it is not secure.
 ALLOW_JSONPATH_EVAL=false

.travis.yml

@@ -1,5 +1,7 @@
 language: ruby
 bundler_args: --without development production
+env:
+  - APP_SECRET_TOKEN=b2724973fd81c2f4ac0f92ac48eb3f0152c4a11824c122bcf783419a4c51d8b9bba81c8ba6a66c7de599677c7f486242cf819775c433908e77c739c5c8ae118d
 rvm:
   - 2.0.0
   - 2.1.1
@@ -15,6 +17,6 @@ notifications:
     channels:
       - "chat.freenode.net#huginn"
     template:
-      - "%{repository}#%{build_number} (%{branch} - %{commit} : %{author}): %{message}"
+      - "<%{author}> %{branch} - %{commit} (%{commit_message}): %{message}"
       - "Change view : %{compare_url}"
       - "Build details : %{build_url}"

Gemfile

@@ -1,14 +1,26 @@
 source 'https://rubygems.org'
 
-gem 'rails', '3.2.17'
-gem 'mysql2', '~> 0.3.13'
-gem 'devise', '~> 3.0.0'
-gem 'kaminari', '~> 0.14.1'
+gem 'protected_attributes', '~>1.0.7'
+
+gem 'rails', '4.1.0'
+
+case RUBY_PLATFORM
+when /freebsd/i
+  # Seems FreeBSD's zoneinfo is not exactly what tzinfo expects
+  gem 'tzinfo-data'
+else
+  # Windows does not include zoneinfo files, so bundle the tzinfo-data gem
+  gem 'tzinfo-data', platforms: [:mswin]
+end
+
+gem 'mysql2', '~> 0.3.15'
+gem 'devise', '~> 3.2.4'
+gem 'kaminari', '~> 0.15.1'
 gem 'bootstrap-kaminari-views', '~> 0.0.2'
 gem 'rufus-scheduler', '~> 3.0.7', require: false
-gem 'json', '>= 1.7.7'
+gem 'json', '~> 1.8.1'
 gem 'jsonpath', '~> 0.5.3'
-gem 'twilio-ruby', '~> 3.10.0'
+gem 'twilio-ruby', '~> 3.11.5'
 gem 'ruby-growl', '~> 4.1.0'
 
 gem 'delayed_job', '~> 4.0.0'
@@ -20,27 +32,29 @@ gem 'daemons', '~> 1.1.9'
 
 gem 'foreman', '~> 0.63.0'
 
-gem 'sass-rails',   '~> 3.2.3'
-gem 'coffee-rails', '~> 3.2.1'
-gem 'uglifier', '>= 1.0.3'
-gem 'select2-rails', '~> 3.4.3'
-gem 'jquery-rails', '~> 3.0.4'
+gem 'sass-rails',   '~> 4.0.0'
+gem 'coffee-rails', '~> 4.0.0'
+gem 'uglifier', '>= 1.3.0'
+gem 'select2-rails', '~> 3.5.4'
+gem 'jquery-rails', '~> 3.1.0'
 gem 'ace-rails-ap', '~> 2.0.1'
 
 # geokit-rails doesn't work with geokit 1.8.X but it specifies ~> 1.5
 # in its own Gemfile.
-gem 'geokit', '~> 1.6.7'
-gem 'geokit-rails3', '~> 0.1.5'
+gem 'geokit', '~> 1.8.4'
+gem 'geokit-rails', '~> 2.0.1'
 
-gem 'kramdown', '~> 1.1.0'
+gem 'kramdown', '~> 1.3.3'
+gem 'faraday', '~> 0.9.0'
+gem 'faraday_middleware'
 gem 'typhoeus', '~> 0.6.3'
-gem 'nokogiri', '~> 1.6.0'
+gem 'nokogiri', '~> 1.6.1'
 
-gem 'wunderground', '~> 1.1.0'
+gem 'wunderground', '~> 1.2.0'
 gem 'forecast_io', '~> 2.0.0'
-gem 'rturk', '~> 2.11.0'
+gem 'rturk', '~> 2.12.1'
 
-gem 'twitter', '~> 5.7.1'
+gem 'twitter', '~> 5.8.0'
 gem 'twitter-stream', github: 'cantino/twitter-stream', branch: 'master'
 gem 'em-http-request', '~> 1.1.2'
 gem 'weibo_2', '~> 0.1.4'
@@ -60,6 +74,12 @@ group :development, :test do
   gem 'rspec'
   gem 'shoulda-matchers'
   gem 'rr'
+  gem 'delorean'
   gem 'webmock', require: false
   gem 'coveralls', require: false
 end
+
+group :production do
+  gem 'dotenv-deployment'
+  gem 'rack'
+end

Gemfile.lock

@@ -12,38 +12,35 @@ GEM
   remote: https://rubygems.org/
   specs:
     ace-rails-ap (2.0.1)
-    actionmailer (3.2.17)
-      actionpack (= 3.2.17)
+    actionmailer (4.1.0)
+      actionpack (= 4.1.0)
+      actionview (= 4.1.0)
       mail (~> 2.5.4)
-    actionpack (3.2.17)
-      activemodel (= 3.2.17)
-      activesupport (= 3.2.17)
-      builder (~> 3.0.0)
+    actionpack (4.1.0)
+      actionview (= 4.1.0)
+      activesupport (= 4.1.0)
+      rack (~> 1.5.2)
+      rack-test (~> 0.6.2)
+    actionview (4.1.0)
+      activesupport (= 4.1.0)
+      builder (~> 3.1)
       erubis (~> 2.7.0)
-      journey (~> 1.0.4)
-      rack (~> 1.4.5)
-      rack-cache (~> 1.2)
-      rack-test (~> 0.6.1)
-      sprockets (~> 2.2.1)
-    activemodel (3.2.17)
-      activesupport (= 3.2.17)
-      builder (~> 3.0.0)
-    activerecord (3.2.17)
-      activemodel (= 3.2.17)
-      activesupport (= 3.2.17)
-      arel (~> 3.0.2)
-      tzinfo (~> 0.3.29)
-    activeresource (3.2.17)
-      activemodel (= 3.2.17)
-      activesupport (= 3.2.17)
-    activesupport (3.2.17)
-      i18n (~> 0.6, >= 0.6.4)
-      multi_json (~> 1.0)
+    activemodel (4.1.0)
+      activesupport (= 4.1.0)
+      builder (~> 3.1)
+    activerecord (4.1.0)
+      activemodel (= 4.1.0)
+      activesupport (= 4.1.0)
+      arel (~> 5.0.0)
+    activesupport (4.1.0)
+      i18n (~> 0.6, >= 0.6.9)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.1)
+      tzinfo (~> 1.1)
     addressable (2.3.6)
-    arel (3.0.3)
+    arel (5.0.1.20140414130214)
     bcrypt (3.1.7)
-    bcrypt-ruby (3.1.5)
-      bcrypt (>= 3.1.3)
     better_errors (1.1.0)
       coderay (>= 1.0.0)
       erubis (>= 2.6.6)
@@ -53,11 +50,12 @@ GEM
       kaminari (>= 0.13)
       rails (>= 3.1)
     buftok (0.2.0)
-    builder (3.0.4)
+    builder (3.2.2)
+    chronic (0.10.2)
     coderay (1.1.0)
-    coffee-rails (3.2.2)
+    coffee-rails (4.0.1)
       coffee-script (>= 2.2.0)
-      railties (~> 3.2.0)
+      railties (>= 4.0.0, < 5.0)
     coffee-script (2.2.0)
       coffee-script-source
       execjs
@@ -78,14 +76,18 @@ GEM
     delayed_job_active_record (4.0.1)
       activerecord (>= 3.0, < 4.2)
       delayed_job (>= 3.0, < 4.1)
-    devise (3.0.4)
-      bcrypt-ruby (~> 3.0)
+    delorean (2.1.0)
+      chronic
+    devise (3.2.4)
+      bcrypt (~> 3.0)
       orm_adapter (~> 0.1)
       railties (>= 3.2.6, < 5)
+      thread_safe (~> 0.1)
       warden (~> 1.2.3)
     diff-lcs (1.2.5)
     docile (1.1.3)
     dotenv (0.10.0)
+    dotenv-deployment (0.0.2)
     dotenv-rails (0.10.0)
       dotenv (= 0.10.0)
     em-http-request (1.1.2)
@@ -106,6 +108,8 @@ GEM
     execjs (2.0.2)
     faraday (0.9.0)
       multipart-post (>= 1.2, < 3)
+    faraday_middleware (0.9.1)
+      faraday (>= 0.7.4, < 0.10)
     ffi (1.9.3)
     forecast_io (2.0.0)
       faraday
@@ -114,11 +118,11 @@ GEM
     foreman (0.63.0)
       dotenv (>= 0.7)
       thor (>= 0.13.6)
-    geokit (1.6.7)
+    geokit (1.8.4)
       multi_json (>= 1.3.2)
-    geokit-rails3 (0.1.5)
+    geokit-rails (2.0.1)
       geokit (~> 1.5)
-      rails (~> 3.0)
+      rails (>= 3.0)
     hashie (2.0.5)
     hike (1.2.3)
     hipchat (1.1.0)
@@ -130,8 +134,7 @@ GEM
       json (~> 1.8)
       multi_xml (>= 0.5.2)
     i18n (0.6.9)
-    journey (1.0.4)
-    jquery-rails (3.0.4)
+    jquery-rails (3.1.0)
       railties (>= 3.0, < 5.0)
       thor (>= 0.14, < 2.0)
     json (1.8.1)
@@ -139,10 +142,10 @@ GEM
       multi_json
     jwt (0.1.11)
       multi_json (>= 1.5)
-    kaminari (0.14.1)
+    kaminari (0.15.1)
       actionpack (>= 3.0.0)
       activesupport (>= 3.0.0)
-    kramdown (1.1.0)
+    kramdown (1.3.3)
     libv8 (3.16.14.3)
     macaddr (1.7.1)
       systemu (~> 2.6.2)
@@ -154,6 +157,7 @@ GEM
     method_source (0.8.2)
     mime-types (1.25.1)
     mini_portile (0.5.3)
+    minitest (5.3.3)
     multi_json (1.9.2)
     multi_xml (0.5.5)
     multipart-post (2.0.0)
@@ -169,35 +173,31 @@ GEM
       rack (~> 1.2)
     orm_adapter (0.5.0)
     polyglot (0.3.4)
+    protected_attributes (1.0.7)
+      activemodel (>= 4.0.1, < 5.0)
     pry (0.9.12.6)
       coderay (~> 1.0)
       method_source (~> 0.8)
       slop (~> 3.4)
-    rack (1.4.5)
-    rack-cache (1.2)
-      rack (>= 0.4)
-    rack-ssl (1.3.4)
-      rack
+    rack (1.5.2)
     rack-test (0.6.2)
       rack (>= 1.0)
-    rails (3.2.17)
-      actionmailer (= 3.2.17)
-      actionpack (= 3.2.17)
-      activerecord (= 3.2.17)
-      activeresource (= 3.2.17)
-      activesupport (= 3.2.17)
-      bundler (~> 1.0)
-      railties (= 3.2.17)
-    railties (3.2.17)
-      actionpack (= 3.2.17)
-      activesupport (= 3.2.17)
-      rack-ssl (~> 1.3.2)
+    rails (4.1.0)
+      actionmailer (= 4.1.0)
+      actionpack (= 4.1.0)
+      actionview (= 4.1.0)
+      activemodel (= 4.1.0)
+      activerecord (= 4.1.0)
+      activesupport (= 4.1.0)
+      bundler (>= 1.3.0, < 2.0)
+      railties (= 4.1.0)
+      sprockets-rails (~> 2.0)
+    railties (4.1.0)
+      actionpack (= 4.1.0)
+      activesupport (= 4.1.0)
       rake (>= 0.8.7)
-      rdoc (~> 3.4)
-      thor (>= 0.14.6, < 2.0)
-    rake (10.2.2)
-    rdoc (3.12.2)
-      json (~> 1.4)
+      thor (>= 0.18.1, < 2.0)
+    rake (10.3.1)
     ref (1.0.5)
     rest-client (1.6.7)
       mime-types (>= 1.16)
@@ -218,7 +218,7 @@ GEM
       rspec-core (~> 2.14.0)
       rspec-expectations (~> 2.14.0)
       rspec-mocks (~> 2.14.0)
-    rturk (2.11.3)
+    rturk (2.12.1)
       erector
       nokogiri
       rest-client
@@ -227,13 +227,13 @@ GEM
     rufus-scheduler (3.0.7)
       tzinfo
     safe_yaml (1.0.2)
-    sass (3.3.5)
-    sass-rails (3.2.6)
-      railties (~> 3.2.0)
-      sass (>= 3.1.10)
-      tilt (~> 1.3)
-    select2-rails (3.4.9)
-      sass-rails
+    sass (3.2.19)
+    sass-rails (4.0.3)
+      railties (>= 4.0.0, < 5.0)
+      sass (~> 3.2.0)
+      sprockets (~> 2.8, <= 2.11.0)
+      sprockets-rails (~> 2.0)
+    select2-rails (3.5.4)
       thor (~> 0.14)
     shoulda-matchers (2.6.0)
       activesupport (>= 3.0.0)
@@ -244,11 +244,15 @@ GEM
       simplecov-html (~> 0.8.0)
     simplecov-html (0.8.0)
     slop (3.5.0)
-    sprockets (2.2.2)
+    sprockets (2.11.0)
       hike (~> 1.2)
       multi_json (~> 1.0)
       rack (~> 1.0)
       tilt (~> 1.1, != 1.3.0)
+    sprockets-rails (2.1.3)
+      actionpack (>= 3.0)
+      activesupport (>= 3.0)
+      sprockets (~> 2.8)
     systemu (2.6.4)
     term-ansicolor (1.3.0)
       tins (~> 1.0)
@@ -262,11 +266,11 @@ GEM
     treetop (1.4.15)
       polyglot
       polyglot (>= 0.3.1)
-    twilio-ruby (3.10.1)
+    twilio-ruby (3.11.5)
       builder (>= 2.1.2)
       jwt (>= 0.1.2)
       multi_json (>= 1.3.0)
-    twitter (5.7.1)
+    twitter (5.8.0)
       addressable (~> 2.3)
       buftok (~> 0.2.0)
       equalizer (~> 0.0.9)
@@ -279,7 +283,10 @@ GEM
       simple_oauth (~> 0.2.0)
     typhoeus (0.6.8)
       ethon (>= 0.7.0)
-    tzinfo (0.3.39)
+    tzinfo (1.1.0)
+      thread_safe (~> 0.1)
+    tzinfo-data (1.2014.2)
+      tzinfo (>= 1.0.0)
     uglifier (2.5.0)
       execjs (>= 0.3.0)
       json (>= 1.8.0)
@@ -287,7 +294,7 @@ GEM
       macaddr (~> 1.0)
     warden (1.2.3)
       rack (>= 1.0)
-    webmock (1.13.0)
+    webmock (1.17.4)
       addressable (>= 2.2.7)
       crack (>= 0.3.2)
     weibo_2 (0.1.6)
@@ -295,7 +302,7 @@ GEM
       multi_json (~> 1)
       oauth2 (~> 0.9.1)
       rest-client (~> 1.6.7)
-    wunderground (1.1.0)
+    wunderground (1.2.0)
       addressable
       httparty (> 0.6.0)
       json (> 1.4.0)
@@ -308,43 +315,50 @@ DEPENDENCIES
   better_errors
   binding_of_caller
   bootstrap-kaminari-views (~> 0.0.2)
-  coffee-rails (~> 3.2.1)
+  coffee-rails (~> 4.0.0)
   coveralls
   daemons (~> 1.1.9)
   delayed_job (~> 4.0.0)
   delayed_job_active_record (~> 4.0.0)
-  devise (~> 3.0.0)
+  delorean
+  devise (~> 3.2.4)
+  dotenv-deployment
   dotenv-rails
   em-http-request (~> 1.1.2)
+  faraday (~> 0.9.0)
+  faraday_middleware
   forecast_io (~> 2.0.0)
   foreman (~> 0.63.0)
-  geokit (~> 1.6.7)
-  geokit-rails3 (~> 0.1.5)
+  geokit (~> 1.8.4)
+  geokit-rails (~> 2.0.1)
   hipchat (~> 1.1.0)
-  jquery-rails (~> 3.0.4)
-  json (>= 1.7.7)
+  jquery-rails (~> 3.1.0)
+  json (~> 1.8.1)
   jsonpath (~> 0.5.3)
-  kaminari (~> 0.14.1)
-  kramdown (~> 1.1.0)
-  mysql2 (~> 0.3.13)
-  nokogiri (~> 1.6.0)
+  kaminari (~> 0.15.1)
+  kramdown (~> 1.3.3)
+  mysql2 (~> 0.3.15)
+  nokogiri (~> 1.6.1)
+  protected_attributes (~> 1.0.7)
   pry
-  rails (= 3.2.17)
+  rack
+  rails (= 4.1.0)
   rr
   rspec
   rspec-rails
-  rturk (~> 2.11.0)
+  rturk (~> 2.12.1)
   ruby-growl (~> 4.1.0)
   rufus-scheduler (~> 3.0.7)
-  sass-rails (~> 3.2.3)
-  select2-rails (~> 3.4.3)
+  sass-rails (~> 4.0.0)
+  select2-rails (~> 3.5.4)
   shoulda-matchers
   therubyracer (~> 0.12.1)
-  twilio-ruby (~> 3.10.0)
-  twitter (~> 5.7.1)
+  twilio-ruby (~> 3.11.5)
+  twitter (~> 5.8.0)
   twitter-stream!
   typhoeus (~> 0.6.3)
-  uglifier (>= 1.0.3)
+  tzinfo-data
+  uglifier (>= 1.3.0)
   webmock
   weibo_2 (~> 0.1.4)
-  wunderground (~> 1.1.0)
+  wunderground (~> 1.2.0)

README.md

@@ -104,5 +104,5 @@ Huginn is a work in progress and is hopefully just getting started.  Please get
 
 Please fork, add specs, and send pull requests!
 
-[![Build Status](https://travis-ci.org/cantino/huginn.png)](https://travis-ci.org/cantino/huginn) [![Coverage Status](https://coveralls.io/repos/cantino/huginn/badge.png)](https://coveralls.io/r/cantino/huginn) [![Bitdeli Badge](https://d2weczhvl823v0.cloudfront.net/cantino/huginn/trend.png)](https://bitdeli.com/free "Bitdeli Badge")
+[![Build Status](https://travis-ci.org/cantino/huginn.png)](https://travis-ci.org/cantino/huginn) [![Coverage Status](https://coveralls.io/repos/cantino/huginn/badge.png)](https://coveralls.io/r/cantino/huginn) [![Bitdeli Badge](https://d2weczhvl823v0.cloudfront.net/cantino/huginn/trend.png)](https://bitdeli.com/free "Bitdeli Badge") [![Dependency Status](https://gemnasium.com/cantino/huginn.svg)](https://gemnasium.com/cantino/huginn)
 

app/assets/javascripts/application.js.coffee.erb

@@ -56,9 +56,6 @@ $(document).ready ->
   # JSON Editor
   window.jsonEditor = setupJsonEditor()
 
-  # Select2 Selects
-  $(".select2").select2(width: 'resolve')
-
   # Flash
   if $(".flash").length
     setTimeout((-> $(".flash").slideUp(-> $(".flash").remove())), 5000)
@@ -155,6 +152,9 @@ $(document).ready ->
 
   $("#agent_type").change() if $("#agent_type").length
 
+  # Select2 Selects
+  $(".select2").select2(width: 'resolve')
+
   if $(".schedule-region")
     if $(".schedule-region").data("can-be-scheduled") == true
       showSchedule()

app/controllers/agents_controller.rb

@@ -1,4 +1,6 @@
 class AgentsController < ApplicationController
+  include DotHelper
+
   def index
     @agents = current_user.agents.page(params[:page])
 
@@ -101,7 +103,13 @@ class AgentsController < ApplicationController
   end
 
   def new
-    @agent = current_user.agents.build
+    agents = current_user.agents
+
+    if id = params[:id]
+      @agent = agents.build_clone(agents.find(id))
+    else
+      @agent = agents.build
+    end
 
     respond_to do |format|
       format.html

app/helpers/application_helper.rb

@@ -16,19 +16,4 @@ module ApplicationHelper
       link_to '<span class="label btn-danger">No</span>'.html_safe, agent_path(agent, :tab => (agent.recent_error_logs? ? 'logs' : 'details'))
     end
   end
-
-  def render_dot(dot_format_string)
-    if (command = ENV['USE_GRAPHVIZ_DOT']) &&
-       (svg = IO.popen([command, *%w[-Tsvg -q1 -o/dev/stdout /dev/stdin]], 'w+') { |dot|
-          dot.print dot_format_string
-          dot.close_write
-          dot.read
-        } rescue false)
-      svg.html_safe
-    else
-      tag('img', src: URI('https://chart.googleapis.com/chart').tap { |uri|
-            uri.query = URI.encode_www_form(cht: 'gv', chl: dot_format_string)
-          })
-    end
-  end
 end

app/helpers/dot_helper.rb

@@ -0,0 +1,48 @@
+module DotHelper
+  def render_agents_diagram(agents)
+    if (command = ENV['USE_GRAPHVIZ_DOT']) &&
+       (svg = IO.popen([command, *%w[-Tsvg -q1 -o/dev/stdout /dev/stdin]], 'w+') { |dot|
+          dot.print agents_dot(agents, true)
+          dot.close_write
+          dot.read
+        } rescue false)
+      svg.html_safe
+    else
+      tag('img', src: URI('https://chart.googleapis.com/chart').tap { |uri|
+            uri.query = URI.encode_www_form(cht: 'gv', chl: agents_dot(agents))
+          })
+    end
+  end
+
+  private
+
+  def dot_id(string)
+    # Backslash escaping seems to work for the backslash itself,
+    # despite the DOT language document.
+    '"%s"' % string.gsub(/\\/, "\\\\\\\\").gsub(/"/, "\\\\\"")
+  end
+
+  def agents_dot(agents, rich = false)
+    "digraph foo {".tap { |dot|
+      agents.each.with_index do |agent, index|
+        if rich
+          if agent.disabled
+            dot << '%s[URL=%s] (Disabled);' % [dot_id(agent.name), dot_id(agent_path(agent.id))]
+          else
+            dot << '%s[URL=%s];' % [dot_id(agent.name), dot_id(agent_path(agent.id))]
+          end
+        else
+          if agent.disabled
+            dot << '%s (Disabled);' % dot_id(agent.name)
+          else
+            dot << '%s;' % dot_id(agent.name)
+          end
+        end
+        agent.receivers.each do |receiver|
+          dot << "%s->%s;" % [dot_id(agent.name), dot_id(receiver.name)]
+        end
+      end
+      dot << "}"
+    }
+  end
+end

app/models/agent.rb

@@ -39,10 +39,10 @@ class Agent < ActiveRecord::Base
   after_save :possibly_update_event_expirations
 
   belongs_to :user, :inverse_of => :agents
-  has_many :events, :dependent => :delete_all, :inverse_of => :agent, :order => "events.id desc"
+  has_many :events, -> { order("events.id desc") }, :dependent => :delete_all, :inverse_of => :agent
   has_one  :most_recent_event, :inverse_of => :agent, :class_name => "Event", :order => "events.id desc"
-  has_many :logs, :dependent => :delete_all, :inverse_of => :agent, :class_name => "AgentLog", :order => "agent_logs.id desc"
-  has_many :received_events, :through => :sources, :class_name => "Event", :source => :events, :order => "events.id desc"
+  has_many :logs,  -> { order("agent_logs.id desc") }, :dependent => :delete_all, :inverse_of => :agent, :class_name => "AgentLog"
+  has_many :received_events, -> { order("events.id desc") }, :through => :sources, :class_name => "Event", :source => :events
   has_many :links_as_source, :dependent => :delete_all, :foreign_key => "source_id", :class_name => "Link", :inverse_of => :source
   has_many :links_as_receiver, :dependent => :delete_all, :foreign_key => "receiver_id", :class_name => "Link", :inverse_of => :receiver
   has_many :sources, :through => :links_as_receiver, :class_name => "Agent", :inverse_of => :receivers
@@ -230,6 +230,19 @@ class Agent < ActiveRecord::Base
   # Class Methods
 
   class << self
+    def build_clone(original)
+      new(original.slice(:type, :options, :schedule, :source_ids, :keep_events_for, :propagate_immediately)) { |clone|
+        # Give it a unique name
+        2.upto(count) do |i|
+          name = '%s (%d)' % [original.name, i]
+          unless exists?(name: name)
+            clone.name = name
+            break
+          end
+        end
+      }
+    end
+
     def cannot_be_scheduled!
       @cannot_be_scheduled = true
     end

app/models/agents/stubhub_agent.rb

@@ -0,0 +1,100 @@
+module Agents
+  class StubhubAgent < Agent
+    cannot_receive_events!
+
+    description <<-MD
+      This StubHubAgent creates an event for a given StubHub Event. It can be used to track how many tickets are available for the event and the minimum and maximum price. All that is required is that you paste in the url from the actual event, e.g. http://www.stubhub.com/outside-lands-music-festival-tickets/outside-lands-music-festival-3-day-pass-san-francisco-golden-gate-park-polo-fields-8-8-2014-9020701/
+    MD
+
+    event_description <<-MD
+      Events looks like this:
+        {
+          "url": "http://stubhub.com/valid-event-url"
+          "name": "Event Name"
+          "date": "2014-08-01"
+          "max_price": "999.99"
+          "min_price": "100.99"
+          "total_postings": "50"
+          "total_tickets": "150"
+          "venue_name": "Venue Name"
+        }
+    MD
+
+    default_schedule "every_1d"
+
+    def working?
+      event_created_within?(1) && !recent_error_logs?
+    end
+
+    def default_options
+      { 'url' =>  'http://stubhub.com/enter-your-event-here' }
+    end
+
+    def validate_options
+      errors.add(:base, 'url is required') unless options['url'].present?
+    end
+
+    def url
+      options['url']
+    end
+
+    def check
+      create_event :payload => fetch_stubhub_data(url)
+    end
+
+    def fetch_stubhub_data(url)
+      StubhubFetcher.call(url)
+    end
+
+    class StubhubFetcher
+
+      def self.call(url)
+        new(url).fields
+      end
+
+      def initialize(url)
+        @url = url
+      end
+
+      def event_id
+        /(\d*)\/{0,1}\z/.match(url)[1]
+      end
+
+      def base_url
+       'http://www.stubhub.com/listingCatalog/select/?q='
+      end
+
+      def build_url
+        base_url + "%2B+stubhubDocumentType%3Aevent%0D%0A%2B+event_id%3A#{event_id}%0D%0A&start=0&rows=10&wt=json"
+      end
+
+      def response
+        uri = URI(build_url)
+        Net::HTTP.get(uri)
+      end
+
+      def parse_response
+        JSON.parse(response)
+      end
+
+      def fields
+        stubhub_fields = parse_response['response']['docs'][0]
+        {
+          'url' => url,
+          'name' => stubhub_fields['seo_description_en_US'],
+          'date' => stubhub_fields['event_date_local'],
+          'max_price' => stubhub_fields['maxPrice'].to_s,
+          'min_price' => stubhub_fields['minPrice'].to_s,
+          'total_postings' => stubhub_fields['totalPostings'].to_s,
+          'total_tickets' => stubhub_fields['totalTickets'].to_i.to_s,
+          'venue_name' => stubhub_fields['venue_name']
+        }
+      end
+
+      private
+
+      attr_reader :url
+
+    end
+  end
+end

app/models/agents/trigger_agent.rb

@@ -15,6 +15,8 @@ module Agents
 
       All rules must match for the Agent to match.  The resulting Event will have a payload message of `message`.  You can include extractions in the message, for example: `I saw a bar of: <foo.bar>`
 
+      Set `keep_event` to `true` if you'd like to re-emit the incoming event, optionally merged with 'message' when provided.
+
       Set `expected_receive_period_in_days` to the maximum amount of time that you'd expect to pass between Events being received by this Agent.
     MD
 
@@ -25,15 +27,20 @@ module Agents
     MD
 
     def validate_options
-      unless options['expected_receive_period_in_days'].present? && options['message'].present? && options['rules'].present? &&
+      unless options['expected_receive_period_in_days'].present? && options['rules'].present? &&
           options['rules'].all? { |rule| rule['type'].present? && VALID_COMPARISON_TYPES.include?(rule['type']) && rule['value'].present? && rule['path'].present? }
         errors.add(:base, "expected_receive_period_in_days, message, and rules, with a type, value, and path for every rule, are required")
       end
+
+      errors.add(:base, "message is required unless 'keep_event' is 'true'") unless options['message'].present? || keep_event?
+
+      errors.add(:base, "keep_event, when present, must be 'true' or 'false'") unless options['keep_event'].blank? || %w[true false].include?(options['keep_event'])
     end
 
     def default_options
       {
         'expected_receive_period_in_days' => "2",
+        'keep_event' => 'false',
         'rules' => [{
                       'type' => "regex",
                       'value' => "foo\\d+bar",
@@ -79,10 +86,20 @@ module Agents
         end
 
         if match
-          create_event :payload => { 'message' => make_message(event[:payload]) } # Maybe this should include the
-                                                                                  # original event as well?
+          if keep_event?
+            payload = event.payload.dup
+            payload['message'] = make_message(event[:payload]) if options['message'].present?
+          else
+            payload = { 'message' => make_message(event[:payload]) }
+          end
+
+          create_event :payload => payload
         end
       end
     end
+
+    def keep_event?
+      options['keep_event'] == 'true'
+    end
   end
 end

app/models/agents/twilio_agent.rb

@@ -7,17 +7,16 @@ module Agents
     cannot_create_events!
 
     description <<-MD
-      The TwilioAgent receives and collects events and sends them via text message or gives you a call when scheduled.
+      The TwilioAgent receives and collects events and sends them via text message (up to 160 characters) or gives you a call when scheduled.
 
-      It is assumed that events have a `message`, `text`, or `sms` key, the value of which is sent as the content of the text message/call. You can use Event Formatting Agent if your event does not provide these keys.
+      It is assumed that events have a `message`, `text`, or `sms` key, the value of which is sent as the content of the text message/call. You can use the EventFormattingAgent if your event does not provide these keys.
 
       Set `receiver_cell` to the number to receive text messages/call and `sender_cell` to the number sending them.
 
       `expected_receive_period_in_days` is maximum number of days that you would expect to pass between events being received by this agent.
 
-      If you would like to receive calls, then set `receive_call` to true. `server_url` needs to be 
-      filled only if you are making calls. Dont forget to include http/https in `server_url`.
-
+      If you would like to receive calls, set `receive_call` to `true`. In this case, `server_url` must be set to the URL of your
+      Huginn installation (probably "https://#{ENV['DOMAIN']}"), which must be web-accessible.  Be sure to set http/https correctly.
     MD
 
     def default_options
@@ -43,13 +42,14 @@ module Agents
       @client = Twilio::REST::Client.new options['account_sid'], options['auth_token']
       memory['pending_calls'] ||= {}
       incoming_events.each do |event|
-        message = (event.payload['message'] || event.payload['text'] || event.payload['sms']).to_s
-        if message != ""
+        message = (event.payload['message'].presence || event.payload['text'].presence || event.payload['sms'].presence).to_s
+        if message.present?
           if options['receive_call'].to_s == 'true'
             secret = SecureRandom.hex 3
             memory['pending_calls'][secret] = message
             make_call secret
           end
+
           if options['receive_text'].to_s == 'true'
             message = message.slice 0..160
             send_message message
@@ -71,11 +71,11 @@ module Agents
     def make_call(secret)
       @client.account.calls.create :from => options['sender_cell'],
                                    :to => options['receiver_cell'],
-                                   :url => post_url(options['server_url'],secret)
+                                   :url => post_url(options['server_url'], secret)
     end
 
-    def post_url(server_url,secret)
-      "#{server_url}/users/#{self.user.id}/web_requests/#{self.id}/#{secret}"
+    def post_url(server_url, secret)
+      "#{server_url}/users/#{user.id}/web_requests/#{id}/#{secret}"
     end
 
     def receive_web_request(params, method, format)

app/models/agents/website_agent.rb

@@ -1,10 +1,10 @@
 require 'nokogiri'
-require 'typhoeus'
+require 'faraday'
+require 'faraday_middleware'
 require 'date'
 
 module Agents
   class WebsiteAgent < Agent
-    cannot_receive_events!
 
     default_schedule "every_12h"
 
@@ -22,30 +22,36 @@ module Agents
 
       To tell the Agent how to parse the content, specify `extract` as a hash with keys naming the extractions and values of hashes.
 
-      When parsing HTML or XML, these sub-hashes specify how to extract with either a `css` CSS selector or a `xpath` XPath expression and either `'text': true` or `attr` pointing to an attribute name to grab.  An example:
+      When parsing HTML or XML, these sub-hashes specify how to extract with either a `css` CSS selector or a `xpath` XPath expression and either `"text": true` or `attr` pointing to an attribute name to grab.  An example:
 
-          'extract': {
-            'url': { 'css': "#comic img", 'attr': "src" },
-            'title': { 'css': "#comic img", 'attr': "title" },
-            'body_text': { 'css': "div.main", 'text': true }
+          "extract": {
+            "url": { "css": "#comic img", "attr": "src" },
+            "title": { "css": "#comic img", "attr": "title" },
+            "body_text": { "css": "div.main", "text": true }
           }
 
       When parsing JSON, these sub-hashes specify [JSONPaths](http://goessner.net/articles/JsonPath/) to the values that you care about.  For example:
 
-          'extract': {
-            'title': { 'path': "results.data[*].title" },
-            'description': { 'path': "results.data[*].description" }
+          "extract": {
+            "title": { "path": "results.data[*].title" },
+            "description": { "path": "results.data[*].description" }
           }
 
       Note that for all of the formats, whatever you extract MUST have the same number of matches for each extractor.  E.g., if you're extracting rows, all extractors must match all rows.  For generating CSS selectors, something like [SelectorGadget](http://selectorgadget.com) may be helpful.
 
-      Can be configured to use HTTP basic auth by including the `basic_auth` parameter with `username:password`.
+      Can be configured to use HTTP basic auth by including the `basic_auth` parameter with `"username:password"`, or `["username", "password"]`.
 
       Set `expected_update_period_in_days` to the maximum amount of time that you'd expect to pass between Events being created by this Agent.  This is only used to set the "working" status.
 
       Set `uniqueness_look_back` to limit the number of events checked for uniqueness (typically for performance).  This defaults to the larger of #{UNIQUENESS_LOOK_BACK} or #{UNIQUENESS_FACTOR}x the number of detected received results.
 
       Set `force_encoding` to an encoding name if the website does not return a Content-Type header with a proper charset.
+
+      Set `user_agent` to a custom User-Agent name if the website does not like the default value ("Faraday v#{Faraday::VERSION}").
+
+      The `headers` field is optional.  When present, it should be a hash of headers to send with the request.
+
+      The WebsiteAgent can also scrape based on incoming events. It will scrape the url contained in the `url` key of the incoming event payload.
     MD
 
     event_description do
@@ -102,30 +108,33 @@ module Agents
           errors.add(:base, "force_encoding must be a string")
         end
       end
-    end
 
-    def check
-      hydra = Typhoeus::Hydra.new
-      log "Fetching #{options['url']}"
-      request_opts = { :followlocation => true }
-      request_opts[:userpwd] = options['basic_auth'] if options['basic_auth'].present?
+      if options['user_agent'].present?
+        errors.add(:base, "user_agent must be a string") unless options['user_agent'].is_a?(String)
+      end
 
-      requests = []
+      unless headers.is_a?(Hash)
+        errors.add(:base, "if provided, headers must be a hash")
+      end
 
-      if options['url'].kind_of?(Array)
-        options['url'].each do |url|
-           requests.push(Typhoeus::Request.new(url, request_opts))
-        end
-      else
-        requests.push(Typhoeus::Request.new(options['url'], request_opts))
+      begin
+        basic_auth_credentials()
+      rescue => e
+        errors.add(:base, e.message)
       end
+    end
 
-      requests.each do |request|
-        request.on_failure do |response|
-          error "Failed: #{response.inspect}"
-        end
+    def check
+      check_url options['url']
+    end
 
-        request.on_success do |response|
+    def check_url(in_url)
+      return unless in_url.present?
+
+      Array(in_url).each do |url|
+        log "Fetching #{url}"
+        response = faraday.get(url)
+        if response.success?
           body = response.body
           if (encoding = options['force_encoding']).present?
             body = body.encode(Encoding::UTF_8, encoding)
@@ -150,7 +159,7 @@ module Agents
                 when xpath = extraction_details['xpath']
                   nodes = doc.xpath(xpath)
                 else
-                  error "'css' or 'xpath' is required for HTML or XML extraction"
+                  error '"css" or "xpath" is required for HTML or XML extraction'
                   return
                 end
                 unless Nokogiri::XML::NodeSet === nodes
@@ -163,7 +172,7 @@ module Agents
                   elsif extraction_details['text']
                     node.text()
                   else
-                    error "'attr' or 'text' is required on HTML or XML extraction patterns"
+                    error '"attr" or "text" is required on HTML or XML extraction patterns'
                     return
                   end
                 }
@@ -178,14 +187,14 @@ module Agents
               error "Got an uneven number of matches for #{options['name']}: #{options['extract'].inspect}"
               return
             end
-        
+
             old_events = previous_payloads num_unique_lengths.first
             num_unique_lengths.first.times do |index|
               result = {}
               options['extract'].keys.each do |name|
                 result[name] = output[name][index]
                 if name.to_s == 'url'
-                  result[name] = URI.join(options['url'], result[name]).to_s if (result[name] =~ URI::DEFAULT_PARSER.regexp[:ABS_URI]).nil?
+                  result[name] = (response.env[:url] + result[name]).to_s
                 end
               end
 
@@ -195,10 +204,16 @@ module Agents
               end
             end
           end
+        else
+          error "Failed: #{response.inspect}"
         end
+      end
+    end
 
-        hydra.queue request
-        hydra.run
+    def receive(incoming_events)
+      incoming_events.each do |event|
+        url_to_scrape = event.payload['url']
+        check_url(url_to_scrape) if url_to_scrape =~ /^https?:\/\//i
       end
     end
 
@@ -275,5 +290,47 @@ module Agents
         false
       end
     end
+
+    def faraday
+      @faraday ||= Faraday.new { |builder|
+        builder.headers = headers if headers.length > 0
+
+        if (user_agent = options['user_agent']).present?
+          builder.headers[:user_agent] = user_agent
+        end
+
+        builder.use FaradayMiddleware::FollowRedirects
+        builder.request :url_encoded
+        if userinfo = basic_auth_credentials()
+          builder.request :basic_auth, *userinfo
+        end
+
+        case backend = faraday_backend
+        when :typhoeus
+          require 'typhoeus/adapters/faraday'
+        end
+        builder.adapter backend
+      }
+    end
+
+    def faraday_backend
+      ENV.fetch('FARADAY_HTTP_BACKEND', 'typhoeus').to_sym
+    end
+
+    def basic_auth_credentials
+      case value = options['basic_auth']
+      when nil, ''
+        return nil
+      when Array
+        return value if value.size == 2
+      when /:/
+        return value.split(/:/, 2)
+      end
+      raise "bad value for basic_auth: #{value.inspect}"
+    end
+
+    def headers
+      options['headers'].presence || {}
+    end
   end
 end

app/models/user.rb

@@ -23,8 +23,8 @@ class User < ActiveRecord::Base
   validates_inclusion_of :invitation_code, :on => :create, :in => INVITATION_CODES, :message => "is not valid"
 
   has_many :user_credentials, :dependent => :destroy, :inverse_of => :user
-  has_many :events, :order => "events.created_at desc", :dependent => :delete_all, :inverse_of => :user
-  has_many :agents, :order => "agents.created_at desc", :dependent => :destroy, :inverse_of => :user
+  has_many :events, -> { order("events.created_at desc") }, :dependent => :delete_all, :inverse_of => :user
+  has_many :agents, -> { order("agents.created_at desc") }, :dependent => :destroy, :inverse_of => :user
   has_many :logs, :through => :agents, :class_name => "AgentLog"
 
   # Allow users to login via either email or username.

app/views/agents/diagram.html.erb

@@ -9,17 +9,7 @@
       </div>
 
       <div class='digraph'>
-        <%
-          dot_format_string = "digraph foo {"
-          @agents.each.with_index do |agent, index|
-            dot_format_string += "\"#{agent.name}#{" (Disabled)" if agent.disabled?}\";"
-            agent.receivers.each do |receiver|
-              dot_format_string += "\"#{agent.name}#{" (Disabled)" if agent.disabled?}\"->\"#{receiver.name}#{" (Disabled)" if receiver.disabled?}\";"
-            end
-           end
-           dot_format_string = dot_format_string + "}"
-        %>
-        <%= render_dot(dot_format_string) %>
+        <%= render_agents_diagram(@agents) %>
       </div>
     </div>
   </div>

app/views/agents/show.html.erb

@@ -14,12 +14,10 @@
           <% end %>
           <li><a href="#logs" data-toggle="tab" data-agent-id="<%= @agent.id %>" class='<%= @agent.recent_error_logs? ? 'recent-errors' : '' %>'><i class='icon-list-alt'></i> Logs</a></li>
 
-          <% if @agent.can_create_events? %>
-            <% if @agent.events.count > 0 %>
-              <li><%= link_to '<i class="icon-random"></i> Events'.html_safe, events_path(:agent => @agent.to_param) %></li>
-            <% else %>
-              <li class='disabled'><a><i class='icon-random'></i> Events</a></li>
-            <% end %>
+          <% if @agent.can_create_events? && @agent.events.count > 0 %>
+            <li><%= link_to '<i class="icon-random"></i> Events'.html_safe, events_path(:agent => @agent.to_param) %></li>
+          <% else %>
+            <li class='disabled'><a><i class='icon-random'></i> Events</a></li>
           <% end %>
 
           <li class="dropdown">
@@ -32,6 +30,7 @@
               <% end %>
 
               <li><%= link_to '<i class="icon-pencil"></i> Edit'.html_safe, edit_agent_path(@agent) %></li>
+              <li><%= link_to '<i class="icon-plus"></i> Clone'.html_safe, new_agent_path(id: @agent) %></li>
 
               <li>
                 <% if !@agent.disabled? %>

app/views/layouts/_messages.html.erb

@@ -1,7 +1,7 @@
 <% if flash.keys.length > 0 %>
   <div class="flash">
     <% flash.each do |name, msg| %>
-      <div class="alert alert-<%= name == :notice ? "success" : "error" %>">
+      <div class="alert alert-<%= name.to_sym == :notice ? "success" : "error" %>">
         <a class="close" data-dismiss="alert">&#215;</a>
         <%= content_tag :div, msg, :id => "flash_#{name}" if msg.is_a?(String) %>
       </div>

bin/bundle

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__)
+load Gem.bin_path('bundler', 'bundle')

bin/rails

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+APP_PATH = File.expand_path('../../config/application',  __FILE__)
+require_relative '../config/boot'
+require 'rails/commands'

bin/rake

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require_relative '../config/boot'
+require 'rake'
+Rake.application.run

config/application.rb

@@ -2,12 +2,7 @@ require File.expand_path('../boot', __FILE__)
 
 require 'rails/all'
 
-if defined?(Bundler)
-  # If you precompile assets before deploying to production, use this line
-  Bundler.require(*Rails.groups(:assets => %w(development test)))
-  # If you want your assets lazily compiled in production, use this line
-  # Bundler.require(:default, :assets, Rails.env)
-end
+Bundler.require(:default, Rails.env)
 
 module Huginn
   class Application < Rails::Application
@@ -18,10 +13,6 @@ module Huginn
     # Custom directories with classes and modules you want to be autoloadable.
     config.autoload_paths += %W(#{config.root}/lib)
 
-    # Only load the plugins named here, in the order given (default is alphabetical).
-    # :all can be used as a placeholder for all plugins not explicitly named.
-    # config.plugins = [ :exception_notification, :ssl_requirement, :all ]
-
     # Activate observers that should always be running.
     # config.active_record.observers = :cacher, :garbage_collector, :forum_observer
 
@@ -56,8 +47,5 @@ module Huginn
     # Enable the asset pipeline
     config.assets.enabled = true
     config.assets.initialize_on_precompile = false
-
-    # Version of your assets, change this if you want to expire all your assets
-    config.assets.version = '1.0'
   end
 end

config/environments/development.rb

@@ -8,8 +8,11 @@ Huginn::Application.configure do
   # since you don't have to restart the web server when you make code changes.
   config.cache_classes = false
 
-  # Log error messages when you accidentally call methods on nil.
-  config.whiny_nils = true
+  # Eager load code on boot. This eager loads most of Rails and
+  # your application in memory, allowing both threaded web servers
+  # and those relying on copy on write to perform better.
+  # Rake tasks automatically ignore this option for performance.
+  config.eager_load = false
 
   # Show full error reports and disable caching
   config.consider_all_requests_local       = true
@@ -24,12 +27,8 @@ Huginn::Application.configure do
   # Raise exception on mass assignment protection for Active Record models
   config.active_record.mass_assignment_sanitizer = :strict
 
-  # Log the query plan for queries taking more than this (works
-  # with SQLite, MySQL, and PostgreSQL)
-  config.active_record.auto_explain_threshold_in_seconds = 0.5
-
-  # Do not compress assets
-  config.assets.compress = false
+  # Raise an error on page load if there are pending migrations.
+  config.active_record.migration_error = :page_load
 
   # Expands the lines which load the assets
   config.assets.debug = true

config/environments/production.rb

@@ -4,31 +4,41 @@ Huginn::Application.configure do
   # Code is not reloaded between requests
   config.cache_classes = true
 
+  # Eager load code on boot. This eager loads most of Rails and
+  # your application in memory, allowing both threaded web servers
+  # and those relying on copy on write to perform better.
+  # Rake tasks automatically ignore this option for performance.
+  config.eager_load = true
+
   # Full error reports are disabled and caching is turned on
   config.consider_all_requests_local       = false
   config.action_controller.perform_caching = true
 
+  # Enable Rack::Cache to put a simple HTTP cache in front of your application
+  # Add `rack-cache` to your Gemfile before enabling this.
+  # For large-scale production use, consider using a caching reverse proxy like nginx, varnish or squid.
+  # config.action_dispatch.rack_cache = true
+
   # Disable Rails's static asset server (Apache or nginx will already do this)
   config.serve_static_assets = false
 
   # Compress JavaScripts and CSS
-  config.assets.compress = true
+  config.assets.js_compressor  = :uglifier
+  config.assets.css_compressor = :sass
 
   # Don't fallback to assets pipeline if a precompiled asset is missed
   config.assets.compile = false
 
   # Generate digests for assets URLs
   config.assets.digest = true
+  config.assets.precompile += %w(*.png *.jpg *.jpeg *.gif)
 
-  # Defaults to nil and saved in location specified by config.assets.prefix
-  # config.assets.manifest = YOUR_PATH
-
-  # Specifies the header that your server uses for sending files
+  # Specifies the header that your server uses for sending files.
   # config.action_dispatch.x_sendfile_header = "X-Sendfile" # for apache
   # config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for nginx
 
   # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies.
-  config.force_ssl = true
+  config.force_ssl = ENV['FORCE_SSL'].present? && ENV['FORCE_SSL'] == 'true' ? true : false
 
   # See everything in the log (default is :info)
   # config.log_level = :debug
@@ -50,19 +60,25 @@ Huginn::Application.configure do
   # Precompile additional assets (application.js.coffee.erb, application.css, and all non-JS/CSS are already added)
   config.assets.precompile += %w( graphing.js user_credentials.js )
 
-  # Enable threaded mode
-  # config.threadsafe!
+  # Ignore bad email addresses and do not raise email delivery errors.
+  # Set this to true and configure the email server for immediate delivery to raise delivery errors.
+  # config.action_mailer.raise_delivery_errors = false
 
   # Enable locale fallbacks for I18n (makes lookups for any locale fall back to
-  # the I18n.default_locale when a translation can not be found)
+  # the I18n.default_locale when a translation cannot be found).
   config.i18n.fallbacks = true
 
   # Send deprecation notices to registered listeners
   config.active_support.deprecation = :notify
 
-  # Log the query plan for queries taking more than this (works
-  # with SQLite, MySQL, and PostgreSQL)
-  # config.active_record.auto_explain_threshold_in_seconds = 0.5
+  # Disable automatic flushing of the log to improve performance.
+  # config.autoflush_log = false
+
+  # Use default logging formatter so that PID and timestamp are not suppressed.
+  config.log_formatter = ::Logger::Formatter.new
+
+  # Do not dump schema after migrations.
+  config.active_record.dump_schema_after_migration = false
 
   config.action_mailer.default_url_options = { :host => ENV['DOMAIN'] }
   config.action_mailer.asset_host = ENV['DOMAIN']
@@ -73,4 +89,4 @@ Huginn::Application.configure do
   config.action_mailer.raise_delivery_errors = true
   config.action_mailer.delivery_method = :smtp
   # smtp_settings moved to config/initializers/action_mailer.rb
-end
+end

config/environments/staging.rb

@@ -1,76 +0,0 @@
-Huginn::Application.configure do
-  # Settings specified here will take precedence over those in config/application.rb
-
-  # Code is not reloaded between requests
-  config.cache_classes = true
-
-  # Full error reports are disabled and caching is turned on
-  config.consider_all_requests_local       = false
-  config.action_controller.perform_caching = true
-
-  # Disable Rails's static asset server (Apache or nginx will already do this)
-  config.serve_static_assets = false
-
-  # Compress JavaScripts and CSS
-  config.assets.compress = true
-
-  # Don't fallback to assets pipeline if a precompiled asset is missed
-  config.assets.compile = false
-
-  # Generate digests for assets URLs
-  config.assets.digest = true
-
-  # Defaults to nil and saved in location specified by config.assets.prefix
-  # config.assets.manifest = YOUR_PATH
-
-  # Specifies the header that your server uses for sending files
-  # config.action_dispatch.x_sendfile_header = "X-Sendfile" # for apache
-  # config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for nginx
-
-  # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies.
-  config.force_ssl = true
-
-  # See everything in the log (default is :info)
-  # config.log_level = :debug
-
-  # Prepend all log lines with the following tags
-  config.log_tags = [ :uuid ] # :subdomain
-
-  # Use a different logger for distributed setups
-  # config.logger = ActiveSupport::TaggedLogging.new(SyslogLogger.new)
-
-  # Use a different cache store in production
-  # config.cache_store = :mem_cache_store
-
-  # Enable serving of images, stylesheets, and JavaScripts from an asset server
-  if ENV['ASSET_HOST'].present?
-    config.action_controller.asset_host = ENV['ASSET_HOST']
-  end
-
-  # Precompile additional assets (application.js.coffee.erb, application.css, and all non-JS/CSS are already added)
-  config.assets.precompile += %w( graphing.js )
-
-  # Enable threaded mode
-  # config.threadsafe!
-
-  # Enable locale fallbacks for I18n (makes lookups for any locale fall back to
-  # the I18n.default_locale when a translation can not be found)
-  config.i18n.fallbacks = true
-
-  # Send deprecation notices to registered listeners
-  config.active_support.deprecation = :notify
-
-  # Log the query plan for queries taking more than this (works
-  # with SQLite, MySQL, and PostgreSQL)
-  # config.active_record.auto_explain_threshold_in_seconds = 0.5
-
-  config.action_mailer.default_url_options = { :host => ENV['DOMAIN'] }
-  config.action_mailer.asset_host = ENV['DOMAIN']
-  if ENV['ASSET_HOST']
-    config.action_mailer.asset_host = ENV['ASSET_HOST']
-  end
-  config.action_mailer.perform_deliveries = true
-  config.action_mailer.raise_delivery_errors = true
-  config.action_mailer.delivery_method = :smtp
-  # smtp_settings moved to config/initializers/action_mailer.rb
-end

config/environments/test.rb

@@ -7,13 +7,15 @@ Huginn::Application.configure do
   # and recreated between test runs. Don't rely on the data there!
   config.cache_classes = true
 
+  # Do not eager load code on boot. This avoids loading your whole application
+  # just for the purpose of running a single test. If you are using a tool that
+  # preloads Rails for running tests, you may have to set it to true.
+  config.eager_load = false
+
   # Configure static asset server for tests with Cache-Control for performance
   config.serve_static_assets = true
   config.static_cache_control = "public, max-age=3600"
 
-  # Log error messages when you accidentally call methods on nil
-  config.whiny_nils = true
-
   # Show full error reports and disable caching
   config.consider_all_requests_local       = true
   config.action_controller.perform_caching = false
@@ -22,7 +24,7 @@ Huginn::Application.configure do
   config.action_dispatch.show_exceptions = false
 
   # Disable request forgery protection in test environment
-  config.action_controller.allow_forgery_protection    = false
+  config.action_controller.allow_forgery_protection = false
 
   # Tell Action Mailer not to deliver emails to the real world.
   # The :test delivery method accumulates sent emails in the

config/initializers/devise.rb

@@ -3,7 +3,8 @@
 Devise.setup do |config|
   # ==> Mailer Configuration
   # Configure the e-mail address which will be shown in Devise::Mailer,
-  # note that it will be overwritten if you use your own mailer class with default "from" parameter.
+  # note that it will be overwritten if you use your own mailer class
+  # with default "from" parameter.
   config.mailer_sender = "please-change-me-at-config-initializers-devise@example.com"
 
   # Configure the class responsible to send e-mails.
@@ -72,6 +73,12 @@ Devise.setup do |config|
   # passing :skip => :sessions to `devise_for` in your config/routes.rb
   config.skip_session_storage = [:http_auth]
 
+  # By default, Devise cleans up the CSRF token on authentication to
+  # avoid CSRF token fixation attacks. This means that, when using AJAX
+  # requests for sign in and sign up, you need to get a new CSRF token
+  # from the server. You can disable this option at your own risk.
+  # config.clean_up_csrf_token_on_authentication = true
+
   # ==> Configuration for :database_authenticatable
   # For bcrypt, this is the cost for hashing the password and defaults to 10. If
   # using other encryptors, it sets how many times you want the password re-encrypted.
@@ -174,10 +181,6 @@ Devise.setup do |config|
   # REST_AUTH_SITE_KEY to pepper)
   # config.encryptor = :sha512
 
-  # ==> Configuration for :token_authenticatable
-  # Defines name of the authentication token params key
-  # config.token_authentication_key = :auth_token
-
   # ==> Scopes configuration
   # Turn scoped views on. Before rendering "sessions/new", it will first check for
   # "users/sessions/new". It's turned off by default because it's slower if you

config/initializers/secret_token.rb

@@ -4,4 +4,4 @@
 # If you change this key, all old signed cookies will become invalid!
 # Make sure the secret is at least 30 characters and all random,
 # no regular words or you'll be exposed to dictionary attacks.
-Huginn::Application.config.secret_token = ENV['APP_SECRET_TOKEN']
+Huginn::Application.config.secret_key_base = ENV['APP_SECRET_TOKEN']

config/routes.rb

@@ -29,18 +29,18 @@ Huginn::Application.routes.draw do
 
   resources :user_credentials, :except => :show
 
-  match "/worker_status" => "worker_status#show"
+  get "/worker_status" => "worker_status#show"
 
   post "/users/:user_id/update_location/:secret" => "user_location_updates#create"
 
-  match "/users/:user_id/web_requests/:agent_id/:secret" => "web_requests#handle_request", :as => :web_requests
+  match  "/users/:user_id/web_requests/:agent_id/:secret" => "web_requests#handle_request", :as => :web_requests, :via => [:get, :post, :put, :delete]
   post "/users/:user_id/webhooks/:agent_id/:secret" => "web_requests#handle_request" # legacy
 
 # To enable DelayedJobWeb, see the 'Enable DelayedJobWeb' section of the README.
-#  match "/delayed_job" => DelayedJobWeb, :anchor => false
+#  get "/delayed_job" => DelayedJobWeb, :anchor => false
 
   devise_for :users, :sign_out_via => [ :post, :delete ]
 
-  match "/about" => "home#about"
+  get "/about" => "home#about"
   root :to => "home#index"
 end

db/seeds.rb

@@ -1,7 +1,7 @@
 # This file should contain all the record creation needed to seed the database with its default values.
 # The data can then be loaded with the rake db:seed (or created alongside the db with db:setup).
 
-user = User.find_or_initialize_by_email("admin@example.com")
+user = User.find_or_initialize_by(:email => "admin@example.com")
 user.username = "admin"
 user.password = "password"
 user.password_confirmation = "password"

deployment/.chef/knife.rb

@@ -0,0 +1,3 @@
+cookbook_path ["cookbooks", "site-cookbooks"]
+role_path     "roles"
+data_bag_path "data_bags"

deployment/Cheffile.lock

@@ -0,0 +1,71 @@
+SITE
+  remote: http://community.opscode.com/api/v1
+  specs:
+    apt (2.3.8)
+    bluepill (2.3.1)
+      rsyslog (>= 0.0.0)
+    build-essential (2.0.0)
+    chef_handler (1.1.6)
+    dmg (2.2.0)
+    ohai (1.1.12)
+    rsyslog (1.12.2)
+    runit (1.5.10)
+      build-essential (>= 0.0.0)
+      yum (~> 3.0)
+      yum-epel (>= 0.0.0)
+    windows (1.30.2)
+      chef_handler (>= 0.0.0)
+    yum (3.2.0)
+    yum-epel (0.3.6)
+      yum (~> 3.0)
+
+GIT
+  remote: git://github.com/mdxp/nodejs-cookbook.git
+  ref: master
+  sha: e2415cd8c4e03dccf21d7ef6ca31e1c5c81467ca
+  specs:
+    nodejs (1.3.0)
+      apt (>= 0.0.0)
+      build-essential (>= 0.0.0)
+      yum-epel (>= 0.0.0)
+
+GIT
+  remote: git://github.com/opscode-cookbooks/git.git
+  ref: master
+  sha: 76b0f9bb08fdd9e2e201fd70b72298097accdf96
+  specs:
+    git (4.0.1)
+      build-essential (>= 0.0.0)
+      dmg (>= 0.0.0)
+      runit (>= 1.0)
+      windows (>= 0.0.0)
+      yum (~> 3.0)
+      yum-epel (>= 0.0.0)
+
+GIT
+  remote: git://github.com/opscode-cookbooks/mysql.git
+  ref: master
+  sha: a2ff53f0ca6deca75aebf6da55ac381194ec7728
+  specs:
+    mysql (5.1.9)
+
+GIT
+  remote: git://github.com/opscode-cookbooks/nginx.git
+  ref: master
+  sha: 05b3a613f53a0b05c96f9206c5d67aa420f337fb
+  specs:
+    nginx (2.6.3)
+      apt (~> 2.2)
+      bluepill (~> 2.3)
+      build-essential (~> 2.0)
+      ohai (~> 1.1)
+      runit (~> 1.2)
+      yum-epel (~> 0.3)
+
+DEPENDENCIES
+  git (>= 0)
+  mysql (>= 0)
+  nginx (>= 0)
+  nodejs (>= 0)
+  runit (>= 0)
+

deployment/Vagrantfile

@@ -3,47 +3,22 @@
 
 Vagrant.configure("2") do |config|
   config.omnibus.chef_version = :latest
-  config.vm.define :vb do |vb|
-    vb.vm.box = "precise32"
-    vb.vm.box_url = "http://files.vagrantup.com/precise32.box"
-    vb.vm.network :forwarded_port, host: 3000, guest: 3000
 
-    vb.vm.provision :chef_solo do |chef|
-      chef.roles_path = "roles"
-      chef.cookbooks_path = ["cookbooks", "site-cookbooks"]
-      chef.add_role("huginn_development")
-    end
+  config.vm.provision :chef_solo do |chef|
+    chef.roles_path = "roles"
+    chef.cookbooks_path = ["cookbooks", "site-cookbooks"]
+    chef.add_role("huginn_development")
+    # chef.add_role("huginn_production")
   end
 
-  config.vm.define :prl do |prl|
-    prl.vm.box = "parallels/ubuntu-12.04"
-
-    prl.vm.provision :chef_solo do |chef|
-      chef.roles_path = "roles"
-      chef.cookbooks_path = ["cookbooks", "site-cookbooks"]
-      chef.add_role("huginn_development")
-    end
+  config.vm.provider :virtualbox do |vb, override|
+    #vb.memory = 1024
+    #vb.cpus = 4
+    override.vm.box = "hashicorp/precise64"
+    override.vm.network :forwarded_port, host: 3000, guest: 3000
   end
 
-  config.vm.define :ec2 do |ec2|
-    ec2.vm.box = "dummy"
-    ec2.vm.box_url = "https://github.com/mitchellh/vagrant-aws/raw/master/dummy.box"
-
-    ec2.vm.provider :aws do |aws, override|
-      aws.access_key_id = ""
-      aws.secret_access_key = ""
-      aws.keypair_name = ""
-      aws.region = "us-east-1"
-      aws.ami = "ami-d0f89fb9"
-
-      override.ssh.username = "ubuntu"
-      override.ssh.private_key_path = ""
-    end
-    ec2.vm.provision :chef_solo do |chef|
-      chef.roles_path = "roles"
-      chef.cookbooks_path = ["cookbooks", "site-cookbooks"]
-      chef.add_role("huginn_production")
-    
-    end
+  config.vm.provider :parallels do |prl, override|
+    override.vm.box = "parallels/ubuntu-12.04"
   end
 end

deployment/roles/huginn_development.json

@@ -23,6 +23,7 @@
              "recipe[git]",
              "recipe[apt]",
              "recipe[mysql::server]",
+             "recipe[mysql::client]",
              "recipe[nodejs::install_from_binary]",
              "recipe[huginn_development]"
            ]

deployment/roles/huginn_production.json

@@ -10,7 +10,7 @@
 
 "default_attributes" : {
   "mysql": {
-    "server_root_password": "",
+    "server_root_password": "password",
     "server_repl_password": "",
     "server_debian_password": ""
   },

deployment/site-cookbooks/huginn_development/recipes/default.rb

@@ -55,7 +55,7 @@ bash "huginn dependencies" do
     export LANG="en_US.UTF-8"
     export LC_ALL="en_US.UTF-8"
     sudo bundle install
-    sed s/REPLACE_ME_NOW\!/$(sudo rake secret)/ .env.example > .env
+    sed s/REPLACE_ME_NOW\!/$(sudo bundle exec rake secret)/ .env.example > .env
     sudo bundle exec rake db:create
     sudo bundle exec rake db:migrate
     sudo bundle exec rake db:seed

deployment/site-cookbooks/huginn_production/files/default/Gemfile

@@ -1,58 +0,0 @@
-source 'https://rubygems.org'
-
-gem 'rails'
-gem 'rake'
-gem 'mysql2'
-gem 'devise'
-gem 'kaminari'
-gem 'bootstrap-kaminari-views'
-gem "rufus-scheduler", :require => false
-gem 'json', '>= 1.7.7'
-gem 'jsonpath'
-gem 'twilio-ruby'
-
-gem 'delayed_job', :git => 'https://github.com/wok/delayed_job' # Until the YAML issues are fixed in master.
-gem 'delayed_job_active_record', "~> 0.3.3" # newer was giving a strange MySQL error
-gem "daemons"
-# gem "delayed_job_web"
-group :production do
-  gem 'unicorn'
-end
-gem 'foreman'
-gem 'dotenv-rails', :groups => [:development, :test]
-
-group :assets do
-  gem 'sass-rails',   '~> 3.2.3'
-  gem 'coffee-rails', '~> 3.2.1'
-  gem 'uglifier', '>= 1.0.3'
-  gem 'select2-rails'
-  gem 'jquery-rails'
-end
-
-gem 'geokit-rails3'
-gem 'kramdown'
-gem "typhoeus"
-gem 'nokogiri'
-gem 'wunderground'
-
-gem "twitter"
-gem 'twitter-stream', '>=0.1.16'
-gem 'em-http-request'
-
-platforms :ruby_18 do
-  gem 'system_timer'
-  gem 'fastercsv'
-end
-
-group :development do
-  gem 'pry'
-end
-
-group :development, :test do
-  gem 'rspec-rails'
-  gem 'rspec'
-  gem 'shoulda-matchers'
-  gem 'rr'
-  gem 'webmock', :require => false
-  gem 'rake'
-end

deployment/site-cookbooks/huginn_production/files/default/Procfile

@@ -1,4 +1,4 @@
-web: sudo bundle exec unicorn_rails -c config/unicorn.rb
-schedule: sudo bundle exec rails runner bin/schedule.rb
-twitter: sudo bundle exec rails runner bin/twitter_stream.rb
-dj: sudo bundle exec script/delayed_job run
+web: sudo bundle exec unicorn_rails -c config/unicorn.rb -E production
+schedule: sudo RAILS_ENV=production bundle exec rails runner bin/schedule.rb
+twitter: sudo RAILS_ENV=production bundle exec rails runner bin/twitter_stream.rb
+dj: sudo RAILS_ENV=production bundle exec script/delayed_job run

deployment/site-cookbooks/huginn_production/files/default/env.example

@@ -14,7 +14,7 @@ DATABASE_RECONNECT=true
 DATABASE_NAME=huginn_production
 DATABASE_POOL=5
 DATABASE_USERNAME=root
-DATABASE_PASSWORD=
+DATABASE_PASSWORD=password
 #DATABASE_HOST=your-domain-here.com
 #DATABASE_PORT=3306
 #DATABASE_SOCKET=/tmp/mysql.sock
@@ -23,6 +23,7 @@ DATABASE_PASSWORD=
 
 # Configure Rails environment.  This should only be needed in production and may cause errors in development.
 RAILS_ENV=production
+FORCE_SSL=false
 
 # Outgoing email settings.  To use Gmail or Google Apps, put your Google Apps domain or gmail.com
 # as the SMTP_DOMAIN and your Gmail username and password as the SMTP_USER_NAME and SMTP_PASSWORD.

deployment/site-cookbooks/huginn_production/files/default/nginx.conf

@@ -1,15 +1,18 @@
 #worker_process 2;
 user huginn huginn;
 
-events { 
+events {
   worker_connections 1024;
   accept_mutex on;
 }
 
 http {
+  types_hash_max_size 2048;
+  include    mime.types;
+
   upstream huginn_server {
     server unix:/home/huginn/shared/tmp/sockets/unicorn.sock;
-}
+  }
 
   server {
     listen 80;
@@ -23,13 +26,9 @@ http {
     }
     location @app {
       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-
       proxy_set_header X-Forwarded-Proto $scheme;
-
       proxy_set_header Host $http_host;
-
       proxy_redirect off;
-
       proxy_pass http://huginn_server;
     }
 }

deployment/site-cookbooks/huginn_production/files/default/unicorn.rb

@@ -17,7 +17,8 @@ stdout_path "log/unicorn_err.log"
 pid '/home/huginn/shared/tmp/pids/unicorn.pid'
 
 before_fork do |server, worker|
-  ActiveRecord::Base.connection.disconnect!
+  defined?(ActiveRecord::Base) and
+    ActiveRecord::Base.connection.disconnect!
   old_pid = "#{server.config[:pid]}.oldbin"
   if File.exists?(old_pid) && server.pid != old_pid
     begin
@@ -29,5 +30,6 @@ before_fork do |server, worker|
 end
 
 after_fork do |server, worker|
-  ActiveRecord::Base.establish_connection
+  defined?(ActiveRecord::Base) and
+    ActiveRecord::Base.establish_connection
 end

deployment/site-cookbooks/huginn_production/recipes/default.rb

@@ -14,10 +14,17 @@ group "huginn" do
   members ["huginn"]
 end
 
-%w("ruby1.9.1" "ruby1.9.1-dev" "libxslt-dev" "libxml2-dev" "curl" "libshadow-ruby1.8" "libmysqlclient-dev").each do |pkg|
+%w("ruby1.9.1" "ruby1.9.1-dev" "libxslt-dev" "libxml2-dev" "curl" "libshadow-ruby1.8" "libmysqlclient-dev" "libffi-dev" "libssl-dev" "rubygems").each do |pkg|
   package("#{pkg}")
 end
 
+bash "Setting default ruby version to 1.9" do
+  code <<-EOH
+    update-alternatives --set ruby /usr/bin/ruby1.9.1
+    update-alternatives --set gem /usr/bin/gem1.9.1
+  EOH
+end
+
 gem_package("rake")
 gem_package("bundle")
 
@@ -36,6 +43,7 @@ end
 
 deploy "/home/huginn" do
   repo "https://github.com/cantino/huginn.git"
+  branch "master"
   user "huginn"
   group "huginn"
   environment "RAILS_ENV" => "production"
@@ -56,7 +64,7 @@ deploy "/home/huginn" do
     end
     directory("/home/huginn/shared/tmp/pids")
     directory("/home/huginn/shared/tmp/sockets")
-    %w(Procfile unicorn.rb Gemfile nginx.conf).each do |file|
+    %w(Procfile unicorn.rb nginx.conf).each do |file|
       cookbook_file "/home/huginn/shared/config/#{file}" do
       owner "huginn"
       action :create_if_missing
@@ -77,16 +85,17 @@ deploy "/home/huginn" do
       code <<-EOH
       export LANG="en_US.UTF-8"
       export LC_ALL="en_US.UTF-8"
-      ln -nfs /home/huginn/shared/config/Gemfile ./Gemfile
       ln -nfs /home/huginn/shared/config/Procfile ./Procfile
       ln -nfs /home/huginn/shared/config/.env ./.env
       ln -nfs /home/huginn/shared/config/unicorn.rb ./config/unicorn.rb
-      sudo cp /home/huginn/shared/config/nginx.conf /etc/nginx/ 
-      sudo bundle install
-      sed -i s/REPLACE_ME_NOW\!/$(sudo rake secret)/ .env
-      sudo bundle exec rake db:create
-      sudo bundle exec rake db:migrate
-      sudo bundle exec rake db:seed
+      sudo cp /home/huginn/shared/config/nginx.conf /etc/nginx/
+      echo 'gem "unicorn", :group => :production' >> Gemfile
+      sudo bundle install --without=development --without=test
+      sed -i s/REPLACE_ME_NOW\!/$(sudo bundle exec rake secret)/ .env
+      sudo RAILS_ENV=production bundle exec rake db:create
+      sudo RAILS_ENV=production bundle exec rake db:migrate
+      sudo RAILS_ENV=production bundle exec rake db:seed
+      sudo RAILS_ENV=production bundle exec rake assets:precompile
       sudo foreman export upstart /etc/init -a huginn -u huginn -l log
       sudo start huginn
       EOH

deployment/solo.rb

@@ -1,6 +0,0 @@
-file_cache_path           "/tmp/chef-solo"
-data_bag_path             "/tmp/chef-solo/data_bags"
-encrypted_data_bag_secret "/tmp/chef-solo/data_bag_key"
-cookbook_path             [ "/tmp/chef-solo/site-cookbooks",
-                            "/tmp/chef-solo/cookbooks" ]
-role_path                 "/tmp/chef-solo/roles"

lib/rdbms_functions.rb

@@ -1,10 +1,10 @@
 module RDBMSFunctions
   def rdbms_date_add(source, unit, amount)
-    adapter_type = connection.adapter_name.downcase.to_sym
+    adapter_type = ActiveRecord::Base.connection.adapter_name.downcase.to_sym
     case adapter_type
       when :mysql, :mysql2
         "DATE_ADD(`#{source}`, INTERVAL #{amount} #{unit})"
-      when :postgresql    
+      when :postgresql
         "(#{source} + INTERVAL '#{amount} #{unit}')"
       else
         raise NotImplementedError, "Unknown adapter type '#{adapter_type}'"

spec/controllers/agents_controller_spec.rb

@@ -46,6 +46,22 @@ describe AgentsController do
     end
   end
 
+  describe "GET new with :id" do
+    it "opens a clone of a given Agent" do
+      sign_in users(:bob)
+      get :new, :id => agents(:bob_website_agent).to_param
+      assigns(:agent).attributes.should eq(users(:bob).agents.build_clone(agents(:bob_website_agent)).attributes)
+    end
+
+    it "only allows the current user to clone his own Agent" do
+      sign_in users(:bob)
+
+      lambda {
+        get :new, :id => agents(:jane_website_agent).to_param
+      }.should raise_error(ActiveRecord::RecordNotFound)
+    end
+  end
+
   describe "GET edit" do
     it "only shows Agents for the current user" do
       sign_in users(:bob)

spec/data_fixtures/stubhub_data.json

@@ -0,0 +1,17 @@
+{
+  "response":{
+    "docs":[
+      {
+      "url": "http://www.stubhub.com/event/name-1-1-2014-12345",
+      "seo_description_en_US": "name",
+      "event_date_local": "2014-01-01",
+      "maxPrice": "100",
+      "minPrice": "50",
+      "totalPostings": "100",
+      "totalTickets": "200",
+      "venue_name": "Venue Name"
+    }
+    ]
+  }
+}
+

spec/helpers/dot_helper_spec.rb

@@ -0,0 +1,48 @@
+require 'spec_helper'
+
+describe DotHelper do
+  describe "#dot_id" do
+    it "properly escapes double quotaion and backslash" do
+      dot_id('hello\\"').should == '"hello\\\\\\""'
+    end
+  end
+
+  describe "with example Agents" do
+    class Agents::DotFoo < Agent
+      default_schedule "2pm"
+
+      def check
+        create_event :payload => {}
+      end
+    end
+
+    class Agents::DotBar < Agent
+      cannot_be_scheduled!
+
+      def check
+        create_event :payload => {}
+      end
+    end
+
+    before do
+      stub(Agents::DotFoo).valid_type?("Agents::DotFoo") { true }
+      stub(Agents::DotBar).valid_type?("Agents::DotBar") { true }
+    end
+
+    describe "#agents_dot" do
+      it "generates a DOT script" do
+        @foo = Agents::DotFoo.new(:name => "foo")
+        @foo.user = users(:bob)
+        @foo.save!
+
+        @bar = Agents::DotBar.new(:name => "bar")
+        @bar.user = users(:bob)
+        @bar.sources << @foo
+        @bar.save!
+
+        agents_dot([@foo, @bar]).should == 'digraph foo {"foo";"foo"->"bar";"bar";}'
+        agents_dot([@foo, @bar], true).should == 'digraph foo {"foo"[URL="/agents/%d"];"foo"->"bar";"bar"[URL="/agents/%d"];}' % [@foo.id, @bar.id]
+      end
+    end
+  end
+end

spec/lib/utils_spec.rb

@@ -97,7 +97,7 @@ describe Utils do
     it "escapes </script> tags in the output JSON" do
       cleaned_json = Utils.jsonify(:foo => "bar", :xss => "</script><script>alert('oh no!')</script>")
       cleaned_json.should_not include("</script>")
-      cleaned_json.should include("<\\/script>")
+      cleaned_json.should include('\\u003c/script\\u003e')
     end
 
     it "html_safes the output unless :skip_safe is passed in" do

spec/models/agent_spec.rb

@@ -514,6 +514,51 @@ describe Agent do
         end
       end
     end
+
+    describe "Agent.build_clone" do
+      before do
+        Event.delete_all
+        @sender = Agents::SomethingSource.new(
+          name: 'Agent (2)',
+          options: { foo: 'bar2' },
+          schedule: '5pm')
+        @sender.user = users(:bob)
+        @sender.save!
+        @sender.create_event :payload => {}
+        @sender.create_event :payload => {}
+        @sender.events.count.should == 2
+
+        @receiver = Agents::CannotBeScheduled.new(
+          name: 'Agent',
+          options: { foo: 'bar3' },
+          keep_events_for: 3,
+          propagate_immediately: true)
+        @receiver.user = users(:bob)
+        @receiver.sources << @sender
+        @receiver.memory[:test] = 1
+        @receiver.save!
+      end
+
+      it "should create a clone of a given agent for editing" do
+        sender_clone = users(:bob).agents.build_clone(@sender)
+
+        sender_clone.attributes.should == Agent.new.attributes.
+          update(@sender.slice(:user_id, :type,
+            :options, :schedule, :keep_events_for, :propagate_immediately)).
+          update('name' => 'Agent (2) (2)', 'options' => { 'foo' => 'bar2' })
+
+        sender_clone.source_ids.should == []
+
+        receiver_clone = users(:bob).agents.build_clone(@receiver)
+
+        receiver_clone.attributes.should == Agent.new.attributes.
+          update(@receiver.slice(:user_id, :type,
+            :options, :schedule, :keep_events_for, :propagate_immediately)).
+          update('name' => 'Agent (3)', 'options' => { 'foo' => 'bar3' })
+
+        receiver_clone.source_ids.should == [@sender.id]
+      end
+    end
   end
 
   describe ".trigger_web_request" do

spec/models/agents/hipchat_agent_spec.rb

@@ -62,7 +62,7 @@ describe Agents::HipchatAgent do
     end
 
     it "should merge all options" do
-      @checker.send(:merge_options, @event).should == {
+      @checker.send(:merge_options, @event).deep_symbolize_keys.should == {
         :room_name => "test",
         :username => "Huggin user",
         :message => "Looks like its going to rain",

spec/models/agents/public_transport_agent_spec.rb

@@ -19,7 +19,6 @@ describe Agents::PublicTransportAgent do
       stub_request(:get, "http://webservices.nextbus.com/service/publicXMLFeed?a=sf-muni&command=predictionsForMultiStops&stops=N%7C5215").
          with(:headers => {'User-Agent'=>'Typhoeus - https://github.com/typhoeus/typhoeus'}).
          to_return(:status => 200, :body => File.read(Rails.root.join("spec/data_fixtures/public_transport_agent.xml")), :headers => {})
-      stub(Time).now {"2014-01-14 20:21:30 +0500".to_time}
     end
 
     it "should create 4 events" do
@@ -27,15 +26,18 @@ describe Agents::PublicTransportAgent do
     end
 
     it "should add 4 items to memory" do
-      @agent.memory.should == {}
-      @agent.check
-      @agent.memory.should == {"existing_routes" => [
-          {"stopTag"=>"5221", "tripTag"=>"5840324", "epochTime"=>"1389706393991", "currentTime"=>"2014-01-14 20:21:30 +0500"},
-          {"stopTag"=>"5221", "tripTag"=>"5840083", "epochTime"=>"1389706512784", "currentTime"=>"2014-01-14 20:21:30 +0500"},
-          {"stopTag"=>"5215", "tripTag"=>"5840324", "epochTime"=>"1389706282012", "currentTime"=>"2014-01-14 20:21:30 +0500"},
-          {"stopTag"=>"5215", "tripTag"=>"5840083", "epochTime"=>"1389706400805", "currentTime"=>"2014-01-14 20:21:30 +0500"}
-        ]
-      }
+      time_travel_to Time.parse("2014-01-14 20:21:30 +0500") do
+        @agent.memory.should == {}
+        @agent.check
+        @agent.save
+        @agent.reload.memory.should == {"existing_routes" => [
+            {"stopTag"=>"5221", "tripTag"=>"5840324", "epochTime"=>"1389706393991", "currentTime"=>Time.now.to_s},
+            {"stopTag"=>"5221", "tripTag"=>"5840083", "epochTime"=>"1389706512784", "currentTime"=>Time.now.to_s},
+            {"stopTag"=>"5215", "tripTag"=>"5840324", "epochTime"=>"1389706282012", "currentTime"=>Time.now.to_s},
+            {"stopTag"=>"5215", "tripTag"=>"5840083", "epochTime"=>"1389706400805", "currentTime"=>Time.now.to_s}
+          ]
+        }
+      end
     end
 
     it "should not create events twice" do
@@ -44,10 +46,13 @@ describe Agents::PublicTransportAgent do
     end
 
     it "should reset memory after 2 hours" do
-      lambda { @agent.check }.should change {@agent.events.count}.by(4)
-      stub(Time).now {"2014-01-14 20:21:30 +0500".to_time + 3.hours}
-      @agent.cleanup_old_memory
-      lambda { @agent.check }.should change {@agent.events.count}.by(4)
+      time_travel_to Time.parse("2014-01-14 20:21:30 +0500") do
+        lambda { @agent.check }.should change {@agent.events.count}.by(4)
+      end
+      time_travel_to "2014-01-14 23:21:30 +0500".to_time do
+        @agent.cleanup_old_memory
+        lambda { @agent.check }.should change {@agent.events.count}.by(4)
+      end
     end
   end
 

spec/models/agents/stubhub_agent_spec.rb

@@ -0,0 +1,67 @@
+require 'spec_helper'
+
+describe Agents::StubhubAgent do
+
+  let(:name) { 'Agent Name' }
+  let(:url) { 'http://www.stubhub.com/event/name-1-1-2014-12345' }
+  let(:parsed_body) { JSON.parse(body)['response']['docs'][0] }
+  let(:valid_params) { { 'url' => parsed_body['url'] } }
+  let(:body) { File.read(Rails.root.join('spec/data_fixtures/stubhub_data.json')) }
+  let(:stubhub_event_id) { 12345 }
+  let(:response_payload) { {
+                            'url' => url,
+                            'name' => parsed_body['seo_description_en_US'],
+                            'date' => parsed_body['event_date_local'],
+                            'max_price' => parsed_body['maxPrice'],
+                            'min_price' => parsed_body['minPrice'],
+                            'total_postings' => parsed_body['totalPostings'],
+                            'total_tickets' => parsed_body['totalTickets'],
+                            'venue_name' => parsed_body['venue_name']
+                            } }
+
+  before do
+      stub_request(:get, "http://www.stubhub.com/listingCatalog/select/?q=%2B%20stubhubDocumentType:event%0D%0A%2B%20event_id:#{stubhub_event_id}%0D%0A&rows=10&start=0&wt=json").
+         to_return(:status => 200, :body => body, :headers => {})
+
+    @stubhub_agent = described_class.new(name: name, options: valid_params)
+    @stubhub_agent.user = users(:jane)
+    @stubhub_agent.save!
+  end
+
+
+  describe "#check" do
+
+    it 'should create an event' do
+      expect { @stubhub_agent.check }.to change { Event.count }.by(1)
+    end
+
+    it 'should properly parse the response' do
+      event = @stubhub_agent.check
+      event.payload.should == response_payload
+    end
+  end
+
+  describe "validations" do
+    before do
+      @stubhub_agent.should be_valid
+    end
+
+    it "should require a url" do
+      @stubhub_agent.options['url'] = nil
+      @stubhub_agent.should_not be_valid
+    end
+
+  end
+
+  describe "#working?" do
+    it "checks if events have been received within the expected receive period" do
+      @stubhub_agent.should_not be_working
+
+      Agents::StubhubAgent.async_check @stubhub_agent.id
+      @stubhub_agent.reload.should be_working
+      two_days_from_now = 2.days.from_now
+      stub(Time).now { two_days_from_now }
+      @stubhub_agent.reload.should_not be_working
+    end
+  end
+end

spec/models/agents/trigger_agent_spec.rb

@@ -30,9 +30,32 @@ describe Agents::TriggerAgent do
       @checker.should be_valid
     end
 
-    it "should validate presence of options" do
+    it "should validate presence of message" do
       @checker.options['message'] = nil
       @checker.should_not be_valid
+
+      @checker.options['message'] = ''
+      @checker.should_not be_valid
+    end
+
+    it "should be valid without a message when 'keep_event' is set" do
+      @checker.options['keep_event'] = 'true'
+      @checker.options['message'] = ''
+      @checker.should be_valid
+    end
+
+    it "if present, 'keep_event' must equal true or false" do
+      @checker.options['keep_event'] = 'true'
+      @checker.should be_valid
+
+      @checker.options['keep_event'] = 'false'
+      @checker.should be_valid
+
+      @checker.options['keep_event'] = ''
+      @checker.should be_valid
+
+      @checker.options['keep_event'] = 'tralse'
+      @checker.should_not be_valid
     end
 
     it "should validate the three fields in each rule" do
@@ -278,5 +301,38 @@ describe Agents::TriggerAgent do
         @checker.receive([@event])
       }.should_not change { Event.count }
     end
+
+    describe "when 'keep_event' is true" do
+      before do
+        @checker.options['keep_event'] = 'true'
+        @event.payload['foo']['bar']['baz'] = "5"
+        @checker.options['rules'].first['type'] = "field<value"
+      end
+
+      it "can re-emit the origin event" do
+        @checker.options['rules'].first['value'] = 3
+        @checker.options['message'] = ''
+        @event.payload['message'] = 'hi there'
+
+        lambda {
+          @checker.receive([@event])
+        }.should_not change { Event.count }
+
+        @checker.options['rules'].first['value'] = 6
+        lambda {
+          @checker.receive([@event])
+        }.should change { Event.count }.by(1)
+
+        @checker.most_recent_event.payload.should == @event.payload
+      end
+
+      it "merges 'message' into the original event when present" do
+        @checker.options['rules'].first['value'] = 6
+
+        @checker.receive([@event])
+
+        @checker.most_recent_event.payload.should == @event.payload.merge(:message => "I saw '5' from Joe")
+      end
+    end
   end
 end

spec/models/agents/website_agent_spec.rb

@@ -21,28 +21,71 @@ describe Agents::WebsiteAgent do
       @checker.save!
     end
 
-    describe "#check" do
+    describe "validations" do
+      before do
+        @checker.should be_valid
+      end
+
       it "should validate the integer fields" do
-        @checker.options['expected_update_period_in_days'] = "nonsense"
-        lambda { @checker.save! }.should raise_error;
         @checker.options['expected_update_period_in_days'] = "2"
+        @checker.should be_valid
+
+        @checker.options['expected_update_period_in_days'] = "nonsense"
+        @checker.should_not be_valid
+      end
+
+      it "should validate uniqueness_look_back" do
         @checker.options['uniqueness_look_back'] = "nonsense"
-        lambda { @checker.save! }.should raise_error;
+        @checker.should_not be_valid
+
+        @checker.options['uniqueness_look_back'] = "2"
+        @checker.should be_valid
+      end
+
+      it "should validate headers" do
+        @checker.options['headers'] = "blah"
+        @checker.should_not be_valid
+
+        @checker.options['headers'] = ""
+        @checker.should be_valid
+
+        @checker.options['headers'] = {}
+        @checker.should be_valid
+
+        @checker.options['headers'] = { 'foo' => 'bar' }
+        @checker.should be_valid
+      end
+
+      it "should validate mode" do
         @checker.options['mode'] = "nonsense"
-        lambda { @checker.save! }.should raise_error;
-        @checker.options = @site
+        @checker.should_not be_valid
+
+        @checker.options['mode'] = "on_change"
+        @checker.should be_valid
+
+        @checker.options['mode'] = "all"
+        @checker.should be_valid
+
+        @checker.options['mode'] = ""
+        @checker.should be_valid
       end
 
       it "should validate the force_encoding option" do
+        @checker.options['force_encoding'] = ''
+        @checker.should be_valid
+
         @checker.options['force_encoding'] = 'UTF-8'
-        lambda { @checker.save! }.should_not raise_error;
+        @checker.should be_valid
+
         @checker.options['force_encoding'] = ['UTF-8']
-        lambda { @checker.save! }.should raise_error;
+        @checker.should_not be_valid
+
         @checker.options['force_encoding'] = 'UTF-42'
-        lambda { @checker.save! }.should raise_error;
-        @checker.options = @site
+        @checker.should_not be_valid
       end
+    end
 
+    describe "#check" do
       it "should check for changes (and update Event.expires_at)" do
         lambda { @checker.check }.should change { Event.count }.by(1)
         event = Event.last
@@ -331,11 +374,26 @@ describe Agents::WebsiteAgent do
         end
       end
     end
+
+    describe "#receive" do
+      it "should scrape from the url element in incoming event payload" do
+        @event = Event.new
+        @event.agent = agents(:bob_rain_notifier_agent)
+        @event.payload = { 'url' => "http://xkcd.com" }
+
+        lambda {
+          @checker.options = @site
+          @checker.receive([@event])
+        }.should change { Event.count }.by(1)
+      end
+    end
   end
 
   describe "checking with http basic auth" do
     before do
-      stub_request(:any, /user:pass/).to_return(:body => File.read(Rails.root.join("spec/data_fixtures/xkcd.html")), :status => 200)
+      stub_request(:any, /example/).
+        with(headers: { 'Authorization' => "Basic #{['user:pass'].pack('m').chomp}" }).
+        to_return(:body => File.read(Rails.root.join("spec/data_fixtures/xkcd.html")), :status => 200)
       @site = {
         'name' => "XKCD",
         'expected_update_period_in_days' => 2,
@@ -361,4 +419,32 @@ describe Agents::WebsiteAgent do
       end
     end
   end
+
+  describe "checking with headers" do
+    before do
+      stub_request(:any, /example/).
+        with(headers: { 'foo' => 'bar', 'user_agent' => /Faraday/ }).
+        to_return(:body => File.read(Rails.root.join("spec/data_fixtures/xkcd.html")), :status => 200)
+      @site = {
+        'name' => "XKCD",
+        'expected_update_period_in_days' => 2,
+        'type' => "html",
+        'url' => "http://www.example.com",
+        'mode' => 'on_change',
+        'headers' => { 'foo' => 'bar' },
+        'extract' => {
+          'url' => { 'css' => "#comic img", 'attr' => "src" },
+        }
+      }
+      @checker = Agents::WebsiteAgent.new(:name => "ua", :options => @site)
+      @checker.user = users(:bob)
+      @checker.save!
+    end
+
+    describe "#check" do
+      it "should check for changes" do
+        lambda { @checker.check }.should change { Event.count }.by(1)
+      end
+    end
+  end
 end

spec/spec_helper.rb

@@ -42,4 +42,5 @@ RSpec.configure do |config|
 
   config.include Devise::TestHelpers, :type => :controller
   config.include SpecHelpers
+  config.include Delorean
 end