Merge branch 'master' of https://github.com/huginn/huginn into issue_2150

.buildpacks

@@ -1,3 +1,4 @@
 https://github.com/cantino/heroku-selectable-procfile.git
 https://github.com/heroku/heroku-buildpack-ruby.git
 https://github.com/weibeld/heroku-buildpack-graphviz.git
+https://github.com/chrismytton/heroku-buildpack-jq.git

.env.example

@@ -105,6 +105,10 @@ IMPORT_DEFAULT_SCENARIO_FOR_ALL_USERS=true
 # SMTP_USER_NAME must be set to none or else you will receive
 # errors that AUTH not enabled.
 
+# Uncomment if you want to use `/usr/sbin/sendmail` to send email instead of SMTP.
+# This option is ignored unless RAILS_ENV=production, and setting it to `sendmail` causes the settings in the rest of this section (except EMAIL_FROM_ADDRESS) to be ignored.
+# SMTP_DELIVERY_METHOD=sendmail
+
 SMTP_DOMAIN=your-domain-here.com
 SMTP_USER_NAME=you@gmail.com
 SMTP_PASSWORD=somepassword
@@ -112,6 +116,7 @@ SMTP_SERVER=smtp.gmail.com
 SMTP_PORT=587
 SMTP_AUTHENTICATION=plain
 SMTP_ENABLE_STARTTLS_AUTO=true
+SMTP_SSL=false
 
 # Set to true to send real emails via SMTP when running in the development Rails environment.
 # Set to false to have emails intercepted in development and displayed at http://localhost:3000/letter_opener
@@ -218,6 +223,13 @@ SCHEDULER_FREQUENCY=0.3
 # You can use `m` for minutes, `h` for hours, and `d` for days.
 EVENT_EXPIRATION_CHECK=6h
 
+# Enable JqAgent which uses jq.  Specify a file path to the jq(1)
+# command or just `jq`.  This is not enabled by default because jq can
+# cause infinite loop and is not suitable for public service.
+# Only uncomment this when you trust everyone using your Huginn
+# installation.
+#USE_JQ=jq
+
 # Use Graphviz for generating diagrams instead of using Google Chart
 # Tools.  Specify a dot(1) command path built with SVG support
 # enabled.

.github/FUNDING.yml

@@ -0,0 +1,13 @@
+# These are supported funding model platforms
+
+github: # up to 4 GitHub Sponsors-enabled usernames
+- knu
+patreon: # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.gitignore

@@ -12,7 +12,7 @@ capybara-*.html
 /public/system/*
 /coverage/
 /spec/tmp/*
-**.orig
+*.orig
 rerun.txt
 pickle-email-*.html
 .idea/

.graphviz

@@ -1 +0,0 @@
-http://www.graphviz.org/pub/graphviz/stable/SOURCES/graphviz-2.38.0.tar.gz

.travis.yml

@@ -1,4 +1,4 @@
-dist: trusty
+dist: bionic
 sudo: required
 language: ruby
 services:
@@ -6,12 +6,13 @@ services:
   - mysql
   - postgresql
 before_install:
-  - gem update --system
-  - gem install bundler
+  - sudo apt-get -y install jq
+  - gem install bundler -v '< 2' --conservative --force
+before_script:
+  - sudo systemctl start postgresql
 env:
   global:
     - APP_SECRET_TOKEN=b2724973fd81c2f4ac0f92ac48eb3f0152c4a11824c122bcf783419a4c51d8b9bba81c8ba6a66c7de599677c7f486242cf819775c433908e77c739c5c8ae118d
-    - RSPEC_TASK=spec:nofeatures
     - COVERALLS_PARALLEL=true
     - secure: fzmSI7PQz6CJiIJNAtLAuy3TMmYCrK4bUil3uufh8JkHfpSGWOZt2i6fZ8yZ7pzwG5Aw7eZDgdFsNcEPJlgUDJhlwjg+QxCJslhotTQ9qI3Ieo85peWlU9dZFTOZcrCu0net/hY2FE4ZpTRb5r8A/DRv9ukA8P8tShhePCjckgg=
     - secure: YjW07LpRSiC9xB6PhLQ4LVv2VphvF3IacV43PLfvzdagjy14yAwKXTUlSadgRaMbndB2dlCTe3YcY11a/xtX/2HDrF14NHPXQdL7e2dJUS3CDLSKZK26x1SOiaaDIrl1jO1xr5kOUd+564MAcNUzDTJQR4CrWl/5t6EwW4iYQVc=
@@ -19,49 +20,25 @@ env:
   matrix:
     - DATABASE_ADAPTER=mysql2
     - DATABASE_ADAPTER=postgresql DATABASE_USERNAME=postgres
-    - DOCKER_IMAGE=cantino/huginn-single-process DOCKERFILE=docker/single-process/Dockerfile
-    - DOCKER_IMAGE=cantino/huginn DOCKERFILE=docker/multi-process/Dockerfile
     - DOCKER_IMAGE=huginn/huginn-single-process DOCKERFILE=docker/single-process/Dockerfile
     - DOCKER_IMAGE=huginn/huginn DOCKERFILE=docker/multi-process/Dockerfile
-    - RSPEC_TASK=spec:features
 matrix:
   exclude:
-    - env: DOCKER_IMAGE=cantino/huginn-single-process DOCKERFILE=docker/single-process/Dockerfile
-    - env: DOCKER_IMAGE=cantino/huginn DOCKERFILE=docker/multi-process/Dockerfile
     - env: DOCKER_IMAGE=huginn/huginn-single-process DOCKERFILE=docker/single-process/Dockerfile
     - env: DOCKER_IMAGE=huginn/huginn DOCKERFILE=docker/multi-process/Dockerfile
-    - env: RSPEC_TASK=spec:features
   include:
-    - rvm: 2.4.4
-      env: DATABASE_ADAPTER=mysql2 DOCKER_IMAGE=cantino/huginn-single-process DOCKERFILE=docker/single-process/Dockerfile
-    - rvm: 2.4.4
-      env: DATABASE_ADAPTER=mysql2 DOCKER_IMAGE=cantino/huginn DOCKERFILE=docker/multi-process/Dockerfile
-    - rvm: 2.4.4
+    - rvm: 2.6.5
       env: DATABASE_ADAPTER=mysql2 DOCKER_IMAGE=huginn/huginn-single-process DOCKERFILE=docker/single-process/Dockerfile
-    - rvm: 2.4.4
+    - rvm: 2.6.5
       env: DATABASE_ADAPTER=mysql2 DOCKER_IMAGE=huginn/huginn DOCKERFILE=docker/multi-process/Dockerfile
-    - rvm: 2.5.1
-      env: RSPEC_TASK=spec:features DATABASE_ADAPTER=mysql2
-    - rvm: 2.5.1
-      env: RSPEC_TASK=spec:features DATABASE_ADAPTER=postgresql DATABASE_USERNAME=postgres
-    - rvm: 2.4.4
-      env: RSPEC_TASK=spec:features DATABASE_ADAPTER=mysql2
-    - rvm: 2.4.4
-      env: RSPEC_TASK=spec:features DATABASE_ADAPTER=postgresql DATABASE_USERNAME=postgres
-    - rvm: 2.3.7
-      env: RSPEC_TASK=spec:features DATABASE_ADAPTER=mysql2
-    - rvm: 2.3.7
-      env: RSPEC_TASK=spec:features DATABASE_ADAPTER=postgresql DATABASE_USERNAME=postgres
 rvm:
-- 2.2.10
-- 2.3.7
-- 2.4.4
-- 2.5.1
+  - 2.5.8
+  - 2.6.6
 cache: bundler
 bundler_args: --without development production
 script:
   - if [ -z "${DOCKER_IMAGE}" ]; then bundle exec rake db:create db:migrate; else true; fi
-  - if [ -z "${DOCKER_IMAGE}" ]; then bundle exec rake $RSPEC_TASK; else ./build_docker_image.sh; fi
+  - if [ -z "${DOCKER_IMAGE}" ]; then bundle exec rake; else ./build_docker_image.sh; fi
 notifications:
   irc:
     channels:

CHANGES.md

@@ -2,52 +2,294 @@
 
 | DateOfChange   | Changes                                                                                                      |
 |----------------|--------------------------------------------------------------------------------------------------------------|
+| Apr 04, 2020   | Upgrade ubuntu versions of docker images to 18.04. [2603](https://github.com/huginn/huginn/pull/2603) **If you are using the `huginn/huginn` image with a internal MySQL database, back up your database volume before updating**  |
+| Mar 31, 2020   | Add FUNDING.yml [2728](https://github.com/huginn/huginn/pull/2728) |
+| Mar 30, 2020   | Improve formatting of OpenShift documentation [2724](https://github.com/huginn/huginn/pull/2724) |
+| Mar 30, 2020   | Fix deployment via OpenShift [2726](https://github.com/huginn/huginn/pull/2726) |
+| Mar 29, 2020   | Fix UserLocationAgent never displayed the course of the user [2709](https://github.com/huginn/huginn/pull/2709) |
+| Mar 29, 2020   | Allow email agent to dry run [2706](https://github.com/huginn/huginn/pull/2706) |
+| Feb 29, 2020   | Allow slack agent to dry run [2694](https://github.com/huginn/huginn/pull/2694) |
+| Feb 29, 2020   | Upgrade rake and nokogiri to fix CVEs [2698](https://github.com/huginn/huginn/pull/2698) |
+| Feb 16, 2020   | Simplify the download of the latest version of jq [2683](https://github.com/huginn/huginn/pull/2683) |
+| Feb 11, 2020   | Make sure to download the latest release of jq from GitHub [2681](https://github.com/huginn/huginn/pull/2681) |
+| Feb 06, 2020   | Install the latest jq for JqAgent [2675](https://github.com/huginn/huginn/pull/2675) |
+| Jan 11, 2020   | Drop support for Ruby 2.3 which reached EOL |
+| Jan 11, 2020   | Add Jq Agent [2665](https://github.com/huginn/huginn/pull/2665) |
+| Jan 10, 2020   | Update README.md about dockers images, adds details about seeding [2669](https://github.com/huginn/huginn/pull/2669) |
+| Jan 09, 2020   | Upgrade mini-racer to 0.2.9 and libv8 to 7.3.492.27.1 [2664](https://github.com/huginn/huginn/pull/2664) |
+| Jan 07, 2020   | Allow Agents injected as GEMs to define a UI [2659](https://github.com/huginn/huginn/pull/2659) |
+| Dec 19, 2019   | Upgrade rack to 2.0.8 to fix CVE [2651](https://github.com/huginn/huginn/pull/2651) |
+| Dec 19, 2019   | Fix TravisCI build [2650](https://github.com/huginn/huginn/pull/2650) |
+| Dec 18, 2019   | Fix link to andrewcurioso/huginn [2647](https://github.com/huginn/huginn/pull/2647) |
+| Dec 14, 2019   | Sql installation info [2646](https://github.com/huginn/huginn/pull/2646) |
+| Dec 14, 2019   | FIX broken link to docker container [2645](https://github.com/huginn/huginn/pull/2645) |
+| Dec 13, 2019   | fix: password prompt at sql secure step [2643](https://github.com/huginn/huginn/pull/2643) |
+| Dec 08, 2019   | Allow dry-run of PushoverAgent [2640](https://github.com/huginn/huginn/pull/2640) |
+| Dec 03, 2019   | Prevent `GoogleCalendar.open` from raising exception in ensure block [2634](https://github.com/huginn/huginn/pull/2634) |
+| Dec 03, 2019   | Upgrade rubies to the latest minor releases [2630](https://github.com/huginn/huginn/pull/2630) |
+| Nov 27, 2019   | Upgrade Feedjira to 3.1.0 that improves RssAgent [2629](https://github.com/huginn/huginn/pull/2629) |
+| Nov 16, 2019   | email is the plural of email [2623](https://github.com/huginn/huginn/pull/2623) |
+| Nov 10, 2019   | Upgrade loofah and nokogiri [2621](https://github.com/huginn/huginn/pull/2621) |
+| Oct 06, 2019   | Escape MySQL database name during migration [2608](https://github.com/huginn/huginn/pull/2608) |
+| Oct 03, 2019   | Upgrade typhoeus to fix obscure error SSL validation error [2602](https://github.com/huginn/huginn/pull/2602) |
+| Sep 14, 2019   | Do not sanitize `@body` in a text part in EmailAgent [2595](https://github.com/huginn/huginn/pull/2595) |
+| Aug 25, 2019   | Make JavaScript Agent optional [2590](https://github.com/huginn/huginn/pull/2590) |
+| Aug 17, 2019   | Remove support for the Weather Underground API [2396](https://github.com/huginn/huginn/pull/2396) |
+| Aug 17, 2019   | Add ability to reemit all of an Agent's events from the UI [2573](https://github.com/huginn/huginn/pull/2573) |
+| Aug 17, 2019   | Improve Utils.normalize_uri() [2585](https://github.com/huginn/huginn/pull/2585) |
+| Aug 12, 2019   | Upgrade `nokogiri` to 1.10.4 for CVE-2019-5477 [2582](https://github.com/huginn/huginn/pull/2582) |
+| Aug 01, 2019   | Add `group_by` liquid filter [2572](https://github.com/huginn/huginn/pull/2572) |
+| Jul 29, 2019   | Upgrade `mini_magic`, fix focus rspec filter limiting test suit on CI [2568](https://github.com/huginn/huginn/pull/2568) |
+| Jul 28, 2019   | Fix description of `EmailDigestAgent` [2567](https://github.com/huginn/huginn/pull/2567) |
+| Jul 27, 2019   | Increase `TelegramAgent` Caption Length [2560](https://github.com/huginn/huginn/pull/2560) |
+| Jun 30, 2019   | Install Node 10 instead of Node 0.12 in the install guide [2551](https://github.com/huginn/huginn/pull/2551) |
+| Jun 10, 2019   | Improve Docker README.md [2546](https://github.com/huginn/huginn/pull/2546) |
+| May 19, 2019   | Update `liquid` to the latest version 4.0.3 [2536](https://github.com/huginn/huginn/pull/2536) |
+| May 07, 2019   | Add `drop_pending_events` option to `AgentControllerConcern` [2532](https://github.com/huginn/huginn/pull/2532) |
+| Apr 30, 2019   | Update `nokogiri` for CVEs [2531](https://github.com/huginn/huginn/pull/2531) |
+| Apr 28, 2019   | Set `inbound_event` when creating `AgentLog` entries [2530](https://github.com/huginn/huginn/pull/2530) |
+| Apr 18, 2019   | Replace rubyracer with mini_racer, dropped support for Debian 7 aka Wheezy [1961](https://github.com/huginn/huginn/pull/1961) |
+| Apr 16, 2019   | Force SNI support in Net modules for IMAP/POP3/SMTP [2523](https://github.com/huginn/huginn/pull/2523) |
+| Apr 15, 2019   | Upgrade `devise` to fix CVE [2525](https://github.com/huginn/huginn/pull/2525) |
+| Apr 14, 2019   | Set Heroku stack to `heroku-18` in app.json [2524](https://github.com/huginn/huginn/pull/2524) |
+| Mar 29, 2019   | Add digest filters to our Liquid engine [2516](https://github.com/huginn/huginn/pull/2516) |
+| Mar 29, 2019   | Add Liquid based rule support to `TriggerAgent` [2514](https://github.com/huginn/huginn/pull/2514) |
+| Mar 29, 2019   | Add a `delete` option to `ImapFolderAgent` [2515](https://github.com/huginn/huginn/pull/2515) |
+| Mar 17, 2019   | Update `rails` for CVEs [2508](https://github.com/huginn/huginn/pull/2508) |
+| Mar 02, 2019   | Revert "Update DataOutputAgent accept header for browser compatibilit… [2499](https://github.com/huginn/huginn/pull/2499) |
+| Feb 14, 2019   | Improve description of `TriggerAgent` [2489](https://github.com/huginn/huginn/pull/2489) |
+| Jan 29, 2019   | Add `event_headers` support to `ImapFolderAgent` [2476](https://github.com/huginn/huginn/pull/2476) |
+| Jan 26, 2019   | Make the `mail` gem use real-world encoders for some Japanese charsets [2477](https://github.com/huginn/huginn/pull/2477) |
+| Jan 25, 2019   | Update `jsonpath` gem [2474](https://github.com/huginn/huginn/pull/2474) |
+| Jan 25, 2019   | Ddd jpg as renderType to phantomjscloud setting [2470](https://github.com/huginn/huginn/pull/2470) |
+| Jan 09, 2019   | Update manual installation instructions [2461](https://github.com/huginn/huginn/pull/2461) |
+| Jan 08, 2019   | Improve #2434 (Add HTTP Headers to Webhook Agent) [2454](https://github.com/huginn/huginn/pull/2454) |
+| Jan 08, 2019   | Fix for Bundler 2 [2455](https://github.com/huginn/huginn/pull/2455) |
+| Jan 07, 2019   | Winter cleaning [2452](https://github.com/huginn/huginn/pull/2452) |
+| Jan 07, 2019   | Persist override of HUGINN_PORT [2448](https://github.com/huginn/huginn/pull/2448) |
+| Dec 15, 2018   | Bump `capistrano` to latest version to avoid OpenSSL error [2437](https://github.com/huginn/huginn/pull/2437) |
+| Dec 02, 2018   | Update `rails` to 5.2.1.1 for CVE-2018-16476 [2428](https://github.com/huginn/huginn/pull/2428) |
+| Nov 22, 2018   | Bump `rack and` `nokogiri` [2425](https://github.com/huginn/huginn/pull/2425) |
+| Nov 21, 2018   | Improve usability of `DelayAgent` [2422](https://github.com/huginn/huginn/pull/2422) |
+| Nov 15, 2018   | Bug fix: restrict IFS to only the read builtin [2413](https://github.com/huginn/huginn/pull/2413) |
+| Nov 14, 2018   | Added better validations for the `WatherAgent`. [2414](https://github.com/huginn/huginn/pull/2414) |
+| Nov 03, 2018   | Updated link to Kitematic [2406](https://github.com/huginn/huginn/pull/2406) |
+| Oct 29, 2018   | Move method `is_positive_integer?` to "agent.rb" [2402](https://github.com/huginn/huginn/pull/2402) |
+| Oct 12, 2018   | Do not seed databnase when a user exists [2386](https://github.com/huginn/huginn/pull/2386) |
+| Oct 12, 2018   | Fix name of Twitter event field in `TwitterUserAgent` [2388](https://github.com/huginn/huginn/pull/2388) |
+| Oct 03, 2018   | Delete stale delayed_job pid file when starting docker containers [2385](https://github.com/huginn/huginn/pull/2385) |
+| Oct 03, 2018   | Update `jsonpath` to 0.9.4 [2383](https://github.com/huginn/huginn/pull/2383) |
+| Oct 03, 2018   | Added support for MySQL 8 and MariaDB 10.3. Dropped support for MySQL < 5.5 and PostgreSQL < 9.2 [2384](https://github.com/huginn/huginn/pull/2384) |
+| Aug 30, 2018   | Support merge mode in `JsonParseAgent`  [2353](https://github.com/huginn/huginn/pull/2353) |
+| Aug 27, 2018   | Docker: extract heredocs from multi-process init and update docker-compose files [2298](https://github.com/huginn/huginn/pull/2298) |
+| Aug 18, 2018   | Remove `GoogleFlightsAgent` [2351](https://github.com/huginn/huginn/pull/2351) |
+| Aug 07, 2018   | Fix IMAP encoding issues [2346](https://github.com/huginn/huginn/pull/2346) |
+| Aug 07, 2018   | Upgrade rubies to their latest patch release [2267](https://github.com/huginn/huginn/pull/2267) |
+| Aug 07, 2018   | Fix "already retweeted" error detection [2174](https://github.com/huginn/huginn/pull/2174) |
+| Aug 07, 2018   | Update twitter-stream and its dependency, eventmachine [2345](https://github.com/huginn/huginn/pull/2345) |
+| Aug 01, 2018   | Respect WEB_CONCURRENCY env in unicorn.rb.example [2342](https://github.com/huginn/huginn/pull/2342) |
+| Jul 31, 2018   | Update DataOutputAgent accept header for browser compatibility [2338](https://github.com/huginn/huginn/pull/2338) |
+| Jul 16, 2018   | Make 'expected_receive_period_in_days' optional [2333](https://github.com/huginn/huginn/pull/2333) |
+| Jul 14, 2018   | Fix "working?" of PostAgent [2329](https://github.com/huginn/huginn/pull/2329) |
+| Jun 22, 2018   | Bump sprockets version [2321](https://github.com/huginn/huginn/pull/2321) |
+| Jun 17, 2018   | Upgrade to Rails 5.2 [2266](https://github.com/huginn/huginn/pull/2266) |
+| Jun 17, 2018   | Add link for darksky api key since wunderground no longer free [2313](https://github.com/huginn/huginn/pull/2313) |
+| Jun 08, 2018   | Update google_translation_agent.rb [2309](https://github.com/huginn/huginn/pull/2309) |
+| Jun 04, 2018   | Improve logic of "working?" in MqttAgent [2307](https://github.com/huginn/huginn/pull/2307) |
+| May 18, 2018   | Pass URLs to Telegram API directly [2285](https://github.com/huginn/huginn/pull/2285) |
+| May 02, 2018   | Bump binding of caller to fix incompatibility with ruby 2.5 [2276](https://github.com/huginn/huginn/pull/2276) |
+| Apr 30, 2018   | Bump rails-html-sanitizer to 1.0.4 to address CVE [2274](https://github.com/huginn/huginn/pull/2274) |
+| Apr 28, 2018   | Add ssl support for mail config [2270](https://github.com/huginn/huginn/pull/2270) |
+| Apr 21, 2018   | PushoverAgent: HTML message support [2264](https://github.com/huginn/huginn/pull/2264) |
+| Apr 04, 2018   | Allow to run on ruby 2.5 and fix warnings [2216](https://github.com/huginn/huginn/pull/2216) |
+| Apr 04, 2018   | [#2248] Allow the `PostAgent` to consume arrays [2249](https://github.com/huginn/huginn/pull/2249) |
+| Mar 21, 2018   | Update loofah due to CVE-2018-8048 [2243](https://github.com/huginn/huginn/pull/2243) |
+| Feb 13, 2018   | Clarify xpath usage with example [2217](https://github.com/huginn/huginn/pull/2217) |
+| Feb 03, 2018   | Fix syntax error in Website Agent description [2207](https://github.com/huginn/huginn/pull/2207) |
+| Jan 31, 2018   | Upgrade nokogiri to 1.8.2 to address vulnerabilities found in libxml2 [2205](https://github.com/huginn/huginn/pull/2205) |
+| Jan 29, 2018   | Split long Telegram messages [2171](https://github.com/huginn/huginn/pull/2171) |
+| Jan 08, 2018   | Rescue ZeroDivisionError on validation [2189](https://github.com/huginn/huginn/pull/2189) |
+| Jan 05, 2018   | Fix Liquid interpolation in TwilioAgent helper methods [2187](https://github.com/huginn/huginn/pull/2187) |
+| Jan 02, 2018   | Fix Docker testing README for better GitHub readability [2183](https://github.com/huginn/huginn/pull/2183) |
+| Dec 01, 2017   | Add `array` extraction option to WebsiteAgent in HTML/XML mode [2170](https://github.com/huginn/huginn/pull/2170) |
+| Nov 30, 2017   | Add options to Telegram Agent [2168](https://github.com/huginn/huginn/pull/2168) |
+| Nov 11, 2017   | Upgrade Dropbox Agents to new v2 API [2146](https://github.com/huginn/huginn/pull/2146) |
+| Nov 11, 2017   | Add proxy support for WebRequestConcern [2157](https://github.com/huginn/huginn/pull/2157) |
+| Nov 11, 2017   | Allow usage of custom Liquid tags in LiquidOutputAgent [2160](https://github.com/huginn/huginn/pull/2160) |
+| Oct 30, 2017   | Add a workaround for broken AlreadyRetweeted error detection [2155](https://github.com/huginn/huginn/pull/2155) |
+| Oct 27, 2017   | Clarifying no authentication scenario [2153](https://github.com/huginn/huginn/pull/2153) |
+| Oct 22, 2017   | Make Docker image runnable as non-root user [2112](https://github.com/huginn/huginn/pull/2112) |
+| Oct 16, 2017   | Fix running specs with guard and spring [2145](https://github.com/huginn/huginn/pull/2145) |
+| Oct 11, 2017   | Do not treat already retweeted/favorited error as failure [2140](https://github.com/huginn/huginn/pull/2140) |
+| Oct 03, 2017   | Make TelegramAgent FormConfigurable, DryRunable and add logging [2138](https://github.com/huginn/huginn/pull/2138) |
+| Sep 21, 2017   | Fix Run Event Propagation search action [2124](https://github.com/huginn/huginn/pull/2124) |
+| Sep 21, 2017   | Update to nokogiri 1.8.1 [2132](https://github.com/huginn/huginn/pull/2132) |
+| Sep 20, 2017   | Update rubies: 2017-09 [2129](https://github.com/huginn/huginn/pull/2129) |
+| Sep 19, 2017   | Fix default scenario links [2123](https://github.com/huginn/huginn/pull/2123) |
+| Sep 19, 2017   | Upgrade jquery.jsoneditor to handle null values [2127](https://github.com/huginn/huginn/pull/2127) |
+| Sep 16, 2017   | Update Ruby version in instructions [2015](https://github.com/huginn/huginn/pull/2015) |
+| Sep 16, 2017   | Handle lazy loading of Agents in gems during Agent.receive! [2125](https://github.com/huginn/huginn/pull/2125) |
+| Sep 16, 2017   | Fix dry-run modal when clicking on icon in 'Dry Run' button [2126](https://github.com/huginn/huginn/pull/2126) |
+| Sep 16, 2017   | OpenShift v3 quickstart [2108](https://github.com/huginn/huginn/pull/2108) |
 | Sep 15, 2017   | Tweets view of `TwitterStreamAgent` has been enhanced. [2122](https://github.com/huginn/huginn/pull/2122) |
+| Sep 11, 2017   | Do not instantiate all records when liquidizing a record collection [2119](https://github.com/huginn/huginn/pull/2119) |
 | Sep 09, 2017   | Agent objects in Liquid templating now have new properties `working` and `url`. [2118](https://github.com/huginn/huginn/pull/2118) |
 | Sep 06, 2017   | `DataOutputAgent` includes an icon in a podcast feed. [2114](https://github.com/huginn/huginn/pull/2114) |
+| Sep 06, 2017   | Add documentation for `force_stop` rake task [2115](https://github.com/huginn/huginn/pull/2115) |
+| Sep 05, 2017   | Fix flaky spec [2113](https://github.com/huginn/huginn/pull/2113) |
 | Sep 05, 2017   | `DataOutputAgent` can properly output an RSS feed with items containing  multiple categories, enclosures, etc. [2110](https://github.com/huginn/huginn/pull/2110) |
+| Sep 04, 2017   | Replace references to https://github.com/cantino/huginn with huginn/huginn [2106](https://github.com/huginn/huginn/pull/2106) |
+| Sep 01, 2017   | Switch back to the upstream heroku-buildpack-graphviz [2105](https://github.com/huginn/huginn/pull/2105) |
+| Aug 31, 2017   | Prevent PeakDetectorAgent from storing invalid data in it's memory [2103](https://github.com/huginn/huginn/pull/2103) |
+| Aug 10, 2017   | Upgrade omniauth to prevent Hashie warning [2084](https://github.com/huginn/huginn/pull/2084) |
+| Aug 10, 2017   | Load .env.test instead of .env.development when running rspec [2083](https://github.com/huginn/huginn/pull/2083) |
+| Aug 08, 2017   | Add logging output for pushover agent [2081](https://github.com/huginn/huginn/pull/2081) |
 | Aug 07, 2017   | `ImapFolderAgent` can now include a `Message-ID` and optionally a raw mail blob in each created event. [2076](https://github.com/huginn/huginn/pull/2076) |
+| Aug 06, 2017   | Increase upper user name limit to 190 [2078](https://github.com/huginn/huginn/pull/2078) |
 | Aug 01, 2017   | `GrowlAgent` supports new API parameters `sticky`, `priority` and `callback_url`. [2074](https://github.com/huginn/huginn/pull/2074) |
 | Jul 31, 2017   | `PostAgent` now has a `merge` mode that can be enabled via the `output_mode` new option. [2069](https://github.com/huginn/huginn/pull/2069) |
+| Jul 27, 2017   | Add validations for `mode` values in EventFormattingAgent [2070](https://github.com/huginn/huginn/pull/2070) |
+| Jul 25, 2017   | Improve documentation of Website Agent [2066](https://github.com/huginn/huginn/pull/2066) |
+| Jul 21, 2017   | Upgrade mysql2 gem to 0.4.8 [2065](https://github.com/huginn/huginn/pull/2065) |
 | Jul 20, 2017   | Agent editor has a selectable `contollers` field. [2063](https://github.com/huginn/huginn/pull/2063) |
 | Jul 20, 2017   | Receivers are now inherited when cloning an agent. [2063](https://github.com/huginn/huginn/pull/2063)|
+| Jul 19, 2017   | Replace references of `cantino/huginn` with `huginn/huginn` [2062](https://github.com/huginn/huginn/pull/2062) |
 | Jul 18, 2017   | `DigestAgent` gets a new option `retained_events`. [2041](https://github.com/huginn/huginn/pull/2041) |
 | Jul 10, 2017   | `CommanderAgent` can now refer to `target` to determine what to do for each target agent. [2053](https://github.com/huginn/huginn/pull/2053)   |
 | Jul 10, 2017   | Update Google API Client. May break backwards compatibility for GoogleCalendarPublishAgent. [2047](https://github.com/huginn/huginn/pull/2047)   |
+| Jul 09, 2017   | Add runit-information for Debian Stretch [2048](https://github.com/huginn/huginn/pull/2048) |
+| Jul 01, 2017   | Improve Capistrano configuration [2045](https://github.com/huginn/huginn/pull/2045) |
 | Jun 06, 2017   | Addressed the problem with MySQL resetting auto_increment after events get emptied and the server reboots. [1974](https://github.com/huginn/huginn/pull/1974), [2014](https://github.com/huginn/huginn/pull/2014) |
+| Jun 01, 2017   | Upgrade letter_opener_web version to 1.3.1 to support Rails 5. [2026](https://github.com/huginn/huginn/pull/2026) |
+| May 31, 2017   | Downgrade ruby in Gemfile.lock to Heroku supported version [2024](https://github.com/huginn/huginn/pull/2024) |
+| May 31, 2017   | Build and publish huginn/huginn-test [2016](https://github.com/huginn/huginn/pull/2016) |
 | May 30, 2017   | Support Ruby 2.4. [1876](https://github.com/huginn/huginn/pull/1876) |
 | May 25, 2017   | PeakDetectorAgent now has a configurable `search_url`. [2013](https://github.com/huginn/huginn/pull/2013) |
+| May 24, 2017   | Add test docker image [2002](https://github.com/huginn/huginn/pull/2002) |
+| May 19, 2017   | Improve docker environment variable documentation [2007](https://github.com/huginn/huginn/pull/2007) |
 | May 19, 2017   | Upgrade Rails to 5.1. [1912](https://github.com/huginn/huginn/pull/1912) |
 | May 18, 2017   | `ShellCommandAgent` gets a new option `unbundle`. [1990](https://github.com/huginn/huginn/pull/1990) |
+| May 17, 2017   | Change docker image namespace [2004](https://github.com/huginn/huginn/pull/2004) |
+| May 11, 2017   | Switch graphviz heroku buildpack to a fork to support new heroku stack [1998](https://github.com/huginn/huginn/pull/1998) |
+| May 11, 2017   | Update Nokogiri to 1.7.2 [2000](https://github.com/huginn/huginn/pull/2000) |
 | May 11, 2017   | `PeakDetectorAgent` gets a new option `min_events`. [1924](https://github.com/huginn/huginn/pull/1924) |
 | May 11, 2017   | Switch back to the much improved `jsonpath` gem after `jsonpathv2` gets merged to mainline. [1996](https://github.com/huginn/huginn/pull/1996), [1997](https://github.com/huginn/huginn/pull/1997), [2017](https://github.com/huginn/huginn/pull/2017) |
+| May 01, 2017   | Revert "Protect the latest event from automatic deletion when using MySQL" [1993](https://github.com/huginn/huginn/pull/1993) |
 | Apr 27, 2017   | Add custom response header support to DataOutputAgent, WebhookAgent and LiquidOutputAgent. [1977](https://github.com/huginn/huginn/pull/1977) |
 | Apr 27, 2017   | Upgrade Liquid to 4.0. [1982](https://github.com/huginn/huginn/pull/1982) |
 | Apr 26, 2017   | Add `GoogleTranslationAgent`. [1978](https://github.com/huginn/huginn/pull/1978) |
+| Apr 23, 2017   | Rss agent dynamic cleanup [1733](https://github.com/huginn/huginn/pull/1733) |
+| Apr 21, 2017   | Twilio receiver fix [1980](https://github.com/huginn/huginn/pull/1980) |
+| Apr 19, 2017   | Add another method to ping yourself from huginn [1970](https://github.com/huginn/huginn/pull/1970) |
 | Apr 19, 2017   | `DataOutputAgent` now serves RSS output as `application/rss+xml` by default. (existing agents are automatically configured to use `text/xml`) [1973](https://github.com/huginn/huginn/pull/1973) |
+| Apr 12, 2017   | Set created_at of dry-runned event to the current time [1965](https://github.com/huginn/huginn/pull/1965) |
+| Apr 10, 2017   | Cleanup openshift configuration since it is not supported [1954](https://github.com/huginn/huginn/pull/1954) |
 | Apr 08, 2017   | Add `TumblrLikesAgent`. [1923](https://github.com/huginn/huginn/pull/1923) |
+| Apr 05, 2017   | Fix #1799 by linking to Liquid docs [1953](https://github.com/huginn/huginn/pull/1953) |
 | Mar 31, 2017   | `ChangeDetectorAgent` can now refer to `last_property`. [1950](https://github.com/huginn/huginn/pull/1950) |
+| Mar 26, 2017   | Update Nokogiri to 1.7.1 [1947](https://github.com/huginn/huginn/pull/1947) |
+| Mar 21, 2017   | action_mailer initializer usability fixes [1942](https://github.com/huginn/huginn/pull/1942) |
+| Mar 18, 2017   | Do not allow to "become" a deactivated user [1938](https://github.com/huginn/huginn/pull/1938) |
+| Mar 17, 2017   | docker: generate Rails secret at first run if not configured [1931](https://github.com/huginn/huginn/pull/1931) |
+| Mar 17, 2017   | Fix multi-process image when not specifying  POSTGRES_PORT_5432_TCP_ADDR [1922](https://github.com/huginn/huginn/pull/1922) |
+| Mar 16, 2017   | Fix View diagram : Too many agent to display (#1664) [1935](https://github.com/huginn/huginn/pull/1935) |
+| Mar 12, 2017   | docker: DEBIAN_FRONTEND was missing "export" [1932](https://github.com/huginn/huginn/pull/1932) |
+| Mar 12, 2017   | Update README.md with info on huginn_agent gem [1929](https://github.com/huginn/huginn/pull/1929) |
+| Mar 07, 2017   | Remove some empty documentation files [1926](https://github.com/huginn/huginn/pull/1926) |
+| Mar 07, 2017   | Enable include_sort_info in RssAgent [1925](https://github.com/huginn/huginn/pull/1925) |
+| Feb 27, 2017   | Make feature specs more robust [1917](https://github.com/huginn/huginn/pull/1917) |
+| Feb 27, 2017   | Allow path to accept stored credentials in shell_command_agent.rb [1911](https://github.com/huginn/huginn/pull/1911) |
+| Feb 12, 2017   | Check for agent class file to determine if it's valid [1907](https://github.com/huginn/huginn/pull/1907) |
+| Feb 05, 2017   | Explain time zone labels [1902](https://github.com/huginn/huginn/pull/1902) |
+| Feb 03, 2017   | Fix devise confirmation form, unify unlock form [1897](https://github.com/huginn/huginn/pull/1897) |
+| Feb 01, 2017   | Order TwitterStreamAgents in setup_workers [1890](https://github.com/huginn/huginn/pull/1890) |
 | Feb 01, 2017   | `GoogleFlightsAgent` supports choice of carrier and alliance. [1878](https://github.com/huginn/huginn/pull/1878) |
+| Jan 31, 2017   | Rename Delayed::Job failed scope to prevent warning [1889](https://github.com/huginn/huginn/pull/1889) |
+| Jan 31, 2017   | Add titles to all pages [1884](https://github.com/huginn/huginn/pull/1884) |
+| Jan 30, 2017   | Change "Abort" button to say "Cancel" [1885](https://github.com/huginn/huginn/pull/1885) |
 | Jan 29, 2017   | `WebhookAgent` can redirect to any URL after successful submission. [1923](https://github.com/huginn/huginn/pull/1923) |
+| Jan 28, 2017   | Let default table inherit the boostrap #table css style [1883](https://github.com/huginn/huginn/pull/1883) |
+| Jan 28, 2017   | Allow Redirect Requests [1881](https://github.com/huginn/huginn/pull/1881) |
+| Jan 20, 2017   | Fix scenario import when merges are required [1877](https://github.com/huginn/huginn/pull/1877) |
+| Jan 08, 2017   | Fix #1863 validation issue with schedule names [1864](https://github.com/huginn/huginn/pull/1864) |
+| Jan 08, 2017   | Fix #1853 by hardcoding protocol on image. [1854](https://github.com/huginn/huginn/pull/1854) |
+| Jan 08, 2017   | Credential create should use ace too [1865](https://github.com/huginn/huginn/pull/1865) |
+| Jan 08, 2017   | Cleanup [1866](https://github.com/huginn/huginn/pull/1866) |
+| Jan 07, 2017   | Default selected value for scenario.icon select [1861](https://github.com/huginn/huginn/pull/1861) |
 | Jan 06, 2017   | Agent's id of each incoming event is accessible from Liquid and JavaScriptAgent. [1860](https://github.com/huginn/huginn/pull/1860) |
 | Jan 06, 2017   | "Every X" schedules now run on fixed times. [1844](https://github.com/huginn/huginn/pull/1844) |
 | Jan 03, 2017   | Twitter agents support "extended" tweets that are longer than 140 characters. [1847](https://github.com/huginn/huginn/pull/1847) |
+| Jan 03, 2017   | Make migrations compatible with SQLite [1842](https://github.com/huginn/huginn/pull/1842) |
 | Jan 01, 2017   | A new `include_sort_info` Agent option is added to help sort out an Nth event of a series of events created in a run. [1772](https://github.com/huginn/huginn/pull/1772) |
+| Dec 31, 2016   | Fix HttpStatusAgent [1776](https://github.com/huginn/huginn/pull/1776) |
+| Dec 29, 2016   | Upgrade rails to 5.0.1 [1841](https://github.com/huginn/huginn/pull/1841) |
+| Dec 27, 2016   | Don't use the insecure git:// protocol when fetching git gems [1763](https://github.com/huginn/huginn/pull/1763) |
 | Nov 30, 2016   | `RssAgent` includes podcast tag values in events created from a podcast feed. [1782](https://github.com/huginn/huginn/pull/1782) |
 | Nov 28, 2016   | Remove `BeeperAgent` after Beeper.io shuts down. [1808](https://github.com/huginn/huginn/pull/1808) |
 | Nov 27, 2016   | `WebsiteAgent` can interpolate via the `template` option after extraction. [1743](https://github.com/huginn/huginn/pull/1743), [1816](https://github.com/huginn/huginn/pull/1816) |
+| Nov 27, 2016   | Disable automatic URL normalization and absolutization on `url` [1771](https://github.com/huginn/huginn/pull/1771) |
+| Nov 26, 2016   | Add class of service chooser for Google Flights Agent [1778](https://github.com/huginn/huginn/pull/1778) |
+| Nov 23, 2016   | Fix a double-decoding problem in RssAgent [1813](https://github.com/huginn/huginn/pull/1813) |
+| Nov 22, 2016   | Cache Agent type select options in Agent#new [1804](https://github.com/huginn/huginn/pull/1804) |
+| Nov 20, 2016   | Increase default database pool size to 20 [1805](https://github.com/huginn/huginn/pull/1805) |
 | Nov 20, 2016   | `WebsiteAgent` provides a new extractor option `repeat`. [1769](https://github.com/huginn/huginn/pull/1769) |
+| Nov 19, 2016   | Fixed the online documentation for the Weather Agent class. [1803](https://github.com/huginn/huginn/pull/1803) |
+| Nov 14, 2016   | Fix typos in docker documentation [1792](https://github.com/huginn/huginn/pull/1792) |
+| Nov 13, 2016   | Nitrous.io is shutting down [1789](https://github.com/huginn/huginn/pull/1789) |
+| Nov 13, 2016   | Prevent submit from disabling on invalid json [1790](https://github.com/huginn/huginn/pull/1790) |
+| Nov 13, 2016   | Remove additional nitrous files [1791](https://github.com/huginn/huginn/pull/1791) |
+| Nov 03, 2016   | Revert the special treatment for CDATA introduced in #1071 [1770](https://github.com/huginn/huginn/pull/1770) |
+| Nov 02, 2016   | Fix `url` handling of WebsiteAgent  [1766](https://github.com/huginn/huginn/pull/1766) |
+| Nov 01, 2016   | Fix Stubhub test failures [1764](https://github.com/huginn/huginn/pull/1764) |
+| Oct 31, 2016   | Convert a bunch of HTTP links to HTTPS [1757](https://github.com/huginn/huginn/pull/1757) |
 | Oct 27, 2016   | `WebsiteAgent` now has improved encoding detection for HTML/XML documents. [1751](https://github.com/huginn/huginn/pull/1751) |
+| Oct 27, 2016   | Ignore empty author and link entries in RssAgent [1754](https://github.com/huginn/huginn/pull/1754) |
+| Oct 23, 2016   | Use the XPath expression `string(.)` instead of `.//text()` [1744](https://github.com/huginn/huginn/pull/1744) |
 | Oct 17, 2016   | Normalize URL in `to_uri` and `uri_expand` liquid filters.                                                   |
+| Oct 15, 2016   | Fix delayed_job_active_record overriding defaults [1736](https://github.com/huginn/huginn/pull/1736) |
+| Oct 14, 2016   | Add as_object Liquid filter [1716](https://github.com/huginn/huginn/pull/1716) |
+| Oct 14, 2016   | Retire ar_mysql_column_charset [1729](https://github.com/huginn/huginn/pull/1729) |
+| Oct 11, 2016   | Agent form: ace-editor highlighting and theme [1727](https://github.com/huginn/huginn/pull/1727) |
+| Oct 11, 2016   | Manual event agent validate JSON field before form submit [1728](https://github.com/huginn/huginn/pull/1728) |
+| Oct 09, 2016   | Update forecast_io gem and language [1722](https://github.com/huginn/huginn/pull/1722) |
+| Oct 09, 2016   | Update documentation [1725](https://github.com/huginn/huginn/pull/1725) |
 | Oct 06, 2016   | `RssAgent` is reimplemented migrating its underlying feed parser from FeedNormalizer to Feedjira. [1564](https://github.com/huginn/huginn/pull/1564)     |
 | Oct 05, 2016   | Migrate to Rails 5. [1688](https://github.com/huginn/huginn/pull/1688)                                      |
 | Oct 05, 2016   | Improve URL normalization in `WebsiteAgent`. [1719](https://github.com/huginn/huginn/pull/1719)             |
 | Oct 05, 2016   | `PushoverAgent` now treats parameter options as templates rather than default values. [1720](https://github.com/huginn/huginn/pull/1720) |
+| Sep 30, 2016   | Fix escape characters of events when dry running [1715](https://github.com/huginn/huginn/pull/1715) |
+| Sep 28, 2016   | Allow style tags in sanitized HTML [1712](https://github.com/huginn/huginn/pull/1712) |
+| Sep 23, 2016   | Clarify path for a simple body_text event. [1705](https://github.com/huginn/huginn/pull/1705) |
+| Sep 21, 2016   | Rescue from AR:SubclassNotFound and allow to delete agents [1695](https://github.com/huginn/huginn/pull/1695) |
+| Sep 21, 2016   | Replace jquery.serializeObject with new implementation [1698](https://github.com/huginn/huginn/pull/1698) |
+| Sep 21, 2016   | Ignore the fixture API key in WeatherAgent [1694](https://github.com/huginn/huginn/pull/1694) |
 | Sep 19, 2016   | Add multipart file upload to `PostAgent`. [1690](https://github.com/huginn/huginn/pull/1690)                |
+| Sep 16, 2016   | docker-compose version 2 [1681](https://github.com/huginn/huginn/pull/1681) |
+| Sep 15, 2016   | `service_id` is a valid part of the agent_params [1686](https://github.com/huginn/huginn/pull/1686) |
+| Sep 15, 2016   | interpolated response [1682](https://github.com/huginn/huginn/pull/1682) |
+| Sep 14, 2016   | bundle update net-ssh to 3.0.x (to avoid deprecation warn on Ruby 2.3) [1683](https://github.com/huginn/huginn/pull/1683) |
+| Sep 14, 2016   | storage_engine removed since 5.7.5 replaced by default_storage_engine… [1684](https://github.com/huginn/huginn/pull/1684) |
+| Sep 12, 2016   | Use strong_parameters and drop protected_attributes [1679](https://github.com/huginn/huginn/pull/1679) |
+| Sep 11, 2016   | Updated JsonPathV2 version to latest. [1674](https://github.com/huginn/huginn/pull/1674) |
+| Sep 10, 2016   | Remove unused contact model [1680](https://github.com/huginn/huginn/pull/1680) |
+| Sep 09, 2016   | Option to set response code for webhook agent [1676](https://github.com/huginn/huginn/pull/1676) |
+| Sep 09, 2016   | can_enqueue? propagation detection that does not depend on Rails [1672](https://github.com/huginn/huginn/pull/1672) |
+| Sep 08, 2016   | Add cache_response option to completable form fields [1673](https://github.com/huginn/huginn/pull/1673) |
 | Sep 08, 2016   | Allow `TwitterUserAgent` to retry failed actions. [1645](https://github.com/huginn/huginn/pull/1645)        |
+| Sep 06, 2016   | Update libv8 to fix build issues on OS X [1671](https://github.com/huginn/huginn/pull/1671) |
+| Sep 06, 2016   | minor tweaks [1669](https://github.com/huginn/huginn/pull/1669) |
+| Sep 04, 2016   | Extract service option prviders for non-standard Omniauth payloads [1655](https://github.com/huginn/huginn/pull/1655) |
+| Sep 03, 2016   | Fix SMTP config regression when not using authentication [1663](https://github.com/huginn/huginn/pull/1663) |
+| Sep 02, 2016   | Admin "become" method [1659](https://github.com/huginn/huginn/pull/1659) |
 | Aug 16, 2016   | `EmailDigestAgent` now relies on received events, rather in memory. [1624](https://github.com/huginn/huginn/pull/1624) |
+| Aug 15, 2016   | Update rails to 4.2.7.1 [1630](https://github.com/huginn/huginn/pull/1630) |
+| Aug 15, 2016   | Handle removed events when rendering logs [1634](https://github.com/huginn/huginn/pull/1634) |
+| Aug 09, 2016   | Docker: fix usage of ENV variables that are not in .env.example [1620](https://github.com/huginn/huginn/pull/1620) |
 | Aug 08, 2016   | `DataOutputAgent` now limits events after ordering. [1444](https://github.com/huginn/huginn/pull/1444)      |
+| Aug 05, 2016   | Fix dependency check for Tumblr gem [1615](https://github.com/huginn/huginn/pull/1615) |
 | Aug 05, 2016   | Add `api_key` option to `UserLocationAgent`. [1613](https://github.com/huginn/huginn/pull/1613)             |
+| Jul 27, 2016   | DataOutputAgent cannot create events [1608](https://github.com/huginn/huginn/pull/1608) |
+| Jul 26, 2016   | Monkey patch faraday to fix encoding issue in URLs [1607](https://github.com/huginn/huginn/pull/1607) |
 | Jul 25, 2016   | Add `LiquidOutputAgent`. [1587](https://github.com/huginn/huginn/pull/1587)                                 |
 | Jul 25, 2016   | Allow `PostAgent` headers to interpolate event data. [1606](https://github.com/huginn/huginn/pull/1606)     |
 | Jul 25, 2016   | Remove `smtp.yml` configuration file, the SMTP configuration now needs to be done via environment variables. [1595](https://github.com/huginn/huginn/pull/1595) |

Gemfile

@@ -1,13 +1,11 @@
 source 'https://rubygems.org'
 
-# Ruby 2.2.2 is the minimum requirement
-ruby [Gem::Version.new('2.2.2'), Gem::Version.new(RUBY_VERSION)].max
+ruby '>=2.5.0'
 
 # Ensure github repositories are fetched using HTTPS
 git_source(:github) do |repo_name|
-  repo_name = "#{repo_name}/#{repo_name}" unless repo_name.include?("/")
   "https://github.com/#{repo_name}.git"
-end if Gem::Version.new(Bundler::VERSION) < Gem::Version.new('2')
+end
 
 # Load vendored dotenv gem and .env file
 require File.join(File.dirname(__FILE__), 'lib/gemfile_helper.rb')
@@ -34,11 +32,11 @@ end
 gem 'twilio-ruby', '~> 3.11.5'    # TwilioAgent
 gem 'ruby-growl', '~> 4.1.0'      # GrowlAgent
 gem 'net-ftp-list', '~> 3.2.8'    # FtpsiteAgent
-gem 'wunderground', '~> 1.2.0'    # WeatherAgent
 gem 'forecast_io', '~> 2.0.0'     # WeatherAgent
 gem 'rturk', '~> 2.12.1'          # HumanTaskAgent
-gem 'erector', github: 'dsander/erector', branch: 'fix-fixnum-warning'
+gem 'erector', github: 'dsander/erector', branch: 'rails6'
 gem 'hipchat', '~> 1.2.0'         # HipchatAgent
+gem 'mini_racer', '~> 0.2.4'      # JavaScriptAgent
 gem 'xmpp4r',  '~> 0.5.6'         # JabberAgent
 gem 'mqtt'                        # MQTTAgent
 gem 'slack-notifier', '~> 1.0.0'  # SlackAgent
@@ -59,7 +57,7 @@ gem 'omniauth-twitter', '~> 1.3.0'
 
 # Tumblr Agents
 # until merge of https://github.com/tumblr/tumblr_client/pull/61
-gem 'tumblr_client', github: 'albertsun/tumblr_client', branch: 'master', ref: 'e046fe6e39291c173add0a49081630c7b60a36c7' 
+gem 'tumblr_client', github: 'albertsun/tumblr_client', branch: 'master', ref: 'e046fe6e39291c173add0a49081630c7b60a36c7'
 gem 'omniauth-tumblr', '~> 1.2'
 
 # Dropbox Agents
@@ -93,15 +91,15 @@ end
 gem 'ace-rails-ap', '~> 2.0.1'
 gem 'bootstrap-kaminari-views', '~> 0.0.3'
 gem 'bundler', '>= 1.5.0'
-gem 'coffee-rails', '~> 4.2'
+gem 'coffee-rails', '~> 5'
 gem 'daemons', '~> 1.1.9'
-gem 'delayed_job', '~> 4.1.5'
-gem 'delayed_job_active_record', github: 'dsander/delayed_job_active_record', branch: 'rails52'
-gem 'devise', '~> 4.4.3'
+gem 'delayed_job', '~> 4.1.8'
+gem 'delayed_job_active_record', github: 'dsander/delayed_job_active_record', branch: 'rails6-zeitwerk'
+gem 'devise', '~> 4.7.1'
 gem 'em-http-request', '~> 1.1.2'
 gem 'faraday', '~> 0.9'
 gem 'faraday_middleware', '~> 0.12.2'
-gem 'feedjira', '~> 2.1'
+gem 'feedjira', '~> 3.1'
 gem 'font-awesome-sass', '~> 4.7.0'
 gem 'foreman', '~> 0.63.0'
 gem 'geokit', '~> 1.8.4'
@@ -110,27 +108,27 @@ gem 'httparty', '~> 0.13'
 gem 'httmultiparty', '~> 0.3.16'
 gem 'jquery-rails', '~> 4.2.1'
 gem 'huginn_agent', '~> 0.4.0'
-gem 'json', '~> 1.8.1'
-gem 'jsonpath', '~> 0.8.3'
+gem 'json', '~> 2.3'
+gem 'jsonpath', '~> 1.0.1'
 gem 'kaminari', '~> 1.1.1'
 gem 'kramdown', '~> 1.3.3'
-gem 'liquid', '~> 4.0'
+gem 'liquid', '~> 4.0.3'
 gem 'loofah', '~> 2.0'
-gem 'mini_magick'
+gem 'mini_magick', ">= 4.9.4"
 gem 'multi_xml'
-gem 'nokogiri'
+gem "nokogiri", ">= 1.10.8"
 gem 'omniauth', '~> 1.6.1'
-gem 'rails', '~> 5.2.0'
+gem 'rails', '~> 6.0.3.1'
 gem 'sprockets', '~> 3.7.2'
-gem 'rails-html-sanitizer', '~> 1.0.4'
+gem 'rails-html-sanitizer', '~> 1.2'
 gem 'rufus-scheduler', '~> 3.4.2', require: false
-gem 'sass-rails', '~> 5.0'
+gem 'sass-rails', '>= 6.0'
 gem 'select2-rails', '~> 3.5.4'
 gem 'spectrum-rails'
-gem 'therubyracer', '~> 0.12.3'
-gem 'typhoeus', '~> 0.6.3'
+gem 'execjs', '~> 2.7.0'
+gem 'typhoeus', '~> 1.3.1'
 gem 'uglifier', '~> 2.7.2'
-gem 'bootsnap', '>= 1.1.0', require: false
+gem 'bootsnap', '~> 1.4.4', require: false
 
 group :development do
   gem 'better_errors', '~> 1.1'
@@ -142,7 +140,7 @@ group :development do
   gem 'letter_opener_web', '~> 1.3.1'
   gem 'web-console', '>= 3.3.0'
 
-  gem 'capistrano', '~> 3.4.0'
+  gem 'capistrano', '~> 3.11.0'
   gem 'capistrano-rails', '~> 1.1'
   gem 'capistrano-bundler', '~> 1.1.4'
 
@@ -153,23 +151,22 @@ group :development do
   end
 
   group :test do
-    gem 'coveralls', '~> 0.8.12', require: false
+    gem 'coveralls', '~> 0.8.23', require: false
     gem 'capybara', '~> 2.18'
     gem 'capybara-screenshot'
-    gem 'capybara-select2', require: false
-    gem 'delorean'
+    gem 'capybara-select-2', github: 'Hirurg103/capybara_select2', ref: 'fbf22fb74dec10fa0edcd26da7c5184ba8fa2c76', require: false
     gem 'poltergeist'
     gem 'pry-rails'
     gem 'pry-byebug'
     gem 'rr'
-    gem 'rspec', '~> 3.7'
+    gem 'rspec', '~> 3.8'
+    gem 'rspec-rails'
     gem 'rspec-collection_matchers', '~> 1.1.0'
-    gem 'rspec-rails', '~> 3.7'
     gem 'rspec-html-matchers', '~> 0.8'
     gem 'rails-controller-testing'
     gem 'shoulda-matchers'
     gem 'vcr'
-    gem 'webmock', '~> 2.3'
+    gem 'webmock', '~> 3.5.1'
   end
 end
 
@@ -200,11 +197,11 @@ ENV['DATABASE_ADAPTER'] ||=
   end
 
 if_true(ENV['DATABASE_ADAPTER'].strip == 'postgresql') do
-  gem 'pg', '~> 0.18.3'
+  gem 'pg', '~> 1.1.3'
 end
 
 if_true(ENV['DATABASE_ADAPTER'].strip == 'mysql2') do
-  gem 'mysql2', ">= 0.3.18", "< 0.5"
+  gem 'mysql2' , "~> 0.5.2"
 end
 
 GemfileHelper.parse_each_agent_gem(ENV['ADDITIONAL_GEMS']) do |args|

Gemfile.lock

@@ -1,3 +1,10 @@
+GIT
+  remote: https://github.com/Hirurg103/capybara_select2.git
+  revision: fbf22fb74dec10fa0edcd26da7c5184ba8fa2c76
+  ref: fbf22fb74dec10fa0edcd26da7c5184ba8fa2c76
+  specs:
+    capybara-select-2 (0.3.2)
+
 GIT
   remote: https://github.com/albertsun/tumblr_client.git
   revision: e046fe6e39291c173add0a49081630c7b60a36c7
@@ -43,11 +50,11 @@ GIT
 
 GIT
   remote: https://github.com/dsander/delayed_job_active_record.git
-  revision: 8efc7b10a3e899a75515430b2216556f4e6c3de0
-  branch: rails52
+  revision: ac2dfef97be39ae0fee2de8006158878ed65a719
+  branch: rails6-zeitwerk
   specs:
-    delayed_job_active_record (4.1.3)
-      activerecord (>= 3.0, < 5.3)
+    delayed_job_active_record (4.1.4)
+      activerecord (>= 3.0, < 6.1)
       delayed_job (>= 3.0, < 5)
 
 GIT
@@ -62,8 +69,8 @@ GIT
 
 GIT
   remote: https://github.com/dsander/erector.git
-  revision: 9ee3667e4b26f586ba72ee74406b3bd65ddf5c16
-  branch: fix-fixnum-warning
+  revision: 821c2fa9174b56cc39e203883d83b18b60912a36
+  branch: rails6
   specs:
     erector (0.10.0)
       treetop (>= 1.2.3)
@@ -95,70 +102,86 @@ GEM
   remote: https://rubygems.org/
   specs:
     ace-rails-ap (2.0.1)
-    actioncable (5.2.0)
-      actionpack (= 5.2.0)
+    actioncable (6.0.3.1)
+      actionpack (= 6.0.3.1)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
-    actionmailer (5.2.0)
-      actionpack (= 5.2.0)
-      actionview (= 5.2.0)
-      activejob (= 5.2.0)
+    actionmailbox (6.0.3.1)
+      actionpack (= 6.0.3.1)
+      activejob (= 6.0.3.1)
+      activerecord (= 6.0.3.1)
+      activestorage (= 6.0.3.1)
+      activesupport (= 6.0.3.1)
+      mail (>= 2.7.1)
+    actionmailer (6.0.3.1)
+      actionpack (= 6.0.3.1)
+      actionview (= 6.0.3.1)
+      activejob (= 6.0.3.1)
       mail (~> 2.5, >= 2.5.4)
       rails-dom-testing (~> 2.0)
-    actionpack (5.2.0)
-      actionview (= 5.2.0)
-      activesupport (= 5.2.0)
-      rack (~> 2.0)
+    actionpack (6.0.3.1)
+      actionview (= 6.0.3.1)
+      activesupport (= 6.0.3.1)
+      rack (~> 2.0, >= 2.0.8)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
-      rails-html-sanitizer (~> 1.0, >= 1.0.2)
-    actionview (5.2.0)
-      activesupport (= 5.2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.2.0)
+    actiontext (6.0.3.1)
+      actionpack (= 6.0.3.1)
+      activerecord (= 6.0.3.1)
+      activestorage (= 6.0.3.1)
+      activesupport (= 6.0.3.1)
+      nokogiri (>= 1.8.5)
+    actionview (6.0.3.1)
+      activesupport (= 6.0.3.1)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
-      rails-html-sanitizer (~> 1.0, >= 1.0.3)
-    activejob (5.2.0)
-      activesupport (= 5.2.0)
+      rails-html-sanitizer (~> 1.1, >= 1.2.0)
+    activejob (6.0.3.1)
+      activesupport (= 6.0.3.1)
       globalid (>= 0.3.6)
-    activemodel (5.2.0)
-      activesupport (= 5.2.0)
-    activerecord (5.2.0)
-      activemodel (= 5.2.0)
-      activesupport (= 5.2.0)
-      arel (>= 9.0)
-    activestorage (5.2.0)
-      actionpack (= 5.2.0)
-      activerecord (= 5.2.0)
+    activemodel (6.0.3.1)
+      activesupport (= 6.0.3.1)
+    activerecord (6.0.3.1)
+      activemodel (= 6.0.3.1)
+      activesupport (= 6.0.3.1)
+    activestorage (6.0.3.1)
+      actionpack (= 6.0.3.1)
+      activejob (= 6.0.3.1)
+      activerecord (= 6.0.3.1)
       marcel (~> 0.3.1)
-    activesupport (5.2.0)
+    activesupport (6.0.3.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
+      zeitwerk (~> 2.2, >= 2.2.2)
     addressable (2.5.2)
       public_suffix (>= 2.0.2, < 4.0)
-    arel (9.0.0)
+    airbrussh (1.3.1)
+      sshkit (>= 1.6.1, != 1.7.0)
     aws-sdk-core (2.2.15)
       jmespath (~> 1.0)
-    bcrypt (3.1.11)
+    bcrypt (3.1.13)
     better_errors (1.1.0)
       coderay (>= 1.0.0)
       erubis (>= 2.6.6)
     binding_of_caller (0.8.0)
       debug_inspector (>= 0.0.1)
-    bootsnap (1.3.0)
+    bootsnap (1.4.5)
       msgpack (~> 1.0)
     bootstrap-kaminari-views (0.0.5)
       kaminari (>= 0.13)
       rails (>= 3.1)
     buftok (0.2.0)
-    builder (3.2.3)
+    builder (3.2.4)
     byebug (8.2.5)
-    capistrano (3.4.0)
+    capistrano (3.11.0)
+      airbrussh (>= 1.0.0)
       i18n
       rake (>= 10.0.0)
-      sshkit (~> 1.3)
+      sshkit (>= 1.9.0)
     capistrano-bundler (1.1.4)
       capistrano (~> 3.1)
       sshkit (~> 1.2)
@@ -175,47 +198,40 @@ GEM
     capybara-screenshot (1.0.17)
       capybara (>= 1.0, < 3)
       launchy
-    capybara-select2 (1.0.1)
-      capybara
-      rspec
-    chronic (0.10.2)
     cliver (0.3.2)
     coderay (1.1.2)
-    coffee-rails (4.2.2)
+    coffee-rails (5.0.0)
       coffee-script (>= 2.2.0)
-      railties (>= 4.0.0)
+      railties (>= 5.2.0)
     coffee-script (2.4.1)
       coffee-script-source
       execjs
     coffee-script-source (1.12.2)
-    colorize (0.7.7)
-    concurrent-ruby (1.0.5)
+    concurrent-ruby (1.1.6)
     cookiejar (0.3.2)
-    coveralls (0.8.21)
+    coveralls (0.8.23)
       json (>= 1.8, < 3)
-      simplecov (~> 0.14.1)
+      simplecov (~> 0.16.1)
       term-ansicolor (~> 1.3)
-      thor (~> 0.19.4)
+      thor (>= 0.19.4, < 2.0)
       tins (~> 1.6)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
-    crass (1.0.4)
+    crass (1.0.6)
     daemons (1.1.9)
     debug_inspector (0.0.3)
     declarative (0.0.9)
     declarative-option (0.1.0)
-    delayed_job (4.1.5)
-      activesupport (>= 3.0, < 5.3)
-    delorean (2.1.0)
-      chronic
-    devise (4.4.3)
+    delayed_job (4.1.8)
+      activesupport (>= 3.0, < 6.1)
+    devise (4.7.1)
       bcrypt (~> 3.0)
       orm_adapter (~> 0.1)
-      railties (>= 4.1.0, < 6.0)
+      railties (>= 4.1.0)
       responders
       warden (~> 1.2.3)
     diff-lcs (1.3)
-    docile (1.1.5)
+    docile (1.3.2)
     domain_name (0.5.20170404)
       unf (>= 0.0.5, < 1.0.0)
     em-http-request (1.1.2)
@@ -230,28 +246,26 @@ GEM
       eventmachine (>= 0.12.9)
       http_parser.rb (~> 0.6.0)
     equalizer (0.0.11)
-    erubi (1.7.1)
+    erubi (1.9.0)
     erubis (2.7.0)
     et-orbi (1.0.9)
       tzinfo
-    ethon (0.7.1)
+    ethon (0.12.0)
       ffi (>= 1.3.0)
     eventmachine (1.2.7)
     evernote-thrift (1.25.1)
     evernote_oauth (0.2.3)
       evernote-thrift
       oauth (>= 0.4.1)
-    execjs (2.6.0)
+    execjs (2.7.0)
     faraday (0.12.1)
       multipart-post (>= 1.2, < 3)
     faraday_middleware (0.12.2)
       faraday (>= 0.7.4, < 1.0)
-    feedjira (2.1.2)
-      faraday (>= 0.9)
-      faraday_middleware (>= 0.9)
-      loofah (>= 2.0)
+    feedjira (3.1.0)
+      loofah (>= 2.3.1)
       sax-machine (>= 1.0)
-    ffi (1.9.18)
+    ffi (1.12.2)
     font-awesome-sass (4.7.0)
       sass (>= 3.2)
     forecast_io (2.0.1)
@@ -267,7 +281,7 @@ GEM
     geokit-rails (2.2.0)
       geokit (~> 1.5)
       rails (>= 3.0)
-    globalid (0.4.1)
+    globalid (0.4.2)
       activesupport (>= 4.2.0)
     google-api-client (0.13.0)
       addressable (~> 2.5, >= 2.5.1)
@@ -311,7 +325,7 @@ GEM
       guard (~> 2.1)
       guard-compat (~> 1.1)
       rspec (>= 2.99.0, < 4.0)
-    hashdiff (0.3.2)
+    hashdiff (0.3.8)
     hashie (3.5.6)
     haversine (0.3.0)
     hipchat (1.2.0)
@@ -337,16 +351,17 @@ GEM
     hypdf (1.0.10)
       httmultiparty (~> 0.3)
       httparty (~> 0.13)
-    i18n (1.0.1)
+    i18n (1.8.2)
       concurrent-ruby (~> 1.0)
     jmespath (1.1.3)
     jquery-rails (4.2.2)
       rails-dom-testing (>= 1, < 3)
       railties (>= 4.2.0)
       thor (>= 0.14, < 2.0)
-    json (1.8.6)
-    jsonpath (0.8.3)
+    json (2.3.0)
+    jsonpath (1.0.1)
       multi_json
+      to_regexp (~> 0.2.1)
     jwt (1.5.6)
     kaminari (1.1.1)
       activesupport (>= 4.1.0)
@@ -370,8 +385,8 @@ GEM
       actionmailer (>= 3.2)
       letter_opener (~> 1.0)
       railties (>= 3.2)
-    libv8 (3.16.14.19)
-    liquid (4.0.0)
+    libv8 (7.3.492.27.1)
+    liquid (4.0.3)
     listen (3.0.8)
       rb-fsevent (~> 0.9, >= 0.9.4)
       rb-inotify (~> 0.9, >= 0.9.7)
@@ -379,44 +394,46 @@ GEM
     logging (2.2.2)
       little-plugger (~> 1.1)
       multi_json (~> 1.10)
-    loofah (2.2.2)
+    loofah (2.5.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     lumberjack (1.0.12)
     macaddr (1.7.1)
       systemu (~> 2.6.2)
-    mail (2.7.0)
+    mail (2.7.1)
       mini_mime (>= 0.1.1)
-    marcel (0.3.2)
+    marcel (0.3.3)
       mimemagic (~> 0.3.2)
     memoist (0.16.0)
     memoizable (0.4.2)
       thread_safe (~> 0.3, >= 0.3.1)
-    method_source (0.9.0)
+    method_source (0.9.2)
     mime-types (3.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2016.0521)
-    mimemagic (0.3.2)
-    mini_magick (4.2.3)
-    mini_mime (1.0.0)
-    mini_portile2 (2.3.0)
-    minitest (5.11.3)
+    mimemagic (0.3.5)
+    mini_magick (4.9.5)
+    mini_mime (1.0.2)
+    mini_portile2 (2.4.0)
+    mini_racer (0.2.9)
+      libv8 (>= 6.9.411)
+    minitest (5.14.1)
     mqtt (0.3.1)
-    msgpack (1.2.4)
-    multi_json (1.12.1)
+    msgpack (1.3.1)
+    multi_json (1.13.1)
     multi_xml (0.6.0)
     multipart-post (2.0.0)
-    mysql2 (0.4.8)
+    mysql2 (0.5.2)
     naught (1.1.0)
     nenv (0.3.0)
     net-ftp-list (3.2.8)
     net-scp (1.2.1)
       net-ssh (>= 2.6.5)
-    net-ssh (3.0.2)
+    net-ssh (5.0.2)
     netrc (0.11.0)
-    nio4r (2.3.0)
-    nokogiri (1.8.2)
-      mini_portile2 (~> 2.3.0)
+    nio4r (2.5.2)
+    nokogiri (1.10.9)
+      mini_portile2 (~> 2.4.0)
     notiffany (0.1.1)
       nenv (~> 0.1)
       shellany (~> 0.0)
@@ -454,7 +471,7 @@ GEM
       omniauth-oauth2 (~> 1.1)
     orm_adapter (0.5.0)
     os (0.9.6)
-    pg (0.18.3)
+    pg (1.1.3)
     poltergeist (1.8.1)
       capybara (~> 2.1)
       cliver (~> 0.3.1)
@@ -470,86 +487,87 @@ GEM
     pry-rails (0.3.4)
       pry (>= 0.9.10)
     public_suffix (3.0.2)
-    rack (2.0.5)
+    rack (2.2.2)
     rack-livereload (0.3.16)
       rack
-    rack-test (1.0.0)
+    rack-test (1.1.0)
       rack (>= 1.0, < 3)
-    rails (5.2.0)
-      actioncable (= 5.2.0)
-      actionmailer (= 5.2.0)
-      actionpack (= 5.2.0)
-      actionview (= 5.2.0)
-      activejob (= 5.2.0)
-      activemodel (= 5.2.0)
-      activerecord (= 5.2.0)
-      activestorage (= 5.2.0)
-      activesupport (= 5.2.0)
+    rails (6.0.3.1)
+      actioncable (= 6.0.3.1)
+      actionmailbox (= 6.0.3.1)
+      actionmailer (= 6.0.3.1)
+      actionpack (= 6.0.3.1)
+      actiontext (= 6.0.3.1)
+      actionview (= 6.0.3.1)
+      activejob (= 6.0.3.1)
+      activemodel (= 6.0.3.1)
+      activerecord (= 6.0.3.1)
+      activestorage (= 6.0.3.1)
+      activesupport (= 6.0.3.1)
       bundler (>= 1.3.0)
-      railties (= 5.2.0)
+      railties (= 6.0.3.1)
       sprockets-rails (>= 2.0.0)
-    rails-controller-testing (1.0.1)
-      actionpack (~> 5.x)
-      actionview (~> 5.x)
-      activesupport (~> 5.x)
+    rails-controller-testing (1.0.4)
+      actionpack (>= 5.0.1.x)
+      actionview (>= 5.0.1.x)
+      activesupport (>= 5.0.1.x)
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
       nokogiri (>= 1.6)
-    rails-html-sanitizer (1.0.4)
-      loofah (~> 2.2, >= 2.2.2)
-    railties (5.2.0)
-      actionpack (= 5.2.0)
-      activesupport (= 5.2.0)
+    rails-html-sanitizer (1.3.0)
+      loofah (~> 2.3)
+    railties (6.0.3.1)
+      actionpack (= 6.0.3.1)
+      activesupport (= 6.0.3.1)
       method_source
       rake (>= 0.8.7)
-      thor (>= 0.18.1, < 2.0)
+      thor (>= 0.20.3, < 2.0)
     raindrops (0.17.0)
-    rake (12.3.1)
+    rake (13.0.1)
     rb-fsevent (0.10.2)
     rb-inotify (0.9.10)
       ffi (>= 0.5.0, < 2)
     rb-kqueue (0.2.4)
       ffi (>= 0.5.0)
-    ref (2.0.0)
     representable (3.0.4)
       declarative (< 0.1.0)
       declarative-option (< 0.2.0)
       uber (< 0.2.0)
-    responders (2.4.0)
-      actionpack (>= 4.2.0, < 5.3)
-      railties (>= 4.2.0, < 5.3)
+    responders (3.0.0)
+      actionpack (>= 5.0)
+      railties (>= 5.0)
     rest-client (2.0.2)
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
     retriable (3.0.2)
     rr (1.1.2)
-    rspec (3.7.0)
-      rspec-core (~> 3.7.0)
-      rspec-expectations (~> 3.7.0)
-      rspec-mocks (~> 3.7.0)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
     rspec-collection_matchers (1.1.2)
       rspec-expectations (>= 2.99.0.beta1)
-    rspec-core (3.7.1)
-      rspec-support (~> 3.7.0)
-    rspec-expectations (3.7.0)
+    rspec-core (3.8.2)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.5)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
+      rspec-support (~> 3.8.0)
     rspec-html-matchers (0.9.1)
       nokogiri (~> 1)
       rspec (>= 3.0.0.a, < 4)
-    rspec-mocks (3.7.0)
+    rspec-mocks (3.8.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-rails (3.7.2)
+      rspec-support (~> 3.8.0)
+    rspec-rails (3.8.2)
       actionpack (>= 3.0)
       activesupport (>= 3.0)
       railties (>= 3.0)
-      rspec-core (~> 3.7.0)
-      rspec-expectations (~> 3.7.0)
-      rspec-mocks (~> 3.7.0)
-      rspec-support (~> 3.7.0)
-    rspec-support (3.7.1)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.3)
     rturk (2.12.1)
       erector
       nokogiri
@@ -559,30 +577,38 @@ GEM
     rufus-scheduler (3.4.2)
       et-orbi (~> 1.0)
     safe_yaml (1.0.4)
-    sass (3.4.23)
-    sass-rails (5.0.6)
-      railties (>= 4.0.0, < 6)
-      sass (~> 3.1)
-      sprockets (>= 2.8, < 4.0)
-      sprockets-rails (>= 2.0, < 4.0)
-      tilt (>= 1.1, < 3)
+    sass (3.7.4)
+      sass-listen (~> 4.0.0)
+    sass-listen (4.0.0)
+      rb-fsevent (~> 0.9, >= 0.9.4)
+      rb-inotify (~> 0.9, >= 0.9.7)
+    sass-rails (6.0.0)
+      sassc-rails (~> 2.1, >= 2.1.1)
+    sassc (2.3.0)
+      ffi (~> 1.9)
+    sassc-rails (2.1.2)
+      railties (>= 4.0.0)
+      sassc (>= 2.0)
+      sprockets (> 3.0)
+      sprockets-rails
+      tilt
     sax-machine (1.3.2)
     select2-rails (3.5.9.3)
       thor (~> 0.14)
     shellany (0.0.1)
-    shoulda-matchers (3.0.0)
-      activesupport (>= 4.0.0)
+    shoulda-matchers (4.0.1)
+      activesupport (>= 4.2.0)
     signet (0.7.3)
       addressable (~> 2.3)
       faraday (~> 0.9)
       jwt (~> 1.5)
       multi_json (~> 1.10)
     simple_oauth (0.3.1)
-    simplecov (0.14.1)
-      docile (~> 1.1.0)
+    simplecov (0.16.1)
+      docile (~> 1.1)
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
-    simplecov-html (0.10.1)
+    simplecov-html (0.10.2)
     slack-notifier (1.0.0)
     spectrum-rails (1.3.4)
       railties (>= 3.1)
@@ -600,29 +626,26 @@ GEM
       actionpack (>= 4.0)
       activesupport (>= 4.0)
       sprockets (>= 3.0.0)
-    sshkit (1.7.1)
-      colorize (>= 0.7.0)
+    sshkit (1.18.0)
       net-scp (>= 1.1.2)
       net-ssh (>= 2.8.0)
     systemu (2.6.4)
-    term-ansicolor (1.6.0)
+    term-ansicolor (1.7.1)
       tins (~> 1.0)
-    therubyracer (0.12.3)
-      libv8 (~> 3.16.14.15)
-      ref
-    thor (0.19.4)
+    thor (0.20.3)
     thread_safe (0.3.6)
-    tilt (2.0.7)
-    tins (1.15.0)
-    treetop (1.6.9)
+    tilt (2.0.10)
+    tins (1.21.1)
+    to_regexp (0.2.1)
+    treetop (1.6.10)
       polyglot (~> 0.3)
     twilio-ruby (3.11.6)
       builder (>= 2.1.2)
       jwt (>= 0.1.2)
       multi_json (>= 1.3.0)
-    typhoeus (0.6.9)
-      ethon (>= 0.7.1)
-    tzinfo (1.2.5)
+    typhoeus (1.3.1)
+      ethon (>= 0.9.0)
+    tzinfo (1.2.7)
       thread_safe (~> 0.1)
     uber (0.1.0)
     uglifier (2.7.2)
@@ -637,27 +660,24 @@ GEM
     uuid (2.3.7)
       macaddr (~> 1.0)
     vcr (3.0.3)
-    warden (1.2.7)
-      rack (>= 1.0)
+    warden (1.2.8)
+      rack (>= 2.0.6)
     web-console (3.3.1)
       actionview (>= 5.0)
       activemodel (>= 5.0)
       debug_inspector
       railties (>= 5.0)
-    webmock (2.3.2)
+    webmock (3.5.1)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff
-    websocket-driver (0.7.0)
+    websocket-driver (0.7.2)
       websocket-extensions (>= 0.1.0)
-    websocket-extensions (0.1.3)
-    wunderground (1.2.0)
-      addressable
-      httparty (> 0.6.0)
-      json (> 1.4.0)
+    websocket-extensions (0.1.4)
     xmpp4r (0.5.6)
     xpath (3.0.0)
       nokogiri (~> 1.8)
+    zeitwerk (2.3.0)
 
 PLATFORMS
   ruby
@@ -667,31 +687,31 @@ DEPENDENCIES
   aws-sdk-core (~> 2.2.15)
   better_errors (~> 1.1)
   binding_of_caller (~> 0.8.0)
-  bootsnap (>= 1.1.0)
+  bootsnap (~> 1.4.4)
   bootstrap-kaminari-views (~> 0.0.3)
   bundler (>= 1.5.0)
-  capistrano (~> 3.4.0)
+  capistrano (~> 3.11.0)
   capistrano-bundler (~> 1.1.4)
   capistrano-rails (~> 1.1)
   capybara (~> 2.18)
   capybara-screenshot
-  capybara-select2
-  coffee-rails (~> 4.2)
-  coveralls (~> 0.8.12)
+  capybara-select-2!
+  coffee-rails (~> 5)
+  coveralls (~> 0.8.23)
   daemons (~> 1.1.9)
-  delayed_job (~> 4.1.5)
+  delayed_job (~> 4.1.8)
   delayed_job_active_record!
-  delorean
-  devise (~> 4.4.3)
+  devise (~> 4.7.1)
   dotenv!
   dotenv-rails!
   dropbox-api!
   em-http-request (~> 1.1.2)
   erector!
   evernote_oauth
+  execjs (~> 2.7.0)
   faraday (~> 0.9)
   faraday_middleware (~> 0.12.2)
-  feedjira (~> 2.1)
+  feedjira (~> 3.1)
   ffi (>= 1.9.4)
   font-awesome-sass (~> 4.7.0)
   forecast_io (~> 2.0.0)
@@ -710,20 +730,21 @@ DEPENDENCIES
   huginn_agent (~> 0.4.0)
   hypdf (~> 1.0.10)
   jquery-rails (~> 4.2.1)
-  json (~> 1.8.1)
-  jsonpath (~> 0.8.3)
+  json (~> 2.3)
+  jsonpath (~> 1.0.1)
   kaminari (~> 1.1.1)
   kramdown (~> 1.3.3)
   letter_opener_web (~> 1.3.1)
-  liquid (~> 4.0)
+  liquid (~> 4.0.3)
   listen (~> 3.0.5)
   loofah (~> 2.0)
-  mini_magick
+  mini_magick (>= 4.9.4)
+  mini_racer (~> 0.2.4)
   mqtt
   multi_xml
-  mysql2 (>= 0.3.18, < 0.5)
+  mysql2 (~> 0.5.2)
   net-ftp-list (~> 3.2.8)
-  nokogiri
+  nokogiri (>= 1.10.8)
   omniauth (~> 1.6.1)
   omniauth-37signals
   omniauth-dropbox-oauth2!
@@ -731,24 +752,24 @@ DEPENDENCIES
   omniauth-tumblr (~> 1.2)
   omniauth-twitter (~> 1.3.0)
   omniauth-wunderlist
-  pg (~> 0.18.3)
+  pg (~> 1.1.3)
   poltergeist
   pry-byebug
   pry-rails
   rack-livereload (~> 0.3.16)
-  rails (~> 5.2.0)
+  rails (~> 6.0.3.1)
   rails-controller-testing
-  rails-html-sanitizer (~> 1.0.4)
+  rails-html-sanitizer (~> 1.2)
   rb-kqueue (>= 0.2)
   rr
-  rspec (~> 3.7)
+  rspec (~> 3.8)
   rspec-collection_matchers (~> 1.1.0)
   rspec-html-matchers (~> 0.8)
-  rspec-rails (~> 3.7)
+  rspec-rails
   rturk (~> 2.12.1)
   ruby-growl (~> 4.1.0)
   rufus-scheduler (~> 3.4.2)
-  sass-rails (~> 5.0)
+  sass-rails (>= 6.0)
   select2-rails (~> 3.5.4)
   shoulda-matchers
   slack-notifier (~> 1.0.0)
@@ -757,25 +778,23 @@ DEPENDENCIES
   spring-commands-rspec (~> 1.0.4)
   spring-watcher-listen (~> 2.0.1)
   sprockets (~> 3.7.2)
-  therubyracer (~> 0.12.3)
   tumblr_client!
   twilio-ruby (~> 3.11.5)
   twitter!
   twitter-stream!
-  typhoeus (~> 0.6.3)
+  typhoeus (~> 1.3.1)
   tzinfo (>= 1.2.0)
   tzinfo-data
   uglifier (~> 2.7.2)
   unicorn (~> 5.1.0)
   vcr
   web-console (>= 3.3.0)
-  webmock (~> 2.3)
+  webmock (~> 3.5.1)
   weibo_2!
-  wunderground (~> 1.2.0)
   xmpp4r (~> 0.5.6)
 
 RUBY VERSION
-   ruby 2.5.1p57
+   ruby 2.6.5p114
 
 BUNDLED WITH
-   1.16.3
+   1.17.3

README.md

@@ -11,25 +11,25 @@ Huginn is a system for building agents that perform automated tasks for you onli
 #### Here are some of the things that you can do with Huginn:
 
 * Track the weather and get an email when it's going to rain (or snow) tomorrow ("Don't forget your umbrella!")
-* List terms that you care about and receive emails when their occurrence on Twitter changes.  (For example, want to know when something interesting has happened in the world of Machine Learning?  Huginn will watch the term "machine learning" on Twitter and tell you when there is a spike in discussion.)
+* List terms that you care about and receive email when their occurrence on Twitter changes.  (For example, want to know when something interesting has happened in the world of Machine Learning?  Huginn will watch the term "machine learning" on Twitter and tell you when there is a spike in discussion.)
 * Watch for air travel or shopping deals
 * Follow your project names on Twitter and get updates when people mention them
-* Scrape websites and receive emails when they change
+* Scrape websites and receive email when they change
 * Connect to Adioso, HipChat, Basecamp, Growl, FTP, IMAP, Jabber, JIRA, MQTT, nextbus, Pushbullet, Pushover, RSS, Bash, Slack, StubHub, translation APIs, Twilio, Twitter, Wunderground, and Weibo, to name a few.
-* Send digest emails with things that you care about at specific times during the day
+* Send digest email with things that you care about at specific times during the day
 * Track counts of high frequency events and send an SMS within moments when they spike, such as the term "san francisco emergency"
 * Send and receive WebHooks
 * Run custom JavaScript or CoffeeScript functions
 * Track your location over time
 * Create Amazon Mechanical Turk workflows as the inputs, or outputs, of agents (the Amazon Turk Agent is called the "HumanTaskAgent"). For example: "Once a day, ask 5 people for a funny cat photo; send the results to 5 more people to be rated; send the top-rated photo to 5 people for a funny caption; send to 5 final people to rate for funniest caption; finally, post the best captioned photo on my blog."
 
-[![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/huginn/huginn?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Changelog #199](https://img.shields.io/badge/changelog-%23199-lightgrey.svg)](https://changelog.com/199)
+[![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/huginn/huginn?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Changelog #199](https://img.shields.io/badge/changelog-%23199-lightgrey.svg)](https://changelog.com/podcast/199)
 
 Join us in our [Gitter room](https://gitter.im/huginn/huginn) to discuss the project.
 
 ### Join us!
 
-Want to help with Huginn?  All contributions are encouraged!  You could make UI improvements, [add new Agents](https://github.com/huginn/huginn/wiki/Creating-a-new-agent), write [documentation and tutorials](https://github.com/huginn/huginn/wiki), or try tackling [issues tagged with #help-wanted](https://github.com/huginn/huginn/issues?direction=desc&labels=help-wanted&page=1&sort=created&state=open).  Please fork, add specs, and send pull requests!
+Want to help with Huginn?  All contributions are encouraged!  You could make UI improvements, [add new Agents](https://github.com/huginn/huginn/wiki/Creating-a-new-agent), write [documentation and tutorials](https://github.com/huginn/huginn/wiki), or try tackling [issues tagged with #"help wanted"](https://github.com/huginn/huginn/issues?direction=desc&labels=help+wanted&page=1&sort=created&state=open).  Please fork, add specs, and send pull requests!
 
 Really want a fix or feature? Want to solve some community issues and earn some extra coffee money? Take a look at the [current bounties on Bountysource](https://www.bountysource.com/trackers/282580-huginn).
 
@@ -71,8 +71,8 @@ If you just want to play around, you can simply fork this repository, then perfo
 * Read the [wiki][wiki] for usage examples and to get started making new Agents.
 * Periodically run `git fetch upstream` and then `git checkout master && git merge upstream/master` to merge in the newest version of Huginn.
 
-Note: By default, emails are intercepted in the `development` Rails environment, which is what you just setup.  You can view
-them at [http://localhost:3000/letter_opener](http://localhost:3000/letter_opener). If you'd like to send real emails via SMTP when playing
+Note: By default, email messages are intercepted in the `development` Rails environment, which is what you just setup.  You can view
+them at [http://localhost:3000/letter_opener](http://localhost:3000/letter_opener). If you'd like to send real email via SMTP when playing
 with Huginn locally, set `SEND_EMAIL_IN_DEVELOPMENT` to `true` in your `.env` file.
 
 If you need more detailed instructions, see the [Novice setup guide][novice-setup-guide].
@@ -110,12 +110,26 @@ Try Huginn on Heroku: [![Deploy](https://www.herokucdn.com/deploy/button.png)](h
 
 Huginn launches on the free version of Heroku [with significant limitations](https://github.com/huginn/huginn/blob/master/doc/heroku/install.md). For non-experimental use, we strongly recommend Heroku's 1GB paid plan or our Docker container.
 
-### OpenShift Online (v3)
+### OpenShift
 
-Try Huginn on OpenShift Online (v3): `oc new-app -f https://raw.githubusercontent.com/huginn/huginn/master/openshift/templates/huginn-mysql.json` or `oc new-app -f https://raw.githubusercontent.com/huginn/huginn/master/openshift/templates/huginn-postgresql.json`. You can also use the web console to import either json file by going to "Add to Project" -> "Import YAML/JSON".
+#### OpenShift Online
+
+Try Huginn on OpenShift Online
+
+Create a new app with either `mysql` or `postgres`:
+```bash
+oc new-app -f https://raw.githubusercontent.com/huginn/huginn/master/openshift/templates/huginn-mysql.json
+```
+or
+```bash
+oc new-app -f https://raw.githubusercontent.com/huginn/huginn/master/openshift/templates/huginn-postgresql.json
+```
+**Note**: You can also use the web console to import either json file by going to "Add to Project" -> "Import YAML/JSON".
 
 If you are on the Starter plan, make sure to follow the [guide](https://docs.openshift.com/online/getting_started/beyond_the_basics.html#btb-creating-a-new-application-from-source-code) to remove any existing application.
 
+The templates should work on a v3 installation or the current v4 online.
+
 ### Manual installation on any server
 
 Have a look at the [installation guide](https://github.com/huginn/huginn/blob/master/doc/manual/README.md).
@@ -142,4 +156,4 @@ Huginn is provided under the MIT License.
 
 Huginn was originally created by [@cantino](https://github.com/cantino) in 2013. Since then, many people's dedicated contributions have made it what it is today.
 
-[![Build Status](https://travis-ci.org/huginn/huginn.svg)](https://travis-ci.org/huginn/huginn) [![Coverage Status](https://coveralls.io/repos/cantino/huginn/badge.svg)](https://coveralls.io/r/cantino/huginn) [![Dependency Status](https://gemnasium.com/huginn/huginn.svg)](https://gemnasium.com/huginn/huginn) [![Bountysource](https://www.bountysource.com/badge/tracker?tracker_id=282580)](https://www.bountysource.com/trackers/282580-huginn?utm_source=282580&utm_medium=shield&utm_campaign=TRACKER_BADGE)
+[![Build Status](https://travis-ci.org/huginn/huginn.svg)](https://travis-ci.org/huginn/huginn) [![Coverage Status](https://coveralls.io/repos/huginn/huginn/badge.svg)](https://coveralls.io/r/huginn/huginn) [![Dependency Status](https://gemnasium.com/huginn/huginn.svg)](https://gemnasium.com/huginn/huginn) [![Bountysource](https://www.bountysource.com/badge/tracker?tracker_id=282580)](https://www.bountysource.com/trackers/282580-huginn?utm_source=282580&utm_medium=shield&utm_campaign=TRACKER_BADGE)

app.json

@@ -21,5 +21,6 @@
       "postdeploy": "bundle exec rake db:migrate"
     },
     "addons": ["heroku-postgresql"],
-    "success_url": "/users/sign_up"
+    "success_url": "/users/sign_up",
+    "stack": "heroku-18"
 }

app/assets/javascripts/map_marker.js.coffee

@@ -12,14 +12,7 @@ window.map_marker = (map, options = {}) ->
       center: pos
       radius: options.radius
     return marker
-  else
-    marker = new google.maps.Marker
-      map: map
-      position: pos
-      title: 'Recorded Location'
-    return marker
-
-  if options.course
+  else if options.course
     p1 = new LatLon(pos.lat(), pos.lng())
     speed = options.speed ? 1
     p2 = p1.destinationPoint(options.course, Math.max(0.2, speed) * 0.1)
@@ -41,5 +34,10 @@ window.map_marker = (map, options = {}) ->
           offset: '100%'
         }
       ]
-
     return arrow
+  else
+    marker = new google.maps.Marker
+      map: map
+      position: pos
+      title: 'Recorded Location'
+    return marker

app/assets/stylesheets/application.scss.erb

@@ -219,7 +219,7 @@ span.not-applicable:after {
   font-size: 14px;
 }
 
-// Position tweeks
+// Position tweaks
 
 .hover-help {
   top: 2px;
@@ -231,6 +231,12 @@ span.not-applicable:after {
   }
 }
 
+.page-header .btn-group {
+  position: relative;
+  top: -5px;
+  left: 4px;
+}
+
 h2 .scenario, a span.label.scenario {
   position: relative;
   top: -2px;

app/concerns/agent_controller_concern.rb

@@ -55,6 +55,9 @@ module AgentControllerConcern
           when 'enable'
             case
             when target.disabled?
+              if boolify(interpolated['drop_pending_events'])
+                target.drop_pending_events = true
+              end
               target.update!(disabled: false)
               log "Agent '#{target.name}' is enabled"
             else

app/concerns/event_headers_concern.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module EventHeadersConcern
+  private
+
+  def validate_event_headers_options!
+    event_headers_payload({})
+  rescue ArgumentError => e
+    errors.add(:base, e.message)
+  rescue Liquid::Error => e
+    errors.add(:base, "has an error with Liquid templating: #{e.message}")
+  end
+
+  def event_headers_normalizer
+    case interpolated['event_headers_style']
+    when nil, '', 'capitalized'
+      ->name { name.gsub(/[^-]+/, &:capitalize) }
+    when 'downcased'
+      :downcase.to_proc
+    when 'snakecased', nil
+      ->name { name.tr('A-Z-', 'a-z_') }
+    when 'raw'
+      :itself.to_proc
+    else
+      raise ArgumentError, "if provided, event_headers_style must be 'capitalized', 'downcased', 'snakecased' or 'raw'"
+    end
+  end
+
+  def event_headers_key
+    case key = interpolated['event_headers_key']
+    when nil, String
+      key.presence
+    else
+      raise ArgumentError, "if provided, event_headers_key must be a string"
+    end
+  end
+
+  def event_headers_payload(headers)
+    key = event_headers_key or return {}
+
+    normalize = event_headers_normalizer
+
+    hash = headers.transform_keys(&normalize)
+
+    names =
+      case event_headers = interpolated['event_headers']
+      when Array
+        event_headers.map(&:to_s)
+      when String
+        event_headers.split(',')
+      when nil
+        nil
+      else
+        raise ArgumentError, "if provided, event_headers must be an array of strings or a comma separated string"
+      end
+
+    {
+      key => names ? hash.slice(*names.map(&normalize)) : hash
+    }
+  end
+end

app/concerns/json_serialized_field.rb

@@ -1,13 +1,13 @@
 require 'json_with_indifferent_access'
 
-module JSONSerializedField
+module JsonSerializedField
   extend ActiveSupport::Concern
 
   module ClassMethods
     def json_serialize(*fields)
       fields.each do |field|
         class_eval <<-CODE
-          serialize :#{field}, JSONWithIndifferentAccess
+          serialize :#{field}, JsonWithIndifferentAccess
 
           validate :#{field}_has_no_errors
 
@@ -39,4 +39,4 @@ module JSONSerializedField
       end
     end
   end
-end
+end

app/concerns/liquid_interpolatable.rb

@@ -71,6 +71,7 @@ module LiquidInterpolatable
   def interpolate_with_each(array)
     array.each do |object|
       interpolate_with(object) do
+        self.current_event = object
         yield object
       end
     end
@@ -243,6 +244,26 @@ module LiquidInterpolatable
       JSON.dump(input)
     end
 
+    def md5(input)
+      Digest::MD5.hexdigest(input.to_s)
+    end
+
+    def sha1(input)
+      Digest::SHA1.hexdigest(input.to_s)
+    end
+
+    def sha256(input)
+      Digest::SHA256.hexdigest(input.to_s)
+    end
+
+    def hmac_sha1(input, key)
+      OpenSSL::HMAC.hexdigest('sha1', key.to_s, input.to_s)
+    end
+
+    def hmac_sha256(input, key)
+      OpenSSL::HMAC.hexdigest('sha256', key.to_s, input.to_s)
+    end
+
     # Returns a Ruby object
     #
     # It can be used as a JSONPath replacement for Agents that only support Liquid:
@@ -262,6 +283,27 @@ module LiquidInterpolatable
       throw :as_object, object.as_json
     end
 
+    # Group an array of items by a property
+    #
+    # Example usage:
+    #
+    # {% assign posts_by_author = site.posts | group_by: "author" %}
+    # {% for author in posts_by_author %}
+    #   <dt>{{author.name}}</dt>
+    #   {% for post in author.items %}
+    #   <dd><a href="{{post.url}}">{{post.title}}</a></dd>
+    #   {% endfor %}
+    # {% endfor %}
+    def group_by(input, property)
+      if input.respond_to?(:group_by)
+        input.group_by { |item| item[property] }.map do |value, items|
+          { 'name' => value, 'items' => items }
+        end
+      else
+        input
+      end
+    end
+
     private
 
     def logger

app/controllers/agents_controller.rb

@@ -75,6 +75,18 @@ class AgentsController < ApplicationController
     render :json => { :description_html => html }
   end
 
+  def reemit_events
+    @agent = current_user.agents.find(params[:id])
+
+    AgentReemitJob.perform_later(@agent, @agent.most_recent_event.id,
+                                 params[:delete_old_events] == '1')
+
+    respond_to do |format|
+      format.html { redirect_back "Enqueued job to re-emit all events for '#{@agent.name}'" }
+      format.json { head :ok }
+    end
+  end
+
   def remove_events
     @agent = current_user.agents.find(params[:id])
     @agent.events.delete_all
@@ -160,7 +172,7 @@ class AgentsController < ApplicationController
     @agent = current_user.agents.find(params[:id])
 
     respond_to do |format|
-      if @agent.update_attributes(agent_params)
+      if @agent.update(agent_params)
         format.html { redirect_back "'#{@agent.name}' was successfully updated.", return: agents_path }
         format.json { render json: @agent, status: :ok, location: agent_path(@agent) }
       else

app/controllers/application_controller.rb

@@ -30,7 +30,6 @@ class ApplicationController < ActionController::Base
     return unless current_user
     twitter_oauth_check
     basecamp_auth_check
-    outdated_docker_image_namespace_check
     outdated_google_auth_check
   end
 
@@ -65,10 +64,6 @@ class ApplicationController < ActionController::Base
     end
   end
 
-  def outdated_docker_image_namespace_check
-    @outdated_docker_image_namespace = ENV['OUTDATED_DOCKER_IMAGE_NAMESPACE'] == 'true'
-  end
-
   def outdated_google_auth_check
     @outdated_google_cal_agents = current_user.agents.of_type('Agents::GoogleCalendarPublishAgent').select do |agent|
       agent.options['google']['key_secret'].present?

app/controllers/jobs_controller.rb

@@ -29,7 +29,7 @@ class JobsController < ApplicationController
     @job.last_error = nil
 
     respond_to do |format|
-      if !running? && @job.update_attributes!(run_at: Time.now, failed_at: nil)
+      if !running? && @job.update!(run_at: Time.now, failed_at: nil)
         format.html { redirect_to jobs_path, notice: "Job enqueued." }
         format.json { render json: @job, status: :ok }
       else

app/controllers/scenarios_controller.rb

@@ -88,7 +88,7 @@ class ScenariosController < ApplicationController
     @scenario = current_user.scenarios.find(params[:id])
 
     respond_to do |format|
-      if @scenario.update_attributes(scenario_params)
+      if @scenario.update(scenario_params)
         format.html { redirect_to @scenario, notice: 'This Scenario was successfully updated.' }
         format.json { head :no_content }
       else

app/controllers/user_credentials_controller.rb

@@ -65,7 +65,7 @@ class UserCredentialsController < ApplicationController
     @user_credential = current_user.user_credentials.find(params[:id])
 
     respond_to do |format|
-      if @user_credential.update_attributes(user_credential_params)
+      if @user_credential.update(user_credential_params)
         format.html { redirect_to user_credentials_path, notice: 'Your credential was successfully updated.' }
         format.json { head :no_content }
       else

app/controllers/web_requests_controller.rb

@@ -20,6 +20,7 @@
 class WebRequestsController < ApplicationController
   skip_before_action :verify_authenticity_token
   skip_before_action :authenticate_user!
+  wrap_parameters false
 
   def handle_request
     user = User.find_by_id(params[:user_id])

app/helpers/agent_helper.rb

@@ -1,9 +1,8 @@
 module AgentHelper
+
   def agent_show_view(agent)
-    name = agent.short_type.underscore
-    if File.exist?(Rails.root.join("app", "views", "agents", "agent_views", name, "_show.html.erb"))
-      File.join("agents", "agent_views", name, "show")
-    end
+    path = File.join('agents', 'agent_views', @agent.short_type.underscore, 'show')
+    return self.controller.template_exists?(path, [], true) ? path : nil
   end
 
   def toggle_disabled_text

app/importers/scenario_import.rb

@@ -66,7 +66,7 @@ class ScenarioImport
     icon = parsed_data['icon']
     source_url = parsed_data['source_url'].presence || nil
     @scenario = user.scenarios.where(:guid => guid).first_or_initialize
-    @scenario.update_attributes!(name: name, description: description,
+    @scenario.update!(name: name, description: description,
                                  source_url: source_url, public: false,
                                  tag_fg_color: tag_fg_color,
                                  tag_bg_color: tag_bg_color,

app/jobs/agent_reemit_job.rb

@@ -0,0 +1,10 @@
+class AgentReemitJob < ActiveJob::Base
+  # Given an Agent, re-emit all of agent's events up to (and including) `most_recent_event_id`
+  def perform(agent, most_recent_event_id, delete_old_events = false)
+    # `find_each` orders by PK, so events get re-created in the same order
+    agent.events.where("id <= ?", most_recent_event_id).find_each do |event|
+      event.reemit!
+      event.destroy if delete_old_events
+    end
+  end
+end

app/models/agent.rb

@@ -6,8 +6,8 @@ require 'utils'
 class Agent < ActiveRecord::Base
   include AssignableTypes
   include MarkdownClassAttributes
-  include JSONSerializedField
-  include RDBMSFunctions
+  include JsonSerializedField
+  include RdbmsFunctions
   include WorkingHelpers
   include LiquidInterpolatable
   include HasGuid
@@ -222,7 +222,7 @@ class Agent < ActiveRecord::Base
   end
 
   def log(message, options = {})
-    AgentLog.log_for_agent(self, message, options)
+    AgentLog.log_for_agent(self, message, options.merge(inbound_event: current_event))
   end
 
   def error(message, options = {})
@@ -265,7 +265,9 @@ class Agent < ActiveRecord::Base
   #Validation Methods
   
   private
-  
+
+  attr_accessor :current_event
+
   def validate_schedule
     unless cannot_be_scheduled?
       errors.add(:schedule, "is not a valid schedule") unless SCHEDULES.include?(schedule.to_s)
@@ -289,6 +291,12 @@ class Agent < ActiveRecord::Base
     end
   end
 
+  def is_positive_integer?(value)
+    Integer(value) >= 0
+  rescue
+    false
+  end
+
   # Class Methods
 
   class << self

app/models/agents/commander_agent.rb

@@ -16,6 +16,7 @@ module Agents
       * `disable`: Target Agents are disabled (if not) when this agent is triggered.
 
       * `enable`: Target Agents are enabled (if not) when this agent is triggered.
+        * If the option `drop_pending_events` is set to `true`, pending events will be cleared before the agent is enabled.
 
       * `configure`: Target Agents have their options updated with the contents of `configure_options`.
 

app/models/agents/data_output_agent.rb

@@ -27,7 +27,7 @@ module Agents
           * `ttl` - A value for the \\<ttl\\> element in RSS output. (default: `60`)
           * `ns_media` - Add [yahoo media namespace](https://en.wikipedia.org/wiki/Media_RSS) in output xml
           * `ns_itunes` - Add [itunes compatible namespace](http://lists.apple.com/archives/syndication-dev/2005/Nov/msg00002.html) in output xml
-          * `rss_content_type` - Content-Type for RSS output (default: `application/rss+xml, application/rdf+xml;q=0.8, application/atom+xml;q=0.6, application/xml;q=0.4, text/xml;q=0.4`)
+          * `rss_content_type` - Content-Type for RSS output (default: `application/rss+xml`)
           * `response_headers` - An object with any custom response headers. (example: `{"Access-Control-Allow-Origin": "*"}`)
           * `push_hubs` - Set to a list of PubSubHubbub endpoints you want to publish an update to every time this agent receives an event. (default: none)  Popular hubs include [Superfeedr](https://pubsubhubbub.superfeedr.com/) and [Google](https://pubsubhubbub.appspot.com/).  Note that publishing updates will make your feed URL known to the public, so if you want to keep it secret, set up a reverse proxy to serve your feed via a safe URL and specify it in `template.self`.
 
@@ -174,7 +174,7 @@ module Agents
     end
 
     def rss_content_type
-      interpolated['rss_content_type'].presence || 'application/rss+xml, application/rdf+xml;q=0.8, application/atom+xml;q=0.6, application/xml;q=0.4, text/xml;q=0.4'
+      interpolated['rss_content_type'].presence || 'application/rss+xml'
     end
 
     def xml_namespace

app/models/agents/delay_agent.rb

@@ -1,6 +1,8 @@
 module Agents
   class DelayAgent < Agent
-    default_schedule "every_12h"
+    include FormConfigurable
+
+    default_schedule 'every_12h'
 
     description <<-MD
       The DelayAgent stores received Events and emits copies of them on a schedule. Use this as a buffer or queue of Events.
@@ -17,12 +19,18 @@ module Agents
 
     def default_options
       {
-        'expected_receive_period_in_days' => "10",
-        'max_events' => "100",
-        'keep' => 'newest'
+        'expected_receive_period_in_days' => '10',
+        'max_events' => '100',
+        'keep' => 'newest',
+        'max_emitted_events' => ''
       }
     end
 
+    form_configurable :expected_receive_period_in_days, type: :string
+    form_configurable :max_events, type: :string
+    form_configurable :keep, type: :array, values: %w[newest oldest]
+    form_configurable :max_emitted_events, type: :string
+
     def validate_options
       unless options['expected_receive_period_in_days'].present? && options['expected_receive_period_in_days'].to_i > 0
         errors.add(:base, "Please provide 'expected_receive_period_in_days' to indicate how many days can pass before this Agent is considered to be not working")
@@ -32,9 +40,15 @@ module Agents
         errors.add(:base, "The 'keep' option is required and must be set to 'oldest' or 'newest'")
       end
 
-      unless options['max_events'].present? && options['max_events'].to_i > 0
+      unless interpolated['max_events'].present? && interpolated['max_events'].to_i > 0
         errors.add(:base, "The 'max_events' option is required and must be an integer greater than 0")
       end
+
+      if interpolated['max_emitted_events'].present?
+        unless interpolated['max_emitted_events'].to_i > 0
+          errors.add(:base, "The 'max_emitted_events' option is optional and should be an integer greater than 0")
+        end
+      end
     end
 
     def working?
@@ -46,7 +60,7 @@ module Agents
         memory['event_ids'] ||= []
         memory['event_ids'] << event.id
         if memory['event_ids'].length > interpolated['max_events'].to_i
-          if interpolated['keep'] == 'newest'
+          if options['keep'] == 'newest'
             memory['event_ids'].shift
           else
             memory['event_ids'].pop
@@ -59,8 +73,8 @@ module Agents
       if memory['event_ids'] && memory['event_ids'].length > 0
         events = received_events.where(id: memory['event_ids']).reorder('events.id asc')
 
-        if options['max_emitted_events'].present?
-          events = events.limit(options['max_emitted_events'].to_i)
+        if interpolated['max_emitted_events'].present?
+          events = events.limit(interpolated['max_emitted_events'].to_i)
         end
 
         events.each do |event|

app/models/agents/dropbox_watch_agent.rb

@@ -51,12 +51,6 @@ module Agents
 
     private
 
-    def is_positive_integer?(value)
-      Integer(value) >= 0
-    rescue
-      false
-    end
-
     def ls(dir_to_watch)
       dropbox.ls(dir_to_watch).map { |file| { 'path' => file.path, 'rev' => file.rev, 'modified' => file.server_modified } }
     end

app/models/agents/email_agent.rb

@@ -2,6 +2,7 @@ module Agents
   class EmailAgent < Agent
     include EmailConcern
 
+    can_dry_run!
     cannot_be_scheduled!
     cannot_create_events!
     no_bulk_receive!

app/models/agents/email_digest_agent.rb

@@ -11,7 +11,7 @@ module Agents
       used events also relies on the `Keep events` option of the emitting Agent, meaning that if events expire before
       this agent is scheduled to run, they will not appear in the email.
 
-      By default, the will have a `subject` and an optional `headline` before listing the Events.  If the Events'
+      By default, the email will have a `subject` and an optional `headline` before listing the Events.  If the Events'
       payloads contain a `message`, that will be highlighted, otherwise everything in
       their payloads will be shown.
 

app/models/agents/ftpsite_agent.rb

@@ -257,12 +257,6 @@ module Agents
 
     private
 
-    def is_positive_integer?(value)
-      Integer(value) >= 0
-    rescue
-      false
-    end
-
     def uri_path_escape(string)
       str = string.b
       str.gsub!(/([^A-Za-z0-9\-._~!$&()*+,=@]+)/) { |m|

app/models/agents/imap_folder_agent.rb

@@ -5,6 +5,8 @@ require 'mail'
 
 module Agents
   class ImapFolderAgent < Agent
+    include EventHeadersConcern
+
     cannot_receive_events!
 
     can_dry_run!
@@ -54,6 +56,15 @@ module Agents
           If this key is unspecified or set to null, it is ignored.
 
       Set `mark_as_read` to true to mark found mails as read.
+      Set `delete` to true to delete found mails.
+
+      Set `event_headers` to a list of header names you want to include in a `headers` hash in each created event, either in an array of string or in a comma-separated string.
+
+      Set `event_headers_style` to one of the following values to normalize the keys of "headers" for downstream agents' convenience:
+
+        * `capitalized` (default) - Header names are capitalized; e.g. "Content-Type"
+        * `downcased` - Header names are downcased; e.g. "content-type"
+        * `snakecased` - Header names are snakecased; e.g. "content_type"
 
       Set `include_raw_mail` to true to add a `raw_mail` value to each created event, which contains a *Base64-encoded* blob in the "RFC822" format defined in [the IMAP4 standard](https://tools.ietf.org/html/rfc3501).  Note that while the result of Base64 encoding will be LF-terminated, its raw content will often be CRLF-terminated because of the nature of the e-mail protocols and formats.  The primary use case for a raw mail blob is to pass to a Shell Command Agent with a command like `openssl enc -d -base64 | tr -d '\r' | procmail -Yf-`.
 
@@ -79,7 +90,7 @@ module Agents
             }
           }
 
-      Additionally, "raw_mail" will be included if the `include_raw_mail` option is set.
+      Additionally, "headers" will be included if the `event_headers` option is set, and "raw_mail" if the `include_raw_mail` option is set.
     MD
 
     IDCACHE_SIZE = 100
@@ -118,7 +129,7 @@ module Agents
         errors.add(:base, "port must be a positive integer") unless is_positive_integer?(options['port'])
       end
 
-      %w[ssl mark_as_read include_raw_mail].each { |key|
+      %w[ssl mark_as_read delete include_raw_mail].each { |key|
         if options[key].present?
           if boolify(options[key]).nil?
             errors.add(:base, '%s must be a boolean value' % key)
@@ -288,6 +299,14 @@ module Agents
             payload['raw_mail'] = Base64.encode64(mail.raw_mail)
           end
 
+          if interpolated['event_headers'].present?
+            headers = mail.header.each_with_object({}) { |field, hash|
+              name = field.name
+              hash[name] = (v = hash[name]) ? "#{v}\n#{field.value.to_s}" : field.value.to_s
+            }
+            payload.update(event_headers_payload(headers))
+          end
+
           create_event payload: payload
 
           notified << mail.message_id if mail.message_id
@@ -297,6 +316,11 @@ module Agents
           log 'Marking as read'
           mail.mark_as_read unless dry_run?
         end
+
+        if boolify(interpolated['delete'])
+          log 'Deleting'
+          mail.delete unless dry_run?
+        end
       }
     end
 
@@ -405,12 +429,6 @@ module Agents
 
     private
 
-    def is_positive_integer?(value)
-      Integer(value) >= 0
-    rescue
-      false
-    end
-
     def glob_match?(pattern, value)
       File.fnmatch?(pattern, value, FNM_FLAGS)
     end
@@ -419,6 +437,10 @@ module Agents
       "%d %s" % [count, noun.pluralize(count)]
     end
 
+    def event_headers_key
+      super || 'headers'
+    end
+
     class Client < ::Net::IMAP
       class << self
         def open(host, *args)
@@ -552,6 +574,11 @@ module Agents
         @client.uid_store(@uid, '+FLAGS', [:Seen])
       end
 
+      def delete
+        @client.uid_store(@uid, '+FLAGS', [:Deleted])
+        @client.expunge
+      end
+
       private
 
       def struct_has_attachment?(struct)

app/models/agents/java_script_agent.rb

@@ -9,9 +9,13 @@ module Agents
 
     default_schedule "never"
 
+    gem_dependency_check { defined?(MiniRacer) }
+
     description <<-MD
       The JavaScript Agent allows you to write code in JavaScript that can create and receive events.  If other Agents aren't meeting your needs, try this one!
 
+      #{'## Include `mini_racer` in your Gemfile to use this Agent!' if dependencies_missing?}
+
       You can put code in the `code` option, or put your code in a Credential and reference it from `code` with `credential:<name>` (recommended).
 
       You can implement `Agent.check` and `Agent.receive` as you see fit.  The following methods will be available on Agent in the JavaScript environment:
@@ -107,26 +111,22 @@ module Agents
 
     def execute_js(js_function, incoming_events = [])
       js_function = js_function == "check" ? "check" : "receive"
-      context = V8::Context.new
+      context = MiniRacer::Context.new
       context.eval(setup_javascript)
 
-      context["doCreateEvent"] = lambda { |a, y| create_event(payload: clean_nans(JSON.parse(y))).payload.to_json }
-      context["getIncomingEvents"] = lambda { |a| incoming_events.to_json }
-      context["getOptions"] = lambda { |a, x| interpolated.to_json }
-      context["doLog"] = lambda { |a, x| log x }
-      context["doError"] = lambda { |a, x| error x }
-      context["getMemory"] = lambda { |a| memory.to_json }
-      context["setMemoryKey"] = lambda do |a, x, y|
-        memory[x] = clean_nans(y)
-      end
-      context["setMemory"] = lambda do |a, x|
-        memory.replace(clean_nans(x))
-      end
-      context["deleteKey"] = lambda { |a, x| memory.delete(x).to_json }
-      context["escapeHtml"] = lambda { |a, x| CGI.escapeHTML(x) }
-      context["unescapeHtml"] = lambda { |a, x| CGI.unescapeHTML(x) }
-      context['getCredential'] = lambda { |a, k| credential(k); }
-      context['setCredential'] = lambda { |a, k, v| set_credential(k, v) }
+      context.attach("doCreateEvent", -> (y) { create_event(payload: clean_nans(JSON.parse(y))).payload.to_json })
+      context.attach("getIncomingEvents", -> { incoming_events.to_json })
+      context.attach("getOptions", -> { interpolated.to_json })
+      context.attach("doLog", -> (x) { log x })
+      context.attach("doError", -> (x) { error x })
+      context.attach("getMemory", -> { memory.to_json })
+      context.attach("setMemoryKey", -> (x, y) { memory[x] = clean_nans(y) })
+      context.attach("setMemory", -> (x) { memory.replace(clean_nans(x)) })
+      context.attach("deleteKey", -> (x) { memory.delete(x).to_json })
+      context.attach("escapeHtml", -> (x) { CGI.escapeHTML(x) })
+      context.attach("unescapeHtml", -> (x) { CGI.unescapeHTML(x) })
+      context.attach('getCredential', -> (k) { credential(k); })
+      context.attach('setCredential', -> (k, v) { set_credential(k, v) })
 
       if (options['language'] || '').downcase == 'coffeescript'
         context.eval(CoffeeScript.compile code)
@@ -225,15 +225,15 @@ module Agents
     def log_errors
       begin
         yield
-      rescue V8::Error => e
+      rescue MiniRacer::Error => e
         error "JavaScript error: #{e.message}"
       end
     end
 
     def clean_nans(input)
-      if input.is_a?(V8::Array)
+      if input.is_a?(Array)
         input.map {|v| clean_nans(v) }
-      elsif input.is_a?(V8::Object)
+      elsif input.is_a?(Hash)
         input.inject({}) { |m, (k, v)| m[k] = clean_nans(v); m }
       elsif input.is_a?(Float) && input.nan?
         'NaN'

app/models/agents/jq_agent.rb

@@ -0,0 +1,205 @@
+require 'open3'
+
+module Agents
+  class JqAgent < Agent
+    cannot_be_scheduled!
+    can_dry_run!
+
+    def self.should_run?
+      !!jq_version
+    end
+
+    def self.jq_command
+      ENV['USE_JQ'].presence
+    end
+
+    def self.jq_version
+      if command = jq_command
+        Open3.capture2(command, '--version', 2 => IO::NULL).first[/\Ajq-\K\S+/]
+      end
+    end
+
+    def self.jq_info
+      if version = jq_version
+        "jq version #{version} is installed"
+      else
+        "**This agent is not enabled on this server**"
+      end
+    end
+
+    gem_dependency_check { jq_version }
+
+    description <<-MD
+      The Jq Agent allows you to process incoming Events with [jq](https://stedolan.github.io/jq/) the JSON processor. (#{jq_info})
+
+      It allows you to filter, transform and restructure Events in the way you want using jq's powerful features.
+
+      You can specify a jq filter expression to apply to each incoming event in `filter`, and results it produces will become Events to be emitted.
+
+      You can optionally pass in variables to the filter program by specifying key-value pairs of a variable name and an associated value in the `variables` key, each of which becomes a predefined variable.
+
+      This Agent can be used to parse a complex JSON structure that is too hard to handle with JSONPath or Liquid templating.
+
+      For example, suppose that a Post Agent created an Event which contains a `body` key with a value of the JSON formatted string of the following response body:
+
+          {
+            "status": "1",
+            "since": "1245626956",
+            "list": {
+              "93817": {
+                "item_id": "93817",
+                "url": "http://url.com",
+                "title": "Page Title",
+                "time_updated": "1245626956",
+                "time_added": "1245626956",
+                "tags": "comma,seperated,list",
+                "state": "0"
+              },
+              "935812": {
+                "item_id": "935812",
+                "url": "http://google.com",
+                "title": "Google",
+                "time_updated": "1245635279",
+                "time_added": "1245635279",
+                "tags": "comma,seperated,list",
+                "state": "1"
+              }
+            }
+          }
+
+      Then you could have a Jq Agent with the following jq filter:
+
+          .body | fromjson | .list | to_entries | map(.value) | map(try(.tags |= split(",")) // .) | sort_by(.time_added | tonumber)
+
+      To get the following two Events emitted out of the said incoming Event from Post Agent:
+
+          [
+            {
+              "item_id": "93817",
+              "url": "http://url.com",
+              "title": "Page Title",
+              "time_updated": "1245626956",
+              "time_added": "1245626956",
+              "tags": ["comma", "seperated", "list"],
+              "state": "0"
+            },
+            {
+              "item_id": "935812",
+              "url": "http://google.com",
+              "title": "Google",
+              "time_updated": "1245626956",
+              "time_added": "1245626956",
+              "tags": ["comma", "seperated", "list"],
+              "state": "1"
+            }
+          ]
+    MD
+
+    def validate_options
+      errors.add(:base, "filter needs to be present.") if !options['filter'].is_a?(String)
+      errors.add(:base, "variables must be a hash if present.") if options.key?('variables') && !options['variables'].is_a?(Hash)
+    end
+
+    def default_options
+      {
+        'filter' => '.',
+        'variables' => {}
+      }
+    end
+
+    def working?
+      self.class.should_run? && !recent_error_logs?
+    end
+
+    def receive(incoming_events)
+      if !self.class.should_run?
+        log("Unable to run because this agent is not enabled.  Edit the USE_JQ environment variable.")
+        return
+      end
+
+      incoming_events.each do |event|
+        interpolate_with(event) do
+          process_event(event)
+        end
+      end
+    end
+
+    private
+
+    def get_variables
+      variables = interpolated['variables']
+      return {} if !variables.is_a?(Hash)
+
+      variables.map { |name, value|
+        [name.to_s, value.to_json]
+      }.to_h
+    end
+
+    def process_event(event)
+      Tempfile.create do |file|
+        filter = interpolated['filter'].to_s
+
+        # There seems to be no way to force jq to treat an arbitrary
+        # string as a filter without being confused with a command
+        # line option, so pass one via file.
+        file.print filter
+        file.close
+
+        variables = get_variables
+
+        command_args = [
+          self.class.jq_command,
+          '--compact-output',
+          '--sort-keys',
+          '--from-file', file.path,
+          *variables.flat_map { |name, json|
+            ['--argjson', name, json]
+          }
+        ]
+
+        log [
+          "Running jq with filter: #{filter}",
+          *variables.map { |name, json| "variable: #{name} = #{json}" }
+        ].join("\n")
+
+        Open3.popen3(*command_args) do |stdin, stdout, stderr, wait_thread|
+          stderr_reader = Thread.new { stderr.read }
+          stdout_reader = Thread.new { stdout.each_line.flat_map { |line| JSON.parse(line) } }
+
+          results, errout, status =
+            begin
+              JSON.dump(event.payload, stdin)
+              stdin.close
+
+              [
+                stdout_reader.value,
+                stderr_reader.value,
+                wait_thread.value
+              ]
+            rescue Errno::EPIPE
+            end
+
+          if !status.success?
+            error "Error output from jq:\n#{errout}"
+            return
+          end
+
+          results.keep_if do |result|
+            if result.is_a?(Hash)
+              true
+            else
+              error "Ignoring a non-object result: #{result.to_json}"
+              false
+            end
+          end
+
+          log "Creating #{results.size} events"
+
+          results.each do |payload|
+            create_event payload: payload
+          end
+        end
+      end
+    end
+  end
+end

app/models/agents/json_parse_agent.rb

@@ -3,19 +3,23 @@ module Agents
     include FormConfigurable
 
     cannot_be_scheduled!
+    can_dry_run!
 
     description <<-MD
-      The JSON Parse Agent parses a JSON string and emits the data in a new event
+      The JSON Parse Agent parses a JSON string and emits the data in a new event or merge with with the original event.
 
       `data` is the JSON to parse. Use [Liquid](https://github.com/huginn/huginn/wiki/Formatting-Events-using-Liquid) templating to specify the JSON string.
 
       `data_key` sets the key which contains the parsed JSON data in emitted events
+
+      `mode` determines whether create a new `clean` event or `merge` old payload with new values (default: `clean`)
     MD
 
     def default_options
       {
         'data' => '{{ data }}',
         'data_key' => 'data',
+        'mode' => 'clean',
       }
     end
 
@@ -25,10 +29,14 @@ module Agents
 
     form_configurable :data
     form_configurable :data_key
+    form_configurable :mode, type: :array, values: ['clean', 'merge']
 
     def validate_options
       errors.add(:base, "data needs to be present") if options['data'].blank?
       errors.add(:base, "data_key needs to be present") if options['data_key'].blank?
+      if options['mode'].present? && !options['mode'].to_s.include?('{{') && !%[clean merge].include?(options['mode'].to_s)
+        errors.add(:base, "mode must be 'clean' or 'merge'")
+      end
     end
 
     def working?
@@ -39,7 +47,8 @@ module Agents
       incoming_events.each do |event|
         begin
           mo = interpolated(event)
-          create_event payload: { mo['data_key'] => JSON.parse(mo['data']) }
+          existing_payload = mo['mode'].to_s == 'merge' ? event.payload : {}
+          create_event payload: existing_payload.merge({ mo['data_key'] => JSON.parse(mo['data']) })
         rescue JSON::JSONError => e
           error("Could not parse JSON: #{e.class} '#{e.message}'")
         end

app/models/agents/liquid_output_agent.rb

@@ -129,8 +129,8 @@ EOF
       end
 
       if options['event_limit'].present?
-        if((Integer(options['event_limit']) rescue false) == false)
-          errors.add(:base, "Event limit must be an integer that is less than 1001.")
+        if (Integer(options['event_limit']) rescue false) == false && date_limit.blank?
+          errors.add(:base, "Event limit must be an integer that is less than 1001 or an integer plus a valid unit.")
         elsif (options['event_limit'].to_i > 1000)
           errors.add(:base, "For performance reasons, you cannot have an event limit greater than 1000.")
         end

app/models/agents/manual_event_agent.rb

@@ -6,14 +6,18 @@ module Agents
     description <<-MD
       The Manual Event Agent is used to manually create Events for testing or other purposes.
 
-      Do not set options for this Agent.  Instead, connect it to other Agents and create Events
-      using the UI provided on this Agent's Summary page.
+      Connect this Agent to other Agents and create Events using the UI provided on this Agent's Summary page.
+
+      You can set the default event payload via the "payload" option.
     MD
 
-    event_description "User determined"
+    event_description do
+      "Events are editable in the UI.  The default value is this:\n\n    " +
+        Utils.pretty_print(options["payload"].presence || {})
+    end
 
     def default_options
-      { "no options" => "are needed" }
+      { "payload" => {} }
     end
 
     def handle_details_post(params)

app/models/agents/mqtt_agent.rb

@@ -94,16 +94,16 @@ module Agents
     end
 
     def mqtt_client
-      @client ||= MQTT::Client.new(interpolated['uri'])
-
-      if interpolated['ssl']
-        @client.ssl = interpolated['ssl'].to_sym
-        @client.ca_file = interpolated['ca_file']
-        @client.cert_file = interpolated['cert_file']
-        @client.key_file = interpolated['key_file']
+      @client ||= begin
+        MQTT::Client.new(interpolated['uri']).tap do |c|
+          if interpolated['ssl']
+            c.ssl = interpolated['ssl'].to_sym
+            c.ca_file = interpolated['ca_file']
+            c.cert_file = interpolated['cert_file']
+            c.key_file = interpolated['key_file']
+          end
+        end
       end
-
-      @client
     end
 
     def receive(incoming_events)
@@ -117,35 +117,36 @@ module Agents
 
     def check
       last_message = memory['last_message']
+      mqtt_client.connect
 
-      mqtt_client.connect do |c|
-        begin
-          Timeout.timeout((interpolated['max_read_time'].presence || 15).to_i) {
-            c.get_packet(interpolated['topic']) do |packet|
-              topic, payload = message = [packet.topic, packet.payload]
-
-              # Ignore a message if it is previously received
-              next if (packet.retain || packet.duplicate) && message == last_message
-
-              last_message = message
-
-              # A lot of services generate JSON, so try that.
-              begin
-                payload = JSON.parse(payload)
-              rescue
-              end
-
-              create_event payload: {
-                'topic' => topic,
-                'message' => payload,
-                'time' => Time.now.to_i
-              }
-            end
+      poll_thread = Thread.new do
+        mqtt_client.get_packet(interpolated['topic']) do |packet|
+          topic, payload = message = [packet.topic, packet.payload]
+
+          # Ignore a message if it is previously received
+          next if (packet.retain || packet.duplicate) && message == last_message
+
+          last_message = message
+
+          # A lot of services generate JSON, so try that.
+          begin
+            payload = JSON.parse(payload)
+          rescue
+          end
+
+          create_event payload: {
+            'topic' => topic,
+            'message' => payload,
+            'time' => Time.now.to_i
           }
-        rescue Timeout::Error
         end
       end
 
+      sleep (interpolated['max_read_time'].presence || 15).to_f
+
+      mqtt_client.disconnect
+      poll_thread.kill
+
       # Remember the last original (non-retain, non-duplicate) message
       self.memory['last_message'] = last_message
       save!

app/models/agents/phantom_js_cloud_agent.rb

@@ -26,8 +26,8 @@ module Agents
       * `Api key` - PhantomJs Cloud API Key credential stored in Huginn
       * `Url` - The url to render
       * `Mode` - Create a new `clean` event or `merge` old payload with new values (default: `clean`)
-      * `Render type` - Render as html or plain text without html tags (default: `html`)
-      * `Output as json` - Return the page conents and metadata as a JSON object (default: `false`)
+      * `Render type` - Render as html, plain text without html tags, or jpg as screenshot of the page (default: `html`)
+      * `Output as json` - Return the page contents and metadata as a JSON object (default: `false`)
       * `Ignore images` - Skip loading of inlined images (default: `false`)
       * `Url agent` - A custom User-Agent name (default: `#{default_user_agent}`)
       * `Wait interval` - Milliseconds to delay rendering after the last resource is finished loading.
@@ -59,7 +59,7 @@ module Agents
     form_configurable :mode, type: :array, values: ['clean', 'merge']
     form_configurable :api_key, roles: :completable
     form_configurable :url
-    form_configurable :render_type, type: :array, values: ['html', 'plainText']
+    form_configurable :render_type, type: :array, values: ['html', 'plainText', 'jpg']
     form_configurable :output_as_json, type: :boolean
     form_configurable :ignore_images, type: :boolean
     form_configurable :user_agent, type: :text

app/models/agents/post_agent.rb

@@ -1,5 +1,6 @@
 module Agents
   class PostAgent < Agent
+    include EventHeadersConcern
     include WebRequestConcern
     include FileHandling
 
@@ -33,6 +34,8 @@ module Agents
 
         If `output_mode` is set to `merge`, the emitted Event will be merged into the original contents of the received Event.
 
+        Set `event_headers` to a list of header names, either in an array of string or in a comma-separated string, to include only some of the header values.
+
         Set `event_headers_style` to one of the following values to normalize the keys of "headers" for downstream agents' convenience:
 
           * `capitalized` (default) - Header names are capitalized; e.g. "Content-Type"
@@ -125,11 +128,7 @@ module Agents
         errors.add(:base, "if provided, emit_events must be true or false")
       end
 
-      begin
-        normalize_response_headers({})
-      rescue ArgumentError => e
-        errors.add(:base, e.message)
-      end
+      validate_event_headers_options!
 
       unless %w[post get put delete patch].include?(method)
         errors.add(:base, "method must be 'post', 'get', 'put', 'delete', or 'patch'")
@@ -169,29 +168,6 @@ module Agents
 
     private
 
-    def normalize_response_headers(headers)
-      case interpolated['event_headers_style']
-      when nil, '', 'capitalized'
-        normalize = ->name {
-          name.gsub(/(?:\A|(?<=-))([[:alpha:]])|([[:alpha:]]+)/) {
-            $1 ? $1.upcase : $2.downcase
-          }
-        }
-      when 'downcased'
-        normalize = :downcase.to_proc
-      when 'snakecased', nil
-        normalize = ->name { name.tr('A-Z-', 'a-z_') }
-      when 'raw'
-        normalize = ->name { name }  # :itself.to_proc in Ruby >= 2.2
-      else
-        raise ArgumentError, "if provided, event_headers_style must be 'capitalized', 'downcased', 'snakecased' or 'raw'"
-      end
-
-      headers.each_with_object({}) { |(key, value), hash|
-        hash[normalize[key]] = value
-      }
-    end
-
     def handle(data, event = Event.new, headers)
       url = interpolated(event.payload)[:post_url]
 
@@ -234,10 +210,15 @@ module Agents
         new_event = interpolated['output_mode'].to_s == 'merge' ? event.payload.dup : {}
         create_event payload: new_event.merge(
           body: response.body,
-          headers: normalize_response_headers(response.headers),
           status: response.status
+        ).merge(
+          event_headers_payload(response.headers)
         )
       end
     end
+
+    def event_headers_key
+      super || 'headers'
+    end
   end
 end

app/models/agents/pushover_agent.rb

@@ -1,5 +1,6 @@
 module Agents
   class PushoverAgent < Agent
+    can_dry_run!
     cannot_be_scheduled!
     cannot_create_events!
     no_bulk_receive!

app/models/agents/rss_agent.rb

@@ -6,7 +6,7 @@ module Agents
     can_dry_run!
     default_schedule "every_1d"
 
-    gem_dependency_check { defined?(Feedjira::Feed) }
+    gem_dependency_check { defined?(Feedjira) }
 
     DEFAULT_EVENTS_ORDER = [['{{date_published}}', 'time'], ['{{last_updated}}', 'time']]
 
@@ -164,7 +164,7 @@ module Agents
         begin
           response = faraday.get(url)
           if response.success?
-            feed = Feedjira::Feed.parse(preprocessed_body(response))
+            feed = Feedjira.parse(preprocessed_body(response))
             new_events.concat feed_to_events(feed)
           else
             error "Failed to fetch #{url}: #{response.inspect}"
@@ -275,7 +275,7 @@ module Agents
       {
         id: entry.id,
         url: entry.url,
-        urls: entry.links.map(&:href),
+        urls: Array(entry.url) | entry.links.map(&:href),
         links: entry.links,
         title: entry.title,
         description: clean_fragment(entry.summary),

app/models/agents/scheduler_agent.rb

@@ -24,6 +24,7 @@ module Agents
       * `disable`: Target Agents are disabled (if not) at intervals.
 
       * `enable`: Target Agents are enabled (if not) at intervals.
+        * If the option `drop_pending_events` is set to `true`, pending events will be cleared before the agent is enabled.
 
       # Targets
 

app/models/agents/slack_agent.rb

@@ -1,8 +1,9 @@
 module Agents
   class SlackAgent < Agent
     DEFAULT_USERNAME = 'Huginn'
-    ALLOWED_PARAMS = ['channel', 'username', 'unfurl_links', 'attachments']
+    ALLOWED_PARAMS = ['channel', 'username', 'unfurl_links', 'attachments', 'blocks']
 
+    can_dry_run!
     cannot_be_scheduled!
     cannot_create_events!
     no_bulk_receive!

app/models/agents/telegram_agent.rb

@@ -31,7 +31,7 @@ module Agents
 
       **Options**
 
-      * `caption`: caption for a media content (0-200 characters)
+      * `caption`: caption for a media content (0-1024 characters)
       * `disable_notification`: send a message silently in a channel
       * `disable_web_page_preview`: disable link previews for links in a text message
       * `long_message`: truncate (default) or split text messages and captions that exceed Telegram API limits. Markdown and HTML tags can't span across messages and, if not opened or closed properly, will render as plain text.
@@ -68,7 +68,7 @@ module Agents
     def validate_options
       errors.add(:base, 'auth_token is required') unless options['auth_token'].present?
       errors.add(:base, 'chat_id is required') unless options['chat_id'].present?
-      errors.add(:base, 'caption should be 200 characters ol less') if interpolated['caption'].present? && interpolated['caption'].length > 200 && (!interpolated['long_message'].present? || interpolated['long_message'] != 'split')
+      errors.add(:base, 'caption should be 1024 characters or less') if interpolated['caption'].present? && interpolated['caption'].length > 1024 && (!interpolated['long_message'].present? || interpolated['long_message'] != 'split')
       errors.add(:base, "disable_notification has invalid value: should be 'true' or 'false'") if interpolated['disable_notification'].present? && !%w(true false).include?(interpolated['disable_notification'])
       errors.add(:base, "disable_web_page_preview has invalid value: should be 'true' or 'false'") if interpolated['disable_web_page_preview'].present? && !%w(true false).include?(interpolated['disable_web_page_preview'])
       errors.add(:base, "long_message has invalid value: should be 'split' or 'truncate'") if interpolated['long_message'].present? && !%w(split truncate).include?(interpolated['long_message'])
@@ -130,19 +130,20 @@ module Agents
     def send_telegram_messages(field, params)
       if interpolated['long_message'] == 'split'
         if field == :text
-          params[:text].scan(/\G(?:\w{4096}|.{1,4096}(?=\b|\z))/m) do |message|
-            send_message field, configure_params(field => message.strip) unless message.strip.blank?
+          params[:text].scan(/\G\s*(?:\w{4096}|.{1,4096}(?=\b|\z))/m) do |message|
+            message.strip!
+            send_message field, configure_params(field => message) unless message.blank?
           end
         else
-          caption_array = params[:caption].scan(/\G(?:\w{200}|.{1,200}(?=\b|\z))/m)
-          params[:caption] = caption_array.first.strip
+          caption_array = (params[:caption].presence || '').scan(/\G\s*\K(?:\w{1024}|.{1,1024}(?=\b|\z))/m).map(&:strip)
+          params[:caption] = caption_array.shift
           send_message field, params
-          caption_array.drop(1).each do |caption|
-            send_message(:text, configure_params(text: caption.strip)) unless caption.strip.blank?
-            end
+          caption_array.each do |caption|
+            send_message(:text, configure_params(text: caption)) unless caption.blank?
+          end
         end
       else
-        params[:caption] = params[:caption][0..199] if params[:caption]
+        params[:caption] = params[:caption][0..1023] if params[:caption]
         params[:text] = params[:text][0..4095] if params[:text]
         send_message field, params
       end

app/models/agents/trigger_agent.rb

@@ -8,10 +8,18 @@ module Agents
     description <<-MD
       The Trigger Agent will watch for a specific value in an Event payload.
 
-      The `rules` array contains hashes of `path`, `value`, and `type`.  The `path` value is a dotted path through a hash in [JSONPaths](http://goessner.net/articles/JsonPath/) syntax. For simple events, this is usually just the name of the field you want, like 'text' for the text key of the event.
+      The `rules` array contains a mixture of strings and hashes.
+
+      A string rule is a Liquid template and counts as a match when it expands to `true`.
+
+      A hash rule consists of the following keys: `path`, `value`, and `type`.
+
+      The `path` value is a dotted path through a hash in [JSONPaths](http://goessner.net/articles/JsonPath/) syntax. For simple events, this is usually just the name of the field you want, like 'text' for the text key of the event.
 
       The `type` can be one of #{VALID_COMPARISON_TYPES.map { |t| "`#{t}`" }.to_sentence} and compares with the `value`.  Note that regex patterns are matched case insensitively.  If you want case sensitive matching, prefix your pattern with `(?-i)`.
 
+      In any `type` including regex Liquid variables can be used normally. To search for just a word matching the concatenation of `foo` and variable `bar` would use `value` of `foo{{bar}}`. Note that note that starting/ending delimiters like `/` or `|` are not required for regex.
+
       The `value` can be a single value or an array of values. In the case of an array, all items must be strings, and if one or more values match, then the rule matches. Note: avoid using `field!=value` with arrays, you should use `not in` instead.
 
       By default, all rules must match for the Agent to trigger. You can switch this so that only one rule must match by
@@ -30,9 +38,22 @@ module Agents
           { "message": "Your message" }
     MD
 
+    private def valid_rule?(rule)
+      case rule
+      when String
+        true
+      when Hash
+        rule.values_at('type', 'value', 'path').all?(&:present?) &&
+          VALID_COMPARISON_TYPES.include?(rule['type'])
+      else
+        false
+      end
+    end
+
     def validate_options
-      unless options['expected_receive_period_in_days'].present? && options['rules'].present? &&
-             options['rules'].all? { |rule| rule['type'].present? && VALID_COMPARISON_TYPES.include?(rule['type']) && rule['value'].present? && rule['path'].present? }
+      unless options['expected_receive_period_in_days'].present? &&
+             options['rules'].present? &&
+             options['rules'].all? { |rule| valid_rule?(rule) }
         errors.add(:base, "expected_receive_period_in_days, message, and rules, with a type, value, and path for every rule, are required")
       end
 
@@ -72,6 +93,10 @@ module Agents
         opts = interpolated(event)
 
         match_results = opts['rules'].map do |rule|
+          if rule.is_a?(String)
+            next boolify(rule)
+          end
+
           value_at_path = Utils.value_at(event['payload'], rule['path'])
           rule_values = rule['value']
           rule_values = [rule_values] unless rule_values.is_a?(Array)

app/models/agents/twilio_agent.rb

@@ -54,7 +54,7 @@ module Agents
           end
 
           if boolify(interpolated['receive_text'])
-            message = message.slice 0..160
+            message = message.slice 0..1600
             send_message message
           end
         end

app/models/agents/twitter_user_agent.rb

@@ -28,7 +28,7 @@ module Agents
       Events are the raw JSON provided by the [Twitter API](https://dev.twitter.com/docs/api/1.1/get/statuses/user_timeline). Should look something like:
           {
              ... every Tweet field, including ...
-            "text": "something",
+            "full_text": "something",
             "user": {
               "name": "Mr. Someone",
               "screen_name": "Someone",

app/models/agents/weather_agent.rb

@@ -5,29 +5,23 @@ module Agents
   class WeatherAgent < Agent
     cannot_receive_events!
 
-    gem_dependency_check { defined?(Wunderground) && defined?(ForecastIO) }
+    gem_dependency_check { defined?(ForecastIO) }
 
     description <<-MD
       The Weather Agent creates an event for the day's weather at a given `location`.
 
-      #{'## Include `forecast_io` and `wunderground` in your Gemfile to use this Agent!' if dependencies_missing?}
+      #{'## Include `forecast_io` in your Gemfile to use this Agent!' if dependencies_missing?}
 
       You also must select when you would like to get the weather forecast for using the `which_day` option, where the number 1 represents today, 2 represents tomorrow and so on. Weather forecast inforation is only returned for at most one week at a time.
 
-      The weather forecast information can be provided by either Wunderground or Dark Sky. To choose which `service` to use, enter either `darksky` or `wunderground`.
+      The weather forecast information is provided by Dark Sky. 
 
-      The `location` should be:
+      The `location` must be a comma-separated string of map co-ordinates (longitude, latitude). For example, San Francisco would be `37.7771,-122.4196`.
 
-      * For Wunderground: A US zipcode, or any location that Wunderground supports. To find one, search [wunderground.com](https://wunderground.com) and copy the location part of the URL.  For example, a result for San Francisco gives `https://www.wunderground.com/US/CA/San_Francisco.html` and London, England gives `https://www.wunderground.com/q/zmw:00000.1.03772`.  The locations in each are `US/CA/San_Francisco` and `zmw:00000.1.03772`, respectively.
-      * For Dark Sky: `location` must be a comma-separated string of map co-ordinates (longitude, latitude). For example, San Francisco would be `37.7771,-122.4196`.
-
-      You must set up an [API key for Wunderground](https://www.wunderground.com/weather/api/) in order to use this Agent with Wunderground.
-
-      You must set up an [API key for Dark Sky](https://darksky.net/dev/) in order to use this Agent with Dark Sky.
+      You must set up an [API key for Dark Sky](https://darksky.net/dev/) in order to use this Agent.
 
       Set `expected_update_period_in_days` to the maximum amount of time that you'd expect to pass between Events being created by this Agent.
 
-      If you want to see the returned texts in your language, set the `language` parameter in ISO 639-1 format.
     MD
 
     event_description <<-MD
@@ -67,27 +61,22 @@ module Agents
 
     def default_options
       {
-        'service' => 'wunderground',
         'api_key' => 'your-key',
-        'location' => '94103',
+        'location' => '37.779329,-122.41915',
         'which_day' => '1',
-        'language' => 'EN',
-        'expected_update_period_in_days' => '2'
+        'expected_update_period_in_days' => '2',
+        'language' => 'en'
       }
     end
 
     def check
       if key_setup?
-        create_event :payload => model(weather_provider, which_day).merge('location' => location)
+        create_event :payload => model(which_day).merge('location' => location)
       end
     end
 
     private
 
-    def weather_provider
-      interpolated["service"].presence || "wunderground"
-    end
-
     def which_day
       (interpolated["which_day"].presence || 1).to_i
     end
@@ -96,103 +85,115 @@ module Agents
       interpolated["location"].presence || interpolated["zipcode"]
     end
 
+    def coordinates
+      location.split(',').map { |e| e.to_f }
+    end
+
     def language
-      interpolated['language'].presence || 'EN'
+      interpolated["language"].presence || "en"
     end
 
-    def validate_options
-      errors.add(:base, "service must be set to 'darksky' or 'wunderground'") unless %w[darksky forecastio wunderground].include?(weather_provider)
-      errors.add(:base, "location is required") unless location.present?
-      errors.add(:base, "api_key is required") unless interpolated['api_key'].present?
-      errors.add(:base, "which_day selection is required") unless which_day.present?
+    def wunderground? 
+      interpolated["service"].presence && interpolated["service"].presence.downcase == "wunderground"
     end
 
-    def wunderground
-      if key_setup?
-        forecast = Wunderground.new(interpolated['api_key'], language: language.upcase).forecast_for(location)
-        merged = {}
-        forecast['forecast']['simpleforecast']['forecastday'].each { |daily| merged[daily['period']] = daily }
-        forecast['forecast']['txt_forecast']['forecastday'].each { |daily| (merged[daily['period']] || {}).merge!(daily) }
-        merged
+    VALID_COORDS_REGEX = /^\s*-?\d{1,3}\.\d+\s*,\s*-?\d{1,3}\.\d+\s*$/
+
+    def validate_location
+      errors.add(:base, "location is required") unless location.present?
+      if location =~ VALID_COORDS_REGEX
+        lat, lon = coordinates
+        errors.add :base, "too low of a latitude" unless lat > -90
+        errors.add :base, "too big of a latitude" unless lat < 90
+        errors.add :base, "too low of a longitude" unless lon > -180
+        errors.add :base, "too high of a longitude" unless lon < 180
+      else
+        errors.add(
+          :base,
+          "Location #{location} is malformed. Location for " +
+          'Dark Sky must be in the format "-00.000,-00.00000". The ' +
+          "number of decimal places does not matter.")
       end
     end
 
+    def validate_options
+      errors.add(:base, "The Weather Underground API has been disabled since Jan 1st 2018, please switch to DarkSky") if wunderground?
+      validate_location
+      errors.add(:base, "api_key is required") unless interpolated['api_key'].present?
+      errors.add(:base, "which_day selection is required") unless which_day.present?
+    end
+
     def dark_sky
       if key_setup?
         ForecastIO.api_key = interpolated['api_key']
-        lat, lng = location.split(',')
+        lat, lng = coordinates
         ForecastIO.forecast(lat, lng, params: {lang: language.downcase})['daily']['data']
       end
     end
 
-    def model(weather_provider,which_day)
-      if weather_provider == "wunderground"
-        wunderground[which_day]
-      elsif weather_provider == "darksky" || weather_provider == "forecastio"
-        dark_sky.each do |value|
-          timestamp = Time.at(value.time)
-          if (timestamp.to_date - Time.now.to_date).to_i == which_day
-            day = {
-              'date' => {
-                'epoch' => value.time.to_s,
-                'pretty' => timestamp.strftime("%l:%M %p %Z on %B %d, %Y"),
-                'day' => timestamp.day,
-                'month' => timestamp.month,
-                'year' => timestamp.year,
-                'yday' => timestamp.yday,
-                'hour' => timestamp.hour,
-                'min' => timestamp.strftime("%M"),
-                'sec' => timestamp.sec,
-                'isdst' => timestamp.isdst ? 1 : 0 ,
-                'monthname' => timestamp.strftime("%B"),
-                'monthname_short' => timestamp.strftime("%b"),
-                'weekday_short' => timestamp.strftime("%a"),
-                'weekday' => timestamp.strftime("%A"),
-                'ampm' => timestamp.strftime("%p"),
-                'tz_short' => timestamp.zone
-              },
-              'period' => which_day.to_i,
-              'high' => {
-                'fahrenheit' => value.temperatureMax.round().to_s,
-                'epoch' => value.temperatureMaxTime.to_s,
-                'fahrenheit_apparent' => value.apparentTemperatureMax.round().to_s,
-                'epoch_apparent' => value.apparentTemperatureMaxTime.to_s,
-                'celsius' => ((5*(Float(value.temperatureMax) - 32))/9).round().to_s
-              },
-              'low' => {
-                'fahrenheit' => value.temperatureMin.round().to_s,
-                'epoch' => value.temperatureMinTime.to_s,
-                'fahrenheit_apparent' => value.apparentTemperatureMin.round().to_s,
-                'epoch_apparent' => value.apparentTemperatureMinTime.to_s,
-                'celsius' => ((5*(Float(value.temperatureMin) - 32))/9).round().to_s
-              },
-              'conditions' => value.summary,
-              'icon' => value.icon,
-              'avehumidity' => (value.humidity * 100).to_i,
-              'sunriseTime' => value.sunriseTime.to_s,
-              'sunsetTime' => value.sunsetTime.to_s,
-              'moonPhase' => value.moonPhase.to_s,
-              'precip' => {
-                'intensity' => value.precipIntensity.to_s,
-                'intensity_max' => value.precipIntensityMax.to_s,
-                'intensity_max_epoch' => value.precipIntensityMaxTime.to_s,
-                'probability' => value.precipProbability.to_s,
-                'type' => value.precipType
-              },
-              'dewPoint' => value.dewPoint.to_s,
-              'avewind' => {
-                'mph' => value.windSpeed.round().to_s,
-                'kph' =>  (Float(value.windSpeed) * 1.609344).round().to_s,
-                'degrees' => value.windBearing.to_s
-              },
-              'visibility' => value.visibility.to_s,
-              'cloudCover' => value.cloudCover.to_s,
-              'pressure' => value.pressure.to_s,
-              'ozone' => value.ozone.to_s
-            }
-            return day
-          end
-        end
+    def model(which_day)
+      value = dark_sky[which_day - 1]
+      if value
+        timestamp = Time.at(value.time)
+        day = {
+          'date' => {
+            'epoch' => value.time.to_s,
+            'pretty' => timestamp.strftime("%l:%M %p %Z on %B %d, %Y"),
+            'day' => timestamp.day,
+            'month' => timestamp.month,
+            'year' => timestamp.year,
+            'yday' => timestamp.yday,
+            'hour' => timestamp.hour,
+            'min' => timestamp.strftime("%M"),
+            'sec' => timestamp.sec,
+            'isdst' => timestamp.isdst ? 1 : 0 ,
+            'monthname' => timestamp.strftime("%B"),
+            'monthname_short' => timestamp.strftime("%b"),
+            'weekday_short' => timestamp.strftime("%a"),
+            'weekday' => timestamp.strftime("%A"),
+            'ampm' => timestamp.strftime("%p"),
+            'tz_short' => timestamp.zone
+          },
+          'period' => which_day.to_i,
+          'high' => {
+            'fahrenheit' => value.temperatureMax.round().to_s,
+            'epoch' => value.temperatureMaxTime.to_s,
+            'fahrenheit_apparent' => value.apparentTemperatureMax.round().to_s,
+            'epoch_apparent' => value.apparentTemperatureMaxTime.to_s,
+            'celsius' => ((5*(Float(value.temperatureMax) - 32))/9).round().to_s
+          },
+          'low' => {
+            'fahrenheit' => value.temperatureMin.round().to_s,
+            'epoch' => value.temperatureMinTime.to_s,
+            'fahrenheit_apparent' => value.apparentTemperatureMin.round().to_s,
+            'epoch_apparent' => value.apparentTemperatureMinTime.to_s,
+            'celsius' => ((5*(Float(value.temperatureMin) - 32))/9).round().to_s
+          },
+          'conditions' => value.summary,
+          'icon' => value.icon,
+          'avehumidity' => (value.humidity * 100).to_i,
+          'sunriseTime' => value.sunriseTime.to_s,
+          'sunsetTime' => value.sunsetTime.to_s,
+          'moonPhase' => value.moonPhase.to_s,
+          'precip' => {
+            'intensity' => value.precipIntensity.to_s,
+            'intensity_max' => value.precipIntensityMax.to_s,
+            'intensity_max_epoch' => value.precipIntensityMaxTime.to_s,
+            'probability' => value.precipProbability.to_s,
+            'type' => value.precipType
+          },
+          'dewPoint' => value.dewPoint.to_s,
+          'avewind' => {
+            'mph' => value.windSpeed.round().to_s,
+            'kph' =>  (Float(value.windSpeed) * 1.609344).round().to_s,
+            'degrees' => value.windBearing.to_s
+          },
+          'visibility' => value.visibility.to_s,
+          'cloudCover' => value.cloudCover.to_s,
+          'pressure' => value.pressure.to_s,
+          'ozone' => value.ozone.to_s
+        }
+        return day
       end
     end
   end

app/models/agents/webhook_agent.rb

@@ -1,6 +1,7 @@
 module Agents
   class WebhookAgent < Agent
-    include WebRequestConcern
+    include EventHeadersConcern
+    include WebRequestConcern  # to make reCAPTCHA verification requests
 
     cannot_be_scheduled!
     cannot_receive_events!
@@ -22,6 +23,8 @@ module Agents
         * `payload_path` - JSONPath of the attribute in the POST body to be
           used as the Event payload.  Set to `.` to return the entire message.
           If `payload_path` points to an array, Events will be created for each element.
+        * `event_headers` - Comma-separated list of HTTP headers your agent will include in the payload.
+        * `event_headers_key` - The key to use to store all the headers received
         * `verbs` - Comma-separated list of http verbs your agent will accept.
           For example, "post,get" will enable POST and GET requests. Defaults
           to "post".
@@ -30,6 +33,7 @@ module Agents
         * `code` - The response code to the request. Defaults to '201'. If the code is '301' or '302' the request will automatically be redirected to the url defined in "response".
         * `recaptcha_secret` - Setting this to a reCAPTCHA "secret" key makes your agent verify incoming requests with reCAPTCHA.  Don't forget to embed a reCAPTCHA snippet including your "site" key in the originating form(s).
         * `recaptcha_send_remote_addr` - Set this to true if your server is properly configured to set REMOTE_ADDR to the IP address of each visitor (instead of that of a proxy server).
+        * `score_threshold` - Setting this when using reCAPTCHA v3 to define the treshold when a submission is verified. Defaults to 0.5
       MD
     end
 
@@ -43,15 +47,32 @@ module Agents
     def default_options
       { "secret" => "supersecretstring",
         "expected_receive_period_in_days" => 1,
-        "payload_path" => "some_key"
+        "payload_path" => "some_key",
+        "event_headers" => "",
+        "event_headers_key" => "headers",
+        "score_threshold" => 0.5
       }
     end
 
-    def receive_web_request(params, method, format)
+    def receive_web_request(request)
       # check the secret
-      secret = params.delete('secret')
+      secret = request.path_parameters[:secret]
       return ["Not Authorized", 401] unless secret == interpolated['secret']
 
+      params = request.query_parameters.dup
+      begin
+        params.update(request.request_parameters)
+      rescue EOFError
+      end
+
+      method = request.method_symbol.to_s
+      headers = request.headers.each_with_object({}) { |(name, value), hash|
+        case name
+        when /\AHTTP_([A-Z0-9_]+)\z/
+          hash[$1.tr('_', '-').gsub(/[^-]+/, &:capitalize)] = value
+        end
+      }
+
       # check the verbs
       verbs = (interpolated['verbs'] || 'post').split(/,/).map { |x| x.strip.downcase }.select { |x| x.present? }
       return ["Please use #{verbs.join('/').upcase} requests only", 401] unless verbs.include?(method)
@@ -81,12 +102,18 @@ module Agents
           return ["Not Authorized", 401]
         end
 
-        JSON.parse(response.body)['success'] or
-          return ["Not Authorized", 401]
+        body = JSON.parse(response.body)
+        if interpolated['score_threshold'].present? && body['score'].present?
+          body['score'] > interpolated['score_threshold'].to_f or
+            return ["Not Authorized", 401]
+        else
+          body['success'] or
+            return ["Not Authorized", 401]
+        end
       end
 
       [payload_for(params)].flatten.each do |payload|
-        create_event(payload: payload)
+        create_event(payload: payload.merge(event_headers_payload(headers)))
       end
 
       if interpolated['response_headers'].presence
@@ -112,6 +139,8 @@ module Agents
       if options['code'].to_s.in?(['301', '302']) && !options['response'].present?
         errors.add(:base, "Must specify a url for request redirect")
       end
+
+      validate_event_headers_options!
     end
 
     def payload_for(params)

app/models/agents/website_agent.rb

@@ -462,26 +462,24 @@ module Agents
     end
 
     def receive(incoming_events)
-      incoming_events.each do |event|
-        interpolate_with(event) do
-          existing_payload = interpolated['mode'].to_s == "merge" ? event.payload : {}
-
-          if data_from_event = options['data_from_event'].presence
-            data = interpolate_options(data_from_event)
-            if data.present?
-              handle_event_data(data, event, existing_payload)
-            else
-              error "No data was found in the Event payload using the template #{data_from_event}", inbound_event: event
-            end
+      interpolate_with_each(incoming_events) do |event|
+        existing_payload = interpolated['mode'].to_s == "merge" ? event.payload : {}
+
+        if data_from_event = options['data_from_event'].presence
+          data = interpolate_options(data_from_event)
+          if data.present?
+            handle_event_data(data, event, existing_payload)
           else
-            url_to_scrape =
-              if url_template = options['url_from_event'].presence
-                interpolate_options(url_template)
-              else
-                interpolated['url']
-              end
-            check_urls(url_to_scrape, existing_payload)
+            error "No data was found in the Event payload using the template #{data_from_event}", inbound_event: event
           end
+        else
+          url_to_scrape =
+            if url_template = options['url_from_event'].presence
+              interpolate_options(url_template)
+            else
+              interpolated['url']
+            end
+          check_urls(url_to_scrape, existing_payload)
         end
       end
     end
@@ -500,7 +498,7 @@ module Agents
         handle_data(data, event.payload['url'].presence, existing_payload)
       }
     rescue => e
-      error "Error when handling event data: #{e.message}\n#{e.backtrace.join("\n")}", inbound_event: event
+      error "Error when handling event data: #{e.message}\n#{e.backtrace.join("\n")}"
     end
 
     # This method returns true if the result should be stored as a new event.
@@ -660,12 +658,6 @@ module Agents
       end
     end
 
-    def is_positive_integer?(value)
-      Integer(value) >= 0
-    rescue
-      false
-    end
-
     class UnevenSizeError < ArgumentError
     end
 

app/models/event.rb

@@ -4,7 +4,7 @@ require 'location'
 # Agents.  They contain a serialized `payload` of arbitrary JSON data, as well as optional `lat`, `lng`, and `expires_at`
 # fields.
 class Event < ActiveRecord::Base
-  include JSONSerializedField
+  include JsonSerializedField
   include LiquidDroppable
 
   acts_as_mappable

app/views/admin/users/_form.html.erb

@@ -1,5 +1,5 @@
 <%= form_for([:admin, @user], html: { class: 'form-horizontal' }) do |f| %>
-  <%= devise_error_messages! %>
+  <%= render "devise/shared/error_messages", resource: resource %>
   <%= render partial: '/devise/registrations/common_registration_fields', locals: { f: f } %>
 
   <div class="form-group">

app/views/agents/_action_menu.html.erb

@@ -26,7 +26,7 @@
   </li>
 
   <li>
-    <%= link_to '#', 'data-toggle' => 'modal', 'data-target' => "#confirm-agent#{agent.id}" do %>
+    <%= link_to '#', 'data-toggle' => 'modal', 'data-target' => "#confirm-agent-enable-disable#{agent.id}" do %>
       <% if agent.disabled? %>
         <%= icon_tag('glyphicon-play') %> Enable agent
       <% else %>
@@ -45,6 +45,16 @@
     <% end %>
   <% end %>
 
+  <% if agent.can_create_events? && agent.events_count > 0 %>
+    <li class="divider"></li>
+
+    <li>
+      <%= link_to '#', 'data-toggle' => 'modal', 'data-target' => "#confirm-agent-reemit-events#{agent.id}" do %>
+        <%= icon_tag('glyphicon-refresh') %> Re-emit all events
+      <% end %>
+    </li>
+  <% end %>
+
   <li class="divider"></li>
 
   <% if agent.can_create_events? && agent.events_count > 0 %>
@@ -58,7 +68,7 @@
   </li>
 </ul>
 
-<div id="confirm-agent<%= agent.id %>" class="confirm-agent modal fade" tabindex="-1" role="dialog" aria-labelledby="confirmAgentLabel" aria-hidden="true">
+<div id="confirm-agent-enable-disable<%= agent.id %>" class="confirm-agent modal fade" tabindex="-1" role="dialog" aria-labelledby="confirmAgentLabel" aria-hidden="true">
   <div class="modal-dialog modal-sm">
     <div class="modal-content">
       <div class="modal-header">
@@ -86,3 +96,29 @@
   </div>
 </div>
 
+<div id="confirm-agent-reemit-events<%= agent.id %>" class="confirm-agent modal fade" tabindex="-1" role="dialog" aria-labelledby="confirmAgentLabel" aria-hidden="true">
+  <div class="modal-dialog modal-sm">
+    <div class="modal-content">
+      <div class="modal-header">
+        <button type="button" class="close" data-dismiss="modal"><span aria-hidden="true">&times;</span><span class="sr-only">Close</span></button>
+        <h4 class="modal-title">Confirm</h4>
+      </div>
+      <div class="modal-body">
+        <p>Re-emit all events for &quot;<%= agent.name %>&quot;?</p>
+      </div>
+      <div class="modal-footer">
+        <%= form_for(agent, as: :agent, url: reemit_events_agent_path(agent, return: return_to), method: 'POST') do |f| %>
+          <% if agent.can_create_events? && agent.events_count > 0 %>
+            <div class="form-group">
+              <%= check_box_tag :delete_old_events %>
+              <%= label_tag :delete_old_events, 'Delete old events' %>
+              <span class="glyphicon glyphicon-question-sign hover-help" data-content="If this is checked, the original copies of the events will be deleted once they are re-emitted."></span>
+            </div>
+          <% end %>
+          <%= f.button 'Cancel', class: 'btn btn-default', 'data-dismiss' => 'modal' %>
+          <%= f.submit 'Re-emit all events', class: 'btn btn-primary' %>
+        <% end %>
+      </div>
+    </div>
+  </div>
+</div>

app/views/agents/agent_views/manual_event_agent/_show.html.erb

@@ -16,7 +16,7 @@
 <%= form_tag handle_details_post_agent_path(@agent), :id => "create-event-form" do %>
   <div class="form-group">
     <textarea rows="10" id="payload" name="payload" class="payload-editor" data-height="200">
-      {}
+      <%= (@agent.options["payload"].presence || {}).to_json %>
     </textarea>
   </div>
 
@@ -55,4 +55,4 @@
       });
     });
   });
-</script>
+</script>

app/views/application/_upgrade_warning.html.erb

@@ -45,20 +45,3 @@ TWITTER_OAUTH_SECRET=<%= @twitter_oauth_secret %>
     </div>
   </dir>
 <% end -%>
-<% if @outdated_docker_image_namespace %>
-  <dir class="container">
-    <div class="alert alert-danger" role="alert">
-      <p>
-        <b>Warning!</b> You need to change the namespace of the Huginn Docker image you are using.
-      </p>
-      <br/>
-      <p>
-        Since Huginn moved to a GitHub organization we also changed the namespace of the Docker images. Please change your configuration to the new namespace:
-        <ul>
-          <li><code>cantino/huginn</code> is now <code>huginn/huginn</code></li>
-          <li><code>cantino/huginn-single-process</code> is now <code>huginn/huginn-single-process</code></li>
-        </ul>
-      </p>
-    </div>
-  </dir>
-<% end -%>

app/views/devise/confirmations/new.html.erb

@@ -6,7 +6,7 @@
       <h2>Resend confirmation instructions</h2>
 
       <%= form_for(resource, as: resource_name, url: confirmation_path(resource_name), html: { method: :post, class: 'form-horizontal' }) do |f| %>
-        <%= devise_error_messages! %>
+        <%= render "devise/shared/error_messages", resource: resource %>
 
         <div class="form-group">
           <%= f.label :login, class: 'col-md-2 col-md-offset-2 control-label' %>

app/views/devise/passwords/edit.html.erb

@@ -7,7 +7,7 @@
         <h2>Change your password</h2>
 
         <%= form_for(resource, as: resource_name, url: password_path(resource_name), html: { method: :put, class: 'form-horizontal' }) do |f| %>
-          <%= devise_error_messages! %>
+          <%= render "devise/shared/error_messages", resource: resource %>
           <%= f.hidden_field :reset_password_token %>
 
           <div class="control-group">

app/views/devise/passwords/new.html.erb

@@ -6,7 +6,7 @@
       <h2>Forgot your password?</h2>
 
       <%= form_for(resource, as: resource_name, url: password_path(resource_name), html: { method: :post, class: 'form-horizontal' }) do |f| %>
-        <%= devise_error_messages! %>
+        <%= render "devise/shared/error_messages", resource: resource %>
 
         <div class="form-group">
           <%= f.label :login, :class => 'col-md-2 col-md-offset-2 control-label' %>

app/views/devise/registrations/edit.html.erb

@@ -8,7 +8,7 @@
         <h2>Edit <%= resource_name.to_s.humanize %></h2>
 
         <%= form_for(resource, as: resource_name, url: registration_path(resource_name), html: { method: :put, class: 'form-horizontal' }) do |f| %>
-          <%= devise_error_messages! %>
+          <%= render "devise/shared/error_messages", resource: resource %>
 
           <div class="form-group">
             <%= f.label :email, class: 'col-md-4 control-label' %>
@@ -45,7 +45,7 @@
 
         <h3>Change password</h3>
         <%= form_for(resource, as: resource_name, url: registration_path(resource_name), html: { method: :put, class: 'form-horizontal' }) do |f| %>
-          <%= devise_error_messages! %>
+          <%= render "devise/shared/error_messages", resource: resource %>
           <div class="form-group">
             <%= f.label :current_password, class: 'col-md-4 control-label' %>
             <div class="col-md-6">

app/views/devise/registrations/new.html.erb

@@ -8,7 +8,7 @@
         <h2>Sign up</h2>
 
         <%= form_for(resource, as: resource_name, url: registration_path(resource_name), html: { class: 'form-horizontal' }) do |f| %>
-          <%= devise_error_messages! %>
+          <%= render "devise/shared/error_messages", resource: resource %>
           <% if ENV['ON_HEROKU'] && User.count.zero? %>
           <div class="heroku-instructions">
             <% app_name = request.host[/\A[^.]+/] %>

app/views/devise/shared/_error_messages.html.erb

@@ -0,0 +1,15 @@
+<% if resource.errors.any? %>
+  <div id="error_explanation">
+    <h2>
+      <%= I18n.t("errors.messages.not_saved",
+                 count: resource.errors.count,
+                 resource: resource.class.model_name.human.downcase)
+       %>
+    </h2>
+    <ul>
+      <% resource.errors.full_messages.each do |message| %>
+        <li><%= message %></li>
+      <% end %>
+    </ul>
+  </div>
+<% end %>

app/views/devise/unlocks/new.html.erb

@@ -6,7 +6,7 @@
       <h2>Resend unlock instructions</h2>
 
       <%= form_for(resource, as: resource_name, url: unlock_path(resource_name), html: { method: :post, class: 'form-horizontal' }) do |f| %>
-        <%= devise_error_messages! %>
+        <%= render "devise/shared/error_messages", resource: resource %>
 
         <div class="form-group">
          <%= f.label :login, class: 'col-md-2 col-md-offset-2 control-label' %>

app/views/events/index.html.erb

@@ -4,13 +4,15 @@
   <div class='row'>
     <div class='col-md-12'>
       <div class="page-header">
-        <h2>
+        <span class='h2'>
           Your Events
           <% if @agent %>
             from <%= @agent.name %>
-            <%= render 'agents/mini_action_menu', agent: @agent, return_to: request.path %>
           <% end %>
-        </h2>
+        </span>
+        <% if @agent %>
+          <%= render 'agents/mini_action_menu', agent: @agent, return_to: request.path %>
+        <% end %>
       </div>
 
       <div class='table-responsive'>

app/views/events/show.html.erb

@@ -4,10 +4,10 @@
   <div class='row'>
     <div class='col-md-12'>
       <div class="page-header">
-        <h2>
+        <span class='h2'>
           Event from <%= @event.agent.name %>
-          <%= render 'agents/mini_action_menu', agent: @event.agent, return_to: event_path(@event) %>
-        </h2>
+        </span>
+        <%= render 'agents/mini_action_menu', agent: @event.agent, return_to: event_path(@event) %>
       </div>
 
       <p>

app/views/layouts/_navigation.html.erb

@@ -28,7 +28,7 @@
   
   <ul class="nav navbar-nav navbar-right">
     <% if user_signed_in? %>
-      <form class="navbar-form navbar-left visible-lg" role="search">
+      <form class="navbar-form navbar-left" role="search">
         <div class="form-group">
           <input type="text" class="form-control" id='agent-navigate' autocomplete="off" placeholder="Search">
           <%= image_tag "spinner-arrows.gif", :class => "spinner" %>
@@ -36,12 +36,12 @@
       </form>
       
       <li class='job-indicator' role='pending'>
-        <%= link_to current_user.admin? ? jobs_path : '#', class: 'visible-lg' do %>
+        <%= link_to current_user.admin? ? jobs_path : '#' do %>
           <span class="badge"><%= icon_tag('glyphicon-refresh', class: 'icon-white') %> <span class='number'>0</span></span>
         <% end %>
       </li>
       <li class='job-indicator' role='awaiting_retry'>
-        <%= link_to current_user.admin? ? jobs_path : '#', class: 'visible-lg' do %>
+        <%= link_to current_user.admin? ? jobs_path : '#' do %>
           <span class="badge"><%= icon_tag('glyphicon-question-sign', class: 'icon-yellow') %> <span class='number'>0</span></span>
         <% end %>
       </li>

app/views/system_mailer/send_message.text.erb

@@ -1,4 +1,4 @@
-<% if @body.present? %><%= sanitize @body -%>
+<% if @body.present? %><%= @body -%>
 <% else -%>
 <% if @headline %><%= @headline %>
 

bin/agent_runner.rb

@@ -1,3 +0,0 @@
-#!/usr/bin/env ruby
-ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__)
-load Gem.bin_path('bundler', 'bundle')

bin/pre_runner_boot.rb

@@ -9,5 +9,3 @@ end
 Rails.configuration.cache_classes = true
 
 Dotenv.load if ENV['APP_SECRET_TOKEN'].blank?
-
-require 'agent_runner'

bin/spring

@@ -7,9 +7,13 @@ unless defined?(Spring)
   require 'rubygems'
   require 'bundler'
 
-  lockfile = Bundler::LockfileParser.new(Bundler.default_lockfile.read)
-  spring = lockfile.specs.detect { |spec| spec.name == "spring" }
-  if spring
+  require File.join(File.dirname(__FILE__), '../lib/gemfile_helper.rb')
+  GemfileHelper.load_dotenv
+
+  if ENV['SPRING']
+    lockfile = Bundler::LockfileParser.new(Bundler.default_lockfile.read)
+    spring = lockfile.specs.find { |spec| spec.name == "spring" }
+
     Gem.use_paths Gem.dir, Bundler.bundle_path.to_s, *Gem.path
     gem 'spring', spring.version
     require 'spring/binstub'

bin/threaded.rb

@@ -1,13 +1,7 @@
 #!/bin/bash
 set -ev
 
-docker pull $DOCKER_IMAGE
-
-if [[ $DOCKER_IMAGE =~ "cantino" ]]; then
-  bin/docker_wrapper build --build-arg OUTDATED_DOCKER_IMAGE_NAMESPACE='true' -t $DOCKER_IMAGE -f $DOCKERFILE .
-else
-  bin/docker_wrapper build -t $DOCKER_IMAGE -f $DOCKERFILE .
-fi
+bin/docker_wrapper build -t $DOCKER_IMAGE -f $DOCKERFILE .
 
 if [[ -n "${DOCKER_USER}" && "${TRAVIS_PULL_REQUEST}" = 'false' && "${TRAVIS_BRANCH}" = "master" ]]; then
   docker login -u $DOCKER_USER -p $DOCKER_PASS

config/application.rb

@@ -11,11 +11,12 @@ module Huginn
     Dotenv.overload File.expand_path('../../spec/env.test', __FILE__) if Rails.env.test?
 
     # Initialize configuration defaults for originally generated Rails version.
-    config.load_defaults 5.2
+    config.load_defaults 6.0
 
     # Settings in config/environments/* take precedence over those specified here.
-    # Application configuration should go into files in config/initializers
-    # -- all .rb files in that directory are automatically loaded.
+    # Application configuration can go into files in config/initializers
+    # -- all .rb files in that directory are automatically loaded after loading
+    # the framework and any gems in your application.
 
     # Custom directories with classes and modules you want to be autoloadable.
     config.autoload_paths += %W(#{config.root}/lib #{config.root}/app/presenters #{config.root}/app/jobs)
@@ -43,5 +44,8 @@ module Huginn
     # config.active_record.schema_format = :sql
 
     config.active_job.queue_adapter = :delayed_job
+
+    config.action_view.sanitized_allowed_tags = %w[strong em b i p code pre tt samp kbd var sub sup dfn cite big small address hr br div span h1 h2 h3 h4 h5 h6 ul ol li dl dt dd abbr acronym a img blockquote del ins style table thead tbody tr th td]
+    config.action_view.sanitized_allowed_attributes = %w[href src width height alt cite datetime title class name xml:lang abbr border cellspacing cellpadding valign style]
   end
 end

config/boot.rb

@@ -1,6 +1,3 @@
-require 'rubygems'
-
-# Set up gems listed in the Gemfile.
 ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__)
 
 require 'bundler/setup' # Set up gems listed in the Gemfile.

config/deploy.rb

@@ -2,7 +2,7 @@ require 'dotenv'
 Dotenv.load
 
 # config valid only for current version of Capistrano
-lock '3.4.0'
+lock '3.11.0'
 
 set :application, 'huginn'
 set :repo_url, ENV['CAPISTRANO_DEPLOY_REPO_URL'] || 'https://github.com/huginn/huginn.git'

config/environment.rb

@@ -1,5 +1,5 @@
-# Load the rails application
+# Load the Rails application.
 require_relative 'application'
 
-# Initialize the rails application
-Huginn::Application.initialize!
+# Initialize the Rails application.
+Rails.application.initialize!

config/environments/development.rb

@@ -1,6 +1,6 @@
 $stdout.sync = true
 
-Huginn::Application.configure do
+Rails.application.configure do
   config.middleware.insert_after ActionDispatch::Static, Rack::LiveReload
 
   # Settings specified here will take precedence over those in config/application.rb
@@ -10,10 +10,7 @@ Huginn::Application.configure do
   # since you don't have to restart the web server when you make code changes.
   config.cache_classes = false
 
-  # Eager load code on boot. This eager loads most of Rails and
-  # your application in memory, allowing both threaded web servers
-  # and those relying on copy on write to perform better.
-  # Rake tasks automatically ignore this option for performance.
+  # Do not eager load code on boot.
   config.eager_load = false
 
   # Show full error reports.
@@ -23,6 +20,7 @@ Huginn::Application.configure do
   # Run rails dev:cache to toggle caching.
   if Rails.root.join('tmp', 'caching-dev.txt').exist?
     config.action_controller.perform_caching = true
+    config.action_controller.enable_fragment_cache_logging = true
 
     config.cache_store = :memory_store
     config.public_file_server.headers = {

config/environments/production.rb

@@ -95,7 +95,7 @@ Huginn::Application.configure do
   end
   config.action_mailer.perform_deliveries = true
   config.action_mailer.raise_delivery_errors = true
-  config.action_mailer.delivery_method = :smtp
+  config.action_mailer.delivery_method = ENV.fetch('SMTP_DELIVERY_METHOD', 'smtp').to_sym
   config.action_mailer.perform_caching = false
   # smtp_settings moved to config/initializers/action_mailer.rb
 end

config/environments/test.rb

@@ -1,11 +1,7 @@
 Huginn::Application.configure do
   # Settings specified here will take precedence over those in config/application.rb
 
-  # The test environment is used exclusively to run your application's
-  # test suite. You never need to work with it otherwise. Remember that
-  # your test database is "scratch space" for the test suite and is wiped
-  # and recreated between test runs. Don't rely on the data there!
-  config.cache_classes = true
+  config.cache_classes = false
 
   # Do not eager load code on boot. This avoids loading your whole application
   # just for the purpose of running a single test. If you are using a tool that
@@ -21,6 +17,7 @@ Huginn::Application.configure do
   # Show full error reports and disable caching
   config.consider_all_requests_local       = true
   config.action_controller.perform_caching = false
+  config.cache_store = :null_store
 
   # Raise exceptions instead of rendering exception templates
   config.action_dispatch.show_exceptions = false

config/initializers/devise.rb

@@ -1,3 +1,5 @@
+require 'utils'
+
 # Use this hook to configure devise mailer, warden hooks and so forth.
 # Many of these configuration options can be set straight in your model.
 Devise.setup do |config|

config/initializers/force_sni.rb

@@ -0,0 +1,11 @@
+require 'net/protocol'
+
+class Net::Protocol
+  module ForceSNI
+    def ssl_socket_connect(*)
+      @sock.hostname = @host if @sock.respond_to? :hostname=
+      super
+    end
+  end
+  prepend ForceSNI
+end

config/initializers/mail_encoding_patch.rb

@@ -0,0 +1,18 @@
+module Mail
+  class Ruby19
+    class ImprovedEncoder < BestEffortCharsetEncoder
+      def pick_encoding(charset)
+        case charset
+        when /\Aiso-2022-jp\z/i
+          Encoding::CP50220
+        when /\Ashift_jis\z/i
+          Encoding::Windows_31J
+        else
+          super
+        end
+      end
+    end
+
+    self.charset_encoder = ImprovedEncoder.new
+  end
+end

config/initializers/sanitizer.rb

@@ -1,2 +0,0 @@
-ActionView::Base.sanitized_allowed_tags += Set.new(%w(style table thead tbody tr th td))
-ActionView::Base.sanitized_allowed_attributes += Set.new(%w(border cellspacing cellpadding valign style))

config/routes.rb

@@ -4,6 +4,7 @@ Huginn::Application.routes.draw do
       post :run
       post :handle_details_post
       put :leave_scenario
+      post :reemit_events
       delete :remove_events
       delete :memory, action: :destroy_memory
     end
@@ -100,12 +101,12 @@ Huginn::Application.routes.draw do
   post  "/users/:user_id/update_location/:secret" => "web_requests#update_location" # legacy
 
   devise_for :users,
-             controllers: { 
+             controllers: {
                omniauth_callbacks: 'omniauth_callbacks',
                registrations: 'users/registrations'
              },
              sign_out_via: [:post, :delete]
-  
+
   if Rails.env.development?
     mount LetterOpenerWeb::Engine, at: "/letter_opener"
   end

config/spring.rb

@@ -1,6 +1,6 @@
-%w(
-  .ruby-version
-  .rbenv-vars
-  tmp/restart.txt
-  tmp/caching-dev.txt
-).each { |path| Spring.watch(path) }
+Spring.watch(
+  ".ruby-version",
+  ".rbenv-vars",
+  "tmp/restart.txt",
+  "tmp/caching-dev.txt"
+)

data/default_scenario.json

@@ -85,7 +85,7 @@
       "disabled": false,
       "guid": "bdae6dfdf9d01a123ddd513e695fd466",
       "options": {
-        "location": "94103",
+        "location": "42.3601,-71.0589",
         "api_key": "put-your-key-here"
       },
       "schedule": "10pm",

db/migrate/20140505201716_migrate_agents_to_liquid_templating.rb

@@ -1,31 +1,6 @@
 require 'liquid_migrator'
 
 class MigrateAgentsToLiquidTemplating < ActiveRecord::Migration[4.2]
-  class Agent < ActiveRecord::Base
-    include JSONSerializedField
-    json_serialize :options, :memory
-  end
-  class Agents::HipchatAgent < Agent
-  end
-  class Agents::EventFormattingAgent < Agent
-  end
-  class Agents::PushbulletAgent < Agent
-  end
-  class Agents::JabberAgent < Agent
-  end
-  class Agents::DataOutputAgent < Agent
-  end
-  class Agents::TranslationAgent < Agent
-  end
-  class Agents::TwitterPublishAgent < Agent
-  end
-  class Agents::TriggerAgent < Agent
-  end
-  class Agents::PeakDetectorAgent < Agent
-  end
-  class Agents::HumanTaskAgent < Agent
-  end
-
   def up
     Agent.where(:type => 'Agents::HipchatAgent').each do |agent|
       LiquidMigrator.convert_all_agent_options(agent)

db/migrate/20140723110551_adopt_xpath_in_website_agent.rb

@@ -1,6 +1,6 @@
 class AdoptXpathInWebsiteAgent < ActiveRecord::Migration[4.2]
   class Agent < ActiveRecord::Base
-    include JSONSerializedField
+    include JsonSerializedField
     json_serialize :options
   end
 

db/migrate/20140813110107_set_charset_for_mysql.rb

@@ -59,7 +59,7 @@ class SetCharsetForMysql < ActiveRecord::Migration[4.2]
           execute 'ALTER TABLE %s CHARACTER SET utf8 COLLATE utf8_unicode_ci' % table_name
         }
 
-        execute 'ALTER DATABASE %s CHARACTER SET utf8 COLLATE utf8_unicode_ci' % connection.current_database
+        execute 'ALTER DATABASE `%s` CHARACTER SET utf8 COLLATE utf8_unicode_ci' % connection.current_database
       end
 
       dir.down do

db/seeds/seeder.rb

@@ -1,11 +1,11 @@
 class Seeder
   def self.seed
-    user = User.find_or_initialize_by(:email => ENV['SEED_EMAIL'].presence || "admin@example.com")
-    if user.persisted?
-      puts "User with email '#{user.email}' already exists, not seeding."
+    if User.any?
+      puts "At least one User already exists, not seeding."
       exit
     end
 
+    user = User.find_or_initialize_by(:email => ENV['SEED_EMAIL'].presence || "admin@example.com")
     user.username = ENV['SEED_USERNAME'].presence || "admin"
     user.password = ENV['SEED_PASSWORD'].presence || "password"
     user.password_confirmation = ENV['SEED_PASSWORD'].presence || "password"

deployment/nginx/huginn

@@ -59,8 +59,7 @@ server {
   ## WARNING: If you are using relative urls remove the block below
   ## See config/application.rb under "Relative url support" for the list of
   ## other files that need to be changed for relative url support
-  location ~ ^/(assets)/ {
-    root /home/huginn/huginn/public;
+  location /assets/ {
     gzip_static on; # to serve pre-gzipped version
     expires max;
     add_header Cache-Control public;

deployment/nginx/huginn-ssl

@@ -108,8 +108,7 @@ server {
   ## WARNING: If you are using relative urls remove the block below
   ## See config/application.rb under "Relative url support" for the list of
   ## other files that need to be changed for relative url support
-  location ~ ^/(assets)/ {
-    root /home/huginn/huginn/public;
+  location /assets/ {
     gzip_static on; # to serve pre-gzipped version
     expires max;
     add_header Cache-Control public;

doc/docker/install.md

@@ -10,7 +10,7 @@ Getting Huginn up and running using docker is quick and painless once you have d
 
 #### OSX GUI using Kitematic
 
-1. Download and install [Kitematic](https://www.docker.com/docker-kitematic)
+1. Download and install [Kitematic](https://kitematic.com/)
 * Start Kitematic and search for `huginn/huginn`
 * Click `create` and wait for the container to be downloaded and booted
 * Click on the link icon next to 'WEB PREVIEW'
@@ -22,7 +22,7 @@ Getting Huginn up and running using docker is quick and painless once you have d
 * Follow the installation instructions untill you can successfully run `docker ps`
 * Get the the IP of the VM running docker by running `docker-machine ls`
 * Start your Huginn container using `docker run -it -p 3000:3000 huginn/huginn`
-* Open Huginn in the browser [http://docker-machine ip:3000](http://<docker-machine ip>:3000)
+* Open Huginn in the browser `http://<docker-machine ip>:3000`
 * Log in to your Huginn instance using the username `admin` and password `password`
 
 #### Linux
@@ -34,7 +34,7 @@ Getting Huginn up and running using docker is quick and painless once you have d
 
 ## Configuration and linking to a database container
 
-Follow the [instructions on the docker hub registry](https://registry.hub.docker.com/u/huginn/huginn/) on how to configure Huginn using environment variables and linking the container to an external MySQL or PostgreSQL database.
+Follow the [instructions on the docker hub registry](https://registry.hub.docker.com/r/huginn/huginn/) on how to configure Huginn using environment variables and linking the container to an external MySQL or PostgreSQL database.
 
 ## Running each Huginn process in a seperate container
 
@@ -44,5 +44,5 @@ With the `cantino/huginn-single-process` image you can easily run each process n
 
 Other Docker options:
 
-* If you don't want to use the official repo, see also: https://registry.hub.docker.com/u/andrewcurioso/huginn/
+* If you don't want to use the official repo, see also: https://registry.hub.docker.com/r/andrewcurioso/huginn
 * If you'd like to run Huginn's web process and job worker process in separate containers, another option is https://github.com/hackedu/huginn-docker. It also uses Unicorn as the web server and serves precompiled assets.

doc/manual/installation.md

@@ -44,13 +44,9 @@ up-to-date and install it.
     sudo apt-get install -y vim
     sudo update-alternatives --set editor /usr/bin/vim.basic
 
-Import node.js repository (can be skipped on Ubuntu and Debian Jessie):
-
-    curl -sL https://deb.nodesource.com/setup_0.12 | sudo bash -
-
 Install the required packages (needed to compile Ruby and native extensions to Ruby gems):
 
-    sudo apt-get install -y runit build-essential git zlib1g-dev libyaml-dev libssl-dev libgdbm-dev libreadline-dev libncurses5-dev libffi-dev curl openssh-server checkinstall libxml2-dev libxslt-dev libcurl4-openssl-dev libicu-dev logrotate python-docutils pkg-config cmake nodejs graphviz
+    sudo apt-get install -y runit build-essential git zlib1g-dev libyaml-dev libssl-dev libgdbm-dev libreadline-dev libncurses5-dev libffi-dev curl openssh-server checkinstall libxml2-dev libxslt-dev libcurl4-openssl-dev libicu-dev logrotate python-docutils pkg-config cmake nodejs graphviz jq
 
 
 ### Debian Stretch
@@ -59,6 +55,11 @@ Since Debian Stretch, `runit` isn't started anymore automatically, but this gets
 
      sudo apt-get install -y runit-systemd libssl1.0-dev
 
+### Ubuntu 18.04 Bionic
+
+To start `runit` automatically on Ubuntu Bionic, we need to install `runit-systemd`:
+
+    sudo apt-get install -y runit-systemd
 
 ## 2. Ruby
 
@@ -71,15 +72,20 @@ Remove the old Ruby versions if present:
 Download Ruby and compile it:
 
     mkdir /tmp/ruby && cd /tmp/ruby
-    curl -L --progress https://cache.ruby-lang.org/pub/ruby/2.5/ruby-2.5.1.tar.bz2 | tar xj
-    cd ruby-2.5.1
+    curl -L --progress https://cache.ruby-lang.org/pub/ruby/2.6/ruby-2.6.5.tar.bz2 | tar xj
+    cd ruby-2.6.5
     ./configure --disable-install-rdoc
     make -j`nproc`
     sudo make install
 
 Install the bundler and foreman gems:
 
-    sudo gem install rake bundler foreman --no-ri --no-rdoc
+    sudo gem install rake foreman --no-document
+    sudo gem install bundler -v '< 2' --no-document
+
+Update rubygems:
+
+    sudo gem update --system --no-document
 
 ## 3. System Users
 
@@ -93,20 +99,19 @@ Install the database packages
 
     sudo apt-get install -y mysql-server mysql-client libmysqlclient-dev
 
-    # Pick a MySQL root password (can be anything), type it and press enter,
-    # retype the MySQL root password and press enter
-
 For Debian Stretch, replace `libmysqlclient-dev` with `default-libmysqlclient-dev`. See the [additional notes section](#additional-notes) for more information.
 
 Check the installed MySQL version (remember if its >= 5.5.3 for the `.env` configuration done later):
 
     mysql --version
 
-Secure your installation
+Secure your installation. During this step, you will be prompted to pick a MySQL root password (can be anything)
 
     sudo mysql_secure_installation
 
-Login to MySQL
+The `mysql_secure_installation` script does not apply the user-provided password to the MySQL root user on Ubuntu systems. To apply a password to the MySQL root user on Ubuntu systems, see the [additional notes section](#set-password-for-root-MySQL-user-on-Ubuntu) for more information before proceeding.
+
+Login to MySQL using the root password you set in the previous steps
 
     mysql -u root -p
 
@@ -127,6 +132,9 @@ Grant the Huginn user necessary permissions on the database
 
     mysql> GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, INDEX, ALTER, LOCK TABLES ON `huginn_production`.* TO 'huginn'@'localhost';
 
+Use the flush privileges command to save the new permissions
+    mysql> FLUSH PRIVILEGES;
+
 Quit the database session
 
     mysql> \q
@@ -257,6 +265,8 @@ Enable (remove the comment) [from these lines](https://github.com/huginn/huginn/
     # web: bundle exec unicorn -c config/unicorn.rb
     # jobs: bundle exec rails runner bin/threaded.rb
 
+**Note:** Ensure you have no leading spaces before `web:` or `jobs:` in your `Procfile` file.
+
 Export the init scripts:
 
     sudo bundle exec rake production:export
@@ -420,3 +430,21 @@ You probably found an error message or exception backtrace you could not resolve
 ### Additional notes
 
 Debian Stretch switched from MySQL to [MariaDB](https://mariadb.org/). All packages with `mysql` in the name are just wrappers around the MariaDB ones, with some containing some compatibility symlinks. Huginn should also work fine with the MariaDB packages directly, although to keep the installation instructions more compact, they still use the MySQL packages.
+
+#### Set password for root MySQL user on Ubuntu
+
+MySQL installations (>= 5.7.26) on Ubuntu use the UNIX `auth_socket` plugin by default, such that authentication is handled by system user credientials. In order to access the MySQL root user from any system user, you have to set the MySQL root user password in the user database. Sign into the MySQL shell 
+
+    sudo mysql -u root -p
+
+    # The default password upon installation is blank
+
+Once in the MySQL shell, run the following command to set the password for the root user by replacing `new-password` with a password of your choice
+
+    ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY 'new-password';
+
+After the change has been made, exit the MySQL shell with `\q`. 
+
+For the change to propogate, restart the MySQL server
+
+    sudo service mysql restart

doc/manual/requirements.md

@@ -4,8 +4,8 @@
 
 ### Supported Unix distributions by this guide
 
-- Ubuntu (16.04, 14.04 and 12.04)
-- Debian (Jessie and Wheezy)
+- Ubuntu (18.04, 16.04 and 14.04)
+- Debian (Stretch and Jessie)
 
 ### Unsupported Unix distributions
 

doc/manual/update.md

@@ -2,7 +2,7 @@
 
 You can also use [Capistrano](./capistrano.md) to keep your installation up to date.
 
-### 0. Ensure depencies are up to date
+### 0. Ensure dependencies are up to date
 
 ```
 cd /home/huginn/huginn
@@ -50,7 +50,36 @@ Restore backed up files
 sudo -u huginn -H cp Procfile.bak Procfile
 ```
 
-### 4. Install gems, migrate and precompile assets
+### 4. Update ruby version
+
+Ensure you have Ruby 2.5+ installed:
+
+```
+ruby -v
+```
+
+Upgrade when required:
+
+```
+mkdir /tmp/ruby && cd /tmp/ruby
+curl -L --progress https://cache.ruby-lang.org/pub/ruby/2.6/ruby-2.6.5.tar.bz2 | tar xj
+cd ruby-2.6.5
+./configure --disable-install-rdoc
+make -j`nproc`
+sudo make install
+sudo gem install rake bundler foreman --no-document
+```
+
+### 5. Install gems, migrate and precompile assets
+
+Ensure you have rubygems 2.7.0+ installed:
+
+```
+gem -v
+
+# Update rubygems if the version is too old
+sudo gem update --system --no-document
+```
 
 ```
 cd /home/huginn/huginn
@@ -65,7 +94,7 @@ sudo -u huginn -H bundle exec rake assets:clean assets:precompile tmp:cache:clea
 
 ```
 
-### 5. Update the Procfile
+### 6. Update the Procfile
 
 Check for changes made to the default `Procfile`
 ```
@@ -77,7 +106,7 @@ Update your `Procfile` if the default options of the version you are using chang
 sudo -u huginn -H editor Procfile
 ```
 
-### 6. Update the .env file
+### 7. Update the .env file
 
 Check for changes made to the example `.env`
 ```
@@ -90,7 +119,7 @@ sudo -u huginn -H editor .env
 ```
 
 
-### 7. Export init script and start Huginn
+### 8. Export init script and start Huginn
 
 ```
 # Export the init script

docker/multi-process/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:14.04
+FROM ubuntu:18.04
 
 COPY docker/scripts/prepare /scripts/
 RUN /scripts/prepare
@@ -28,7 +28,14 @@ RUN umask 002 && \
 
 EXPOSE 3000
 
-COPY ["docker/scripts/setup_env", "docker/multi-process/scripts/init", "/scripts/"]
+COPY docker/multi-process/scripts/supervisord.conf /etc/supervisor/
+COPY ["docker/multi-process/scripts/bootstrap.conf", \
+      "docker/multi-process/scripts/foreman.conf", \
+      "docker/multi-process/scripts/mysqld.conf", "/etc/supervisor/conf.d/"]
+COPY ["docker/multi-process/scripts/bootstrap.sh", \
+      "docker/multi-process/scripts/foreman.sh", \
+      "docker/multi-process/scripts/init", \
+      "docker/scripts/setup_env", "/scripts/"]
 CMD ["/scripts/init"]
 
 USER 1001

docker/multi-process/README.md

@@ -10,6 +10,10 @@ This was patterned after [sameersbn/gitlab](https://hub.docker.com/r/sameersbn/g
 The scripts/init script generates a .env file containing the variables as passed as per normal Huginn documentation.
 The same environment variables that would be used for Heroku PaaS deployment are used by this script.
 
+It is possible to use a separate mysql/mariadb/postgres container for the database. If you do not use a separate database container, a built-in mysql database will be started. There is an exported docker volume of `/var/lib/mysql` to allow persistence of that mysql database. Please be aware that to improve security, newer versions of this docker image do not run the processes as root. If the permissions of the mounted volume are not correct (777 / chown 1001), mysql can not be started. There are no permission environment variables (GUID/PUID) for this image.
+
+__NOTE:__ If you do not export the volume, or use a separate database container, you cannot update Huginn without losing your data.
+
 The scripts/init script is aware of mysql and postgres linked containers through the environment variables:
 
     MYSQL_PORT_3306_TCP_ADDR
@@ -20,12 +24,7 @@ and
     POSTGRES_PORT_5432_TCP_ADDR
     POSTGRES_PORT_5432_TCP_PORT
 
-Its recommended to use an image that allows you to create a database via environmental variables at docker run, like `paintedfox / postgresql` or `centurylink / mysql`, so the db is populated when this script runs.
-
-If you do not link a database container, a built-in mysql database will be started.
-There is an exported docker volume of `/var/lib/mysql` to allow persistence of that mysql database.
-
-__NOTE:__ If you do not export the volme, or use a linked database container, you cannot update Huginn without losing your data.
+It is recommended to use an image that allows you to create a database via environmental variables at docker run, so the db is populated when this script runs. The offical images of the mentioned databases all support this.
 
 Additionally, the database variables may be overridden from the above as per the standard Huginn documentation:
 
@@ -42,6 +41,7 @@ It will also seed the database (rake db:seed) unless this is defined:
     DO_NOT_SEED
 
 This same seeding initially defines the "admin" user with a default password of "password" as per the standard Huginn documentation.
+You can customize the admin account name with the environment variable ``SEED_USERNAME`` and ``SEED_PASSWORD``.
 
 If you do not wish to have the default 6 agents, you will want to set the above environment variable after your initially deploy, otherwise they will be added automatically the next time a container pointing at the database is spun up.
 
@@ -88,6 +88,17 @@ To link to another container named 'postgres':
         -e HUGINN_DATABASE_ADAPTER=postgresql \
         huginn/huginn
 
+To use a separate, non-linked mysql container:
+
+    docker run --rm --name huginn \
+        -p 3000:3000 \
+        -e HUGINN_DATABASE_NAME=huginn \
+        -e HUGINN_DATABASE_USERNAME=huginn \
+        -e HUGINN_DATABASE_PASSWORD=mysecretpassword \
+        -e HUGINN_DATABASE_HOST=myname.mydomain \
+        -e HUGINN_DATABASE_PORT=3306
+        huginn/huginn
+
 The `docker/multi-process` folder also has a `docker-compose.yml` that allows for a sample database formation with a data volume container:
 
     cd docker/multi-process