Merge branch 'master' into rss_agent_logs_exact_feed_on_error

.env.example

@@ -167,7 +167,7 @@ TIMEZONE="Pacific Time (US & Canada)"
 FAILED_JOBS_TO_KEEP=100
 
 # Maximum runtime of background jobs in minutes
-DELAYED_JOB_MAX_RUNTIME=20
+DELAYED_JOB_MAX_RUNTIME=2
 
 # Amount of seconds for delayed_job to sleep before checking for new jobs
 DELAYED_JOB_SLEEP_DELAY=10

CHANGES.md

@@ -1,12 +1,19 @@
 # Changes
 
+* Jul 30, 2015   - RssAgent can configure the order of events created via `events_order`.
+* Jul 29, 2015   - WebsiteAgent can configure the order of events created via `events_order`.
+* Jul 29, 2015   - DataOutputAgent can configure the order of events in the output via `events_order`.
 * Jul 20, 2015   - Control Links (used by the SchedularAgent) are correctly exported in Scenarios.
 * Jul 20, 2015   - keep\_events\_for was moved from days to seconds; Scenarios have a schema verison.
+* Jul 8, 2015    - DataOutputAgent supports feed icon, and a new template variable `events`.
 * Jul 1, 2015    - DeDuplicationAgent properly handles destruction of memory.
 * Jun 26, 2015   - Add `max_events_per_run` to RssAgent.
 * Jun 19, 2015   - Add `url_from_event` to WebsiteAgent.
 * Jun 17, 2015   - RssAgent emits events for new feed items in chronological order.
+* Jun 17, 2015   - Liquid filter `unescape` added.
+* Jun 17, 2015   - Liquid filter `regex_replace` and `regex_replace_first` added, with escape sequence support.
 * Jun 15, 2015   - Liquid filter `uri_expand` added.
+* Jun 13, 2015   - Liquid templating engine is upgraded to version 3.
 * Jun 12, 2015   - RSSAgent can now accept an array of URLs.
 * Jun 8, 2015    - WebsiteAgent includes a `use_namespaces` option to enable XML namespaces.
 * May 27, 2015   - Validation warns user if they have not provided a `path` when using JSONPath in WebsiteAgent.

Gemfile

@@ -1,5 +1,8 @@
 source 'https://rubygems.org'
 
+# Ruby 2.0 is the minimum requirement
+ruby ['2.0.0', RUBY_VERSION].max
+
 # Optional libraries.  To conserve RAM, comment out any that you don't need,
 # then run `bundle` and commit the updated Gemfile and Gemfile.lock.
 gem 'twilio-ruby', '~> 3.11.5'    # TwilioAgent

Gemfile.lock

@@ -235,7 +235,7 @@ GEM
     launchy (2.4.2)
       addressable (~> 2.3)
     libv8 (3.16.14.7)
-    liquid (3.0.3)
+    liquid (3.0.6)
     listen (2.7.9)
       celluloid (>= 0.15.2)
       rb-fsevent (>= 0.9.3)
@@ -580,6 +580,3 @@ DEPENDENCIES
   weibo_2!
   wunderground (~> 1.2.0)
   xmpp4r (~> 0.5.6)
-
-BUNDLED WITH
-   1.10.5

README.md

@@ -80,9 +80,11 @@ All agents have specs! Test all specs with `bundle exec rspec`, or test a specif
 
 ## Deployment
 
-[![Deploy](https://www.herokucdn.com/deploy/button.png)](https://heroku.com/deploy) (Takes a few minutes to setup.  Be sure to click 'View it' after launch!)
+Try Huginn on Heroku: [![Deploy](https://www.herokucdn.com/deploy/button.png)](https://heroku.com/deploy) (Takes a few minutes to setup.  Be sure to click 'View it' after launch!)
 
-Huginn can run on Heroku for free!  Please see [the Huginn Wiki](https://github.com/cantino/huginn/wiki#deploying-huginn) for detailed deployment strategies for different providers.
+Huginn works on the free version of Heroku [with limitations](https://github.com/cantino/huginn/wiki/Run-Huginn-for-free-on-Heroku). For non-experimental use, we recommend Heroku's cheapest paid plan or our Docker container.
+
+Please see [the Huginn Wiki](https://github.com/cantino/huginn/wiki#deploying-huginn) for detailed deployment strategies for different providers.
 
 ### Optional Setup
 

app/concerns/dry_runnable.rb

@@ -1,10 +1,8 @@
 module DryRunnable
-  def dry_run!
-    readonly!
+  extend ActiveSupport::Concern
 
-    class << self
-      prepend Sandbox
-    end
+  def dry_run!
+    @dry_run = true
 
     log = StringIO.new
     @dry_run_logger = Logger.new(log)
@@ -14,6 +12,7 @@ module DryRunnable
 
     begin
       raise "#{short_type} does not support dry-run" unless can_dry_run?
+      readonly!
       check
     rescue => e
       error "Exception during dry-run. #{e.message}: #{e.backtrace.join("\n")}"
@@ -23,28 +22,38 @@ module DryRunnable
       memory: memory,
       log: log.string,
     )
+  ensure
+    @dry_run = false
   end
 
   def dry_run?
-    is_a? Sandbox
+    !!@dry_run
+  end
+
+  included do
+    prepend Wrapper
   end
 
-  module Sandbox
+  module Wrapper
     attr_accessor :results
 
     def logger
+      return super unless dry_run?
       @dry_run_logger
     end
 
-    def save
-      valid?
+    def save(options = {})
+      return super unless dry_run?
+      perform_validations(options)
     end
 
-    def save!
-      save or raise ActiveRecord::RecordNotSaved
+    def save!(options = {})
+      return super unless dry_run?
+      save(options) or raise_record_invalid
     end
 
     def log(message, options = {})
+      return super unless dry_run?
       case options[:level] || 3
       when 0..2
         sev = Logger::DEBUG
@@ -57,10 +66,12 @@ module DryRunnable
       logger.log(sev, message)
     end
 
-    def create_event(event_hash)
+    def create_event(event)
+      return super unless dry_run?
       if can_create_events?
-        @dry_run_results[:events] << event_hash[:payload]
-        events.build({ user: user, expires_at: new_event_expiration_date }.merge(event_hash))
+        event = build_event(event)
+        @dry_run_results[:events] << event.payload
+        event
       else
         error "This Agent cannot create events!"
       end

app/concerns/sortable_events.rb

@@ -0,0 +1,161 @@
+module SortableEvents
+  extend ActiveSupport::Concern
+
+  included do
+    validate :validate_events_order
+  end
+
+  def description_events_order(*args)
+    self.class.description_events_order(*args)
+  end
+
+  module ClassMethods
+    def can_order_created_events!
+      raise if cannot_create_events?
+      prepend AutomaticSorter
+    end
+
+    def can_order_created_events?
+      include? AutomaticSorter
+    end
+
+    def cannot_order_created_events?
+      !can_order_created_events?
+    end
+
+    def description_events_order(events = 'events created in each run')
+      <<-MD.lstrip
+        To specify the order of #{events}, set `events_order` to an array of sort keys, each of which looks like either `expression` or `[expression, type, descending]`, as described as follows:
+
+        * _expression_ is a Liquid template to generate a string to be used as sort key.
+
+        * _type_ (optional) is one of `string` (default), `number` and `time`, which specifies how to evaluate _expression_ for comparison.
+
+        * _descending_ (optional) is a boolean value to determine if comparison should be done in descending (reverse) order, which defaults to `false`.
+
+        Sort keys listed earlier take precedence over ones listed later.  For example, if you want to sort articles by the date and then by the author, specify `[["{{date}}", "time"], "{{author}}"]`.
+
+        Sorting is done stably, so even if all events have the same set of sort key values the original order is retained.  Also, a special Liquid variable `_index_` is provided, which contains the zero-based index number of each event, which means you can exactly reverse the order of events by specifying `[["{{_index_}}", "number", true]]`.
+      MD
+    end
+  end
+
+  def can_order_created_events?
+    self.class.can_order_created_events?
+  end
+
+  def cannot_order_created_events?
+    self.class.cannot_order_created_events?
+  end
+
+  def events_order
+    options['events_order']
+  end
+
+  module AutomaticSorter
+    def check
+      return super unless events_order
+      sorting_events do
+        super
+      end
+    end
+
+    def receive(incoming_events)
+      return super unless events_order
+      # incoming events should be processed sequentially
+      incoming_events.each do |event|
+        sorting_events do
+          super([event])
+        end
+      end
+    end
+
+    def create_event(event)
+      if @sortable_events
+        event = build_event(event)
+        @sortable_events << event
+        event
+      else
+        super
+      end
+    end
+
+    private
+
+    def sorting_events(&block)
+      @sortable_events = []
+      yield
+    ensure
+      events, @sortable_events = @sortable_events, nil
+      sort_events(events).each do |event|
+        create_event(event)
+      end
+    end
+  end
+
+  private
+
+  EXPRESSION_PARSER = {
+    'string' => ->string { string },
+    'number' => ->string { string.to_f },
+    'time'   => ->string { Time.zone.parse(string) },
+  }
+  EXPRESSION_TYPES = EXPRESSION_PARSER.keys.freeze
+
+  def validate_events_order
+    case order_by = events_order
+    when nil
+    when Array
+      # Each tuple may be either [expression, type, desc] or just
+      # expression.
+      order_by.each do |expression, type, desc|
+        case expression
+        when String
+          # ok
+        else
+          errors.add(:base, "first element of each events_order tuple must be a Liquid template")
+          break
+        end
+        case type
+        when nil, *EXPRESSION_TYPES
+          # ok
+        else
+          errors.add(:base, "second element of each events_order tuple must be #{EXPRESSION_TYPES.to_sentence(last_word_connector: ' or ')}")
+          break
+        end
+        if !desc.nil? && boolify(desc).nil?
+          errors.add(:base, "third element of each events_order tuple must be a boolean value")
+          break
+        end
+      end
+    else
+      errors.add(:base, "events_order must be an array of arrays")
+    end
+  end
+
+  # Sort given events in order specified by the "events_order" option
+  def sort_events(events)
+    order_by = events_order.presence or
+      return events
+
+    orders = order_by.map { |_, _, desc = false| boolify(desc) }
+
+    Utils.sort_tuples!(
+      events.map.with_index { |event, index|
+        interpolate_with(event) {
+          interpolation_context['_index_'] = index
+          order_by.map { |expression, type, _|
+            string = interpolate_string(expression)
+            begin
+              EXPRESSION_PARSER[type || 'string'.freeze][string]
+            rescue
+              error "Cannot parse #{string.inspect} as #{type}; treating it as string"
+              string
+            end
+          }
+        } << index << event  # index is to make sorting stable
+      },
+      orders
+    ).collect!(&:last)
+  end
+end

app/helpers/application_helper.rb

@@ -80,6 +80,7 @@ module ApplicationHelper
   end
 
   def service_label(service)
+    return if service.nil?
     content_tag :span, [
       omniauth_provider_icon(service.provider),
       service_label_text(service)

app/models/agent.rb

@@ -13,6 +13,7 @@ class Agent < ActiveRecord::Base
   include HasGuid
   include LiquidDroppable
   include DryRunnable
+  include SortableEvents
 
   markdown_class_attributes :description, :event_description
 
@@ -104,12 +105,19 @@ class Agent < ActiveRecord::Base
     raise "Implement me in your subclass"
   end
 
-  def create_event(attrs)
+  def build_event(event)
+    event = events.build(event) if event.is_a?(Hash)
+    event.agent = self
+    event.user = user
+    event.expires_at ||= new_event_expiration_date
+    event
+  end
+
+  def create_event(event)
     if can_create_events?
-      events.create!({
-         :user => user,
-         :expires_at => new_event_expiration_date
-      }.merge(attrs))
+      event = build_event(event)
+      event.save!
+      event
     else
       error "This Agent cannot create events!"
     end

app/models/agents/data_output_agent.rb

@@ -40,11 +40,15 @@ module Agents
               "_contents": "tag contents (can be an object for nesting)"
             }
 
+        # Ordering events in the output
+
+        #{description_events_order('events in the output')}
+
         # Liquid Templating
 
         In Liquid templating, the following variable is available:
 
-        * `events`: An array of events being output, sorted in descending order up to `events_to_show` in number.  For example, if source events contain a site title in the `site_title` key, you can refer to it in `template.title` by putting `{{events.first.site_title}}`.
+        * `events`: An array of events being output, sorted in the given order, up to `events_to_show` in number.  For example, if source events contain a site title in the `site_title` key, you can refer to it in `template.title` by putting `{{events.first.site_title}}`.
 
       MD
     end
@@ -134,7 +138,7 @@ module Agents
         end
       end
 
-      source_events = received_events.order(id: :desc).limit(events_to_show).to_a
+      source_events = sort_events(received_events.order(id: :desc).limit(events_to_show).to_a)
 
       interpolation_context.stack do
         interpolation_context['events'] = source_events

app/models/agents/rss_agent.rb

@@ -9,6 +9,8 @@ module Agents
     can_dry_run!
     default_schedule "every_1d"
 
+    DEFAULT_EVENTS_ORDER = [['{{date_published}}', 'time'], ['{{last_updated}}', 'time']]
+
     description do
       <<-MD
         This Agent consumes RSS feeds and emits events when they change.
@@ -29,6 +31,12 @@ module Agents
           * `disable_url_encoding` - Set to `true` to disable url encoding.
           * `user_agent` - A custom User-Agent name (default: "Faraday v#{Faraday::VERSION}").
           * `max_events_per_run` - Limit number of events created (items parsed) per run for feed.
+
+        # Ordering Events
+
+        #{description_events_order}
+
+        In this Agent, the default value for `events_order` is `#{DEFAULT_EVENTS_ORDER.to_json}`.
       MD
     end
 
@@ -70,6 +78,11 @@ module Agents
       end
 
       validate_web_request_options!
+      validate_events_order
+    end
+
+    def events_order
+      super.presence || DEFAULT_EVENTS_ORDER
     end
 
     def check
@@ -84,26 +97,15 @@ module Agents
       response = faraday.get(url)
       if response.success?
         feed = FeedNormalizer::FeedNormalizer.parse(response.body)
-        feed.clean! if interpolated['clean'] == 'true'
+        feed.clean! if boolify(interpolated['clean'])
         max_events = (interpolated['max_events_per_run'].presence || 0).to_i
         created_event_count = 0
-        feed.entries.sort_by { |entry| [entry.date_published, entry.last_updated] }.each.with_index do |entry, index|
+        sort_events(feed_to_events(feed)).each.with_index do |event, index|
           break if max_events && max_events > 0 && index >= max_events
-          entry_id = get_entry_id(entry)
+          entry_id = event.payload[:id]
           if check_and_track(entry_id)
             created_event_count += 1
-            create_event(payload: {
-                           id: entry_id,
-                           date_published: entry.date_published,
-                           last_updated: entry.last_updated,
-                           url: entry.url,
-                           urls: entry.urls,
-                           description: entry.description,
-                           content: entry.content,
-                           title: entry.title,
-                           authors: entry.authors,
-                           categories: entry.categories
-                         })
+            create_event(event)
           end
         end
         log "Fetched #{url} and created #{created_event_count} event(s)."
@@ -128,5 +130,22 @@ module Agents
         true
       end
     end
+
+    def feed_to_events(feed)
+      feed.entries.map { |entry|
+        Event.new(payload: {
+                    id: get_entry_id(entry),
+                    date_published: entry.date_published,
+                    last_updated: entry.last_updated,
+                    url: entry.url,
+                    urls: entry.urls,
+                    description: entry.description,
+                    content: entry.content,
+                    title: entry.title,
+                    authors: entry.authors,
+                    categories: entry.categories
+                  })
+      }
+    end
   end
 end

app/models/agents/website_agent.rb

@@ -6,6 +6,7 @@ module Agents
     include WebRequestConcern
 
     can_dry_run!
+    can_order_created_events!
 
     default_schedule "every_12h"
 
@@ -105,6 +106,10 @@ module Agents
           * `status`: HTTP status as integer. (Almost always 200)
 
           * `headers`: Response headers; for example, `{{ _response_.headers.Content-Type }}` expands to the value of the Content-Type header.  Keys are insensitive to cases and -/_.
+
+      # Ordering Events
+
+      #{description_events_order}
     MD
 
     event_description do

app/models/scenario_import.rb

@@ -278,21 +278,6 @@ class ScenarioImport
       yield 'disabled', disabled, boolean if disabled.requires_merge?
     end
 
-    # Unfortunately Ruby 1.9's OpenStruct doesn't expose [] and []=.
-    unless instance_methods.include?(:[]=)
-      def [](key)
-        self.send(sanitize key)
-      end
-
-      def []=(key, val)
-        self.send("#{sanitize key}=", val)
-      end
-
-      def sanitize(key)
-        key.gsub(/[^a-zA-Z0-9_-]/, '')
-      end
-    end
-
     def agent_instance
       "Agents::#{self.type.updated}".constantize.new
     end

config/initializers/delayed_job.rb

@@ -1,6 +1,6 @@
 Delayed::Worker.destroy_failed_jobs = false
 Delayed::Worker.max_attempts = 5
-Delayed::Worker.max_run_time = (ENV['DELAYED_JOB_MAX_RUNTIME'].presence || 20).to_i.minutes
+Delayed::Worker.max_run_time = (ENV['DELAYED_JOB_MAX_RUNTIME'].presence || 2).to_i.minutes
 Delayed::Worker.read_ahead = 5
 Delayed::Worker.default_priority = 10
 Delayed::Worker.delay_jobs = !Rails.env.test?

lib/ar_mysql_column_charset.rb

@@ -1,6 +1,3 @@
-# Module#prepend support for Ruby 1.9
-require 'prepend' unless Module.method_defined?(:prepend)
-
 require 'active_support'
 
 ActiveSupport.on_load :active_record do

lib/prepend.rb

@@ -1,85 +0,0 @@
-# Fake implementation of prepend(), which does not support overriding
-# inherited methods nor methods that are formerly overridden by
-# another invocation of prepend().
-#
-# Here's what <Original>.prepend(<Wrapper>) does:
-#
-# - Create an anonymous stub module (hereinafter <Stub>) and define
-#   <Stub>#<method> that calls #<method>_without_<Wrapper> for each
-#   instance method of <Wrapper>.
-#
-# - Rename <Original>#<method> to #<method>_without_<Wrapper> for each
-#   instance method of <Wrapper>.
-#
-# - Include <Stub> and <Wrapper> into <Original> in that order.
-#
-# This way, a call of <Original>#<method> is dispatched to
-# <Wrapper><method>, which may call super which is dispatched to
-# <Stub>#<method>, which finally calls
-# <Original>#<method>_without_<Wrapper> which is used to be called
-# <Original>#<method>.
-#
-# Usage:
-#
-#     class Mechanize
-#       # module with methods that overrides those of X
-#       module Y
-#       end
-#
-#       unless X.respond_to?(:prepend, true)
-#         require 'mechanize/prependable'
-#         X.extend(Prependable)
-#       end
-#
-#       class X
-#         prepend Y
-#       end
-#     end
-class Module
-  def prepend(mod)
-    stub = Module.new
-
-    mod_id = (mod.name || 'Module__%d' % mod.object_id).gsub(/::/, '__')
-
-    mod.instance_methods.each { |name|
-      method_defined?(name) or next
-
-      original = instance_method(name)
-      next if original.owner != self
-
-      name = name.to_s
-      name_without = name.sub(/(?=[?!=]?\z)/) { '_without_%s' % mod_id }
-
-      arity = original.arity
-      arglist = (
-        if arity >= 0
-          (1..arity).map { |i| 'x%d' % i }
-        else
-          (1..(-arity - 1)).map { |i| 'x%d' % i } << '*a'
-        end << '&b'
-      ).join(', ')
-
-      if name.end_with?('=')
-        stub.module_eval %{
-          def #{name}(#{arglist})
-            __send__(:#{name_without}, #{arglist})
-          end
-        }
-      else
-        stub.module_eval %{
-          def #{name}(#{arglist})
-            #{name_without}(#{arglist})
-          end
-        }
-      end
-      module_eval {
-        alias_method name_without, name
-        remove_method name
-      }
-    }
-
-    include stub
-    include mod
-  end
-  private :prepend
-end unless Module.method_defined?(:prepend)

lib/utils.rb

@@ -79,4 +79,43 @@ module Utils
   def self.pretty_jsonify(thing)
     JSON.pretty_generate(thing).gsub('</', '<\/')
   end
+
+  class TupleSorter
+    class SortableTuple
+      attr_reader :array
+
+      # The <=> method will call orders[n] to determine if the nth element
+      # should be compared in descending order.
+      def initialize(array, orders = [])
+        @array = array
+        @orders = orders
+      end
+
+      def <=> other
+        other = other.array
+        @array.each_with_index do |e, i|
+          o = other[i]
+          case cmp = e <=> o || e.to_s <=> o.to_s
+          when 0
+            next
+          else
+            return @orders[i] ? -cmp : cmp
+          end
+        end
+        0
+      end
+    end
+
+    class << self
+      def sort!(array, orders = [])
+        array.sort_by! do |e|
+          SortableTuple.new(e, orders)
+        end
+      end
+    end
+  end
+
+  def self.sort_tuples!(array, orders = [])
+    TupleSorter.sort!(array, orders)
+  end
 end

spec/concerns/sortable_events_spec.rb

@@ -0,0 +1,264 @@
+require 'spec_helper'
+
+describe SortableEvents do
+  let(:agent_class) {
+    Class.new(Agent) do
+      include SortableEvents
+
+      default_schedule 'never'
+
+      def self.valid_type?(name)
+        true
+      end
+    end
+  }
+
+  def new_agent(events_order = nil)
+    options = {}
+    options['events_order'] = events_order if events_order
+    agent_class.new(name: 'test', options: options) { |agent|
+      agent.user = users(:bob)
+    }
+  end
+
+  describe 'validations' do
+    let(:agent_class) {
+      Class.new(Agent) do
+        include SortableEvents
+
+        default_schedule 'never'
+
+        def self.valid_type?(name)
+          true
+        end
+      end
+    }
+
+    def new_agent(events_order = nil)
+      options = {}
+      options['events_order'] = events_order if events_order
+      agent_class.new(name: 'test', options: options) { |agent|
+        agent.user = users(:bob)
+      }
+    end
+
+    it 'should allow events_order to be unspecified, null or an empty array' do
+      expect(new_agent()).to be_valid
+      expect(new_agent(nil)).to be_valid
+      expect(new_agent([])).to be_valid
+    end
+
+    it 'should not allow events_order to be a non-array object' do
+      agent = new_agent(0)
+      expect(agent).not_to be_valid
+      expect(agent.errors[:base]).to include(/events_order/)
+
+      agent = new_agent('')
+      expect(agent).not_to be_valid
+      expect(agent.errors[:base]).to include(/events_order/)
+
+      agent = new_agent({})
+      expect(agent).not_to be_valid
+      expect(agent.errors[:base]).to include(/events_order/)
+    end
+
+    it 'should not allow events_order to be an array containing unexpected objects' do
+      agent = new_agent(['{{key}}', 1])
+      expect(agent).not_to be_valid
+      expect(agent.errors[:base]).to include(/events_order/)
+
+      agent = new_agent(['{{key1}}', ['{{key2}}', 'unknown']])
+      expect(agent).not_to be_valid
+      expect(agent.errors[:base]).to include(/events_order/)
+    end
+
+    it 'should allow events_order to be an array containing strings and valid tuples' do
+      agent = new_agent(['{{key1}}', ['{{key2}}'], ['{{key3}}', 'number']])
+      expect(agent).to be_valid
+
+      agent = new_agent(['{{key1}}', ['{{key2}}'], ['{{key3}}', 'number'], ['{{key4}}', 'time', true]])
+      expect(agent).to be_valid
+    end
+  end
+
+  describe 'sort_events' do
+    let(:payloads) {
+      [
+        { 'title' => 'TitleA', 'score' => 4,  'updated_on' => '7 Jul 2015' },
+        { 'title' => 'TitleB', 'score' => 2,  'updated_on' => '25 Jun 2014' },
+        { 'title' => 'TitleD', 'score' => 10, 'updated_on' => '10 Jan 2015' },
+        { 'title' => 'TitleC', 'score' => 10, 'updated_on' => '9 Feb 2015' },
+      ]
+    }
+
+    let(:events) {
+      payloads.map { |payload| Event.new(payload: payload) }
+    }
+
+    it 'should sort events by a given key' do
+      agent = new_agent(['{{title}}'])
+      expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleA TitleB TitleC TitleD])
+
+      agent = new_agent([['{{title}}', 'string', true]])
+      expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleD TitleC TitleB TitleA])
+    end
+
+    it 'should sort events by multiple keys' do
+      agent = new_agent([['{{score}}', 'number'], '{{title}}'])
+      expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleB TitleA TitleC TitleD])
+
+      agent = new_agent([['{{score}}', 'number'], ['{{title}}', 'string', true]])
+      expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleB TitleA TitleD TitleC])
+    end
+
+    it 'should sort events by time' do
+      agent = new_agent([['{{updated_on}}', 'time']])
+      expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleB TitleD TitleC TitleA])
+    end
+
+    it 'should sort events stably' do
+      agent = new_agent(['<constant>'])
+      expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleA TitleB TitleD TitleC])
+
+      agent = new_agent([['<constant>', 'string', true]])
+      expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleA TitleB TitleD TitleC])
+    end
+
+    it 'should support _index_' do
+      agent = new_agent([['{{_index_}}', 'number', true]])
+      expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleC TitleD TitleB TitleA])
+    end
+  end
+
+  describe 'automatic event sorter' do
+    describe 'declaration' do
+      let(:passive_agent_class) {
+        Class.new(Agent) do
+          include SortableEvents
+
+          cannot_create_events!
+        end
+      }
+
+      let(:active_agent_class) {
+        Class.new(Agent) do
+          include SortableEvents
+        end
+      }
+
+      describe 'can_order_created_events!' do
+        it 'should refuse to work if called from an Agent that cannot create events' do
+          expect {
+            passive_agent_class.class_eval do
+              can_order_created_events!
+            end
+          }.to raise_error
+        end
+
+        it 'should work if called from an Agent that can create events' do
+          expect {
+            active_agent_class.class_eval do
+              can_order_created_events!
+            end
+          }.not_to raise_error
+        end
+      end
+
+      describe 'can_order_created_events?' do
+        it 'should return false unless an Agent declares can_order_created_events!' do
+          expect(active_agent_class.can_order_created_events?).to eq(false)
+          expect(active_agent_class.new.can_order_created_events?).to eq(false)
+        end
+
+        it 'should return true if an Agent declares can_order_created_events!' do
+          active_agent_class.class_eval do
+            can_order_created_events!
+          end
+
+          expect(active_agent_class.can_order_created_events?).to eq(true)
+          expect(active_agent_class.new.can_order_created_events?).to eq(true)
+        end
+      end
+    end
+
+    describe 'behavior' do
+      class Agents::EventOrderableAgent < Agent
+        include SortableEvents
+
+        default_schedule 'never'
+
+        can_order_created_events!
+
+        attr_accessor :payloads_to_emit
+
+        def self.valid_type?(name)
+          true
+        end
+
+        def check
+          payloads_to_emit.each do |payload|
+            create_event payload: payload
+          end
+        end
+
+        def receive(events)
+          events.each do |event|
+            payloads_to_emit.each do |payload|
+              create_event payload: payload.merge('title' => payload['title'] + event.payload['title_suffix'])
+            end
+          end
+        end
+      end
+
+      def new_agent(events_order = nil)
+        options = {}
+        options['events_order'] = events_order if events_order
+        Agents::EventOrderableAgent.new(name: 'test', options: options) { |agent|
+          agent.user = users(:bob)
+          agent.payloads_to_emit = payloads
+        }
+      end
+
+      let(:payloads) {
+        [
+          { 'title' => 'TitleA', 'score' => 4,  'updated_on' => '7 Jul 2015' },
+          { 'title' => 'TitleB', 'score' => 2,  'updated_on' => '25 Jun 2014' },
+          { 'title' => 'TitleD', 'score' => 10, 'updated_on' => '10 Jan 2015' },
+          { 'title' => 'TitleC', 'score' => 10, 'updated_on' => '9 Feb 2015' },
+        ]
+      }
+
+      it 'should keep the order of created events unless events_order is specified' do
+        [[], [nil], [[]]].each do |args|
+          agent = new_agent(*args)
+          agent.save!
+          expect { agent.check }.to change { Event.count }.by(4)
+          events = agent.events.last(4).sort_by(&:id)
+          expect(events.map { |event| event.payload['title'] }).to eq(%w[TitleA TitleB TitleD TitleC])
+        end
+      end
+
+      it 'should sort events created in check() in the order specified in events_order' do
+        agent = new_agent([['{{score}}', 'number'], ['{{title}}', 'string', true]])
+        agent.save!
+        expect { agent.check }.to change { Event.count }.by(4)
+        events = agent.events.last(4).sort_by(&:id)
+        expect(events.map { |event| event.payload['title'] }).to eq(%w[TitleB TitleA TitleD TitleC])
+      end
+
+      it 'should sort events created in receive() in the order specified in events_order' do
+        agent = new_agent([['{{score}}', 'number'], ['{{title}}', 'string', true]])
+        agent.save!
+        expect {
+          agent.receive([Event.new(payload: { 'title_suffix' => ' [new]' }),
+                         Event.new(payload: { 'title_suffix' => ' [popular]' })])
+        }.to change { Event.count }.by(8)
+        events = agent.events.last(8).sort_by(&:id)
+        expect(events.map { |event| event.payload['title'] }).to eq([
+          'TitleB [new]',     'TitleA [new]',     'TitleD [new]',     'TitleC [new]',
+          'TitleB [popular]', 'TitleA [popular]', 'TitleD [popular]', 'TitleC [popular]',
+        ])
+      end
+    end
+  end
+end

spec/controllers/agents_controller_spec.rb

@@ -372,7 +372,7 @@ describe AgentsController do
       sign_in users(:bob)
       agent = agents(:bob_weather_agent)
       expect {
-        post :dry_run, id: agents(:bob_website_agent), agent: valid_attributes(name: 'New Name')
+        post :dry_run, id: agent, agent: valid_attributes(name: 'New Name')
       }.not_to change {
         [users(:bob).agents.count, users(:bob).events.count, users(:bob).logs.count, agent.name, agent.updated_at]
       }

spec/lib/utils_spec.rb

@@ -114,4 +114,62 @@ describe Utils do
       expect(cleaned_json).to include("<\\/script>")
     end
   end
+
+  describe "#sort_tuples!" do
+    let(:tuples) {
+      time = Time.now
+      [
+        [2, "a", time - 1],  # 0
+        [2, "b", time - 1],  # 1
+        [1, "b", time - 1],  # 2
+        [1, "b", time],      # 3
+        [1, "a", time],      # 4
+        [2, "a", time + 1],  # 5
+        [2, "a", time],      # 6
+      ]
+    }
+
+    it "sorts tuples like arrays by default" do
+      expected = tuples.values_at(4, 2, 3, 0, 6, 5, 1)
+
+      Utils.sort_tuples!(tuples)
+      expect(tuples).to eq expected
+    end
+
+    it "sorts tuples in order specified: case 1" do
+      # order by x1 asc, x2 desc, c3 asc
+      orders = [false, true, false]
+      expected = tuples.values_at(2, 3, 4, 1, 0, 6, 5)
+
+      Utils.sort_tuples!(tuples, orders)
+      expect(tuples).to eq expected
+    end
+
+    it "sorts tuples in order specified: case 2" do
+      # order by x1 desc, x2 asc, c3 desc
+      orders = [true, false, true]
+      expected = tuples.values_at(5, 6, 0, 1, 4, 3, 2)
+
+      Utils.sort_tuples!(tuples, orders)
+      expect(tuples).to eq expected
+    end
+
+    it "always succeeds in sorting even if it finds pairs of incomparable objects" do
+      time = Time.now
+      tuples = [
+        [2,   "a", time - 1],  # 0
+        [1,   "b", nil],       # 1
+        [1,   "b", time],      # 2
+        ["2", nil, time],      # 3
+        [1,   nil, time],      # 4
+        [nil, "a", time + 1],  # 5
+        [2,   "a", time],      # 6
+      ]
+      orders = [true, false, true]
+      expected = tuples.values_at(3, 6, 0, 4, 2, 1, 5)
+
+      Utils.sort_tuples!(tuples, orders)
+      expect(tuples).to eq expected
+    end
+  end
 end

spec/models/agents/data_output_agent_spec.rb

@@ -209,6 +209,22 @@ describe Agents::DataOutputAgent do
         })
       end
 
+      describe 'ordering' do
+        before do
+          agent.options['events_order'] = ['{{title}}']
+        end
+
+        it 'can reorder the events_to_show last events based on a Liquid expression' do
+          asc_content, _status, _content_type = agent.receive_web_request({ 'secret' => 'secret2' }, 'get', 'application/json')
+          expect(asc_content['items'].map {|i| i["title"] }).to eq(["Evolving", "Evolving again", "Evolving yet again with a past date"])
+
+          agent.options['events_order'] = [['{{title}}', 'string', true]]
+
+          desc_content, _status, _content_type = agent.receive_web_request({ 'secret' => 'secret2' }, 'get', 'application/json')
+          expect(desc_content['items']).to eq(asc_content['items'].reverse)
+        end
+      end
+
       describe "interpolating \"events\"" do
         before do
           agent.options['template']['title'] = "XKCD comics as a feed{% if events.first.site_title %} ({{events.first.site_title}}){% endif %}"

spec/models/agents/rss_agent_spec.rb

@@ -66,6 +66,21 @@ describe Agents::RssAgent do
       expect(last.payload['urls']).to eq(["https://github.com/cantino/huginn/commit/d465158f77dcd9078697e6167b50abbfdfa8b1af"])
     end
 
+    it "should emit items as events in the order specified in the events_order option" do
+      expect {
+        agent.options['events_order'] = ['{{title | replace_regex: "^[[:space:]]+", "" }}']
+        agent.check
+      }.to change { agent.events.count }.by(20)
+
+      first, *, last = agent.events.last(20)
+      expect(first.payload['title'].strip).to eq('upgrade rails and gems')
+      expect(first.payload['url']).to eq("https://github.com/cantino/huginn/commit/87a7abda23a82305d7050ac0bb400ce36c863d01")
+      expect(first.payload['urls']).to eq(["https://github.com/cantino/huginn/commit/87a7abda23a82305d7050ac0bb400ce36c863d01"])
+      expect(last.payload['title'].strip).to eq('Dashed line in a diagram indicates propagate_immediately being false.')
+      expect(last.payload['url']).to eq("https://github.com/cantino/huginn/commit/0e80f5341587aace2c023b06eb9265b776ac4535")
+      expect(last.payload['urls']).to eq(["https://github.com/cantino/huginn/commit/0e80f5341587aace2c023b06eb9265b776ac4535"])
+    end
+
     it "should track ids and not re-emit the same item when seen again" do
       agent.check
       expect(agent.memory['seen_ids']).to eq(agent.events.map {|e| e.payload['id'] })