Merge branch 'master' into user-credentials

app/models/agent_log.rb

@@ -24,6 +24,6 @@ class AgentLog < ActiveRecord::Base
   end
 
   def self.log_length
-    ENV['AGENT_LOG_LENGTH'].present? ? ENV['AGENT_LOG_LENGTH'].to_i : 100
+    ENV['AGENT_LOG_LENGTH'].present? ? ENV['AGENT_LOG_LENGTH'].to_i : 200
   end
 end

app/models/agents/website_agent.rb

@@ -32,6 +32,8 @@ module Agents
 
       Note that for all of the formats, whatever you extract MUST have the same number of matches for each extractor.  E.g., if you're extracting rows, all extractors must match all rows.  For generating CSS selectors, something like [SelectorGadget](http://selectorgadget.com) may be helpful.
 
+      Can be configured to use HTTP basic auth by including the `basic_auth` parameter with `username:password`.
+
       Set `expected_update_period_in_days` to the maximum amount of time that you'd expect to pass between Events being created by this Agent.
     MD
 
@@ -70,7 +72,11 @@ module Agents
     def check
       hydra = Typhoeus::Hydra.new
       log "Fetching #{options['url']}"
-      request = Typhoeus::Request.new(options['url'], :followlocation => true)
+      request_opts = {:followlocation => true}
+      if options['basic_auth'].present?
+        request_opts[:userpwd] = options['basic_auth']
+      end
+      request = Typhoeus::Request.new(options['url'], request_opts)
       request.on_failure do |response|
         error "Failed: #{response.inspect}"
       end

spec/models/agent_log_spec.rb

@@ -78,8 +78,8 @@ describe AgentLog do
   end
 
   describe "#log_length" do
-    it "defaults to 100" do
-      AgentLog.log_length.should == 100
+    it "defaults to 200" do
+      AgentLog.log_length.should == 200
     end
   end
 end

spec/models/agents/website_agent_spec.rb

@@ -1,189 +1,220 @@
 require 'spec_helper'
 
 describe Agents::WebsiteAgent do
-  before do
-    stub_request(:any, /xkcd/).to_return(:body => File.read(Rails.root.join("spec/data_fixtures/xkcd.html")), :status => 200)
-    @site = {
-      'name' => "XKCD",
-      'expected_update_period_in_days' => 2,
-      'type' => "html",
-      'url' => "http://xkcd.com",
-      'mode' => 'on_change',
-      'extract' => {
-        'url' => {'css' => "#comic img", 'attr' => "src"},
-        'title' => {'css' => "#comic img", 'attr' => "title"}
+  describe "checking without basic auth" do
+    before do
+      stub_request(:any, /xkcd/).to_return(:body => File.read(Rails.root.join("spec/data_fixtures/xkcd.html")), :status => 200)
+      @site = {
+        'name' => "XKCD",
+        'expected_update_period_in_days' => 2,
+        'type' => "html",
+        'url' => "http://xkcd.com",
+        'mode' => 'on_change',
+        'extract' => {
+          'url' => {'css' => "#comic img", 'attr' => "src"},
+          'title' => {'css' => "#comic img", 'attr' => "title"}
+        }
       }
-    }
-    @checker = Agents::WebsiteAgent.new(:name => "xkcd", :options => @site)
-    @checker.user = users(:bob)
-    @checker.save!
-  end
-
-  describe "#check" do
-    it "should check for changes" do
-      lambda { @checker.check }.should change { Event.count }.by(1)
-      lambda { @checker.check }.should_not change { Event.count }
+      @checker = Agents::WebsiteAgent.new(:name => "xkcd", :options => @site)
+      @checker.user = users(:bob)
+      @checker.save!
     end
 
-    it "should always save events when in :all mode" do
-      lambda {
-        @site['mode'] = 'all'
+    describe "#check" do
+      it "should check for changes" do
+        lambda { @checker.check }.should change { Event.count }.by(1)
+        lambda { @checker.check }.should_not change { Event.count }
+      end
+
+      it "should always save events when in :all mode" do
+        lambda {
+          @site['mode'] = 'all'
+          @checker.options = @site
+          @checker.check
+          @checker.check
+        }.should change { Event.count }.by(2)
+      end
+
+      it "should log an error if the number of results for a set of extraction patterns differs" do
+        @site['extract']['url']['css'] = "div"
         @checker.options = @site
         @checker.check
-        @checker.check
-      }.should change { Event.count }.by(2)
-    end
-
-    it "should log an error if the number of results for a set of extraction patterns differs" do
-      @site['extract']['url']['css'] = "div"
-      @checker.options = @site
-      @checker.check
-      @checker.logs.first.message.should =~ /Got an uneven number of matches/
+        @checker.logs.first.message.should =~ /Got an uneven number of matches/
+      end
     end
-  end
 
-  describe '#working?' do
-    it 'checks if events have been received within the expected receive period' do
-      stubbed_time = Time.now
-      stub(Time).now { stubbed_time }
+    describe '#working?' do
+      it 'checks if events have been received within the expected receive period' do
+        stubbed_time = Time.now
+        stub(Time).now { stubbed_time }
 
-      @checker.should_not be_working # No events created
-      @checker.check
-      @checker.reload.should be_working # Just created events
-
-      @checker.error "oh no!"
-      @checker.reload.should_not be_working # There is a recent error
+        @checker.should_not be_working # No events created
+        @checker.check
+        @checker.reload.should be_working # Just created events
 
-      stubbed_time = 20.minutes.from_now
-      @checker.events.delete_all
-      @checker.check
-      @checker.reload.should be_working # There is a newer event now
+        @checker.error "oh no!"
+        @checker.reload.should_not be_working # There is a recent error
 
-      stubbed_time = 2.days.from_now
-      @checker.reload.should_not be_working # Two days have passed without a new event having been created
-    end
-  end
+        stubbed_time = 20.minutes.from_now
+        @checker.events.delete_all
+        @checker.check
+        @checker.reload.should be_working # There is a newer event now
 
-  describe "parsing" do
-    it "parses CSS" do
-      @checker.check
-      event = Event.last
-      event.payload['url'].should == "http://imgs.xkcd.com/comics/evolving.png"
-      event.payload['title'].should =~ /^Biologists play reverse/
+        stubbed_time = 2.days.from_now
+        @checker.reload.should_not be_working # Two days have passed without a new event having been created
+      end
     end
 
-    it "should turn relative urls to absolute" do
-      rel_site = {
-        'name' => "XKCD",
-        'expected_update_period_in_days' => 2,
-        'type' => "html",
-        'url' => "http://xkcd.com",
-        'mode' => :on_change,
-        'extract' => {
-          'url' => {'css' => "#topLeft a", 'attr' => "href"},
-          'title' => {'css' => "#topLeft a", 'text' => "true"}
-        }
-      }
-      rel = Agents::WebsiteAgent.new(:name => "xkcd", :options => rel_site)
-      rel.user = users(:bob)
-      rel.save!
-      rel.check
-      event = Event.last
-      event.payload['url'].should == "http://xkcd.com/about"
-    end
+    describe "parsing" do
+      it "parses CSS" do
+        @checker.check
+        event = Event.last
+        event.payload['url'].should == "http://imgs.xkcd.com/comics/evolving.png"
+        event.payload['title'].should =~ /^Biologists play reverse/
+      end
 
-    describe "JSON" do
-      it "works with paths" do
-        json = {
-          'response' => {
-            'version' => 2,
-            'title' => "hello!"
-          }
-        }
-        stub_request(:any, /json-site/).to_return(:body => json.to_json, :status => 200)
-        site = {
-          'name' => "Some JSON Response",
+      it "should turn relative urls to absolute" do
+        rel_site = {
+          'name' => "XKCD",
           'expected_update_period_in_days' => 2,
-          'type' => "json",
-          'url' => "http://json-site.com",
-          'mode' => 'on_change',
+          'type' => "html",
+          'url' => "http://xkcd.com",
+          'mode' => :on_change,
           'extract' => {
-            'version' => {'path' => "response.version"},
-            'title' => {'path' => "response.title"}
+            'url' => {'css' => "#topLeft a", 'attr' => "href"},
+            'title' => {'css' => "#topLeft a", 'text' => "true"}
           }
         }
-        checker = Agents::WebsiteAgent.new(:name => "Weather Site", :options => site)
-        checker.user = users(:bob)
-        checker.save!
-
-        checker.check
+        rel = Agents::WebsiteAgent.new(:name => "xkcd", :options => rel_site)
+        rel.user = users(:bob)
+        rel.save!
+        rel.check
         event = Event.last
-        event.payload['version'].should == 2
-        event.payload['title'].should == "hello!"
+        event.payload['url'].should == "http://xkcd.com/about"
       end
 
-      it "can handle arrays" do
-        json = {
-          'response' => {
-            'data' => [
-              {'title' => "first", 'version' => 2},
-              {'title' => "second", 'version' => 2.5}
-            ]
+      describe "JSON" do
+        it "works with paths" do
+          json = {
+            'response' => {
+              'version' => 2,
+              'title' => "hello!"
+            }
           }
-        }
-        stub_request(:any, /json-site/).to_return(:body => json.to_json, :status => 200)
-        site = {
-          'name' => "Some JSON Response",
-          'expected_update_period_in_days' => 2,
-          'type' => "json",
-          'url' => "http://json-site.com",
-          'mode' => 'on_change',
-          'extract' => {
-            :title => {'path' => "response.data[*].title"},
-            :version => {'path' => "response.data[*].version"}
+          stub_request(:any, /json-site/).to_return(:body => json.to_json, :status => 200)
+          site = {
+            'name' => "Some JSON Response",
+            'expected_update_period_in_days' => 2,
+            'type' => "json",
+            'url' => "http://json-site.com",
+            'mode' => 'on_change',
+            'extract' => {
+              'version' => {'path' => "response.version"},
+              'title' => {'path' => "response.title"}
+            }
           }
-        }
-        checker = Agents::WebsiteAgent.new(:name => "Weather Site", :options => site)
-        checker.user = users(:bob)
-        checker.save!
+          checker = Agents::WebsiteAgent.new(:name => "Weather Site", :options => site)
+          checker.user = users(:bob)
+          checker.save!
 
-        lambda {
           checker.check
-        }.should change { Event.count }.by(2)
-
-        event = Event.all[-1]
-        event.payload['version'].should == 2.5
-        event.payload['title'].should == "second"
+          event = Event.last
+          event.payload['version'].should == 2
+          event.payload['title'].should == "hello!"
+        end
+
+        it "can handle arrays" do
+          json = {
+            'response' => {
+              'data' => [
+                {'title' => "first", 'version' => 2},
+                {'title' => "second", 'version' => 2.5}
+              ]
+            }
+          }
+          stub_request(:any, /json-site/).to_return(:body => json.to_json, :status => 200)
+          site = {
+            'name' => "Some JSON Response",
+            'expected_update_period_in_days' => 2,
+            'type' => "json",
+            'url' => "http://json-site.com",
+            'mode' => 'on_change',
+            'extract' => {
+              :title => {'path' => "response.data[*].title"},
+              :version => {'path' => "response.data[*].version"}
+            }
+          }
+          checker = Agents::WebsiteAgent.new(:name => "Weather Site", :options => site)
+          checker.user = users(:bob)
+          checker.save!
+
+          lambda {
+            checker.check
+          }.should change { Event.count }.by(2)
+
+          event = Event.all[-1]
+          event.payload['version'].should == 2.5
+          event.payload['title'].should == "second"
+
+          event = Event.all[-2]
+          event.payload['version'].should == 2
+          event.payload['title'].should == "first"
+        end
+
+        it "stores the whole object if :extract is not specified" do
+          json = {
+            'response' => {
+              'version' => 2,
+              'title' => "hello!"
+            }
+          }
+          stub_request(:any, /json-site/).to_return(:body => json.to_json, :status => 200)
+          site = {
+            'name' => "Some JSON Response",
+            'expected_update_period_in_days' => 2,
+            'type' => "json",
+            'url' => "http://json-site.com",
+            'mode' => 'on_change'
+          }
+          checker = Agents::WebsiteAgent.new(:name => "Weather Site", :options => site)
+          checker.user = users(:bob)
+          checker.save!
 
-        event = Event.all[-2]
-        event.payload['version'].should == 2
-        event.payload['title'].should == "first"
+          checker.check
+          event = Event.last
+          event.payload['response']['version'].should == 2
+          event.payload['response']['title'].should == "hello!"
+        end
       end
+    end
+  end
 
-      it "stores the whole object if :extract is not specified" do
-        json = {
-          'response' => {
-            'version' => 2,
-            'title' => "hello!"
-          }
-        }
-        stub_request(:any, /json-site/).to_return(:body => json.to_json, :status => 200)
-        site = {
-          'name' => "Some JSON Response",
-          'expected_update_period_in_days' => 2,
-          'type' => "json",
-          'url' => "http://json-site.com",
-          'mode' => 'on_change'
-        }
-        checker = Agents::WebsiteAgent.new(:name => "Weather Site", :options => site)
-        checker.user = users(:bob)
-        checker.save!
+  describe "checking with http basic auth" do
+    before do
+      stub_request(:any, /user:pass/).to_return(:body => File.read(Rails.root.join("spec/data_fixtures/xkcd.html")), :status => 200)
+      @site = {
+        'name' => "XKCD",
+        'expected_update_period_in_days' => 2,
+        'type' => "html",
+        'url' => "http://www.example.com",
+        'mode' => 'on_change',
+        'extract' => {
+          'url' => {'css' => "#comic img", 'attr' => "src"},
+          'title' => {'css' => "#comic img", 'attr' => "title"}
+        },
+        'basic_auth' => "user:pass"
+      }
+      @checker = Agents::WebsiteAgent.new(:name => "auth", :options => @site)
+      @checker.user = users(:bob)
+      @checker.save!
+    end
 
-        checker.check
-        event = Event.last
-        event.payload['response']['version'].should == 2
-        event.payload['response']['title'].should == "hello!"
+    describe "#check" do
+      it "should check for changes" do
+        lambda { @checker.check }.should change { Event.count }.by(1)
+        lambda { @checker.check }.should_not change { Event.count }
       end
     end
   end
+
 end