Merge pull request #495 from cantino/fix_rss_agent_ids

Calculate IDs in RssAgent if none are available

Gemfile

@@ -108,8 +108,8 @@ group :development, :test do
   gem 'delorean'
   gem 'webmock', '~> 1.17.4', require: false
   gem 'coveralls', require: false
-  gem 'spring'
-  gem 'spring-commands-rspec'
+  # gem 'spring'
+  # gem 'spring-commands-rspec'
 end
 
 group :production do

Gemfile.lock

@@ -317,9 +317,6 @@ GEM
     slop (3.6.0)
     spectrum-rails (1.3.4)
       railties (>= 3.1)
-    spring (1.1.3)
-    spring-commands-rspec (1.0.2)
-      spring (>= 0.9.1)
     sprockets (2.11.0)
       hike (~> 1.2)
       multi_json (~> 1.0)
@@ -450,8 +447,6 @@ DEPENDENCIES
   shoulda-matchers
   slack-notifier (~> 0.5.0)
   spectrum-rails
-  spring
-  spring-commands-rspec
   therubyracer (~> 0.12.1)
   twilio-ruby (~> 3.11.5)
   twitter (~> 5.8.0)

app/models/agents/rss_agent.rb

@@ -34,6 +34,23 @@ module Agents
       }
     end
 
+    event_description <<-MD
+      Events look like:
+
+          {
+            "id": "829f845279611d7925146725317b868d",
+            "date_published": "2014-09-11 01:30:00 -0700",
+            "last_updated": "Thu, 11 Sep 2014 01:30:00 -0700",
+            "urls": [ "http://example.com/..." ],
+            "description": "Some description",
+            "content": "Some content",
+            "title": "Some title",
+            "authors": [ ... ],
+            "categories": [ ... ]
+          }
+
+    MD
+
     def working?
       event_created_within?((interpolated['expected_update_period_in_days'].presence || 10).to_i) && !recent_error_logs?
     end
@@ -55,10 +72,11 @@ module Agents
         feed.clean! if interpolated['clean'] == 'true'
         created_event_count = 0
         feed.entries.each do |entry|
-          if check_and_track(entry.id)
+          entry_id = get_entry_id(entry)
+          if check_and_track(entry_id)
             created_event_count += 1
             create_event(:payload => {
-              :id => entry.id,
+              :id => entry_id,
               :date_published => entry.date_published,
               :last_updated => entry.last_updated,
               :urls => entry.urls,
@@ -78,6 +96,10 @@ module Agents
 
     protected
 
+    def get_entry_id(entry)
+      entry.id.presence || Digest::MD5.hexdigest(entry.content)
+    end
+
     def check_and_track(entry_id)
       memory['seen_ids'] ||= []
       if memory['seen_ids'].include?(entry_id)

spec/models/agents/rss_agent_spec.rb

@@ -8,10 +8,11 @@ describe Agents::RssAgent do
     }
 
     stub_request(:any, /github.com/).to_return(:body => File.read(Rails.root.join("spec/data_fixtures/github_rss.atom")), :status => 200)
+    stub_request(:any, /SlickdealsnetFP/).to_return(:body => File.read(Rails.root.join("spec/data_fixtures/slickdeals.atom")), :status => 200)
   end
 
   let(:agent) do
-    _agent = Agents::RssAgent.new(:name => "github rss feed", :options => @valid_options)
+    _agent = Agents::RssAgent.new(:name => "rss feed", :options => @valid_options)
     _agent.user = users(:bob)
     _agent.save!
     _agent
@@ -78,4 +79,17 @@ describe Agents::RssAgent do
       agent.memory['seen_ids'].length.should == 500
     end
   end
+
+  context "when no ids are available" do
+    before do
+      @valid_options['url'] = 'http://feeds.feedburner.com/SlickdealsnetFP?format=atom'
+    end
+
+    it "calculates content MD5 sums" do
+      lambda {
+        agent.check
+      }.should change { agent.events.count }.by(79)
+      agent.memory['seen_ids'].should == agent.events.map {|e| Digest::MD5.hexdigest(e.payload['content']) }
+    end
+  end
 end