Add a Liquid filter `uri_expand`

This implements #853.

CHANGES.md

@@ -1,5 +1,6 @@
 # Changes
 
+* Jun 15, 2015   - Liquid filter `uri_expand` added.
 * Jun 12, 2015   - RSSAgent can now accept an array of URLs.
 * Jun 8, 2015    - WebsiteAgent includes a `use_namespaces` option to enable XML namespaces.
 * May 27, 2015   - Validation warns user if they have not provided a `path` when using JSONPath in WebsiteAgent.

app/concerns/liquid_interpolatable.rb

@@ -132,6 +132,42 @@ module LiquidInterpolatable
       nil
     end
 
+    # Get the destination URL of a given URL by recursively following
+    # redirects, up to 5 times in a row.  If a given string is not a
+    # valid absolute HTTP URL, or any error occurs while following
+    # redirects, the original string is returned.
+    def uri_expand(url, limit = 5)
+      uri = URI(url)
+
+      http = Faraday.new do |builder|
+        builder.adapter :net_http
+        # builder.use FaradayMiddleware::FollowRedirects, limit: limit
+        # ...does not handle non-HTTP URLs.
+      end
+
+      limit.times do
+        begin
+          case uri
+          when URI::HTTP
+            response = http.head(uri)
+            case response.status
+            when 301, 302, 303, 307
+              if location = response['location']
+                uri += location
+                next
+              end
+            end
+          end
+        rescue
+        end
+
+        return uri.to_s
+      end
+
+      # too many redirections
+      url
+    end
+
     # Escape a string for use in XPath expression
     def to_xpath(string)
       subs = string.to_s.scan(/\G(?:\A\z|[^"]+|[^']+)/).map { |x|

spec/concerns/liquid_interpolatable_spec.rb

@@ -96,4 +96,72 @@ describe LiquidInterpolatable::Filters do
       expect(@agent.interpolated['foo']).to eq('/dir/foo/index.html')
     end
   end
+
+  describe 'uri_expand' do
+    before do
+      stub_request(:head, 'https://t.co.x/aaaa').
+        to_return(status: 301, headers: { Location: 'https://bit.ly.x/bbbb' })
+      stub_request(:head, 'https://bit.ly.x/bbbb').
+        to_return(status: 301, headers: { Location: 'http://tinyurl.com.x/cccc' })
+      stub_request(:head, 'http://tinyurl.com.x/cccc').
+        to_return(status: 301, headers: { Location: 'http://www.example.com/welcome' })
+
+      (1..5).each do |i|
+        stub_request(:head, "http://2many.x/#{i}").
+          to_return(status: 301, headers: { Location: "http://2many.x/#{i+1}" })
+      end
+      stub_request(:head, 'http://2many.x/6').
+        to_return(status: 301, headers: { 'Content-Length' => '5' })
+    end
+
+    it 'should follow redirects' do
+      expect(@filter.uri_expand('https://t.co.x/aaaa')).to eq('http://www.example.com/welcome')
+    end
+
+    it 'should respect the limit for the number of redirects' do
+      expect(@filter.uri_expand('http://2many.x/1')).to eq('http://2many.x/1')
+      expect(@filter.uri_expand('http://2many.x/1', 6)).to eq('http://2many.x/6')
+    end
+
+    it 'should detect a redirect loop' do
+      stub_request(:head, 'http://bad.x/aaaa').
+        to_return(status: 301, headers: { Location: 'http://bad.x/bbbb' })
+      stub_request(:head, 'http://bad.x/bbbb').
+        to_return(status: 301, headers: { Location: 'http://bad.x/aaaa' })
+
+      expect(@filter.uri_expand('http://bad.x/aaaa')).to eq('http://bad.x/aaaa')
+    end
+
+    it 'should be able to handle an FTP URL' do
+      stub_request(:head, 'http://downloads.x/aaaa').
+        to_return(status: 301, headers: { Location: 'http://downloads.x/download?file=aaaa.zip' })
+      stub_request(:head, 'http://downloads.x/download').
+        with(query: { file: 'aaaa.zip' }).
+        to_return(status: 301, headers: { Location: 'ftp://downloads.x/pub/aaaa.zip' })
+
+      expect(@filter.uri_expand('http://downloads.x/aaaa')).to eq('ftp://downloads.x/pub/aaaa.zip')
+    end
+
+    describe 'used in interpolation' do
+      before do
+        @agent = Agents::InterpolatableAgent.new(name: "test")
+      end
+
+      it 'should follow redirects' do
+        @agent.interpolation_context['short_url'] = 'https://t.co.x/aaaa'
+        @agent.options['long_url'] = '{{ short_url | uri_expand }}'
+        expect(@agent.interpolated['long_url']).to eq('http://www.example.com/welcome')
+      end
+
+      it 'should respect the limit for the number of redirects' do
+        @agent.interpolation_context['short_url'] = 'http://2many.x/1'
+        @agent.options['long_url'] = '{{ short_url | uri_expand }}'
+        expect(@agent.interpolated['long_url']).to eq('http://2many.x/1')
+
+        @agent.interpolation_context['short_url'] = 'http://2many.x/1'
+        @agent.options['long_url'] = '{{ short_url | uri_expand:6 }}'
+        expect(@agent.interpolated['long_url']).to eq('http://2many.x/6')
+      end
+    end
+  end
 end