huginn/app/models/agents/rss_agent.rb

module Agents
  class RssAgent < Agent
    include WebRequestConcern

    cannot_receive_events!
    can_dry_run!
    default_schedule "every_1d"

    gem_dependency_check { defined?(Feedjira) }

    DEFAULT_EVENTS_ORDER = [['{{date_published}}', 'time'], ['{{last_updated}}', 'time']]

    description do
      <<~MD
        The RSS Agent consumes RSS feeds and emits events when they change.

        This agent, using [Feedjira](https://github.com/feedjira/feedjira) as a base, can parse various types of RSS and Atom feeds and has some special handlers for FeedBurner, iTunes RSS, and so on.  However, supported fields are limited by its general and abstract nature.  For complex feeds with additional field types, we recommend using a WebsiteAgent.  See [this example](https://github.com/huginn/huginn/wiki/Agent-configuration-examples#itunes-trailers).

        If you want to *output* an RSS feed, use the DataOutputAgent.

        Options:

          * `url` - The URL of the RSS feed (an array of URLs can also be used; items with identical guids across feeds will be considered duplicates).
          * `include_feed_info` - Set to `true` to include feed information in each event.
          * `clean` - Set to `true` to sanitize `description` and `content` as HTML fragments, removing unknown/unsafe elements and attributes.
          * `expected_update_period_in_days` - How often you expect this RSS feed to change.  If more than this amount of time passes without an update, the Agent will mark itself as not working.
          * `headers` - When present, it should be a hash of headers to send with the request.
          * `basic_auth` - Specify HTTP basic auth parameters: `"username:password"`, or `["username", "password"]`.
          * `disable_ssl_verification` - Set to `true` to disable ssl verification.
          * `disable_url_encoding` - Set to `true` to disable url encoding.
          * `force_encoding` - Set `force_encoding` to an encoding name if the website is known to respond with a missing, invalid or wrong charset in the Content-Type header.  Note that a text content without a charset is taken as encoded in UTF-8 (not ISO-8859-1).
          * `user_agent` - A custom User-Agent name (default: "Faraday v#{Faraday::VERSION}").
          * `max_events_per_run` - Limit number of events created (items parsed) per run for feed.
          * `remembered_id_count` - Number of IDs to keep track of and avoid re-emitting (default: 500).

        # Ordering Events

        #{description_events_order}

        In this Agent, the default value for `events_order` is `#{DEFAULT_EVENTS_ORDER.to_json}`.
      MD
    end

    def default_options
      {
        'expected_update_period_in_days' => "5",
        'clean' => 'false',
        'url' => "https://github.com/huginn/huginn/commits/master.atom"
      }
    end

    event_description <<~MD
      Events look like:

          {
            "feed": {
              "id": "...",
              "type": "atom",
              "generator": "...",
              "url": "http://example.com/",
              "links": [
                { "href": "http://example.com/", "rel": "alternate", "type": "text/html" },
                { "href": "http://example.com/index.atom", "rel": "self", "type": "application/atom+xml" }
              ],
              "title": "Some site title",
              "description": "Some site description",
              "copyright": "...",
              "icon": "http://example.com/icon.png",
              "authors": [ "..." ],

              "itunes_block": "no",
              "itunes_categories": [
                "Technology", "Gadgets",
                "TV & Film",
                "Arts", "Food"
              ],
              "itunes_complete": "yes",
              "itunes_explicit": "yes",
              "itunes_image": "http://...",
              "itunes_new_feed_url": "http://...",
              "itunes_owners": [ "John Doe <john.doe@example.com>" ],
              "itunes_subtitle": "...",
              "itunes_summary": "...",
              "language": "en-US",

              "date_published": "2014-09-11T01:30:00-07:00",
              "last_updated": "2014-09-11T01:30:00-07:00"
            },
            "id": "829f845279611d7925146725317b868d",
            "url": "http://example.com/...",
            "urls": [ "http://example.com/..." ],
            "links": [
              { "href": "http://example.com/...", "rel": "alternate" },
            ],
            "title": "Some title",
            "description": "Some description",
            "content": "Some content",
            "authors": [ "Some Author <email@address>" ],
            "categories": [ "..." ],
            "image": "http://example.com/...",
            "enclosure": {
              "url" => "http://example.com/file.mp3", "type" => "audio/mpeg", "length" => "123456789"
            },

            "itunes_block": "no",
            "itunes_closed_captioned": "yes",
            "itunes_duration": "04:34",
            "itunes_explicit": "yes",
            "itunes_image": "http://...",
            "itunes_order": "1",
            "itunes_subtitle": "...",
            "itunes_summary": "...",

            "date_published": "2014-09-11T01:30:00-0700",
            "last_updated": "2014-09-11T01:30:00-0700"
          }

      Some notes:

      - The `feed` key is present only if `include_feed_info` is set to true.
      - The keys starting with `itunes_`, and `language` are only present when the feed is a podcast.  See [Podcasts Connect Help](https://help.apple.com/itc/podcasts_connect/#/itcb54353390) for details.
      - Each element in `authors` and `itunes_owners` is a string normalized in the format "*name* <*email*> (*url*)", where each space-separated part is optional.
      - Timestamps are converted to the ISO 8601 format.
    MD

    def working?
      event_created_within?((interpolated['expected_update_period_in_days'].presence || 10).to_i) && !recent_error_logs?
    end

    def validate_options
      errors.add(:base, "url is required") unless options['url'].present?

      unless options['expected_update_period_in_days'].present? && options['expected_update_period_in_days'].to_i > 0
        errors.add(:base,
                   "Please provide 'expected_update_period_in_days' to indicate how many days can pass without an update before this Agent is considered to not be working")
      end

      if options['remembered_id_count'].present? && options['remembered_id_count'].to_i < 1
        errors.add(:base,
                   "Please provide 'remembered_id_count' as a number bigger than 0 indicating how many IDs should be saved to distinguish between new and old IDs in RSS feeds. Delete option to use default (500).")
      end

      validate_web_request_options!
      validate_events_order
    end

    def events_order(key = SortableEvents::EVENTS_ORDER_KEY)
      if key == SortableEvents::EVENTS_ORDER_KEY
        super.presence || DEFAULT_EVENTS_ORDER
      else
        raise ArgumentError, "unsupported key: #{key}"
      end
    end

    def check
      check_urls(Array(interpolated['url']))
    end

    protected

    def check_urls(urls)
      new_events = []
      max_events = (interpolated['max_events_per_run'].presence || 0).to_i

      urls.each do |url|
        response = faraday.get(url)
        if response.success?
          feed = Feedjira.parse(preprocessed_body(response))
          new_events.concat feed_to_events(feed)
        else
          error "Failed to fetch #{url}: #{response.inspect}"
        end
      rescue StandardError => e
        error "Failed to fetch #{url} with message '#{e.message}': #{e.backtrace}"
      end

      events = sort_events(new_events).select.with_index { |event, index|
        check_and_track(event.payload[:id]) &&
          !(max_events && max_events > 0 && index >= max_events)
      }
      create_events(events)
      log "Fetched #{urls.to_sentence} and created #{events.size} event(s)."
    end

    def remembered_id_count
      (options['remembered_id_count'].presence || 500).to_i
    end

    def check_and_track(entry_id)
      memory['seen_ids'] ||= []
      if memory['seen_ids'].include?(entry_id)
        false
      else
        memory['seen_ids'].unshift entry_id
        memory['seen_ids'].pop(memory['seen_ids'].length - remembered_id_count) if memory['seen_ids'].length > remembered_id_count
        true
      end
    end

    unless dependencies_missing?
      require 'feedjira_extension'
    end

    def preprocessed_body(response)
      body = response.body
      case body.encoding
      when Encoding::ASCII_8BIT
        # Encoding is unknown from the Content-Type, so let the SAX
        # parser detect it from the content.
      else
        # Encoding is already known, so do not let the parser detect
        # it from the XML declaration in the content.
        body.sub!(/(?<noenc>\A\u{FEFF}?\s*<\?xml(?:\s+\w+(?<av>\s*=\s*(?:'[^']*'|"[^"]*")))*?)\s+encoding\g<av>/,
                  '\\k<noenc>')
      end
      body
    end

    def feed_data(feed)
      type =
        case feed.class.name
        when /Atom/
          'atom'
        else
          'rss'
        end

      {
        id: feed.feed_id,
        type:,
        url: feed.url,
        links: feed.links,
        title: feed.title,
        description: feed.description,
        copyright: feed.copyright,
        generator: feed.generator,
        icon: feed.icon,
        authors: feed.authors,
        date_published: feed.date_published,
        last_updated: feed.last_updated,
        **itunes_feed_data(feed)
      }
    end

    def itunes_feed_data(feed)
      data = {}
      case feed
      when Feedjira::Parser::ITunesRSS
        %i[
          itunes_block
          itunes_categories
          itunes_complete
          itunes_explicit
          itunes_image
          itunes_new_feed_url
          itunes_owners
          itunes_subtitle
          itunes_summary
          language
        ].each { |attr|
          next unless value = feed.try(attr).presence

          data[attr] =
            case attr
            when :itunes_summary
              clean_fragment(value)
            else
              value
            end
        }
      end
      data
    end

    def entry_data(entry)
      {
        id: entry.id,
        url: entry.url,
        urls: Array(entry.url) | entry.links.map(&:href),
        links: entry.links,
        title: entry.title,
        description: clean_fragment(entry.summary),
        content: clean_fragment(entry.content || entry.summary),
        image: entry.try(:image),
        enclosure: entry.enclosure,
        authors: entry.authors,
        categories: Array(entry.try(:categories)),
        date_published: entry.date_published,
        last_updated: entry.last_updated,
        **itunes_entry_data(entry)
      }
    end

    def itunes_entry_data(entry)
      data = {}
      case entry
      when Feedjira::Parser::ITunesRSSItem
        %i[
          itunes_block
          itunes_closed_captioned
          itunes_duration
          itunes_explicit
          itunes_image
          itunes_order
          itunes_subtitle
          itunes_summary
        ].each { |attr|
          if value = entry.try(attr).presence
            data[attr] = value
          end
        }
      end
      data
    end

    def feed_to_events(feed)
      payload_base = {}

      if boolify(interpolated['include_feed_info'])
        payload_base[:feed] = feed_data(feed)
      end

      feed.entries.map { |entry|
        Event.new(payload: payload_base.merge(entry_data(entry)))
      }
    end

    def clean_fragment(fragment)
      if boolify(interpolated['clean']) && fragment.present?
        Loofah.scrub_fragment(fragment, :prune).to_s
      else
        fragment
      end
    end
  end
end