Huginn scenarios

rss-full-text-scrape-to-stevedore.json

{
  "schema_version": 1,
  "name": "RSS Full Text Scrape to Stevedore",
  "description": "Follows an RSS feed and visits every feed item to save the full text into Stevedore",
  "source_url": "https://mighty-retreat-48764.herokuapp.com/scenarios/7/export",
  "guid": "1dcba6751110ed460f0a83273953535e",
  "tag_fg_color": "#ffffff",
  "tag_bg_color": "#5bc0de",
  "exported_at": "2016-04-18T22:41:41Z",
  "agents": [
    {
      "type": "Agents::PostAgent",
      "name": "EEOC Post to Stevedore",
      "disabled": false,
      "guid": "a3817606515748c31c3bfa32465ff8e2",
      "options": {
        "post_url": "http://example.com/stevedore",
        "expected_receive_period_in_days": "1",
        "content_type": "json",
        "method": "post",
        "payload": {
          "id": "{{ id }}",
          "title": "{{ title }}",
          "source_url": "{{ url }}",
          "file": {
            "title": "{{ title }}",
            "file": "{{ body_text }}"
          },
          "analyzed": {
            "body": "{{ body_text }}",
            "metadata": {
              "Content-Type": "text/plain"
            }
          }
        },
        "headers": {
        },
        "emit_events": "false",
        "no_merge": "true"
      },
      "schedule": "never",
      "keep_events_for": 0,
      "propagate_immediately": true
    },
    {
      "type": "Agents::RssAgent",
      "name": "EEOC RSS",
      "disabled": false,
      "guid": "bcff007b3576fb801ffbb1288e95e920",
      "options": {
        "expected_update_period_in_days": "5",
        "clean": "false",
        "url": "https://www1.eeoc.gov/eeoc/newsroom/index.cfm?xml=eeoc_press,RSS2.0full"
      },
      "schedule": "every_5h",
      "keep_events_for": 2592000
    },
    {
      "type": "Agents::WebsiteAgent",
      "name": "EEOC News Page Scrape",
      "disabled": false,
      "guid": "f68d5d70c4d0265eef284486a299dbd4",
      "options": {
        "expected_update_period_in_days": "2",
        "url_from_event": "{{url}}",
        "type": "html",
        "mode": "merge",
        "extract": {
          "title": {
            "css": "div h1",
            "value": "normalize-space(.)"
          },
          "body_text": {
            "css": "#CS_Element_Content",
            "value": "normalize-space(.)"
          }
        }
      },
      "schedule": "never",
      "keep_events_for": 0,
      "propagate_immediately": true
    }
  ],
  "links": [
    {
      "source": 1,
      "receiver": 2
    },
    {
      "source": 2,
      "receiver": 0
    }
  ],
  "control_links": [

  ]
}

website-email-notifier.json

{
  "schema_version": 1,
  "name": "Website Notifier",
  "description": "Scrape a website and notify via email when it changes",
  "source_url": "https://mighty-retreat-48764.herokuapp.com/scenarios/1/export",
  "guid": "ce4d15de1dc26cd4dbc17c88cfc03c1c",
  "tag_fg_color": "#ffffff",
  "tag_bg_color": "#5bc0de",
  "exported_at": "2016-04-18T16:56:11Z",
  "agents": [
    {
      "type": "Agents::EmailAgent",
      "name": "Donald Trump Email Notifier",
      "disabled": false,
      "guid": "9c61eccfd02f22c23d8fe461bbc46805",
      "options": {
        "subject": "Donald Trump's Website Changed",
        "headline": "Your notification:",
        "expected_receive_period_in_days": "2"
      },
      "propagate_immediately": true
    },
    {
      "type": "Agents::WebsiteAgent",
      "name": "Donald Trump Wall",
      "disabled": false,
      "guid": "f820bf7d25c0fc527280a0c05d905b1f",
      "options": {
        "expected_update_period_in_days": "2",
        "url": "https://www.donaldjtrump.com/positions/pay-for-the-wall",
        "type": "html",
        "mode": "on_change",
        "extract": {
          "text": {
            "css": "article.press_item.entry",
            "value": "normalize-space(.)"
          }
        }
      },
      "schedule": "every_30m",
      "keep_events_for": 0,
      "propagate_immediately": false
    }
  ],
  "links": [
    {
      "source": 1,
      "receiver": 0
    }
  ],
  "control_links": [

  ]
}

website-slack-notifier.json

{
  "schema_version": 1,
  "name": "Website Slack Notifier",
  "description": "Scrape a website into slack",
  "source_url": false,
  "guid": "2176cbdc72468e0044c7895dff5cb6e4",
  "tag_fg_color": "#ffffff",
  "tag_bg_color": "#5bc0de",
  "exported_at": "2016-04-18T21:05:01Z",
  "agents": [
    {
      "type": "Agents::SlackAgent",
      "name": "codeconvening-test slack",
      "disabled": false,
      "guid": "b90e88f7babfdef67940437f852e90f8",
      "options": {
        "webhook_url": "{% credential slack_webhook %}",
        "channel": "#codeconvening-test",
        "username": "Huginn",
        "message": "{{text}}",
        "icon": ""
      },
      "propagate_immediately": false
    },
    {
      "type": "Agents::WebsiteAgent",
      "name": "Donald Trump Wall",
      "disabled": false,
      "guid": "f820bf7d25c0fc527280a0c05d905b1f",
      "options": {
        "expected_update_period_in_days": "2",
        "url": "https://www.donaldjtrump.com/positions/pay-for-the-wall",
        "type": "html",
        "mode": "on_change",
        "extract": {
          "text": {
            "css": "article.press_item.entry",
            "value": "normalize-space(.)"
          }
        }
      },
      "schedule": "every_30m",
      "keep_events_for": 0,
      "propagate_immediately": false
    }
  ],
  "links": [
    {
      "source": 1,
      "receiver": 0
    }
  ],
  "control_links": [

  ]
}

website-to-stevedore.json

{
  "schema_version": 1,
  "name": "Website to Stevedore",
  "description": "Scrape a website and insert it's changes into Stevedore",
  "source_url": false,
  "guid": "4c970fc0789eb9c9117a33b4a60bf776",
  "tag_fg_color": "#ffffff",
  "tag_bg_color": "#5bc0de",
  "exported_at": "2016-04-18T21:42:51Z",
  "agents": [
    {
      "type": "Agents::PostAgent",
      "name": "Donald Trump Stevedore Insert Agent",
      "disabled": false,
      "guid": "83ebd85c425a91a222711563e83f6425",
      "options": {
        "post_url": "http://example.com/stevedore",
        "expected_receive_period_in_days": "1",
        "content_type": "json",
        "method": "post",
        "payload": {
          "id": "https://www.donaldjtrump.com/positions/pay-for-the-wall",
          "title": "Donald Trump Pay for the Wall",
          "source_url": "https://www.donaldjtrump.com/positions/pay-for-the-wall",
          "file": {
            "title": "Donald Trump Pay for the Wall",
            "file": "{{ text }}"
          },
          "analyzed": {
            "body": "{{ text }}",
            "metadata": {
              "Content-Type": "text/plain"
            }
          }
        },
        "headers": {
        },
        "emit_events": "false",
        "no_merge": "true"
      },
      "schedule": "never",
      "keep_events_for": 0,
      "propagate_immediately": true
    },
    {
      "type": "Agents::WebsiteAgent",
      "name": "Donald Trump Wall",
      "disabled": false,
      "guid": "f820bf7d25c0fc527280a0c05d905b1f",
      "options": {
        "expected_update_period_in_days": "2",
        "url": "https://www.donaldjtrump.com/positions/pay-for-the-wall",
        "type": "html",
        "mode": "on_change",
        "extract": {
          "text": {
            "css": "article.press_item.entry",
            "value": "normalize-space(.)"
          }
        }
      },
      "schedule": "every_30m",
      "keep_events_for": 0,
      "propagate_immediately": false
    }
  ],
  "links": [
    {
      "source": 1,
      "receiver": 0
    }
  ],
  "control_links": [

  ]
}