Python requests scraper for https://l3com.taleo.net/careersection/l3_ext_us/jobsearch.ftl#

l3com_scraper.py

#!/usr/bin/env python

import json
import requests

from bs4 import BeautifulSoup

class L3ComScraper(object):
    def __init__(self):
        self.session = requests.Session()
        self.url = 'https://l3com.taleo.net/careersection/rest/jobboard/searchjobs'
        self.params = {
            'lang': 'en',
            'portal': '2140452562'
        }
        self.data = {
            "multilineEnabled": False,
            "sortingSelection": {
                "sortBySelectionParam":"1",
                "ascendingSortingOrder":"false"
            },
            "fieldData": {
                "fields": {
                    "KEYWORD":"",
                    "LOCATION":""
                },
                "valid": True
            },
            "filterSelectionParam": {
                "searchFilterSelections": [
                    {
                        "id": "LOCATION",
                        "selectedValues":[]
                    },
                    {
                        "id":"JOB_FIELD",
                        "selectedValues":[]
                    },
                    {
                        "id": "ORGANIZATION",
                        "selectedValues": []
                    },
                    {
                        "id": "JOB_SCHEDULE",
                        "selectedValues":[]
                    },
                    {
                        "id": "JOB_TYPE",
                        "selectedValues":[]
                    },
                    {
                        "id": "JOB_LEVEL",
                        "selectedValues": []
                    },
                    {
                        "id": "WILL_TRAVEL",
                        "selectedValues": []
                    },
                    {
                        "id": "POSTING_DATE",
                        "selectedValues": []
                    }
                ]
            },
            "advancedSearchFiltersSelectionParam": {
                "searchFilterSelections": [
                    {
                        "id": "LOCATION",
                        "selectedValues": []
                    },
                    {
                        "id": "JOB_FIELD",
                        "selectedValues": []
                    },
                    {
                        "id": "ORGANIZATION",
                        "selectedValues": []
                    },
                    {
                        "id": "JOB_NUMBER",
                        "selectedValues": []
                    },
                    {
                        "id": "JOB_LEVEL",
                        "selectedValues": []
                    },
                    {
                        "id": "JOB_SHIFT",
                        "selectedValues": []
                    },
                    {
                        "id": "JOB_SCHEDULE",
                        "selectedValues": []
                    },
                    {
                        "id": "JOB_TYPE",
                        "selectedValues": []
                    },
                    {
                        "id": "WILL_TRAVEL",
                        "selectedValues": []
                    },
                    {
                        "id": "URGENT_JOB",
                        "selectedValues": []
                    },
                    {
                        "id": "STUDY_LEVEL",
                        "selectedValues":[]
                    }
                ]
            },
            "pageNo":1
        }

    def scrape(self):
        #
        # From SearchHandler.js (search for progressIndicator)
        #
        # $.ajax({
        #   type : 'POST',
        #   url : '/careersection/rest/jobboard/searchjobs?lang='
        #         + FacetedSearchSettings.lang
        #         + '&portal='
        #         + FacetedSearchSettings.portalNo
        #         + URLBuilder.getReqSource(),
        #   data : currentInstance.formToJSON(event),
        #   dataType : 'json',
        #   headers : {
        #     'tz' : new DateUtils().getTimeZone()
        #   },
        #   contentType : 'application/json',
        headers = {
            'Content-Type': 'application/json',
            'tz': 'GMT-04:00'
        }

        resp = self.session.post(self.url, headers=headers, params=self.params, data=json.dumps(self.data))
        data = resp.json()
        
        print json.dumps(data, indent=2)
        
if __name__ == '__main__':
    scraper = L3ComScraper()
    scraper.scrape()