scrape_example.py

#!/usr/bin/python
#
import os, sys
import pycurl
import time
import datetime
from StringIO import StringIO
from datetime import date, timedelta
from HTMLParser import HTMLParser

class LinksParser(HTMLParser):
  def __init__(self):
    HTMLParser.__init__(self)
    self.recording = 0
    self.data = []

  def handle_starttag(self, tag, attributes):
    if tag == 'input':
      previous_val = 'none'
      for name, value in attributes:
        if name == 'name' and value == 'sessionid':
          print ''
          print 'name: '+ name
          print 'value: '+ value
          previous_val = 'sessionid'
        if name == 'value' and previous_val == 'sessionid':
          print ''
          print 'name of value: '+ name
          print 'value: '+ value
          curSessIdPseg = value
          global curSessIdPseg
    else:
      return

curl = pycurl.Curl()
curl.setopt(pycurl.URL, "http://example.com")
curl.setopt(pycurl.POSTFIELDS, 'val1=1&pdf=0')
cookie_file_name = os.tempnam()
curl.setopt(pycurl.COOKIEFILE, 'cookie.txt')
curl.setopt(pycurl.VERBOSE, True)
curl.setopt(pycurl.SSL_VERIFYPEER, 0)
curl.setopt(pycurl.SSL_VERIFYHOST, 0)
curl.setopt(pycurl.SSLVERSION, 3)
curl.setopt(pycurl.CAINFO, "https.crt")
curl.setopt(pycurl.WRITEFUNCTION, memString1.write)
# 
#
# perform first page request
# this submits the login info and gets you to the second page
#
#
curl.perform()

page1 = memString1.getvalue()

p = LinksParser()
p.feed(page1)
session_id_page1 = curSessIdPseg
print session_id_page1
p.close()

time.sleep(2)