LinkedinScrape.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Linkedin Scraper"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To approach this problem systematically first writing down all the points which need to be taken care of:\n",
    "1. Take care of Ip blocking by giving time difference between request\n",
    "2. Cookie experiation as simple account"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "99081419\n",
      "30377\n",
      "77339\n",
      "180393\n",
      "192331\n",
      "207134\n",
      "217588\n",
      "271635\n",
      "295961\n",
      "324907\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "cookies = {\n",
    "    'sessionid': 'eyJkamFuZ29fdGltZXpvbmUiOiJBc2lhL0tvbGthdGEifQ:1Ykwc5:_4Now3f6hlj8wA0T0JUU5RZfGiQ',\n",
    "'li_at': 'AQECARkM06IEzXSDAAABTR5yjMsAAAFNH1RRBUst4ENksw2guJrXARM-cB48luIqEa9-98sCbygbQe-nsFvoK2WI4fycfdkHj22NmL5fJok-QARaOD_KX5olzwllultd27Xr-WKz_-aOgaS4Sio7BYk'\n",
    "}\n",
    "\n",
    "headers = {\n",
    "    'X-LinkedIn-traceDataContext': 'X-LI-ORIGIN-UUID=DJyFss0E2xNgKGIJsSoAAA==',\n",
    "    'Accept-Encoding': 'gzip, deflate, sdch',\n",
    "    'Accept-Language': 'en-US,en;q=0.8,hi;q=0.6',\n",
    "    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36',\n",
    "    'Accept': '*/*',\n",
    "    'Referer': 'https://www.linkedin.com/vsearch/p?title=certification&openAdvancedForm=true&titleScope=C&count=100&locationType=Y&rsid=4202710101430742588356&orig=ADVS&openFacets=N,G,CC&page_num=2&pt=people',\n",
    "    'X-Requested-With': 'XMLHttpRequest',\n",
    "    'Connection': 'keep-alive',\n",
    "    'X-IsAJAXForm': '1',\n",
    "}\n",
    "\n",
    "r = requests.get('https://www.linkedin.com/vsearch/pj?title=certification&openAdvancedForm=true&titleScope=C&locationType=Y&count=100&rsid=4202710101430742588356&orig=ADVS&openFacets=N,G,CC&page_num=2&pt=people&rnd=1430742607615', headers=headers, cookies=cookies)\n",
    "#r = requests.get('https://www.linkedin.com/vsearch/pj?title=certification&openAdvancedForm=true&titleScope=C&locationType=Y&count=100&rsid=4202710101430742588356&orig=ADVS&openFacets=N,G,CC&page_num=2&pt=people&rnd=1430742607615', headers=headers, cookies=cookies)\n",
    "\n",
    "for profile in r.json()[\"content\"][\"page\"][\"voltron_unified_search_json\"][\"search\"][\"results\"]:\n",
    "    print(profile[\"person\"][\"id\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('Jenny', 'Hayes, MSOD', 'UC Santa Cruz Extension Silicon Valley')"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import requests\n",
    "cookies = {\n",
    "    'sessionid': 'eyJkamFuZ29fdGltZXpvbmUiOiJBc2lhL0tvbGthdGEifQ:1Ykwc5:_4Now3f6hlj8wA0T0JUU5RZfGiQ',\n",
    "'li_at': 'AQECARkM06IEzXSDAAABTR5yjMsAAAFNH1RRBUst4ENksw2guJrXARM-cB48luIqEa9-98sCbygbQe-nsFvoK2WI4fycfdkHj22NmL5fJok-QARaOD_KX5olzwllultd27Xr-WKz_-aOgaS4Sio7BYk'\n",
    "}\n",
    "\n",
    "headers = {\n",
    "    'Accept-Encoding': 'gzip, deflate, sdch',\n",
    "    'Accept-Language': 'en-US,en;q=0.8,hi;q=0.6',\n",
    "    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36',\n",
    "    'Accept': '*/*',\n",
    "    'Referer': 'https://www.linkedin.com/sales/profile/12146173,X40s,name?memberId=12146173&authToken=X40s&authType=name',\n",
    "    'X-Requested-With': 'XMLHttpRequest',\n",
    "    'Connection': 'keep-alive',\n",
    "}\n",
    "\n",
    "\n",
    "r = requests.get('https://www.linkedin.com/sales/profile/12146173,X40s,name/pathfinder?_=1430750115778', headers=headers, cookies=cookies)\n",
    "\n",
    "r.json()[\"viewee\"][\"firstName\"], r.json()[\"viewee\"][\"lastName\"], r.json()[\"viewee\"][\"company\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "twitter_account:  ['neildaswani']\n",
      "websites:  [['Personal Website', 'http://www.neildaswani.com/'], ['My Book on Security', 'http://www.amazon.com/gp/product/1590597842?ie=UTF8&tag=learnsecurity-20&linkCode=as2&camp=1789&creative=9325&creativeASIN=1590597842'], ['Company', 'http://www.dasient.com']]\n",
      "firstname:  Neil\n",
      "lastname:  Daswani\n",
      "currentPostion:  Twitter\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "import json\n",
    "import re\n",
    "\n",
    "cookies = {\n",
    "    'sessionid': 'eyJkamFuZ29fdGltZXpvbmUiOiJBc2lhL0tvbGthdGEifQ:1Ykwc5:_4Now3f6hlj8wA0T0JUU5RZfGiQ',\n",
    "'li_at': 'AQECARkM06IEzXSDAAABTR5yjMsAAAFNH1RRBUst4ENksw2guJrXARM-cB48luIqEa9-98sCbygbQe-nsFvoK2WI4fycfdkHj22NmL5fJok-QARaOD_KX5olzwllultd27Xr-WKz_-aOgaS4Sio7BYk'\n",
    "}\n",
    "\n",
    "headers = {\n",
    "    'Accept-Encoding': 'gzip, deflate, sdch',\n",
    "    'Accept-Language': 'en-US,en;q=0.8,hi;q=0.6',\n",
    "    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36',\n",
    "    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',\n",
    "    'Cache-Control': 'max-age=0',\n",
    "    'Connection': 'keep-alive',\n",
    "}\n",
    "\n",
    "r = requests.get('https://www.linkedin.com/sales/profile/30377,cPSh,name?memberId=30377&authToken=cPSh&authType=name', headers=headers, cookies=cookies)\n",
    "\n",
    "s = r.text[39000:]\n",
    "start = 'display: none;\"><!--'\n",
    "end = '--></code>'\n",
    "result = re.search('%s(.*)%s' % (start, end), s).group(1)\n",
    "k = json.loads(result)\n",
    "\n",
    "print('twitter_account: ',k['profile']['contactInfo']['twitterAccounts'])\n",
    "\n",
    "print('websites: ',k['profile']['contactInfo']['websites'])\n",
    "\n",
    "print('firstname: ',k['profile']['firstName'])\n",
    "\n",
    "print('lastname: ', k['profile']['lastName'])\n",
    "\n",
    "print('currentcompany: ',k['currentPosition']['position']['companyName'])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###Rapportive API in python\n",
    "\n",
    "We can make a method which returns true and false for person if the email exists and if exits twitter link also and profile link also.\n",
    "\n",
    "The problem with rapportive API is that token is valid for fixed number of request and only valid for 5 minutes, looks like."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10\n"
     ]
    }
   ],
   "source": [
    "#its simple but work only for few request so has toimplement our own api as 3rd party app"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "##Sidekick API for validating email\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'twitterDetails': {'id': '380923338', 'name': 'deepender singla', 'profileImageUrlHttps': 'https://abs.twimg.com/sticky/default_profile_images/default_profile_5_normal.png', 'createdAt': 1317129550000, 'listedCount': 0, 'profileBannerUrl': 'null', 'url': 'http://www.twitter.com/deependersingla', 'utcOffset': -1, 'description': 'Conventionally happily unemployed. Is in love with Start-ups, Music -addict, Passive smoker )Wanter of change.', 'location': '', 'statusesCount': 44, 'followerCount': 24, 'screenname': 'deependersingla', 'friendsCount': 137}, 'email': 'deepender281190@gmail.com', 'fullcontactDetails': {'contactinfo': {'familyname': 'Singla', 'websites': [], 'fullname': 'Deepender Singla'}, 'digitalfootprint': [{'topics': [{'value': 'Behavioral Economics', 'provider': 'klout'}, {'value': 'R/GA', 'provider': 'klout'}, {'value': 'Internet Security', 'provider': 'klout'}, {'value': 'Information Security', 'provider': 'klout'}, {'value': 'Software', 'provider': 'klout'}], 'scores': [{'value': 12.0, 'provider': 'klout', 'type': 'general'}]}], 'demographics': {'gender': 'Male', 'locationgeneral': 'panipat'}, 'photos': [{'typename': 'Facebook', 'typeid': 'facebook', 'isprimary': True, 'url': 'https://d2ojpxxtu63wzl.cloudfront.net/static/1e84e64c28ddde686f6c99ffc3318907_6f7bf2162769f60b318f881a8e61df2430b077e3e480360dcc03a0ad1d4e10b7'}, {'typename': 'Google Plus', 'typeid': 'googleplus', 'isprimary': False, 'url': 'https://d2ojpxxtu63wzl.cloudfront.net/static/3b96ffb95f9e8ab9cf8a1c52a5231517_ca7fb9054f684a0292028b176ddf64debaa1f880012005c7d07fa98c7c5e91cb'}], 'organizations': [], 'socialprofiles': [{'typeid': 'googleplus', 'typename': 'Google Plus', 'id': '109209844366163555823', 'url': 'https://plus.google.com/109209844366163555823'}, {'followers': 521, 'typeid': 'facebook', 'following': 521, 'username': 'deepender.singla', 'typename': 'Facebook', 'id': '100000578171275', 'url': 'https://www.facebook.com/deepender.singla'}, {'followers': 13, 'typeid': 'twitter', 'typename': 'Twitter', 'following': 101, 'username': 'deependersingla', 'bio': 'Conventionally happily unemployed. Is in love with Start-ups, Music -addict, Passive smoker )Wanter of change.', 'id': '380923338', 'url': 'http://www.twitter.com/deependersingla'}, {'username': 'deependersingla', 'typeid': 'klout', 'typename': 'Klout', 'id': '113434426119068062', 'url': 'http://www.klout.com/user/deependersingla'}]}, 'facebookDetails': {'username': 'deepender.singla', 'gender': 'Male', 'id': '100000578171275', 'name': 'Deepender Singla', 'url': 'https://www.facebook.com/deepender.singla'}}\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "\n",
    "def sidekick_check(email):\n",
    "    cookies = {\n",
    "    'utm_source_first': 'www.quora.com',\n",
    "    'utm_medium_first': 'referral',\n",
    "    'utm_source': 'www.quora.com',\n",
    "    'utm_medium': 'referral',\n",
    "    '_gat': '1',\n",
    "    'hsfirstvisit': 'http%3A%2F%2Fwww.getsidekick.com%2F|http%3A%2F%2Fwww.quora.com%2FWhat-is-the-best-alternative-to-Rapportive|1430735795811',\n",
    "    'ab_test': 'test',\n",
    "    'SACSID': 'AJKiYcGU2qoy7gNSaZKwpGw8WK6Flq03anDBY0fPgBvQVS2K4J1nhcziVqJHNNsFHsiaZuvKU4D0B70lWWkwDNkoEzh4waaPZ7vbY_0UO4bwzPmnnq1QSjCWQRjAi4T8sE4OSuagBVlzjYOGOUexKsZ_0u4Vr-UHWaeWFOsG0Kb7Q9lVOmKyg1R-LSazDyqibUu7_GdTzYocc0UhX80wzn30Et1JU6ps446bL9-scZk-R4Wbg2hvMR-jideOfdeHPOKmBBfrbz16uq0sBZnOt1dxJCwPvLVgVqEEJIK9Wkh_-W1-811HDDkrCqo8vRsFmBCDZftTeC3EFVGaCZ0UfcsBFr2Z0oIsMEpZehJOtrCWaYAL3aIuj9qzwjeannD9PvrTXQjpMuvg529FRX0O51-NZL0DwAcBENc6-A8XHqU6wbtJ7dxuBt6m5H5ASyUwO67juVv1bH4Xkwvma27Tm7TqbrUYdkXOsDomgGeyder5BGmCJyhmi53Ex_jl0fN0995tzQvYpitVa4zvVHiQhhiQnbe_niDQc_VITJof0ERD8352LI0HiWhlFMrW79siU0MMhCOb-Mbu5mrg7M2_gHuWRCB2JppGkFsNJLJTrx3sBh8nJolqOCzNZwDqmth9Y2Mz90XR2EjgI77dVCYOmiFGseOpq_J2Ri8RlBHb2Cdl4eXVgA2FrrMVKR0c-hKs1XO9BwhNxbk4PK3A2CRQ5iQX5LNDbLUF16N1-dCDZ4QZhZ3kxZ9PG2RJWz_fmoQ_zLm2dXv5DwAM1nCQ_JRImtLuuHHVE6ze8KnuqT7u9-0oSDy2w-bBo3sLox4HscAS4DBhijXs4c0y-tWvAaUi4VNTrbbLzQ7hYm5hqozdMbmyvrPf0ovv22DXPQT2YwKIx1g9u33ze5vXUXFLSk3Fql5U_y-e0Fnw_HG9YJqn0-dpYEze3kgeBWMkngtqkbrntAcbOMEwU24hPBDhNdcOCharO638wDKVNHbijOd-lf9OZpOhqJRcu2BwEMw0370BcFFU7clY1i3FsJqIh5RRniPPDeIhna9JXZazesXgO00EkJkoWz1UQpJNjkz28dg689m_ODXmACPUFLjx8U8EzI71VSiTDzdgp7XNxuuX7OSqK8OfupCkLfAEZ-lAr0spLN9uBnH_2PVTUpZvnOu5L2M9Jo3usZtXQ_9ocy37DlYpOeuPXECu3Kv5LB_62I5oEhdowxfHSmU0GnQ59Jw3lfFacsmUgnisFQb6KFFSTKUSo3vXeIMvtlO_eHjQArIKBvnp45zt7jM85IaUH3uvbNI8AfxyGF3SnqU5EfijPxlxiBw5iS0ULkghYPD2j9A6L9HHVlt7eukr5rYZL_rwDUwgeI-kGp6PcJk-OB2GVV6JM_z5P8zhAIx7mL2vymzDDNSzr5LT3DmzU-8a3KGtnTOscOyDShm1JAJV_eJ9kCugIOzy1BcXq3jsPo5kF58yWM93_hWJJhcMieGOb4hwoTWb7SBr-gGwYjGR7oioQbwOjMfXiFeyWoIqzhcgX1ep-rLwz1X-ISpD8g5OqqT1mAlkAuyKs0uwMLnHUpo8qQmirgsTqKz3dQjVgpjOWTPEDvRGc-aBUZJiwc5c1l8w2YpjR3dLK4tX_EJfACA6wPya0XFur5_MGMQ',\n",
    "    '_sig': 'YWd4emZuTnBaMjVoYkhOamNuaHlHQXNTQzFWelpYSlFjbTltYVd4bEdJQ0E0S1Rqb2ZVTERB|1430735855|637137ada567d1b2af96b6d0f01d74d66162c68b',\n",
    "    '_sign': 'YWd4emZuTnBaMjVoYkhOamNuaHlHQXNTQzFWelpYSlFjbTltYVd4bEdJQ0E0S1Rqb2ZVTERB|1430735855|9df4bb6ff9867adbc6662efe77e4258a98a641c0',\n",
    "    '__hlstore': '{',\n",
    "    }\n",
    "    \n",
    "    headers = {\n",
    "    'Accept-Encoding': 'gzip, deflate, sdch',\n",
    "    'Accept-Language': 'en-US,en;q=0.8,hi;q=0.6',\n",
    "    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36',\n",
    "    'Accept': 'application/json, text/javascript, */*; q=0.01',\n",
    "    'Referer': 'https://app.getsidekick.com/sidebar/contacts/deepender.singla@yifp.in?gmail=true&gmail_email=null&is_collapsed=false',\n",
    "    'Connection': 'keep-alive',\n",
    "    }\n",
    "    \n",
    "    k = requests.get('https://app.getsidekick.com/sidebar/contacts/{0}/socialintel.json'.format(email), headers=headers, cookies=cookies)\n",
    "    print(k.json())\n",
    "\n",
    "sidekick_check(\"deepender281190@gmail.com\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}