New Finra Data

finra_org_.idea_encodings.xml

<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
  <component name="Encoding" useUTFGuessing="true" native2AsciiForPropertiesFiles="false" />
</project>

finra_org_.idea_finra_org.iml

<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
  <component name="NewModuleRootManager">
    <content url="file://$MODULE_DIR$" />
    <orderEntry type="inheritedJdk" />
    <orderEntry type="sourceFolder" forTests="false" />
  </component>
</module>

finra_org_.idea_misc.xml

<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
  <component name="DaemonCodeAnalyzer">
    <disable_hints />
  </component>
  <component name="DependencyValidationManager">
    <option name="SKIP_IMPORT_STATEMENTS" value="false" />
  </component>
  <component name="Encoding" useUTFGuessing="true" native2AsciiForPropertiesFiles="false" />
  <component name="FileColors">
    <fileColor scope="Project Files" color="ffcc39" />
    <fileColor scope="All" color="fffbf7" />
  </component>
  <component name="ProjectLevelVcsManager" settingsEditedManually="false">
    <OptionsSetting value="true" id="Add" />
    <OptionsSetting value="true" id="Remove" />
    <OptionsSetting value="true" id="Checkout" />
    <OptionsSetting value="true" id="Update" />
    <OptionsSetting value="true" id="Status" />
    <OptionsSetting value="true" id="Edit" />
    <ConfirmationsSetting value="0" id="Add" />
    <ConfirmationsSetting value="0" id="Remove" />
  </component>
  <component name="ProjectModuleManager">
    <modules />
  </component>
  <component name="ProjectRootManager" version="2" project-jdk-name="Python 2.7.6 (/usr/bin/python2.7)" project-jdk-type="Python SDK" />
  <component name="PropertiesComponent">
    <property name="options.lastSelected" value="reference.settings.ide.settings.file-colors" />
    <property name="options.splitter.main.proportions" value="0.3" />
    <property name="options.splitter.details.proportions" value="0.2" />
  </component>
  <component name="PyConsoleOptionsProvider">
    <option name="myPythonConsoleState">
      <console-settings />
    </option>
  </component>
  <component name="RunManager">
    <list size="0" />
  </component>
  <component name="masterDetails">
    <states>
      <state key="ScopeChooserConfigurable.UI">
        <settings>
          <splitter-proportions>
            <option name="proportions">
              <list>
                <option value="0.2" />
              </list>
            </option>
          </splitter-proportions>
        </settings>
      </state>
    </states>
  </component>
</project>

finra_org_.idea_modules.xml

<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
  <component name="ProjectModuleManager">
    <modules>
      <module fileurl="file://$PROJECT_DIR$/.idea/finra_org.iml" filepath="$PROJECT_DIR$/.idea/finra_org.iml" />
    </modules>
  </component>
</project>

finra_org_.idea_scopes_scope_settings.xml

<component name="DependencyValidationManager">
  <state>
    <option name="SKIP_IMPORT_STATEMENTS" value="false" />
  </state>
</component>

finra_org_.idea_vcs.xml

<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
  <component name="VcsDirectoryMappings">
    <mapping directory="" vcs="" />
  </component>
</project>

finra_org_.idea_workspace.xml

<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
  <component name="ChangeListManager">
    <option name="TRACKING_ENABLED" value="true" />
    <option name="SHOW_DIALOG" value="false" />
    <option name="HIGHLIGHT_CONFLICTS" value="true" />
    <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
    <option name="LAST_RESOLUTION" value="IGNORE" />
  </component>
  <component name="ChangesViewManager" flattened_view="true" show_ignored="false" />
  <component name="CreatePatchCommitExecutor">
    <option name="PATCH_PATH" value="" />
  </component>
  <component name="DaemonCodeAnalyzer">
    <disable_hints />
  </component>
  <component name="ProjectLevelVcsManager" settingsEditedManually="false">
    <OptionsSetting value="true" id="Add" />
    <OptionsSetting value="true" id="Remove" />
    <OptionsSetting value="true" id="Checkout" />
    <OptionsSetting value="true" id="Update" />
    <OptionsSetting value="true" id="Status" />
    <OptionsSetting value="true" id="Edit" />
    <ConfirmationsSetting value="0" id="Add" />
    <ConfirmationsSetting value="0" id="Remove" />
  </component>
  <component name="PropertiesComponent">
    <property name="options.lastSelected" value="reference.settings.ide.settings.file-colors" />
    <property name="options.splitter.main.proportions" value="0.3" />
    <property name="options.splitter.details.proportions" value="0.2" />
  </component>
  <component name="RunManager">
    <list size="0" />
  </component>
  <component name="ShelveChangesManager" show_recycled="false" />
  <component name="TaskManager">
    <task active="true" id="Default" summary="Default task">
      <created>1420625536105</created>
      <option name="number" value="Default" />
      <updated>1420625536105</updated>
    </task>
    <servers />
  </component>
  <component name="VcsContentAnnotationSettings">
    <option name="myLimit" value="2678400000" />
  </component>
  <component name="VcsManagerConfiguration">
    <option name="myTodoPanelSettings">
      <TodoPanelSettings />
    </option>
  </component>
  <component name="XDebuggerManager">
    <breakpoint-manager />
    <watches-manager />
  </component>
</project>

finra_org_finra.csv

finra_org_finra_org_items.py

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.html

import scrapy


class FinraOrgItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    pass

finra_org_finra_org_pipelines.py

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html


class FinraOrgPipeline(object):
    def process_item(self, item, spider):
        return item

finra_org_finra_org_settings.py

# -*- coding: utf-8 -*-

# Scrapy settings for finra_org project
#
# For simplicity, this file contains only the most important settings by
# default. All the other settings are documented here:
#
#     http://doc.scrapy.org/en/latest/topics/settings.html
#

BOT_NAME = 'finra_org'

SPIDER_MODULES = ['finra_org.spiders']
NEWSPIDER_MODULE = 'finra_org.spiders'

# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'finra_org (+http://www.yourdomain.com)'

finra_org_finra_org_spiders___init__.py

# This package will contain the spiders of your Scrapy project
#
# Please refer to the documentation for information on how to create and manage
# your spiders.

finra_org_finra_org_spiders_finra.py

from scrapy.spider import BaseSpider
from scrapy.selector import HtmlXPathSelector
from pymongo import MongoClient, ReadPreference


class MySpider(BaseSpider):
    name = "finra"

    start_urls = ['http://www.finra.org/AboutFINRA/MemberFirms/ListOfMembers/P012908',
                  'http://www.finra.org/AboutFINRA/MemberFirms/ListOfMembers/P012909',
                  'http://www.finra.org/AboutFINRA/MemberFirms/ListOfMembers/P012912',
                  'http://www.finra.org/AboutFINRA/MemberFirms/ListOfMembers/P012914',
                  'http://www.finra.org/AboutFINRA/MemberFirms/ListOfMembers/P012915',
                  'http://www.finra.org/AboutFINRA/MemberFirms/ListOfMembers/P012916',
                  'http://www.finra.org/AboutFINRA/MemberFirms/ListOfMembers/P012917',
                  'http://www.finra.org/AboutFINRA/MemberFirms/ListOfMembers/P012918',
                  'http://www.finra.org/AboutFINRA/MemberFirms/ListOfMembers/P012919',
                  'http://www.finra.org/AboutFINRA/MemberFirms/ListOfMembers/P012920',
                  'http://www.finra.org/AboutFINRA/MemberFirms/ListOfMembers/P012921',
                  'http://www.finra.org/AboutFINRA/MemberFirms/ListOfMembers/P012922',
                  'http://www.finra.org/AboutFINRA/MemberFirms/ListOfMembers/P012923',
                  'http://www.finra.org/AboutFINRA/MemberFirms/ListOfMembers/P012924',
                  'http://www.finra.org/AboutFINRA/MemberFirms/ListOfMembers/P012925',
                  'http://www.finra.org/AboutFINRA/MemberFirms/ListOfMembers/P012926',
                  'http://www.finra.org/AboutFINRA/MemberFirms/ListOfMembers/P012927',
                  'http://www.finra.org/AboutFINRA/MemberFirms/ListOfMembers/P012928',
                  'http://www.finra.org/AboutFINRA/MemberFirms/ListOfMembers/P012929',
                  'http://www.finra.org/AboutFINRA/MemberFirms/ListOfMembers/P012930',
                  'http://www.finra.org/AboutFINRA/MemberFirms/ListOfMembers/P012931',
                  'http://www.finra.org/AboutFINRA/MemberFirms/ListOfMembers/P012932',
                  'http://www.finra.org/AboutFINRA/MemberFirms/ListOfMembers/P012933',
                  'http://www.finra.org/AboutFINRA/MemberFirms/ListOfMembers/P012934',
                  'http://www.finra.org/AboutFINRA/MemberFirms/ListOfMembers/P012935',
                  'http://www.finra.org/AboutFINRA/MemberFirms/ListOfMembers/P012936',
                  'http://www.finra.org/AboutFINRA/MemberFirms/ListOfMembers/P012937'
                  ]



    MAIN_DB_HOST1 = 'localhost'
    MAIN_DB_PORT = 27017
    REL_DB = 'finra'
    REL_COLL = 'data'

    client = MongoClient(MAIN_DB_HOST1, MAIN_DB_PORT)
    read_preference = ReadPreference.SECONDARY
    rel_coll = client[REL_DB][REL_COLL]



    def parse(self, response):

        hxs = HtmlXPathSelector(response)


        companies = hxs.select('//p[@class="MsoNormal"]//text()').extract()

        index = 0
        flag_mailling = 0
        while index < len(companies):
            count = 0
            name = ''
            address = ''
            mailing = ''
            name = companies[index]
            index+=1
            if index >= len(companies):
                break
            address = companies[index]
            index+=1
            if index >= len(companies):
                break

            if len(companies[index])!= 1:
                print '===================',companies[index]

                mailing = companies[index]
                if mailing in address:
                    mailing = address

            self.rel_coll.insert({"company": name, "address": address,"mailing":mailing})

            index+=1
            if len(companies[index]) ==1:
                index+=1

            if index >= len(companies):
                break

finra_org_scrapy.cfg

# Automatically created by: scrapy startproject
#
# For more information about the [deploy] section see:
# http://doc.scrapy.org/en/latest/topics/scrapyd.html

[settings]
default = finra_org.settings

[deploy]
#url = http://localhost:6800/
project = finra_org