Skip to content

Instantly share code, notes, and snippets.

@alfondotnet
Created September 24, 2016 00:08
Show Gist options
  • Save alfondotnet/506b6fde1a6024f3cb62b61b9ab9ff76 to your computer and use it in GitHub Desktop.
Save alfondotnet/506b6fde1a6024f3cb62b61b9ab9ff76 to your computer and use it in GitHub Desktop.
# -*- encoding: utf-8 -*-
from django.core.management.base import BaseCommand, CommandError
from recommendations.models import *
import httplib, json
from optparse import make_option
import re
from random import randrange
from urllib import quote
import time
'''
######################
Discover Reviews
This is a command that goes through each item and get the reviews done by users
(check the other gist for discovering businesses)
######################
'''
businesses = Item.objects.all() # <--- in here your businesses will be
conn = httplib.HTTPConnection('www.yelp.es')
for b in businesses:
# Sleep a little bit
time.sleep(1)
id_business = b.item_id.encode('utf-8')
print 'Saving ratings for '+ id_business + '...'
url = '/biz/'+ id_business + '/review_feed'
# We try to make Yelp happy as much as we can :-)
user_agents = ['Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0',
'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/29.0',
'Mozilla/5.0 (Windows NT 6.0; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1944.0 Safari/537.36',
'Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36',
'Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25',
'Mozilla/5.0 (compatible; MSIE 10.6; Windows NT 6.1; Trident/5.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727) 3gpp-gba UNTRUSTED/1.0',
'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; Media Center PC 6.0; InfoPath.3; MS-RTC LM 8; Zune 4.7)'
]
conn.request('GET', quote(url), headers={"Connection":" keep-alive", "User-agent": user_agents[randrange(len(user_agents)-1)]})
response = conn.getresponse()
response_string = response.read()
if response.status != 200:
print ('Got status '+ str(response.status) + '!! calm down')
return False
#json_results = {"snippet_serial": "20130924", "header": "\u003ch2\u003eRecomendados\u003c/h2\u003e\n\n \u003cdiv class=\"feed-sorts-filters\"\u003e\n \u003cul class=\"tabs-pill feed-sorts pull-left\"\u003e\n \u003cli class=\"inline-block tab selected\"\u003e\n \u003ca href=\"http://www.yelp.es/biz/casa-f%C3%A9lix-sevilla-2?sort_by=relevance_desc\u0026amp;start=0\"\u003e\n Relevancia en Yelp\n \u003cspan class=\"offscreen\"\u003eSeleccionado\u003c/span\u003e\n \u003c/a\u003e\n \u003c/li\u003e\n \u003cli class=\"inline-block tab tab-resortable\"\u003e\n \u003ca href=\"http://www.yelp.es/biz/casa-f%C3%A9lix-sevilla-2?sort_by=date_desc\u0026amp;start=0\"\u003e\n Fecha\n \u003c/a\u003e\n \u003c/li\u003e\n \u003cli class=\"inline-block tab tab-resortable\"\u003e\n \u003ca href=\"http://www.yelp.es/biz/casa-f%C3%A9lix-sevilla-2?sort_by=rating_desc\u0026amp;start=0\"\u003e\n Valoraci\u00f3n\n \u003c/a\u003e\n \u003c/li\u003e\n \u003cli class=\"inline-block\"\u003e\n \u003cspan class=\"subtle-text\"\u003e\u00c9lites\u003c/span\u003e\n \u003c/li\u003e\n \u003c/ul\u003e\n\n \u003cul class=\"tabs-pill feed-language-filters pull-right\"\u003e\n \n \u003cli class=\"tab inline-block js-language-link selected\"\u003e\n \u003ca href=\"http://www.yelp.es/biz/casa-f%C3%A9lix-sevilla-2\" data-lang=\"es\"\u003eespa\u00f1ol\u003cspan class=\"offscreen\"\u003eSeleccionado\u003c/span\u003e\u003c/a\u003e\u003cspan class=\"count\"\u003e3\u003c/span\u003e\n \u003c/li\u003e\n\n \n \u003c/ul\u003e\n \u003c/div\u003e\n \u003cdiv class=\"feed-trust-banner\"\u003e\n \u003cdiv class=\"trust-banner-inner\"\u003e\n \u003ca class=\"alert-dismiss dismiss-trust-banner js-dismiss-trust-banner\" href=\"#\"\u003e\u0026times;\u003c/a\u003e\n \u003cdiv class=\"media-block media-block-large clearfix trust-banner-text\"\u003e\n \u003cdiv class=\"media-avatar\"\u003e\n \u003ci class=\"i ig-biz_details i-large-burst-biz_details inline-block trust-banner-burst\"\u003e\u003c/i\u003e\n \u003c/div\u003e\n \u003cdiv class=\"media-story\"\u003e\n \u003cspan class=\"legal-copy\"\u003e\n \u003cb\u003ePorque tu confianza es nuestra preocupaci\u00f3n principal,\u003c/b\u003e los negocios no pueden pagar para borrar o modificar sus comentarios. \u003ca href=\"/advertiser_faq\"\u003eSeguir leyendo.\u003c/a\u003e\n \u003c/span\u003e\n \u003c/div\u003e\n \u003c/div\u003e\n \u003c/div\u003e\n\n \u003c/div\u003e", "pagination": "\u003cdiv class=\"pagination-block clearfix\"\u003e\n \u003cdiv class=\"page-of-pages\"\u003e\n P\u00e1gina 1 de 1\n \u003c/div\u003e\n\n \u003c/div\u003e", "review_list": "\u003cli\u003e\n \u003cdiv class=\"review review-with-no-actions\" data-review-id=\"7PdC921zwryDKB5-wDDlPQ\" itemprop=\"review\" itemscope itemtype=\"http://schema.org/Review\"\u003e\n \u003cmeta itemprop=\"author\" content=\"\u00c1ngel A.\"\u003e\n \u003cdiv class=\"review-sidebar\"\u003e\n \u003cdiv class=\"review-sidebar-content\"\u003e\n \u003cdiv class=\"ypassport media-block clearfix\"\u003e\n \u003cdiv class=\"media-avatar\"\u003e\n \n\n\n \u003cdiv class=\"photo-box pb-60s\" data-hovercard-id=\"tVhxWsJMxshWwyawviAKrA\"\u003e\n \u003ca href=\"/user_details?userid=q86P_eyfjthuh_Lli25hsw\"\u003e\n \n\n \u003cimg alt=\"\u00c1ngel A.\" class=\"photo-box-img\" height=\"60\" src=\"//s3-media2.ak.yelpcdn.com/photo/YxZZcLEz7Zhp1K4CpIEeOw/60s.jpg\" width=\"60\"\u003e\n\n \u003c/a\u003e\n\n \u003c/div\u003e\n\n\n\n \u003c/div\u003e\n \u003cdiv class=\"media-story\"\u003e\n \u003cul class=\"user-passport-info\"\u003e\n \u003cli class=\"user-name\"\u003e\n \u003ca class=\"user-display-name\" href=\"/user_details?userid=q86P_eyfjthuh_Lli25hsw\" data-hovercard-id=\"tVhxWsJMxshWwyawviAKrA\"\u003e\u00c1ngel A.\u003c/a\u003e\n\n \u003c/li\u003e\n \u003cli class=\"user-location\"\u003e\n \u003cb\u003eSevilla\u003c/b\u003e\n \u003c/li\u003e\n \u003c/ul\u003e\n\n \u003cul class=\"user-passport-stats\"\u003e\n \u003cli class=\"friend-count\"\u003e\n \u003cspan class=\"i-wrap ig-wrap-common i-friends-orange-common-wrap\"\u003e\u003ci class=\"i ig-common i-friends-orange-common\"\u003e\u003c/i\u003e \u003cb\u003e23\u003c/b\u003e amigos\u003c/span\u003e\n \u003c/li\u003e\n \u003cli class=\"review-count\"\u003e\n \u003cspan class=\"i-wrap ig-wrap-common i-star-orange-common-wrap\"\u003e\u003ci class=\"i ig-common i-star-orange-common\"\u003e\u003c/i\u003e \u003cb\u003e223\u003c/b\u003e comentarios\u003c/span\u003e\n \u003c/li\u003e\n \u003c/ul\u003e\n\n \u003c/div\u003e\n \u003c/div\u003e\n\n \u003cul class=\"iconed-list action-link-list\"\u003e\n\n \u003cli class=\"iconed-list-item\"\u003e\n\n \u003ca class=\"action-link send-to-friend\" data-pop-uri=\"/send_to_friend/review/7PdC921zwryDKB5-wDDlPQ\" href=\"/biz_share?bizid=cn_ideovo7uyosgGfGOzwg\u0026amp;return_url=http%3A%2F%2Fwww.yelp.es%2Fbiz%2Fcasa-f%25C3%25A9lix-sevilla-2\u0026amp;reviewid=7PdC921zwryDKB5-wDDlPQ\"\u003e\n \u003cdiv class=\"media-block\"\u003e\n \u003cdiv class=\"media-avatar\"\u003e\n \u003ci class=\"i ig-common i-action-share-common\"\u003e\u003c/i\u003e\n \u003c/div\u003e\n \u003cdiv class=\"media-story\"\u003e\n Compartir\n \u003c/div\u003e\n \u003c/div\u003e\n \u003c/a\u003e\n\n \u003c/li\u003e\n\n \u003cli class=\"iconed-list-item\"\u003e\n\n \u003ca class=\"action-link send-compliment\" href=\"/thanx?complimentable_id=7PdC921zwryDKB5-wDDlPQ\u0026amp;complimentable_type=REVIEW\u0026amp;previous_url=http%3A%2F%2Fwww.yelp.es%2Fbiz%2Fcasa-f%25C3%25A9lix-sevilla-2\u0026amp;user_id=q86P_eyfjthuh_Lli25hsw\"\u003e\n \u003cdiv class=\"media-block\"\u003e\n \u003cdiv class=\"media-avatar\"\u003e\n \u003ci class=\"i ig-common i-action-compliment-common\"\u003e\u003c/i\u003e\n \u003c/div\u003e\n \u003cdiv class=\"media-story\"\u003e\n Piropear\n \u003c/div\u003e\n \u003c/div\u003e\n \u003c/a\u003e\n\n \u003c/li\u003e\n\n \u003cli class=\"iconed-list-item\"\u003e\n\n \u003ca class=\"action-link send-pm\" href=\"/mail?action_send_form=1\u0026amp;dst=q86P_eyfjthuh_Lli25hsw\u0026amp;return_url=http%3A%2F%2Fwww.yelp.es%2Fbiz%2Fcasa-f%25C3%25A9lix-sevilla-2\" rel=\"\u00c1ngel A.\"\u003e\n \u003cdiv class=\"media-block\"\u003e\n \u003cdiv class=\"media-avatar\"\u003e\n \u003ci class=\"i ig-common i-action-message-common\"\u003e\u003c/i\u003e\n \u003c/div\u003e\n \u003cdiv class=\"media-story\"\u003e\n Enviar mensaje\n \u003c/div\u003e\n \u003c/div\u003e\n \u003c/a\u003e\n\n \u003c/li\u003e\n\n \u003cli class=\"iconed-list-item manage-follow-container\"\u003e\n \n\n \u003ca class=\"action-link manage-following-add\" href=\"/following_user/add?dst_user_id=q86P_eyfjthuh_Lli25hsw\u0026amp;previous_url=http%3A%2F%2Fwww.yelp.es%2Fbiz%2Fcasa-f%25C3%25A9lix-sevilla-2\" rel=\"q86P_eyfjthuh_Lli25hsw\" style=\"\"\u003e\n \u003cdiv class=\"media-block\"\u003e\n \u003cdiv class=\"media-avatar\"\u003e\n \u003ci class=\"i ig-common i-action-follow-common\"\u003e\u003c/i\u003e\n \u003c/div\u003e\n \u003cdiv class=\"media-story\"\u003e\n Seguir a \u00c1ngel A.\n \u003c/div\u003e\n \u003c/div\u003e\n \u003c/a\u003e\n\n\n\n \u003ca class=\"action-link manage-following-remove\" href=\"/following_user/remove?dst_user_id=q86P_eyfjthuh_Lli25hsw\u0026amp;previous_url=http%3A%2F%2Fwww.yelp.es%2Fbiz%2Fcasa-f%25C3%25A9lix-sevilla-2\" rel=\"q86P_eyfjthuh_Lli25hsw\" style=\"display: none;\"\u003e\n \u003cdiv class=\"media-block\"\u003e\n \u003cdiv class=\"media-avatar\"\u003e\n \u003ci class=\"i ig-common i-action-unfollow-common\"\u003e\u003c/i\u003e\n \u003c/div\u003e\n \u003cdiv class=\"media-story\"\u003e\n Dejar de seguir a \u00c1ngel A.\n \u003c/div\u003e\n \u003c/div\u003e\n \u003c/a\u003e\n\n \u003c/li\u003e\n \u003c/ul\u003e\n\n \u003c/div\u003e\n \u003c/div\u003e\n\n \u003cdiv class=\"review-wrapper\"\u003e\n \u003cdiv class=\"review-content\"\u003e\n \u003cdiv class=\"biz-rating biz-rating-very-large clearfix\"\u003e\n \u003cdiv itemprop=\"reviewRating\" itemscope itemtype=\"http://schema.org/Rating\"\u003e\n \n \u003cdiv class=\"rating-very-large\"\u003e\n \u003ci class=\"star-img stars_4\" title=\"Valoraci\u00f3n de 4.0 estrellas\"\u003e\n \u003cimg alt=\"Valoraci\u00f3n de 4.0 estrellas\" class=\"offscreen\" height=\"303\" src=\"http://s3-media3.ak.yelpcdn.com/assets/2/www/img/c2252a4cd43e/ico/stars/v2/stars_map.png\" width=\"84\"\u003e\n \u003c/i\u003e\n \u003cmeta itemprop=\"ratingValue\" content=\"4.0\"\u003e\n \u003c/div\u003e\n\n\n \u003c/div\u003e\n \u003cspan class=\"rating-qualifier\"\u003e\n \u003cmeta itemprop=\"datePublished\" content=\"2011-11-20\"\u003e\n 21/11/2011\n \u003c/span\u003e\n\n \u003c/div\u003e\n\n \u003cspan class=\"review-tags\"\u003e\n \u003cspan class=\"i-wrap ig-wrap-common i-first-burst-common-wrap badge first-to-review-marker\"\u003e\u003ci class=\"i ig-common i-first-burst-common\"\u003e\u003c/i\u003e Primero en comentar\u003c/span\u003e\n\n \u003c/span\u003e\n\n \u003cp class=\"review_comment ieSucks\" itemprop=\"description\" lang=\"es\"\u003eMe pregunto cu\u00e1ntas veces habr\u00e9 ido a desayunar a Casa F\u00e9lix, que ya incluso me pon\u00edan lo que quer\u00eda sin tener que pedirlo. Aqu\u00ed ven\u00eda a diario con un amigo a desayunar; entre los dos compart\u00edamos dos colacaos y una tostada con aceite, tomate triturado y jam\u00f3n, a 2 euros por persona y lleg\u00e1bamos a la biblioteca o a clase con las pilas cargadas. El tomate triturado te lo ponen en un cuenco que es casi gazpacho y est\u00e1 riqu\u00edsimo.\u003cbr\u003e\u003cbr\u003eEn Casa F\u00e9lix siempre nos han atendido de maravilla. El personal va a las mesas s\u00f3lo a retirar lo que haya dejado el cliente anterior, pero en cuanto a comida es auto servicio. Los d\u00edas que hace fr\u00edo estar dentro es un poco agobiante por la estrechez y el jaleo, pero compensa, sobre todo si ese d\u00eda cambias y te pides unos churros, que aqu\u00ed tambi\u00e9n tienen fama y para los que somos un poco sensibles con las comidas excesivamente grasas te digo que por suerte nunca me dieron problema. Pero, como digo, donde m\u00e1s agusto me he encontrado ha sido en las mesas del exterior, comiendo a la sombrita un buen desayuno con mis amigos y debatiendo sobre lo divino y humano. Un peque\u00f1o gustazo enmedio de las obligaciones.\u003c/p\u003e \n \u003cul class=\"photo-box-grid clearfix js-content-expandable\"\u003e\n \u003cli \u003e\n \u003cdiv class=\"photo-box has-overlay\"\u003e\n \n \u003cimg alt=\"Casa F\u00e9lix\" class=\"photo-box-img\" height=\"348\" src=\"//s3-media4.ak.yelpcdn.com/bphoto/EZV2ISnEqs5rE2V4ZvvsWg/348s.jpg\" width=\"348\"\u003e\n\n\n\n \u003cdiv class=\"photo-box-overlay js-overlay\"\u003e\n Casa F\u00e9lix\n \u003c/div\u003e\n\n \n\n \u003ca class=\"biz-shim\" href=\"/biz_photos/casa-f%C3%A9lix-sevilla-2?filter_by_userid=True\u0026amp;select=EZV2ISnEqs5rE2V4ZvvsWg\u0026amp;userid=q86P_eyfjthuh_Lli25hsw\"\u003e\n \u003cspan class=\"offscreen\"\u003eCasa F\u00e9lix\u003c/span\u003e\n \u003c/a\u003e\n\n \u003c/div\u003e\n\n \u003c/li\u003e\n\n \u003c/ul\u003e\n\n \u003c/div\u003e\n \u003cdiv class=\"review-footer clearfix\"\u003e\n \u003cdiv class=\"rateReview ufc-feedback clearfix\" data-review-id=\"7PdC921zwryDKB5-wDDlPQ\"\u003e\n \u003cp class=\"review-intro review-message\"\u003e\n \u00bfEl comentario es \u0026hellip;?\n \u003c/p\u003e\n\n \u003cul class=\"big-ufc\" data-csrf-token=\"79d0ffc00e41a215f4b802aa68a69d49294d02e2c4c16e208c5c702321321697\"\u003e\n \n\n\n \u003cli class=\"ufc-stat inline-block ytype\"\u003e\n \u003ca href=\"javascript:;\" rel=\"useful\" class=\"ybtn ybtn-tertiary ybtn-small useful ytype\"\u003e\u003cspan class=\"i-wrap ig-wrap-common i-big-ufc-useful-common-wrap button-content\"\u003e\u003ci class=\"i ig-common i-big-ufc-useful-common\"\u003e\u003c/i\u003e \u003cspan class=\"vote-type\"\u003e\u00datil\u003c/span\u003e\n \u003cspan class=\"count\"\u003e1\u003c/span\u003e\u003c/span\u003e\u003c/a\u003e\n \u003c/li\u003e\n\n \n\n\n \u003cli class=\"ufc-stat inline-block ytype\"\u003e\n \u003ca href=\"javascript:;\" rel=\"funny\" class=\"ybtn ybtn-tertiary ybtn-small funny ytype\"\u003e\u003cspan class=\"i-wrap ig-wrap-common i-big-ufc-funny-common-wrap button-content\"\u003e\u003ci class=\"i ig-common i-big-ufc-funny-common\"\u003e\u003c/i\u003e \u003cspan class=\"vote-type\"\u003eDivertido\u003c/span\u003e\n \u003cspan class=\"count\"\u003e\u003c/span\u003e\u003c/span\u003e\u003c/a\u003e\n \u003c/li\u003e\n\n \n\n\n \u003cli class=\"ufc-stat inline-block ytype\"\u003e\n \u003ca href=\"javascript:;\" rel=\"cool\" class=\"ybtn ybtn-tertiary ybtn-small cool ytype\"\u003e\u003cspan class=\"i-wrap ig-wrap-common i-big-ufc-cool-common-wrap button-content\"\u003e\u003ci class=\"i ig-common i-big-ufc-cool-common\"\u003e\u003c/i\u003e \u003cspan class=\"vote-type\"\u003eLo m\u00e1s\u003c/span\u003e\n \u003cspan class=\"count\"\u003e\u003c/span\u003e\u003c/span\u003e\u003c/a\u003e\n \u003c/li\u003e\n\n \u003c/ul\u003e\n \u003c/div\u003e\n\n \u003c/div\u003e\n \n \n\n \u003c/div\u003e\n \u003c/div\u003e\n\n \u003c/li\u003e\n \u003cli\u003e\n \u003cdiv class=\"review review-with-no-actions\" data-review-id=\"ttYegtnpcY5plGfs06X1kA\" itemprop=\"review\" itemscope itemtype=\"http://schema.org/Review\"\u003e\n \u003cmeta itemprop=\"author\" content=\"Marciana D.\"\u003e\n \u003cdiv class=\"review-sidebar\"\u003e\n \u003cdiv class=\"review-sidebar-content\"\u003e\n \u003cdiv class=\"ypassport media-block clearfix\"\u003e\n \u003cdiv class=\"media-avatar\"\u003e\n \n\n\n \u003cdiv class=\"photo-box pb-60s\" data-hovercard-id=\"G85jQCLBbhhBRF6P0oHEpw\"\u003e\n \u003ca href=\"/user_details?userid=Qlz-8GxOfg67TJTY1fya5g\"\u003e\n \n\n \u003cimg alt=\"Marciana D.\" class=\"photo-box-img\" height=\"60\" src=\"//s3-media1.ak.yelpcdn.com/photo/Pz-UzoETj72I_Kjdux-sew/60s.jpg\" width=\"60\"\u003e\n\n \u003c/a\u003e\n\n \u003c/div\u003e\n\n\n\n \u003c/div\u003e\n \u003cdiv class=\"media-story\"\u003e\n \u003cul class=\"user-passport-info\"\u003e\n \u003cli class=\"user-name\"\u003e\n \u003ca class=\"user-display-name\" href=\"/user_details?userid=Qlz-8GxOfg67TJTY1fya5g\" data-hovercard-id=\"G85jQCLBbhhBRF6P0oHEpw\"\u003eMarciana D.\u003c/a\u003e\n\n \u003c/li\u003e\n \u003cli class=\"user-location\"\u003e\n \u003cb\u003eSevilla\u003c/b\u003e\n \u003c/li\u003e\n \u003c/ul\u003e\n\n \u003cul class=\"user-passport-stats\"\u003e\n \u003cli class=\"friend-count\"\u003e\n \u003cspan class=\"i-wrap ig-wrap-common i-friends-orange-common-wrap\"\u003e\u003ci class=\"i ig-common i-friends-orange-common\"\u003e\u003c/i\u003e \u003cb\u003e4\u003c/b\u003e amigos\u003c/span\u003e\n \u003c/li\u003e\n \u003cli class=\"review-count\"\u003e\n \u003cspan class=\"i-wrap ig-wrap-common i-star-orange-common-wrap\"\u003e\u003ci class=\"i ig-common i-star-orange-common\"\u003e\u003c/i\u003e \u003cb\u003e23\u003c/b\u003e comentarios\u003c/span\u003e\n \u003c/li\u003e\n \u003c/ul\u003e\n\n \u003c/div\u003e\n \u003c/div\u003e\n\n \u003cul class=\"iconed-list action-link-list\"\u003e\n\n \u003cli class=\"iconed-list-item\"\u003e\n\n \u003ca class=\"action-link send-to-friend\" data-pop-uri=\"/send_to_friend/review/ttYegtnpcY5plGfs06X1kA\" href=\"/biz_share?bizid=cn_ideovo7uyosgGfGOzwg\u0026amp;return_url=http%3A%2F%2Fwww.yelp.es%2Fbiz%2Fcasa-f%25C3%25A9lix-sevilla-2\u0026amp;reviewid=ttYegtnpcY5plGfs06X1kA\"\u003e\n \u003cdiv class=\"media-block\"\u003e\n \u003cdiv class=\"media-avatar\"\u003e\n \u003ci class=\"i ig-common i-action-share-common\"\u003e\u003c/i\u003e\n \u003c/div\u003e\n \u003cdiv class=\"media-story\"\u003e\n Compartir\n \u003c/div\u003e\n \u003c/div\u003e\n \u003c/a\u003e\n\n \u003c/li\u003e\n\n \u003cli class=\"iconed-list-item\"\u003e\n\n \u003ca class=\"action-link send-compliment\" href=\"/thanx?complimentable_id=ttYegtnpcY5plGfs06X1kA\u0026amp;complimentable_type=REVIEW\u0026amp;previous_url=http%3A%2F%2Fwww.yelp.es%2Fbiz%2Fcasa-f%25C3%25A9lix-sevilla-2\u0026amp;user_id=Qlz-8GxOfg67TJTY1fya5g\"\u003e\n \u003cdiv class=\"media-block\"\u003e\n \u003cdiv class=\"media-avatar\"\u003e\n \u003ci class=\"i ig-common i-action-compliment-common\"\u003e\u003c/i\u003e\n \u003c/div\u003e\n \u003cdiv class=\"media-story\"\u003e\n Piropear\n \u003c/div\u003e\n \u003c/div\u003e\n \u003c/a\u003e\n\n \u003c/li\u003e\n\n \u003cli class=\"iconed-list-item\"\u003e\n\n \u003ca class=\"action-link send-pm\" href=\"/mail?action_send_form=1\u0026amp;dst=Qlz-8GxOfg67TJTY1fya5g\u0026amp;return_url=http%3A%2F%2Fwww.yelp.es%2Fbiz%2Fcasa-f%25C3%25A9lix-sevilla-2\" rel=\"Marciana D.\"\u003e\n \u003cdiv class=\"media-block\"\u003e\n \u003cdiv class=\"media-avatar\"\u003e\n \u003ci class=\"i ig-common i-action-message-common\"\u003e\u003c/i\u003e\n \u003c/div\u003e\n \u003cdiv class=\"media-story\"\u003e\n Enviar mensaje\n \u003c/div\u003e\n \u003c/div\u003e\n \u003c/a\u003e\n\n \u003c/li\u003e\n\n \u003cli class=\"iconed-list-item manage-follow-container\"\u003e\n \n\n \u003ca class=\"action-link manage-following-add\" href=\"/following_user/add?dst_user_id=Qlz-8GxOfg67TJTY1fya5g\u0026amp;previous_url=http%3A%2F%2Fwww.yelp.es%2Fbiz%2Fcasa-f%25C3%25A9lix-sevilla-2\" rel=\"Qlz-8GxOfg67TJTY1fya5g\" style=\"\"\u003e\n \u003cdiv class=\"media-block\"\u003e\n \u003cdiv class=\"media-avatar\"\u003e\n \u003ci class=\"i ig-common i-action-follow-common\"\u003e\u003c/i\u003e\n \u003c/div\u003e\n \u003cdiv class=\"media-story\"\u003e\n Seguir a Marciana D.\n \u003c/div\u003e\n \u003c/div\u003e\n \u003c/a\u003e\n\n\n\n \u003ca class=\"action-link manage-following-remove\" href=\"/following_user/remove?dst_user_id=Qlz-8GxOfg67TJTY1fya5g\u0026amp;previous_url=http%3A%2F%2Fwww.yelp.es%2Fbiz%2Fcasa-f%25C3%25A9lix-sevilla-2\" rel=\"Qlz-8GxOfg67TJTY1fya5g\" style=\"display: none;\"\u003e\n \u003cdiv class=\"media-block\"\u003e\n \u003cdiv class=\"media-avatar\"\u003e\n \u003ci class=\"i ig-common i-action-unfollow-common\"\u003e\u003c/i\u003e\n \u003c/div\u003e\n \u003cdiv class=\"media-story\"\u003e\n Dejar de seguir a Marciana D.\n \u003c/div\u003e\n \u003c/div\u003e\n \u003c/a\u003e\n\n \u003c/li\u003e\n \u003c/ul\u003e\n\n \u003c/div\u003e\n \u003c/div\u003e\n\n \u003cdiv class=\"review-wrapper\"\u003e\n \u003cdiv class=\"review-content\"\u003e\n \u003cdiv class=\"biz-rating biz-rating-very-large clearfix\"\u003e\n \u003cdiv itemprop=\"reviewRating\" itemscope itemtype=\"http://schema.org/Rating\"\u003e\n \n \u003cdiv class=\"rating-very-large\"\u003e\n \u003ci class=\"star-img stars_5\" title=\"Valoraci\u00f3n de 5.0 estrellas\"\u003e\n \u003cimg alt=\"Valoraci\u00f3n de 5.0 estrellas\" class=\"offscreen\" height=\"303\" src=\"http://s3-media3.ak.yelpcdn.com/assets/2/www/img/c2252a4cd43e/ico/stars/v2/stars_map.png\" width=\"84\"\u003e\n \u003c/i\u003e\n \u003cmeta itemprop=\"ratingValue\" content=\"5.0\"\u003e\n \u003c/div\u003e\n\n\n \u003c/div\u003e\n \u003cspan class=\"rating-qualifier\"\u003e\n \u003cmeta itemprop=\"datePublished\" content=\"2013-05-08\"\u003e\n 8/5/2013\n \u003c/span\u003e\n\n \u003c/div\u003e\n\n \n \u003cp class=\"review_comment ieSucks\" itemprop=\"description\" lang=\"es\"\u003eChurros con un chocolate exquisito, buenos desayunos, pescaito del bueno y en cantidad...Es un cl\u00e1sico de Reina Mercedes y mi favorito para comer bien en la avenida. Lo recomiendo, todo lo hacen bien.\u003c/p\u003e \n \n \u003c/div\u003e\n \u003cdiv class=\"review-footer clearfix\"\u003e\n \u003cdiv class=\"rateReview ufc-feedback clearfix\" data-review-id=\"ttYegtnpcY5plGfs06X1kA\"\u003e\n \u003cp class=\"review-intro review-message\"\u003e\n \u00bfEl comentario es \u0026hellip;?\n \u003c/p\u003e\n\n \u003cul class=\"big-ufc\" data-csrf-token=\"79d0ffc00e41a215f4b802aa68a69d49294d02e2c4c16e208c5c702321321697\"\u003e\n \n\n\n \u003cli class=\"ufc-stat inline-block ytype\"\u003e\n \u003ca href=\"javascript:;\" rel=\"useful\" class=\"ybtn ybtn-tertiary ybtn-small useful ytype\"\u003e\u003cspan class=\"i-wrap ig-wrap-common i-big-ufc-useful-common-wrap button-content\"\u003e\u003ci class=\"i ig-common i-big-ufc-useful-common\"\u003e\u003c/i\u003e \u003cspan class=\"vote-type\"\u003e\u00datil\u003c/span\u003e\n \u003cspan class=\"count\"\u003e3\u003c/span\u003e\u003c/span\u003e\u003c/a\u003e\n \u003c/li\u003e\n\n \n\n\n \u003cli class=\"ufc-stat inline-block ytype\"\u003e\n \u003ca href=\"javascript:;\" rel=\"funny\" class=\"ybtn ybtn-tertiary ybtn-small funny ytype\"\u003e\u003cspan class=\"i-wrap ig-wrap-common i-big-ufc-funny-common-wrap button-content\"\u003e\u003ci class=\"i ig-common i-big-ufc-funny-common\"\u003e\u003c/i\u003e \u003cspan class=\"vote-type\"\u003eDivertido\u003c/span\u003e\n \u003cspan class=\"count\"\u003e\u003c/span\u003e\u003c/span\u003e\u003c/a\u003e\n \u003c/li\u003e\n\n \n\n\n \u003cli class=\"ufc-stat inline-block ytype\"\u003e\n \u003ca href=\"javascript:;\" rel=\"cool\" class=\"ybtn ybtn-tertiary ybtn-small cool ytype\"\u003e\u003cspan class=\"i-wrap ig-wrap-common i-big-ufc-cool-common-wrap button-content\"\u003e\u003ci class=\"i ig-common i-big-ufc-cool-common\"\u003e\u003c/i\u003e \u003cspan class=\"vote-type\"\u003eLo m\u00e1s\u003c/span\u003e\n \u003cspan class=\"count\"\u003e\u003c/span\u003e\u003c/span\u003e\u003c/a\u003e\n \u003c/li\u003e\n\n \u003c/ul\u003e\n \u003c/div\u003e\n\n \u003c/div\u003e\n \n \n\n \u003c/div\u003e\n \u003c/div\u003e\n\n \u003c/li\u003e\n \u003cli\u003e\n \u003cdiv class=\"review review-with-no-actions\" data-review-id=\"wnfqFC83791yr_FXVd3v8Q\" itemprop=\"review\" itemscope itemtype=\"http://schema.org/Review\"\u003e\n \u003cmeta itemprop=\"author\" content=\"Pepa L.\"\u003e\n \u003cdiv class=\"review-sidebar\"\u003e\n \u003cdiv class=\"review-sidebar-content\"\u003e\n \u003cdiv class=\"ypassport media-block clearfix\"\u003e\n \u003cdiv class=\"media-avatar\"\u003e\n \n\n\n \u003cdiv class=\"photo-box pb-60s\" data-hovercard-id=\"QrS3kMuTCmZBH1571D0OKA\"\u003e\n \u003ca href=\"/user_details?userid=evo0GqKqi-KCjSSa_QMxIw\"\u003e\n \n\n \u003cimg alt=\"Pepa L.\" class=\"photo-box-img\" height=\"60\" src=\"//s3-media1.ak.yelpcdn.com/photo/QbYua6sJYFbPrbCfcPboaw/60s.jpg\" width=\"60\"\u003e\n\n \u003c/a\u003e\n\n \u003c/div\u003e\n\n\n\n \u003c/div\u003e\n \u003cdiv class=\"media-story\"\u003e\n \u003cul class=\"user-passport-info\"\u003e\n \u003cli class=\"user-name\"\u003e\n \u003ca class=\"user-display-name\" href=\"/user_details?userid=evo0GqKqi-KCjSSa_QMxIw\" data-hovercard-id=\"QrS3kMuTCmZBH1571D0OKA\"\u003ePepa L.\u003c/a\u003e\n\n \u003c/li\u003e\n \u003cli class=\"user-location\"\u003e\n \u003cb\u003eSevilla\u003c/b\u003e\n \u003c/li\u003e\n \u003c/ul\u003e\n\n \u003cul class=\"user-passport-stats\"\u003e\n \u003cli class=\"friend-count\"\u003e\n \u003cspan class=\"i-wrap ig-wrap-common i-friends-orange-common-wrap\"\u003e\u003ci class=\"i ig-common i-friends-orange-common\"\u003e\u003c/i\u003e \u003cb\u003e35\u003c/b\u003e amigos\u003c/span\u003e\n \u003c/li\u003e\n \u003cli class=\"review-count\"\u003e\n \u003cspan class=\"i-wrap ig-wrap-common i-star-orange-common-wrap\"\u003e\u003ci class=\"i ig-common i-star-orange-common\"\u003e\u003c/i\u003e \u003cb\u003e252\u003c/b\u003e comentarios\u003c/span\u003e\n \u003c/li\u003e\n \u003c/ul\u003e\n\n \u003c/div\u003e\n \u003c/div\u003e\n\n \u003cul class=\"iconed-list action-link-list\"\u003e\n\n \u003cli class=\"iconed-list-item\"\u003e\n\n \u003ca class=\"action-link send-to-friend\" data-pop-uri=\"/send_to_friend/review/wnfqFC83791yr_FXVd3v8Q\" href=\"/biz_share?bizid=cn_ideovo7uyosgGfGOzwg\u0026amp;return_url=http%3A%2F%2Fwww.yelp.es%2Fbiz%2Fcasa-f%25C3%25A9lix-sevilla-2\u0026amp;reviewid=wnfqFC83791yr_FXVd3v8Q\"\u003e\n \u003cdiv class=\"media-block\"\u003e\n \u003cdiv class=\"media-avatar\"\u003e\n \u003ci class=\"i ig-common i-action-share-common\"\u003e\u003c/i\u003e\n \u003c/div\u003e\n \u003cdiv class=\"media-story\"\u003e\n Compartir\n \u003c/div\u003e\n \u003c/div\u003e\n \u003c/a\u003e\n\n \u003c/li\u003e\n\n \u003cli class=\"iconed-list-item\"\u003e\n\n \u003ca class=\"action-link send-compliment\" href=\"/thanx?complimentable_id=wnfqFC83791yr_FXVd3v8Q\u0026amp;complimentable_type=REVIEW\u0026amp;previous_url=http%3A%2F%2Fwww.yelp.es%2Fbiz%2Fcasa-f%25C3%25A9lix-sevilla-2\u0026amp;user_id=evo0GqKqi-KCjSSa_QMxIw\"\u003e\n \u003cdiv class=\"media-block\"\u003e\n \u003cdiv class=\"media-avatar\"\u003e\n \u003ci class=\"i ig-common i-action-compliment-common\"\u003e\u003c/i\u003e\n \u003c/div\u003e\n \u003cdiv class=\"media-story\"\u003e\n Piropear\n \u003c/div\u003e\n \u003c/div\u003e\n \u003c/a\u003e\n\n \u003c/li\u003e\n\n \u003cli class=\"iconed-list-item\"\u003e\n\n \u003ca class=\"action-link send-pm\" href=\"/mail?action_send_form=1\u0026amp;dst=evo0GqKqi-KCjSSa_QMxIw\u0026amp;return_url=http%3A%2F%2Fwww.yelp.es%2Fbiz%2Fcasa-f%25C3%25A9lix-sevilla-2\" rel=\"Pepa L.\"\u003e\n \u003cdiv class=\"media-block\"\u003e\n \u003cdiv class=\"media-avatar\"\u003e\n \u003ci class=\"i ig-common i-action-message-common\"\u003e\u003c/i\u003e\n \u003c/div\u003e\n \u003cdiv class=\"media-story\"\u003e\n Enviar mensaje\n \u003c/div\u003e\n \u003c/div\u003e\n \u003c/a\u003e\n\n \u003c/li\u003e\n\n \u003cli class=\"iconed-list-item manage-follow-container\"\u003e\n \n\n \u003ca class=\"action-link manage-following-add\" href=\"/following_user/add?dst_user_id=evo0GqKqi-KCjSSa_QMxIw\u0026amp;previous_url=http%3A%2F%2Fwww.yelp.es%2Fbiz%2Fcasa-f%25C3%25A9lix-sevilla-2\" rel=\"evo0GqKqi-KCjSSa_QMxIw\" style=\"\"\u003e\n \u003cdiv class=\"media-block\"\u003e\n \u003cdiv class=\"media-avatar\"\u003e\n \u003ci class=\"i ig-common i-action-follow-common\"\u003e\u003c/i\u003e\n \u003c/div\u003e\n \u003cdiv class=\"media-story\"\u003e\n Seguir a Pepa L.\n \u003c/div\u003e\n \u003c/div\u003e\n \u003c/a\u003e\n\n\n\n \u003ca class=\"action-link manage-following-remove\" href=\"/following_user/remove?dst_user_id=evo0GqKqi-KCjSSa_QMxIw\u0026amp;previous_url=http%3A%2F%2Fwww.yelp.es%2Fbiz%2Fcasa-f%25C3%25A9lix-sevilla-2\" rel=\"evo0GqKqi-KCjSSa_QMxIw\" style=\"display: none;\"\u003e\n \u003cdiv class=\"media-block\"\u003e\n \u003cdiv class=\"media-avatar\"\u003e\n \u003ci class=\"i ig-common i-action-unfollow-common\"\u003e\u003c/i\u003e\n \u003c/div\u003e\n \u003cdiv class=\"media-story\"\u003e\n Dejar de seguir a Pepa L.\n \u003c/div\u003e\n \u003c/div\u003e\n \u003c/a\u003e\n\n \u003c/li\u003e\n \u003c/ul\u003e\n\n \u003c/div\u003e\n \u003c/div\u003e\n\n \u003cdiv class=\"review-wrapper\"\u003e\n \u003cdiv class=\"review-content\"\u003e\n \u003cdiv class=\"biz-rating biz-rating-very-large clearfix\"\u003e\n \u003cdiv itemprop=\"reviewRating\" itemscope itemtype=\"http://schema.org/Rating\"\u003e\n \n \u003cdiv class=\"rating-very-large\"\u003e\n \u003ci class=\"star-img stars_3\" title=\"Valoraci\u00f3n de 3.0 estrellas\"\u003e\n \u003cimg alt=\"Valoraci\u00f3n de 3.0 estrellas\" class=\"offscreen\" height=\"303\" src=\"http://s3-media3.ak.yelpcdn.com/assets/2/www/img/c2252a4cd43e/ico/stars/v2/stars_map.png\" width=\"84\"\u003e\n \u003c/i\u003e\n \u003cmeta itemprop=\"ratingValue\" content=\"3.0\"\u003e\n \u003c/div\u003e\n\n\n \u003c/div\u003e\n \u003cspan class=\"rating-qualifier\"\u003e\n \u003cmeta itemprop=\"datePublished\" content=\"2011-12-07\"\u003e\n 7/12/2011\n \u003c/span\u003e\n\n \u003c/div\u003e\n\n \n \u003cp class=\"review_comment ieSucks\" itemprop=\"description\" lang=\"es\"\u003eMi amiga Mele ha sido mam\u00e1.\u003cbr\u003eElla vive en Vejer y vino a traer el ni\u00f1o a su familia que vive en Reina Mercedes. \u003cbr\u003eCuando estaba en casa de su madre, aprovechamos para quedar con ella y ver al peque\u00f1\u00edn.\u003cbr\u003e\u003cbr\u003eComo el beb\u00e9 es casi reci\u00e9n nacido y aquella ma\u00f1ana de s\u00e1bado hac\u00eda bastante fr\u00edo, nos quedamos a tomar unas cervecitas y unas tapitas en un bar que hab\u00eda justo al lado del portal donde vive su madre. Casa Felix.\u003cbr\u003e\u003cbr\u003eNos sentamos en la terraza exterior, algunas mesas est\u00e1n bajo los soportales y quedan refugiadas del fr\u00edo.\u003cbr\u003eDentro tambi\u00e9n hay unas cuatro mesas; pero el local es bastante ruidoso. Camareros que entran y salen gritando las comandas en un local decorado con motivos marineros. Y es que casa Felix est\u00e1 especializado en pescados y mariscos.\u003cbr\u003eNos pedimos unas huevas a la plancha bastante ricas. Luego pedimos salmonetes, caz\u00f3n en adobo y puntillitas. El pescado est\u00e1 bien frito y la relaci\u00f3n calidad/precio est\u00e1 bien.\u003cbr\u003eEn la carta tambi\u00e9n hab\u00eda carnes y revueltos.\u003cbr\u003e\u003cbr\u003eEn un momento se llen\u00f3 la terraza y el sol comenz\u00f3 a asomar. \u003cbr\u003eEchamos un buen ratito y conocimos a Teo, el precioso beb\u00e9 de Mele.\u003c/p\u003e \n \u003cul class=\"photo-box-grid clearfix js-content-expandable\"\u003e\n \u003cli \u003e\n \u003cdiv class=\"photo-box has-overlay\"\u003e\n \n \u003cimg alt=\"\" class=\"photo-box-img\" height=\"168\" src=\"//s3-media1.ak.yelpcdn.com/bphoto/ylf3jcKNwfvbiS-E9KruMg/168s.jpg\" width=\"168\"\u003e\n\n\n\n\n \n\n \u003ca class=\"biz-shim\" href=\"/biz_photos/casa-f%C3%A9lix-sevilla-2?filter_by_userid=True\u0026amp;select=ylf3jcKNwfvbiS-E9KruMg\u0026amp;userid=evo0GqKqi-KCjSSa_QMxIw\"\u003e\n \u003c/a\u003e\n\n \u003c/div\u003e\n\n \u003c/li\u003e\n \u003cli \u003e\n \u003cdiv class=\"photo-box has-overlay\"\u003e\n \n \u003cimg alt=\"\" class=\"photo-box-img\" height=\"168\" src=\"//s3-media4.ak.yelpcdn.com/bphoto/BXbYufp3J1bFDm0bcfhaaQ/168s.jpg\" width=\"168\"\u003e\n\n\n\n\n \n\n \u003ca class=\"biz-shim\" href=\"/biz_photos/casa-f%C3%A9lix-sevilla-2?filter_by_userid=True\u0026amp;select=BXbYufp3J1bFDm0bcfhaaQ\u0026amp;userid=evo0GqKqi-KCjSSa_QMxIw\"\u003e\n \u003c/a\u003e\n\n \u003c/div\u003e\n\n \u003c/li\u003e\n\n \u003c/ul\u003e\n\n \u003c/div\u003e\n \u003cdiv class=\"review-footer clearfix\"\u003e\n \u003cdiv class=\"rateReview ufc-feedback clearfix\" data-review-id=\"wnfqFC83791yr_FXVd3v8Q\"\u003e\n \u003cp class=\"review-intro review-message\"\u003e\n \u00bfEl comentario es \u0026hellip;?\n \u003c/p\u003e\n\n \u003cul class=\"big-ufc\" data-csrf-token=\"79d0ffc00e41a215f4b802aa68a69d49294d02e2c4c16e208c5c702321321697\"\u003e\n \n\n\n \u003cli class=\"ufc-stat inline-block ytype\"\u003e\n \u003ca href=\"javascript:;\" rel=\"useful\" class=\"ybtn ybtn-tertiary ybtn-small useful ytype\"\u003e\u003cspan class=\"i-wrap ig-wrap-common i-big-ufc-useful-common-wrap button-content\"\u003e\u003ci class=\"i ig-common i-big-ufc-useful-common\"\u003e\u003c/i\u003e \u003cspan class=\"vote-type\"\u003e\u00datil\u003c/span\u003e\n \u003cspan class=\"count\"\u003e1\u003c/span\u003e\u003c/span\u003e\u003c/a\u003e\n \u003c/li\u003e\n\n \n\n\n \u003cli class=\"ufc-stat inline-block ytype\"\u003e\n \u003ca href=\"javascript:;\" rel=\"funny\" class=\"ybtn ybtn-tertiary ybtn-small funny ytype\"\u003e\u003cspan class=\"i-wrap ig-wrap-common i-big-ufc-funny-common-wrap button-content\"\u003e\u003ci class=\"i ig-common i-big-ufc-funny-common\"\u003e\u003c/i\u003e \u003cspan class=\"vote-type\"\u003eDivertido\u003c/span\u003e\n \u003cspan class=\"count\"\u003e\u003c/span\u003e\u003c/span\u003e\u003c/a\u003e\n \u003c/li\u003e\n\n \n\n\n \u003cli class=\"ufc-stat inline-block ytype\"\u003e\n \u003ca href=\"javascript:;\" rel=\"cool\" class=\"ybtn ybtn-tertiary ybtn-small cool ytype\"\u003e\u003cspan class=\"i-wrap ig-wrap-common i-big-ufc-cool-common-wrap button-content\"\u003e\u003ci class=\"i ig-common i-big-ufc-cool-common\"\u003e\u003c/i\u003e \u003cspan class=\"vote-type\"\u003eLo m\u00e1s\u003c/span\u003e\n \u003cspan class=\"count\"\u003e\u003c/span\u003e\u003c/span\u003e\u003c/a\u003e\n \u003c/li\u003e\n\n \u003c/ul\u003e\n \u003c/div\u003e\n\n \u003c/div\u003e\n \n \n\n \u003c/div\u003e\n \u003c/div\u003e\n\n \u003c/li\u003e", "full_page_url": "http://www.yelp.es/biz/casa-f%C3%A9lix-sevilla-2"}
json_results = json.loads(response_string)
#print json_results
reviews = json_results['review_list'].encode('utf-8')
regexp_users = 'class="user-display-name" href="/user_details\?userid=([^\""]+)".+?">(.+?)</a>'
regexp_ratings = '<i class="star-img stars_([0-9])'
# Hack to get some of the Qype users as well, those ones
# who have at least an avatar, otherwise is impossible to referenciate them :-D
regexp_qype_users = 'ak.yelpcdn.com/photo/([^/]+)/60s.jpg"\s+width="60">\s+</div>.+?<span class="ghost-qyper"><b>Usuario Qyper</b>'
regexp = re.compile(regexp_users+'.+?'+regexp_ratings, re.DOTALL)
regexp_qype = re.compile(regexp_qype_users+'.+?'+regexp_ratings, re.DOTALL)
users = re.findall(regexp, reviews)
users_qype = re.findall(regexp_qype, reviews)
all_users = users + users_qype
for user in all_users:
id_user = user[0]
numerical_rating = user[-1]
# Save a new RatingUser in the DB or Update
rating_user = RatingUser(user_id=id_user)
rating_user.save()
try:
# Save a new Rating
rating = Rating(user_id=rating_user, rating=numerical_rating)
rating.item_id_id = id_business
rating.save()
print "Saved rating for "+ str(id_business)
except Exception:
# In case the record is already there
pass
conn.close()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment