facescraper.py

#-*- coding: utf8 -*-
import json
from json import load
import sqlite3
import hmac
import hashlib
from facepy import GraphAPI
from django.core.serializers.json import DjangoJSONEncoder
import json

'''
 For this script to work, you first need to create a sqlite3 database
 named fs.db with the following command:
 CREATE TABLE main (encryptedpostid TEXT, group_name TEXT, encryptedidfield TEXT, thedate TEXT, message TEXT, UNIQUE(encryptedpostid));
'''
conn = sqlite3.connect('fs.db')

addedrows = 0

# Get a token from https://developers.facebook.com/tools/explorer/
access_token = ""

groups = {
        "Sverigedemokraterna": 676246449160270,
        "Moderaterna": 7719591155,
        "Centerpartiet": 12690894340,
        "Socialdemokraterna": 8040892957,
        "Liberalerna": 132266740623,
        "Feministiskt initiativ": 112513375470294,
        "Vänsterpartiet": 323156205347,
        "Miljöpartiet": 34780126150,
        "Kristdemokraterna": 115198178364,
        "Piratpartiet": 7336236196,
        "Liberala partiet": 194555880406,
        "Kommunistiska partiet": 152953178070559,
        "Avpixlat": 1456773041274824,
        "Galago": 115702118491956,
        "Dagens Nyheter": 321961491679,
        "Aftonbladet": 124292684247609,
        "SVT": 152358601473403,
        "Expressen": 37134975344,
        "GP Ledare": 915684658452138,
        "Bubbla": 244909922366444,
        "Nyheter Idag": 647421441984111,
        "ETC": 369582590803,
        "Vi som stödjer sveriges medborgargarde": 124275634443303,
        "Vi som älskar fredagsmys": 290035680194,
        "Glänta": 128439233871779,
        "Kommunal": 179931025357058,
        "LO": 163820136238,
        "Greenpeace Sverige": 95735961890,
        "Nätverket Linje 17": 192100317482100,
        "planka.nu": 60883417696,
        "Svd": 96178346173,
        "Refugees Welcome Stockholm": 538286309669991,
        "Supersnippan": 169647209730395,
        "Stoppa maktmissbruket": 1054122034605760,
        "Vi avslöjar rasisterna": 1642166096066888,
        "Vi gillar olika": 117963821590139,
        "Världsnaturfonden WWF": 371791194529,
        "Läkare utan gränser": 800360196753998,
        "Musikerförbundet": 63554872384,
        "Svenska kyrkan": 15223945415,
        "Muslimsk kvinna": 299623940189416,
        "Scouterna": 111005714391,
        "Sverok": 391759218754,
        "Hemslöjden": 296285327106241,
        "Allt om islam": 317629638324311,
        "Judiska församlingen Stockholm": 105131542852545,
        "ABF Göteborg": 204628709604397,
        "Afrosvenskarnas riksförbund": 1423043744632690,
        "SDNejTack": 142027660912,
        "Vi som älskar att mysa": 140964994153,
        "Djurens rätt": 19746853632,
        "Kungahuset": 133515803374007,
        "Unicef Sverige": 59324455950,
        "Inte rasist men": 210114715755977,
        "Romers rättigheter": 183309435339674,
        "Folkkamp för asylrätt": 195574290793670,
        "Vetenskap och Folkbildning": 2362213620,
        "Vaken.se": 4926532303,
        "Allt åt alla Stockholm": 180369687403,
        "Feminism underifrån": 1422891258003981,
        "Kontrapunkt": 176219345754217,
        "Cyklopen": 528677750491192,
        "Kulturhuset Jönköping": 103180474992,
        "Antifascism är självförsvar": 256030537889875,
        "Soldiers of Odin Sweden": 253750258297512,
        "Skattebetalarnas förening": 72832167834,
        "Företagarnas Riksförbund": 124892400864698,
        "Nätverket Ofog": 284662214922092,
        "Freds och skiljedomsföreningen": 127063960674693,
        "Individuell människhjälp": 113006852188985,
        "Motormännens riksförbund": 281307768603480,
        "Anonymous Sweden": 228172710552909,
        "Occupy Sweden": 171089306306925,
        "Grön ungdom": 113234935374946,
        "Ung vänster": 293573868390,
        "Moderata ungdomsförbundet": 139375052786521,
        "SSU": 45101268520,
        "Centerpartiets ungdomsförbund": 110793008936879,
        "Kristdemokratiska ungdomsförbundet": 177549278946958,
        "Liberala ungdomsförbundet": 108941769135355,
        "Sverigedemokratisk ungdom": 160333517324943,
        "Ung pirat": 92683221988,
        "LCHF Fakta och Debatt": 515119345177700,
        "Vi som vill avveckla Livsmedelsverket": 667917029919842,
        "Livsmedelsverket": 117067621647704,
        "Svenska kommitten mot antisemitism": 101762947469,
        "Yimby Göteborg": 18858735822,
        "Nej till trängselskatt i Göteborg": 104188036293500,
        "Ja till trängselskatt i Göteborg": 356775831071782,
        "Stoppa västlänken nu": 35630384565,
        "Stoppa TTIP Sweden": 1481053882109899,
        "No Pegida Sweden": 391683337659508,
        "Tidskriften EXPO": 53779680285,
        "Pegida Sweden": 1436446579978630,
        "Riv upp DÖ": 479191938888400,
        "Nyval NU": 536557109828124,
        "SR P1": 102331493143871,
        "Sveriges Radio": 146516072147644,
        "Sveriges Kvinnolobby": 181409445272366,
        "Rättviseförmedlingen": 361481469749
        }


def addtodatabase(group_name, group_id):
    graph = GraphAPI(access_token)
    pages = graph.get(group_id + "/feed", page=True, retry=3, limit=1)
    for p in pages:
        print('Downloading page')
        jsondata = p
        addedrows = 0
        integritycounter = 0
        for item in jsondata['data']:
            if item:
                try:
                    if item['comments']['data']:
                        for i in item['comments']['data']:
                            postid = i['id']
                            thedate = i['created_time']
                            idfield = i['from']['id']
                            message = i['message']
                            commentid = i['id']
                            digest_maker1 = hmac.new(b'password_replace_me', postid.encode(), hashlib.sha256)
                            encryptedpostid = digest_maker1.hexdigest()
                            digest_maker2 = hmac.new(b'password_replace_me', idfield.encode(), hashlib.sha256)
                            encryptedidfield = digest_maker2.hexdigest()
                            conn.execute("INSERT INTO main (encryptedpostid, group_name, encryptedidfield, thedate, message) \
                            VALUES (?, ?, ?, ?, ?);", (encryptedpostid, group_name, encryptedidfield, thedate, message))
                            conn.commit()
                            print("SUCCESS! Added 1 Row to database for the group " + group_name)
                            addedrows += 1
                except KeyError:
                    print("FAIL! Found no comment in this post.")
                except sqlite3.IntegrityError:
                    while integritycounter < 600:
                        print("***** Already in Database: for " + group_name + " SKIPPING *****")
                        integritycounter += 1
                    print("***** No new comments in this post. Continuing to the next one")
                    integritycounter = 0
                    return

def selectgroup():
    for key, value in groups.items():
        group_name = key
        group_id = str(value)
        addtodatabase(group_name, group_id)

selectgroup()