vk_scraper.py

import urllib.request
import time
import datetime
import re

def req(url):
    u = urllib.request.urlopen(url)
    r = u.read()
    response = eval(r)
    return response['response']
  
class Post:
    def __repr__(self):
        return "https://vk.com/wall" + str(self.owner_id) + "_" + str(self.id)
    
    def who_likes(self):            
        times = int(self.likes / 1000) + 1
        fans = []
        for offset in range(0, times * 1000, 1000):
            response = req("https://api.vk.com/method/likes.getList?owner_id=" + str(self.owner_id) + "&item_id=" + str(self.id) + "&type=post&filter=likes&count=1000&offset=" + str(offset))
            fans.extend(response['users'])
        return fans
        
    def get_likes(self):
        r = req("https://api.vk.com/method/likes.getList?owner_id=" + str(self.owner_id) + "&item_id=" + str(self.id) + "&type=post&filter=likes&count=1&offset=0")
        self.likes = r['count'] 

    def __init__(self, owner_id, id, post = None, members = -1):
        self.owner_id = owner_id
        self.id = id
        self.likes = post['likes']['count']
        self.date = post['date']
        self.text = post['text']
        self.reposts = post['reposts']['count']
        self.comments = post['comments']['count']
        self.virality = float(self.likes + self.reposts) / float(members)
        

class Posts:
    def __repr__(self):
        s = ""
        for post in self.posts:
            s += str(post) + "\n"
        return s
    
    def sort(self, criteria = 'likes'):
        if criteria == 'likes':
            self.posts.sort(key = lambda Post: Post.likes, reverse = True)
        elif criteria == 'reposts':
            self.posts.sort(key = lambda Post: Post.likes, reverse = True)
        elif criteria == 'date':
            self.posts.sort(key = lambda Post: Post.date, reverse = True)
        elif criteria == 'virality':
            self.posts.sort(key = lambda Post: Post.virality, reverse = True)
    
    def user_likes(self, user):
        for post in self.posts:
            if user in post.who_likes():
                print('user likes ' + str(post))
                
    def __init__(self, posts):
        self.posts = posts

class Page(Posts):
    def get_posts(self, since_time, until_time, filter_words, stop_words):     
        times = int(self.number_of_posts / 100) + 1
        posts = []
        right_time = 0
        
        for offset in range(0, times * 100, 100):
            response = req("https://api.vk.com/method/wall.get?owner_id=" + str(self.id) + "&count=1000&offset=" + str(offset))[1:]
            for post in response:
                if post['date'] <= until_time and post['date'] >= since_time and self.filter_by_words(post['text'], filter_words, True) and self.filter_by_words(post['text'], stop_words, False):
                    #record = Post(self.id, post['id'], post['likes']['count'], post['date'], post['text'], post['reposts']['count'], post['comments']['count'], float(post['likes']['count'] + post['reposts']['count']) / float(self.members))
                    record = Post(self.id, post['id'], post, self.members)
                    if record.likes > 1000:
                        posts.append(record)
                if post['date'] < since_time:
                    right_time += 1
                if right_time == 2:
                    break
                    
            if right_time == 2:
                break
                
        return posts
    
    def word_filter(self, text, word):
        word = word.lower()
        text = text.lower()
#             pattern = re.compile(words)
#             bool(pattern.match(text))
        return word in text
    
    def filter_by_words(self, text, words, flag = True):
        if type(words) is str:
            result = self.word_filter(text, words)
            return result if flag else not result
        elif words is None:
            return True
        else:
            text = text.lower()
            if flag:
                for word in words:
                    if not self.word_filter(text, word):
                        return False
                return True
            else:
                for word in words:
                    if self.word_filter(text, word):
                        return False
                return True
    
    def __init__(self, id, since_time = 0, until_time = 9223372036854775807, filter_words = None, stop_words = None):
        if type(id) is str:
            id = re.sub(r'(https://|vk.com)', r'', id)
            response = req("https://api.vk.com/method/groups.getById?group_ids=" + id)
            self.id = - response[0]['gid']
        else:
            self.id = id

        response = req("https://api.vk.com/method/groups.getMembers?group_id=" + str(-self.id) + "&count=1")
        self.members = response['count']
    
        response = req("https://api.vk.com/method/wall.get?owner_id=" + str(self.id) + "&count=1")
        self.number_of_posts = response[0] 
        self.posts = self.get_posts(since_time, until_time, filter_words, stop_words)
        Posts.__init__(self, self.posts)
    

get_cur_time = lambda: int(round(time.time()))
cur_time = get_cur_time()

delta1 = datetime.timedelta(days = 0)
delta2 = datetime.timedelta(days = 90)

start_time = cur_time - delta1.total_seconds() - delta2.total_seconds()
end_time = cur_time - delta2.total_seconds()

page = Page('itmozg', 0, end_time) #, 'Подборка')
page.sort()

print(page)