scrape1.py

import lxml.html

odluki = 'http://ustavensud.mk/?cat=82'

def iterate_pages_from_category(start_page):
    page_url = start_page
    while True:
        p = lxml.html.parse(page_url)
        root = p.getroot()
        yield from [ el.attrib['href'] for el in root.cssselect('h3.gdlr-blog-title a') ]

        next_url = root.cssselect('a.page-numbers.next')
        if len(next_url) == 0:
            return
        page_url = next_url[0].attrib['href']
            
pages = list(iterate_pages_from_category(odluki))


# find duplicates (!?)
from collections import Counter
cnt = Counter(pages)
print([item for item, count in cnt.items() if count > 1])

scrape2.py

import lxml.html
import requests

odluki = 'http://ustavensud.mk/?cat=82'

def iterate_pages_from_category(start_page):
    page_url = start_page
    session = requests.Session()
    while True:
        resp = session.get(page_url)
        root = lxml.html.fromstring(resp.content, base_url=page_url)
        yield from [ el.attrib['href'] for el in root.cssselect('h3.gdlr-blog-title a') ]

        next_url = root.cssselect('a.page-numbers.next')
        if len(next_url) == 0:
            return
        page_url = next_url[0].attrib['href']
            
pages = list(iterate_pages_from_category(odluki))


# find duplicates (!?)
from collections import Counter
cnt = Counter(pages)
print([item for item, count in cnt.items() if count > 1])