Simple spam detection of Common Search host-level page rank list: detect blocks of hosts with similar rank and host names which ev. form link farms

cs_despam_host_pagerank.py

import fileinput
import sys
import tldextract
from _collections import defaultdict
from math import log


RANK_DIVERGENCE_THR = 0.02
HOST_LENGTH_DIVERGENCE_THR = 0.15

LEN_NORM = .2
SPAM_SKIP_THR = .2

MIN_PAGERANK = 0.0  # for testing or to select only higher ranking hosts
FAST_DECISION = True

counters = defaultdict(int)
spam_domains = defaultdict(int)


def mark_spam(buffer):
    for (host, rank, len_host, hasNum, suffix, domain) in buffer:
        score = 0.0
        if suffix == '':
            # ip address or failed to parse
            print('-\t{}\t{}\t{}\t{}'.format(score, rank, host,
                                             convert_rank(rank)))
            counters['bad host'] += 1
            continue
        for (h, r, l, n, s, d) in buffer:
            if host is h or s == '':
                continue
            sc = RANK_DIVERGENCE_THR - abs(1.0 - abs(r/rank))
            if hasNum and n:
                # both host names contain numbers
                sc *= 2.0
            if domain == d:
                # same domain
                sc *= 2.0
            elif suffix == s:
                # from same tld
                sc *= 1.2
            # decrease score if lengths differ
            sc /= (1.0 + LEN_NORM * abs(len_host-l))
            score += sc
            if FAST_DECISION and score > (2.0*SPAM_SKIP_THR):
                break
        mark = '+'
        if score > SPAM_SKIP_THR or score > (rank/2.0):
            mark = '-'
            counters['spam'] += 1
            spam_domains[domain] += 1
        else:
            counters['good'] += 1
        print('{}\t{}\t{}\t{}\t{}'.format(mark, score, rank, host,
                                          convert_rank(rank)))


def convert_rank(rank):
    """convert page rank to value between 0.0 and 10.0
        0.15 => 0.0"""
    rank = log(.85+rank)
    if rank > 10.0:
        rank = 10.0
    elif rank < 0.0:
        rank = 0.0
    return '%.4e' % rank


buffer = list()
buffer_str_len = 0

for line in fileinput.input():
    counters['total'] += 1
    fields = line.split('\t')
    if len(fields) != 2:
        sys.stderr.write('bad line: ' + line)
        counters['bad line'] += 1
        continue
    (host, rank) = fields
    rank = float(rank)
    if rank < MIN_PAGERANK:
        break
    len_host = len(host)
    if len_host == 0:
        sys.stderr.write('empty host: ' + line)
        counters['bad line'] += 1
        continue
    host_contains_number = any(char.isdigit() for char in host)
    parsed_host = tldextract.extract(host)
    domain = '.'.join([parsed_host.domain, parsed_host.suffix])
    if len(buffer) > 0:
        avg_buffer_host_len = buffer_str_len / len(buffer)
        len_dif = abs(1.0 - (len_host / avg_buffer_host_len))
        if ((buffer[0][1] / rank) > (1.0 + RANK_DIVERGENCE_THR)) or \
           (len_dif > HOST_LENGTH_DIVERGENCE_THR and
           domain != buffer[-1][5]):
            # host or rank are different: start next block
            mark_spam(buffer)
            buffer = list()
            buffer_str_len = 0
    buffer.append([host, rank, len_host, host_contains_number,
                   parsed_host.suffix, domain])
    buffer_str_len += len(host)

mark_spam(buffer)

sys.stderr.write('statistics:\n')
for name, count in counters.items():
    percent = 100.0 * count / counters['total']
    sys.stderr.write('%8d  %6.2f%%\t%s\n' % (count, percent, name))

sys.stderr.write('spam domains total: %d\n' % len(spam_domains))
sys.stderr.write('top spam domains:\n')
domains_shown = 0
for domain in sorted(spam_domains, key=spam_domains.get, reverse=True):
    domains_shown += 1
    if domains_shown <= 20:
        sys.stderr.write('%6d\t%s\n' % (spam_domains[domain], domain))
    print('D', spam_domains[domain], domain)

cs_despam_host_pagerank.sh

#!/bin/bash

# download data
wget https://dumps.commonsearch.org/webgraph/201606/host-level/pagerank/pagerank.txt.gz

# sort by decreasing rank and reverse host name,
# use tab as separator
zcat pagerank.txt.gz \
    | perl -lne 's/\s+([0-9.]+)$//; print scalar reverse, "\t", $1' \
    | LC_ALL=C sort --compress-program=gzip -t$'\t' -k2,2nr -k1,1 \
    | perl -lne 's/\t([0-9.]+)$//; print scalar reverse, "\t", $1' \
    | gzip \
    > pagerank_sort.txt.gz

# try to mark spam
zcat pagerank_sort.txt.gz \
    | LC_ALL=en_US.utf8 python3 cs_despam_host_pagerank.py \
    | gzip \
    > pagerank_spam_marked.txt.gz

# write seed file for Nutch
zcat pagerank_spam_marked.txt.gz \
    | perl -lne '($ok, $sp, $pr, $host, $sc) = split /\s+/; print "http://", $host, "/\tnutch.score=", $sc if $ok eq "+"' \
    | bzip2 \
    > cs_host_pagerank_inject.txt.bz2