dedup.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
import pandas

#Open workbook with Shodan results
shodan_xl = pandas.ExcelFile('shodan.xls')
#Read dorks from workbook to DataFrame
second_df = pandas.read_excel('second_shodan.xls', encoding='utf=8')

results = []   #List with dicts of all results

i = 0
for second_index, second_row in second_df.iterrows():
    i += 1
    dup = False
    result = {}
    second_ip = second_row['IP']
    second_port = second_row['Port']
    for sheet in shodan_xl.sheet_names:
        if sheet == 'Stats' or sheet == 'Vulns':
            continue
        sheet_df = shodan_xl.parse(sheet, encoding='utf=8', na_values=['NaN'], keep_default_na=False)
        for shodan_index,shodan_row in sheet_df.iterrows():
            shodan_ip = shodan_row["IP"]
            shodan_port = shodan_row["Port"]

            if second_ip == shodan_ip and second_port == shodan_port:
                print 'IP %s' %  i, second_ip, ':', second_port,': DUPLICATED'
                dup = True
                break

    if not dup:
        print i, second_row["IP"], second_row["Port"]
        result['IP'] = second_row["IP"]
        result['Port'] = second_row["Port"]
        result['Service'] = second_row["Service"]
        result['Banner'] = second_row["Banner"]
        results.append(result)
writer = pandas.ExcelWriter('result.xls')

result_df = pandas.DataFrame(results)
result_df.to_excel(writer, 'Dorks finded')
writer.save()