Skip to content

Instantly share code, notes, and snippets.

@Gjum

Gjum/chunk.py Secret

Last active March 11, 2016 03:50
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save Gjum/0375b643ec13a42ab3c0 to your computer and use it in GitHub Desktop.
Save Gjum/0375b643ec13a42ab3c0 to your computer and use it in GitHub Desktop.
import sys, glob, json, struct
from collections import defaultdict
import minecraft_data.v1_9 as mcd
# Minecraft varints are 32-bit signed values
# packed into Google Protobuf varints
def unpack_varint(bbuff):
total = 0
shift = 0
val = 0x80
while val & 0x80:
val = struct.unpack('B', bbuff.read(1))[0]
total |= (val & 0x7F) << shift
shift += 7
if total >= (1 << 32):
return None
if total & (1 << 31):
total -= 1 << 32
return total
class BufferUnderflowException(Exception):
pass
def read_section_blocks(buff):
"""returns iterable over the 4096 blocks in the section"""
results = {}
results['unusual'] = defaultdict(list)
block_bits = buff.read(1)[0]
results['block_bits'] = block_bits
uses_palette = block_bits > 0
palette = {}
if uses_palette:
palette_len = unpack_varint(buff)
results['palette_len'] = palette_len
results['palette'] = palette
assert palette_len <= 2 ** block_bits, \
'%s < %s = %s^2' % (palette_len, 2 ** block_bits, block_bits)
for i in range(palette_len):
palette[i] = unpack_varint(buff)
else: # use global palette
block_bits = 13
data_len = unpack_varint(buff) * 8
if data_len * 8 < 4096 * block_bits:
raise BufferUnderflowException(
'Too little block data: %s < %s' % (data_len * 8, 4096 * block_bits))
data = buff.read(data_len)
results['blocks'] = [None] * 4096
for i in range(4096):
block = 0
# blocks are not byte-aligned, read bitwise
# TODO might be faster to combine bytes and shift+AND
for bit_offset in range(i * block_bits, (i + 1) * block_bits):
bit = 1 & (data[bit_offset // 8] >> (7 - (bit_offset % 8)))
block = (block << 1) | bit
if uses_palette: # convert to global palette
try:
block = palette[block]
except KeyError:
results['unusual']['%s (key)' % block].append(i)
continue # next block
if block < 0:
results['unusual']['%s (val)' % block].append(i)
results['blocks'][i] = block
return results
def main():
columns_read = 0
sections_read = 0
unusual_sections = 0
for file_name in glob.glob('chunks-1.9.1-pre2/chunk_*_*.dump'):
buff = open(file_name, 'rb')
packet_fn = file_name.replace('chunk_', 'packet_').replace('.dump', '.data')
packet = json.load(open(packet_fn, 'r'))
has_skylight = True
for y in range(16):
if not (packet['bitMap'] >> y) & 1:
continue
prev_pos = buff.tell()
try:
results = read_section_blocks(buff)
except Exception as e:
exc_str = 'Exception: %s: %s' % (type(e).__name__, e)
print(exc_str)
now_pos = buff.tell()
buff.seek(prev_pos)
excess = buff.read(now_pos - prev_pos)
print('excess data:', len(excess), excess[:256])
print('chunk column ended after', prev_pos, now_pos)
print_data(exc_str, **results)
raise
if results['unusual']:
unusual_sections += 1
print('\n=====', file_name, 'section', y, 'at byte', prev_pos, '=====\n')
print_data('Unusual blocks found', **results)
block_light = buff.read(2048)
if has_skylight:
sky_light = buff.read(2048) if has_skylight else None
sections_read += 1
if packet['groundUp']:
biome_data = buff.read(256)
remainder = buff.read()
if remainder: # should not occur since 1.9.1-pre2
print(len(remainder), 'byte of data left after biome data')
columns_read += 1
if columns_read % 50 == 0:
print(columns_read, 'columns read so far')
print('columns read:', columns_read)
print('sections read:', sections_read)
print('unusual sections:', unusual_sections)
def print_data(name, block_bits=0, palette_len=0, palette={}, blocks=[], unusual={}):
print('%s:' % name)
print(block_bits, 'block_bits,', palette_len, 'palette entries')
print('\nPalette:')
found_from_palette = 0
lines = []
for k, v in palette.items():
usages = tuple(i for i, b in enumerate(blocks) if b == v)
lines.append('%3s: %4s %2s %-20s %4sx: %s' % (k, v >> 4, v & 0xf, data_to_name(v), len(usages), ' '.join(map(str, usages))))
found_from_palette += len(usages)
print_shortened(lines)
if found_from_palette != len(blocks):
print('\npalette only knows %s of the 4096 blocks' % found_from_palette)
if unusual:
lines = list('%10s: %3sx at %s' % (key, len(founds), ' '.join(map(str, founds))) for key, founds in unusual.items())
print()
print(len(lines), 'unusual palette usages:')
print_shortened(lines)
def print_shortened(lines):
for line in sorted(lines):
print(shorten_str(line, 120, 1, '...'))
def shorten_str(s, n, frac=.5, sep='~'):
"""
shorten_str('abcxyz', 5) => 'ab~yz'
shorten_str('abcxyz', 5, 1, sep='...') => 'ab...' # asymmetric
shorten_str('a', 42) => 'a' # no padding applied
"""
s = str(s)
if len(s) <= n: return s
n -= len(sep)
a = int(n * frac)
b = n - a
return s[:a] + sep + (s[-b:] if b > 0 else '')
def data_to_name(data):
bid, meta = data >> 4, data & 0xf
block = mcd.find_item_or_block(bid)
if not block:
return '(unknown block)'
for b in block.get('variations', []):
if b['metadata'] == meta:
block.update(b)
break
return block['displayName']
if __name__ == '__main__':
main()
full log at https://gist.github.com/Gjum/8f1c0344e73f564abc65
columns read: 389
sections read: 1806
unusual sections: 143
-1 sections: 143
unknown key sections: 140
===== chunks-1.9.1-pre2/chunk_-9_4_1457663363663.dump section 2 at byte 12329 =====
Unusual blocks found:
5 block_bits, 15 palette entries
Palette:
0: 0 0 Air 508x: 0 1 3 4 5 6 14 51 115 122 179 243 256 257 260 307 371 435 499 518 519 520 56...
1: 1 0 Stone 648x: 26 27 28 29 30 31 32 33 34 35 36 37 90 91 92 93 94 95 96 97 98 99 100 101 12...
2: 16 0 Coal Ore 631x: 2 7 8 9 10 11 12 64 65 66 67 68 69 70 71 72 73 74 75 76 128 129 131 132 133 ...
3: 13 0 Gravel 19x: 543 799 844 1100 1356 1376 1377 1440 1612 1632 1676 1696 1824 1932 1952 2080...
4: 1 3 Diorite 673x: 38 39 40 41 42 43 44 45 46 47 48 49 50 102 103 104 105 106 107 108 109 110 1...
5: 1 5 Andesite 45x: 990 991 1201 1245 1246 1247 1306 1457 1501 1502 1503 1562 1757 1758 1759 196...
6: 11 12 Stationary Lava 12x: 1207 1463 1909 1973 2566 2567 2820 2821 2822 2823 2824 3080
7: 10 1 Lava 0x:
8: 15 0 Iron Ore 614x: 13 15 16 17 18 19 20 21 22 23 24 77 78 79 80 81 82 83 84 85 86 87 88 141 142...
9: 11 0 Stationary Lava 95x: 25 89 153 217 281 345 409 473 537 601 665 729 793 848 854 857 921 981 1038 1...
10: 11 2 Stationary Lava 41x: 1028 1029 1030 1031 1228 1284 1285 1286 1287 1350 1351 1483 1540 1541 1542 1...
11: 11 4 Stationary Lava 9x: 1484 1485 1740 1741 1944 2198 2199 2200 2456
12: 10 4 Lava 16x: 553 809 1066 1067 1322 1323 1386 1387 1450 1510 1642 1643 1702 1706 1766 1958
13: 3 0 Dirt 14x: 1444 1445 1699 1700 1701 1953 1954 1955 1956 1957 2209 2210 2211 2212
14: -1 15 (unknown block) 0x:
palette only knows 3325 of the 4096 blocks
11 unusual palette usages:
16 (key): 608x at 52 53 54 55 56 57 58 59 60 61 62 63 116 117 118 123 124 125 126 127 180 181 182 183 184 185 186 1...
17 (key): 8x at 119 1075 1076 1331 1332 1396 1587 2815
18 (key): 39x at 823 829 889 894 1078 1084 1144 1150 1279 1339 1340 1341 1342 1400 1406 1535 1591 1595 1596 1597 1...
19 (key): 1x at 1141
20 (key): 46x at 999 1000 1001 1193 1199 1255 1256 1257 1330 1449 1456 1467 1511 1512 1513 1586 1705 1712 1767 176...
21 (key): 16x at 1198 1397 1398 1454 1455 1709 1710 1711 1963 1964 1965 1966 1967 2220 2221 2222
22 (key): 19x at 1208 1464 1465 1466 1718 1719 1720 1721 1722 1910 1974 1975 1976 1977 2164 2165 2166 2231 2232
24 (key): 10x at 1111 1112 1367 1368 1408 1431 1687 2831 2832 2833
25 (key): 3x at 1177 2834 3090
26 (key): 20x at 1409 1473 1474 1475 1476 1665 1728 1729 1730 1731 1732 1920 1921 1922 1984 1985 1986 1987 2176 2177
27 (key): 1x at 1943
===== chunks-1.9.1-pre2/chunk_-9_4_1457663363663.dump section 3 at byte 19017 =====
Unusual blocks found:
5 block_bits, 19 palette entries
Palette:
0: 0 0 Air 323x: 51 115 179 307 371 435 508 563 627 683 691 694 755 819 874 885 920 941 951 1...
1: 1 0 Stone 597x: 26 27 28 29 30 31 37 90 91 92 93 94 95 96 97 154 155 156 157 158 159 160 161...
2: 15 0 Iron Ore 563x: 2 5 6 7 8 9 10 11 12 68 69 70 71 72 73 74 75 76 128 129 133 134 135 136 137 ...
3: 1 5 Andesite 57x: 32 33 34 35 36 98 99 100 101 223 354 355 356 357 421 549 610 611 612 613 804...
4: 1 1 Granite 623x: 3 4 38 39 40 41 47 48 49 50 102 103 104 105 106 107 114 166 167 168 169 170 ...
5: 16 0 Coal Ore 86x: 682 873 937 1114 1115 1129 1193 1370 1371 1385 1449 1641 1897 2600 2790 2800...
6: 13 0 Gravel 88x: 0 1 64 65 66 67 130 131 132 257 258 320 321 322 323 385 386 387 388 389 512 ...
7: 1 3 Diorite 8x: 1946 1947 2202 2203 2204 2458 2459 2460
8: 3 0 Dirt 594x: 13 14 15 16 17 18 19 20 21 22 78 79 80 81 82 83 84 85 86 87 143 144 145 146 ...
9: 82 0 Clay 111x: 153 217 281 409 665 729 857 921 1105 1113 1293 1305 1361 1369 1494 1549 1561...
10: 9 0 Stationary Water 85x: 473 918 983 1156 1157 1158 1174 1238 1412 1413 1414 1430 1686 1881 2137 2393...
11: 2 0 Grass Block 5x: 1945 2201 2457 2521 3616
12: 17 0 Wood 62x: 42 43 44 45 46 108 109 110 111 112 113 233 234 235 299 300 301 302 364 365 3...
13: 106 2 Vines 2x: 2736 3626
14: 106 1 Vines 7x: 1928 1929 2184 2185 2186 2441 2442
15: 106 4 Vines 3x: 3939 3941 3999
16: 31 1 Tall Grass 554x: 57 58 59 60 61 62 63 116 117 123 124 125 126 127 180 181 182 183 184 185 186...
17: 111 0 Lily Pad 71x: 52 53 54 55 56 118 119 120 121 122 244 245 308 309 310 311 312 374 375 376 3...
18: -1 15 (unknown block) 90x: 1146 1147 1148 1149 1402 1403 1404 1405 1846 1911 2038 2102 2166 2235 2423 2...
palette only knows 3929 of the 4096 blocks
9 unusual palette usages:
-1 (val): 90x at 1146 1147 1148 1149 1402 1403 1404 1405 1846 1911 2038 2102 2166 2235 2423 2615 2680 3339 3403 34...
19 (key): 6x at 1976 1980 2232 2237 2238 2494
20 (key): 83x at 692 1138 1139 1394 1395 1907 2622 2746 2810 2868 3399 3400 3401 3459 3460 3461 3462 3463 3464 346...
21 (key): 2x at 3596 3636
22 (key): 1x at 3584
24 (key): 56x at 23 24 77 88 141 142 214 333 344 397 398 525 526 527 589 653 654 781 782 783 1037 1906 1939 1966 1...
25 (key): 13x at 25 89 345 537 601 793 1049 1937 1938 2193 2194 2450 2674
26 (key): 3x at 2765 3607 3671
28 (key): 3x at 1970 2151 2407
===== chunks-1.9.1-pre2/chunk_-17_5_1457663363344.dump section 1 at byte 6169 =====
Unusual blocks found:
5 block_bits, 24 palette entries
Palette:
0: 0 0 Air 607x: 32 42 43 44 51 52 53 54 55 64 87 88 98 115 166 179 214 230 243 288 298 299 3...
1: 1 0 Stone 569x: 26 27 28 29 30 31 35 36 37 89 90 91 92 93 94 95 96 97 99 100 101 155 160 161...
2: 1 3 Diorite 628x: 0 1 2 3 4 5 6 7 8 11 12 65 66 67 68 69 70 71 72 73 74 75 76 128 129 130 131 ...
3: 16 0 Coal Ore 50x: 33 34 255 289 290 524 652 704 780 1215 1276 1292 1331 1379 1380 1484 1531 15...
4: 1 1 Granite 616x: 9 10 38 39 40 41 45 46 47 48 49 50 102 103 104 105 106 107 108 109 110 111 1...
5: 21 0 Lapis Lazuli Ore 2x: 154 509
6: 15 0 Iron Ore 20x: 221 549 750 805 1344 1345 1600 1601 1856 1857 1984 1985 2240 2241 2496 2944 ...
7: 1 5 Andesite 60x: 668 669 670 671 697 924 925 926 927 1182 1183 1311 1312 1313 1315 1316 1317 ...
8: 9 7 Stationary Water 583x: 13 14 15 16 17 18 19 20 21 22 23 77 78 79 80 81 82 83 84 85 86 141 142 143 1...
9: 9 5 Stationary Water 47x: 153 476 537 659 793 857 914 921 981 985 1049 1113 1237 1301 1304 1486 1557 1...
10: 9 6 Stationary Water 5x: 217 475 696 3673 3929
11: 9 4 Stationary Water 3x: 1241 1497 1753
12: 10 0 Lava 24x: 514 515 650 651 770 771 906 907 1646 1647 1771 1772 1902 2027 2028 2029 2284...
13: 11 0 Stationary Lava 0x:
14: 10 2 Lava 45x: 715 905 971 972 1024 1025 1028 1029 1227 1228 1280 1284 1285 1286 1356 1536 ...
15: 11 2 Stationary Lava 1x: 716
16: 10 10 Lava 600x: 56 57 58 59 60 61 62 63 116 117 118 119 120 121 122 123 124 125 126 127 167 ...
17: 11 10 Stationary Lava 24x: 190 210 251 447 507 660 764 916 917 1333 1334 1589 1590 1974 2230 2231 2329 ...
18: 10 1 Lava 0x:
19: 10 4 Lava 46x: 959 1277 1332 1532 1533 1588 1786 1787 1788 1844 1845 1846 1977 1978 2100 21...
20: 11 4 Stationary Lava 2x: 678 741
21: 10 6 Lava 1x: 511
22: 11 6 Stationary Lava 1x: 740
23: -1 15 (unknown block) 0x:
palette only knows 3934 of the 4096 blocks
7 unusual palette usages:
24 (key): 31x at 24 231 280 717 726 751 918 982 1232 1238 1295 1302 1488 1551 1622 1624 1744 1881 1934 2000 2008 2...
25 (key): 50x at 25 281 915 1230 1231 1293 1294 1305 1369 1487 1558 1559 1560 1561 1625 1741 1742 1743 1814 1815 1...
26 (key): 1x at 687
27 (key): 2x at 695 1485
28 (key): 75x at 425 679 680 681 935 936 937 1191 1192 1193 1266 1321 1322 1323 1324 1449 1521 1522 1577 1578 1579...
30 (key): 1x at 1476
31 (key): 2x at 706 1219
===== chunks-1.9.1-pre2/chunk_-17_-4_1457663363332.dump section 2 at byte 12327 =====
Unusual blocks found:
5 block_bits, 16 palette entries
Palette:
0: 0 0 Air 733x: 51 115 122 130 131 132 141 165 175 176 177 179 185 243 254 307 333 371 377 3...
1: 1 0 Stone 611x: 26 27 28 29 30 31 32 33 34 35 36 37 91 92 93 94 95 96 97 98 99 100 101 120 1...
2: 1 3 Diorite 620x: 0 1 2 3 4 5 6 7 8 9 10 11 12 64 65 66 67 68 69 70 71 72 73 74 75 76 137 138 ...
3: 13 0 Gravel 10x: 90 162 163 164 332 419 420 588 1036 3532
4: 1 5 Andesite 668x: 38 39 40 41 42 43 44 45 46 47 48 49 50 102 103 104 105 106 107 108 109 110 1...
5: 15 0 Iron Ore 4x: 2019 2020 3576 3578
6: 3 0 Dirt 10x: 128 129 133 136 223 385 389 390 479 3574
7: 1 1 Granite 33x: 859 860 1114 1115 1116 1117 1178 1179 1180 1181 1317 1370 1371 1372 1373 143...
8: 16 0 Coal Ore 567x: 13 14 15 16 17 18 19 20 21 22 23 24 77 81 82 83 84 85 86 87 88 113 145 146 1...
9: 8 7 Water 67x: 25 144 153 214 281 345 408 473 537 601 663 726 729 793 857 913 920 982 985 1...
10: 8 6 Water 8x: 134 135 217 391 2054 2055 2310 2311
11: 9 5 Stationary Water 4x: 89 1305 1561 1817
12: 9 4 Stationary Water 8x: 114 172 173 174 178 429 430 3558
13: 9 7 Stationary Water 2x: 3557 3568
14: 9 6 Stationary Water 52x: 904 905 1026 1027 1028 1029 1030 1159 1160 1161 1162 1224 1225 1281 1282 128...
15: -1 15 (unknown block) 0x:
palette only knows 3397 of the 4096 blocks
12 unusual palette usages:
16 (key): 583x at 52 53 54 55 56 57 58 59 60 61 62 63 78 79 80 111 112 116 117 118 121 127 180 181 184 186 187 188 ...
17 (key): 13x at 119 124 126 182 183 246 380 381 502 2393 2649 2905 3124
18 (key): 5x at 125 373 436 629 885
19 (key): 23x at 894 895 1149 1150 1151 1213 1214 1215 1405 1406 1407 1469 1470 1471 1661 1662 1663 1725 1726 1727...
20 (key): 4x at 2029 2030 2285 3570
21 (key): 1x at 3566
22 (key): 1x at 3577
23 (key): 1x at 3575
24 (key): 21x at 142 233 398 399 400 489 915 1039 1108 1172 1296 1364 1428 1491 1553 1619 1684 1747 1809 1939 2063
25 (key): 35x at 143 914 1037 1038 1106 1107 1169 1170 1171 1293 1294 1295 1361 1362 1363 1425 1426 1427 1490 1549...
26 (key): 1x at 3523
28 (key): 11x at 1127 1329 1330 1383 1447 1585 1586 1639 1703 1841 1842
===== chunks-1.9.1-pre2/chunk_-24_-5_1457663362582.dump section 0 at byte 0 =====
Unusual blocks found:
5 block_bits, 32 palette entries
Palette:
0: 0 0 Air 221x: 51 115 179 243 308 311 371 375 379 437 443 499 501 503 509 563 570 574 628 6...
1: 7 0 Bedrock 661x: 26 27 28 29 30 31 32 33 34 35 36 37 90 91 92 93 94 95 96 97 98 99 100 101 15...
2: 1 0 Stone 616x: 0 1 2 3 4 5 6 7 8 9 10 11 12 64 65 66 67 68 69 70 71 72 73 74 75 76 128 129 ...
3: 13 0 Gravel 41x: 348 385 579 602 605 607 640 863 896 1659 1660 1754 1755 1756 1757 1758 1759 ...
4: 1 3 Diorite 650x: 38 39 40 41 42 43 44 45 46 47 48 49 50 102 103 104 105 106 107 108 109 110 1...
5: 56 0 Diamond Ore 47x: 491 685 747 755 758 1001 1003 1011 1012 1013 1267 1268 1269 1868 2124 2730 2...
6: 73 0 Redstone Ore 23x: 588 647 735 947 991 1056 1057 1144 1203 1312 1313 1461 1569 1891 2146 2147 2...
7: 15 0 Iron Ore 10x: 368 619 624 652 682 878 939 2679 2680 4031
8: 10 0 Lava 678x: 13 14 15 16 17 18 19 20 21 22 23 24 77 78 79 80 81 82 83 84 85 86 87 88 141 ...
9: 16 0 Coal Ore 88x: 25 89 153 217 281 345 463 492 601 729 748 846 1047 1194 1306 1451 1562 1626 ...
10: 1 5 Andesite 50x: 454 472 483 738 985 994 995 996 1177 1241 1361 1617 1872 1873 1874 2128 2129...
11: 49 0 Obsidian 10x: 982 1165 1240 2592 2593 2596 2597 2711 3920 3921
12: 9 9 Stationary Water 29x: 361 409 836 837 873 1767 1768 1859 1860 2023 2024 2025 2086 2115 2184 2279 2...
13: 1 1 Granite 11x: 1185 1187 1188 1189 1443 1444 1445 2816 3613 3869 3870
14: 14 0 Gold Ore 3x: 780 2116 2372
15: 8 1 Water 4x: 630 691 886 887
16: 9 1 Stationary Water 627x: 52 53 54 55 56 57 58 59 60 61 62 63 116 117 118 119 120 121 122 123 124 125 ...
17: 8 4 Water 72x: 309 380 473 508 537 569 571 638 699 702 762 767 789 793 821 824 829 891 893 ...
18: 8 2 Water 86x: 442 665 857 921 1049 1113 1303 1351 1352 1360 1369 1433 1497 1558 1607 1608 ...
19: 9 2 Stationary Water 11x: 726 1176 1753 2009 2265 2521 2710 2968 3919 3990 4091
20: 8 3 Water 24x: 687 704 705 942 960 1198 1455 1598 1919 2110 2111 2175 2367 2431 2667 2685 2...
21: 8 9 Water 17x: 500 502 686 757 941 1195 1196 1197 1452 1453 1454 2666 2818 2922 2923 2924 3879
22: 9 3 Stationary Water 21x: 695 948 949 952 1145 1146 1204 1205 1206 1208 2560 2561 2562 2563 2564 2676 ...
23: 8 6 Water 6x: 662 663 664 919 920 2819
24: 9 4 Stationary Water 35x: 342 369 597 598 620 625 684 745 853 854 880 1046 1394 1650 1746 1747 2001 20...
25: 8 5 Water 2x: 790 940
26: 9 5 Stationary Water 21x: 644 900 901 1153 1154 1156 1216 1409 1410 1411 2573 2583 2584 2585 2820 2852...
27: 9 6 Stationary Water 3x: 683 879 2853
28: 8 7 Water 12x: 384 387 580 643 898 2433 2434 2689 2690 2839 4009 4040
29: 9 7 Stationary Water 4x: 386 641 642 897
30: 3 0 Dirt 7x: 612 613 672 673 929 932 2841
31: -1 15 (unknown block) 6x: 374 378 436 629 634 885
1 unusual palette usages:
-1 (val): 6x at 374 378 436 629 634 885
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment