decrypt

#!/usr/bin/env python
from pwn import *
context(arch='i386', os='linux')


def cons(n):
    return n & 0xffffffff

def rot(n,v):
    # Positive is left
    b = bits(n)
    return unpack(unbits(''.join(map(str, b[v:] + b[:v])).rjust(32, '0')), endianness = 'big')


def decrypt(data, seed):
    out = ""
    for i in range(0, len(data), 4):
        v6 = ((1664525 * seed + 1013904223) >> 16) % 2
        v7 = cons(1664525 * (1664525 * seed + 1013904223) + 1013904223)
        v35 = (v7 >> 16) % 32
        v8 = cons(1664525 * v7 + 1013904223)
        v9 = (v8 >> 16) % 256
        v10 = cons(1664525 * v8 + 1013904223)
        v11 = (v10 >> 16) % 256
        v12 = cons(1664525 * v10 + 1013904223)
        seed = cons(1664525 * v12 + 1013904223)

        v14 = data[i:i+4]

        if v6:
            v13 = rot(v14, -v35)
        else:
            v13 = rot(v14, v35)

        out += chr(((v13 >>  0) & 0xff) ^ ((seed >> 16) % 256))
        out += chr(((v13 >>  8) & 0xff) ^ ((v12 >> 16) % 256))
        out += chr(((v13 >> 16) & 0xff) ^ v11
        out += chr(((v13 >> 24) & 0xff) ^ v9

    return out

data = read('./instr.txt.enc')
seed = 0x691aa561
#seed = 0x61a51a69
write('test', decrypt(data, seed))

unpack.py

#!/usr/bin/env python
from pwn import *
context(arch='i386', os='linux')
#splash()


data = read('./yar_1.dat')

num = u32(data[8:12])
i = 0xc

def decrypt(filename, data):
    res = ""
    for c in data:
        c = ord(c)
        a = ((c << 7) & 0xff) | (c & 0x7e) | ((c >> 7) & 1)
        b = (a ^ (2 * (~(a >> 1) & 0xff)) & 0xff) & 2 ^ a
        r = b ^ ((b ^ (~(b >> 6) << 6) & 0xff) & 0x40)
        res += chr(r)
    write(filename, res)

for f in range(num):
    filename = ""
    for c in data[i:]:
        if c == '\x00':
            break
        filename += c
    start  = u32(data[i+100:i+104])
    length = u32(data[i+104:i+108])
    decrypt(filename, data[start:start+length])
    i += 128