my_decrypt_crack.c

#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include <unistd.h>

struct __attribute__((__packed__)) header_struct {
    uint32_t MAGIC_NUMBER;
    uint32_t file_length;
    char original_filename[256];
    char encrypted_password[32];   
};

int decrypt_queue[4] = {0, 0, 0x9A596725, 0x592C3156};
uint32_t param_hard_1 = 0x2137;
uint32_t param_hard_2 = 0x7a69;
uint32_t param_hard_3 = 0x1234567;


void push_data_to_buffer(uint32_t param_1){  
  uint32_t retval = param_1 ^ decrypt_queue[0];
  decrypt_queue[0] = decrypt_queue[1];
  decrypt_queue[1] = decrypt_queue[2];
  decrypt_queue[2] = decrypt_queue[3];
  decrypt_queue[3] = retval >> 6 | retval << 0x1a;
  return;
}

uint32_t decrypt_easy(uint32_t arg) {
    uint32_t retval = arg ^ decrypt_queue[0];
    uint32_t tmp = decrypt_queue[2] * 0x2137 + retval;
    decrypt_queue[0] = decrypt_queue[1];
    decrypt_queue[1] = decrypt_queue[2];
    decrypt_queue[2] = decrypt_queue[3];
    decrypt_queue[3] = tmp >> 6 | tmp << 0x1a;
    return retval;
}

uint32_t decrypt_hard(uint32_t arg) {
    uint32_t retval = arg ^ decrypt_queue[0];
    uint32_t tmp = decrypt_queue[2] * param_hard_1 + retval;
    decrypt_queue[0] = decrypt_queue[2] * param_hard_2 + decrypt_queue[1];
    decrypt_queue[1] = decrypt_queue[2];
    decrypt_queue[2] = decrypt_queue[2] * param_hard_3 + decrypt_queue[3];
    decrypt_queue[3] = tmp >> 6 | tmp << 0x1a;
    return retval;
}


void strange_function(struct header_struct* header) {    
    char* orig_filename = header->original_filename;
    char* enc_password = header->encrypted_password;
    do {
        push_data_to_buffer(*(uint32_t *)orig_filename);
        orig_filename += 4;
    } while (orig_filename != enc_password);
    
    for(int i = 0; i < 3; i++) {
        push_data_to_buffer(0);
    }
}

int main(int argc,char **argv){
    int retval = 0;
    
    if (argc < 1) {
        printf("usage: %s <input filename>\n", *argv);
        return 0;
    }
    
    FILE *encrypted_file = fopen(argv[1], "rb");
    if (encrypted_file == NULL) {
        perror("fopen");
        retval = 1;
        goto cleanup_1;
    }
    
    struct header_struct file_header;
    memset(&file_header, 0, sizeof(file_header));
    size_t read_chars = fread(&file_header, 0x128, 1, encrypted_file);
    if(read_chars != 1) {
        perror("fread");
        retval = 1;
        goto cleanup_1;
    }
    
    if ((file_header.MAGIC_NUMBER != 0xb542020) && (file_header.MAGIC_NUMBER != 0x20200b54)) {
        fprintf(stderr, "unrecognized file\n");
        retval = 1;
        goto cleanup_1;
    }
    
    printf("Decrypting file %s, resulting filename: %s. Using %s encryption.\n", argv[1], file_header.original_filename, file_header.MAGIC_NUMBER == 0xb542020 ? "easy" : "hard");
    
    strange_function(&file_header);
    
    int unhexlified_pass_buffer[8] = {0};
    int password_processed[8] = {0};
    
    for(int i = 0; i < 8; i++) {
        // Password cracking - originally there was a sscanf that unhexlified entered password
        unhexlified_pass_buffer[i] = ((int*)file_header.encrypted_password)[i] ^ decrypt_queue[0];
        password_processed[i] = decrypt_easy(unhexlified_pass_buffer[i]);
    }
    
    param_hard_1 = unhexlified_pass_buffer[2];
    param_hard_2 = unhexlified_pass_buffer[3];
    param_hard_3 = unhexlified_pass_buffer[5];
    
    int identical = memcmp(file_header.encrypted_password,password_processed,0x20);
    if(identical != 0) {
        fprintf(stderr, "Calculated key is invalid - something is terribly wrong\n");
        retval = 1;
        goto cleanup_1;
    }
    
    fprintf(stderr, "OK, decrypting...\n");
    
    FILE *decrypted_file = fopen(file_header.original_filename,"wb");
    if (decrypted_file == NULL) {
        perror("open");
        retval = 1;
        goto cleanup_1;
    }
    
     if (file_header.file_length == 0) {
         retval = 0;
         goto cleanup_2;
     }
    
    uint32_t current_data;
    for(int i = 0; i < file_header.file_length; i += 4) {
        read_chars = fread(&current_data,4,1,encrypted_file);
        if (read_chars != 1) {
            perror("fread");
            retval = 1;
            goto cleanup_2;
        }
        if (file_header.MAGIC_NUMBER == 0xb542020) {
            current_data = decrypt_easy(current_data);
        }
        else {
            current_data = decrypt_hard(current_data);
        }
        
        int32_t remaining = file_header.file_length - i;
        if (4 < remaining) {
            remaining = 4;
        }
        read_chars = fwrite(&current_data,(long)remaining,1,decrypted_file);
        if (read_chars != 1) {
            retval = 1;
            goto cleanup_2;
        }
    }
    
    cleanup_2:
    fclose(decrypted_file);
    cleanup_1:
    fclose(encrypted_file);
    
    return retval;
}