Compression using RDP API

rdp_pack.cpp

/**
    Compression using undocumented API in rdpbase.dll

    RDPCompress supports four algorithms : MPPC-8K, MPPC-64K, NCRUSH and XCRUSH.

    This code only supports compression and decompression using MPPC algorithms because these were the easiest to figure out.
    The MPPC compression ratio is very similar to LZSS, so this could be quite useful for shellcode trying to evade detection.
    
    I was unable to get decompression working for NCRUSH and XCRUSH, which is why I'm publishing this.
    Maybe someone will figure out how to provide the correct input for these.
    
    Internally, NCRUSH expects the length of uncompressed or compressed data in the buffer. 
    If you discover correct input for NCRUSH or XCRUSH, please let me know.
    
    The parameters to decompress data may be wrong. It appears to work fine for MPPC, but perhaps it won't work correctly
    for the others. 
    
    cl /EHsc rdp_pack.cpp
    compress: rdp_pack e infile outfile.8k
    decompress: rdp_pack d infile.8k outfile
*/
        
#include <windows.h>

#include <cstdio>
#include <cstdint>
#include <cstdlib>
#include <cstring>

#include <string>
#include <vector>
#include <iostream>
#include <fstream>
#include <sstream>
#include <iomanip>

#pragma pack(push, 1)
typedef struct {
    uint8_t header;   // An 8-bit, unsigned integer specifying the compression type and flags
    uint8_t data[1];  // A variable-length array of bytes containing data encoded using RDP 8.0 Bulk Compression techniques
} RDP8_BULK_ENCODED_DATA;
#pragma pack(pop)

typedef struct {
    uint8_t descriptor; // SINGLE (0xE0) or MULTIPART (0xE1)

    // Optional fields
    uint16_t segmentCount; // Only present if descriptor is MULTIPART
    uint32_t uncompressedSize; // Only present if descriptor is MULTIPART
    RDP8_BULK_ENCODED_DATA bulkData; // Only present if descriptor is SINGLE
    //RDP_DATA_SEGMENT *segmentArray; // Only present if descriptor is MULTIPART
} RDP_SEGMENTED_DATA_MP;

#pragma pack(push, 1)
typedef struct {
    uint32_t               uncompressedSize; // Only present if descriptor is MULTIPART
    RDP8_BULK_ENCODED_DATA bulkData; // Only present if descriptor is SINGLE
} RDP_SEGMENTED_DATA;
#pragma pack(pop)

// type of compression
#define PACKET_COMPR_TYPE_8K    0 // MPPC
#define PACKET_COMPR_TYPE_64K   1 // MPPC
#define PACKET_COMPR_TYPE_RDP6  2 // NCRUSH
#define PACKET_COMPR_TYPE_RDP61 3 // XCRUSH
#define PACKET_COMPR_TYPE_RDP8  4 // MSDN references this, but I doubt it's actually supported by OS

#define PACKET_ENCRYPTED  0x10
#define PACKET_COMPRESSED 0x20
#define PACKET_AT_FRONT   0x40
#define PACKET_FLUSHED    0x80

// ******************************************************
// Needed to allocate memory for compression context.
typedef
DWORD (WINAPI *_RDPCompress_GetContextSize)(DWORD ComprType);

// Initialise compression context.
typedef
void (WINAPI *_RDPCompress_InitSendContext)(void *ctx, SIZE_T ctx_len, DWORD ComprType);

// Compress inbuf and store in outbuf
typedef
DWORD (WINAPI *_RDPCompress)(DWORD ComprType, void *inbuf, void *outbuf, PDWORD outlen, void *ctx);

// ******************************************************
typedef
DWORD (WINAPI *_RDPDeCompress_GetContextSize)(DWORD ComprType);

typedef
void (WINAPI *_RDPCompress_InitRecvContext)(void *ctx, SIZE_T ctx_len, DWORD ComprType, void *workspace);

typedef
DWORD (WINAPI *_RDPDecompress)(
    PVOID  inbuf, 
    DWORD  inlen, 
    DWORD  start, 
    PVOID* outbuf, 
    PDWORD outlen, 
    PVOID  RecvContext, 
    DWORD  ComprType
    );

typedef struct _RDP_pack_ctx {
    DWORD                         type;  // compression algorithm
    
    _RDPCompress_GetContextSize   RDPCompress_GetContextSize;
    _RDPCompress_InitSendContext  RDPCompress_InitSendContext;
    _RDPCompress                  RDPCompress;
    
    _RDPDeCompress_GetContextSize RDPDeCompress_GetContextSize;
    _RDPCompress_InitRecvContext  RDPCompress_InitRecvContext;
    _RDPDecompress                RDPDecompress;
    
    std::vector<BYTE> inbuf, outbuf;
} RDP_pack_ctx;

#define RDP_COMPR_BLK_LEN 4096

#pragma pack(push,1)
typedef struct _RDP_pack_blk {
    DWORD len;
    BYTE data[RDP_COMPR_BLK_LEN];
} RDP_pack_blk;
#pragma pack(pop)

bool
init_pack_ctx(RDP_pack_ctx *c) {
    HMODULE rdpbase = LoadLibrary("rdpbase.dll");
    
    if (!rdpbase) return false;
    
    c->RDPCompress_GetContextSize = (_RDPCompress_GetContextSize)GetProcAddress(rdpbase, "RDPCompress_GetContextSize"); 
    c->RDPCompress_InitSendContext = (_RDPCompress_InitSendContext)GetProcAddress(rdpbase, "RDPCompress_InitSendContext");
    c->RDPCompress = (_RDPCompress)GetProcAddress(rdpbase, "RDPCompress");

    c->RDPDeCompress_GetContextSize = (_RDPDeCompress_GetContextSize)GetProcAddress(rdpbase, "RDPDeCompress_GetContextSize");
    c->RDPCompress_InitRecvContext = (_RDPCompress_InitRecvContext)GetProcAddress(rdpbase, "RDPCompress_InitRecvContext");
    c->RDPDecompress = (_RDPDecompress)GetProcAddress(rdpbase, "RDPDecompress");
    
    return c->RDPCompress_GetContextSize != NULL &&
           c->RDPCompress_InitSendContext != NULL &&
           c->RDPCompress != NULL &&
           c->RDPDeCompress_GetContextSize != NULL &&
           c->RDPCompress_InitRecvContext != NULL &&
           c->RDPDecompress != NULL;
}

std::vector<BYTE>
ReadFileData(std::string path) {
    std::ifstream instream(path, std::ios::in | std::ios::binary);
    std::vector<BYTE> data((std::istreambuf_iterator<char>(instream)), std::istreambuf_iterator<char>());
    return data;
}

bool
WriteFileData(std::string path, std::vector<BYTE> data) {
    std::ofstream outstream(path, std::ios::out | std::ios::binary);
    if (!outstream) return false;
    std::copy(data.begin(), data.end(), std::ostreambuf_iterator<char>(outstream));
    return outstream.good();
}

//
// Use RDP algorithms to compress file. 
//
bool
rdp_encode(RDP_pack_ctx *c) {
    void *SendCtx = NULL;
    bool result = false;
    
    do {
        DWORD ctx_len = c->RDPCompress_GetContextSize(c->type);
        if (!ctx_len) break;
        
        SendCtx = malloc(ctx_len);
        if (!SendCtx) break;
    
        c->RDPCompress_InitSendContext(SendCtx, ctx_len, c->type);
        
        PBYTE inbuf = (PBYTE)c->inbuf.data();
        DWORD inlen = c->inbuf.size();
        
        while (inlen) {
            RDP_pack_blk in={0}, out={0};
            
            in.len = inlen > RDP_COMPR_BLK_LEN ? RDP_COMPR_BLK_LEN : inlen;
            out.len = in.len;
            memcpy(in.data, inbuf, in.len);
            
            
            c->RDPCompress(c->type, in.data, out.data, &out.len, SendCtx);
            
            if (!out.len) {
                printf("RDPCompress failed.\n");
                break;
            }
            
            // no compression?
            if (out.len == RDP_COMPR_BLK_LEN) 
            {
                // copy uncompressed
                memcpy(&out, &in, sizeof(RDP_pack_blk));
            }
            
            c->outbuf.insert(c->outbuf.end(), (PBYTE)&out, (PBYTE)&out + out.len + sizeof(DWORD));
            
            inlen -= in.len;
            inbuf += in.len;
        }
        result = true;
    } while (false);
    
    if (SendCtx) free(SendCtx);
    return result;
}

//
//
//
bool
rdp_decode(RDP_pack_ctx *c) {
    void *RecvContext = NULL;
    bool result = false;
    
    do {
        DWORD ctx_len = c->RDPDeCompress_GetContextSize(c->type);
        RecvContext = malloc(ctx_len);
        if (!RecvContext) break;
        
        c->RDPCompress_InitRecvContext(RecvContext, ctx_len, c->type, NULL);
        
        PBYTE inbuf = (PBYTE)c->inbuf.data();
        DWORD inlen = c->inbuf.size();
        DWORD start = 0;
        
        while (inlen) {
            RDP_pack_blk *in = (RDP_pack_blk*)inbuf;
            
            PBYTE outbuf = NULL;
            DWORD outlen = 0;
            
            if (in->len == RDP_COMPR_BLK_LEN) {
                outbuf = in->data;
                outlen = RDP_COMPR_BLK_LEN;
            } else {
                c->RDPDecompress(in->data, in->len, start, (PVOID*)&outbuf, &outlen, RecvContext, c->type);
                
                if (!outbuf) {
                    printf("RDPDecompress() failed.\n");
                    break;
                }
            }
            
            c->outbuf.insert(c->outbuf.end(), outbuf, outbuf + outlen);
            start = 1;
            
            inlen -= in->len + sizeof(DWORD);
            inbuf += in->len + sizeof(DWORD);
        }
    } while (false);
    
    if (RecvContext) free(RecvContext);
    return result;
}

int
main(int argc, char *argv[]) {
    if ((argc != 4) || ((argv[1][0] != 'e') && (argv[1][0] != 'd'))) {
        printf("usage: rdp_pack [e/d] <infile> <outfile>\n");
        return 0;
    }
    
    bool encode = (argv[1][0] == 'e');
    const char* infile = argv[2];
    const char* outfile = argv[3];
    
    RDP_pack_ctx c;
    
    if (!init_pack_ctx(&c)) {
        printf("Unable to initialise RDP API.\n");
        return 0;
    }
    
    // set compression and input data
    c.type  = PACKET_COMPR_TYPE_8K;
    c.inbuf = ReadFileData(infile);
    
    if (encode) {
        rdp_encode(&c);
    } else {
        rdp_decode(&c);
    }
    
    // save output to file
    WriteFileData(outfile, c.outbuf);
    printf("OK\n");
    return 0;
}