streaming openpgp s3

Dockerfile

FROM node:12.9.1-buster-slim

RUN mkdir /opt/pgp-aws

WORKDIR /opt/pgp-aws/
RUN apt-get update && apt-get install -y git gnupg2 curl nano wget time
 
COPY . /opt/pgp-aws/

RUN npm cache clean -f
RUN rm -rf node_modules && ls -la && npm install

CMD ["time", "node", "--max-old-space-size=4096", "streamer.js"]

package.json

{
  "name": "dockerpgp",
  "version": "1.0.0",
  "description": "",
  "main": "streamer.js",
  "scripts": {
    "d25gb": ""
  },
  "author": "",
  "license": "ISC",
  "devDependencies": {
    "buffer": "5.6.0"
  },
  "dependencies": {
    "aws-sdk": "^2.673.0",
    "buffer": "5.6.0",
    "faker": "^4.1.0",
    "gpg": "^0.6.0",
    "openpgp": "^4.10.4",
    "stream-tester": "0.0.5"
  }
}

streamer.js

const openpgp = require('openpgp');
const fs = require('fs');
const AWS = require('aws-sdk');
const stream = require('stream');
const faker = require('faker');


AWS.config.update({
    secretsmanager: {
        apiVersion: '2017-10-17',
    },
    region: 'us-east-2',
    s3: {
        apiVersion: '2006-03-01'
    }
});


openpgp.config.aead_protect = true;
openpgp.config.allow_unauthenticated_stream = true;
//openpgp.config.integrity_protect = false
//openpgp.config.ignore_mdc_error = true

const secretName = process.env.SECRET,
    Action = process.env.ACTION,
    Bucket = process.env.BUCKET,
    Client = process.env.CLIENT,
    DocName = process.env.DOCNAME,
    Region = process.env.REGION,
    GenerateNum = process.env.GENUM 

const PublicKeyPath = `public/${Client}`,
    PrivateKeyPath = `private/${Client}`,
    ClientPath = `clients/${Client}`,
    EncryptedDoc = `${ClientPath}/${DocName}`,
    DecryptedDoc = `${ClientPath}/decrypted/${DocName}`;

const s3 = new AWS.S3();

const PassPhrase = async () => {
    const secrets = new AWS.SecretsManager({ region: Region });
    return await secrets.getSecretValue({
        SecretId: secretName,
    }).promise();
};

const GetS3Object = async (path) => {
    return await s3.getObject({
        Bucket,
        Key: path
    }).promise();
};

const OpenKey = async () => {
    const privateKeyArray = new Uint8Array((await GetS3Object(PrivateKeyPath)).Body);
    const privateKey = await openpgp.key.read(privateKeyArray);
    const passwd = JSON.parse((await PassPhrase()).SecretString)[Client];

    const publicKey = await openpgp.key.readArmored((await GetS3Object(PublicKeyPath)).Body);

    await privateKey.keys[0].decrypt(passwd);
    return {
        privateKey,
        privateKeyArray,
        publicKey
    }
}

function UploadFromStream(Key) {
    var pass = new stream.PassThrough();
    var options = {partSize: 100 * 1024 * 1024, queueSize: 4};
    var params = { Bucket, Key, Body: pass };
    const uploader = s3.upload(params, options, function (err, data) {
        if (err) { console.log(err) }
        else { console.log("Task complete:", data) }
    });
    uploader.on('httpUploadProgress', (progress) => {
        console.log(`${progress.loaded} - ${progress.total} - ${progress.part} - ${progress.key}`);
    });
    return pass;
}


const DecryptMessage = async () => {
    const keys = await OpenKey();
    const reader = s3.getObject({ Bucket, Key: EncryptedDoc }).createReadStream();
    const decrypted = await openpgp.decrypt({
        message: (await openpgp.message.readArmored(reader, 'node')),
        privateKeys: keys.privateKey.keys,
    });
    const plainText = decrypted.data;
    await plainText.pipe(UploadFromStream(DecryptedDoc));
}

const EncryptMessage = async () => {
    const keys = await OpenKey();
    const reader = s3.getObject({ Bucket, Key: `tests/${DocName}` }).createReadStream();
    const decrypted = await openpgp.encrypt({
        message: (await openpgp.message.fromText(reader, 'node')),
        publicKeys: keys.publicKey.keys,
    });
    const cipherText = decrypted.data;
    await cipherText.pipe(UploadFromStream(EncryptedDoc));
}

const EncryptLocalFile = async () => {
    console.log('start encrypt local file');
    const keys = await OpenKey();
    var readableStream = fs.createReadStream(`test-files/${DocName}`);
    const encryptedMessage = await openpgp.encrypt({
        message: (openpgp.message.fromText(readableStream)),
        publicKeys: keys.publicKey.keys
    });
    const cipherText = encryptedMessage.data;
    cipherText.pipe(UploadFromStream(EncryptedDoc));
}

const GenerateTestData = async () =>  {
    var tester = require('stream-tester');
    return await tester.createRandomStream(() => {
        const username = faker.internet.userName();
        const avatar = faker.image.avatar();
        const email = faker.internet.email();
        const ua = faker.internet.userAgent();
        const dir = faker.system.directoryPath();
        const data = `${username},${avatar},${email},${ua},${dir}\n`;
        return data
    }, GenerateNum).pipe(UploadFromStream(`tests/${DocName}`));
}

console.log(`Action: ${Action} Source: ${DecryptedDoc} Target: ${EncryptedDoc} DocName: ${DocName}`)
switch (Action) {
    case "encrypt-local":
        EncryptLocalFile();
        break;
    case "decrypt":
        DecryptMessage();
        break;
    case "encrypt":
        EncryptMessage();
        break;
    case "generate":
        GenerateTestData();
        break;
    default:
        break;
}