rclone.js

const ApolloClient = require('apollo-boost').default;
const gql = require('graphql-tag');
const util = require('util');
const exec = util.promisify(require('child_process').exec);

require('cross-fetch/polyfill');
require('events').EventEmitter.defaultMaxListeners = 100;

function ql_mutation(client, m, variables) {
  return client.mutate({
    mutation: gql`${m}`,
    variables 
  })
}

function ql_request(client, q, variables) {
  return client.query({
    query: gql`${q}`,
    variables
  })
}

async function createConfig(config) {
  const configArr = config.split('\n');
  let configCreate = "";
  for (configLine of configArr){
    configCreate = `echo '${configLine}' >> /tmp/rclone.conf`;
    const { stdout, stderr } = await exec(configCreate);
  }
}


const rcloneCopy = async (fileData, client, oriRemote, userId) => {

  fileData = fileData.filter(function (el) {
    return el != '';
  });

  let result = {};
  const response = await Promise.all(fileData.map( async function (file) {
    const filePath = file.substring(10);
    const filePathArr = filePath.split('/');
    const fileName = filePathArr.pop();
    const variables = { fileName };

    const fileTypes = ['txt', 'jpeg', 'png', 'pdf', 'docx', 'pptx', 'ppt', 'doc'];
    let fileType = fileName.split('.').pop();
    fileType = fileTypes.includes(fileType) ? fileType : null;
    let capType = null;
    let exType = null;
    if(fileType){
      capType = fileType.toUpperCase();
      exType = `.${fileType}`;
    }

    const mutation = `
      mutation uploadDocument($fileName: String!) {
        CreateDocument(
          fileName: $fileName
          fileType: ${capType}
          fileExtension: "${exType}"
          fileSize: 1024
          data: "linkToS3"
        ) {
          id
        }
      }
    `;

    const { data: { CreateDocument: { id: fileId } } } = await ql_mutation(client, mutation, variables);
    // console.log('first mutation');
    // console.log(fileId);

    const fileIdName = fileId + exType;
    const rcloneCopyCmd = `./rclone/rclone copyto '${oriRemote}${filePath}' s3-remote:tagai-alpha-store/${fileIdName} --config='/tmp/rclone.conf'`
    // console.log(rcloneCopyCmd);


    const mutation2 = `
      mutation addDocumentToUser($userId: ID!, $fileId: ID!) {
        AddUserDocuments(
          from: {
            id: $userId
          },
          to: {
            id: $fileId
          },
          data: {
            privilege: UPLOADER
          }
        ) {
          privilege
        }
      }
    `;

    const output = await exec(rcloneCopyCmd);
    const variables2 = { userId, fileId }
    const { data: { AddUserDocuments }, errors } = await ql_mutation(client, mutation2, variables2);
    // console.log('AddUserDocuments');
    // console.log(AddUserDocuments);

    const query = `
      query documentLookup($documentId: ID!) {
        Document(id: $documentId) {
          fileName
          fileExtension
          suggestedTags
        }
      }
    `;

    const variables3 = { 'documentId': fileId }
    const { data: { Document: documents } } = await ql_request(client, query, variables3);
    // console.log('documents');
    // console.log(documents);

    const { suggestedTags } = documents[0];
    const tags = suggestedTags.concat(filePathArr);
    const url = 'https://tagai-alpha-store.s3.amazonaws.com/' + fileId + exType;
    result[url] = {tags, 'name': fileName, fileId};
    return Promise.resolve(result);
  }));
  return response[0];
};

const rclone = async (authToken, userId, oriRemoteConfig, desRemoteConfig) => {

  await createConfig(oriRemoteConfig);
  await createConfig(desRemoteConfig);

  const client = new ApolloClient({
    uri: 'https://XXX.execute-api.us-east-1.amazonaws.com/dev/graphql',
    headers: { Authorization: `Bearer ${authToken}`}
  });

  const oriRemote = "cloud-remote:"
  const rcloneListCmd = "./rclone/rclone ls " + oriRemote + " --config='/tmp/rclone.conf'";

  const { stdout, stderr } = await exec(rcloneListCmd);
  const fileData = stdout.split('\n');
  const res = await rcloneCopy(fileData, client, oriRemote, userId);
  return res;
};

exports.handler = async (event) => {
  const userId = event.queryStringParameters.userId;
  const authToken = event.queryStringParameters.token;
  const oriRemoteConfig = event.queryStringParameters.config;
  const desRemoteConfig = '\n[s3-remote]\ntype = s3\nprovider = AWS\nenv_auth = false\naccess_key_id = XXX\nsecret_access_key = XXX\nregion = us-east-1\nacl = private\nbucket_acl = private';
  const result = await rclone(authToken, userId, oriRemoteConfig, desRemoteConfig);
  const res = {
    statusCode: 200,
    headers: {
      'Access-Control-Allow-Origin' : '*',
      'Content-Type': 'application/json'
    },
    body: JSON.stringify(result),
  };
  const { stdout, stderr } = await exec("rm /tmp/rclone.conf");
  return res;
};


const {
  "queryStringParameters": {
    "config": "[cloud-remote]\ntype = drive\nclient_id = \nclient_secret = \nscope = drive\ntoken = {\"access_token\":\"XXX\",\"token_type\":\"Bearer\",\"expiry\":\"0001-01-01T00:00:00Z\"}",
    "userId": "5d532213114ecb0d493b991b",
    "token": "XXX"
  }
}