CopadoADDDataLoader.cls

public class CopadoADDDataLoader {
    
    public static List<List<SObject>> allNewRecords;
    
    public static void loadData(){
        //prepare data for github call
        String githubToken = '6121aa9eb9deb0c20a920d8fee7f238e6a5e7894';
        String githubEndpoint = 'https://api.github.com/repos/CopadoSolutions/PartnerTraining';
        String sha = '';
        Http http = new Http();
        
        //get master branch sha
        HttpRequest shaRequest = new HttpRequest();
        shaRequest.setEndpoint(githubEndpoint + '/git/ref/heads/master');
        shaRequest.setHeader('Content-Type', 'application/json');
        shaRequest.setHeader('Authorization', 'Bearer ' + githubToken);
        shaRequest.setMethod('GET');
        HttpResponse shaResponse = http.send(shaRequest);
        System.debug('We have a response with status: ' + shaResponse.getStatus());
        
        if(shaResponse.getStatus() != 'OK') return;
        
        System.debug('Starting to parse the response and get the sha of master branch');
        Map<String, Object> shaResultMap = ( Map<String, Object>) JSON.deserializeUntyped(shaResponse.getBody());
        
        if(shaResultMap.get('object') != null){
            String objectJson = JSON.serialize(shaResultMap.get('object'));
            Map<String, Object> objectMap = (Map<String, Object>) JSON.deserializeUntyped(objectJson);
            sha = objectMap.get('sha') != null ? String.valueOf(objectMap.get('sha')) : sha;
        }
        
        System.debug('sha of master branch: ' + sha);
        
        //get all files
        System.debug('Preparing to get all file names from CopadoTrails repo...');
        HttpRequest fetchFilesRequest = new HttpRequest();
        fetchFilesRequest.setEndpoint(githubEndpoint + '/git/trees/' + sha);
        fetchFilesRequest.setHeader('Content-Type', 'application/json');
        fetchFilesRequest.setHeader('Authorization', 'Bearer ' + githubToken);
        fetchFilesRequest.setMethod('GET');
        HttpResponse filesResponse = http.send(fetchFilesRequest);
        System.debug('We have a response with status: ' + filesResponse.getStatus());
        
        if(filesResponse.getStatus() != 'OK') return;
        
        //get files starts with ADD_
        System.debug('Starting to parse the response and get all the file names that starts with ADD_');
        List<String> addFiles = new List<String>();
        Map<String, Object> resultMap = ( Map<String, Object>) JSON.deserializeUntyped(filesResponse.getBody());
        
        if(resultMap.get('tree') != null){
            List<Object> treeList = (List<Object>) resultMap.get('tree');
            for(Object node : treeList){
                String nodeJSON = JSON.serialize(node);
                Map<String, Object> nodeObj = (Map<String, Object>) JSON.deserializeUntyped(nodeJSON);
                String pathValue = (String) nodeObj.get('path');
                if(pathValue != null && pathValue.startsWith('ADD_')){
                    addFiles.add(pathValue);
                }
            } 
            
        }
        System.debug('We have all file names we need to import. Total template count: ' + addFiles.size());
        
        //get file content
        //NOTE: if there are more than 100 files this script will give limit exception
        allNewRecords = new List<List<SObject>>();
        for(String filePath : addFiles){
            System.debug('Getting records for the template... ' + filePath);
            HttpRequest fetchFileRequest = new HttpRequest();
            fetchFileRequest.setEndpoint(githubEndpoint + '/contents/' + filePath);
            fetchFileRequest.setHeader('Content-Type', 'application/json');
            fetchFileRequest.setHeader('Authorization', 'Bearer ' + githubToken);
            fetchFileRequest.setMethod('GET');
            HttpResponse fileResponse = http.send(fetchFileRequest);
            System.debug('We have a response with status: ' + fileResponse.getStatus());
            
            if(fileResponse.getStatus() != 'OK') return;
            
            CopadoADDDataLoader.parseAndInsertData(filePath, fileResponse.getBody());
            System.debug('Added records to the list for insert operation...'); 
        }
        for(List<SObject> records : allNewRecords){
            System.debug('Inserting total number of records ' + records.size());
            insert records;
        }
        System.debug('Data import completed...');
        
        
    }
    
    public static void parseAndInsertData(String filePath, String fileBody){
        //parse file content
        System.debug('Parsing file content...');
        Map<String, Object> jsonResponse = (Map<String, Object>) JSON.deserializeUntyped(fileBody);
        System.debug(EncodingUtil.base64Decode((String)jsonResponse.get('content')).toString());
        String fileContent = EncodingUtil.base64Decode((String) jsonResponse.get('content')).toString();
        
        if(String.isBlank(fileContent)) return;
        
        //get the number of rows in the csv
        List<String> rows = fileContent.split('\n');
        System.debug('Getting the rows of template: ' + (rows.size()-1));
        
        if(rows.size() < 2) return;
        List<String> fieldNames = rows[0].split(',');
        System.debug('Total field names to be mapped: ' + fieldNames.size());
        
        //insert data
        String objectName = filePath.substringBetween('_', '.csv').substringAfter('_');
        System.debug('Preparing SObjects for object name: ' + objectName);
        List<SObject> newRecords = new List<SObject>();
        for(Integer i = 1 ; i < rows.size(); i++){
            List<String> values = rows[i].split(',');
            SObject newRecord =  Schema.getGlobalDescribe().get(objectName).newSObject();
            Schema.SObjectType objectType = Schema.getGlobalDescribe().get(objectName);
            Schema.DescribeSObjectResult objectDescribe = objectType.getDescribe() ;
            Map<String,Schema.SObjectField> fields = objectDescribe.fields.getMap();
            Integer j = 0;
            
            for(String value : values){
                value = value.trim().removeEnd('\n');
                fieldNames[j] = fieldNames[j].trim().removeEnd('\n');
                System.debug('field name: ' + fieldNames[j] + ' value ' + value);
                if(fieldNames[j].contains('.')){
                    System.debug('Reference field found with external id');
                    String referenceObjectName = fieldNames[j].split('\\.')[0];
                    Sobject relationObj = Schema.getGlobalDescribe().get(referenceObjectName).newSObject();
                    relationObj.put(fieldNames[j].split('\\.')[1], String.valueOf(value));
                    newRecord.putsObject(referenceObjectName, relationObj);
                } else {
                    String fieldTypeName = fields.get(fieldNames[j]).getDescribe().getType().name().toLowerCase(); 
                    Object valueObject = (Object) value;
                    if (fieldTypeName == 'textarea' || fieldTypeName == 'picklist' || fieldTypeName == 'reference' || fieldTypeName == 'string') newRecord.put(fieldNames[j], String.valueOf(value));
                    if(fieldTypeName == 'boolean') newRecord.put(fieldNames[j], Boolean.valueOf(value));
                    if(fieldTypeName == 'blob') newRecord.put(fieldNames[j], (Blob) valueObject);
                    if(fieldTypeName == 'date') newRecord.put(fieldNames[j], Date.valueOf(value));
                    if(fieldTypeName == 'datetime') newRecord.put(fieldNames[j], Datetime.valueOf(value));
                    if(fieldTypeName == 'time') newRecord.put(fieldNames[j], (Time) valueObject);
                    if(fieldTypeName == 'integer') newRecord.put(fieldNames[j], Integer.valueOf(value));
                    if(fieldTypeName == 'long') newRecord.put(fieldNames[j], (Long) valueObject);
                    if(fieldTypeName == 'decimal') newRecord.put(fieldNames[j], (Decimal) valueObject);
                    if(fieldTypeName == 'double' || fieldTypeName == 'currency') newRecord.put(fieldNames[j], Double.valueOf(value));
                }
                j+=1;
                if(j == fieldNames.size()) break;
            }
            newRecords.add(newRecord);
            j = 0;
            
        }
        allNewRecords.add(newRecords);
    }
    
}