Programmatically import data into Dynamics 365

ImportData.cs

public void Run()
{
var url = "https://[].crm.dynamics.com";
    var userName = "";
    var password = "";

    // Default Azure application
    var conn = $@"
Url = {url};
AuthType = OAuth;
UserName = {userName};
Password = {password};
AppId = 51f81489-12ee-4a9e-aaae-a2591f45987d;;
RedirectUri = app://58145B91-0C36-4500-8554-080854F2AC97;
LoginPrompt=Auto;
RequireNewInstance = True";

    const string folderName = @"C:\downloads\";
    const string fileName = "Properties Import.csv";
    const string entityName = "tld_property";
    const string importMapName = "TLD Property Master"; 
    const bool debug = true;

    Console.WriteLine("Reading the .CSV file...");
    
    var content = ReadCsvFile(Path.Combine(folderName, fileName));

    var crmService = new CrmServiceClient(conn);

    var importId = CreateImportRecord(crmService, importMapName, fileName, content, entityName, out var importFileId, debug);

    
    // *** Method 1 ***
    PerformParse(crmService, importId, importFileId, debug);
    PerformTransform(crmService, importId, importFileId);

    // ReSharper disable once ConditionIsAlwaysTrueOrFalse
    if (!debug)
    {
        PerformImport(crmService, importId, importFileId);
    }
    // ****************
   
    /*
    // *** Method 2 ***
    PerformImportOperation(crmService, importId, importFileId, "ParseImport", "Parsing");

    // ReSharper disable once ConditionIsAlwaysTrueOrFalse
    if (debug)
    {
        DebugParse(crmService, importFileId);
    }

    PerformImportOperation(crmService, importId, importFileId, "TransformImport", "Transforming");

    // ReSharper disable once ConditionIsAlwaysTrueOrFalse
    if (!debug)
    {
        PerformImportOperation(crmService, importId, importFileId, "ImportRecordsImport", "Importing");
    }
    // ****************
    /*

    /*
    // *** Production ***
    ImportRecords(crmService, folderName, "Subdivisions.csv", "tld_subdivision", "TLD Subdivision Master-0725", debug);
    ImportRecords(crmService, folderName, "Owners.csv", Contact.EntityLogicalName, "TLD Owner Master-0808", debug);
    ImportRecords(crmService, folderName, "Do Not Mail Properties.csv", "tld_property", "TLD Do Not Mail Properties-0816", debug);
    ImportRecords(crmService, folderName, "Properties.csv", "tld_property", "TLD Property Master-0807", debug);
    ImportRecords(crmService, folderName, "Letters.csv", Letter.EntityLogicalName, "TLD Offer Letters Master (with ContactID)", debug);
    // ****************
    */

    Console.WriteLine();
    Console.Write("Import complete. Press Enter to continue.");

    Console.ReadLine();
}

private Guid CreateImportRecord(CrmServiceClient crmService, string importMapName, string fileName, string content, string entityName,
    out Guid importFileId, bool debug)
{
    Console.WriteLine("Creating the Import Record...");

    var importMap = GetImportMap(crmService, importMapName);
    var sourceEntityName = GetSourceEntityName(crmService, importMap);

    var import = new Import
    {
        ModeCode = new OptionSetValue((int)ImportModeCode.Create),
        Name = fileName
    };

    var importId = crmService.Create(import);

    var importFile = new ImportFile
    {
        EnableDuplicateDetection = true,
        IsFirstRowHeader = true,
        UseSystemMap = true,
        Name = fileName,
        Source = fileName,
        SourceEntityName = sourceEntityName,   
        TargetEntityName = entityName,
        Content = content,
        Size = content.Length.ToString(),
        ImportMapId = importMap,
        ImportId = new EntityReference(Import.EntityLogicalName, importId),
        FieldDelimiterCode = new OptionSetValue((int)ImportFileFieldDelimiterCode.Comma),
        DataDelimiterCode = new OptionSetValue((int)ImportFileDataDelimiterCode.DoubleQuote),
        ProcessCode = new OptionSetValue((int)ImportFileProcessCode.Process),
        RecordsOwnerId = GetWhoAmI(crmService)
    };

    importFileId = crmService.Create(importFile);

    if (debug)
    {
        DebugCreateImport(crmService, importFileId);
    }

    return importId;
}

private static void PerformParse(CrmServiceClient crmService, Guid importId, Guid importFileId, bool debug)
{
    Console.WriteLine("Parsing the .CSV file...");

    var request = new ParseImportRequest
    {
        ImportId = importId
    };

    var response = (ParseImportResponse)crmService.Execute(request);

    Console.WriteLine("Waiting for Parse async job to complete...");

    WaitForAsyncJobCompletion(crmService, response.AsyncOperationId);

    ReportErrors(crmService, importFileId);

    if (debug)
    {
        DebugParse(crmService, importFileId);
    }
}

private static void PerformTransform(CrmServiceClient crmService, Guid importId, Guid importFileId)
{
    Console.WriteLine("Transforming .CSV file...");

    var request = new TransformImportRequest
    {
        ImportId = importId
    };

    var response = (TransformImportResponse)crmService.Execute(request);

    Console.WriteLine("Waiting for Transform async job to complete...");

    WaitForAsyncJobCompletion(crmService, response.AsyncOperationId);

    ReportErrors(crmService, importFileId);
}

private static void PerformImport(CrmServiceClient crmService, Guid importId, Guid importFileId)
{
    Console.WriteLine("Importing the .CSV file...");

    var request = new ImportRecordsImportRequest
    {
        ImportId = importId
    };

    var response = (ImportRecordsImportResponse)crmService.Execute(request);

    Console.WriteLine("Waiting for ImportRecords async job to complete...");

    WaitForAsyncJobCompletion(crmService, response.AsyncOperationId);

    ReportErrors(crmService, importFileId);
}

private static void PerformImportOperation(CrmServiceClient crmService, Guid importId, Guid importFileId, string requestName, string description)
{
    Console.WriteLine($"{description} the .CSV file...");

    var request = new OrganizationRequest(requestName);

    request.Parameters.Add(new KeyValuePair<string, object>("ImportId", importId));

    var response = crmService.Execute(request);

    var asyncId = (Guid)response.Results["AsyncOperationId"];

    Console.WriteLine($"Waiting for {requestName} async job to complete...");

    WaitForAsyncJobCompletion(crmService, asyncId);

    ReportErrors(crmService, importFileId);
}

private void ImportRecords(CrmServiceClient crmService, string folderName, string fileName, string entityName, string importMapName, bool debug)
{
    Console.WriteLine("Reading the .CSV file...");

    var content = ReadCsvFile(Path.Combine(folderName, fileName));

    var importId = CreateImportRecord(crmService, importMapName, fileName, content, entityName, out var importFileId, debug);

    PerformImportOperation(crmService, importId, importFileId, "ParseImport", "Parsing");

    // ReSharper disable once ConditionIsAlwaysTrueOrFalse
    if (debug)
    {
        DebugParse(crmService, importFileId);
    }

    PerformImportOperation(crmService, importId, importFileId, "TransformImport", "Transforming");

    // ReSharper disable once ConditionIsAlwaysTrueOrFalse
    if (!debug)
    {
        PerformImportOperation(crmService, importId, importFileId, "ImportRecordsImport", "Importing");
    }
}

private static void DebugCreateImport(CrmServiceClient crmService, Guid importFileId)
{
    var headerColumnsRequest = new GetHeaderColumnsImportFileRequest
    {
        ImportFileId = importFileId
    };

    var headerColumnsResponse = (GetHeaderColumnsImportFileResponse)crmService.Execute(headerColumnsRequest);

    Console.WriteLine("\tHere is the Import File Header...");

    for (var columnNumber = 0; columnNumber < headerColumnsResponse.Columns.Length; columnNumber++)
    {
        var headerName = headerColumnsResponse.Columns[columnNumber];

        Console.WriteLine($"\t\tColumn[{columnNumber+1}] = {headerName}");
    }

    Console.WriteLine();
}

private static void DebugParse(CrmServiceClient crmService, Guid importFileId)
{
    // Retrieve data from the parse table.
    // NOTE: You must create the parse table first using the ParseImport message.
    // The parse table is not accessible after ImportRecordsImportResponse is called.
    var request = new RetrieveParsedDataImportFileRequest
    {
        ImportFileId = importFileId,
        PagingInfo = new PagingInfo
        {
            Count = 2, // Specify the number of entity instances returned per page.
            PageNumber = 1, // Specify the number of pages returned from the query.
            PagingCookie = "1" // Specify a total number of entity instances returned.
        }
    };

    var response = (RetrieveParsedDataImportFileResponse)crmService.Execute(request);

    Console.WriteLine("\tHere are the first two parsed records:");

    for (var rowCount = 0; rowCount < response.Values.Length; rowCount++)
    {
        var rows = response.Values[rowCount];

        for (var columnCount = 0; columnCount < rows.Length; columnCount++)
        {
            var value = rows[columnCount];

            Console.WriteLine($"\t\t({rowCount + 1}, {columnCount + 1}) = {value}");
        }

        Console.WriteLine();
    }
}

public static EntityReference GetWhoAmI(CrmServiceClient crmService)
{
    var response = (WhoAmIResponse)crmService.Execute(new WhoAmIRequest());

    return new EntityReference(SystemUser.EntityLogicalName, response.UserId);
}

/// <summary>
/// Waits for the async job to complete.
/// </summary>
/// <param name="crmService"></param>
/// <param name="asyncJobId"></param>
public static void WaitForAsyncJobCompletion(CrmServiceClient crmService, Guid asyncJobId)
{
    var columnSet = new ColumnSet(true);
    var asyncJob = (AsyncOperation)crmService.Retrieve(AsyncOperation.EntityLogicalName, asyncJobId, columnSet);

    var retryCount = 100;

    while (asyncJob.StateCode != null &&
           asyncJob.StateCode.Value != AsyncOperationState.Completed &&
           retryCount > 0)
    {
        asyncJob = (AsyncOperation)crmService.Retrieve(AsyncOperation.EntityLogicalName, asyncJobId, columnSet);

        Thread.Sleep(2000);     // wait for two seconds

        retryCount--;

        if (asyncJob.StateCode != null)
        {
            Console.WriteLine($"\tAsync operation state is {asyncJob.StateCode.Value}");
        }
    }

    if (asyncJob.StateCode == null)
    {
        return;
    }

    Console.WriteLine($"Async job is {asyncJob.StateCode.Value} with status {(asyncoperation_statuscode)asyncJob.StatusCode.Value}");

    if ((asyncoperation_statuscode)asyncJob.StatusCode.Value == asyncoperation_statuscode.Failed)
    {
        Console.WriteLine($"\t!!! Operation Failed with the following error: {asyncJob.FriendlyMessage}");
    }
}

/// <summary>
/// Reads data from the specified .csv file
/// </summary>
/// <param name="filePath"></param>
/// <returns></returns>
public static string ReadCsvFile(string filePath)
{
    return File.ReadAllText(filePath);
}

/// <summary>
/// Check for importlog records
/// </summary>
/// <param name="crmService"></param>
/// <param name="importFileId"></param>
public static void ReportErrors(CrmServiceClient crmService, Guid importFileId)
{
    var query = new QueryByAttribute
    {
        EntityName = ImportLog.EntityLogicalName,
        ColumnSet = new ColumnSet(true)
    };

    query.AddAttributeValue("importfileid", importFileId);

    var results = crmService.RetrieveMultiple(query);

    if (results.Entities.Count <= 0)
    {
        return;
    }

    Console.WriteLine($"Number of Failures: {results.Entities.Count}");
    Console.WriteLine();

    foreach (var entity in results.Entities)
    {
        var log = (ImportLog)entity;

        var stringBuilder = new StringBuilder();

        if (log.SequenceNumber != null) stringBuilder.AppendLine($"Sequence Number: {log.SequenceNumber.Value}");
        if (log.ErrorNumber != null) stringBuilder.AppendLine($"Error Number: {log.ErrorNumber.Value}");
        if (log.LineNumber != null) stringBuilder.AppendLine($"Line Number: {log.LineNumber.Value}");

        stringBuilder.AppendLine($"Description: {log.AdditionalInfo}");
        stringBuilder.AppendLine($"Column Header: {log.HeaderColumn}");
        stringBuilder.AppendLine($"Column Value: {log.ColumnValue}");
        stringBuilder.AppendLine();

        Console.WriteLine(stringBuilder.ToString());
    }
}

public static EntityReference GetImportMap(CrmServiceClient crmService, string importMapName)
{
    var query = new QueryExpression(ImportMap.EntityLogicalName);

    query.Criteria.AddCondition("name", ConditionOperator.Equal, importMapName);

    var results = crmService.RetrieveMultiple(query);

    if (results.Entities == null || results.Entities.Count == 0)
    {
        throw new Exception($"Unable to locate the ImportMap with the name of: {importMapName}");
    }

    return results.Entities[0].ToEntityReference();
}

public string GetSourceEntityName(CrmServiceClient crmService, EntityReference importMap)
{
    var query = new QueryExpression(ColumnMapping.EntityLogicalName)
    {
        ColumnSet = new ColumnSet("sourceentityname")
    };

    query.Criteria.AddCondition("importmapid", ConditionOperator.Equal, importMap.Id);

    var results = crmService.RetrieveMultiple(query);

    if (results.Entities == null || results.Entities.Count == 0)
    {
        throw new Exception($"Unable to locate the ImportMap with the name of: {importMap.Name}");
    }

    var entity = results.Entities[0];

    return entity["sourceentityname"].ToString();
}