Backpack For Laravel Import Functionality

2018_12_29_113449_create_csv_data_table.php

<?php

use Illuminate\Support\Facades\Schema;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Database\Migrations\Migration;

class CreateCsvDataTable extends Migration
{
    /**
     * Run the migrations.
     *
     * @return void
     */
    public function up()
    {
        Schema::create('csv_data', function (Blueprint $table) {
            $table->bigIncrements('id');
            $table->string('created_by', 100)->nullable();
            $table->string('last_upd_by', 100)->nullable();
            $table->softDeletes();
            $table->timestamps();

            // now add the actual fields.
            $table->string('csv_filename', 64)->default(null)->nullable();
            $table->longText('csv_data')->default(null)->nullable();

        });
    }

    /**
     * Reverse the migrations.
     *
     * @return void
     */
    public function down()
    {
        Schema::dropIfExists('csv_data');
    }
}

button: import.blade.php

<a href="{{ route('admin.'.strtolower(str_replace(' ', '', $crud->entity_name)).'.import') }}" class="btn btn-sm btn-default import-button">
    <i class="fa fa-file-image-o"></i> Import
</a>

CsvData.php

<?php

namespace App\Models;


use Illuminate\Database\Eloquent\Model;

class CsvData extends Model
{
    protected $table = 'csv_data';
    protected $fillable = ['csv_filename', 'csv_data'];
}

import.blade.php

@extends('backpack::layout')

@section('header')
    <section class="content-header">
        <h1>
            <span class="text-capitalize">{!! $crud->getHeading() ?? $crud->entity_name_plural !!}</span>
            <small>{!! $crud->getSubheading() ?? 'Import '.$crud->entity_name !!}.</small>
        </h1>
        <ol class="breadcrumb">
            <li>
                <a href="{{ url(config('backpack.base.route_prefix'), 'dashboard') }}">{{ trans('backpack::crud.admin') }}</a>
            </li>
            <li><a href="{{ url($crud->route) }}" class="text-capitalize">{{ $crud->entity_name_plural }}</a></li>
            <li class="active">Import</li>
        </ol>
    </section>
@endsection

@section('content')

    <div class="container">
        <div class="row">
            <div class="col-md-8 col-md-offset-2">
                <div class="panel panel-default">
                    <div class="panel-heading">CSV Import</div>

                    <div class="panel-body">
                        <form class="form-horizontal" method="POST"
                              action="{{ route('admin.'.strtolower($crud->entity_name).'.importParse') }}"
                              action="{{ route('admin.'.strtolower(str_replace(' ', '', $crud->entity_name)).'.importParse') }}"
                              enctype="multipart/form-data">
                            {{ csrf_field() }}

                            <div class="form-group{{ $errors->has('csv_file') ? ' has-error' : '' }}">
                                <label for="csv_file" class="col-md-4 control-label">CSV file to import: </label>

                                <div class="col-md-6">
                                    <input id="csv_file" type="file" class="form-control" name="csv_file" required>

                                    @if ($errors->has('csv_file'))
                                        <span class="help-block">
                                        <strong>{{ $errors->first('csv_file') }}</strong>
                                    </span>
                                    @endif
                                </div>
                            </div>

                            <div class="form-group">
                                <label for="csv_file" class="col-md-4 control-label">Instructions:</label>

                                <div class="col-md-6">
                                    1. Upload a CSV in the following format:
                                    <a href="{{ route('admin.'.strtolower($crud->entity_name).'.importFormatDownload') }}">Download</a><br>
                                    <a href="{{ route('admin.'.strtolower(str_replace(' ', '', $crud->entity_name)).'.importFormatDownload') }}">Download</a><br>
                                    @foreach($instructions as $idx => $instruction)
                                        {{ $idx+2 }}. {{$instruction}}<br>
                                    @endforeach
                                </div>
                            </div>

                            <div class="form-group">
                                <div class="col-md-8 col-md-offset-4">
                                    <button type="submit" class="btn btn-primary">
                                        Parse CSV
                                    </button>
                                </div>
                            </div>
                        </form>
                    </div>
                </div>
            </div>
        </div>
    </div>

@endsection

import_parse.blade.php

@extends('backpack::layout')

@section('header')
    <section class="content-header">
        <h1>
            <span class="text-capitalize">{!! $crud->getHeading() ?? $crud->entity_name_plural !!}</span>
            <small>{!! $crud->getSubheading() ?? 'Import Mapping for '.$crud->entity_name !!}.</small>
        </h1>
        <ol class="breadcrumb">
            <li>
                <a href="{{ url(config('backpack.base.route_prefix'), 'dashboard') }}">{{ trans('backpack::crud.admin') }}</a>
            </li>
            <li><a href="{{ url($crud->route) }}" class="text-capitalize">{{ $crud->entity_name_plural }}</a></li>
            <li class="active">Import Mapping</li>
        </ol>
    </section>
@endsection

@section('content')

    <div class="container m-l-5 m-r-5" style="width: 100%;">
        <div class="row">
            <form class="form-horizontal" method="POST" action="{{ route('admin.'.strtolower(str_replace(' ', '', $crud->entity_name)).'.importProcess') }}">
                {{ csrf_field() }}
                <input type="hidden" name="csv_data_file_id" value="{{ $csv_data_file->id }}"/>

                <div style="overflow: auto; width: 100%;">
                    <table class="box table table-striped table-hover responsive nowrap m-t-0" style="width: 100%">
                        @foreach ($csv_data as $row)
                            <tr>
                                @foreach ($row as $key => $value)
                                    <td>{{ $value }}</td>
                                @endforeach
                            </tr>
                        @endforeach
                        <tr>
                            @foreach ($csv_data[0] as $key => $value)
                                <td>
                                    <select name="fields[{{ $key }}]" class="form-control select2">
                                        <option value="">Select mapping</option>
                                        @foreach ($import_fields as $key => $import_field)
                                            <option
                                                    @if ($import_field['name']===$value || $import_field['label']===$value)
                                                    selected
                                                    @endif
                                                    value="{{ $import_field['name'] }}">{{ $import_field['label'] }}
                                            </option>
                                        @endforeach
                                    </select>
                                </td>
                            @endforeach
                        </tr>
                    </table>
                </div>

                <div style="float: left;">
                    <a href="{{ route('admin.'.strtolower(str_replace(' ', '', $crud->entity_name)).'.import') }}" class="btn btn-primary">Back</a>
                </div>
                <div style="float: right;">
                    <button type="submit" class="btn btn-primary">Import Data</button>
                </div>
            </form>
        </div>
    </div>

@endsection


@section('after_styles')
    <!-- DATA TABLES -->
    <link href="{{ asset('vendor/adminlte/bower_components/select2/dist/css/select2.min.css') }}" rel="stylesheet"
          type="text/css"/>
    <link href="https://cdnjs.cloudflare.com/ajax/libs/select2-bootstrap-theme/0.1.0-beta.10/select2-bootstrap.min.css"
          rel="stylesheet" type="text/css"/>

    <style>
        .select2-container .select2-selection--single {
            box-sizing: border-box;
            cursor: pointer;
            display: block;
            height: 35px;
            user-select: none;
            -webkit-user-select: none;
        }

        .select2-container {
            width: auto !important;
        }
    </style>

    <!-- CRUD LIST CONTENT - crud_list_styles stack -->
    @stack('crud_fields_styles')
@endsection

@section('after_scripts')

    <script src="{{ asset('vendor/adminlte/bower_components/select2/dist/js/select2.min.js') }}"></script>

    <style type="text/css">
        .select2-selection__clear::after {
            content: ' Clear';
        }
    </style>

    <script>
        $(document).ready(function () {
            $('select.select2').select2();
        });
    </script>

    <!-- CRUD LIST CONTENT - crud_list_scripts stack -->
    @stack('crud_fields_scripts')
@endsection

import_process.blade.php

@extends('backpack::layout')

@section('header')
    <section class="content-header">
        <h1>
            <span class="text-capitalize">{!! $crud->getHeading() ?? $crud->entity_name_plural !!}</span>
            <small>{!! $crud->getSubheading() ?? 'Import Results for '.$crud->entity_name !!}.</small>
        </h1>
        <ol class="breadcrumb">
            <li>
                <a href="{{ url(config('backpack.base.route_prefix'), 'dashboard') }}">{{ trans('backpack::crud.admin') }}</a>
            </li>
            <li><a href="{{ url($crud->route) }}" class="text-capitalize">{{ $crud->entity_name_plural }}</a></li>
            <li class="active">Import Result</li>
        </ol>
    </section>
@endsection

@section('content')

    <div class="container m-l-5 m-r-5" style="width: 100%;">
        <div class="row">
            @if (empty($errors))
                <h1 style="text-align: center; margin: 150px;">Imported successfully</h1>
            @else
                <div style="overflow: auto;">
                    <table class="box table table-striped table-hover display responsive nowrap m-t-0 dataTable">
                        <tr>
                            <td>Row Number</td>
                            <td>Error Messages</td>
                        </tr>
                        @foreach ($errors as $idx_1 => $error)
                            <tr>
                                <td>{{ $error['row_number'] }}</td>
                                <td>
                                    <ul style="padding-left: 15px;">
                                        @foreach($error['errors'] as $idx_2 => $e)
                                            <li>{{ $e }}</li>
                                        @endforeach
                                    </ul>
                                </td>
                            </tr>
                        @endforeach
                    </table>
                </div>
            @endif

            <div style="float: left;">
                <a href="{{ route('admin.'.strtolower(str_replace(' ', '', $crud->entity_name)).'.import') }}" class="btn btn-primary">Import More</a>
            </div>
            <div style="float: right;">
                <a href="{{ url($crud->route) }}" class="btn btn-primary">Finished</a>
            </div>

        </div>
    </div>

@endsection

ImportAwareTrait.php

<?php
namespace App\Services;

use Illuminate\Http\Request;
use App\Models\CsvData;
use Illuminate\Support\Facades\Validator;

/**
 *
 * Include this trait in any CrudController and make provision for enabling configuration driven import functionality on that entity.
 * To enable import on any entity the following steps need to be undertaken.
 *
 * 1. use this trait in the respective entity controller.
 *
 * 2. Override the following methods.
 * a) importSampleFilename()
 * b) importInstructions()
 * c) importValidationRules()
 * d) importValidationMessages()
 * e) importCreate($entity)
 *
 * Instructions of how to override each of these methods is provided against each methods documentation.
 *
 * 3. Once you have imported this trait it allows you to add the following routes.
 * a) GET entityName/import
 * b) POST entityName/importParse
 * c) POST entityName/importProcess
 * d) GET entityName/importFormatDownload
 *
 * These routes get mapped to the methods with the same names which are present in this trait.
 *
 * 4. Optionally to have a more fine grained control over the import process you can override the following two methods.
 * a) beforeImport($entities)
 * b) afterImport($entities)
 *
 * Again both these methods specify the exact nature of the functionality they are exposing.
 *
 * Trait ImportAwareTrait
 * @package App\Services
 */
trait ImportAwareTrait
{
    /**
     * The entity should override this method to provide a new name for the sample CSV which will be
     * downloaded during the first step.
     *
     * Recommended To Override
     *
     * @return string
     */
    protected function importSampleFilename()
    {
        return 'export_sample.csv';
    }

    /**
     * Optionally you can override this method to provide a completely different view for the first step of the import process.
     *
     * Optional To Override
     *
     * @return string
     */
    protected function importView()
    {
        return 'common/import';
    }

    /**
     * You will never have to override this, however left this as a convenience just incase someone wants to override this method.
     *
     * Optional To Override
     *
     * @return string
     */
    protected function importParseView()
    {
        return 'common/import_parse';
    }

    /**
     * This view represents the last step of the import process, again optionally you can override this method to customise the
     * last step of the import process.
     *
     * Optional To Override
     *
     * @return string
     */
    protected function importProcessView()
    {
        return 'common/import_process';
    }

    /**
     * Override this method to provide an array of instructions. These instructions are then displayed in step 1
     * as a numbered list. The first instruction is always the default instruction to download the sample csv.
     *
     * Required To Override
     *
     * @return array
     */
    protected function importInstructions()
    {
        return [];
    }

    /**
     *
     * Override this method to provide an array of validation rules. The syntax to follow is the same as the syntax we
     * use for defining Laravel validation rules.
     * return [
     *      'user_id' => 'required',
     *      'package_id' => 'required',
     *      'company_id' => 'required',
     * ];
     *
     * Required To Override
     *
     * @return array
     *
     */
    protected function importValidationRules()
    {
        return [];
    }

    /**
     * Override this method to provide an array of validation rule failure messages. A sample return could be
     *
     * return [
     *      'user_id.required' => 'Specifying a user is mandatory when creating an appointment, please check if you have specified a valid name, email, date of birth (yyyy-mm-dd format) & contact number of the user.',
     *      'package_id.required' => 'Specifying a package name is mandatory when creating an appointment, please check if you have specified a valid package name.',
     *      'company_id.required' => 'Specifying a company name is mandatory when creating an appointment, please check if you have specified a valid company name.',
     * ];
     *
     * Required To Override
     *
     * @return array
     */
    protected function importValidationMessages()
    {
        return [];
    }

    /**
     *
     * After all foreign keys have been resolved and validation rules have run successfully for all rows, this method
     * is called with an array representing the entity which is to be created.
     *
     * Required To Override
     *
     * @param array $entity
     * @return mixed You need to return the created entity object, generally you can return the result of calling the create method on the entity class.
     */
    protected abstract function importCreate($entity);

    /**
     * Takes an array of entities representing all rows in the CSV as the argument.
     * Each entity passed to this method is actually an array object representing the fully resolved object where all validations have passed and foreign keys resolved.
     * You can optionally override this method to do any processing that is required to be done across all rows which are being imported.
     * To do pre-create activities on a per entity basis you can simply write the code when implemeting importCreate() method.
     *
     * Optional To Override
     *
     * @param array $entities
     */
    protected function beforeImport($entities)
    {

    }

    /**
     * Takes an array of entities representing all rows in the CSV as the argument. Each entity passed to this method is actually an instance of the entity class.
     * You can optionally override this method to do any processing that is required to be done across all rows which are being imported.
     * To do post-create activities on a per entity basis you can simply write the code when implemeting importCreate() method.
     *
     * Optional To Override
     *
     * @param array $entities
     */
    protected function afterImport($entities)
    {

    }

    /**
     * Action method linked to the GET <entityname>/import route.
     * Simply renders the first step which allows one to upload hte CSV file & see the import instructions.
     *
     * @return \Illuminate\Contracts\View\Factory|\Illuminate\View\View
     */
    public function import()
    {
        return view($this->importView(), [
            'crud' => $this->crud,
            'instructions' => $this->importInstructions(),
        ]);
    }

    /**
     * Action method linked to the GET <entityname>/importFormatDownload route.
     * Generates a CSV and streams the response to the browser. The CSV is generated based on cofiguration provided
     * in the setup() method of the entity controller.
     *
     * @return \Symfony\Component\HttpFoundation\StreamedResponse
     */
    public function importFormatDownload()
    {
        // figure out the importable fields.
        $importFields = $this->getImportableFields();

        $headers = array(
            "Content-type" => "text/csv",
            "Content-Disposition" => "attachment; filename=" . $this->importSampleFilename(),
            "Pragma" => "no-cache",
            "Cache-Control" => "must-revalidate, post-check=0, pre-check=0",
            "Expires" => "0"
        );

        $columns = [];
        foreach ($importFields as $idx_1 => $importField) {
            $columns[] = $importField['label'];
        }

        $callback = function () use ($columns) {
            $file = fopen('php://output', 'w');
            fputcsv($file, $columns);

            fclose($file);
        };
        return response()->stream($callback, 200, $headers);
    }

    /**
     * Action method linked to the POST <entityname>/importParse route.
     * This method consumes the file upload, saves it to the CsvData entity which stores the uploaded file in interim storage.
     * This method then renders a view which allows the user to do the mapping between the csv header fields & the field names
     * as specified in the setup() method of the CrudController.
     *
     * @param Request $request
     * @return \Illuminate\Contracts\View\Factory|\Illuminate\View\View
     */
    public function importParse(Request $request)
    {
        $path = $request->file('csv_file')->getRealPath();
        $data = array_map('str_getcsv', file($path));

        $csvDataFile = CsvData::create([
            'csv_filename' => $request->file('csv_file')->getClientOriginalName(),
            'csv_data' => json_encode(mb_convert_encoding($data, 'UTF-8', 'auto'))
        ]);

        $csvData = array_slice($data, 0, 10);

        return view($this->importParseView(), [
            'import_fields' => $this->getImportableFields(),
            'csv_data' => $csvData,
            'csv_header' => $csvData[0],
            'csv_data_file' => $csvDataFile,
            'crud' => $this->crud,
        ]);
    }

    /**
     *
     * Action method linked to the POST <entityname>/importProcess route.
     * Here is where the bulk of the logic of importing the data is written. This method uses the mapping provided in the
     * previous step and uses the CSV data stored earlier in the CsvData entity.
     * It then runs all resolutions / validations as configured.
     * Finally the resolved entities are passed to the importCreate() method. The lifecycle methods are also invoked
     * while running this method.
     *
     * @param Request $request
     * @return \Illuminate\Contracts\View\Factory|\Illuminate\View\View
     */
    public function importProcess(Request $request)
    {
        // figure out the importable fields.
        $importFields = $this->getImportableFields();

        // load the data stored in the database.
        $requestFields = $request->fields;
        $data = CsvData::find($request->csv_data_file_id);
        $csv_data = json_decode($data->csv_data, true);

        $entities = [];
        foreach ($csv_data as $idx_1 => $row) {
            if ($idx_1 == 0) {
                continue;
            }
            $entity = [];

            foreach ($importFields as $idx_2 => $importField) {
                $t = array_search($importField['name'], $requestFields);
                if ($t !== false) {
                    $entity[$importField['name']] = isset($row[$t]) ? $row[$t] : null;
                }
            }

            $entities[] = $entity;
        }

        // resolve all the foreign keys.
        foreach ($entities as $idx_1 => &$entity) {
            foreach ($importFields as $idx_2 => $importField) {
                if (isset($importField['import_resolver'])) {
                    $importField['import_resolver']($entity);
                }
            }
        }

        // run validations.
        $errors = [];
        foreach ($entities as $idx => &$entity) {
            $rules = $this->importValidationRules();
            $messages = $this->importValidationMessages();
            $validator = Validator::make($entity, $rules, $messages);
            if ($validator->fails()) {
                $errors[] = [
                    'row_number' => $idx + 1,
                    'errors' => $validator->errors()->all()
                ];
            }
        }

        // create the user appointments.
        $entityObjects = [];
        $this->beforeImport($entities);
        if (empty($errors)) {
            foreach ($entities as $idx => &$entity) {
                $entityObjects[] = $this->importCreate($entity);
            }
        }
        $this->afterImport($entityObjects);

        return view($this->importProcessView(), [
            'crud' => $this->crud,
            'errors' => $errors,
        ]);
    }

    /**
     *
     * Used internally to fetch a list of importable fields. This method uses the CrudPanel metadata and infers
     * importable fields based on the "importable => true" configuration attribute.
     *
     * @return array
     */
    private function getImportableFields()
    {
        $createFields = $this->crud->getFields('create');
        $importFields = [];
        foreach ($createFields as $idx_1 => $createField) {
            if (isset($createField['importable']) && $createField['importable']) {
                if (isset($createField['importable_fields'])) {
                    foreach ($createField['importable_fields'] as $idx_2 => $nested) {
                        $importFields[] = $nested;
                    }
                } else {
                    $importFields[] = $createField;
                }
            }
        }

        return $importFields;
    }

}

I used laravel backpack 4.1
but $this->crud->getFields('create'); is null value. How to resolve.

I have same error

@gumpon and @ckhandla94 I'm sure y'all figured this out a while ago, so this is mostly for the benefit of others who may stumble upon this page: In order to get $this->crud->getFields('create') to return the importable fields, you need to add the following to your CRUD controller:

    protected function setupImportOperation()
    {
        $this->setupCreateOperation();
    }

(alternatively, you could re-define the fields here, but I don't see any reason that would be necessary...)

The other things I had to do to get this code working in backpack 4.1:

in ImportAwareTrait, add the following method:

    protected function setupImportRoutes($segment, $routeName, $controller)
    {
        Route::get($segment . '/import', [
            'as' => "admin.$routeName.import",
            'uses' => "$controller@import",
            'operation' => 'import',
        ]);
        Route::get($segment . '/import/download-format', [
            'as' => "admin.$routeName.importFormatDownload",
            'uses' => "$controller@importFormatDownload",
            'operation' => 'import',
        ]);
        Route::post($segment . '/import/parse', [
            'as' => "admin.$routeName.importParse",
            'uses' => "$controller@importParse",
            'operation' => 'import',
        ]);
        Route::post($segment . '/import/process', [
            'as' => "admin.$routeName.importProcess",
            'uses' => "$controller@importProcess",
            'operation' => 'import',
        ]);
    }

There were a number of changes I made to the blade templates to get them to look and feel like they were a part of Backpack, but the only one that was strictly necessary for this code to work was to change the Backpack template name in the @extends() directive from backpack::layout to backpack::blank.

Other than that, the documentation covers the normal things you need to do to set up the trait's integration with your CRUD controller.

Hope this helps someone in the future!

@gumpon and @ckhandla94 I'm sure y'all figured this out a while ago, so this is mostly for the benefit of others who may stumble upon this page: In order to get $this->crud->getFields('create') to return the importable fields, you need to add the following to your CRUD controller:

    protected function setupImportOperation()
    {
        $this->setupCreateOperation();
    }

(alternatively, you could re-define the fields here, but I don't see any reason that would be necessary...)

The other things I had to do to get this code working in backpack 4.1:

in ImportAwareTrait, add the following method:

    protected function setupImportRoutes($segment, $routeName, $controller)
    {
        Route::get($segment . '/import', [
            'as' => "admin.$routeName.import",
            'uses' => "$controller@import",
            'operation' => 'import',
        ]);
        Route::get($segment . '/import/download-format', [
            'as' => "admin.$routeName.importFormatDownload",
            'uses' => "$controller@importFormatDownload",
            'operation' => 'import',
        ]);
        Route::post($segment . '/import/parse', [
            'as' => "admin.$routeName.importParse",
            'uses' => "$controller@importParse",
            'operation' => 'import',
        ]);
        Route::post($segment . '/import/process', [
            'as' => "admin.$routeName.importProcess",
            'uses' => "$controller@importProcess",
            'operation' => 'import',
        ]);
    }

There were a number of changes I made to the blade templates to get them to look and feel like they were a part of Backpack, but the only one that was strictly necessary for this code to work was to change the Backpack template name in the @extends() directive from backpack::layout to backpack::blank.

Other than that, the documentation covers the normal things you need to do to set up the trait's integration with your CRUD controller.

Hope this helps someone in the future!

I used Backpack 4.1
Thanks for this code..
But I have a problem.. I follow this code and some changes.. the result page show Imported successfully, but nothing rows imported..

Problem solved if I commented code below in importProcess method :
if ($idx_1 == 0) { continue; }

Thanks....

Can we turn this into a BackpackForLaravel package? something that can easily connect to the admin?

hello sir can you please share a folder structure for your file