KeyStoreCompat is a simple class that takes care of dealing with KeyStore and using the AndroidKeyStore when it can, or another type if it can't. You're welcome.

KeyStoreCompat.java

package YOUR_PACKAGE;

import android.content.Context;
import android.os.Build;

import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.cert.Certificate;

/**
 * Provides a KeyStore mechanism that will work best for various versions of the OS.
 * <p/>
 * It is meant to be used as an instance and not a singleton.
 */

public class KeyStoreCompat {

    // -------------
    // Static
    // -------------

    private static final String ANDROID_KEYSTORE_TYPE = "AndroidKeyStore";
    private static final char[] PASSWORD = "123456".toCharArray();
    private static final String KEYSTORE_FILE = "keystore";

    // ----------------------
    // Attributes
    // ----------------------

    private Boolean usingAndroidKeyStore; // This prevents from checking every single time (saves a tiny bunch of cycles)
    private KeyStore keyStore; // The actual KeyStore instance that we use internally
    private Context context; // is used for file access

    // These values can be overridden in one of the constructors, but if you're lazy they're all setup
    private String keyStoreType = KeyStore.getDefaultType();
    private char[] password = PASSWORD; // WARNING this can be decompiled, only use this if you store public keys
    private String file = KEYSTORE_FILE;

    // ----------------------
    // Constructors
    // ----------------------

    /**
     * Using all the default values
     * @see KeyStoreCompat(Context, String, String, char[]) for parameter details
     */
    public KeyStoreCompat(Context context) {
        this.context = context;
        initKeyStore();
    }

    /**
     * Allows to override the key store type
     * @see KeyStoreCompat(Context, String, String, char[]) for parameter details
     */
    public KeyStoreCompat(Context context, String type) {
        keyStoreType = type;
        this.context = context;
        initKeyStore();
    }

    /**
     * Allows to provide the file name and password
     * @see KeyStoreCompat(Context, String, String, char[]) for parameter details
     */
    public KeyStoreCompat(Context context, String file, char[] password) {
        this.file = file;
        this.password = password;
        this.context = context;
        initKeyStore();
    }

    /**
     * Allows to override all the parameters needed for a key store access
     * @param context needed for I/O file access
     * @param type type of required key store
     * @param file name of the file that will store the key data
     * @param password password allowing to protect the file
     */
    public KeyStoreCompat(Context context, String type, String file, char[] password) {
        keyStoreType = type;
        this.file = file;
        this.password = password;
        this.context = context;
        initKeyStore();
    }

    // ----------------------
    // Public Methods
    // ----------------------

    /**
     * @return true if the key store is not null (initialized at creation of the instance)
     */
    public boolean isValid() {
        return keyStore != null;
    }

    /**
     * @return the type of the key store that has been instantiated
     */
    public String type() {
        return isValid() ? keyStore.getType() : "invalid";
    }

    /**
     * Loads the content from the file (see constructors) into the key store instance
     *
     * @param forReading true if the goal is to read from the file, but not write. Note:
     *                   if API level is 18+, this parameter is not taken into account.
     * @throws IllegalStateException if key store is invalid {@link KeyStoreCompat#isValid()}
     * @throws KeyStoreCompatException if something wrong happened during the execution of the command
     */
    public void load(boolean forReading) {
        if (keyStore == null) {
            throw new IllegalStateException("KeyStore is null, could not find instance");
        }

        try {
            if (usingAndroidKeyStore() || !forReading) {
                keyStore.load(null);
            } else {
                FileInputStream fis = context.openFileInput(file);
                keyStore.load(fis, password);
                fis.close();
            }
        } catch (Exception e) {
            throw new KeyStoreCompatException("An exception occurred loading the keystore", e);
        }
    }

    /**
     * Stores the whole content of the key store object to the file.
     *
     * <p>Note: for API level 18+ this is not necessary as the AndroidKeyStore automatically stores
     * data as it is added </p>
     * @throws IllegalStateException if key store is invalid {@link KeyStoreCompat#isValid()}
     * @throws KeyStoreCompatException if something wrong happened during the execution of the command
     */
    public void store() {
        if (keyStore == null) {
            throw new IllegalStateException("KeyStore is null, could not find instance");
        }

        try {
            if (!usingAndroidKeyStore()) {
                FileOutputStream fos = context.openFileOutput(file, Context.MODE_PRIVATE);
                keyStore.store(fos, password);
                fos.close();
            }
        } catch (Exception e) {
            throw new KeyStoreCompatException("An exception occurred storing the keystore", e);
        }
    }

    /**
     * Safely delete an entry, will be skipped if the alias is not found
     * @param alias name of the entry stored in the store
     * @throws IllegalStateException if key store is invalid {@link KeyStoreCompat#isValid()}
     * @throws KeyStoreCompatException if something wrong happened during the execution of the command
     */
    public void deleteEntryIfExists(String alias) {
        if (keyStore == null) {
            throw new IllegalStateException("KeyStore is null, could not find instance");
        }

        try {
            if (keyStore.containsAlias(alias)) {
                keyStore.deleteEntry(alias);
            }
        } catch (Exception e) {
            throw new KeyStoreCompatException("An exception occurred deleting an entry in keystore", e);
        }
    }

    /**
     * Set the certificate corresponding to the alias entry in the key store. If an entry already exists for this
     * alias, it will be overridden.
     *
     * @param alias alias for the entry
     * @param certificate certificate to store
     * @throws IllegalStateException if key store is invalid {@link KeyStoreCompat#isValid()}
     * @throws KeyStoreCompatException if something wrong happened during the execution of the command
     */
    public void setCertificateEntry(String alias, Certificate certificate) {
        if (keyStore == null) {
            throw new IllegalStateException("KeyStore is null, could not find instance");
        }

        try {
            keyStore.setCertificateEntry(alias, certificate);
        } catch (Exception e) {
            throw new KeyStoreCompatException("An exception occurred deleting an entry in keystore", e);
        }
    }

    /**
     * Checks whether the key store contains the given alias as an entry
     * @param alias name of the entry to verify
     * @return true if the alias was found
     * @throws IllegalStateException if key store is invalid {@link KeyStoreCompat#isValid()}
     * @throws KeyStoreCompatException if something wrong happened during the execution of the command
     */
    public boolean containsAlias(String alias) {
        if (keyStore == null) {
            throw new IllegalStateException("KeyStore is null, could not find instance");
        }

        try {
            return keyStore.containsAlias(alias);
        } catch (Exception e) {
            throw new KeyStoreCompatException("An exception occurred deleting an entry in keystore", e);
        }
    }

    /**
     * Retrieves the certificate for the given alias
     * @param alias name of the certificate given when stored
     * @return The certificate for the given alias if found, or null if the alias doesn't exist or if it is not a certificate
     * @throws IllegalStateException if key store is invalid {@link KeyStoreCompat#isValid()}
     * @throws KeyStoreCompatException if something wrong happened during the execution of the command
     */
    public Certificate getCertificate(String alias) {
        if (keyStore == null) {
            throw new IllegalStateException("KeyStore is null, could not find instance");
        }

        try {
            return keyStore.getCertificate(alias);
        } catch (Exception e) {
            throw new IllegalStateException("An exception occurred deleting an entry in keystore", e);
        }
    }

    // ----------------------
    // Private Methods
    // ----------------------

    /**
     * Initialization of the key store instance, called at construction
     */
    private void initKeyStore() {
        keyStore = safelyGetKeyStore();
    }

    /**
     * @return true if API level is high enough to use the AndroidKeyStore instance
     */
    private boolean canUserAndroidKeyStore() {
        return Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN_MR2;
    }

    /**
     * @return true if currently using the AndroidKeyStore instance
     */
    private boolean usingAndroidKeyStore() {
        if (usingAndroidKeyStore == null) {
            usingAndroidKeyStore = keyStore != null && keyStore.getType().equals(ANDROID_KEYSTORE_TYPE);
        }
        return usingAndroidKeyStore;
    }

    /**
     * @return get the keystore without worrying (too much) about null safety
     */
    private KeyStore safelyGetKeyStore() {
        if (keyStore == null) {

            String instanceType = canUserAndroidKeyStore() ? ANDROID_KEYSTORE_TYPE : keyStoreType;
            keyStore = internalGetKeyStore(instanceType);

            if (keyStore == null) {
                keyStore = internalGetKeyStore(KeyStore.getDefaultType());
            }
        }

        return keyStore;
    }

    /**
     * Internal key store instance getter
     * @param type type of required key store
     * @return key store instance
     */
    private KeyStore internalGetKeyStore(String type) {
        KeyStore keyStore;
        try {
            keyStore = KeyStore.getInstance(type);
        } catch (KeyStoreException e) {
            keyStore = null;
        }
        return keyStore;
    }

    // ----------------------
    // Inner Classes
    // ----------------------

    /**
     * Exception used in the code so that you know that if it crashes, it crashed with style!
     */
    public static class KeyStoreCompatException extends RuntimeException {
        public KeyStoreCompatException(String message, Throwable cause) {
            super(message, cause);
        }
    }
}

First off, thanks for providing this code snippet! I have 2 questions about the implementation.
Is the public method public void store() meant to be called after any modifications to the entries in the keystore, or how is that meant to be used?
Also, in the method public void load(boolean forReading) if forReading is set to false, we load the keystore the same way as for api levels > 18, that is keyStore.load(null), can you explain why that is?

To add a bit more context to my second question, I would expect us to read from the file for read only operations as well as for write operations. Wouldn't keyStore.load(null) create an entirely new keystore instead of modifying the existing one?