HiveMetaResolver.class

HiveMetaResolver.class

public class HiveMetaResolver {

    private String dbTable;
    private String hiveJdbcUrl;
    private String hiveJdbcUser;
    private String hiveJdbcPassword;
    private String hiveMetastoreUrl;
    private String hiveMetastoreUser;
    private String hiveMetastorePassword;

    private StructType sparkSchema;
    private Map<String, HiveMetadata> hiveMetadataMap;


    public HiveMetaResolver(String dbTable,
                            String hiveJdbcUrl,
                            String hiveJdbcUser,
                            String hiveJdbcPassword,
                            String hiveMetastoreUrl,
                            String hiveMetastoreUser,
                            String hiveMetastorePassword)
    {
        this.dbTable = dbTable;

        this.hiveJdbcUrl = hiveJdbcUrl;
        this.hiveJdbcUser = hiveJdbcUser;
        this.hiveJdbcPassword = hiveJdbcPassword;

        this.hiveMetastoreUrl = hiveMetastoreUrl;
        this.hiveMetastoreUser = hiveMetastoreUser;
        this.hiveMetastorePassword = hiveMetastorePassword;

        convertHiveTypeToSparkType();
    }

    private void convertHiveTypeToSparkType()
    {
        hiveMetadataMap = getMeta();
      
     
        Map<String, HiveTableSchema> hiveTableSchemaMap = getHiveTableSchema();
     
      
        for(String columnName : hiveTableSchemaMap.keySet())
        {
            String hiveColumnType = hiveTableSchemaMap.get(columnName).getColumnType();
            hiveMetadataMap.get(columnName).setHiveColumnType(hiveColumnType);
        }

        List<StructField> structFields = new ArrayList<StructField>();

        // sort hive metadata list by index.
        List<HiveMetadata> hiveMetadataList = new ArrayList<>(hiveMetadataMap.values());
        Collections.sort(hiveMetadataList, (o1, o2) -> {
            return o1.getIndex() - o2.getIndex();
        });

        for(HiveMetadata hiveMetadata : hiveMetadataList)
        {
            String columnName = hiveMetadata.getColumnName();

            int sqlType = hiveMetadata.getDataType();
            int precision = hiveMetadata.getFieldSize();
            int scale = hiveMetadata.getFieldScale();
            boolean signed = hiveMetadata.isSigned;
            boolean nullable = hiveMetadata.nullable;
            String hiveColumnType = hiveMetadata.getHiveColumnType();
            
            // convert hive type to spark schema.
            ...
        }

        StructType schema = DataTypes.createStructType(structFields);

        this.sparkSchema = schema;
    }


    public StructType getSparkSchema()
    {
        return sparkSchema;
    }

     ...

}