Basic IO for dbf file format.

dbf.lua

local printf = function(...) return print(string.format(...)) end

local function prequire(...)
  local ok, mod = pcall(require, ...)
  if ok then return mod end
  return nil, mod
end

local function iif(cond, val1, val2)
  if cond then return val1 end return val2 
end

local function get_arg(...)
  return select('#',...), {...}
end

local function coalesce(...)
  local argc, argv = get_arg(...)
  for i = 1, argc do
    if argv[i] ~= nil then 
      return argv[i]
    end
  end  
end

local function round_str(x, digits)
  return assert(string.format("%."..(digits or '0').."f",x))
end;

local function fitstrr(str, ch, width)
  if #str >= width then
    return str:sub(1,width)
  end
  return str .. string.rep(ch,width-#str)
end

local function fitstrl(str, ch, width)
  if #str >= width then
    return str:sub(-width)
  end
  return string.rep(ch,width-#str) .. str
end

local function byte_concat(t)
  local res = ''
  for _,v in ipairs(t) do
    res = res .. string.char(v)
  end
  return res
end

local function at(s,i)
  return string.sub(s,i,i)
end

local function byte_at(s,i)
  return string.byte(at(s,i))
end

local function toint(str)
  return byte_at(str,4) * (256*256*256) + 
         byte_at(str,3) * (256*256) + 
         byte_at(str,2) * 256 + 
         byte_at(str,1)
end

local function toshort(str)
  return byte_at(str,2) * 256 + 
         byte_at(str,1)
end

local function intto(v)
  local str = ''
  str = str .. string.char(math.mod(v,256))
  v = math.floor(v/256)
  str = str .. string.char(math.mod(v,256))
  v = math.floor(v/256)
  str = str .. string.char(math.mod(v,256))
  v = math.floor(v/256)
  str = str .. string.char(math.mod(v,256))
  return str
end

local function shortto(v)
  local str = ''
  str = str .. string.char(math.mod(v,256))
  v = math.floor(v/256)
  str = str .. string.char(math.mod(v,256))
  return str
end

----------------------------------------------------------------------
-- File IO
----------------------------------------------------------------------

local function read_byte(f)
  return string.byte(f:read(1))
end

local function read_int(f)
  return toint(f:read(4))
end

local function read_short(f)
  return toshort( f:read(2) )
end

local function read_char(f)
  return f:read(1)
end

local function read_str(f,len)
  return string.gsub(f:read(len),'%z.*$','')
end

local function read_n(f,n)
  return f:read(n)
end

local function read_date(f)
  return {y=1900+read_byte(f),m=read_byte(f),d=read_byte(f)};
end

local function write_byte(f,v)
  return f:write(string.char(v))
end

local function write_int(f,v)
  return f:write(intto(v))
end

local function write_short(f,v)
  return f:write(shortto(v))
end

local function write_char(f,v)
  return f:write(at(v,1))
end

local function write_n(f,str,len,ch)
  return f:write(fitstrr(str, ch or '\0', len))
end

local function write_str(f,str)
  return f:write(str)
end

local function write_str_n(f,str,len)
  return write_n(f,str,len,'\000')
end

local function write_date(f,v)
  write_byte(f, math.mod((v.y - 1900),256))
  write_byte(f, v.m)
  return write_byte(f, v.d)
end

local function set_pos(f, pos)
  return f:seek("set", pos)
end

local function get_pos(f)
  return f:seek()
end

local function file_size(f)
  local pos = get_pos(f)
  local size = f:seek('end')
  set_pos(f,pos);
  return size
end

----------------------------------------------------------------------
-- DBF Low Level
----------------------------------------------------------------------

local DBF_HEADER_SIZE = 32
local DBF_FIELD_REC_SIZE = 32
local DBF_EOR = 0x0D
local DBF_EOF = 0x1A
local DBF_ID = {
  [0x03] = "FoxBASE+/dBASE III +";
  [0x83] = "FoxBASE+/dBASE III +, memo";
  [0x03] = "FoxPro/dBASE IV";
  [0xF5] = "FoxPro, memo";
  [0x8B] = "dBASE IV, memo";
}

local function dbf_read_header(f)
--[==========================================================[
  ¦ Байты :              Описание                            ¦
  ¦==========================================================¦
  ¦ 00    : Типы файлов с данными:                           ¦
  ¦       : * FoxBASE+/dBASE III +, без memo - 0х03          ¦
  ¦       : * FoxBASE+/dBASE III +, с memo   - 0х83          ¦
  ¦       : * FoxPro/dBASE IV, без memo      - 0х03          ¦
  ¦       : * FoxPro с memo                  - 0хF5          ¦
  ¦       : * dBASE IV с memo                - 0x8B          ¦
  ¦----------------------------------------------------------¦
  ¦ 01-03 : Последнее изменение (ГГММДД)                     ¦
  ¦----------------------------------------------------------¦
  ¦ 04-07 : Число записей в файле                            ¦
  ¦----------------------------------------------------------¦
  ¦ 08-09 : Положение первой записи с данными                ¦
  ¦----------------------------------------------------------¦
  ¦ 10-11 : Длина одной записи с данными (включая признак    ¦
  ¦       : удаления)                                        ¦
  ¦----------------------------------------------------------¦
  ¦ 12-27 : Зарезервированы                                  ¦
  ¦----------------------------------------------------------¦
  ¦ 28    : 1-есть структ.составной инд.файл (типа .CDX)     ¦
  ¦       : 0-нет                                            ¦
  ¦----------------------------------------------------------¦
  ¦ 29-31 : Зарезервированы                                  ¦
  ¦----------------------------------------------------------¦
  ¦ 32-n  : Подзаписи полей**                                ¦
  ¦----------------------------------------------------------¦
  ¦  n+1  : Признак завершения записи заголовка (0х0D)       ¦
--]==========================================================]
  set_pos(f, 0)
  local DBF_HEAD = {
    dbf_id         = assert( read_byte (f)    );
    last_update    = assert( read_date (f)    );
    last_rec       = assert( read_int  (f)    );
    data_offset    = assert( read_short(f)    );
    rec_size       = assert( read_short(f)    );
    reserved1      = assert( read_n    (f,16) );
    flag_ext       = assert( read_byte (f)    );
    reserved2      = assert( read_n    (f,3)  );
  }
  return DBF_HEAD
end

local function dbf_write_header(f, HEADER)
  set_pos(f, 0)
  assert(write_byte  (f,HEADER.dbf_id)       )
  assert(write_date  (f,HEADER.last_update)  )
  assert(write_int   (f,HEADER.last_rec)     )
  assert(write_short (f,HEADER.data_offset)  )
  assert(write_short (f,HEADER.rec_size)     )
  assert(write_n     (f,HEADER.reserved1,16) )
  assert(write_byte  (f,HEADER.flag_ext)     )
  assert(write_n     (f,HEADER.reserved2,3)  )
  return DBF_HEADER_SIZE
end

local function dbf_get_reader(FIELD_REC)
  local readers = {
    ["C"] = function(f) return read_str (f, FIELD_REC.len_info.char_len) end;
    ["L"] = function(f) return read_char(f) end;
    ["N"] = function(f) return tonumber((read_str(f, FIELD_REC.len_info.num_size._len))) end;
    ["D"] = function(f) return read_str(f, 4) .. '-' .. read_str(f, 2) .. '-' .. read_str(f, 2) end;
    ["B"] = function(f) return read_n(f, 10) end;
  }
  return assert(readers[FIELD_REC.field_type], "UNSUPPORTED FIELD TYPE '" .. FIELD_REC.field_type .. "'")
end

local function dbf_get_writer(FIELD_REC)
  local writers = {
    ["C"] = function(f, v) return write_str_n(f, v, FIELD_REC.len_info.char_len) end;
    ["L"] = function(f, v) return write_char(f, v) end;
    ["N"] = function(f, v) 
      local width = FIELD_REC.len_info.num_size._len
      local str = round_str(v, FIELD_REC.len_info.num_size._dec)
      return write_str(f, fitstrl(str, ' ', width))
    end;
    ["D"] = function(f, v) 
      write_str(f, v:sub(1,4))
      write_str(f, v:sub(6,7))
      return write_str(f, v:sub(9,10))
    end;
    ["B"] = function(f, v) return write_n(f, v, 10) end;
  }
  return assert(writers[FIELD_REC.field_type], "UNSUPPORTED FIELD TYPE '" .. FIELD_REC.field_type .. "'")
end

local function dbf_read_field_info(f)
  local FIELD_REC
  repeat
    local n = assert(read_byte(f))
    if DBF_EOR == n then break end
    assert(n ~= 0)
    FIELD_REC = {
      field_name = string.char(n) .. assert(read_str(f,10));
      field_type = assert(read_char(f));
      offset     = assert(read_int(f, 4));
      len_info   = {
        num_size = {
          _len = assert(read_byte(f));
          _dec = assert(read_byte(f));
        };
      };
      flags    = assert(read_byte(f));
      reserved = read_n(f, 13);
    };
    if FIELD_REC.field_type == 'C' then
      FIELD_REC.len_info.char_len = toshort( string.char(FIELD_REC.len_info.num_size._len) .. string.char(FIELD_REC.len_info.num_size._dec) )
    else
      FIELD_REC.len_info.char_len = FIELD_REC.len_info.num_size._len
    end
    FIELD_REC.reader = dbf_get_reader(FIELD_REC)
    FIELD_REC.writer = dbf_get_writer(FIELD_REC)
  until true;
  return FIELD_REC
end

local function dbf_write_field_info(f, FIELD_REC)
  if FIELD_REC then
    assert(write_n    (f, FIELD_REC.field_name, 11)         )
    assert(write_char (f, FIELD_REC.field_type)             )
    assert(write_int  (f, FIELD_REC.offset)                 )
    assert(write_byte (f, FIELD_REC.len_info.num_size._len) )
    assert(write_byte (f, FIELD_REC.len_info.num_size._dec) )
    assert(write_byte (f, FIELD_REC.flags)                  )
    assert(write_n    (f, FIELD_REC.reserved, 13)           )
    return DBF_FIELD_REC_SIZE
  end
  write_byte(f, DBF_EOR)
  return 1
end

local function dbf_read_fields(f)
  set_pos(f, DBF_HEADER_SIZE)
  local DBF_FIELDS = {}
  while true do
    local FIELD_REC = dbf_read_field_info(f)
    if not FIELD_REC then break end
    DBF_FIELDS[#DBF_FIELDS + 1] = FIELD_REC
  end
  return DBF_FIELDS
end

local function dbf_write_fields(f, DBF_FIELDS)
  set_pos(f, DBF_HEADER_SIZE)
  local i = 1
  repeat
    local n = dbf_write_field_info(f, DBF_FIELDS[i])
    i = i + 1
  until n == 1
  return i - 1
end

local function dbf_set_pos(f, HEADER, FIELDS, n)
  -- assert(HEADER.last_rec >= n)
  set_pos(f, HEADER.data_offset + HEADER.rec_size * (n-1))
end

local function dbf_get_pos(f, HEADER, FIELDS)
  local pos  = 1 + math.floor( (get_pos(f) - HEADER.data_offset) / HEADER.rec_size )
  return pos
end

local function dbf_read_record(f, HEADER, FIELDS)
  local record = {}
  record[0] = read_byte(f)
  for _,field in ipairs(FIELDS) do
    local val = field.reader(f)
    record[#record + 1] = val
  end
  return record
end

local function dbf_read_all_records(f, HEADER, FIELDS)
  local records = {}
  dbf_set_pos(f, HEADER, FIELDS, 1)
  for i = 1, HEADER.last_rec do
    records[#records + 1] = dbf_read_record(f, HEADER, FIELDS)
  end
  return records
end;

local function dbf_read_record_n(f, HEADER, FIELDS, n)
  dbf_set_pos(f, HEADER, FIELDS, n)
  return dbf_read_record(f, HEADER, FIELDS)
end;

local function dbf_write_record(f, record, HEADER, FIELDS)
  -- assert(dbf_get_pos(f, HEADER, FIELDS) ~= -1)
  -- assert(dbf_get_pos(f, HEADER, FIELDS) ~= -1)
  -- 0x20 - valid record
  -- 0x2A - deleted record
  local n = dbf_get_pos(f, HEADER, FIELDS)
  assert(write_byte(f, record[0] or 0x20))
  for i,field in ipairs(FIELDS) do
    assert(field.writer(f, record[i]))
  end
  if HEADER.last_rec < n then
    HEADER.last_rec = n
  end
  return true
end

local function dbf_write_record_n(f, record, HEADER, FIELDS, n)
  dbf_set_pos(f, HEADER, FIELDS, n)
  return dbf_write_record(f, record, HEADER, FIELDS)
end

local function dbf_append_record(f, record, HEADER, FIELDS)
  HEADER.last_rec = HEADER.last_rec + 1
  return dbf_write_record_n(f, record, HEADER, FIELDS, HEADER.last_rec)
end

local function dbf_write_all_records(f, records, HEADER, FIELDS )
  dbf_set_pos(f, HEADER, FIELDS, 1)
  for i, record in ipairs(records) do
    dbf_write_record(f, record, HEADER, FIELDS)
  end
  return true
end;

local function dbf_write_eof(f, HEADER, FIELDS)
  dbf_set_pos(f, HEADER, FIELDS, HEADER.last_rec + 1)
  write_byte(f, DBF_EOF)
end

----------------------------------------------------------------------
local function dbf_get_field_size(FIELD_REC)
  if FIELD_REC.field_type == "C" then
    return FIELD_REC.len_info.char_len
  elseif FIELD_REC.field_type == "L" then
    assert (FIELD_REC.len_info.char_len == 1)
    return 1
  elseif FIELD_REC.field_type == "N" then
    return FIELD_REC.len_info.num_size._len
  elseif FIELD_REC.field_type == "D" then
    assert (FIELD_REC.len_info.char_len == 8)
    return 8
  else
    assert(false, "UNSUPPORTED FIELD TYPE '" .. FIELD_REC.field_type .. "'")
  end
end

local function dbf_set_field_size(FIELD_REC, s1, s2)
  if FIELD_REC.field_type == "C" then
    assert(s1)
    assert(s2 == nil)
    FIELD_REC.len_info.char_len = s1
    local len = shortto(FIELD_REC.len_info.char_len)
    FIELD_REC.len_info.num_size._len = byte_at(len,1)
    FIELD_REC.len_info.num_size._dec = byte_at(len,2)
  elseif FIELD_REC.field_type == "L" then
    assert(s1==nil)
    assert(s2==nil)
    FIELD_REC.len_info.char_len = 1
    FIELD_REC.len_info.num_size._len = 1
    FIELD_REC.len_info.num_size._dec = 0
  elseif FIELD_REC.field_type == "N" then
    FIELD_REC.len_info.num_size._len = assert(s1)
    FIELD_REC.len_info.num_size._dec = coalesce(s2,0)
    -- FIELD_REC.len_info.char_len = toshort( string.char(FIELD_REC.len_info.num_size._dec) .. string.char(FIELD_REC.len_info.num_size._len))
    FIELD_REC.len_info.char_len = toshort( string.char(FIELD_REC.len_info.num_size._dec) .. string.char(FIELD_REC.len_info.num_size._len))
  elseif FIELD_REC.field_type == "D" then
    assert(s1==nil)
    assert(s2==nil)
    FIELD_REC.len_info.char_len = 8
    FIELD_REC.len_info.num_size._len = 8
    FIELD_REC.len_info.num_size._dec = 0
  else
    assert(false, "UNSUPPORTED FIELD TYPE '" .. FIELD_REC.field_type .. "'")
  end
end

local function dbf_create_header(f)
  local HEADER = {  
    dbf_id         = 0x03;
    last_update    = {y=1900; d=0; m=0};
    last_rec       = 0;
    data_offset    = DBF_HEADER_SIZE;
    rec_size       = 1;
    reserved1      = byte_concat{0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00};
    flag_ext       = 0;
    reserved2      = byte_concat{0x65,0x00,0x00};
  };
  dbf_write_header(f,HEADER)
  return HEADER
end

local function dbf_create_field(f, HEADER, fieldName, fieldType, fieldSize1, fieldSize2)
  local FIELD_REC = {
    field_name = fieldName:sub(1,11);
    field_type = fieldType:sub(1,1);
    offset     = HEADER.rec_size;
    len_info   = {
      char_len = 0;
      num_size = {
        _len = 0;
        _dec = 0;
      };
    };
    flags = 0x00;
    reserved = byte_concat{0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}; 
  };
  dbf_set_field_size(FIELD_REC, fieldSize1, fieldSize2)
  FIELD_REC.reader = dbf_get_reader(FIELD_REC)
  FIELD_REC.writer = dbf_get_writer(FIELD_REC)
  HEADER.rec_size = HEADER.rec_size + dbf_get_field_size(FIELD_REC)
  dbf_write_field_info(f, FIELD_REC)
  HEADER.data_offset = get_pos(f) + 1

  return FIELD_REC
end

----------------------------------------------------------------------
-- DBF class
----------------------------------------------------------------------

local BIG_FILE_SIZE = 20 * 1024^3

local function NewDBF(file_or_fileName)
  local real_f, mem_f, f
  local DBF_HEADER, DBF_FIELDS
  local self
  self = {
    open             = function (file_or_fileName)
      self.close()
      if type(file_or_fileName) == 'string' then
        local err real_f, err = io.open(file_or_fileName,'rb')
        if not real_f then return nil, err end
        f = real_f
        if file_size(f) < BIG_FILE_SIZE then
          local memfile = prequire "memoryfile"
          if memfile then 
            mem_f = memfile.open(f:read("*all"),'rb')
            if mem_f then f = mem_f end
          end
        end
      else
        real_f = file_or_fileName
        f = real_f 
      end
      DBF_HEADER = dbf_read_header(f)
      DBF_FIELDS = dbf_read_fields(f)
      return self
    end;
    close            = function ()
      if mem_f  then mem_f:close()  end
      if real_f then real_f:close() end
      real_f, mem_f, f = nil
      DBF_HEADER, DBF_FIELDS = nil
    end;
    read_record      = function ()    return dbf_read_record(f, DBF_HEADER, DBF_FIELDS) end;
    read_record_n    = function (n)   return dbf_read_record_n(f, DBF_HEADER, DBF_FIELDS, n) end;
    read_all_records = function ()    return dbf_read_all_records(f, DBF_HEADER, DBF_FIELDS) end;
    set_pos          = function (n)   return dbf_set_pos(f, DBF_HEADER, DBF_FIELDS, n) end;
    get_pos          = function ()    return dbf_get_pos(f, DBF_HEADER, DBF_FIELDS) end;
    count            = function ()    return DBF_HEADER.last_rec end;
    record_size      = function ()    return DBF_HEADER.rec_size end;
    write_record     = function (rec) return dbf_write_record(f, rec, DBF_HEADER, DBF_FIELDS) end;

    header_          = function ()  return DBF_HEADER end;
    fields_          = function ()  return DBF_FIELDS end;
  }

  if file_or_fileName then return self.open(file_or_fileName) end

  return self
end

----------------------------------------------------------------------
-- Usage
----------------------------------------------------------------------

local function dump_dbf(fname)
  local f = io.open(fname,"rb")

  local headers = dbf_read_header(f)
  local fields  = dbf_read_fields(f)
  local records = dbf_read_all_records(f, headers, fields)

  local pp = require "pp"
  pp(headers)
  pp(fields)
  pp(records)
end

local function make_simple_file(fname)
  local f = io.open(fname,"wb")

  -- create empty header
  local header = dbf_create_header(f)

  -- append fields
  local fields = {
    dbf_create_field(f, header, "TYPE",     "C", 12    );
    dbf_create_field(f, header, "ID",       "C", 12    );
    dbf_create_field(f, header, "NAME",     "C", 24    );
    dbf_create_field(f, header, "READONLY", "L"        );
    dbf_create_field(f, header, "CKVAL",    "N", 6,  2 );
    dbf_create_field(f, header, "KOL",      "N", 10, 0 );
    dbf_create_field(f, header, "UPDATED",  "D"        );
  }

  -- end of fields description
  dbf_write_field_info(f) 

  -- insert record
  dbf_write_record_n(f, {"PREF2.5", "COLORSET", "#TLa­L# #­aZ­ - DOS", 'T', 577.20, 8, '2000-01-01'}, header, fields, 1)

  -- Update header (new date and record count)
  header.last_update={y=2016, m=8, d=22}
  dbf_write_header(f, header)

  -- Mark EOF and close
  dbf_write_eof(f, header, fields)
  f:close()
end

make_simple_file("sample_.dbf")

dump_dbf("sample_.dbf")

Hello, Alexey.

I am interested in using your utility.

My scenario is:

Win7 (and/or Win8x and Win10) and Linux
Lua v5.3.xx preferably (or Lua v5.1 or Lua v5.2, both for testing)
several versions of the DBF structure.

Can you give me some instructions for me to start?

Does dbf.lua has any dependency? Does it need any external DLLs for being able to run?

Thanks.

By.

HERNAN CANO M