Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Basic IO for dbf file format.
local printf = function(...) return print(string.format(...)) end
local function prequire(...)
local ok, mod = pcall(require, ...)
if ok then return mod end
return nil, mod
end
local function iif(cond, val1, val2)
if cond then return val1 end return val2
end
local function get_arg(...)
return select('#',...), {...}
end
local function coalesce(...)
local argc, argv = get_arg(...)
for i = 1, argc do
if argv[i] ~= nil then
return argv[i]
end
end
end
local function round_str(x, digits)
return assert(string.format("%."..(digits or '0').."f",x))
end;
local function fitstrr(str, ch, width)
if #str >= width then
return str:sub(1,width)
end
return str .. string.rep(ch,width-#str)
end
local function fitstrl(str, ch, width)
if #str >= width then
return str:sub(-width)
end
return string.rep(ch,width-#str) .. str
end
local function byte_concat(t)
local res = ''
for _,v in ipairs(t) do
res = res .. string.char(v)
end
return res
end
local function at(s,i)
return string.sub(s,i,i)
end
local function byte_at(s,i)
return string.byte(at(s,i))
end
local function toint(str)
return byte_at(str,4) * (256*256*256) +
byte_at(str,3) * (256*256) +
byte_at(str,2) * 256 +
byte_at(str,1)
end
local function toshort(str)
return byte_at(str,2) * 256 +
byte_at(str,1)
end
local function intto(v)
local str = ''
str = str .. string.char(math.mod(v,256))
v = math.floor(v/256)
str = str .. string.char(math.mod(v,256))
v = math.floor(v/256)
str = str .. string.char(math.mod(v,256))
v = math.floor(v/256)
str = str .. string.char(math.mod(v,256))
return str
end
local function shortto(v)
local str = ''
str = str .. string.char(math.mod(v,256))
v = math.floor(v/256)
str = str .. string.char(math.mod(v,256))
return str
end
----------------------------------------------------------------------
-- File IO
----------------------------------------------------------------------
local function read_byte(f)
return string.byte(f:read(1))
end
local function read_int(f)
return toint(f:read(4))
end
local function read_short(f)
return toshort( f:read(2) )
end
local function read_char(f)
return f:read(1)
end
local function read_str(f,len)
return string.gsub(f:read(len),'%z.*$','')
end
local function read_n(f,n)
return f:read(n)
end
local function read_date(f)
return {y=1900+read_byte(f),m=read_byte(f),d=read_byte(f)};
end
local function write_byte(f,v)
return f:write(string.char(v))
end
local function write_int(f,v)
return f:write(intto(v))
end
local function write_short(f,v)
return f:write(shortto(v))
end
local function write_char(f,v)
return f:write(at(v,1))
end
local function write_n(f,str,len,ch)
return f:write(fitstrr(str, ch or '\0', len))
end
local function write_str(f,str)
return f:write(str)
end
local function write_str_n(f,str,len)
return write_n(f,str,len,'\000')
end
local function write_date(f,v)
write_byte(f, math.mod((v.y - 1900),256))
write_byte(f, v.m)
return write_byte(f, v.d)
end
local function set_pos(f, pos)
return f:seek("set", pos)
end
local function get_pos(f)
return f:seek()
end
local function file_size(f)
local pos = get_pos(f)
local size = f:seek('end')
set_pos(f,pos);
return size
end
----------------------------------------------------------------------
-- DBF Low Level
----------------------------------------------------------------------
local DBF_HEADER_SIZE = 32
local DBF_FIELD_REC_SIZE = 32
local DBF_EOR = 0x0D
local DBF_EOF = 0x1A
local DBF_ID = {
[0x03] = "FoxBASE+/dBASE III +";
[0x83] = "FoxBASE+/dBASE III +, memo";
[0x03] = "FoxPro/dBASE IV";
[0xF5] = "FoxPro, memo";
[0x8B] = "dBASE IV, memo";
}
local function dbf_read_header(f)
--[==========================================================[
¦ Байты : Описание ¦
¦==========================================================¦
¦ 00 : Типы файлов с данными: ¦
¦ : * FoxBASE+/dBASE III +, без memo - 0х03 ¦
¦ : * FoxBASE+/dBASE III +, с memo - 0х83 ¦
¦ : * FoxPro/dBASE IV, без memo - 0х03 ¦
¦ : * FoxPro с memo - 0хF5 ¦
¦ : * dBASE IV с memo - 0x8B ¦
¦----------------------------------------------------------¦
¦ 01-03 : Последнее изменение (ГГММДД) ¦
¦----------------------------------------------------------¦
¦ 04-07 : Число записей в файле ¦
¦----------------------------------------------------------¦
¦ 08-09 : Положение первой записи с данными ¦
¦----------------------------------------------------------¦
¦ 10-11 : Длина одной записи с данными (включая признак ¦
¦ : удаления) ¦
¦----------------------------------------------------------¦
¦ 12-27 : Зарезервированы ¦
¦----------------------------------------------------------¦
¦ 28 : 1-есть структ.составной инд.файл (типа .CDX) ¦
¦ : 0-нет ¦
¦----------------------------------------------------------¦
¦ 29-31 : Зарезервированы ¦
¦----------------------------------------------------------¦
¦ 32-n : Подзаписи полей** ¦
¦----------------------------------------------------------¦
¦ n+1 : Признак завершения записи заголовка (0х0D) ¦
--]==========================================================]
set_pos(f, 0)
local DBF_HEAD = {
dbf_id = assert( read_byte (f) );
last_update = assert( read_date (f) );
last_rec = assert( read_int (f) );
data_offset = assert( read_short(f) );
rec_size = assert( read_short(f) );
reserved1 = assert( read_n (f,16) );
flag_ext = assert( read_byte (f) );
reserved2 = assert( read_n (f,3) );
}
return DBF_HEAD
end
local function dbf_write_header(f, HEADER)
set_pos(f, 0)
assert(write_byte (f,HEADER.dbf_id) )
assert(write_date (f,HEADER.last_update) )
assert(write_int (f,HEADER.last_rec) )
assert(write_short (f,HEADER.data_offset) )
assert(write_short (f,HEADER.rec_size) )
assert(write_n (f,HEADER.reserved1,16) )
assert(write_byte (f,HEADER.flag_ext) )
assert(write_n (f,HEADER.reserved2,3) )
return DBF_HEADER_SIZE
end
local function dbf_get_reader(FIELD_REC)
local readers = {
["C"] = function(f) return read_str (f, FIELD_REC.len_info.char_len) end;
["L"] = function(f) return read_char(f) end;
["N"] = function(f) return tonumber((read_str(f, FIELD_REC.len_info.num_size._len))) end;
["D"] = function(f) return read_str(f, 4) .. '-' .. read_str(f, 2) .. '-' .. read_str(f, 2) end;
["B"] = function(f) return read_n(f, 10) end;
}
return assert(readers[FIELD_REC.field_type], "UNSUPPORTED FIELD TYPE '" .. FIELD_REC.field_type .. "'")
end
local function dbf_get_writer(FIELD_REC)
local writers = {
["C"] = function(f, v) return write_str_n(f, v, FIELD_REC.len_info.char_len) end;
["L"] = function(f, v) return write_char(f, v) end;
["N"] = function(f, v)
local width = FIELD_REC.len_info.num_size._len
local str = round_str(v, FIELD_REC.len_info.num_size._dec)
return write_str(f, fitstrl(str, ' ', width))
end;
["D"] = function(f, v)
write_str(f, v:sub(1,4))
write_str(f, v:sub(6,7))
return write_str(f, v:sub(9,10))
end;
["B"] = function(f, v) return write_n(f, v, 10) end;
}
return assert(writers[FIELD_REC.field_type], "UNSUPPORTED FIELD TYPE '" .. FIELD_REC.field_type .. "'")
end
local function dbf_read_field_info(f)
local FIELD_REC
repeat
local n = assert(read_byte(f))
if DBF_EOR == n then break end
assert(n ~= 0)
FIELD_REC = {
field_name = string.char(n) .. assert(read_str(f,10));
field_type = assert(read_char(f));
offset = assert(read_int(f, 4));
len_info = {
num_size = {
_len = assert(read_byte(f));
_dec = assert(read_byte(f));
};
};
flags = assert(read_byte(f));
reserved = read_n(f, 13);
};
if FIELD_REC.field_type == 'C' then
FIELD_REC.len_info.char_len = toshort( string.char(FIELD_REC.len_info.num_size._len) .. string.char(FIELD_REC.len_info.num_size._dec) )
else
FIELD_REC.len_info.char_len = FIELD_REC.len_info.num_size._len
end
FIELD_REC.reader = dbf_get_reader(FIELD_REC)
FIELD_REC.writer = dbf_get_writer(FIELD_REC)
until true;
return FIELD_REC
end
local function dbf_write_field_info(f, FIELD_REC)
if FIELD_REC then
assert(write_n (f, FIELD_REC.field_name, 11) )
assert(write_char (f, FIELD_REC.field_type) )
assert(write_int (f, FIELD_REC.offset) )
assert(write_byte (f, FIELD_REC.len_info.num_size._len) )
assert(write_byte (f, FIELD_REC.len_info.num_size._dec) )
assert(write_byte (f, FIELD_REC.flags) )
assert(write_n (f, FIELD_REC.reserved, 13) )
return DBF_FIELD_REC_SIZE
end
write_byte(f, DBF_EOR)
return 1
end
local function dbf_read_fields(f)
set_pos(f, DBF_HEADER_SIZE)
local DBF_FIELDS = {}
while true do
local FIELD_REC = dbf_read_field_info(f)
if not FIELD_REC then break end
DBF_FIELDS[#DBF_FIELDS + 1] = FIELD_REC
end
return DBF_FIELDS
end
local function dbf_write_fields(f, DBF_FIELDS)
set_pos(f, DBF_HEADER_SIZE)
local i = 1
repeat
local n = dbf_write_field_info(f, DBF_FIELDS[i])
i = i + 1
until n == 1
return i - 1
end
local function dbf_set_pos(f, HEADER, FIELDS, n)
-- assert(HEADER.last_rec >= n)
set_pos(f, HEADER.data_offset + HEADER.rec_size * (n-1))
end
local function dbf_get_pos(f, HEADER, FIELDS)
local pos = 1 + math.floor( (get_pos(f) - HEADER.data_offset) / HEADER.rec_size )
return pos
end
local function dbf_read_record(f, HEADER, FIELDS)
local record = {}
record[0] = read_byte(f)
for _,field in ipairs(FIELDS) do
local val = field.reader(f)
record[#record + 1] = val
end
return record
end
local function dbf_read_all_records(f, HEADER, FIELDS)
local records = {}
dbf_set_pos(f, HEADER, FIELDS, 1)
for i = 1, HEADER.last_rec do
records[#records + 1] = dbf_read_record(f, HEADER, FIELDS)
end
return records
end;
local function dbf_read_record_n(f, HEADER, FIELDS, n)
dbf_set_pos(f, HEADER, FIELDS, n)
return dbf_read_record(f, HEADER, FIELDS)
end;
local function dbf_write_record(f, record, HEADER, FIELDS)
-- assert(dbf_get_pos(f, HEADER, FIELDS) ~= -1)
-- assert(dbf_get_pos(f, HEADER, FIELDS) ~= -1)
-- 0x20 - valid record
-- 0x2A - deleted record
local n = dbf_get_pos(f, HEADER, FIELDS)
assert(write_byte(f, record[0] or 0x20))
for i,field in ipairs(FIELDS) do
assert(field.writer(f, record[i]))
end
if HEADER.last_rec < n then
HEADER.last_rec = n
end
return true
end
local function dbf_write_record_n(f, record, HEADER, FIELDS, n)
dbf_set_pos(f, HEADER, FIELDS, n)
return dbf_write_record(f, record, HEADER, FIELDS)
end
local function dbf_append_record(f, record, HEADER, FIELDS)
HEADER.last_rec = HEADER.last_rec + 1
return dbf_write_record_n(f, record, HEADER, FIELDS, HEADER.last_rec)
end
local function dbf_write_all_records(f, records, HEADER, FIELDS )
dbf_set_pos(f, HEADER, FIELDS, 1)
for i, record in ipairs(records) do
dbf_write_record(f, record, HEADER, FIELDS)
end
return true
end;
local function dbf_write_eof(f, HEADER, FIELDS)
dbf_set_pos(f, HEADER, FIELDS, HEADER.last_rec + 1)
write_byte(f, DBF_EOF)
end
----------------------------------------------------------------------
local function dbf_get_field_size(FIELD_REC)
if FIELD_REC.field_type == "C" then
return FIELD_REC.len_info.char_len
elseif FIELD_REC.field_type == "L" then
assert (FIELD_REC.len_info.char_len == 1)
return 1
elseif FIELD_REC.field_type == "N" then
return FIELD_REC.len_info.num_size._len
elseif FIELD_REC.field_type == "D" then
assert (FIELD_REC.len_info.char_len == 8)
return 8
else
assert(false, "UNSUPPORTED FIELD TYPE '" .. FIELD_REC.field_type .. "'")
end
end
local function dbf_set_field_size(FIELD_REC, s1, s2)
if FIELD_REC.field_type == "C" then
assert(s1)
assert(s2 == nil)
FIELD_REC.len_info.char_len = s1
local len = shortto(FIELD_REC.len_info.char_len)
FIELD_REC.len_info.num_size._len = byte_at(len,1)
FIELD_REC.len_info.num_size._dec = byte_at(len,2)
elseif FIELD_REC.field_type == "L" then
assert(s1==nil)
assert(s2==nil)
FIELD_REC.len_info.char_len = 1
FIELD_REC.len_info.num_size._len = 1
FIELD_REC.len_info.num_size._dec = 0
elseif FIELD_REC.field_type == "N" then
FIELD_REC.len_info.num_size._len = assert(s1)
FIELD_REC.len_info.num_size._dec = coalesce(s2,0)
-- FIELD_REC.len_info.char_len = toshort( string.char(FIELD_REC.len_info.num_size._dec) .. string.char(FIELD_REC.len_info.num_size._len))
FIELD_REC.len_info.char_len = toshort( string.char(FIELD_REC.len_info.num_size._dec) .. string.char(FIELD_REC.len_info.num_size._len))
elseif FIELD_REC.field_type == "D" then
assert(s1==nil)
assert(s2==nil)
FIELD_REC.len_info.char_len = 8
FIELD_REC.len_info.num_size._len = 8
FIELD_REC.len_info.num_size._dec = 0
else
assert(false, "UNSUPPORTED FIELD TYPE '" .. FIELD_REC.field_type .. "'")
end
end
local function dbf_create_header(f)
local HEADER = {
dbf_id = 0x03;
last_update = {y=1900; d=0; m=0};
last_rec = 0;
data_offset = DBF_HEADER_SIZE;
rec_size = 1;
reserved1 = byte_concat{0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00};
flag_ext = 0;
reserved2 = byte_concat{0x65,0x00,0x00};
};
dbf_write_header(f,HEADER)
return HEADER
end
local function dbf_create_field(f, HEADER, fieldName, fieldType, fieldSize1, fieldSize2)
local FIELD_REC = {
field_name = fieldName:sub(1,11);
field_type = fieldType:sub(1,1);
offset = HEADER.rec_size;
len_info = {
char_len = 0;
num_size = {
_len = 0;
_dec = 0;
};
};
flags = 0x00;
reserved = byte_concat{0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00};
};
dbf_set_field_size(FIELD_REC, fieldSize1, fieldSize2)
FIELD_REC.reader = dbf_get_reader(FIELD_REC)
FIELD_REC.writer = dbf_get_writer(FIELD_REC)
HEADER.rec_size = HEADER.rec_size + dbf_get_field_size(FIELD_REC)
dbf_write_field_info(f, FIELD_REC)
HEADER.data_offset = get_pos(f) + 1
return FIELD_REC
end
----------------------------------------------------------------------
-- DBF class
----------------------------------------------------------------------
local BIG_FILE_SIZE = 20 * 1024^3
local function NewDBF(file_or_fileName)
local real_f, mem_f, f
local DBF_HEADER, DBF_FIELDS
local self
self = {
open = function (file_or_fileName)
self.close()
if type(file_or_fileName) == 'string' then
local err real_f, err = io.open(file_or_fileName,'rb')
if not real_f then return nil, err end
f = real_f
if file_size(f) < BIG_FILE_SIZE then
local memfile = prequire "memoryfile"
if memfile then
mem_f = memfile.open(f:read("*all"),'rb')
if mem_f then f = mem_f end
end
end
else
real_f = file_or_fileName
f = real_f
end
DBF_HEADER = dbf_read_header(f)
DBF_FIELDS = dbf_read_fields(f)
return self
end;
close = function ()
if mem_f then mem_f:close() end
if real_f then real_f:close() end
real_f, mem_f, f = nil
DBF_HEADER, DBF_FIELDS = nil
end;
read_record = function () return dbf_read_record(f, DBF_HEADER, DBF_FIELDS) end;
read_record_n = function (n) return dbf_read_record_n(f, DBF_HEADER, DBF_FIELDS, n) end;
read_all_records = function () return dbf_read_all_records(f, DBF_HEADER, DBF_FIELDS) end;
set_pos = function (n) return dbf_set_pos(f, DBF_HEADER, DBF_FIELDS, n) end;
get_pos = function () return dbf_get_pos(f, DBF_HEADER, DBF_FIELDS) end;
count = function () return DBF_HEADER.last_rec end;
record_size = function () return DBF_HEADER.rec_size end;
write_record = function (rec) return dbf_write_record(f, rec, DBF_HEADER, DBF_FIELDS) end;
header_ = function () return DBF_HEADER end;
fields_ = function () return DBF_FIELDS end;
}
if file_or_fileName then return self.open(file_or_fileName) end
return self
end
----------------------------------------------------------------------
-- Usage
----------------------------------------------------------------------
local function dump_dbf(fname)
local f = io.open(fname,"rb")
local headers = dbf_read_header(f)
local fields = dbf_read_fields(f)
local records = dbf_read_all_records(f, headers, fields)
local pp = require "pp"
pp(headers)
pp(fields)
pp(records)
end
local function make_simple_file(fname)
local f = io.open(fname,"wb")
-- create empty header
local header = dbf_create_header(f)
-- append fields
local fields = {
dbf_create_field(f, header, "TYPE", "C", 12 );
dbf_create_field(f, header, "ID", "C", 12 );
dbf_create_field(f, header, "NAME", "C", 24 );
dbf_create_field(f, header, "READONLY", "L" );
dbf_create_field(f, header, "CKVAL", "N", 6, 2 );
dbf_create_field(f, header, "KOL", "N", 10, 0 );
dbf_create_field(f, header, "UPDATED", "D" );
}
-- end of fields description
dbf_write_field_info(f)
-- insert record
dbf_write_record_n(f, {"PREF2.5", "COLORSET", "#TLa­L# #­aZ­ - DOS", 'T', 577.20, 8, '2000-01-01'}, header, fields, 1)
-- Update header (new date and record count)
header.last_update={y=2016, m=8, d=22}
dbf_write_header(f, header)
-- Mark EOF and close
dbf_write_eof(f, header, fields)
f:close()
end
make_simple_file("sample_.dbf")
dump_dbf("sample_.dbf")
@jhernancanom

This comment has been minimized.

Copy link

commented Feb 9, 2017

Hello, Alexey.

I am interested in using your utility.

My scenario is:

Win7 (and/or Win8x and Win10) and Linux
Lua v5.3.xx preferably (or Lua v5.1 or Lua v5.2, both for testing)
several versions of the DBF structure.

Can you give me some instructions for me to start?

Does dbf.lua has any dependency? Does it need any external DLLs for being able to run?

Thanks.

By.

HERNAN CANO M

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.