Skip to content

Instantly share code, notes, and snippets.

@cashiwamochi
Last active September 23, 2018 15:27
Show Gist options
  • Save cashiwamochi/7a938761c37c8493f2f7afe9f3b45d34 to your computer and use it in GitHub Desktop.
Save cashiwamochi/7a938761c37c8493f2f7afe9f3b45d34 to your computer and use it in GitHub Desktop.
This nim source converts binary of cifar10 to PNG. Some packages are needed, if you use this, please install them for yourself.
import os
import streams
import sequtils
import nimPNG, math, tables, base64
import private.buffer
import progress
const
height: int = 32
width: int = 32
type
cifarFileNameArray = array[0..5, string]
patchData = object
id: uint8
seq_r_values: seq[uint8]
seq_g_values: seq[uint8]
seq_b_values: seq[uint8]
Image = ref object
data: string
width, height: int
colorType: PNGcolorType
bitDepth: int
patchPngData = object
id: uint8
image: Image
proc checkCifarExist(cifar_file_array: cifarFileNameArray) : bool =
for file_name in cifar_file_array:
if not existsFile(file_name):
return false
return true
proc loadCifar( cifar_file: string ): seq[patchData] =
echo "|| ", cifar_file, " is being loaded ... "
result = newSeq[patchData](10000)
const num_data_in_row = 3073
const max_row_num = 10000
var row_idx : int = 0
var col_idx : int = 0
var fs = newFileStream(cifar_file, fmRead)
var id : uint8
var b = newSeq[uint8](1024)
var g = newSeq[uint8](1024)
var r = newSeq[uint8](1024)
if not isNil(fs):
while not fs.atEnd:
let value : uint8 = fs.readUint8()
if col_idx == 0:
id = value
elif col_idx <= 1024:
b[col_idx-1] = value
elif col_idx <= 2048:
g[col_idx-1-1024] = value
elif col_idx <= 3072:
r[col_idx-1-1024-1024] = value
else:
echo "|| [ERROR] Ploblem in parsing"
break
col_idx = col_idx + 1
if col_idx == num_data_in_row:
col_idx = 0
result[row_idx] = patchData(id: id, seq_b_values: b, seq_g_values: g, seq_r_values: r)
row_idx = row_idx + 1
fs.close()
assert row_idx == max_row_num
proc encodePNG(data: seq[patchData]) : seq[patchPngData] =
result = newSeq[patchPngData](len(data))
for i in 0..len(data)-1:
var image: Image
new(image)
image.width = width
image.height = height
image.colorType = LCT_RGBA
image.bitDepth = 8
image.data = newString(width * height * 4)
for v in 0 .. height-1:
for u in 0 .. width-1:
image.data[v*width*4 + u*4 + 0] = chr(data[i].seq_b_values[v*width + u])
image.data[v*width*4 + u*4 + 1] = chr(data[i].seq_g_values[v*width + u])
image.data[v*width*4 + u*4 + 2] = chr(data[i].seq_r_values[v*width + u])
image.data[v*width*4 + u*4 + 3] = chr(255)
result[i].image = image
result[i].id = data[i].id
# main
echo "||================================= Cifar-10 (Nim) ================================= "
let
root_path : string = os.getAppDir()
var
cifar_files: cifarFileNameArray
seq_patch_data : seq[patchData] # -> nil
seq_patch_data = @[] # necessary initialization
cifar_files = [root_path&"/cifar10/data_batch_1.bin",
root_path&"/cifar10/data_batch_2.bin",
root_path&"/cifar10/data_batch_3.bin",
root_path&"/cifar10/data_batch_4.bin",
root_path&"/cifar10/data_batch_5.bin",
root_path&"/cifar10/test_batch.bin"]
if not checkCifarExist(cifar_files):
quit(1)
createDir(root_path & "/png/train")
createDir(root_path & "/png/test")
for i in 0 .. 9:
createDir(root_path & "/png/train/" & intToStr(i,1))
createDir(root_path & "/png/test/" & intToStr(i,1))
var train_data = newSeq[patchData](50000)
for i in 1 .. 5:
train_data[(i-1)*10000 .. (i)*10000 - 1] = loadCifar(cifar_files[i-1])
var test_png_data = encodePNG(loadCifar(cifar_files[len(cifar_files)-1]))
var train_png_data = encodePNG(train_data)
var bar = newProgressBar(total = 60000)
bar.start()
for idx, png in train_png_data:
discard savePNG32(root_path & "/png/train/" & intToStr(int(png.id)) & "/" & intToStr(idx) & ".png",
png.image.data, png.image.width, png.image.height)
bar.increment()
for idx, png in test_png_data:
discard savePNG32(root_path & "/png/test/" & intToStr(int(png.id)) & "/" & intToStr(idx) & ".png",
png.image.data, png.image.width, png.image.height)
bar.increment()
bar.finish()
echo "|| Converting has been done ... "
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment