Trying out FlexBuffers to encode a log file in efficient way

LogTest.swift

//
//  main.swift
//  LogTest
//
//  Created by Maxim Zaks on 21.03.17.
//  Copyright © 2017 Maxim Zaks. All rights reserved.
//

import Foundation



func toIP(text : String) -> [UInt]{
    if text.contains(".") {
        return [192, 168, 0, 1]
    } else {
        return [0xfe, 0x80, 0x21, 0x9, 0xe3, 0xff, 0xfe, 0xe7, 0x5d, 0x23]
    }
}

func protoFrom(text : String) -> UInt {
    switch text {
    case "tcp":
        return 1
    case "upd":
        return 2
    default:
        return 0
    }
}

func serviceFrom(text : String) -> UInt {
    switch text {
    case "dns":
        return 1
    case "http":
        return 2
    case "ssl":
        return 3
    default:
        return 0
    }
}

func tableString(text : String) -> String? {
    switch text {
    case "(empty)":
        return nil
    default:
        return text
    }
}

func process () {
    
    let path = "/Users/mzaks/dev/PlayGrounds/LogTest/LogTest/log.txt"
        do {
            let binaryData = try Data(contentsOf: URL(fileURLWithPath: path))
            print("Text data lenght: \(binaryData.count)")
        
            let data = try String(contentsOfFile: path, encoding: .utf8)
            let rows = data.components(separatedBy: .newlines)
            let flx = FlexBuffer()
            flx.addVector {
                for row in rows {
                    if row.characters.count < 4 {
                        continue
                    }
                    flx.addMap {
                        let values = row.components(separatedBy: .whitespaces)
                        let time = Double(values[0]) ?? 0
                        flx.add(key: "time", value: time)
                        let uuid = values[1]
                        flx.add(key: "uuid", stringValue: uuid)
                        let originIP = toIP(text: values[2])
                        flx.add(key: "originIP", value: originIP)
                        let originS = UInt(values[3]) ?? 0
                        flx.add(key: "originS", value: originS)
                        let respIP = toIP(text: values[4])
                        flx.add(key: "respIP", value: respIP)
                        let respS = UInt(values[5]) ?? 0
                        flx.add(key: "respS", value: respS)
                        let proto = protoFrom(text:values[6])
                        flx.add(key: "proto", value: proto)
//                        let service = values[7]
//                        flx.add(key: "service", stringValue: service)
                        let service = serviceFrom(text: values[7])
                        flx.add(key: "service", value: service)
                        let duration = Double(values[8]) ?? 0
                        flx.add(key: "duration", value: duration)
                        let orig_bytes = Int(values[9]) ?? 0
                        flx.add(key: "orig_bytes", value: orig_bytes)
                        let resp_bytes = Int(values[10]) ?? 0
                        flx.add(key: "resp_bytes", value: resp_bytes)
                        let conn_state = values[11]
                        flx.add(key: "conn_state", stringValue: conn_state)
                        let local_orig = values[12] == "0" ? false : true
                        flx.add(key: "local_orig", value: local_orig)
                        let missed_bytes = Int(values[13]) ?? 0
                        flx.add(key: "missed_bytes", value: missed_bytes)
                        let history = values[14]
                        flx.add(key: "history", stringValue: history)
                        let orig_pkts = Int(values[15]) ?? 0
                        flx.add(key: "orig_pkts", value: orig_pkts)
                        let orig_ip_bytes = Int(values[16]) ?? 0
                        flx.add(key: "orig_ip_bytes", value: orig_ip_bytes)
                        let resp_pkts = Int(values[17]) ?? 0
                        flx.add(key: "resp_pkts", value: resp_pkts)
                        let resp_ip_bytes = Int(values[18]) ?? 0
                        flx.add(key: "resp_ip_bytes", value: resp_ip_bytes)
                        if let tunnel_parents = tableString(text:values[19]) {
                            flx.add(key: "tunnel_parents", stringValue: tunnel_parents)
                        } else {
                            flx.add(key: "tunnel_parents", value: "") // TODO add (add nil)
                        }
                    }
                }
            }
            let flxData = flx.finish()
            print("FLX data lenght: \(flxData.count)")
            try flxData.write(to: URL(fileURLWithPath: "/Users/mzaks/dev/PlayGrounds/LogTest/LogTest/log_vect_of_dict.flx"))
            
            let map = FlexBuffer.decode(data: flxData)?.asVector?[1]?.asMap
            print(map!.debugDescription)
            
            
        } catch {
            print(error)
        }
}

func process2 () {
    
    let path = "/Users/mzaks/dev/PlayGrounds/LogTest/LogTest/log.txt"
    do {
        let binaryData = try Data(contentsOf: URL(fileURLWithPath: path))
        print("Text data lenght: \(binaryData.count)")
        
        let data = try String(contentsOfFile: path, encoding: .utf8)
        let rows = data.components(separatedBy: .newlines)
        let flx = FlexBuffer()
        
        var time_s : [Double] = []
        var uuid_s : [String] = []
        var originIP_s : [[UInt]] = []
        var originS_s : [UInt] = []
        var respIP_s : [[UInt]] = []
        var respS_s : [UInt] = []
        var proto_s : [UInt] = []
//        var service_s : [String] = []
        var service_s : [UInt] = []
        var duration_s : [Double] = []
        var orig_bytes_s : [Int] = []
        var resp_bytes_s : [Int] = []
        var conn_state_s : [String] = []
        var local_orig_s : [Bool] = []
        var missed_bytes_s : [Int] = []
        var history_s : [String] = []
        var orig_pkts_s : [Int] = []
        var orig_ip_bytes_s : [Int] = []
        var resp_pkts_s : [Int] = []
        var resp_ip_bytes_s : [Int] = []
        var tunnel_parents_s : [String] = []
            
            for row in rows {
                if row.characters.count < 4 {
                    continue
                }
                
                    let values = row.components(separatedBy: .whitespaces)
                    let time = Double(values[0]) ?? 0
                    time_s.append(time)
                    let uuid = values[1]
                    uuid_s.append(uuid)
                    let originIP = toIP(text: values[2])
                    originIP_s.append(originIP)
                    let originS = UInt(values[3]) ?? 0
                    originS_s.append(originS)
                    let respIP = toIP(text: values[4])
                    respIP_s.append(respIP)
                    let respS = UInt(values[5]) ?? 0
                    respS_s.append(respS)
                    let proto = protoFrom(text:values[6])
                    proto_s.append(proto)
//                    let service = values[7]
//                    service_s.append(service)
                    let service = serviceFrom(text: values[7])
                    service_s.append(service)
                    let duration = Double(values[8]) ?? 0
                    duration_s.append(duration)
                    let orig_bytes = Int(values[9]) ?? 0
                    orig_bytes_s.append(orig_bytes)
                    let resp_bytes = Int(values[10]) ?? 0
                    resp_bytes_s.append(resp_bytes)
                    let conn_state = values[11]
                    conn_state_s.append(conn_state)
                    let local_orig = values[12] == "0" ? false : true
                    local_orig_s.append(local_orig)
                    let missed_bytes = Int(values[13]) ?? 0
                    missed_bytes_s.append(missed_bytes)
                    let history = values[14]
                    history_s.append(history)
                    let orig_pkts = Int(values[15]) ?? 0
                    orig_pkts_s.append(orig_pkts)
                    let orig_ip_bytes = Int(values[16]) ?? 0
                    orig_ip_bytes_s.append(orig_ip_bytes)
                    let resp_pkts = Int(values[17]) ?? 0
                    resp_pkts_s.append(resp_pkts)
                    let resp_ip_bytes = Int(values[18]) ?? 0
                    resp_ip_bytes_s.append(resp_ip_bytes)
                    if let tunnel_parents = tableString(text:values[19]) {
                        tunnel_parents_s.append(tunnel_parents)
                    } else {
                        tunnel_parents_s.append("") // TODO add (add nil)
                    }
                
                
            }
        
        flx.addMap {
            flx.add(key: "time", value: time_s)
            flx.add(key: "uuid", value: uuid_s)
            flx.addVector(key: "originIP") {
                for originIP in originIP_s {
                    flx.add(array:originIP)
                }
            }
            flx.add(key: "originS", value: originS_s)
            flx.addVector(key: "respIP") {
                for respIP in respIP_s {
                    flx.add(array:respIP)
                }
            }
            flx.add(key: "respS", value: respS_s)
            flx.add(key: "proto", value: proto_s)
            flx.add(key: "service", value: service_s)
            flx.add(key: "duration", value: duration_s)
            flx.add(key: "orig_bytes", value: orig_bytes_s)
            flx.add(key: "resp_bytes", value: resp_bytes_s)
            flx.add(key: "conn_state", value: conn_state_s)
            flx.add(key: "local_orig", value: local_orig_s)
            flx.add(key: "missed_bytes", value: missed_bytes_s)
            flx.add(key: "history", value: history_s)
            flx.add(key: "orig_pkts", value: orig_pkts_s)
            flx.add(key: "orig_ip_bytes", value: orig_ip_bytes_s)
            flx.add(key: "resp_pkts", value: resp_pkts_s)
            flx.add(key: "resp_ip_bytes", value: resp_ip_bytes_s)
            flx.add(key: "tunnel_parents", value: tunnel_parents_s)
        }
        
        let flxData = flx.finish()
        print("FLX data lenght: \(flxData.count)")
        try flxData.write(to: URL(fileURLWithPath: "/Users/mzaks/dev/PlayGrounds/LogTest/LogTest/log_dict_of_vect.flx"))
        
        let keys : [StaticString] = ["time", "uuid", "originIP", "originS", "respIP", "respS", "proto", "service", "duration", "orig_bytes", "resp_bytes", "conn_state", "local_orig", "missed_bytes", "history", "orig_pkts", "orig_ip_bytes", "resp_pkts", "resp_ip_bytes", "tunnel_parents"]
        let map = FlexBuffer.decode(data: flxData)?.asMap
        for key in keys {
            let value = map?[key]?.asVector?[1]
            print("\(key): \(value)")
        }
    } catch {
        print(error)
    }
}

print("============ Create FlexBuffer vector of maps ============")
process()
print("============ Create FlexBuffer map of vecors ============")
process2()

/*
 OUTPUT:
 
 ============ Create FlexBuffer vector of maps ============
 
 Text data lenght: 1026256
 FLX data lenght: 2802582
 {"conn_state" : "S0", "duration" : 3.78012, "history" : "D", "local_orig" : 1, "missed_bytes" : 0, "orig_bytes" : 350, "orig_ip_bytes" : 546, "orig_pkts" : 7, "originIP" : [192, 168, 0, 1], "originS" : 137, "proto" : 0, "respIP" : [192, 168, 0, 1], "respS" : 137, "resp_bytes" : 0, "resp_ip_bytes" : 0, "resp_pkts" : 0, "service" : 1, "time" : 1.25853e+09, "tunnel_parents" : "", "uuid" : "nkCxlvNN8pi"}
 
 ============ Create FlexBuffer map of vecors ============
 
 Text data lenght: 1026256
 FLX data lenght: 990679
 time: Optional(1.25853e+09)
 uuid: Optional("nkCxlvNN8pi")
 originIP: Optional([192, 168, 0, 1])
 originS: Optional(137)
 respIP: Optional([192, 168, 0, 1])
 respS: Optional(137)
 proto: Optional(0)
 service: Optional(1)
 duration: Optional(3.78012)
 orig_bytes: Optional(350)
 resp_bytes: Optional(0)
 conn_state: Optional("S0")
 local_orig: Optional(1)
 missed_bytes: Optional(0)
 history: Optional("D")
 orig_pkts: Optional(7)
 orig_ip_bytes: Optional(546)
 resp_pkts: Optional(0)
 resp_ip_bytes: Optional(0)
 tunnel_parents: Optional("")
 
 
*/