Skip to content

Instantly share code, notes, and snippets.

@krishnasrinivas

krishnasrinivas/start-nkv

Created February 22, 2019 18:39
Show Gist options
  • Save krishnasrinivas/11bc8f4b6ef58f26ef8515b51545fa59 to your computer and use it in GitHub Desktop.
Save krishnasrinivas/11bc8f4b6ef58f26ef8515b51545fa59 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
start-nkv
[root@colo-client08 minio]# export MINIO_NKV_CONFIG=/home/minio/nkv_config.json
[root@colo-client08 minio]# export MINIO_ACCESS_KEY=minio
[root@colo-client08 minio]# export MINIO_SECRET_KEY=minio123
[root@colo-client08 minio]# export LD_LIBRARY_PATH=/home/minio/nkv_minio_package/lib
[root@colo-client08 minio]# cat nkv_config.json
{
"fm_address": "10.1.20.91",
"contact_fm": 0,
"nkv_transport" : 0,
"min_container_required" : 1,
"min_container_path_required" : 1,
"nkv_container_path_qd" : 64,
"nkv_mounts": [
{
"mount_point": "/dev/nvme0n1",
"remote_nqn_name": "nqn-02",
"remote_target_node_name": "msl-ssg-sk01",
"nqn_transport_address": "101.100.10.31",
"nqn_transport_port": 1023,
"numa_node_attached" : 0
},
{
"mount_point": "/dev/nvme1n1",
"remote_nqn_name": "nqn-02",
"remote_target_node_name": "msl-ssg-sk01",
"nqn_transport_address": "102.100.10.31",
"nqn_transport_port": 1023,
"numa_node_attached" : 0
},
{
"mount_point": "/dev/nvme2n1",
"remote_nqn_name": "nqn-02",
"remote_target_node_name": "msl-ssg-sk01",
"nqn_transport_address": "103.100.10.31",
"nqn_transport_port": 1023,
"numa_node_attached" : 0
},
{
"mount_point": "/dev/nvme3n1",
"remote_nqn_name": "nqn-02",
"remote_target_node_name": "msl-ssg-sk01",
"nqn_transport_address": "104.100.10.31",
"nqn_transport_port": 1023,
"numa_node_attached" : 0
}
],
"subsystem_maps": [
{
"target_server_name": "msl-ssg-sk01",
"subsystem_nqn_id": "dce20d46",
"subsystem_nqn": "nqn-02",
"subsystem_nqn_nsid": 1,
"subsystem_transport": [
{
"subsystem_type": 1,
"subsystem_address": "101.100.10.31",
"subsystem_port": 1023,
"subsystem_addr_fam": 2,
"subsystem_interface_speed": 3,
"subsystem_interface_numa_aligned": false,
"subsystem_interface_status":1
},
{
"subsystem_type": 1,
"subsystem_address": "102.100.10.31",
"subsystem_port": 1023,
"subsystem_addr_fam": 2,
"subsystem_interface_speed": 3,
"subsystem_interface_numa_aligned": false,
"subsystem_interface_status":1
},
{
"subsystem_type": 1,
"subsystem_address": "103.100.10.31",
"subsystem_port": 1023,
"subsystem_addr_fam": 2,
"subsystem_interface_speed": 3,
"subsystem_interface_numa_aligned": false,
"subsystem_interface_status":1
},
{
"subsystem_type": 1,
"subsystem_address": "104.100.10.31",
"subsystem_port": 1023,
"subsystem_addr_fam": 2,
"subsystem_interface_speed": 3,
"subsystem_interface_numa_aligned": false,
"subsystem_interface_status":1
}
],
"subsystem_status": 0,
"subsystem_space_avail_percent": 87
}
]
}
[root@colo-client08 minio]# ./minio server /ip/101.100.10.31 /ip/102.100.10.31 /ip/103.100.10.31 /ip/104.100.10.31
2019-02-22 10:34:54,291 [139882001307392] [INFO] NKV config file = /home/minio/nkv_config.json
2019-02-22 10:34:54,292 [139882001307392] [INFO] Generated NKV handle = 4091013546 , NKV instance uuid = 2775618203 for app uuid = minio
2019-02-22 10:34:54,292 [139882001307392] [INFO] contact_fm = 0, nkv_transport = 0, min_container_required = 1, min_container_path_required = 1, nkv_container_path_qd = 64
2019-02-22 10:34:54,292 [139882001307392] [INFO] Adding path, hash = 3303807887, address = 101.100.10.31, port = 1023, fam = 2, speed = 3, status = 1, numa_aligned = 0
2019-02-22 10:34:54,292 [139882001307392] [INFO] Network path added, ip hash = 3303807887, target_container_name = nqn-02, total path so far = 1
2019-02-22 10:34:54,292 [139882001307392] [INFO] Adding path, hash = 1441750628, address = 102.100.10.31, port = 1023, fam = 2, speed = 3, status = 1, numa_aligned = 0
2019-02-22 10:34:54,292 [139882001307392] [INFO] Network path added, ip hash = 1441750628, target_container_name = nqn-02, total path so far = 2
2019-02-22 10:34:54,292 [139882001307392] [INFO] Adding path, hash = 3559271658, address = 103.100.10.31, port = 1023, fam = 2, speed = 3, status = 1, numa_aligned = 0
2019-02-22 10:34:54,292 [139882001307392] [INFO] Network path added, ip hash = 3559271658, target_container_name = nqn-02, total path so far = 3
2019-02-22 10:34:54,292 [139882001307392] [INFO] Adding path, hash = 173163378, address = 104.100.10.31, port = 1023, fam = 2, speed = 3, status = 1, numa_aligned = 0
2019-02-22 10:34:54,292 [139882001307392] [INFO] Network path added, ip hash = 173163378, target_container_name = nqn-02, total path so far = 4
2019-02-22 10:34:54,292 [139882001307392] [INFO] Container added, hash = 993003492, id = 0, uuid = dce20d46, Node name = msl-ssg-sk01 , NQN name = nqn-02 , container count = 1
2019-02-22 10:34:54,292 [139882001307392] [INFO] Adding device path, mount point = /dev/nvme0n1, address = 101.100.10.31, port = 1023, nqn name = nqn-02, target node = msl-ssg-sk01
2019-02-22 10:34:54,292 [139882001307392] [INFO] Adding device path, mount point = /dev/nvme1n1, address = 102.100.10.31, port = 1023, nqn name = nqn-02, target node = msl-ssg-sk01
2019-02-22 10:34:54,293 [139882001307392] [INFO] Adding device path, mount point = /dev/nvme2n1, address = 103.100.10.31, port = 1023, nqn name = nqn-02, target node = msl-ssg-sk01
2019-02-22 10:34:54,293 [139882001307392] [INFO] Adding device path, mount point = /dev/nvme3n1, address = 104.100.10.31, port = 1023, nqn name = nqn-02, target node = msl-ssg-sk01
2019-02-22 10:34:54,293 [139882001307392] [DEBUG] Inspecting target id = 0, target node = msl-ssg-sk01, container name = nqn-02
2019-02-22 10:34:54,293 [139882001307392] [DEBUG] Inspecting path with address = 104.100.10.31 , dev_path = /dev/nvme3n1, port = 1023, status = 1
2019-02-22 10:34:54,293 [139882001307392] [DEBUG] Inspecting path with address = 103.100.10.31 , dev_path = /dev/nvme2n1, port = 1023, status = 1
2019-02-22 10:34:54,293 [139882001307392] [DEBUG] Inspecting path with address = 102.100.10.31 , dev_path = /dev/nvme1n1, port = 1023, status = 1
2019-02-22 10:34:54,293 [139882001307392] [DEBUG] Inspecting path with address = 101.100.10.31 , dev_path = /dev/nvme0n1, port = 1023, status = 1
2019-02-22 10:34:54,293 [139882001307392] [INFO] **NKV transport is NVMe Over TCP via kernel**
2019-02-22 10:34:54,293 [139882001307392] [INFO] Setting environment for open mpdk is successful for app = minio
2019-02-22 10:34:54,293 [139882001307392] [DEBUG] Opening path for target node = msl-ssg-sk01, container name = nqn-02
2019-02-22 10:34:54,293 [139882001307392] [DEBUG] Opening path with address = 104.100.10.31 , dev_path = /dev/nvme3n1, port = 1023, status = 1
2019-02-22 10:34:54,301 [139882001307392] [INFO] ** Path open successful for path = /dev/nvme3n1 **
2019-02-22 10:34:54,302 [139882001307392] [INFO] ** Path container creation successful for path = /dev/nvme3n1, container name = nkv_minio **
2019-02-22 10:34:54,302 [139882001307392] [INFO] ** Path open container successful for path = /dev/nvme3n1, container name = nkv_minio **
2019-02-22 10:34:54,302 [139882001307392] [DEBUG] Opening path with address = 103.100.10.31 , dev_path = /dev/nvme2n1, port = 1023, status = 1
2019-02-22 10:34:54,310 [139882001307392] [INFO] ** Path open successful for path = /dev/nvme2n1 **
2019-02-22 10:34:54,310 [139882001307392] [INFO] ** Path container creation successful for path = /dev/nvme2n1, container name = nkv_minio **
2019-02-22 10:34:54,310 [139882001307392] [INFO] ** Path open container successful for path = /dev/nvme2n1, container name = nkv_minio **
2019-02-22 10:34:54,310 [139882001307392] [DEBUG] Opening path with address = 102.100.10.31 , dev_path = /dev/nvme1n1, port = 1023, status = 1
2019-02-22 10:34:54,318 [139882001307392] [INFO] ** Path open successful for path = /dev/nvme1n1 **
2019-02-22 10:34:54,318 [139882001307392] [INFO] ** Path container creation successful for path = /dev/nvme1n1, container name = nkv_minio **
2019-02-22 10:34:54,318 [139882001307392] [INFO] ** Path open container successful for path = /dev/nvme1n1, container name = nkv_minio **
2019-02-22 10:34:54,318 [139882001307392] [DEBUG] Opening path with address = 101.100.10.31 , dev_path = /dev/nvme0n1, port = 1023, status = 1
2019-02-22 10:34:54,327 [139882001307392] [INFO] ** Path open successful for path = /dev/nvme0n1 **
2019-02-22 10:34:54,327 [139882001307392] [INFO] ** Path container creation successful for path = /dev/nvme0n1, container name = nkv_minio **
2019-02-22 10:34:54,327 [139882001307392] [INFO] ** Path open container successful for path = /dev/nvme0n1, container name = nkv_minio **
2019-02-22 10:34:54,327 [139882001307392] [INFO] NKV open is successful for app = minio
2019-02-22 10:34:54,328 [139882001307392] [INFO] Number of containers populated = 1
Got container list, count = 1
Container Information :: hash = 993003492l, id = 0l, uuid = dce20d46, name = nqn-02, target node = msl-ssg-sk01, status = 0, space available pcnt = 87
Number of Container transport = 4
Transport information :: hash = 173163378, id = 3, address = 104.100.10.31, port = 1023, family = 2, speed = 3, status = 1, numa_node = 0
Transport information :: hash = 3559271658, id = 2, address = 103.100.10.31, port = 1023, family = 2, speed = 3, status = 1, numa_node = 0
Transport information :: hash = 1441750628, id = 1, address = 102.100.10.31, port = 1023, family = 2, speed = 3, status = 1, numa_node = 0
Transport information :: hash = 3303807887, id = 0, address = 101.100.10.31, port = 1023, family = 2, speed = 3, status = 1, numa_node = 0
2019-02-22 10:34:54,328 [139882001307392] [INFO] Number of containers populated = 1
Got container list, count = 1
Container Information :: hash = 993003492l, id = 0l, uuid = dce20d46, name = nqn-02, target node = msl-ssg-sk01, status = 0, space available pcnt = 87
Number of Container transport = 4
Transport information :: hash = 173163378, id = 3, address = 104.100.10.31, port = 1023, family = 2, speed = 3, status = 1, numa_node = 0
Transport information :: hash = 3559271658, id = 2, address = 103.100.10.31, port = 1023, family = 2, speed = 3, status = 1, numa_node = 0
Transport information :: hash = 1441750628, id = 1, address = 102.100.10.31, port = 1023, family = 2, speed = 3, status = 1, numa_node = 0
2019-02-22 10:34:54,328 [139882001307392] [INFO] Number of containers populated = 1
Got container list, count = 1
Container Information :: hash = 993003492l, id = 0l, uuid = dce20d46, name = nqn-02, target node = msl-ssg-sk01, status = 0, space available pcnt = 87
Number of Container transport = 4
Transport information :: hash = 173163378, id = 3, address = 104.100.10.31, port = 1023, family = 2, speed = 3, status = 1, numa_node = 0
Transport information :: hash = 3559271658, id = 2, address = 103.100.10.31, port = 1023, family = 2, speed = 3, status = 1, numa_node = 0
2019-02-22 10:34:54,328 [139882001307392] [INFO] Number of containers populated = 1
Got container list, count = 1
Container Information :: hash = 993003492l, id = 0l, uuid = dce20d46, name = nqn-02, target node = msl-ssg-sk01, status = 0, space available pcnt = 87
Number of Container transport = 4
Transport information :: hash = 173163378, id = 3, address = 104.100.10.31, port = 1023, family = 2, speed = 3, status = 1, numa_node = 0
2019-02-22 10:34:54,330 [139882043270912] [INFO] Sending IO to dev mount = /dev/nvme3n1, container name = nqn-02, target node = msl-ssg-sk01, path ip = 104.100.10.31, path port = 1023
2019-02-22 10:34:54,330 [139882043270912] [INFO] Retrieve option:: decompression = 0, decryption = 0, compare crc = 0, delete = 0
2019-02-22 10:34:54,331 [139882018092800] [INFO] Sending IO to dev mount = /dev/nvme1n1, container name = nqn-02, target node = msl-ssg-sk01, path ip = 102.100.10.31, path port = 1023
2019-02-22 10:34:54,331 [139882018092800] [INFO] Retrieve option:: decompression = 0, decryption = 0, compare crc = 0, delete = 0
2019-02-22 10:34:54,333 [139881749657344] [INFO] Sending IO to dev mount = /dev/nvme0n1, container name = nqn-02, target node = msl-ssg-sk01, path ip = 101.100.10.31, path port = 1023
2019-02-22 10:34:54,333 [139881732871936] [INFO] Sending IO to dev mount = /dev/nvme2n1, container name = nqn-02, target node = msl-ssg-sk01, path ip = 103.100.10.31, path port = 1023
2019-02-22 10:34:54,333 [139881732871936] [INFO] Retrieve option:: decompression = 0, decryption = 0, compare crc = 0, delete = 0
2019-02-22 10:34:54,333 [139881749657344] [INFO] Retrieve option:: decompression = 0, decryption = 0, compare crc = 0, delete = 0
....
......
....
2019-02-22 10:34:55,002 [139881732871936] [INFO] Sending IO to dev mount = /dev/nvme1n1, container name = nqn-02, target node = msl-ssg-sk01, path ip = 102.100.10.31, path port = 1023
2019-02-22 10:34:55,002 [139880373942016] [INFO] Sending IO to dev mount = /dev/nvme2n1, container name = nqn-02, target node = msl-ssg-sk01, path ip = 103.100.10.31, path port = 1023
2019-02-22 10:34:55,002 [139881732871936] [INFO] Retrieve option:: decompression = 0, decryption = 0, compare crc = 0, delete = 0
2019-02-22 10:34:55,002 [139880373942016] [INFO] Retrieve option:: decompression = 0, decryption = 0, compare crc = 0, delete = 0
2019-02-22 10:34:55,003 [139881288288000] [ERROR] Retrieve tuple failed with error 0x11 - KVS_ERR_KEY_NOT_EXIST, key = .minio.sys/buckets/test/notification.xml/xl.json/minio/admin/v1/profiling/download
2019-02-22 10:34:55,003 [139881288288000] [ERROR] NKV retrieve operation failed for nkv_handle = 4091013546, key = .minio.sys/buckets/test/notification.xml/xl.json/minio/admin/v1/profiling/download, code = 24
2019-02-22 10:34:55,004 [139881313466112] [ERROR] Retrieve tuple failed with error 0x11 - KVS_ERR_KEY_NOT_EXIST, key = .minio.sys/buckets/test/notification.xml/xl.json.minio.sys/config/config.json/xl.json
2019-02-22 10:34:55,004 [139881313466112] [ERROR] NKV retrieve operation failed for nkv_handle = 4091013546, key = .minio.sys/buckets/test/notification.xml/xl.json.minio.sys/config/config.json/xl.json, code = 24
2019-02-22 10:34:55,005 [139881732871936] [ERROR] Retrieve tuple failed with error 0x11 - KVS_ERR_KEY_NOT_EXIST, key = .minio.sys/buckets/test/notification.xml/xl.json
2019-02-22 10:34:55,005 [139880373942016] [ERROR] Retrieve tuple failed with error 0x11 - KVS_ERR_KEY_NOT_EXIST, key = .minio.sys/buckets/test/notification.xml/xl.json
2019-02-22 10:34:55,005 [139881732871936] [ERROR] NKV retrieve operation failed for nkv_handle = 4091013546, key = .minio.sys/buckets/test/notification.xml/xl.json, code = 24
2019-02-22 10:34:55,005 [139880373942016] [ERROR] NKV retrieve operation failed for nkv_handle = 4091013546, key = .minio.sys/buckets/test/notification.xml/xl.json, code = 24
2019-02-22 10:34:55,005 [139880373942016] [INFO] Number of containers populated = 1
Got container list, count = 1
Container Information :: hash = 993003492l, id = 0l, uuid = dce20d46, name = nqn-02, target node = msl-ssg-sk01, status = 0, space available pcnt = 87
Number of Container transport = 4
Transport information :: hash = 173163378, id = 3, address = 104.100.10.31, port = 1023, family = 2, speed = 3, status = 1, numa_node = 0
Transport information :: hash = 3559271658, id = 2, address = 103.100.10.31, port = 1023, family = 2, speed = 3, status = 1, numa_node = 0
Transport information :: hash = 1441750628, id = 1, address = 102.100.10.31, port = 1023, family = 2, speed = 3, status = 1, numa_node = 0
Transport information :: hash = 3303807887, id = 0, address = 101.100.10.31, port = 1023, family = 2, speed = 3, status = 1, numa_node = 0
2019-02-22 10:34:55,005 [139880373942016] [INFO] Number of containers populated = 1
Got container list, count = 1
Container Information :: hash = 993003492l, id = 0l, uuid = dce20d46, name = nqn-02, target node = msl-ssg-sk01, status = 0, space available pcnt = 87
Number of Container transport = 4
Transport information :: hash = 173163378, id = 3, address = 104.100.10.31, port = 1023, family = 2, speed = 3, status = 1, numa_node = 0
Transport information :: hash = 3559271658, id = 2, address = 103.100.10.31, port = 1023, family = 2, speed = 3, status = 1, numa_node = 0
Transport information :: hash = 1441750628, id = 1, address = 102.100.10.31, port = 1023, family = 2, speed = 3, status = 1, numa_node = 0
2019-02-22 10:34:55,006 [139880373942016] [INFO] Number of containers populated = 1
Got container list, count = 1
Container Information :: hash = 993003492l, id = 0l, uuid = dce20d46, name = nqn-02, target node = msl-ssg-sk01, status = 0, space available pcnt = 87
Number of Container transport = 4
Transport information :: hash = 173163378, id = 3, address = 104.100.10.31, port = 1023, family = 2, speed = 3, status = 1, numa_node = 0
Transport information :: hash = 3559271658, id = 2, address = 103.100.10.31, port = 1023, family = 2, speed = 3, status = 1, numa_node = 0
2019-02-22 10:34:55,006 [139880373942016] [INFO] Number of containers populated = 1
Got container list, count = 1
Container Information :: hash = 993003492l, id = 0l, uuid = dce20d46, name = nqn-02, target node = msl-ssg-sk01, status = 0, space available pcnt = 87
Number of Container transport = 4
Transport information :: hash = 173163378, id = 3, address = 104.100.10.31, port = 1023, family = 2, speed = 3, status = 1, numa_node = 0
2019-02-22 10:34:55,006 [139880617199360] [INFO] Sending IO to dev mount = /dev/nvme3n1, container name = nqn-02, target node = msl-ssg-sk01, path ip = 104.100.10.31, path port = 1023
2019-02-22 10:34:55,006 [139880617199360] [INFO] Retrieve option:: decompression = 0, decryption = 0, compare crc = 0, delete = 0
2019-02-22 10:34:55,009 [139881741264640] [INFO] Sending IO to dev mount = /dev/nvme2n1, container name = nqn-02, target node = msl-ssg-sk01, path ip = 103.100.10.31, path port = 1023
2019-02-22 10:34:55,009 [139881741264640] [INFO] Retrieve option:: decompression = 0, decryption = 0, compare crc = 0, delete = 0
2019-02-22 10:34:55,009 [139882001307392] [INFO] Sending IO to dev mount = /dev/nvme0n1, container name = nqn-02, target node = msl-ssg-sk01, path ip = 101.100.10.31, path port = 1023
2019-02-22 10:34:55,009 [139882001307392] [INFO] Retrieve option:: decompression = 0, decryption = 0, compare crc = 0, delete = 0
2019-02-22 10:34:55,009 [139882018092800] [INFO] Sending IO to dev mount = /dev/nvme1n1, container name = nqn-02, target node = msl-ssg-sk01, path ip = 102.100.10.31, path port = 1023
2019-02-22 10:34:55,009 [139882018092800] [INFO] Retrieve option:: decompression = 0, decryption = 0, compare crc = 0, delete = 0
2019-02-22 10:34:55,011 [139880617199360] [INFO] NKV retrieve operation is successful for nkv_handle = 4091013546, key = .minio.sys/format.json
2019-02-22 10:34:55,013 [139881741264640] [INFO] NKV retrieve operation is successful for nkv_handle = 4091013546, key = .minio.sys/format.json
2019-02-22 10:34:55,013 [139882001307392] [INFO] NKV retrieve operation is successful for nkv_handle = 4091013546, key = .minio.sys/format.json
2019-02-22 10:34:55,013 [139882018092800] [INFO] NKV retrieve operation is successful for nkv_handle = 4091013546, key = .minio.sys/format.json
Status: 4 Online, 0 Offline.
Endpoint: http://10.1.12.23:9000 http://100.100.1.8:9000 http://192.168.122.1:9000 http://127.0.0.1:9000
AccessKey: minio
SecretKey: minio123
Browser Access:
http://10.1.12.23:9000 http://100.100.1.8:9000 http://192.168.122.1:9000 http://127.0.0.1:9000
Command-line Access: https://docs.minio.io/docs/minio-client-quickstart-guide
$ mc config host add myminio http://10.1.12.23:9000 minio minio123
Object API (Amazon S3 compatible):
Go: https://docs.minio.io/docs/golang-client-quickstart-guide
Java: https://docs.minio.io/docs/java-client-quickstart-guide
Python: https://docs.minio.io/docs/python-client-quickstart-guide
JavaScript: https://docs.minio.io/docs/javascript-client-quickstart-guide
.NET: https://docs.minio.io/docs/dotnet-client-quickstart-guide
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment