Skip to content

Instantly share code, notes, and snippets.

@krishnasrinivas

krishnasrinivas/gist:81488dc7de347d7f4f0bddd173b7be34

Created February 21, 2019 00:14
Show Gist options
  • Save krishnasrinivas/81488dc7de347d7f4f0bddd173b7be34 to your computer and use it in GitHub Desktop.
Save krishnasrinivas/81488dc7de347d7f4f0bddd173b7be34 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
gistfile1.txt
minio@minio-c2:~$ sudo nvme list
Node SN Model Namespace Usage Format FW Rev
---------------- -------------------- ---------------------------------------- --------- -------------------------- ---------------- --------
/dev/nvme0n1 S3VJNY0J600133 SAMSUNG MZQLB3T8HALS-000AZ 1 0.00 B / 3.84 TB 512 B + 0 B EEA50K0Q
/dev/nvme1n1 S3VJNY0J600117 SAMSUNG MZQLB3T8HALS-000AZ 1 0.00 B / 3.84 TB 512 B + 0 B EEA50K0Q
/dev/nvme2n1 S3VJNY0J600121 SAMSUNG MZQLB3T8HALS-000AZ 1 0.00 B / 3.84 TB 512 B + 0 B EEA50K0Q
/dev/nvme3n1 S3VJNF0K722737 SAMSUNG MZQLB3T8HALS-000AZ 1 0.00 B / 3.84 TB 512 B + 0 B EEA50K0Q
/dev/nvme4n1 S3VJNF0K722736 SAMSUNG MZQLB3T8HALS-000AZ 1 0.00 B / 3.84 TB 512 B + 0 B EEA50K0Q
/dev/nvme5n1 BTLJ802102S21P0FGN INTEL SSDPE2KX010T8 1 1.00 TB / 1.00 TB 512 B + 0 B VDV10131
minio@minio-c2:~$
minio@minio-c2:~$ cat /tmp/nvme0n1.json
{
"fm_address": "10.1.20.91",
"contact_fm": 0,
"nkv_transport" : 0,
"min_container_required" : 1,
"min_container_path_required" : 1,
"nkv_container_path_qd" : 64,
"nkv_mounts": [
{
"mount_point": "/dev/nvme0n1",
"remote_nqn_name": "nqn-02",
"remote_target_node_name": "msl-ssg-sk01",
"nqn_transport_address": "101.100.10.31",
"nqn_transport_port": 1023,
"numa_node_attached" : 0
}
],
"subsystem_maps": [
{
"target_server_name": "msl-ssg-sk01",
"subsystem_nqn_id": "dce20d46",
"subsystem_nqn": "nqn-02",
"subsystem_nqn_nsid": 1,
"subsystem_transport": [
{
"subsystem_type": 1,
"subsystem_address": "101.100.10.31",
"subsystem_port": 1023,
"subsystem_addr_fam": 2,
"subsystem_interface_speed": 3,
"subsystem_interface_numa_aligned": false,
"subsystem_interface_status":1
}
],
"subsystem_status": 0,
"subsystem_space_avail_percent": 87
}
]
}
minio@minio-c2:~$ cat /tmp/nvme1n1.json
{
"fm_address": "10.1.20.91",
"contact_fm": 0,
"nkv_transport" : 0,
"min_container_required" : 1,
"min_container_path_required" : 1,
"nkv_container_path_qd" : 64,
"nkv_mounts": [
{
"mount_point": "/dev/nvme1n1",
"remote_nqn_name": "nqn-02",
"remote_target_node_name": "msl-ssg-sk01",
"nqn_transport_address": "101.100.10.31",
"nqn_transport_port": 1023,
"numa_node_attached" : 0
}
],
"subsystem_maps": [
{
"target_server_name": "msl-ssg-sk01",
"subsystem_nqn_id": "dce20d46",
"subsystem_nqn": "nqn-02",
"subsystem_nqn_nsid": 1,
"subsystem_transport": [
{
"subsystem_type": 1,
"subsystem_address": "101.100.10.31",
"subsystem_port": 1023,
"subsystem_addr_fam": 2,
"subsystem_interface_speed": 3,
"subsystem_interface_numa_aligned": false,
"subsystem_interface_status":1
}
],
"subsystem_status": 0,
"subsystem_space_avail_percent": 87
}
]
}
minio@minio-c2:~$ cat /tmp/nvme3n1.json
{
"fm_address": "10.1.20.91",
"contact_fm": 0,
"nkv_transport" : 0,
"min_container_required" : 1,
"min_container_path_required" : 1,
"nkv_container_path_qd" : 64,
"nkv_mounts": [
{
"mount_point": "/dev/nvme4n1",
"remote_nqn_name": "nqn-02",
"remote_target_node_name": "msl-ssg-sk01",
"nqn_transport_address": "101.100.10.31",
"nqn_transport_port": 1023,
"numa_node_attached" : 0
}
],
"subsystem_maps": [
{
"target_server_name": "msl-ssg-sk01",
"subsystem_nqn_id": "dce20d46",
"subsystem_nqn": "nqn-02",
"subsystem_nqn_nsid": 1,
"subsystem_transport": [
{
"subsystem_type": 1,
"subsystem_address": "101.100.10.31",
"subsystem_port": 1023,
"subsystem_addr_fam": 2,
"subsystem_interface_speed": 3,
"subsystem_interface_numa_aligned": false,
"subsystem_interface_status":1
}
],
"subsystem_status": 0,
"subsystem_space_avail_percent": 87
}
]
}
minio@minio-c2:~$ cat /tmp/nvme4n1.json
{
"fm_address": "10.1.20.91",
"contact_fm": 0,
"nkv_transport" : 0,
"min_container_required" : 1,
"min_container_path_required" : 1,
"nkv_container_path_qd" : 64,
"nkv_mounts": [
{
"mount_point": "/dev/nvme0n1",
"remote_nqn_name": "nqn-02",
"remote_target_node_name": "msl-ssg-sk01",
"nqn_transport_address": "101.100.10.31",
"nqn_transport_port": 1023,
"numa_node_attached" : 0
}
],
"subsystem_maps": [
{
"target_server_name": "msl-ssg-sk01",
"subsystem_nqn_id": "dce20d46",
"subsystem_nqn": "nqn-02",
"subsystem_nqn_nsid": 1,
"subsystem_transport": [
{
"subsystem_type": 1,
"subsystem_address": "101.100.10.31",
"subsystem_port": 1023,
"subsystem_addr_fam": 2,
"subsystem_interface_speed": 3,
"subsystem_interface_numa_aligned": false,
"subsystem_interface_status":1
}
],
"subsystem_status": 0,
"subsystem_space_avail_percent": 87
}
]
}
minio@minio-c2:~$
Then I start 4 minio processes (distributed minio simulation)
MINIO-0 : minio server --address localhost:9000 http://localhost:9000/tmp/nvme0n1.json http://localhost:9001/tmp/nvme1n1.json http://localhost:9003/tmp/nvme3n1.json http://localhost:9004/tmp/nvme4n1.json
MINIO-1 : minio server --address localhost:9001 http://localhost:9000/tmp/nvme0n1.json http://localhost:9001/tmp/nvme1n1.json http://localhost:9003/tmp/nvme3n1.json http://localhost:9004/tmp/nvme4n1.json
MINIO-3 : minio server --address localhost:9003 http://localhost:9000/tmp/nvme0n1.json http://localhost:9001/tmp/nvme1n1.json http://localhost:9003/tmp/nvme3n1.json http://localhost:9004/tmp/nvme4n1.json
MINIO-4 : minio server --address localhost:9004 http://localhost:9000/tmp/nvme0n1.json http://localhost:9001/tmp/nvme1n1.json http://localhost:9003/tmp/nvme3n1.json http://localhost:9004/tmp/nvme4n1.json
This starts 4 minio processes. Each process does nkv_open() and nkv_container_list() nkv_store() nkv_retreive() operations on one separate nvme devices.
Not sure if NKV supports this style of operation. This causes some corruption in the kernel.
MINIO-0 uses /tmp/nvme0n1.json
MINIO-1 uses /tmp/nvme1n1.json
MINIO-3 uses /tmp/nvme3n1.json
MINIO-4 uses /tmp/nvme4n1.json
https://github.com/krishnasrinivas/minio/blob/storageapi-implementation-of-kv-emulator/cmd/kv.go is the related Minio source accessing the NKV.
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment