passive failures

In [4]: s3_redundant = robot.services.get(name='mys3_red2')

In [5]: s3_redundant.data
Out[5]: 
{'data': {'activeS3': '806a53f0-a289-410f-a104-b6f9d9001300',
  'dataShards': 1,
  'farmerIyoOrg': 'bancadati.threefold.grid.farm01',
  'mgmtNic': {'id': 'a0cbf4b62acb21c7', 'ztClient': 'main'},
  'minioBlockSize': 1048576,
  'minioLogin': 'login',
  'minioPassword': 'password',
  'nsPassword': 'tch7ompwjhi2mpigfv5zppwtmzl1ynpi',
  'parityShards': 1,
  'passiveS3': '5df578ad-dcb0-4d15-9c44-16078a4f09e8',
  'reverseProxy': '',
  'storageSize': 200,
  'storageType': 'hdd'},
 'guid': '36373996-8fc9-4aea-a5b7-ca9532abddbd',
 'name': 'mys3_red2',
 'public': False,
 'state': [{'category': 'actions', 'state': 'ok', 'tag': 'install'}],
 'template': 'github.com/threefoldtech/0-templates/s3_redundant/0.0.1',
 'version': '0.0.1'}

In [6]: active = robot.services.get(name='806a53f0-a289-410f-a104-b6f9d9001300',template_name='s3')

In [7]: active.data
Out[7]: 
{'data': {'activeS3': '',
  'current_namespaces_connections': ['172.16.46.52:1025',
   '172.16.46.55:1028',
   '172.16.46.67:1024'],
  'dataShards': 1,
  'deletableNamespaces': [],
  'excludeNodesVM': [],
  'farmerIyoOrg': 'bancadati.threefold.grid.farm01',
  'master': {'address': '', 'name': '', 'node': '', 'url': ''},
  'mgmtNic': {'id': 'a0cbf4b62acb21c7', 'ztClient': 'main'},
  'minioBlockSize': 1048576,
  'minioLogin': 'login',
  'minioPassword': 'password',
  'minioUrls': {'public': 'http://192.168.196.177:1024',
   'storage': 'http://172.16.46.74:1039'},
  'namespaces': [{'address': '172.16.46.55:1028',
    'name': '366aea33-b1f4-459d-b5b0-0871040feeac',
    'node': 'ac1f6b47a214',
    'url': 'http://172.30.34.68:6600'},
   {'address': '172.16.46.52:1025',
    'name': '3966b63f-509a-49d2-b0df-fb50283f494c',
    'node': 'ac1f6b457be0',
    'url': 'http://172.30.116.159:6600'},
   {'address': '172.16.46.67:1024',
    'name': 'b3659e5c-86e7-4e1a-b807-3757e2f1ebda',
    'node': 'ac1f6b47a0d8',
    'url': 'http://172.30.35.37:6600'}],
  'nsName': '36373996-8fc9-4aea-a5b7-ca9532abddbd',
  'nsPassword': 'tch7ompwjhi2mpigfv5zppwtmzl1ynpi',
  'parityShards': 1,
  'passiveS3': '',
  'reverseProxy': '',
  'storageSize': 200,
  'storageType': 'hdd',
  'tlog': {'address': '172.16.46.58:1038',
   'name': 'c3312126-bdd3-4b6b-a2f6-8ce1fcf8c7d4',
   'node': 'ac1f6b457c24',
   'url': 'http://172.30.32.86:6600'}},
 'guid': '806a53f0-a289-410f-a104-b6f9d9001300',
 'name': '806a53f0-a289-410f-a104-b6f9d9001300',
 'public': False,
 'state': [{'category': 'status', 'state': 'ok', 'tag': 'running'},
  {'category': 'actions', 'state': 'ok', 'tag': 'install'},
  {'category': 'vm', 'state': 'ok', 'tag': 'running'}],
 'template': 'github.com/threefoldtech/0-templates/s3/0.0.1',
 'version': '0.0.1'}

In [8]: passive = robot.services.get(name='5df578ad-dcb0-4d15-9c44-16078a4f09e8',template_name='s3')

In [9]: passive.data
Out[9]: 
{'data': {'activeS3': '',
  'current_namespaces_connections': ['172.16.46.52:1025',
   '172.16.46.55:1028',
   '172.16.46.67:1024'],
  'dataShards': 1,
  'deletableNamespaces': [],
  'excludeNodesVM': ['ac1f6b457b6c', 'ac1f6b457c24'],
  'farmerIyoOrg': 'bancadati.threefold.grid.farm01',
  'master': {'address': '172.16.46.58:1038',
   'name': 'c3312126-bdd3-4b6b-a2f6-8ce1fcf8c7d4',
   'node': 'ac1f6b457c24',
   'url': 'http://172.30.32.86:6600'},
  'mgmtNic': {'id': 'a0cbf4b62acb21c7', 'ztClient': 'main'},
  'minioBlockSize': 1048576,
  'minioLogin': 'login',
  'minioPassword': 'password',
  'minioUrls': {'public': 'http://192.168.196.159:1024',
   'storage': 'http://172.16.46.42:1061'},
  'namespaces': [{'address': '172.16.46.55:1028',
    'name': '366aea33-b1f4-459d-b5b0-0871040feeac',
    'node': 'ac1f6b47a214',
    'url': 'http://172.30.34.68:6600'},
   {'address': '172.16.46.52:1025',
    'name': '3966b63f-509a-49d2-b0df-fb50283f494c',
    'node': 'ac1f6b457be0',
    'url': 'http://172.30.116.159:6600'},
   {'address': '172.16.46.67:1024',
    'name': 'b3659e5c-86e7-4e1a-b807-3757e2f1ebda',
    'node': 'ac1f6b47a0d8',
    'url': 'http://172.30.35.37:6600'}],
  'nsName': '36373996-8fc9-4aea-a5b7-ca9532abddbd',
  'nsPassword': 'tch7ompwjhi2mpigfv5zppwtmzl1ynpi',
  'parityShards': 1,
  'passiveS3': '',
  'reverseProxy': '',
  'storageSize': 200,
  'storageType': 'hdd',
  'tlog': {'address': '172.16.46.90:1065',
   'name': 'ef96230d-a97a-4c80-9dce-ef480ca26125',
   'node': 'ac1f6b47a20c',
   'url': 'http://172.30.212.242:6600'}},
 'guid': '5df578ad-dcb0-4d15-9c44-16078a4f09e8',
 'name': '5df578ad-dcb0-4d15-9c44-16078a4f09e8',
 'public': False,
 'state': [{'category': 'status', 'state': 'ok', 'tag': 'running'},
  {'category': 'actions', 'state': 'ok', 'tag': 'install'},
  {'category': 'vm', 'state': 'ok', 'tag': 'running'},
  {'category': 'tlog_shards', 'state': 'error', 'tag': '172.16.46.90:1065'}],
 'template': 'github.com/threefoldtech/0-templates/s3/0.0.1',
 'version': '0.0.1'}




In [12]: tcl = j.clients.zos.get_by_id('ac1f6b47a20c')

In [13]: zrobot = tcl.containers.get('zrobot')
                                                
In [14]: zrobot.client.bash('zrobot godtoken get ').get()
Out[14]: 
STATE: 0 SUCCESS
STDOUT:
god token: eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdXRoZW50aWNhdGlvbiI6ImdvZF90b2tlbiJ9.hw8KrhGgwOuRdB_-Flhe7A3ZLr7zeTydtxLRzktg8Rc 

STDERR:
[Thu29 19:59] - DbConfigManager.py:105 :anager.dbconfigmanager - INFO     - using namespace ac1f6b47a20c

DATA:


In [15]: tcl
Out[15]: Node <172.30.212.242:6379>

In [16]: remote = j.clients.zrobot.new('test',data={'url':'http://172.30.212.242:6600','god_token_':'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdXRoZW50aWNhdGlvbiI6ImdvZF90b2tlbiJ9.hw8KrhGgwOuRdB_-Flhe7A3ZL
    ...: r7zeTydtxLRzktg8Rc'})

In [17]: trobot = j.clients.zrobot.robots.get('test')

In [18]: vdisk =  trobot.services.get(name='ef96230d-a97a-4c80-9dce-ef480ca26125')

In [19]: vdisk.data
Out[19]: 
{'data': {'diskType': 'ssd',
  'mode': 'direct',
  'nsName': '36373996-8fc9-4aea-a5b7-ca9532abddbd_tlog',
  'password': 'tch7ompwjhi2mpigfv5zppwtmzl1ynpi',
  'public': False,
  'size': 10,
  'zerodb': '14c699af-38af-414c-8d4c-a97614663f66'},
 'guid': 'ef96230d-a97a-4c80-9dce-ef480ca26125',
 'name': 'ef96230d-a97a-4c80-9dce-ef480ca26125',
 'public': False,
 'state': [{'category': 'actions', 'state': 'ok', 'tag': 'install'},
  {'category': 'status', 'state': 'ok', 'tag': 'running'}],
 'template': 'github.com/threefoldtech/0-templates/namespace/0.0.1',
 'version': '0.0.1'}

In [20]: zerodb = trobot.services.get(name='14c699af-38af-414c-8d4c-a97614663f66')

In [21]: zerodb.data
Out[21]: 
{'data': {'admin': 'j2ru5cvjepcki137wmkh8815m',
  'diskType': 'ssd',
  'mode': 'direct',
  'namespaces': [{'name': '36373996-8fc9-4aea-a5b7-ca9532abddbd_tlog',
    'password': 'tch7ompwjhi2mpigfv5zppwtmzl1ynpi',
    'public': False,
    'size': 10}],
  'nics': [],
  'nodePort': 1065,
  'path': '/mnt/storagepools/19df685c-c0fd-494d-88f6-3c0aa0cb4103/filesystems/zdb_14c699af-38af-414c-8d4c-a97614663f66',
  'size': 10,
  'sync': False,
  'ztIdentity': 'a6042d60d2:0:92360e9839c94a826dbf8f5e9202ada616f8a2f3eadf8fc272fa9b152a4b4e5b9c3d27451f5a76f836fa0593f508b706e19d9e64d9816b7f9c298f08cfc58b3b:6e70f284c1b769ca704468b6a687c2f9e6cfc1fa459c07e7197fc69baca31f22a3fb0f100ac100acac86e6e4518ea9f951870d97f69cd050a0f4708b497a4168'},
 'guid': '13239a3b-a70d-4ed2-9771-3aa48fd5c059',
 'name': '14c699af-38af-414c-8d4c-a97614663f66',
 'public': False,
 'state': [{'category': 'actions', 'state': 'ok', 'tag': 'start'},
  {'category': 'actions', 'state': 'ok', 'tag': 'install'},
  {'category': 'status', 'state': 'ok', 'tag': 'running'}],
 'template': 'github.com/threefoldtech/0-templates/zerodb/0.0.1',
 'version': '0.0.1'}



In [27]: tcl.client.bash('lsblk | grep /mnt/storagepools/19df685c-c0fd-494d-88f6-3c0aa0cb4103').get()
Out[27]: 
STATE: 0 SUCCESS
STDOUT:
`-sdb1   8:17   0  1.8T  0 part /mnt/storagepools/19df685c-c0fd-494d-88f6-3c0aa0cb4103

STDERR:

DATA:


In [28]: tcl.client.bash('echo 1 > /sys/block/sdb/device/delete').get()
Out[28]: 
STATE: 0 SUCCESS
STDOUT:

STDERR:

DATA:


In [29]: tcl.client.bash('lsblk | grep /mnt/storagepools/19df685c-c0fd-494d-88f6-3c0aa0cb4103').get()
Out[29]: 
STATE: 1001 ERROR
STDOUT:

STDERR:

DATA:

- now upload something on active minio or get urls of S3_red so minio can know failure or wait sometime so vm robot marked as un reachable so self healing occured 

  s3_redundant.schedule_action('urls').wait(die=True).result

- you can check task on passive minio should find redploy rask is running 

  passive.task_list.list_tasks()

- to cehck passive logs 

print(passive.logs)