Skip to content

Instantly share code, notes, and snippets.

@spikeekips

spikeekips/tokennet-FAILED-tokennet-public.md

Created November 13, 2019 03:52
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save spikeekips/d9d0e645d014b65d47e17c3633598a83 to your computer and use it in GitHub Desktop.
Save spikeekips/d9d0e645d014b65d47e17c3633598a83 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
tokennet-FAILED-tokennet-public.md

just restart

ansible-playbook -i env/tokennet-public script/tokennet-core-up-forcescp.yml -e path=$(pwd)/../design/tokennet-public/latest/
ansible -i env/tokennet-public tokennet-core -m shell -a 'mv /opt/bos/tmp/db-*.tar.gz /opt/'

backup db

ansible -i env/tokennet-public tokennet-core -m shell -a 'rm -rf /opt/bos/tmp/db-*'
ansible-playbook -i env/tokennet-public script/tokennet-core-db.yml -e s3_path=/node-db-backup-20171111T1116
for i in sydney0 sydney1 sydney2 sydney3 sydney4 sydney5 sydney6 sydney7 sydney8 tokyo0 tokyo1 tokyo2 tokyo3 tokyo4 tokyo5 tokyo6 tokyo7 tokyo8 california0 california1 california2 california3 california4 california5 california6 california7 california8; do echo $i; dm scp tokennet-public-$i:/opt/db-*.tar.gz /tmp/a/; done

create snapshot

for i in tokennet-public-aws0-seoul tokennet-public-aws0-sydney tokennet-public-aws0-tokyo tokennet-public-aws0-california tokennet-public-aws1-seoul tokennet-public-aws1-sydney tokennet-public-aws1-tokyo tokennet-public-aws1-california; do echo $i; aws --profile $i rds describe-db-instances --db-instance-identifier 'db-tokennet-public'; break; done

for i in tokennet-public-aws0-seoul tokennet-public-aws0-sydney tokennet-public-aws0-tokyo tokennet-public-aws0-california tokennet-public-aws1-seoul tokennet-public-aws1-sydney tokennet-public-aws1-tokyo tokennet-public-aws1-california; do echo $i; aws --profile $i rds create-db-snapshot --db-instance-identifier db-tokennet-public --db-snapshot-identifier failed-synced-20171110T1528 ; done

increase TX SIZE to 4000

perl -pi -e 's|DESIRED_MAX_TX_PER_LEDGER=1000|DESIRED_MAX_TX_PER_LEDGER=4000|g' *.cfg
perl -pi -e 's|MAX_TX_SET_SIZE=1000|MAX_TX_SET_SIZE=4000|g' *.cfg

ansible-playbook -i env/tokennet-public script/tokennet-core-up-forcescp.yml -e path=$(pwd)/../design/tokennet-public/latest/

analysis

AAAAAQAAAABwnhp9sAE4d2F5zp3BkcCRUlS4l+lW/GUCTOOvpkG64wAAAAAAApTiAAAAA7mgvmWx7irkyI9XRVVpor4B/H73USdNxWnEfUCsrHEKAAAABQAAADBTV26lpvqLsSrTi6I6NLSc1rZ6pQuhHcFpCc65SMiAUgAAAABaBWMAAAAAAAAAAAAAAAAwU1dupab6i7Eq04uiOjS0nNa2eqULoR3BaQnOuUjIgFIAAAAAWgVuiQAAAAAAAAAAAAAAMFNXbqWm+ouxKtOLojo0tJzWtnqlC6EdwWkJzrlIyIBSAAAAAFoFcqUAAAAAAAAAAAAAADxTV26lpvqLsSrTi6I6NLSc1rZ6pQuhHcFpCc65SMiAUgAAAABaBbE6AAAAAQAAAAgAAAADAAAPoAAAAAAAAAA8U1dupab6i7Eq04uiOjS0nNa2eqULoR3BaQnOuUjIgFIAAAAAWgWyOgAAAAEAAAAIAAAAAwAAD6AAAAAAAAAAAAAAAEDHm3Fgf4evP2c5uY7FYGwD8MXCYJsZ8WqWZXvF2mKOFRdV0xAXdMjuD3Z1i2jDIG/AqISdlqWNeGb15/59cOoLAAAAATvPLCfv4IYW2hp3m61OJtAzTxFkuEaVKAZ42PV8Bd22AAAAAAAAAAEAAAARAAAAGQAAAAA3tIRV/QeVFAs49kuYd4BgSKqW12omycUOvDlEgolRRQAAAADhtUmc05YPQfH+pWfEdrKeUrFSiUcKSujiKuSFamDAkQAAAADNIUiqowv4Sc6iCt00HHG4AmEo3EirKGxRHRSYl2Q93gAAAACAEF3I5KzHppwefzBp2FaOATl1NaHlveTjtNHfZ0dnEQAAAADpQs+t9qX9eYuPgFW/xpCEYRyKBV/h2HTxyGET+xFrpQAAAAAPm/+CbBi0LX9xSVXa3OHNpt7+XuQCquf4NG+8EHNTxgAAAAAfK2krETwp3/HfNkXVT+bDsj4qbZmBUIFy4Rp7GIQYkAAAAAD8DgiqPIT9XBDFeEKrXB9sQRCw2G6og5UNaHaQUIInSgAAAAAakIHY7Vs+15Mh9/QrrLbFHZGx2+wD+rN/QobVlcBBoAAAAADYUlOwFALEqlHtcqGqK8IbmaO8RCRhjsJLMmlI3KA6hQAAAABMEOY7cpWHSMZdUKfEFJqRFGV6m9qrnCpT2QQxY4AU5QAAAACBb4DBLTtDo0hjwHYV6zY+u/UvRrRF0kRgZ/oHM6eimAAAAADYYDaA6iZdpmHe+xBT2oBcnBQG+VL6RgXLIH1WT38xUwAAAADFf/2RN5doP1GbKNmoVhbujlUhLGQFy3UiIaGCx40kuQAAAACYhyGs3m/S7ofCuAdrCajZN00XwGT4ESQqQgnnC67W7gAAAAC0OPCQ2x/NqAjPjMc6w/wC5WQwOFQtWe59xbGVriF7VwAAAADvvRcyUh4uaJ43ej0d2FCM1ghFZZLRwfW/884OAzLrDwAAAAARB7w8If/vuRK40tqgtTCZB9SVuOwixCQKzsny5+YNFwAAAACsv2SgtUW7PD6LqWRBKT5PfJKRNJWkZXuBaIlcUs4qRQAAAACzkvgLH3WdYp4jcn/BPMZYPMT7S1uqM60/TNQiKPXo8QAAAADWuJa4Tnvu66RWNFLAjcKnLj6dBXAZzmt1q91KV1rAPwAAAADulrA11sA26irxodVZ2DieLG8KCId7BZjVAGwL2laFdwAAAABFgdEYxjIfrnFr0sX3+a19x5npm+v+Ly3bweO2SU589gAAAAC1mr1KK457x3tr9CMDsIjboiTN2TxUemBbxFqtrRG+pwAAAABwnhp9sAE4d2F5zp3BkcCRUlS4l+lW/GUCTOOvpkG64wAAAAA=
ScpStatementConfirm
ballot
counter: 1
value:
nPrepared: 1889409661
nCommit: 2952870007
nH: 1635372701
quorumSetHash: wZHAkVJUuJfpVvxlAkzjr6ZBuuMAAAAAAAKU4gAAAAM=


AAAAAQAAAAA3tIRV/QeVFAs49kuYd4BgSKqW12omycUOvDlEgolRRQAAAAAAApTiAAAAA8MOJ/jh87FEkyNxeg2j5RqtwHW1xhUbOPtTexY9YdgWAAAABQAAADAj7ihaRioteuoXxpzDD9VgkHW2sFpkUe9HmkZViKXHkAAAAABaBWL7AAAAAAAAAAAAAAAwI+4oWkYqLXrqF8acww/VYJB1trBaZFHvR5pGVYilx5AAAAAAWgVuiQAAAAAAAAAAAAAAMCPuKFpGKi166hfGnMMP1WCQdbawWmRR70eaRlWIpceQAAAAAFoFcYMAAAAAAAAAAAAAADwj7ihaRioteuoXxpzDD9VgkHW2sFpkUe9HmkZViKXHkAAAAABaBbE5AAAAAQAAAAgAAAADAAAPoAAAAAAAAAA8I+4oWkYqLXrqF8acww/VYJB1trBaZFHvR5pGVYilx5AAAAAAWgWyOQAAAAEAAAAIAAAAAwAAD6AAAAAAAAAAAAAAAEDhW2zeE669td0cxb8T3xrEP8Ndh0o8DJGb/EWsKzbQMVFFaht/dPO2rbWCqvp8x06iXz14RU11q9fQP75gqy8JAAAAAdQBU3jMTBzguOqJhVApTzmBN58CLbOEm1MgH6hHxrOnAAAAAAAAAAEAAAAQAAAAFwAAAADhtUmc05YPQfH+pWfEdrKeUrFSiUcKSujiKuSFamDAkQAAAADNIUiqowv4Sc6iCt00HHG4AmEo3EirKGxRHRSYl2Q93gAAAACAEF3I5KzHppwefzBp2FaOATl1NaHlveTjtNHfZ0dnEQAAAADpQs+t9qX9eYuPgFW/xpCEYRyKBV/h2HTxyGET+xFrpQAAAAAPm/+CbBi0LX9xSVXa3OHNpt7+XuQCquf4NG+8EHNTxgAAAAAfK2krETwp3/HfNkXVT+bDsj4qbZmBUIFy4Rp7GIQYkAAAAAD8DgiqPIT9XBDFeEKrXB9sQRCw2G6og5UNaHaQUIInSgAAAAB3delBohO35q6KDlhR8rWlgVf5kyuMKzuZdQ/XUn8fMQAAAAD8TcR/oh4W9/X5PaF99WgLCVBvoscORL6j1LRm6gTwVgAAAAAd3jSj1paXmiXEkjUeHA1O7NKQ2/dI5P3AzKZBZhDjEAAAAADamQ/8IYXExA5lnQ/M7E4rFbX2mePZfR2mjfxFSiNsKgAAAACLST+2AacLfGRsmTRRKSC2JfUM5tJbOiF2RmzmGYCifAAAAAA7QNq1ljmgf6oi2F1lXjU/UnyU37FobxbCLk/5McIMlQAAAAA1rV2FgjohWBzymI/+DcMujN8ix9tBgMGKIypM5v/iQQAAAAAakIHY7Vs+15Mh9/QrrLbFHZGx2+wD+rN/QobVlcBBoAAAAADYUlOwFALEqlHtcqGqK8IbmaO8RCRhjsJLMmlI3KA6hQAAAACsv2SgtUW7PD6LqWRBKT5PfJKRNJWkZXuBaIlcUs4qRQAAAACzkvgLH3WdYp4jcn/BPMZYPMT7S1uqM60/TNQiKPXo8QAAAADWuJa4Tnvu66RWNFLAjcKnLj6dBXAZzmt1q91KV1rAPwAAAADulrA11sA26irxodVZ2DieLG8KCId7BZjVAGwL2laFdwAAAABFgdEYxjIfrnFr0sX3+a19x5npm+v+Ly3bweO2SU589gAAAAC1mr1KK457x3tr9CMDsIjboiTN2TxUemBbxFqtrRG+pwAAAAA3tIRV/QeVFAs49kuYd4BgSKqW12omycUOvDlEgolRRQAAAAA=

ScpStatementConfirm
ballot
counter: 1
value:
nPrepared: 934577237
nCommit: 4245132564
nH: 188282443
quorumSetHash: mHeAYEiqltdqJsnFDrw5RIKJUUUAAAAAAAKU4gAAAAM=

db

for i in $(cd db-seoul0-1510324214; ls *.csv); do echo $i; [ $i == 'scphistory.csv' ] && continue; cat db-seoul0-1510324214/$i | sort > db-seoul0-1510324214/$i-sorted; cat db-tokyo0-1510324214/$i | sort > db-tokyo0-1510324214/$i-sorted; done
for i in $(cd db-seoul0-1510324214; ls *.csv-sorted); do echo $i; diff -uNr db-seoul0-1510324214/$i db-tokyo0-1510324214/$i > ./diff/$i.diff; done

94d63095a32eb339a558837e0bac42f99bf568e09421223642f814d5db0ccd39,928ecc8c2016714a862a94ff387e3cf4bc7c1ae817a26f6ab148b47ded0fed69,169185,1510302453,AAAACJTWMJWjLrM5pViDfgusQvmb9WjglCEiNkL4FNXbDM05n8AoQD6FOxtWN6qFaKNszbA01d9TBuBU9UHG+xqCjVMAAAAAWgVi9QAAAAAAAAAAAx90BHtQZrGFB4ZC16dLHPTtuLoyLXXEnWizGYvtSGySjsyMIBZxSoYqlP84fjz0vHwa6Beib2qxSLR97Q/taQAClOEAEcN5N+CAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAnEAAHoSAAAAPo0/fshKQS3kHS4gykefDfL4B3mtZ6Ck9lJTVqids2Mtr3HuEu2Zf+Rl6dsFfbCX/U8Q80ii1PcOMdy06qkSIlFFlA5Jd4iyvEuyZY9oZ9uQ2ahSJ6LAYkHlG/N+vX+zQIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
94d63095a32eb339a558837e0bac42f99bf568e09421223642f814d5db0ccd39,928ecc8c2016714a862a94ff387e3cf4bc7c1ae817a26f6ab148b47ded0fed69,169185,1510302453,AAAACJTWMJWjLrM5pViDfgusQvmb9WjglCEiNkL4FNXbDM05n8AoQD6FOxtWN6qFaKNszbA01d9TBuBU9UHG+xqCjVMAAAAAWgVi9QAAAAAAAAAA3Kjjs+XCqBzU7/woe2CQ4e086j62Mf3+kiieR/hNiEOSjsyMIBZxSoYqlP84fjz0vHwa6Beib2qxSLR97Q/taQAClOEAEcN5N+CAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAnEAAHoSAAAAPo0/fshKQS3kHS4gykefDfL4B3mtZ6Ck9lJTVqids2Mtr3HuEu2Zf+Rl6dsFfbCX/U8Q80ii1PcOMdy06qkSIlFFlA5Jd4iyvEuyZY9oZ9uQ2ahSJ6LAYkHlG/N+vX+zQIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA

169185 is the last ledger sequence

$ grep '\<169185\>' *-sorted | awk -F':' '{print $1}' | sort -u
accounts.csv-sorted
ledgerheaders.csv-sorted
scpquorums.csv-sorted
storestate.csv-sorted
txfeehistory.csv-sorted
txhistory.csv-sorted

Restore tokennet-public to tokennet-test

Scenario

  1. dump db of all nodes
  2. import the table data of seoul0 to all nodes
    • accounts
    • ledgerheaders
    • peers
    • pubsub
    • storestate
    • txfeehistory
    • txhistory
  3. customize cfg
    • NODE_SEED
    • COMMON_BUDGET_ACCOUNT_ID
    • COMMON_BUDGET_INFLATION_MIN_BALANCE
    • COMMON_BUDGET_INFLATION_MAX_ACCOUNTS
    • COMMON_BUDGET_INFLATION_EXCLUDED_ACCOUNTS
    • NODE_NAMES
    • HISTORY.default
    • VALIDATORS
  4. 'forcescp'

Clean up

$ ansible-playbook -i env/tokennet-test script/docker-cleanup-containers.yml -e name='tokennet-core'
$ ansible-playbook -i env/tokennet-test script/docker-cleanup-containers.yml -e name='horizon'
$ ansible-playbook -i env/tokennet-test script/docker-cleanup-containers.yml -e name='logstash'

dump db of all nodes

$ ansible-playbook -i env/tokennet-public script/tokennet-core-db-pg_dump.yml -l seoul0 -e s3_path='/node-db-backup-20171111T1116-sql'

import the table data of seoul0 to all nodes

$ for i in accounts ledgerheaders peers pubsub storestate txfeehistory txhistory; do PGPASSWORD=bosuser pg_dump -h 172.31.15.18 -U bosuser -t $i --schema-only seoul0; done > create-tables.sql

$ ansible-playbook -i env/tokennet-test script/tokennet-core-db-import.yml -l seoul0 -e sqlfile=/Users/spikeekips/workspace/blockchainos/token-infra/tmp/failed-public-20171111/node-db/tmp/data.sql.gz
``

## snippets

$ PGPASSWORD=bosuser psql -h 172.31.15.18 -U bosuser seoul0

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment