Last active
April 11, 2016 12:26
-
-
Save mpiscaer/21b38abeab9497edee3996a9f7649e9e to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| When I create with the pool ssd-test | |
| # ceph osd pool create ssd-test 640 640 replicated replicated_ssd_ruleset | |
| I get the following error: | |
| root@ceph-mon-01-[LOC]:~# ceph health | |
| HEALTH_WARN 640 pgs degraded; 640 pgs stuck unclean; 640 pgs undersized | |
| The tree map says: | |
| root@ceph-mon-01-[LOC]:~# ceph osd tree | |
| ID WEIGHT TYPE NAME UP/DOWN REWEIGHT PRIMARY-AFFINITY | |
| -12 4.00000 root ssd | |
| -11 2.00000 datacenter htc-ssd | |
| -8 1.00000 host ceph-osd-ssd-02 | |
| 27 0.45499 osd.27 up 1.00000 1.00000 | |
| 25 0.45499 osd.25 up 1.00000 1.00000 | |
| 26 0.45499 osd.26 up 1.00000 1.00000 | |
| 20 0.45499 osd.20 up 1.00000 1.00000 | |
| -10 2.00000 datacenter dor-ssd | |
| -9 2.27299 host ceph-osd-ssd-01 | |
| 30 0.45499 osd.30 up 1.00000 1.00000 | |
| 31 0.45499 osd.31 up 1.00000 1.00000 | |
| 32 0.45499 osd.32 up 1.00000 1.00000 | |
| 33 0.45499 osd.33 up 1.00000 1.00000 | |
| -1 4.00000 root hdd | |
| -7 2.00000 datacenter htc-hdd | |
| -3 4.54300 host ceph-osd-02 | |
| 15 0.90900 osd.15 up 1.00000 1.00000 | |
| 16 0.90900 osd.16 up 1.00000 1.00000 | |
| 14 0.90900 osd.14 up 1.00000 1.00000 | |
| 12 0.90900 osd.12 up 1.00000 1.00000 | |
| 13 0.90900 osd.13 up 1.00000 1.00000 | |
| -5 4.54300 host ceph-osd-04 | |
| 4 0.90900 osd.4 up 1.00000 1.00000 | |
| 8 0.90900 osd.8 up 1.00000 1.00000 | |
| 11 0.90900 osd.11 up 1.00000 1.00000 | |
| 10 0.90900 osd.10 up 1.00000 1.00000 | |
| 9 0.90900 osd.9 up 1.00000 1.00000 | |
| -6 2.00000 datacenter dor-hdd | |
| -2 4.54300 host ceph-osd-01 | |
| 0 0.90900 osd.0 up 1.00000 1.00000 | |
| 18 0.90900 osd.18 up 1.00000 1.00000 | |
| 19 0.90900 osd.19 up 1.00000 1.00000 | |
| 17 0.90900 osd.17 up 1.00000 1.00000 | |
| 1 0.90900 osd.1 up 1.00000 1.00000 | |
| -4 4.54300 host ceph-osd-03 | |
| 3 0.90900 osd.3 up 1.00000 1.00000 | |
| 2 0.90900 osd.2 up 1.00000 1.00000 | |
| 5 0.90900 osd.5 up 1.00000 1.00000 | |
| 7 0.90900 osd.7 up 1.00000 1.00000 | |
| 6 0.90900 osd.6 up 1.00000 1.00000 | |
| And I use the following tree map: | |
| root@ceph-mon-01-[LOC]:~# cat /tmp/crushmap.txt | |
| # begin crush map | |
| tunable choose_local_tries 0 | |
| tunable choose_local_fallback_tries 0 | |
| tunable choose_total_tries 50 | |
| tunable chooseleaf_descend_once 1 | |
| tunable straw_calc_version 1 | |
| # devices | |
| device 0 osd.0 | |
| device 1 osd.1 | |
| device 2 osd.2 | |
| device 3 osd.3 | |
| device 4 osd.4 | |
| device 5 osd.5 | |
| device 6 osd.6 | |
| device 7 osd.7 | |
| device 8 osd.8 | |
| device 9 osd.9 | |
| device 10 osd.10 | |
| device 11 osd.11 | |
| device 12 osd.12 | |
| device 13 osd.13 | |
| device 14 osd.14 | |
| device 15 osd.15 | |
| device 16 osd.16 | |
| device 17 osd.17 | |
| device 18 osd.18 | |
| device 19 osd.19 | |
| device 20 osd.20 | |
| device 21 osd.21 | |
| device 22 osd.22 | |
| device 23 osd.23 | |
| device 24 osd.24 | |
| device 25 osd.25 | |
| device 26 osd.26 | |
| device 27 osd.27 | |
| device 28 osd.28 | |
| #device 29 device29 | |
| device 30 osd.30 | |
| device 31 osd.31 | |
| device 32 osd.32 | |
| device 33 osd.33 | |
| # types | |
| type 0 osd | |
| type 1 host | |
| type 2 chassis | |
| type 3 rack | |
| type 4 row | |
| type 5 pdu | |
| type 6 pod | |
| type 7 room | |
| type 8 datacenter | |
| type 9 region | |
| type 10 root | |
| # buckets | |
| host ceph-osd-02 { | |
| id -3 # do not change unnecessarily | |
| # weight 4.545 | |
| alg straw | |
| hash 0 # rjenkins1 | |
| item osd.15 weight 0.909 | |
| item osd.16 weight 0.909 | |
| item osd.14 weight 0.909 | |
| item osd.12 weight 0.909 | |
| item osd.13 weight 0.909 | |
| } | |
| host ceph-osd-04 { | |
| id -5 # do not change unnecessarily | |
| # weight 4.545 | |
| alg straw | |
| hash 0 # rjenkins1 | |
| item osd.4 weight 0.909 | |
| item osd.8 weight 0.909 | |
| item osd.11 weight 0.909 | |
| item osd.10 weight 0.909 | |
| item osd.9 weight 0.909 | |
| } | |
| datacenter htc-hdd { | |
| id -7 # do not change unnecessarily | |
| # weight 9.086 | |
| alg straw | |
| hash 0 # rjenkins1 | |
| item ceph-osd-02 weight 4.543 | |
| item ceph-osd-04 weight 4.543 | |
| } | |
| host ceph-osd-01 { | |
| id -2 # do not change unnecessarily | |
| # weight 4.545 | |
| alg straw | |
| hash 0 # rjenkins1 | |
| item osd.0 weight 0.909 | |
| item osd.18 weight 0.909 | |
| item osd.19 weight 0.909 | |
| item osd.17 weight 0.909 | |
| item osd.1 weight 0.909 | |
| } | |
| host ceph-osd-03 { | |
| id -4 # do not change unnecessarily | |
| # weight 4.545 | |
| alg straw | |
| hash 0 # rjenkins1 | |
| item osd.3 weight 0.909 | |
| item osd.2 weight 0.909 | |
| item osd.5 weight 0.909 | |
| item osd.7 weight 0.909 | |
| item osd.6 weight 0.909 | |
| } | |
| datacenter dor-hdd { | |
| id -6 # do not change unnecessarily | |
| # weight 9.086 | |
| alg straw | |
| hash 0 # rjenkins1 | |
| item ceph-osd-01 weight 4.543 | |
| item ceph-osd-03 weight 4.543 | |
| } | |
| root hdd { | |
| id -1 # do not change unnecessarily | |
| # weight 4.000 | |
| alg straw | |
| hash 0 # rjenkins1 | |
| item htc-hdd weight 2.000 | |
| item dor-hdd weight 2.000 | |
| } | |
| host ceph-osd-ssd-02 { | |
| id -8 # do not change unnecessarily | |
| # weight 1.820 | |
| alg straw | |
| hash 0 # rjenkins1 | |
| item osd.27 weight 0.455 | |
| item osd.25 weight 0.455 | |
| item osd.26 weight 0.455 | |
| item osd.20 weight 0.455 | |
| } | |
| host ceph-osd-ssd-01 { | |
| id -9 # do not change unnecessarily | |
| # weight 1.820 | |
| alg straw | |
| hash 0 # rjenkins1 | |
| item osd.30 weight 0.455 | |
| item osd.31 weight 0.455 | |
| item osd.32 weight 0.455 | |
| item osd.33 weight 0.455 | |
| } | |
| datacenter dor-ssd { | |
| id -10 # do not change unnecessarily | |
| # weight 2.273 | |
| alg straw | |
| hash 0 # rjenkins1 | |
| item ceph-osd-ssd-01 weight 2.273 | |
| } | |
| datacenter htc-ssd { | |
| id -11 # do not change unnecessarily | |
| # weight 1.000 | |
| alg straw | |
| hash 0 # rjenkins1 | |
| item ceph-osd-ssd-02 weight 1.000 | |
| } | |
| root ssd { | |
| id -12 # do not change unnecessarily | |
| # weight 4.000 | |
| alg straw | |
| hash 0 # rjenkins1 | |
| item htc-ssd weight 2.000 | |
| item dor-ssd weight 2.000 | |
| } | |
| # rules | |
| rule replicated_ruleset { | |
| ruleset 0 | |
| type replicated | |
| min_size 1 | |
| max_size 10 | |
| step take hdd | |
| step chooseleaf firstn 0 type datacenter | |
| step emit | |
| } | |
| rule replicated_ssd_ruleset { | |
| ruleset 1 | |
| type replicated | |
| min_size 0 | |
| max_size 10 | |
| step take ssd | |
| step chooseleaf firstn 1 type datacenter | |
| step emit | |
| } | |
| # end crush map | |
| root@ceph-mon-01-[LOC]:~# ceph osd dump | |
| epoch 4072 | |
| fsid 8be348fe-3cdd-443f-a69e-c2693846dc45 | |
| created 2016-03-25 15:32:19.611410 | |
| modified 2016-04-11 14:14:26.062610 | |
| flags sortbitwise | |
| pool 0 'rbd' replicated size 2 min_size 1 crush_ruleset 0 object_hash rjenkins pg_num 640 pgp_num 640 last_change 541 flags hashpspool stripe_width 0 | |
| pool 1 'data' replicated size 2 min_size 1 crush_ruleset 0 object_hash rjenkins pg_num 640 pgp_num 640 last_change 3503 lfor 3355 flags hashpspool stripe_width 0 | |
| removed_snaps [1~5] | |
| pool 2 'mdsdata' replicated size 2 min_size 1 crush_ruleset 0 object_hash rjenkins pg_num 640 pgp_num 640 last_change 542 flags hashpspool stripe_width 0 | |
| pool 3 'mdsdata_metadata' replicated size 2 min_size 1 crush_ruleset 0 object_hash rjenkins pg_num 640 pgp_num 640 last_change 544 flags hashpspool crash_replay_interval 45 stripe_width 0 | |
| pool 11 'ssd-test' replicated size 2 min_size 1 crush_ruleset 1 object_hash rjenkins pg_num 640 pgp_num 640 last_change 4071 flags hashpspool stripe_width 0 | |
| max_osd 34 | |
| osd.0 up in weight 1 up_from 3535 up_thru 4065 down_at 3533 last_clean_interval [3316,3532) 10.114.160.55:6800/14768 10.114.160.55:6801/14768 10.114.160.55:6802/14768 10.114.160.55:6803/14768 exists,up ba88ef5f-b4eb-4f6c-9ee4-be4267bc98d7 | |
| osd.1 up in weight 1 up_from 3536 up_thru 4065 down_at 3531 last_clean_interval [3316,3530) 10.114.160.55:6804/14942 10.114.160.55:6806/14942 10.114.160.55:6807/14942 10.114.160.55:6809/14942 exists,up 160a9f6c-e969-422f-83ad-0110e3558d0f | |
| osd.2 up in weight 1 up_from 3578 up_thru 4065 down_at 3575 last_clean_interval [3282,3574) 10.114.160.57:6800/5452 10.114.160.57:6802/5452 10.114.160.57:6803/5452 10.114.160.57:6804/5452 exists,up 53a841cc-e6bd-4c31-9075-64c0ac956fb6 | |
| osd.3 up in weight 1 up_from 3578 up_thru 4065 down_at 3573 last_clean_interval [3283,3572) 10.114.160.57:6806/5621 10.114.160.57:6809/5621 10.114.160.57:6810/5621 10.114.160.57:6814/5621 exists,up 555419b2-ede7-4cad-87fd-15a98f40ed16 | |
| osd.4 up in weight 1 up_from 3592 up_thru 4065 down_at 3585 last_clean_interval [1471,3584) 10.114.160.58:6824/17452 10.114.160.58:6825/17452 10.114.160.58:6827/17452 10.114.160.58:6828/17452 exists,up 3af22244-d2a1-4c64-bd92-1f7d6b5a5e2f | |
| osd.5 up in weight 1 up_from 3579 up_thru 4065 down_at 3571 last_clean_interval [3283,3570) 10.114.160.57:6815/5792 10.114.160.57:6816/5792 10.114.160.57:6821/5792 10.114.160.57:6822/5792 exists,up 70a2b238-27cc-499d-b23e-f5190ea9c595 | |
| osd.6 up in weight 1 up_from 3579 up_thru 4065 down_at 3569 last_clean_interval [3560,3568) 10.114.160.57:6825/5961 10.114.160.57:6826/5961 10.114.160.57:6828/5961 10.114.160.57:6829/5961 exists,up 4b2adc55-9a39-4278-8753-1ef93381aa51 | |
| osd.7 up in weight 1 up_from 3579 up_thru 4065 down_at 3567 last_clean_interval [3562,3566) 10.114.160.57:6830/6131 10.114.160.57:6831/6131 10.114.160.57:6832/6131 10.114.160.57:6833/6131 exists,up 3c327aa5-11ee-46b6-85c7-dab4a4c89792 | |
| osd.8 up in weight 1 up_from 3593 up_thru 4065 down_at 3583 last_clean_interval [1471,3582) 10.114.160.58:6829/17623 10.114.160.58:6831/17623 10.114.160.58:6832/17623 10.114.160.58:6833/17623 exists,up 7a49de23-b247-4f62-a593-9e5694285db7 | |
| osd.9 up in weight 1 up_from 3593 up_thru 4065 down_at 3581 last_clean_interval [3564,3580) 10.114.160.58:6834/17793 10.114.160.58:6835/17793 10.114.160.58:6836/17793 10.114.160.58:6837/17793 exists,up 3b0d6f95-c5e5-4cca-90d2-6a383d9a12f6 | |
| osd.10 up in weight 1 up_from 3591 up_thru 4065 down_at 3588 last_clean_interval [1470,3587) 10.114.160.58:6803/17104 10.114.160.58:6807/17104 10.114.160.58:6817/17104 10.114.160.58:6818/17104 exists,up f95b4d20-8325-4b6d-8a94-1613da9c6002 | |
| osd.11 up in weight 1 up_from 3592 up_thru 4065 down_at 3587 last_clean_interval [1470,3586) 10.114.160.58:6819/17273 10.114.160.58:6820/17273 10.114.160.58:6821/17273 10.114.160.58:6823/17273 exists,up 29075d8f-4919-4187-a638-5f62f14df0af | |
| osd.12 up in weight 1 up_from 3549 up_thru 4065 down_at 3547 last_clean_interval [1473,3546) 10.114.160.56:6801/26829 10.114.160.56:6802/26829 10.114.160.56:6803/26829 10.114.160.56:6805/26829 exists,up fd4992e4-bf8e-4692-9eb6-e6dbb7c9eb70 | |
| osd.13 up in weight 1 up_from 3549 up_thru 4065 down_at 3546 last_clean_interval [1473,3545) 10.114.160.56:6800/26998 10.114.160.56:6804/26998 10.114.160.56:6806/26998 10.114.160.56:6807/26998 exists,up a53c8161-7972-4088-aa4c-d8b11c33f0b0 | |
| osd.14 up in weight 1 up_from 3549 up_thru 4065 down_at 3544 last_clean_interval [1474,3543) 10.114.160.56:6808/27167 10.114.160.56:6809/27167 10.114.160.56:6810/27167 10.114.160.56:6811/27167 exists,up c2d30df9-86c9-4a67-9b9d-18040cabc9d3 | |
| osd.15 up in weight 1 up_from 3550 up_thru 4065 down_at 3542 last_clean_interval [1475,3541) 10.114.160.56:6812/27337 10.114.160.56:6813/27337 10.114.160.56:6814/27337 10.114.160.56:6815/27337 exists,up a7816596-4087-460d-b1d0-a4005bdfd644 | |
| osd.16 up in weight 1 up_from 3550 up_thru 4065 down_at 3540 last_clean_interval [1475,3539) 10.114.160.56:6816/27506 10.114.160.56:6817/27506 10.114.160.56:6818/27506 10.114.160.56:6819/27506 exists,up 32ad3229-aa6a-4e68-974d-f95ad19480f5 | |
| osd.17 up in weight 1 up_from 3536 up_thru 4065 down_at 3530 last_clean_interval [3317,3529) 10.114.160.55:6810/15112 10.114.160.55:6812/15112 10.114.160.55:6813/15112 10.114.160.55:6815/15112 exists,up 00df2876-aa53-4bdd-b197-40dc871ab9bb | |
| osd.18 up in weight 1 up_from 3538 up_thru 4065 down_at 3528 last_clean_interval [3317,3527) 10.114.160.55:6816/15282 10.114.160.55:6825/15282 10.114.160.55:6826/15282 10.114.160.55:6827/15282 exists,up ab00824e-9bf9-41bd-a524-e821cad59cf7 | |
| osd.19 up in weight 1 up_from 3538 up_thru 4065 down_at 3526 last_clean_interval [3318,3525) 10.114.160.55:6828/15451 10.114.160.55:6829/15451 10.114.160.55:6830/15451 10.114.160.55:6831/15451 exists,up 930301ab-b6d9-4bba-be16-adc56e8f386c | |
| osd.20 up in weight 1 up_from 3617 up_thru 4071 down_at 3614 last_clean_interval [3510,3613) 10.114.160.60:6800/14274 10.114.160.60:6804/14274 10.114.160.60:6806/14274 10.114.160.60:6807/14274 exists,up 6d527262-2c2c-4247-a435-3c37d466e28b | |
| osd.25 up in weight 1 up_from 3618 up_thru 4071 down_at 3612 last_clean_interval [3510,3611) 10.114.160.60:6808/14586 10.114.160.60:6812/14586 10.114.160.60:6813/14586 10.114.160.60:6814/14586 exists,up 1f0e3183-612a-437e-9796-ebed0ae7910b | |
| osd.26 up in weight 1 up_from 3618 up_thru 4071 down_at 3610 last_clean_interval [3510,3609) 10.114.160.60:6815/14801 10.114.160.60:6816/14801 10.114.160.60:6817/14801 10.114.160.60:6818/14801 exists,up 07fb35e3-913a-4f04-b89f-2c847c4792c3 | |
| osd.27 up in weight 1 up_from 3619 up_thru 4071 down_at 3608 last_clean_interval [3511,3607) 10.114.160.60:6819/15273 10.114.160.60:6820/15273 10.114.160.60:6821/15273 10.114.160.60:6822/15273 exists,up 3fc7e607-c34c-48aa-a134-9947e2bc3bd4 | |
| osd.30 up in weight 1 up_from 3605 up_thru 4071 down_at 3602 last_clean_interval [3394,3601) 10.114.160.59:6800/18601 10.114.160.59:6801/18601 10.114.160.59:6804/18601 10.114.160.59:6805/18601 exists,up a2d6814f-bf31-441f-8f0a-56379fc428ea | |
| osd.31 up in weight 1 up_from 3606 up_thru 4071 down_at 3599 last_clean_interval [3394,3598) 10.114.160.59:6806/18816 10.114.160.59:6807/18816 10.114.160.59:6813/18816 10.114.160.59:6817/18816 exists,up da5f9cee-8eed-4531-8b39-d3c8511ca609 | |
| osd.32 up in weight 1 up_from 3606 up_thru 4071 down_at 3597 last_clean_interval [3557,3596) 10.114.160.59:6818/19128 10.114.160.59:6820/19128 10.114.160.59:6821/19128 10.114.160.59:6822/19128 exists,up 7c57cbb5-35c8-430e-986d-6b5f1721737e | |
| osd.33 up in weight 1 up_from 3606 up_thru 4071 down_at 3595 last_clean_interval [3558,3594) 10.114.160.59:6823/19349 10.114.160.59:6824/19349 10.114.160.59:6825/19349 10.114.160.59:6826/19349 exists,up 7ead9940-b45c-449b-bf90-8f61a6ba2550 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment