{"id":445,"date":"2014-02-26T05:53:33","date_gmt":"2014-02-26T05:53:33","guid":{"rendered":"http:\/\/www.selinuxplus.com\/?p=445"},"modified":"2014-02-27T01:56:40","modified_gmt":"2014-02-27T01:56:40","slug":"ceph-%e6%b7%bb%e5%8a%a0%e7%a7%bb%e9%99%a4osd-%e8%ae%be%e5%a4%87","status":"publish","type":"post","link":"http:\/\/www.selinuxplus.com\/?p=445","title":{"rendered":"ceph \u6dfb\u52a0\/\u79fb\u9664osd \u8bbe\u5907"},"content":{"rendered":"<p>1 \u5220\u9664osd\u8bbe\u5907<\/p>\n<pre class=\"lang:default decode:true \" >[root@ceph04 ~]# service ceph stop osd.3 \/\/\u505c\u6b62\u8be5\u8bbe\u5907\r\n=== osd.3 ===\r\nStopping ceph osd.3 on ceph04...kill 12012...done\r\n[root@ceph04 ~]# ceph osd crush remove osd.3 \/\/\u4eceCRUSH\u4e2d\u79fb\u9664\r\nremoved item id 3 name 'osd.3' from crush map\r\n[root@ceph04 ~]# ceph osd tree\r\n\r\n# id    weight  type name       up\/down reweight\r\n-1      3       root default\r\n-3      3               rack unknownrack\r\n-2      1                       host ceph01\r\n0       1                               osd.0   up      1\r\n-4      1                       host ceph02\r\n1       1                               osd.1   up      1\r\n-5      1                       host ceph03\r\n2       1                               osd.2   up      1\r\n-6      0                       host ceph04\r\n\r\n3       0       osd.3   down      1\r\n\r\n[root@ceph04 ~]# ceph auth del osd.3 \/\/\u4ece\u8ba4\u8bc1\u4e2d\u5220\u9664\r\nupdated\r\n\r\n[root@ceph04 ~]# ceph osd rm 3 \/\/\u5220\u9664\r\nremoved osd.3<\/pre>\n<p>2 \u65b0\u589eosd \u8bbe\u5907<\/p>\n<pre class=\"lang:default decode:true \" >[root@ceph04 data]# mkfs.xfs -f \/dev\/sdb \/\/\u683c\u5f0f\u5316\u5206\u533a\r\n[root@ceph04 data]# mount   \/dev\/sdb \/data\/osd.3\/ \/\/\u6302\u8f7d\u5206\u533a\r\n[root@ceph04 data]# ceph-osd -i 3 --mkfs --mkkey \/\/\u5bf9\u6307\u5b9a\u7684\u76ee\u5f55mkcephfs\r\n2014-02-26 11:17:48.014785 7f94ef4947a0 -1 journal FileJournal::_open: disabling aio for non-block journal.  Use journal_force_aio to force use of aio anyway\r\n2014-02-26 11:17:48.049559 7f94ef4947a0 -1 journal FileJournal::_open: disabling aio for non-block journal.  Use journal_force_aio to force use of aio anyway\r\n2014-02-26 11:17:48.059596 7f94ef4947a0 -1 filestore(\/data\/osd.3) could not find 23c2fcde\/osd_superblock\/0\/\/-1 in index: (2) No such file or directory\r\n2014-02-26 11:17:48.150783 7f94ef4947a0 -1 created object store \/data\/osd.3 journal \/data\/osd.3\/journal for osd.3 fsid c9871314-3f0b-42c5-8bc7-ad14d41977a0\r\n2014-02-26 11:17:48.150840 7f94ef4947a0 -1 auth: error reading file: \/data\/osd.3\/keyring: can't open \/data\/osd.3\/keyring: (2) No such file or directory\r\n2014-02-26 11:17:48.150949 7f94ef4947a0 -1 created new key in keyring \/data\/osd.3\/keyring\r\n\r\n[root@ceph04 osd.3]# ceph auth add osd.3 osd 'allow *' mon 'allow rwx' -i \/data\/osd.3\/keyring \/\/\u6dfb\u52a0allow rwx \u89c4\u5219\r\n2014-02-26 11:19:26.004404 7f46b7ee7760 -1 read 56 bytes from \/data\/osd.3\/keyring\r\nadded key for osd.3\r\n\r\n\r\noot@ceph01 ceph]# ceph osd getcrushmap -o map \/\/\u83b7\u5f97crushmap\u4fe1\u606f\r\ngot crush map from osdmap epoch 12\r\n[root@ceph01 ceph]# ls\r\nfetch_config  map  ceph.conf  ceph.keyring\r\n[root@ceph01 ceph]# crushtool -d map  \/\/\u683c\u5f0f\u5316\u8f93\u51facrushmap\u4fe1\u606f\r\n# begin crush map\r\n\r\n# devices\r\ndevice 0 osd.0\r\ndevice 1 osd.1\r\ndevice 2 osd.2\r\n\r\n# types\r\ntype 0 osd\r\ntype 1 host\r\ntype 2 rack\r\ntype 3 row\r\ntype 4 room\r\ntype 5 datacenter\r\ntype 6 root\r\n\r\n# buckets\r\nhost ceph01 {\r\n        id -2           # do not change unnecessarily\r\n        # weight 1.000\r\n        alg straw\r\n        hash 0  # rjenkins1\r\n        item osd.0 weight 1.000\r\n}\r\nhost ceph02 {\r\n        id -4           # do not change unnecessarily\r\n        # weight 1.000\r\n        alg straw\r\n        hash 0  # rjenkins1\r\n        item osd.1 weight 1.000\r\n}\r\nhost ceph03 {\r\n        id -5           # do not change unnecessarily\r\n        # weight 1.000\r\n        alg straw\r\n        hash 0  # rjenkins1\r\n        item osd.2 weight 1.000\r\n}\r\nhost ceph04 {\r\n        id -6           # do not change unnecessarily\r\n        # weight 0.000\r\n        alg straw\r\n        hash 0  # rjenkins1\r\n}\r\nrack unknownrack {\r\n        id -3           # do not change unnecessarily\r\n        # weight 3.000\r\n        alg straw\r\n        hash 0  # rjenkins1\r\n        item ceph01 weight 1.000\r\n        item ceph02 weight 1.000\r\n        item ceph03 weight 1.000\r\n        item ceph04 weight 0.000\r\n}\r\nroot default {\r\n        id -1           # do not change unnecessarily\r\n        # weight 3.000\r\n        alg straw\r\n        hash 0  # rjenkins1\r\n        item unknownrack weight 3.000\r\n}\r\n\r\n# rules\r\nrule data {\r\n        ruleset 0\r\n        type replicated\r\n        min_size 1\r\n        max_size 10\r\n        step take default\r\n        step chooseleaf firstn 0 type host\r\n        step emit\r\n}\r\nrule metadata {\r\n        ruleset 1\r\n        type replicated\r\n        min_size 1\r\n        max_size 10\r\n        step take default\r\n        step chooseleaf firstn 0 type host\r\n        step emit\r\n}\r\nrule rbd {\r\n        ruleset 2\r\n        type replicated\r\n        min_size 1\r\n        max_size 10\r\n        step take default\r\n        step chooseleaf firstn 0 type host\r\n        step emit\r\n}\r\n[root@ceph04 osd.3]# ceph osd crush set 3 1.0 root=default rack=unknownrack host=ceph04 \/\/\u8bbe\u7f6ecrushmap\r\nset item id 3 name 'osd.3' weight 1 at location {host=ceph04,rack=unknownrack,root=default} to crush map\r\n\r\n\r\n\r\nhost ceph04 {\r\n        id -6           # do not change unnecessarily\r\n        # weight 1.000\r\n        alg straw\r\n        hash 0  # rjenkins1\r\n        item osd.3 weight 1.000\r\n}\r\nrack unknownrack {\r\n        id -3           # do not change unnecessarily\r\n        # weight 4.000\r\n        alg straw\r\n        hash 0  # rjenkins1\r\n        item ceph01 weight 1.000\r\n        item ceph02 weight 1.000\r\n        item ceph03 weight 1.000\r\n        item ceph04 weight 1.000\r\n************************\r\nceph osd crush set 3 1.0 root=default rack=unknownrack host=ceph04\u7684\u53e6\u4e00\u79cd\u505a\u6cd5\r\n\r\n1.\u4fee\u6539map.txt\r\nvi map.txt\r\n#devices\r\ndevice 0 osd.0\r\ndevice 1 osd.1\r\ndevice 2 osd.2\r\ndevice 3 osd.3\r\n  \r\nhost osd3 {\r\n        id -1\r\n        alg straw\r\n        hash 0\r\n        item osd.3 weight 1.00\r\n}\r\n\r\n2.\u7f16\u8bd1crushmap\r\n    cephtool -c \/root\/map.txt -o map\r\n3.\u5c06\u7f16\u8bd1\u597d\u7684map\u518d\u6b21\u5bfc\u5165\r\n    ceph osd crushmap -i map\r\n*******************************\r\n\r\n[root@ceph04 osd.3]# service ceph start osd.3 \/\/\u542f\u52a8osd.3\r\n=== osd.3 ===\r\nMounting xfs on ceph04:\/data\/osd.3\r\ncreate-or-move updated item id 3 name 'osd.3' weight 0.02 at location {host=ceph04,root=default} to crush map\r\nStarting Ceph osd.3 on ceph04...\r\nstarting osd.3 at :\/0 osd_data \/data\/osd.3 \/data\/osd.3\/journal\r\n\r\n[root@ceph04 osd.3]# ceph -s\r\n   health HEALTH_WARN 1 pgs recovery_wait; 1 pgs stuck unclean; recovery 1\/42 degraded (2.381%);  recovering 4 o\/s, 3553B\/s; clock skew detected on mon.ceph02, mon.ceph03\r\n   monmap e1: 3 mons at {ceph01=192.168.9.62:6789\/0,ceph02=192.168.9.63:6789\/0,ceph03=192.168.9.73:6789\/0}, election epoch 6, quorum 0,1,2 ceph01,ceph02,ceph03\r\n   osdmap e16: 4 osds: 4 up, 4 in\r\n    pgmap v459: 960 pgs: 14 active, 945 active+clean, 1 active+recovery_wait; 9518 bytes data, 4154 MB used, 77725 MB \/ 81880 MB avail; 1001B\/s wr, 0op\/s; 1\/42 degraded (2.381%);  recovering 4 o\/s, 3553B\/s\r\n   mdsmap e5: 1\/1\/1 up {0=ucms01=up:active}, 1 up:standby\r\n\r\n[root@ceph04 osd.3]# ceph osd tree\r\n\r\n# id    weight  type name       up\/down reweight\r\n-1      4       root default\r\n-3      4               rack unknownrack\r\n-2      1                       host ceph01\r\n0       1                               osd.0   up      1\r\n-4      1                       host ceph02\r\n1       1                               osd.1   up      1\r\n-5      1                       host ceph03\r\n2       1                               osd.2   up      1\r\n-6      1                       host ceph04\r\n3       1                               osd.3   up      1<\/pre>\n<p>\u53ef\u80fd\u9047\u5230\u7684\u95ee\u9898\uff1a<\/p>\n<pre class=\"lang:default decode:true \" >ceph mds stat\r\nmdsmap e63: 1\/1\/1 up {0=ceph02=up:replay}, 1 up:standby<\/pre>\n<p>osd\u65e5\u5fd7\u4fe1\u606f\u4e3a\uff1a<\/p>\n<pre class=\"lang:default decode:true \" >2014-02-26 10:42:15.386552 7f33b5e40700  0 -- 192.168.9.63:6802\/13005 &gt;&gt; 192.168.9.62:6803\/18894 pipe(0x3fa1900 sd=29 :6802 s=0 pgs=0 cs=0 l=0).accept connect_seq 0 vs existing 0 state connecting\r\n2014-02-26 10:42:16.394540 7f33b5639700  0 -- 192.168.9.63:6802\/13005 &gt;&gt; 192.168.9.39:6802\/11369 pipe(0x3fa1680 sd=33 :6802 s=0 pgs=0 cs=0 l=0).accept connect_seq 0 vs existing 0 state wait\r\n2014-02-26 10:42:17.029623 7f33b5033700  0 -- 192.168.9.63:6801\/13005 &gt;&gt; 192.168.9.62:0\/3872604662 pipe(0x3fa4d80 sd=35 :6801 s=0 pgs=0 cs=0 l=0).accept peer addr is really 192.168.9.62:0\/3872604662 (socket is 192.168.9.62:39504\/0)<\/pre>\n<p>mds \u65e5\u5fd7\u4fe1\u606f\u4e3a\uff1a<\/p>\n<pre class=\"lang:default decode:true \" >2014-02-26 10:42:12.824284 7f57ba9de700  0 -- 192.168.9.63:6800\/12877 &gt;&gt; 192.168.9.62:6801\/17332 pipe(0x2450000 sd=17 :37551 s=1 pgs=0 cs=0 l=1).connect claims to be 0.0.0.0:6801\/18894 not 192.168.9.62:6801\/17332 - wrong node!\r\n<\/pre>\n<p>\u9519\u8bef\u7684\u539f\u56e0\u53ef\u80fd\u4e3a\uff1aosd \u6ca1\u6709\u5728crushmap\u4e2d\u3002<\/p>\n","protected":false},"excerpt":{"rendered":"<p>1 \u5220\u9664osd\u8bbe\u5907 [root@ceph04 ~]&hellip;<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":[],"categories":[84,83],"tags":[180,90,89,88],"_links":{"self":[{"href":"http:\/\/www.selinuxplus.com\/index.php?rest_route=\/wp\/v2\/posts\/445"}],"collection":[{"href":"http:\/\/www.selinuxplus.com\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/www.selinuxplus.com\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/www.selinuxplus.com\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/www.selinuxplus.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=445"}],"version-history":[{"count":3,"href":"http:\/\/www.selinuxplus.com\/index.php?rest_route=\/wp\/v2\/posts\/445\/revisions"}],"predecessor-version":[{"id":449,"href":"http:\/\/www.selinuxplus.com\/index.php?rest_route=\/wp\/v2\/posts\/445\/revisions\/449"}],"wp:attachment":[{"href":"http:\/\/www.selinuxplus.com\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=445"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/www.selinuxplus.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=445"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/www.selinuxplus.com\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=445"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}