CRUSH (Controlled Replication Under Scalable Hashing)
CRUSH (Controlled Replication Under Scalable Hashing)
Embedded Files
ceph osd crush tree
ID CLASS WEIGHT (compat) TYPE NAME
-1 0.07037 root default
-9 0.01759 0.01759 host ubuntu-ceph1
6 hdd 0.00980 0.00980 osd.6
7 ssd 0.00780 0.00780 osd.7
-7 0.01759 0.01759 host ubuntu-ceph2
5 hdd 0.00980 0.00980 osd.5
4 ssd 0.00780 0.00780 osd.4
-3 0.01759 0.01759 host ubuntu-ceph3
0 hdd 0.00980 0.00980 osd.0
1 ssd 0.00780 0.00780 osd.1
-5 0.01759 0.01759 host ubuntu-ceph4
2 hdd 0.00980 0.00980 osd.2
3 ssd 0.00780 0.00780 osd.3
Its my current topology, I want to change this to my real topology for gaining a better understanding of my actual situation for decision-making
For example I have two rack in my datacenter and I each rack has two server and I want to make ceph crush to understand my real topology
so we can do it by editing the crush map topology either by cli or editing crush map file manually and applying it.
first I go through bt cli, its easy and could make some decision for number choosing for buckets
Bucket — A container node in the CRUSH hierarchy (e.g. host, rack, root) that groups other buckets or devices together.
Leaf — The lowest, non-divisible node in the CRUSH tree — an actual OSD/device, with nothing nested inside it.
Device — A single storage unit (an OSD, e.g. osd.0) identified by an ID and optionally tagged with a class like hdd or ssd.
Type — A named level in the CRUSH hierarchy (e.g. type 1 host) used to define failure domains and how far apart replicas must be placed.
now in one of nodes that has access to client.admin, get backup first of current crushmap:
ceph osd getcrushmap -o crushmap.bin
crushtool -d crushmap.bin -o crushmap.txt
cp crushmap.txt crushmap.txt.bak
as you see I named my first file to crushmap.bin becauseits a binary file and we need to decomplie it to human readable text file by crush tool, which has been installed before
Notice that we had not create any POOL yet
Making out two Rack
ceph osd crush add-bucket rack1 rack
ceph osd crush add-bucket rack2 rack
Put Racks into default root crush hirearchy
ceph osd crush move rack1 root=default
ceph osd crush move rack2 root=default
Now put you hosts under the owns Racks
ceph osd crush move ubuntu-ceph1 rack=rack1
ceph osd crush move ubuntu-ceph2 rack=rack1
ceph osd crush move ubuntu-ceph3 rack=rack2
ceph osd crush move ubuntu-ceph4 rack=rack2
now look at your crush hirearchy map
ceph osd crush tree
ID CLASS WEIGHT (compat) TYPE NAME
-1 0.07037 root default
-16 0.03519 0.03519 rack rack1
-9 0.01759 0.01759 host ubuntu-ceph1
6 hdd 0.00980 0.00980 osd.6
7 ssd 0.00780 0.00780 osd.7
-7 0.01759 0.01759 host ubuntu-ceph2
5 hdd 0.00980 0.00980 osd.5
4 ssd 0.00780 0.00780 osd.4
-17 0.03519 0.03519 rack rack2
-3 0.01759 0.01759 host ubuntu-ceph3
0 hdd 0.00980 0.00980 osd.0
1 ssd 0.00780 0.00780 osd.1
-5 0.01759 0.01759 host ubuntu-ceph4
2 hdd 0.00980 0.00980 osd.2
3 ssd 0.00780 0.00780 osd.3
Now its better to take a closer look to crushmap file
ceph osd getcrushmap -o crushmap_current.bin
crushtool -d crushmap_current.bin -o crushmap_current.txt
crushmap.txt
crushmap.txt
# begin crush map
tunable choose_local_tries 0
tunable choose_local_fallback_tries 0
tunable choose_total_tries 50
tunable chooseleaf_descend_once 1
tunable chooseleaf_vary_r 1
tunable chooseleaf_stable 1
tunable straw_calc_version 1
tunable allowed_bucket_algs 54
# devices
device 0 osd.0 class hdd
device 1 osd.1 class ssd
device 2 osd.2 class hdd
device 3 osd.3 class ssd
device 4 osd.4 class ssd
device 5 osd.5 class hdd
device 6 osd.6 class hdd
device 7 osd.7 class ssd
# type
type 0 osd
type 1 host
type 2 chassis
type 3 rack
type 4 row
type 5 pdu
type 6 pod
type 7 room
type 8 datacenter
type 9 zone
type 10 region
type 11 root
# buckets
host ubuntu-ceph3 {
id -3 # do not change unnecessarily
id -4 class hdd # do not change unnecessarily
id -11 class ssd # do not change unnecessarily
# weight 0.01759
alg straw2
hash 0 # rjenkins1
item osd.0 weight 0.00980
item osd.1 weight 0.00780
}
host ubuntu-ceph4 {
id -5 # do not change unnecessarily
id -6 class hdd # do not change unnecessarily
id -12 class ssd # do not change unnecessarily
# weight 0.01759
alg straw2
hash 0 # rjenkins1
item osd.2 weight 0.00980
item osd.3 weight 0.00780
}
host ubuntu-ceph2 {
id -7 # do not change unnecessarily
id -8 class hdd # do not change unnecessarily
id -13 class ssd # do not change unnecessarily
# weight 0.01759
alg straw2
hash 0 # rjenkins1
item osd.4 weight 0.00780
item osd.5 weight 0.00980
}
host ubuntu-ceph1 {
id -9 # do not change unnecessarily
id -10 class hdd # do not change unnecessarily
id -14 class ssd # do not change unnecessarily
# weight 0.01759
alg straw2
hash 0 # rjenkins1
item osd.6 weight 0.00980
item osd.7 weight 0.00780
}
root default {
id -1 # do not change unnecessarily
id -2 class hdd # do not change unnecessarily
id -15 class ssd # do not change unnecessarily
# weight 0.07037
alg straw2
hash 0 # rjenkins1
item ubuntu-ceph3 weight 0.01759
item ubuntu-ceph4 weight 0.01759
item ubuntu-ceph2 weight 0.01759
item ubuntu-ceph1 weight 0.01759
}
# rules
rule replicated_rule {
id 0
type replicated
step take default
step chooseleaf firstn 0 type host
step emit
}
rule replicated_hdd {
id 1
type replicated
step take default class hdd
step chooseleaf firstn 0 type host
step emit
}
rule replicated_ssd {
id 2
type replicated
step take default class ssd
step chooseleaf firstn 0 type host
step emit
}
# choose_args
choose_args 18446744073709551615 {
{
bucket_id -1
weight_set [
[ 0.01759 0.01759 0.01759 0.01759 ]
]
}
{
bucket_id -2
weight_set [
[ 0.00980 0.00980 0.00980 0.00980 ]
]
}
{
bucket_id -3
weight_set [
[ 0.00980 0.00780 ]
]
}
{
bucket_id -4
weight_set [
[ 0.00980 ]
]
}
{
bucket_id -5
weight_set [
[ 0.00980 0.00780 ]
]
}
{
bucket_id -6
weight_set [
[ 0.00980 ]
]
}
{
bucket_id -7
weight_set [
[ 0.00780 0.00980 ]
]
}
{
bucket_id -8
weight_set [
[ 0.00980 ]
]
}
{
bucket_id -9
weight_set [
[ 0.00980 0.00780 ]
]
}
{
bucket_id -10
weight_set [
[ 0.00980 ]
]
}
{
bucket_id -11
weight_set [
[ 0.00780 ]
]
}
{
bucket_id -12
weight_set [
[ 0.00780 ]
]
}
{
bucket_id -13
weight_set [
[ 0.00780 ]
]
}
{
bucket_id -14
weight_set [
[ 0.00780 ]
]
}
{
bucket_id -15
weight_set [
[ 0.00780 0.00780 0.00780 0.00780 ]
]
}
}
# end crush map
crushmap_current.txt
crushmap_current.txt
# begin crush map
tunable choose_local_tries 0
tunable choose_local_fallback_tries 0
tunable choose_total_tries 50
tunable chooseleaf_descend_once 1
tunable chooseleaf_vary_r 1
tunable chooseleaf_stable 1
tunable straw_calc_version 1
tunable allowed_bucket_algs 54
# devices
device 0 osd.0 class hdd
device 1 osd.1 class ssd
device 2 osd.2 class hdd
device 3 osd.3 class ssd
device 4 osd.4 class ssd
device 5 osd.5 class hdd
device 6 osd.6 class hdd
device 7 osd.7 class ssd
# types
type 0 osd
type 1 host
type 2 chassis
type 3 rack
type 4 row
type 5 pdu
type 6 pod
type 7 room
type 8 datacenter
type 9 zone
type 10 region
type 11 root
# buckets
host ubuntu-ceph1 {
id -9 # do not change unnecessarily
id -10 class hdd # do not change unnecessarily
id -14 class ssd # do not change unnecessarily
# weight 0.01759
alg straw2
hash 0 # rjenkins1
item osd.6 weight 0.00980
item osd.7 weight 0.00780
}
host ubuntu-ceph2 {
id -7 # do not change unnecessarily
id -8 class hdd # do not change unnecessarily
id -13 class ssd # do not change unnecessarily
# weight 0.01759
alg straw2
hash 0 # rjenkins1
item osd.4 weight 0.00780
item osd.5 weight 0.00980
}
rack rack1 {
id -16 # do not change unnecessarily
id -20 class hdd # do not change unnecessarily
id -21 class ssd # do not change unnecessarily
# weight 0.03519
alg straw2
hash 0 # rjenkins1
item ubuntu-ceph1 weight 0.01759
item ubuntu-ceph2 weight 0.01759
}
host ubuntu-ceph3 {
id -3 # do not change unnecessarily
id -4 class hdd # do not change unnecessarily
id -11 class ssd # do not change unnecessarily
# weight 0.01759
alg straw2
hash 0 # rjenkins1
item osd.0 weight 0.00980
item osd.1 weight 0.00780
}
host ubuntu-ceph4 {
id -5 # do not change unnecessarily
id -6 class hdd # do not change unnecessarily
id -12 class ssd # do not change unnecessarily
# weight 0.01759
alg straw2
hash 0 # rjenkins1
item osd.2 weight 0.00980
item osd.3 weight 0.00780
}
rack rack2 {
id -17 # do not change unnecessarily
id -18 class hdd # do not change unnecessarily
id -19 class ssd # do not change unnecessarily
# weight 0.03519
alg straw2
hash 0 # rjenkins1
item ubuntu-ceph3 weight 0.01759
item ubuntu-ceph4 weight 0.01759
}
root default {
id -1 # do not change unnecessarily
id -2 class hdd # do not change unnecessarily
id -15 class ssd # do not change unnecessarily
# weight 0.07037
alg straw2
hash 0 # rjenkins1
item rack1 weight 0.03519
item rack2 weight 0.03519
}
# rules
rule replicated_rule {
id 0
type replicated
step take default
step chooseleaf firstn 0 type host
step emit
}
rule replicated_hdd {
id 1
type replicated
step take default class hdd
step chooseleaf firstn 0 type host
step emit
}
rule replicated_ssd {
id 2
type replicated
step take default class ssd
step chooseleaf firstn 0 type host
step emit
}
# choose_args
choose_args 18446744073709551615 {
{
bucket_id -1
weight_set [
[ 0.03519 0.03519 ]
]
}
{
bucket_id -2
weight_set [
[ 0.01959 0.01959 ]
]
}
{
bucket_id -3
weight_set [
[ 0.00980 0.00780 ]
]
}
{
bucket_id -4
weight_set [
[ 0.00980 ]
]
}
{
bucket_id -5
weight_set [
[ 0.00980 0.00780 ]
]
}
{
bucket_id -6
weight_set [
[ 0.00980 ]
]
}
{
bucket_id -7
weight_set [
[ 0.00780 0.00980 ]
]
}
{
bucket_id -8
weight_set [
[ 0.00980 ]
]
}
{
bucket_id -9
weight_set [
[ 0.00980 0.00780 ]
]
}
{
bucket_id -10
weight_set [
[ 0.00980 ]
]
}
{
bucket_id -11
weight_set [
[ 0.00780 ]
]
}
{
bucket_id -12
weight_set [
[ 0.00780 ]
]
}
{
bucket_id -13
weight_set [
[ 0.00780 ]
]
}
{
bucket_id -14
weight_set [
[ 0.00780 ]
]
}
{
bucket_id -15
weight_set [
[ 0.01559 0.01559 ]
]
}
{
bucket_id -16
weight_set [
[ 0.01759 0.01759 ]
]
}
{
bucket_id -17
weight_set [
[ 0.01759 0.01759 ]
]
}
{
bucket_id -18
weight_set [
[ 0.00980 0.00980 ]
]
}
{
bucket_id -19
weight_set [
[ 0.00780 0.00780 ]
]
}
{
bucket_id -20
weight_set [
[ 0.00980 0.00980 ]
]
}
{
bucket_id -21
weight_set [
[ 0.00780 0.00780 ]
]
}
}
# end crush map
Page updated
Google Sites
Report abuse