Ceph Cluster Installation

This is a stub for installing a ceph cluster.

Host Setup

Setup the following hosts as container servers with our automated setup.

  • admin1.ceph.bfh.ch
  • admin2.ceph.bfh.ch
  • admin3.ceph.bfh.ch
  • storage1.ceph.bfh.ch
  • storage2.ceph.bfh.ch
  • storage3.ceph.bfh.ch
  • storage4.ceph.bfh.ch
  • storage5.ceph.bfh.ch
  • storage6.ceph.bfh.ch

FIXME: Configure storage hosts

Temporary post-setup on the storage hosts until we’re running the OSDs in containers too.

sudo -i

cat > "/etc/security/access.conf" << EOF
#  ITS Linux System Administration <its-linux-sysadmin@lists.bfh.ch>
+ : sysadmin : ALL
+ : (INR_SERVICES.its-linux-sysadmin) : ALL

# ceph-deploy@master.ceph.bfh.ch
+ : ceph-deploy : ALL

#  All other users should be denied to get access from all sources.
- : ALL : ALL
EOF

echo 'Defaults:ceph-deploy !requiretty' > "/etc/sudoers.d/ceph-deploy"
echo "ceph-deploy ALL=NOPASSWD: ALL" >> "/etc/sudoers.d/ceph-deploy"
chmod 0440 "/etc/sudoers.d"/*

adduser --gecos "ceph-deploy@master.ceph.bfh.ch,,," --disabled-login ceph-deploy
gpasswd -a ceph-deploy sudo
echo ceph-deploy:bfh | chpasswd

cat > "/etc/ssh/authorized_keys/ceph-deploy" << EOF
# ceph-deploy@master.ceph.bfh.ch
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDrTdoDBBRRS8nLi6/LI1J2/b20LB8FAOIeOFU8s1aXAIyn+p8/l1lKgpn98756qIS7P9JoAG/idIJ45UJRXAlnrKTW4G16bD/mXgF23ZHOI2HBuiAoIBkJaFOoRv/ih/08mXW/ips82yuoObvHToXFCypg7HkDssPUutzPseEDto49EysRc2AKe4kEr79oVek/891yqdcgZXkY+Jcs4jp/cKY/YCsmA4hATToHw7JaoJnqyrsvIhS73cqYCfKsqpTCgt9QPMjBRHgHmAc+V+/odp1iyeOwGokMb7jEPmJNvkMjfZFKocWox8A8yJ/jKnA1ibdB//6bcT3guD2k4bH/AydlryuxIlkJQi7OgBZ7ZeSaHuXK0AAPT5FsdxP4GN0TmzsC2YGIJB7N0AS//zci0FLQPc4cWL2ETmvoFMv4w5JJedNKeoA4Nen6xXaD4uqi+ths/EvJjFnhG9pxnnFd1Uegc2MBFbqC1WWDh0hrXdiCED38S9iizJ6oYuGhqav8gcn469zQPj99OOrMRi2H8/yy8GBd55LRAjU3D99cK8fc6yGHmz2m8D9DAyRCSnVd/D0UlCdF0IgSS5e9HevHPI/EgKwH+HLyCwixKc8eGYg8HIT6MuCeaO/6cCHkWC0XH7l3XQ35kn82ZJaeo45cubHVv9BUav+rl1M5FMHOzw== ceph-deploy@master.ceph.bfh.ch
EOF

apt install --yes ceph ceph-osd

Container Setup

Setup the following services as containers with our automated setup.

  • master.ceph.bfh.ch
  • mon[1-3].luminous.ceph.bfh.ch
  • mgr[1-3].luminous.ceph.bfh.ch
  • mds[1-15].luminous.ceph.bfh.ch
  • rgw[1-3].luminous.ceph.bfh.ch
  • nfs.ceph.bfh.ch
  • smb.ceph.bfh.ch

Bootstrap Ceph cluster

ssh ceph-deploy@master.ceph.bfh.ch

# Create config directory
mkdir -p /srv/master.ceph.bfh.ch/ceph-deploy
cd /srv/master.ceph.bfh.ch/ceph-deploy

# Create mon1.ceph.bfh.ch
ceph-deploy new --cluster-network 10.2.0.100/16 --public-network 147.87.226.0/24 mon1

# checkout updated ceph.conf from ceph-deploy.git on git.linux.bfh.ch

ceph-deploy mon create-initial
ceph-deploy mgr create mgr1

# Create storage1.ceph.bfh.ch
ceph-deploy osd create --data /dev/sdc storage1
ceph-deploy osd create --data /dev/sdd storage1
ceph-deploy osd create --data /dev/sde storage1
ceph-deploy osd create --data /dev/sdf storage1
ceph-deploy osd create --data /dev/sdg storage1
ceph-deploy osd create --data /dev/sdh storage1
ceph-deploy osd create --data /dev/sdi storage1
ceph-deploy osd create --data /dev/sdj storage1
ceph-deploy osd create --data /dev/sdk storage1
ceph-deploy osd create --data /dev/sdl storage1
ceph-deploy osd create --data /dev/sdm storage1
ceph-deploy osd create --data /dev/sdn storage1

# Create storage2.ceph.bfh.ch
ceph-deploy osd create --data /dev/sdc storage2
ceph-deploy osd create --data /dev/sdd storage2
ceph-deploy osd create --data /dev/sde storage2
ceph-deploy osd create --data /dev/sdf storage2
ceph-deploy osd create --data /dev/sdg storage2
ceph-deploy osd create --data /dev/sdh storage2
ceph-deploy osd create --data /dev/sdi storage2
ceph-deploy osd create --data /dev/sdj storage2
ceph-deploy osd create --data /dev/sdk storage2
ceph-deploy osd create --data /dev/sdl storage2
ceph-deploy osd create --data /dev/sdm storage2
ceph-deploy osd create --data /dev/sdn storage2

# Create storage3.ceph.bfh.ch
ceph-deploy osd create --data /dev/sdc storage3
ceph-deploy osd create --data /dev/sdd storage3
ceph-deploy osd create --data /dev/sde storage3
ceph-deploy osd create --data /dev/sdf storage3
ceph-deploy osd create --data /dev/sdg storage3
ceph-deploy osd create --data /dev/sdh storage3
ceph-deploy osd create --data /dev/sdi storage3
ceph-deploy osd create --data /dev/sdj storage3
ceph-deploy osd create --data /dev/sdk storage3
ceph-deploy osd create --data /dev/sdl storage3
ceph-deploy osd create --data /dev/sdm storage3
ceph-deploy osd create --data /dev/sdn storage3

# Create storage4.ceph.bfh.ch
ceph-deploy osd create --data /dev/sdc storage4
ceph-deploy osd create --data /dev/sdd storage4
ceph-deploy osd create --data /dev/sde storage4
ceph-deploy osd create --data /dev/sdf storage4
ceph-deploy osd create --data /dev/sdg storage4
ceph-deploy osd create --data /dev/sdh storage4
ceph-deploy osd create --data /dev/sdi storage4
ceph-deploy osd create --data /dev/sdj storage4
ceph-deploy osd create --data /dev/sdk storage4
ceph-deploy osd create --data /dev/sdl storage4
ceph-deploy osd create --data /dev/sdm storage4
ceph-deploy osd create --data /dev/sdn storage4

# Create storage5.ceph.bfh.ch
ceph-deploy osd create --data /dev/sdc storage5
ceph-deploy osd create --data /dev/sdd storage5
ceph-deploy osd create --data /dev/sde storage5
ceph-deploy osd create --data /dev/sdf storage5
ceph-deploy osd create --data /dev/sdg storage5
ceph-deploy osd create --data /dev/sdh storage5
ceph-deploy osd create --data /dev/sdi storage5
ceph-deploy osd create --data /dev/sdj storage5
ceph-deploy osd create --data /dev/sdk storage5
ceph-deploy osd create --data /dev/sdl storage5
ceph-deploy osd create --data /dev/sdm storage5
ceph-deploy osd create --data /dev/sdn storage5

# Create storage6.ceph.bfh.ch
ceph-deploy osd create --data /dev/sdc storage6
ceph-deploy osd create --data /dev/sdd storage6
ceph-deploy osd create --data /dev/sde storage6
ceph-deploy osd create --data /dev/sdf storage6
ceph-deploy osd create --data /dev/sdg storage6
ceph-deploy osd create --data /dev/sdh storage6
ceph-deploy osd create --data /dev/sdi storage6
ceph-deploy osd create --data /dev/sdj storage6
ceph-deploy osd create --data /dev/sdk storage6
ceph-deploy osd create --data /dev/sdl storage6
ceph-deploy osd create --data /dev/sdm storage6
ceph-deploy osd create --data /dev/sdn storage6

# Create mon[2-3].ceph.bfh.ch
ceph-deploy mon add mon2
ceph-deploy mon add mon3

# Create mgr[1-3].ceph.bfh.ch
ceph-deploy mgr create mgr2
ceph-deploy mgr create mgr3

# Create mds[1-15].ceph.bfh.ch
ceph-deploy mds create mds1
ceph-deploy mds create mds2
ceph-deploy mds create mds3
ceph-deploy mds create mds4
ceph-deploy mds create mds5
ceph-deploy mds create mds6
ceph-deploy mds create mds7
ceph-deploy mds create mds8
ceph-deploy mds create mds9
ceph-deploy mds create mds10
ceph-deploy mds create mds11
ceph-deploy mds create mds12
ceph-deploy mds create mds13
ceph-deploy mds create mds14
ceph-deploy mds create mds15

# Create rgw[1-3].ceph.bfh.ch
ceph-deploy rgw create rgw1
ceph-deploy rgw create rgw2
ceph-deploy rgw create rgw3

# Deploy ceph keyrings
ceph-deploy admin \
  master \
  mon1 mon2 mon3 \
  mgr1 mgr2 mgr3 \
  mds1 mds2 mds3 mds4 mds5 mds6 mds7 mds8 mds9 mds10 mds11 mds12 mds13 mds14 mds15 \
  rgw1 rgw2 rgw3 \
  storage1 storage2 storage3 storage4 storage5 storage6

sudo chmod +r /etc/ceph/ceph.client.admin.keyring

# manually scp adjusted ceph.conf to all *.ceph.bfh.ch and restart all ceph daemons