# ha-node
10.10.10.21 ha01
10.10.10.22 ha02

# controller-node
10.10.10.31 controller01
10.10.10.32 controller02
10.10.10.33 controller03

# compute-node
10.10.10.41 compute01
10.10.10.42 compute02

# ceph-node
10.10.10.51 ceph01
10.10.10.52 ceph02
10.10.10.53 ceph03

echo "# ha-node
10.10.10.21 ha01
10.10.10.22 ha02

# controller-node
10.10.10.31 controller01
10.10.10.32 controller02
10.10.10.33 controller03

# compute-node
10.10.10.41 compute01
10.10.10.42 compute02

# ceph-node
10.10.10.51 ceph01
10.10.10.52 ceph02
10.10.10.53 ceph03
" >> /etc/hosts

sed -i 's/#UseDNS yes/UseDNS no/g' /etc/ssh/sshd_config
systemctl restart sshd

echo 'modprobe br_netfilter' >> /etc/rc.d/rc.local
chmod 755 /etc/rc.d/rc.local
modprobe br_netfilter
echo 'net.ipv4.ip_forward = 1' >> /etc/sysctl.conf
echo 'net.bridge.bridge-nf-call-iptables=1' >> /etc/sysctl.conf
echo 'net.bridge.bridge-nf-call-ip6tables=1'  >>/etc/sysctl.conf
sysctl -p

echo 'net.ipv4.ip_nonlocal_bind = 1' >> /etc/sysctl.conf
sysctl -p

mkdir -p /opt/tools
yum install wget crudini -y
wget --no-check-certificate -P /opt/tools https://cbs.centos.org/kojifiles/packages/openstack-utils/2017.1/1.el7/noarch/openstack-utils-2017.1-1.el7.noarch.rpm
rpm -ivh /opt/tools/openstack-utils-2017.1-1.el7.noarch.rpm

yum install mariadb-server-galera mariadb-galera-common galera xinetd rsync -y

systemctl restart mariadb.service
systemctl enable mariadb.service

[root@controller01 ~]# mysql_secure_installation
#输入root用户的当前密码（不输入密码）
Enter current password for root (enter for none):
#设置root密码？
Set root password? [Y/n] y
#新密码：
New password:
#重新输入新的密码：
Re-enter new password:
#删除匿名用户？
Remove anonymous users? [Y/n] y
#禁止远程root登录？
Disallow root login remotely? [Y/n] n
#删除测试数据库并访问它？
Remove test database and access to it? [Y/n] y
#现在重新加载特权表？
Reload privilege tables now? [Y/n] y

/usr/libexec/mysqld --wsrep-new-cluster --user=root &

#重启controller01节点；并在启动前删除contrller01节点之前的数据
pkill -9 mysqld
rm -rf /var/lib/mysql/*

#注意以system unit方式启动mariadb服务时的权限
chown mysql:mysql /var/run/mariadb/mariadb.pid

## 启动后查看节点所在服务状态，正常的话 contrller01 节点会从 contrller02 节点同步数据
systemctl start mariadb.service
systemctl status mariadb.service

wget -P /extend/shell/ https://raw.githubusercontent.com/olafz/percona-clustercheck/master/clustercheck

#!/bin/bash
#
# Script to make a proxy (ie HAProxy) capable of monitoring Percona XtraDB Cluster nodes properly
#
# Author: Olaf van Zandwijk <[email protected]>
# Author: Raghavendra Prabhu <[email protected]>
#
# Documentation and download: https://github.com/olafz/percona-clustercheck
#
# Based on the original script from Unai Rodriguez
#

if [[ $1 == '-h' || $1 == '--help' ]];then
echo "Usage: $0 <user> <pass> <available_when_donor=0|1> <log_file> <available_when_readonly=0|1> <defaults_extra_file>"
exit
fi

# if the disabled file is present, return 503. This allows
# admins to manually remove a node from a cluster easily.
if [ -e "/var/tmp/clustercheck.disabled" ]; then
# Shell return-code is 1
echo -en "HTTP/1.1 503 Service Unavailable\r\n"
echo -en "Content-Type: text/plain\r\n"
echo -en "Connection: close\r\n"
echo -en "Content-Length: 51\r\n"
echo -en "\r\n"
echo -en "Percona XtraDB Cluster Node is manually disabled.\r\n"
sleep 0.1
exit 1
fi

set -e

if [ -f /etc/sysconfig/clustercheck ]; then
. /etc/sysconfig/clustercheck
fi

MYSQL_USERNAME="clustercheck"
MYSQL_PASSWORD="123456"
MYSQL_HOST="localhost"
MYSQL_PORT="3306"

AVAILABLE_WHEN_DONOR=${AVAILABLE_WHEN_DONOR:-0}
ERR_FILE="${ERR_FILE:-/dev/null}"
AVAILABLE_WHEN_READONLY=${AVAILABLE_WHEN_READONLY:-1}
DEFAULTS_EXTRA_FILE=${DEFAULTS_EXTRA_FILE:-/etc/my.cnf}

#Timeout exists for instances where mysqld may be hung
TIMEOUT=10

EXTRA_ARGS=""
if [[ -n "$MYSQL_USERNAME" ]]; then
EXTRA_ARGS="$EXTRA_ARGS --user=${MYSQL_USERNAME}"
fi
if [[ -n "$MYSQL_PASSWORD" ]]; then
EXTRA_ARGS="$EXTRA_ARGS --password=${MYSQL_PASSWORD}"
fi
if [[ -r $DEFAULTS_EXTRA_FILE ]];then
MYSQL_CMDLINE="mysql --defaults-extra-file=$DEFAULTS_EXTRA_FILE -nNE --connect-timeout=$TIMEOUT \
            ${EXTRA_ARGS}"
else
MYSQL_CMDLINE="mysql -nNE --connect-timeout=$TIMEOUT ${EXTRA_ARGS}"
fi
#
# Perform the query to check the wsrep_local_state
#
WSREP_STATUS=$($MYSQL_CMDLINE -e "SHOW STATUS LIKE 'wsrep_local_state';" \
2>${ERR_FILE} | tail -1 2>>${ERR_FILE})

if [[ "${WSREP_STATUS}" == "4" ]] || [[ "${WSREP_STATUS}" == "2" && ${AVAILABLE_WHEN_DONOR} == 1 ]]
then
# Check only when set to 0 to avoid latency in response.
if [[ $AVAILABLE_WHEN_READONLY -eq 0 ]];then
READ_ONLY=$($MYSQL_CMDLINE -e "SHOW GLOBAL VARIABLES LIKE 'read_only';" \
            2>${ERR_FILE} | tail -1 2>>${ERR_FILE})

if [[ "${READ_ONLY}" == "ON" ]];then
    # Percona XtraDB Cluster node local state is 'Synced', but it is in
    # read-only mode. The variable AVAILABLE_WHEN_READONLY is set to 0.
    # => return HTTP 503
    # Shell return-code is 1
    echo -en "HTTP/1.1 503 Service Unavailable\r\n"
    echo -en "Content-Type: text/plain\r\n"
    echo -en "Connection: close\r\n"
    echo -en "Content-Length: 43\r\n"
    echo -en "\r\n"
    echo -en "Percona XtraDB Cluster Node is read-only.\r\n"
    sleep 0.1
    exit 1
fi
fi
# Percona XtraDB Cluster node local state is 'Synced' => return HTTP 200
# Shell return-code is 0
echo -en "HTTP/1.1 200 OK\r\n"
echo -en "Content-Type: text/plain\r\n"
echo -en "Connection: close\r\n"
echo -en "Content-Length: 40\r\n"
echo -en "\r\n"
echo -en "Percona XtraDB Cluster Node is synced.\r\n"
sleep 0.1
exit 0
else
# Percona XtraDB Cluster node local state is not 'Synced' => return HTTP 503
# Shell return-code is 1
echo -en "HTTP/1.1 503 Service Unavailable\r\n"
echo -en "Content-Type: text/plain\r\n"
echo -en "Connection: close\r\n"
echo -en "Content-Length: 44\r\n"
echo -en "\r\n"
echo -en "Percona XtraDB Cluster Node is not synced.\r\n"
sleep 0.1
exit 1
fi

# 全部控制节点都需要启动
systemctl daemon-reload
systemctl enable xinetd
systemctl start xinetd
systemctl status xinetd

第1步：开启galera集群的群主主机的mariadb服务。
第2步：开启galera集群的成员主机的mariadb服务。

异常处理：galera集群的群主主机和成员主机的mysql服务无法启动，如何处理？

#解决方法一：
第1步、删除garlera群主主机的/var/lib/mysql/grastate.dat状态文件
/bin/galera_new_cluster启动服务。启动正常。登录并查看wsrep状态。

第2步：删除galera成员主机中的/var/lib/mysql/grastate.dat状态文件
systemctl restart mariadb重启服务。启动正常。登录并查看wsrep状态。

#解决方法二：
第1步、修改garlera群主主机的/var/lib/mysql/grastate.dat状态文件中的0为1
/bin/galera_new_cluster启动服务。启动正常。登录并查看wsrep状态。

第2步：修改galera成员主机中的/var/lib/mysql/grastate.dat状态文件中的0为1
systemctl restart mariadb重启服务。启动正常。登录并查看wsrep状态。

经过实际发现，以下操作步骤也可以：
第1步、修改garlera群主主机的/var/lib/mysql/grastate.dat状态文件中的0为1
systemctl restart mariadb重启服务

第2步：修改galera成员主机直接使用systemctl restart mariadb重启服务

yum install erlang rabbitmq-server -y
systemctl enable rabbitmq-server.service

sed -i 's|127.0.0.1,::1|0.0.0.0|g' /etc/sysconfig/memcached

systemctl enable memcached.service
systemctl start memcached.service
systemctl status memcached.service
ss -tnlp|grep memcached

systemctl enable haproxy
systemctl start haproxy
systemctl status haproxy
systemctl enable keepalived
systemctl start keepalived
systemctl status keepalived

mkdir /var/log/haproxy
chmod a+w /var/log/haproxy

wget ftp://ftp.pbone.net/mirror/archive.fedoraproject.org/epel/testing/6.2019-05-29/x86_64/Packages/p/python2-qpid-proton-0.28.0-1.el7.x86_64.rpm
wget ftp://ftp.pbone.net/mirror/vault.centos.org/7.8.2003/messaging/x86_64/qpid-proton/Packages/q/qpid-proton-c-0.28.0-1.el7.x86_64.rpm
rpm -ivh qpid-proton-c-0.28.0-1.el7.x86_64.rpm
rpm -ivh python2-qpid-proton-0.28.0-1.el7.x86_64.rpm
# yum install openstack-keystone httpd python3-mod_wsgi mod_ssl -y # centos 8
yum install openstack-keystone httpd mod_wsgi mod_ssl -y

#备份Keystone配置文件
cp /etc/keystone/keystone.conf{,.bak}
egrep -v '^$|^#' /etc/keystone/keystone.conf.bak > /etc/keystone/keystone.conf

openstack-config --set /etc/keystone/keystone.conf cache backend oslo_cache.memcache_pool
openstack-config --set /etc/keystone/keystone.conf cache enabled true
openstack-config --set /etc/keystone/keystone.conf cache memcache_servers controller01:11211,controller02:11211,controller03:11211
openstack-config --set /etc/keystone/keystone.conf database connection mysql+pymysql://keystone:[email protected]/keystone
openstack-config --set /etc/keystone/keystone.conf token provider fernet

# 在/etc/keystone/生成相关秘钥及目录
keystone-manage fernet_setup --keystone-user keystone --keystone-group keystone
keystone-manage credential_setup --keystone-user keystone --keystone-group keystone

#并将初始化的密钥拷贝到其他的控制节点
scp -rp /etc/keystone/fernet-keys /etc/keystone/credential-keys controller02:/etc/keystone/
scp -rp /etc/keystone/fernet-keys /etc/keystone/credential-keys controller03:/etc/keystone/

#同步后修改另外两台控制节点fernet的权限
chown -R keystone:keystone /etc/keystone/credential-keys/
chown -R keystone:keystone /etc/keystone/fernet-keys/

#修改域名为主机名
cp /etc/httpd/conf/httpd.conf{,.bak}
sed -i "s/#ServerName www.example.com:80/ServerName ${HOSTNAME}/" /etc/httpd/conf/httpd.conf

#不同的节点替换不同的ip地址
# controller01
sed -i "s/Listen\ 80/Listen\ 10.10.10.31:80/g" /etc/httpd/conf/httpd.conf

# controller02
sed -i "s/Listen\ 80/Listen\ 10.10.10.32:80/g" /etc/httpd/conf/httpd.conf

# controller03
sed -i "s/Listen\ 80/Listen\ 10.10.10.33:80/g" /etc/httpd/conf/httpd.conf

#创建软连接wsgi-keystone.conf文件
ln -s /usr/share/keystone/wsgi-keystone.conf /etc/httpd/conf.d/

#不同的节点替换不同的ip地址
##controller01
sed -i "s/Listen\ 5000/Listen\ 10.10.10.31:5000/g" /etc/httpd/conf.d/wsgi-keystone.conf
sed -i "s#*:5000#10.10.10.31:5000#g" /etc/httpd/conf.d/wsgi-keystone.conf

##controller02
sed -i "s/Listen\ 5000/Listen\ 10.10.10.32:5000/g" /etc/httpd/conf.d/wsgi-keystone.conf
sed -i "s#*:5000#10.10.10.32:5000#g" /etc/httpd/conf.d/wsgi-keystone.conf

##controller03
sed -i "s/Listen\ 5000/Listen\ 10.10.10.33:5000/g" /etc/httpd/conf.d/wsgi-keystone.conf
sed -i "s#*:5000#10.10.10.33:5000#g" /etc/httpd/conf.d/wsgi-keystone.conf

systemctl restart httpd.service
systemctl enable httpd.service
systemctl status httpd.service

$ cat >> ~/admin-openrc << EOF
# admin-openrc
export OS_USERNAME=admin
export OS_PASSWORD=123456
export OS_PROJECT_NAME=admin
export OS_USER_DOMAIN_NAME=Default
export OS_PROJECT_DOMAIN_NAME=Default
export OS_AUTH_URL=http://10.10.10.10:5000/v3
export OS_IDENTITY_API_VERSION=3
export OS_IMAGE_API_VERSION=2
EOF

$ source  ~/admin-openrc

# 验证
$ openstack domain list
+---------+---------+---------+--------------------+
| ID      | Name    | Enabled | Description        |
+---------+---------+---------+--------------------+
| default | Default | True    | The default domain |
+---------+---------+---------+--------------------+

# 也可以使用下面的命令
$ openstack token issue

#拷贝到其他的控制节点
scp -rp ~/admin-openrc controller02:~/
scp -rp ~/admin-openrc controller03:~/

cat >> ~/demo-openrc << EOF
#demo-openrc
export OS_USERNAME=demo
export OS_PASSWORD=123456
export OS_PROJECT_NAME=
export OS_USER_DOMAIN_NAME=Default
export OS_PROJECT_DOMAIN_NAME=Default
export OS_AUTH_URL=http://10.10.10.10:5000/v3
export OS_IDENTITY_API_VERSION=3
export OS_IMAGE_API_VERSION=2
EOF

source  ~/demo-openrc
openstack token issue

# 拷贝到其他的控制节点
scp -rp ~/demo-openrc controller02:~/
scp -rp ~/demo-openrc controller03:~/

yum install openstack-glance python-glance python-glanceclient -y

# 备份glance配置文件
cp /etc/glance/glance-api.conf{,.bak}
egrep -v '^$|^#' /etc/glance/glance-api.conf.bak >/etc/glance/glance-api.conf

openstack-config --set /etc/glance/glance-api.conf DEFAULT bind_host 10.10.10.31
openstack-config --set /etc/glance/glance-api.conf database connection  mysql+pymysql://glance:[email protected]/glance
openstack-config --set /etc/glance/glance-api.conf glance_store stores file,http
openstack-config --set /etc/glance/glance-api.conf glance_store default_store file
openstack-config --set /etc/glance/glance-api.conf glance_store filesystem_store_datadir /var/lib/glance/images/
openstack-config --set /etc/glance/glance-api.conf keystone_authtoken www_authenticate_uri   http://10.10.10.10:5000
openstack-config --set /etc/glance/glance-api.conf keystone_authtoken auth_url  http://10.10.10.10:5000
openstack-config --set /etc/glance/glance-api.conf keystone_authtoken memcached_servers controller01:11211,controller02:11211,controller03:11211
openstack-config --set /etc/glance/glance-api.conf keystone_authtoken auth_type password
openstack-config --set /etc/glance/glance-api.conf keystone_authtoken project_domain_name Default
openstack-config --set /etc/glance/glance-api.conf keystone_authtoken user_domain_name Default
openstack-config --set /etc/glance/glance-api.conf keystone_authtoken project_name  service
openstack-config --set /etc/glance/glance-api.conf keystone_authtoken username glance
openstack-config --set /etc/glance/glance-api.conf keystone_authtoken password 123456
openstack-config --set /etc/glance/glance-api.conf paste_deploy flavor keystone

mkdir /var/lib/glance/images/
chown glance:nobody /var/lib/glance/images

su -s /bin/sh -c "glance-manage db_sync" glance

systemctl enable openstack-glance-api.service
systemctl restart openstack-glance-api.service
systemctl status openstack-glance-api.service
sleep 3s
lsof -i:9292

[root@controller01 ~]# ls -l /var/lib/glance/images/
total 0

[root@controller02 ~]# ls -l /var/lib/glance/images/
total 15956
-rw-r----- 1 glance glance 16338944 Nov  4 17:25 1c66cd7e-b6d9-4e70-a3d4-f73b27a84230

[root@controller03 ~]# ls -l /var/lib/glance/images/
total 0

scp -pr /var/lib/glance/images/* controller01:/var/lib/glance/images/
scp -pr /var/lib/glance/images/* controller03:/var/lib/glance/images/
chown -R glance. /var/lib/glance/images/*