vm greenplum7单节点安装+rockylinux8.9

作者: 豪威Howie | 来源:发表于2023-12-11 12:57 被阅读0次

openstack环境安装
MQ RabbitMQ 高可用集群（二）：集群搭建
mac安装nacos
ElasticSearch 安装 (单机单节点/单机多节点)
RocketMQ-初体验RocketMQ(02)_单节点Rock
Ubuntu18.0.4折腾日记(二)|如何使用VM安装Ubun
OpenShift开源mini版试用
Docker安装Kafka
ElasticSearch的安装及简单配置
zookeeper 单节点安装

目标：

a.在redhat8/oraclelinux8/rockylinux8/almalinux8下安装greenplum7;
b.支持python3.9
c.单节点安装 vmware 版的greenplum

安装linux

安装 almalinux8.9 或 rockylinux8.9 (略)

如果用云服务，有的默认给的版本是8.6的，这个不符合greenplum7 的需要，需要升级到8.7及以上
本文假定SELINUX 和防火墙都已经关闭,相关操作略

#!/bin/bash

echo 'LANG=en_US.UTF-8' | sudo tee -a /etc/profile


# 安装必要的linux包
sudo dnf group install -y "Development Tools"
INSTALL_PKGS="apr-devel bison bzip2-devel cmake3 epel-release flex gcc gcc-c++ krb5-devel libcurl-devel libevent-devel libkadm5 libxml2-devel libzstd-devel openssl-devel python39 python39-devel python39-psutil python3-pip perl-ExtUtils-MakeMaker.noarch perl-ExtUtils-Embed.noarch readline-devel rsync xerces-c-devel zlib-devel python3-psutil python3-pyyaml python3-psycopg2"
sudo yum install -y $INSTALL_PKGS
sudo yum --enablerepo=powertools install -y libyaml-devel

sudo yum install -y postgresql 
sudo yum install -y postgresql-devel

pip3.9 install -r python-dependencies.txt

sudo rm -rf /usr/bin/python3
sudo ln -s /usr/bin/python3.9 /usr/bin/python3
sudo rm -rf /usr/bin/python
sudo ln -s /usr/bin/python3.9 /usr/bin/python
sudo rm -rf /usr/bin/pip3
sudo ln -s /usr/bin/pip3.9 /usr/bin/pip3
sudo rm -rf /usr/bin/pip
sudo ln -s /usr/bin/pip3.9 /usr/bin/pip



setenforce 0
sudo tee -a /etc/selinux/config << EOF
SELINUX=disabled
EOF

sudo tee -a /etc/sssd/sssd.conf << EOF
selinux_provider=none
EOF

sudo systemctl stop firewalld.service

#Configure kernel settings so the system is optimized for Greenplum Database.
sudo tee -a /etc/sysctl.d/10-gpdb.conf << EOF
kernel.msgmax = 65536
kernel.msgmnb = 65536
kernel.msgmni = 2048
kernel.sem = 500 2048000 200 8192
kernel.shmmni = 1024
kernel.core_uses_pid = 1
kernel.core_pattern=/var/core/core.%h.%t
kernel.sysrq = 1
net.core.netdev_max_backlog = 2000
net.core.rmem_max = 4194304
net.core.wmem_max = 4194304
net.core.rmem_default = 4194304
net.core.wmem_default = 4194304
net.ipv4.tcp_rmem = 4096 4224000 16777216
net.ipv4.tcp_wmem = 4096 4224000 16777216
net.core.optmem_max = 4194304
net.core.somaxconn = 10000
net.ipv4.ip_forward = 0
net.ipv4.tcp_congestion_control = cubic
net.ipv4.tcp_tw_recycle = 0
net.core.default_qdisc = fq_codel
net.ipv4.tcp_mtu_probing = 0
net.ipv4.conf.all.arp_filter = 1
net.ipv4.conf.default.accept_source_route = 0
net.ipv4.ip_local_port_range = 10000 65535
net.ipv4.tcp_max_syn_backlog = 4096
net.ipv4.tcp_syncookies = 1
net.ipv4.ipfrag_high_thresh = 41943040
net.ipv4.ipfrag_low_thresh = 31457280
net.ipv4.ipfrag_time = 60
net.ipv4.ip_local_reserved_ports=65330
vm.overcommit_memory = 2
vm.overcommit_ratio = 95
vm.swappiness = 10
vm.dirty_expire_centisecs = 500
vm.dirty_writeback_centisecs = 100
vm.zone_reclaim_mode = 0
EOF

RAM_IN_KB=`cat /proc/meminfo | grep MemTotal | awk '{print $2}'`
RAM_IN_BYTES=$(($RAM_IN_KB*1024))
echo "vm.min_free_kbytes = $(($RAM_IN_BYTES*3/100/1024))" | sudo tee -a /etc/sysctl.d/10-gpdb.conf > /dev/null
echo "kernel.shmall = $(($RAM_IN_BYTES/2/4096))" | sudo tee -a /etc/sysctl.d/10-gpdb.conf > /dev/null
echo "kernel.shmmax = $(($RAM_IN_BYTES/2))" | sudo tee -a /etc/sysctl.d/10-gpdb.conf > /dev/null
if [ $RAM_IN_BYTES -le $((64*1024*1024*1024)) ]; then
    echo "vm.dirty_background_ratio = 3" | sudo tee -a /etc/sysctl.d/10-gpdb.conf > /dev/null
    echo "vm.dirty_ratio = 10" | sudo tee -a /etc/sysctl.d/10-gpdb.conf > /dev/null
else
    echo "vm.dirty_background_ratio = 0" | sudo tee -a /etc/sysctl.d/10-gpdb.conf > /dev/null
    echo "vm.dirty_ratio = 0" | sudo tee -a /etc/sysctl.d/10-gpdb.conf > /dev/null
    echo "vm.dirty_background_bytes = 1610612736 # 1.5GB" | sudo tee -a /etc/sysctl.d/10-gpdb.conf > /dev/null
    echo "vm.dirty_bytes = 4294967296 # 4GB" | sudo tee -a /etc/sysctl.d/10-gpdb.conf > /dev/null
fi

sudo sysctl -p

sudo tee -a /etc/security/limits.d/10-nproc.conf << EOF
* soft nofile 524288
* hard nofile 524288
* soft nproc 131072
* hard nproc 131072
* soft core unlimited
EOF


ulimit -n 65536 65536

改机器名

hostnamectl set-hostname gphost

创建gpadmin用户，设置ssh并配置免密登录

将相关代码存为 gpadmin_setup.sh,改成 777，并执行

#!/bin/bash



set -euxo pipefail
setup_ssh_for_user() {
  local user="${1}"
  local home_dir
  home_dir=$(eval echo "~${user}")

  mkdir -p "${home_dir}/.ssh"
  touch "${home_dir}/.ssh/authorized_keys" "${home_dir}/.ssh/known_hosts" "${home_dir}/.ssh/config"
  if [ ! -f "${home_dir}/.ssh/id_rsa" ]; then
    ssh-keygen -t rsa -N "" -f "${home_dir}/.ssh/id_rsa"
  fi
  cat "${home_dir}/.ssh/id_rsa.pub" >> "${home_dir}/.ssh/authorized_keys"
  chmod 0600 "${home_dir}/.ssh/authorized_keys"
  cat << 'NOROAMING' >> "${home_dir}/.ssh/config"
Host *
  UseRoaming no
NOROAMING
  chown -R "${user}" "${home_dir}/.ssh"
}

ssh_keyscan_for_user() {
  local user="${1}"
  local home_dir
  home_dir=$(eval echo "~${user}")

  {
    ssh-keyscan localhost
    ssh-keyscan 0.0.0.0
    ssh-keyscan `hostname`
  } >> "${home_dir}/.ssh/known_hosts"
}

transfer_ownership() {
    chmod a+w gpdb_src
    find gpdb_src -type d -exec chmod a+w {} \;
    # Needed for the gpload test
    [ -f gpdb_src/gpMgmt/bin/gpload_test/gpload2/data_file.csv ] && chown gpadmin:gpadmin gpdb_src/gpMgmt/bin/gpload_test/gpload2/data_file.csv
    [ -d /usr/local/gpdb ] && chown -R gpadmin:gpadmin /usr/local/gpdb
    [ -d /usr/local/greenplum-db-devel ] && chown -R gpadmin:gpadmin /usr/local/greenplum-db-devel
    chown -R gpadmin:gpadmin /home/gpadmin
}

set_limits() {
  # Currently same as what's recommended in install guide
  if [ -d /etc/security/limits.d ]; then
    cat > /etc/security/limits.d/gpadmin-limits.conf <<-EOF
        gpadmin soft core unlimited
        gpadmin soft nproc 131072
        gpadmin soft nofile 65536
    EOF
  fi
  # Print now effective limits for gpadmin
  su gpadmin -c 'ulimit -a'
}

create_gpadmin_if_not_existing() {
  gpadmin_exists=`id gpadmin > /dev/null 2>&1;echo $?`
  if [ "0" -eq "$gpadmin_exists" ]; then
      echo "gpadmin user already exists, skipping creating again."
  else
      eval "$*"
      # Add user to sudoers list required for gpinitsystem test
      echo "gpadmin ALL = NOPASSWD : ALL" >> /etc/sudoers
  fi
}

setup_gpadmin_user() {
  groupadd supergroup
  case "$TEST_OS" in
    centos*)
      user_add_cmd="/usr/sbin/useradd -G supergroup,tty gpadmin"
      create_gpadmin_if_not_existing ${user_add_cmd}
      ;;
    ubuntu*)
      user_add_cmd="/usr/sbin/useradd -G supergroup,tty gpadmin -s /bin/bash"
      create_gpadmin_if_not_existing ${user_add_cmd}
      ;;
    *) echo "Unknown OS: $TEST_OS"; exit 1 ;;
  esac
  echo -e "password\npassword" | passwd gpadmin
  setup_ssh_for_user gpadmin
  transfer_ownership
  set_limits
}

setup_sshd() {
  test -e /etc/ssh/ssh_host_rsa_key || ssh-keygen -f /etc/ssh/ssh_host_rsa_key -N '' -t rsa
  test -e /etc/ssh/ssh_host_dsa_key || ssh-keygen -f /etc/ssh/ssh_host_dsa_key -N '' -t dsa
  # See https://gist.github.com/gasi/5691565
  sed -ri 's/UsePAM yes/UsePAM no/g' /etc/ssh/sshd_config
  # Disable password authentication so builds never hang given bad keys
  sed -ri 's/PasswordAuthentication yes/PasswordAuthentication no/g' /etc/ssh/sshd_config

  case "$TEST_OS" in
    centos7)
      test -e /etc/ssh/ssh_host_key || ssh-keygen -f /etc/ssh/ssh_host_key -N '' -t rsa1
      # For Centos 7, disable looking for host key types that older Centos versions don't support.
      sed -ri 's@^HostKey /etc/ssh/ssh_host_ecdsa_key$@#&@' /etc/ssh/sshd_config
      sed -ri 's@^HostKey /etc/ssh/ssh_host_ed25519_key$@#&@' /etc/ssh/sshd_config
      ;;
  esac

  setup_ssh_for_user root

  if [[ "$TEST_OS" == *"ubuntu"* ]]; then
    mkdir -p /var/run/sshd
    chmod 0755 /var/run/sshd
  fi

  /usr/sbin/sshd

  ssh_keyscan_for_user root
  ssh_keyscan_for_user gpadmin
}

determine_os() {
  local name version
  if [ -f /etc/redhat-release ]; then
    name="centos"
    version=$(sed </etc/redhat-release 's/.*release *//' | cut -f1 -d.)
  elif [ -f /etc/SuSE-release ]; then
    name="sles"
    version=$(awk -F " *= *" '$1 == "VERSION" { print $2 }' /etc/SuSE-release)
  elif grep -q ubuntu /etc/os-release ; then
    name="ubuntu"
    version=$(awk -F " *= *" '$1 == "VERSION_ID" { print $2 }' /etc/os-release | tr -d \")
  else
    echo "Could not determine operating system type" >/dev/stderr
    exit 1
  fi
  echo "${name}${version}"
}
# Set the "Set-User-ID" bit of ping, or else gpinitsystem will error by following message:
# [FATAL]:-Unknown host d6f9f630-65a3-4c98-4c03-401fbe5dd60b: ping: socket: Operation not permitted
# This is needed in centos7, sles12sp5, but not for ubuntu18.04
workaround_before_concourse_stops_stripping_suid_bits() {
  chmod u+s $(which ping)
}

_main() {
  TEST_OS=$(determine_os)
  setup_gpadmin_user
  setup_sshd
  workaround_before_concourse_stops_stripping_suid_bits
}

[ "${BASH_SOURCE[0]}" = "$0" ] && _main "$@"

创建需要的目录

mkdir -p /gpdata
mkdir -p /gpdata/coordinator
mkdir -p /gpdata/gpslave1
mkdir -p /gpdata/gpslave2

chown -R gpadmin:gpadmin /gpdata/

3. 安装 greenplum7 的包

3.1 直接下载并安装rpm包

https://github.com/greenplum-db/gpdb/releases/download/7.0.0/open-source-greenplum-db-7.0.0-el8-x86_64.rpm

  dnf  install open-source-greenplum-db-7.0.0-el8-x86_64.rpm

默认安装到/usr/local，授权给gpadmin

chown -R gpadmin /usr/local/greenplum*
chgrp -R gpadmin /usr/local/greenplum*

3.2 切换用户gpadmin

su gpadmin

进入$开头的bash环境

vim .bash_profile

添加以下：

source /usr/local/greenplum-db-7.0.0/greenplum_path.sh

export COORDINATOR_DATA_DIRECTORY=/data/coordinator/gpseg-1
export PGPORT=12345
export PGUSER=gpadmin
export PGDATABASE=gpdb
export LOCALE=en_US.UTF-8

配置gpadmin的ssh

ssh-keygen -t rsa -b 4096

将gpadmin 添加到管理员组，免密码sudo

a. 在 /etc/sudoers 添加

gpadmin        ALL=(ALL)       NOPASSWD: ALL

3.3 设置子节点的host

vim all_hosts

gphost

3.4 权限互通

source /usr/local/greenplum-db-7.0.0/greenplum_path.sh
python   /usr/local/greenplum-db-7.0.0/bin/gpssh-exkeys -f all_hosts

3.5编辑gp初始化文件

vim initgp

SEG_PREFIX=gpseg
PORT_BASE=33000
declare -a DATA_DIRECTORY=(/u01/gpdata/gpslave1 /u01/gpdata/gpslave2 )
MASTER_HOSTNAME=gphost
MASTER_PORT=12345
MASTER_DIRECTORY=/u01/gpdata/gpmaster
DATABASE_NAME=gpdb

4.8设置节点服务器

vim seg_hosts_file

gphost

4.9初始化GP

gpinitsystem -c initgp  -h seg_hosts_file   all_hosts_file

执行初始化命令的时候缺少一个libnsl.so.1
此时修改/etc/ld.so.conf文件

vim /etc/ld.so.conf

include ld.so.conf.d/*.conf
/usr/lib64

/sbin/ldconfig -v

所有缺失的lib包我都放在文件夹中,缺少的lib通过穿件软连接实现

5.连接GP
基本使用
https://www.cnblogs.com/kingle-study/p/10550785.html
登录数据库之前得先将程序运行,执行
bin/gpstart

5.1执行bin/psql -l 查询所有数据库

5.2创建数据库createdb 库名
创建表: CREATE TABLE 表名
bin/psql -d 数据库名 进入指定数据库,接下来就可以执行sql操作了

eg:简单实例:
create table test001(id int,name varchar(128)); --id 为分布键
insert into test001 values (100,‘tom’),(101,‘lily’),(102,‘jack’),(103,‘linda’);
select * from test001;

5.3重新加载配置文件

gpstop -u

附录

其他启停命令

gpstate #查看服务状态
gpstart #正常启动
gpstop #正常关闭
gpstop -M fast #快速关闭
gpstop r #重启

服务器重启后需要在 gpadmin用户下用 gpstart启动gp服务

openstack环境安装
openstack安装一般有两种方式，Devstack单节点安装和多节点安装。单节点适合自己随便玩玩，认真学还是推...
MQ RabbitMQ 高可用集群（二）：集群搭建
1 集群节点安装单节点安装RabbitMQ，可参考博文： MQ RocketMQ安装部署和配置： https:...
mac安装nacos
1、说明 nacos支持集群，支持单节点，这里记录mac单节点下nacos安装 2、maca下安装（1）下载 n...
ElasticSearch 安装 (单机单节点/单机多节点)
ElasticSearch 安装 (单机单节点/单机多节点) ElasticSearch 简介 ElasticSe...
RocketMQ-初体验RocketMQ(02)_单节点Rock
RocketMQ的安装（单节点）接下来，我们从无到有，搭建一个RocketMQ的环境吧，单节点走起。安装虚...
Ubuntu18.0.4折腾日记(二)|如何使用VM安装Ubun
VM安装Ubuntu VM参阅：教你如何安装并使用VM Pro 进入Ubuntu安装页面
OpenShift开源mini版试用
Minishift 简介 Minishift是一个工具，可以通过在VM中运行单节点OpenShift集群来帮助您在...
Docker安装Kafka
单节点安装 docker-compose.yml 配置文件 Kafka集群安装
ElasticSearch的安装及简单配置
操作步骤启动单节点安装插件查看插件查看安装的插件配置多节点集群查看集群查看nodes 相关阅读安装...
zookeeper 单节点安装
下载 zookeeper： wgethttp://mirror.bit.edu.cn/apache/zookeep...