前言

本文参考下面链接提供另外一种完全离线的安装OpenShift3.11的方式 https://www.jianshu.com/p/f9284e02bdcd

操作系统采用CentOS7.6

[root@openshift1 ~]# lsb_release -a
LSB Version: :core-4.1-amd64:core-4.1-noarch
Distributor ID: CentOS
Description: CentOS Linux release 7.6.1810 (Core)
Release: 7.6.1810
Codename: Core

主机	IP	备注
openshift1	192.168.1.25	Master
openshift2	192.168.1.27	Compute
openshift3	192.168.1.28	Compute，infra

下载离线包

docker镜像

找一台能访问网络的机器，下载所有离线镜像

yum install docker -y
systemctl start docker; systemctl enable docker

docker pull docker.io/openshift/origin-node:v3.11
docker pull docker.io/openshift/origin-control-plane:v3.11
docker pull docker.io/openshift/origin-deployer:v3.11.0
docker pull docker.io/openshift/origin-haproxy-router:v3.11
docker pull docker.io/openshift/origin-pod:v3.11.0
docker pull docker.io/openshift/origin-web-console:v3.11
docker pull docker.io/openshift/origin-docker-registry:v3.11
docker pull docker.io/openshift/origin-metrics-server:v3.11
docker pull docker.io/openshift/origin-console:v3.11
docker pull docker.io/openshift/origin-metrics-heapster:v3.11
docker pull docker.io/openshift/origin-metrics-hawkular-metrics:v3.11
docker pull docker.io/openshift/origin-metrics-schema-installer:v3.11
docker pull docker.io/openshift/origin-metrics-cassandra:v3.11
docker pull docker.io/cockpit/kubernetes:latest
docker pull quay.io/coreos/cluster-monitoring-operator:v0.1.1
docker pull quay.io/coreos/prometheus-config-reloader:v0.23.2
docker pull quay.io/coreos/prometheus-operator:v0.23.2
docker pull docker.io/openshift/prometheus-alertmanager:v0.15.2
docker pull docker.io/openshift/prometheus-node-exporter:v0.16.0
docker pull docker.io/openshift/prometheus:v2.3.2
docker pull docker.io/grafana/grafana:5.2.1
docker pull quay.io/coreos/kube-rbac-proxy:v0.3.1
docker pull quay.io/coreos/etcd:v3.2.22
docker pull quay.io/coreos/kube-state-metrics:v1.3.1
docker pull docker.io/openshift/oauth-proxy:v1.1.0
docker pull quay.io/coreos/configmap-reload:v0.0.1

下载Docker镜像后，使用工具批量导出镜像和在新节点上导入
https://github.com/laoshanxi/saveloadimg

缓存YUM rpm包
yum 源中rpm包的离线可以通过第一次安装的时候保留cache，后续基于cache的rpm包制作yum源
sudo sed -i 's/keepcache=0/keepcache=1/g' /etc/yum.conf

所有节点

配置SELINUX为permissive：

sudo sed  -i 's/SELINUX=enforcing/SELINUX=permissive/g'  /etc/selinux/config

所有节点安装基础包

yum install wget git net-tools bind-utils yum-utils iptables-services bridge-utils bash-completion kexec-tools sos psacct vim python-setuptools unzip tree docker –y

安装OpenShift源

sudo yum install -y centos-release-openshift-origin311 ceph-common container-selinux docker epel extras python-docker

关闭防火墙（不能关闭iptable）

sudo systemctl stop firewalld.service; sudo systemctl disable firewalld.service

所有节点配置iptable

cp /etc/sysconfig/iptables /etc/sysconfig/iptables.bak.$(date "+%Y%m%d%H%M%S");
sed -i '/.*--dport 22 -j ACCEPT.*/a\-A INPUT -p tcp -m state --state NEW -m tcp --dport 53 -j ACCEPT' /etc/sysconfig/iptables;
sed -i '/.*--dport 22 -j ACCEPT.*/a\-A INPUT -p udp -m state --state NEW -m udp --dport 53 -j ACCEPT' /etc/sysconfig/iptables;
sed -i '/.*--dport 22 -j ACCEPT.*/a\-A INPUT -p tcp -m state --state NEW -m tcp --dport 5000 -j ACCEPT' /etc/sysconfig/iptables;
sed -i '/.*--dport 22 -j ACCEPT.*/a\-A INPUT -p tcp -m state --state NEW -m tcp --dport 81 -j ACCEPT' /etc/sysconfig/iptables;

# 在master节点允许 8443 for node join
sed -i '/.*--dport 22 -j ACCEPT.*/a\-A INPUT -p tcp -m state --state NEW -m tcp --dport 8443 -j ACCEPT ' /etc/sysconfig/iptables;
sed -i '/.*--dport 22 -j ACCEPT.*/a\-A INPUT -p tcp -m state --state NEW -m tcp --dport 8443 -j ACCEPT ' /etc/sysconfig/iptables;

systemctl restart iptables;systemctl enable iptables

重启

reboot

Master节点操作Ansible

配置ansible SSH免密码

ssh-keygen -f ~/.ssh/id_rsa -N ''
for host in openshift1 openshift2 openshift3
do
     echo $host
     ssh-copy-id -i ~/.ssh/id_rsa.pub $host;
done

安装openshift ansible

yum install -y openshift-ansible

配置ansible

[root@openshift1 ~]# cat /etc/ansible/hosts
# Create an OSEv3 group that contains the masters, nodes, and etcd groups
[OSEv3:children]
masters
nodes
etcd

# Set variables common for all OSEv3 hosts
[OSEv3:vars]
# SSH user, this user should allow ssh based auth without requiring a password
ansible_ssh_user=root
#openshift_deployment_type=openshift-enterprise
openshift_deployment_type=origin
openshift_release="3.11"
openshift_image_tag=v3.11
openshift_pkg_version=-3.11.0
openshift_use_openshift_sdn=true
# If ansible_ssh_user is not root, ansible_become must be set to true
#ansible_become=true
#containerized=false

# default selectors for router and registry services
# openshift_router_selector='node-role.kubernetes.io/infra=true'
# openshift_registry_selector='node-role.kubernetes.io/infra=true'

# uncomment the following to enable htpasswd authentication; defaults to DenyAllPasswordIdentityProvider
openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true', 'challenge': 'true', 'kind': 'HTPasswdPasswordIdentityProvider'}]
#openshift_master_default_subdomain=ai.com
openshift_disable_check=memory_availability,disk_availability,docker_image_availability

os_sdn_network_plugin_name='redhat/openshift-ovs-networkpolicy'

openshift_master_cluster_method=native
openshift_master_cluster_hostname=openshift1
openshift_master_cluster_public_hostname=openshift1
# false
ansible_service_broker_install=false
openshift_enable_service_catalog=false
template_service_broker_install=false
openshift_logging_install_logging=false
enable_excluders=false
# registry passwd
#oreg_url=10.1.236.77:5000/openshift3/ose-${component}:${version}
#oreg_url=10.1.236.77:5000/openshift/origin-${component}:${version}
#openshift_examples_modify_imagestreams=true

# docker config
#openshift_docker_additional_registries=10.1.236.77:5000
#openshift_docker_insecure_registries=10.1.236.77:5000
#openshift_docker_blocked_registries
openshift_docker_options="--log-driver json-file --log-opt max-size=1M --log-opt max-file=3"

# openshift_cluster_monitoring_operator_install=false
# openshift_metrics_install_metrics=true
# openshift_enable_unsupported_configurations=True
#openshift_logging_es_nodeselector='node-role.kubernetes.io/infra: "true"'
#openshift_logging_kibana_nodeselector='node-role.kubernetes.io/infra: "true"'
# host group for masters

[masters]
openshift1

# host group for etcd
[etcd]
openshift1

# host group for nodes, includes region info
[nodes]
openshift1 openshift_node_group_name='node-config-master'
openshift2 openshift_node_group_name='node-config-compute'
openshift3 openshift_node_group_name='node-config-compute'
openshift2  openshift_node_group_name='node-config-infra'

配置DNS

ansible all -m copy -a "src=/etc/hosts dest=/etc/hosts "

启动Docker

ansible all -a 'systemctl start docker';ansible all -a 'systemctl enable docker'

执行检查

ansible-playbook /usr/share/ansible/openshift-ansible/playbooks/prerequisites.yml

执行安装

ansible-playbook /usr/share/ansible/openshift-ansible/playbooks/deploy_cluster.yml –vvv

执行卸载

ansible-playbook  /usr/share/ansible/openshift-ansible/playbooks/adhoc/uninstall.yml

安装后配置

创建用户

htpasswd -cb /etc/origin/master/htpasswd admin abc123
oc adm policy add-cluster-role-to-user cluster-admin admin

安装成功

image.png image.png

安装包列表

通过下面命令最终可以查到master节点上安装的rpm包列表

rpm -qa | awk '{cmd="rpm -qi "$0" | grep \"Install Date\" ";system(cmd);cmd="rpm -qi "$0" | grep \"Source RPM\" ";system(cmd);}' > rpm_list.log

 origin-3.11.0-1.el7.git.0.62803d0.src.rpm
 etcd-3.3.11-2.el7.centos.src.rpm
 origin-3.11.0-1.el7.git.0.62803d0.src.rpm
 origin-3.11.0-1.el7.git.0.62803d0.src.rpm
 origin-3.11.0-1.el7.git.0.62803d0.src.rpm
 lrzsz-0.12.20-36.el7.src.rpm
 autogen-5.18-5.el7.src.rpm
 ntp-4.2.6p5-28.el7.centos.src.rpm
 iscsi-initiator-utils-6.2.0.874-10.el7.src.rpm
 device-mapper-multipath-0.4.9-123.el7.src.rpm
 device-mapper-multipath-0.4.9-123.el7.src.rpm
 device-mapper-multipath-0.4.9-123.el7.src.rpm
 iscsi-initiator-utils-6.2.0.874-10.el7.src.rpm
 glusterfs-3.12.2-18.el7.src.rpm
 glusterfs-3.12.2-18.el7.src.rpm
 glusterfs-3.12.2-18.el7.src.rpm
 glusterfs-3.12.2-18.el7.src.rpm
 ding-libs-0.6.1-32.el7.src.rpm
 libverto-0.2.5-4.el7.src.rpm
 ding-libs-0.6.1-32.el7.src.rpm
 ding-libs-0.6.1-32.el7.src.rpm
 libevent-2.0.21-4.el7.src.rpm
 keyutils-1.5.8-3.el7.src.rpm
 ding-libs-0.6.1-32.el7.src.rpm
 gssproxy-0.7.0-21.el7.src.rpm
 ding-libs-0.6.1-32.el7.src.rpm
 libnfsidmap-0.25-19.el7.src.rpm
 nfs-utils-1.3.0-0.61.el7.src.rpm
 conntrack-tools-1.4.4-4.el7.src.rpm
 socat-1.7.3.2-2.el7.src.rpm
 libnetfilter_cttimeout-1.0.0-6.el7.src.rpm
 libnetfilter_queue-1.0.2-2.el7_2.src.rpm
 libnetfilter_cthelper-1.0.0-9.el7.src.rpm
 dnsmasq-2.76-7.el7.src.rpm
 criu-3.9-5.el7.src.rpm
 skopeo-0.1.35-2.git404c5bd.el7.centos.src.rpm
 protobuf-c-1.0.2-3.el7.src.rpm
 runc-1.0.0-59.dev.git2abd837.el7.centos.src.rpm
 gomtree-0.5.0-0.2.git16da0f8.el7.src.rpm
 atomic-1.22.1-26.gitb507039.el7.centos.src.rpm
 ostree-2018.5-1.el7.src.rpm
 libnet-1.1.6-7.el7.src.rpm
 libarchive-3.1.2-10.el7_2.src.rpm
 python-dateutil-1.5-7.el7.src.rpm
 openshift-ansible-3.11.37-1.git.0.3b8b341.el7.src.rpm
 nss-3.36.0-7.1.el7_6.src.rpm
 openshift-ansible-3.11.37-1.git.0.3b8b341.el7.src.rpm
 openshift-ansible-3.11.37-1.git.0.3b8b341.el7.src.rpm
 openshift-ansible-3.11.37-1.git.0.3b8b341.el7.src.rpm
 ansible-2.6.14-1.el7.src.rpm
 python-jinja2-2.7.2-3.el7_6.src.rpm
 sshpass-1.06-2.el7.src.rpm
 python-markupsafe-0.11-10.el7.src.rpm
 java-1.8.0-openjdk-1.8.0.212.b04-0.el7_6.src.rpm
 copy-jdk-configs-3.3-10.el7_5.src.rpm
 lksctp-tools-1.0.17-2.el7.src.rpm
 python-passlib-1.6.5-2.el7.src.rpm
 babel-0.9.6-8.el7.src.rpm
 python-paramiko-2.1.1-9.el7.src.rpm
 tzdata-2019a-1.el7.src.rpm
 python-idna-2.4-1.el7.src.rpm
 libtommath-0.42.0-6.el7.src.rpm
 python-crypto-2.6.1-15.el7.src.rpm
 python-cryptography-1.7.2-2.el7.src.rpm
 libtomcrypt-1.17-26.el7.src.rpm
 javapackages-tools-3.4.1-11.el7.src.rpm
 nss-softokn-3.36.0-5.el7_5.src.rpm
 python-ply-3.4-11.el7.src.rpm
 javapackages-tools-3.4.1-11.el7.src.rpm
 apr-util-1.5.2-6.el7.src.rpm
 nss-softokn-3.36.0-5.el7_5.src.rpm
 nss-3.36.0-7.1.el7_6.src.rpm
 python-enum34-1.0.4-1.el7.src.rpm
 python-pycparser-2.14-1.el7.src.rpm
 pcsc-lite-1.8.8-8.el7.src.rpm
 httpd-2.4.6-89.el7.centos.src.rpm
 nss-3.36.0-7.1.el7_6.src.rpm
 python-cffi-1.6.0-5.el7.src.rpm
 libxslt-1.1.28-5.el7.src.rpm
 nspr-4.19.0-1.el7_5.src.rpm
 apr-1.4.8-3.el7_4.1.src.rpm
 python-pyasn1-0.1.9-7.el7.src.rpm
 nss-util-3.36.0-1.1.el7_6.src.rpm
 python-lxml-3.2.1-4.el7.src.rpm
 ceph-common-10.2.5-4.el7.src.rpm
 python-docker-py-1.10.6-9.el7_6.src.rpm
 redhat-lsb-4.1-27.el7.centos.1.src.rpm
 redhat-lsb-4.1-27.el7.centos.1.src.rpm
 python-urllib3-1.10.2-5.el7.src.rpm
 hdparm-9.43-5.el7.src.rpm
 boost-1.53.0-27.el7.src.rpm
 python-requests-2.6.0-1.el7_1.src.rpm
 python-docker-py-1.10.6-9.el7_6.src.rpm
 m4-1.4.16-10.el7.src.rpm
 icu-50.1.2-17.el7.src.rpm
 psmisc-22.20-15.el7.src.rpm
 python-websocket-client-0.32.0-116.el7.src.rpm
 ceph-common-10.2.5-4.el7.src.rpm
 cups-1.6.3-35.el7.src.rpm
 boost-1.53.0-27.el7.src.rpm
 ceph-common-10.2.5-4.el7.src.rpm
 ceph-common-10.2.5-4.el7.src.rpm
 cups-1.6.3-35.el7.src.rpm
 ceph-common-10.2.5-4.el7.src.rpm
 star-1.5.2-13.el7.src.rpm
 patch-2.7.1-10.el7_5.src.rpm
 avahi-0.6.31-19.el7.src.rpm
 gdisk-0.8.10-2.el7.src.rpm
 boost-1.53.0-27.el7.src.rpm
 boost-1.53.0-27.el7.src.rpm
 centos-release-openshift-origin311-1-2.el7.centos.src.rpm
 centos-release-configmanagement-1-1.el7.centos.src.rpm
 centos-release-paas-common-1-1.el7.centos.src.rpm
 centos-release-ansible26-1-3.el7.centos.src.rpm
 audit-2.8.4-4.el7.src.rpm
 tree-1.6.0-10.el7.src.rpm
 dracut-033-554.el7.src.rpm
 net-tools-2.0-0.24.20131004git.el7.src.rpm
 yum-utils-1.1.31-50.el7.src.rpm
 bind-9.9.4-73.el7_6.src.rpm
 wget-1.14-18.el7_6.1.src.rpm
 kexec-tools-2.0.15-21.el7_6.3.src.rpm
 sos-3.6-17.el7.centos.src.rpm
 iptables-1.4.21-28.el7.src.rpm
 bind-9.9.4-73.el7_6.src.rpm
 vim-7.4.160-5.el7.src.rpm
 docker-1.13.1-96.gitb2f74b2.el7.centos.src.rpm
 docker-1.13.1-96.gitb2f74b2.el7.centos.src.rpm
 docker-1.13.1-96.gitb2f74b2.el7.centos.src.rpm
 container-selinux-2.95-2.el7_6.src.rpm
 subscription-manager-1.21.10-3.el7.centos.src.rpm
 policycoreutils-2.5-29.el7_6.1.src.rpm
 oci-umount-2.3.4-2.git87f9237.el7.src.rpm
 python-futures-3.1.1-5.el7.src.rpm
 libcgroup-0.41-20.el7.src.rpm
 git-1.8.3.1-20.el7.src.rpm
 container-storage-setup-0.11.0-2.git5eaf76c.el7.src.rpm
 perl-TermReadKey-2.30-20.el7.src.rpm
 vim-7.4.160-5.el7.src.rpm
 git-1.8.3.1-20.el7.src.rpm
 atomic-1.22.1-26.gitb507039.el7.centos.src.rpm
 python-setuptools-0.9.8-7.el7.src.rpm
 skopeo-0.1.35-2.git404c5bd.el7.centos.src.rpm
 python-backports-ssl_match_hostname-3.5.0.1-1.el7.src.rpm
 checkpolicy-2.5-8.el7.src.rpm
 audit-2.8.4-4.el7.src.rpm
 python-ipaddress-1.0.16-2.el7.src.rpm
 python-pytoml-0.1.14-1.git7dea353.el7.src.rpm
 libsemanage-2.5-14.el7.src.rpm
 oci-systemd-hook-0.1.18-3.git8787307.el7_6.src.rpm
 setools-3.3.8-4.el7.src.rpm
 python-IPy-0.75-6.el7.src.rpm
 iptables-1.4.21-28.el7.src.rpm
 python-backports-1.0-8.el7.src.rpm
 dracut-033-554.el7.src.rpm
 libselinux-2.5-14.1.el7.src.rpm
 oci-register-machine-0-6.git2b44233.el7.src.rpm
 selinux-policy-3.13.1-229.el7_6.12.src.rpm
 perl-Error-0.17020-2.el7.src.rpm
 bind-9.9.4-73.el7_6.src.rpm
 bind-9.9.4-73.el7_6.src.rpm
 libsemanage-2.5-14.el7.src.rpm
 selinux-policy-3.13.1-229.el7_6.12.src.rpm
 audit-2.8.4-4.el7.src.rpm
 dracut-033-554.el7.src.rpm
 policycoreutils-2.5-29.el7_6.1.src.rpm
 libselinux-2.5-14.1.el7.src.rpm
 libsepol-2.5-10.el7.src.rpm
 libselinux-2.5-14.1.el7.src.rpm