Kubernetes环境集成GitLab

概述

1. 安装环境

本地k8s 1.19.4版本环境

2. 主要步骤

• 持久化存储（本地nfs服务）

• postgres数据库

• redis缓存服务

• gitlab应用

一、持久化存储服务准备

1. 安装nfs服务（ubuntu环境）

服务器端：

apt-get install nfs-kernel-server

mkdir -p /nfs_dir/{vol1,vol2,vol3,vol4}

vim /etc/exports

# 添加如下内容

/nfs_dir/vol1  192.168.3.0/24(rw,no_root_squash)

/nfs_dir/vol2  192.168.3.0/24(rw,no_root_squash)

/nfs_dir/vol3  192.168.3.0/24(rw,no_root_squash)

/nfs_dir/vol4  192.168.3.0/24(rw,no_root_squash)

# 启动nfs服务

systemctl start nfs-kernel-server

systemctl status nfs-kernel-server

systemctl enable nfs-kernel-server

客户端测试(k8s环境，安装在centos系统)：

客户端安装nfs-utils

yum -y install nfs-utils

手动挂载测试

mount -t nfs 192.168.3.15:/nfs_dir/vol1 /mnt/

2. 创建PV资源

apiVersion: v1

kind: PersistentVolume

metadata:

  name: pv001

  labels:

    name: pv001

spec:

  nfs:

    path: /nfs_dir/vol1

    server: 192.168.3.15

  accessModes: ["ReadWriteMany","ReadWriteOnce"]

  capacity:

    storage: 1Gi

---

apiVersion: v1

kind: PersistentVolume

metadata:

  name: pv002

  labels:

    name: pv002

spec:

  nfs:

    path: /nfs_dir/vol2

    server: 192.168.3.15

  accessModes: ["ReadWriteMany","ReadWriteOnce"]

  capacity:

    storage: 1Gi

---

apiVersion: v1

kind: PersistentVolume

metadata:

  name: pv003

  labels:

    name: pv003

spec:

  nfs:

    path: /nfs_dir/vol1

    server: 192.168.3.15

  accessModes: ["ReadWriteMany","ReadWriteOnce"]

  capacity:

    storage: 1Gi

---

apiVersion: v1

kind: PersistentVolume

metadata:

  name: pv004

  labels:

    name: pv004

spec:

  nfs:

    path: /nfs_dir/vol1

    server: 192.168.3.15

  accessModes: ["ReadWriteMany","ReadWriteOnce"]

  capacity:

    storage: 1Gi

部署PV资源定义

kubectl apply -f nfs-pv.yaml

[root@k8s-node1 gitlab]# kubectl get pv

NAME    CAPACITY  ACCESS MODES  RECLAIM POLICY  STATUS      CLAIM  STORAGECLASS  REASON  AGE

pv001  1Gi        RWO,RWX        Retain          Available                                  4s

pv002  1Gi        RWO,RWX        Retain          Available                                  4s

pv003  1Gi        RWO,RWX        Retain          Available                                  4s

pv004  1Gi        RWO,RWX        Retain          Available                                  4s

二、部署Postgresql数据库

1. 准备PostgreSQL数据库PVC

apiVersion: v1

kind: PersistentVolumeClaim

metadata:

  name: gitlab-postgresql-pvc

  namespace: default

spec:

  accessModes: ["ReadWriteMany"]

  resources:

    requests:

      storage: 1Gi

创建资源

kubectl apply -f gitlab-postgresql-pvc.yaml

2. 部署PostgresSQL数据库实例及Service资源

apiVersion: apps/v1

kind: Deployment

metadata:

  name: postgresql

  namespace: defaults

  labels:

    name: postgresql

spec:

  replicas: 1

  selector:

    matchLabels:

      name: postgresql

  template:

    metadata:

      name: postgresql

      labels:

        name: postgresql

    spec:

      containers:

      - name: postgresql

        image: sameersbn/postgresql

        imagePullPolicy: IfNotPresent

        env:

        - name: DB_USER

          value: gitlab

        - name: DB_PASS

          value: passw0rd

        - name: DB_NAME

          value: gitlab_production

        - name: DB_EXTENSION

          value: pg_trgm

        ports:

        - name: postgres

          containerPort: 5432

        volumeMounts:

        - mountPath: /var/lib/postgresql

          name: data

        livenessProbe:

          exec:

            command:

            - pg_isready

            - -h

            - localhost

            - -U

            - postgres

          initialDelaySeconds: 30

          timeoutSeconds: 5

        readinessProbe:

          exec:

            command:

            - pg_isready

            - -h

            - localhost

            - -U

            - postgres

          initialDelaySeconds: 5

          timeoutSeconds: 1

      volumes:

      - name: data

        persistentVolumeClaim:

          claimName: gitlab-postgresql-pvc

---

apiVersion: v1

kind: Service

metadata:

  name: postgresql

  namespace: default

  labels:

    name: postgresql

spec:

  ports:

    - name: postgres

      port: 5432

      targetPort: postgres

  selector:

    name: postgresql

三、部署redis缓存数据库

1. 存储资源PVC申请

apiVersion: v1

kind: PersistentVolumeClaim

metadata:

  name: gitlab-redis-pvc

  namespace: default

spec:

  accessModes: ["ReadWriteMany"]

  resources:

    requests:

      storage: 1Gi

2. redis实例及Service资源部署

apiVersion: apps/v1

kind: Deployment

metadata:

  name: redis

  namespace: default

  labels:

    name: redis

spec:

  replicas: 1

  selector:

    matchLabels:

      name: redis

  template:

    metadata:

      name: redis

      labels:

        name: redis

    spec:

      containers:

      - name: redis

        image: sameersbn/redis

        imagePullPolicy: IfNotPresent

        ports:

        - name: redis

          containerPort: 6379

        volumeMounts:

        - mountPath: /var/lib/redis

          name: data

        livenessProbe:

          exec:

            command:

            - redis-cli

            - ping

          initialDelaySeconds: 30

          timeoutSeconds: 5

        readinessProbe:

          exec:

            command:

            - redis-cli

            - ping

          initialDelaySeconds: 5

          timeoutSeconds: 1

      volumes:

      - name: data

        persistentVolumeClaim:

          claimName: gitlab-redis-pvc

---

apiVersion: v1

kind: Service

metadata:

  name: redis

  namespace: default

  labels:

    name: redis

spec:

  ports:

    - name: redis

      port: 6379

      targetPort: redis

  selector:

    name: redis

四、部署gitlab服务

1. 申请存储资源

apiVersion: v1

kind: PersistentVolumeClaim

metadata:

  name: gitlab-pvc

  namespace: default

spec:

  accessModes: ["ReadWriteMany"]

  resources:

    requests:

      storage: 1Gi

2. 添加secret资源配置root密码

echo -n "admin" > ./username

echo -n "admin123" > ./password

kubectl create secret generic db-user-pass --from-file=./username --from-file=./password

3. gitlab实例及Service部署

apiVersion: apps/v1

kind: Deployment

metadata:

  name: gitlab

  namespace: default

  labels:

    name: gitlab

spec:

  replicas: 1

    matchLabels:

      name: gitlab

  template:

    metadata:

      name: gitlab

      labels:

        name: gitlab

    spec:

      containers:

      - name: gitlab

        image: sameersbn/gitlab:12.1.6

        imagePullPolicy: IfNotPresent

        env:

        - name: TZ

          value: Asia/Shanghai

        - name: GITLAB_TIMEZONE

          value: Beijing

        - name: GITLAB_SECRETS_DB_KEY_BASE

          value: long-and-random-alpha-numeric-string

        - name: GITLAB_SECRETS_SECRET_KEY_BASE

          value: long-and-random-alpha-numeric-string

        - name: GITLAB_SECRETS_OTP_KEY_BASE

          value: long-and-random-alpha-numeric-string

        - name: GITLAB_ROOT_PASSWORD

          valueFrom:

            secretKeyRef:

              name: db-user-pass

              key: password

          # value: admin321

        - name: GITLAB_ROOT_EMAIL

          value: 1689991551@qq.com

        - name: GITLAB_HOST

          value: gitlab.jzh.com

        - name: GITLAB_PORT

          value: "80"

        - name: GITLAB_SSH_PORT

          value: "30022"

        - name: GITLAB_NOTIFY_ON_BROKEN_BUILDS

          value: "true"

        - name: GITLAB_NOTIFY_PUSHER

          value: "false"

        - name: GITLAB_BACKUP_SCHEDULE

          value: daily

        - name: GITLAB_BACKUP_TIME

          value: 01:00

        - name: DB_TYPE

          value: postgres

        - name: DB_HOST

          value: postgresql

        - name: DB_PORT

          value: "5432"

        - name: DB_USER

          value: gitlab

        - name: DB_PASS

          value: passw0rd

        - name: DB_NAME

          value: gitlab_production

        - name: REDIS_HOST

          value: redis

        - name: REDIS_PORT

          value: "6379"

        ports:

        - name: http

          containerPort: 80

        - name: ssh

          containerPort: 22

        volumeMounts:

        - mountPath: /home/git/data

          name: data

        livenessProbe:

          httpGet:

            path: /

            port: 80

          initialDelaySeconds: 180

          timeoutSeconds: 5

        readinessProbe:

          httpGet:

            path: /

            port: 80

          initialDelaySeconds: 5

          timeoutSeconds: 1

      volumes:

      - name: data

        persistentVolumeClaim:

          claimName: gitlab-pvc

---

apiVersion: v1

kind: Service

metadata:

  name: gitlab

  namespace: default

  labels:

    name: gitlab

spec:

  ports:

    - name: http

      port: 80

      targetPort: http

    - name: ssh

      port: 22

      targetPort: ssh

      nodePort: 30022

  type: NodePort

  selector:

    name: gitlab

4. 部署完成后查看资源运行状态

[root@k8s-node1 ~]# kubectl get po

NAME                          READY  STATUS    RESTARTS  AGE

gitlab-6dd65b86ff-tgp7s      1/1    Running  0          21m

postgresql-67cb84d6f8-bc87p  1/1    Running  0          22m

redis-f68fd858-9mgxr          1/1    Running  0          22m

[root@k8s-node1 ~]# kubectl get svc

NAME        TYPE        CLUSTER-IP      EXTERNAL-IP  PORT(S)                    AGE

gitlab      NodePort    10.97.236.245    <none>        80:31416/TCP,22:30022/TCP  21m

kubernetes  ClusterIP  10.96.0.1        <none>        443/TCP                    23h

postgresql  ClusterIP  10.108.172.207  <none>        5432/TCP                    22m

redis        ClusterIP  10.96.20.55      <none>        6379/TCP                    22m

[root@k8s-node1 ~]#

5. 通过IP:Port访问Gitlab服务，用户名root，密码admin123