gitlab中怎么批量下载项目组中的代码

背景

现在越来越多的公司采用gitlab来管理代码。但是公司越来越大，项目越来越多，一个一个clone比较麻烦，于是写个脚本批量clone

思路

gitlab有提供api来获取git仓库的信息，利用这些信息clone下项目

参见：https://docs.gitlab.com/ee/api/projects.html#list-all-projects

步骤：

1、 申请gitlab token
进入gitlab setting页面 点击生成token

屏幕快照 2019-08-25 23.33.25.png

2 、在脚本中填入信息

脚本

from urllib.request import urlopen
import json
import subprocess, shlex
import time
import os

gitlabToken = '自己gitlab的token'         // 自己gitlab上的tokne
gitlabAddr = '地址'             //gitlab地址名 注意：去掉https://
target =    'group name'  //项目分组名 ， 为空则下载整个gitlab代码，慎用！

def get_next(group_id):
    url = gen_next_url(group_id)
    allProjects     = urlopen(url)
    allProjectsDict = json.loads(allProjects.read().decode())
    if len(allProjectsDict) == 0:
        return
    for thisProject in allProjectsDict: 
        try:
            thisProjectURL  = thisProject['ssh_url_to_repo']
            thisProjectPath = thisProject['path_with_namespace']
            if os.path.exists(thisProjectPath):
                command     = shlex.split('git -C "%s" pull' % (thisProjectPath))
            else:
                command     = shlex.split('git clone %s %s' % (thisProjectURL, thisProjectPath))
            resultCode  = subprocess.Popen(command)
            time.sleep(1)
        except Exception as e:
            print("Error on %s: %s" % (thisProjectURL, e.strerror))
    return resultCode

def have_next_projects(group_id):
    url = gen_next_url(group_id)
    allProjects     = urlopen(url)
    allProjectsDict = json.loads(allProjects.read().decode())
    if len(allProjectsDict) == 0:
        return False
    return True


def get_sub_groups(parent_id):
    url = gen_subgroups_url(parent_id)
    allProjects     = urlopen(url)
    allProjectsDict = json.loads(allProjects.read().decode())
    sub_ids = []
    if len(allProjectsDict) == 0:
        return sub_ids
    for thisProject in allProjectsDict: 
        try:
            id = thisProject['id']
            sub_ids.append(id)
        except Exception as e:
            print("Error on %s: %s" % (id, e.strerror))
    return sub_ids

def cal_next_sub_groupids(parent_id):
    parent = ''
    parent = parent_id
    is_start = 1
    parent_list = []
    sub_ids = get_sub_groups(parent_id)
    ok = have_next_projects(parent_id) 
    if len(sub_ids)!=0 and ok == False:
        for i in range(len(sub_ids)):
            print(sub_ids[i])
            parent = sub_ids[i]
            a = cal_next_sub_groupids(sub_ids[i])
            return a
    if len(sub_ids) !=0 and ok == True:
        for i in range(len(sub_ids)):
            parent = sub_ids[i]
            parent_list.append(sub_ids[i])
            a = cal_next_sub_groupids(sub_ids[i])
            parent_list.extend(a)
    if len(sub_ids) == 0 and ok == True:
        parent_list.append(parent)
        return parent_list
    if len(sub_ids) ==0 and ok == False:
        return parent_list
    return parent_list

def download_code(parent_id):
    data =cal_next_sub_groupids(parent_id)
    for group_id in data:
        get_next(group_id)
    return
 
def gen_next_url(target_id):
    return "https://%s/api/v4/groups/%s/projects?private_token=%s" % (gitlabAddr, target_id, gitlabToken)

def gen_subgroups_url(target_id):
    return "https://%s/api/v4/groups/%s/subgroups?private_token=%s" % (gitlabAddr, target_id, gitlabToken)

def gen_global_url():
    return "http://%s/api/v4/projects?private_token=%s" % (gitlabAddr, gitlabToken)

def download_global_code():
    url = gen_global_url()
    allProjects     = urlopen(url)
    allProjectsDict = json.loads(allProjects.read().decode())
    if len(allProjectsDict) == 0:
        return
    for thisProject in allProjectsDict: 
        try:
            thisProjectURL  = thisProject['ssh_url_to_repo']
            thisProjectPath = thisProject['path_with_namespace']
            print(thisProjectURL + ' ' + thisProjectPath)
            
            if os.path.exists(thisProjectPath):
                command     = shlex.split('git -C "%s" pull' % (thisProjectPath))
            else:
                command     = shlex.split('git clone %s %s' % (thisProjectURL, thisProjectPath))

            resultCode  = subprocess.Popen(command)
            print(resultCode)
            time.sleep(1)
        except Exception as e:
            print("Error on %s: %s" % (thisProjectURL, e.strerror))
    return
 
def main():
    if target == '':
        download_global_code()
    else:
        url     = "https://%s/api/v4/groups?private_token=%s" % (gitlabAddr,gitlabToken)
        allProjects     = urlopen(url)
        allProjectsDict = json.loads(allProjects.read().decode())
        if len(allProjectsDict) == 0:
            return
        target_id = '' 
        for thisProject in allProjectsDict: 
            try:
                this_name = thisProject['name']
                if target == this_name:
                    target_id = thisProject['id']
                    break
            except Exception as e:
                print("Error on %s: %s" % (this_name, e.strerror))
        download_code(target_id)
        return

main()

脚本我上传github了github地址：https://gist.github.com/linjunjj/440ea8b9b687b57f6c28217d58535e79