【分享】gitlab全量迁移到c7n gitlab脚本


#1

图例说明:
1.git mirror简单来说就是1比1镜像拉取gitlab仓库代码,保存所有的commit记录信息,具体信息请看下面的官方说明。

git clone --mirror官方说明:

git push --mirror官方说明:

devops_service数据库信息简单说明:

涉及到的表:

devops_project 项目表

devops_app_template 模板表

devops_application 应用表

devops_branch 分支表

devops_user 用户表

在数据库中各个表之间没有主外键关联关系,可以随便写入,但是实际上使用过程中,表关联主要关系如下图:

准备工作:

1.c7n平台正常运行

2.expose c7n-mysql数据库的service,添加externalIPs

3.本地localhost主机需要mysql客户端

4.本地localhost主机需要有git客户端

5.执行脚本的用户具有对mysql、git的执行权限

6.拉取CI/CD template文件存放到本地,建议拉取choerodon自带的CICD文件,或者通过拉取修改官方自带的文件来生成达到自己需要的效果,目录格式为:

7.编辑CICD/charts/demo/Chart.yaml文件,删除第三行对于name的描述,并把CICD目录存放于脚本同级目录

可能存在的问题:

用户token不存在,devops_service库里面的gitlab_token表里面,不存在对应用户的token,需要手动在c7n的gitlab里面创建token,并写入到c7n-mysql库里

没有找到上传附件的地方,就直接粘贴在后面吧,仅供参考,请根据自己实际情况调整
脚本内容:

#!/bin/sh

# authored by tay 
# time 2019-05-09 ver 0.1
# used for tranfer gitlab to c7n

#######数据库信息配置#########
mysqlname="ipaddr"									#c7n数据库地址
mysqlport="port"									#数据库端口号

username="username"									#数据库用户名
password="password"									#数据库密码

dbname="devops_service"								
tablename1="devops_application"						
tablename2="devops_branch"

#########git配置信息#########
# 需要在脚本对应目录准备CI/CD的demo文件

git_user="username"									#gitlab账户
git_passwd="password"								#gitlab密码

                   
git_new_user="********"								#c7n gitlab账户
git_new_passwd="********"							#c7n gitlab密码
git_new_url="http://********"						#c7n gitlab地址

# c7n mysql部分参数值配置
app_template_code="****"								#模板code,应用创建自哪个模板

basedir=$(cd `dirname $0`; pwd)
# 读取,手动输入要导入的战队信息
# project_id=
#read -p "请输入导入战队(ID):" project_id						#战队ID
read -p "请输入导致战队code编码:" project_code
read -p "请输入指定导入的用户名:" all_user_name
read -p "请输入导入的战队gitlab域名地址:" git_project_url		#战队gitlab域名  #c7n上的gitlab总路径,不包含项目路径,例如:http://
project_id=$(mysql -h${mysqlname}  -P${mysqlport}  -u${username} -p${password} -Diam_service -Nse "select ID from fd_project where CODE='${project_code}'")
all_user_id=$(mysql -h${mysqlname}  -P${mysqlport}  -u${username} -p${password} -D${dbname} -Nse "select gitlab_user_id from devops_user where gitlab_user_name='${all_user_name}'")
user_token=$(mysql -h${mysqlname}  -P${mysqlport}  -u${username} -p${password} -D${dbname} -Nse "select gitlab_token from devops_user where gitlab_user_name='${all_user_name}'")


# 判断战队、用户ID为数字
expr ${project_id} "+" 10 &> /dev/null
if [ $? -ne 0 ];then
	f_logs "[error] 战队ID必须是数字number"
	exit -1;
fi


# 初始化
function f_init(){
	mkdir -p ${basedir}/logs
	mkdir -p ${basedir}/old_lab
	mkdir -p ${basedir}/new_lab
	mkdir -p /tmp/CICD
}

# 第一阶段clone mirror整库到c7n系统的gitlab上
function f_logs(){
	fdate=$(date "+%Y-%m-%d %H:%M:%S")
	echo "$fdate : $1"
	echo "$fdate : $1" >> $basedir/logs/tranfer.log
}

function f_mirror_git(){

	# 读取文件获取git_old_url	
	
	cd ${basedir}/old_lab
	git_dir=${git_old_url##*/}
	expect <<-EOF
	set timeout 1000		
	spawn git clone --mirror ${git_old_url}
	expect "Username"
	send "${git_user}\r"
	expect "Password"
	send "${git_passwd}\r"
	expect eof
	EOF
	wait;
	#判断是否拉取git库成功
	if [[ ! -d ${git_dir} ]]; then
		f_logs "[error]: git clone --mirror ${git_old_url} faild\n"
		exit -1;
	fi
	f_logs "[info] git clone mirror ${git_old_url} successful !"

	# 推送到c7n库
	cd ${basedir}/old_lab/${git_dir}
	git_dir=${git_dir,,}			#转换为小写
	project_name=${git_dir%.*}
	expect <<-EOF	
	set timeout 1000	
	spawn git push --mirror ${git_project_url}/${git_dir}
	expect "Username"
	send "${git_new_user}\r"
	expect "Password"
	send "${git_new_passwd}\r"
	expect eof
	EOF
	wait;
	#git push --mirror http://${git_new_user}:${git_new_passwd}@${git_project_url}/${git_dir}
	if [[ $? -eq '0' ]]; then
		f_logs "[info] git mirror push to ${git_project_url} successful !"
	else
		f_logs "[error] git mirror push to ${git_project_url} faild !"
		exit -1
	fi
}

function f_clone_c7n(){

	# 拉取c7n gitlab
	cd ${basedir}/new_lab/
	expect <<-EOF
	set timeout 1000		
	spawn git clone ${git_project_url}/${git_dir}
	expect "Username"
	send "${git_new_user}\r"
	expect "Password"
	send "${git_new_passwd}\r"
	expect eof
	EOF
	wait;
	if [[ $? -eq '0' ]]; then
		f_logs "[info] git clone from ${git_project_url} successful !"
	else
		f_logs "[error] git clone from ${git_project_url} faild !"
		exit -1
	fi	
}

# 第二阶段clone checkout分支上传CI/CD所需的文件
# 拉取c7n gitlab库
function f_add_cicd(){

	#project_name=${git_dir%.*}
	# 存放CICD修改后的临时文件
	if [[ ! -d /tmp/CICD ]]; then
		mkdir -p /tmp/CICD
	fi
	cp -a ${basedir}/CICD/. /tmp/CICD/.
	mv /tmp/CICD/charts/demo  /tmp/CICD/charts/${project_name}
	sed -i "3a\name\: ${project_name}" /tmp/CICD/charts/${project_name}/Chart.yaml 
	echo "CICD file is Ready !"
	#git clone http://${git_new_user}:${git_new_passwd}@${git_project_url}/${git_dir}
	# 添加CI/CD文件,修改对应内容
	cd ${basedir}/new_lab/${project_name}
	branchs=$(git branch -a|grep -v remotes|awk '{print $NF}')
	for git_branch in ${branchs}; do
		git checkout ${git_branch}
		cp -a /tmp/CICD/. ${basedir}/new_lab/${project_name}/.
		git add .
		git commit -m "add ci/cd file"
		#git push http://${git_new_user}:${git_new_passwd}@${git_project_url}/${git_dir}
		expect <<-EOF
		set timeout 1000		
		spawn git push ${git_project_url}/${git_dir}
		expect "Username"
		send "${git_new_user}\r"
		expect "Password"
		send "${git_new_passwd}\r"
		expect eof
		EOF
		wait;
		if [[ $? -eq '0' ]]; then
			f_logs "[info] git push ${git_branch} ci/cd to ${git_project_url} successful !"
		else
			f_logs "[error] git push ${git_branch} ci/cd to ${git_project_url} faild !"
		fi
	done

	#CICD完成,删除CICD临时文件
	rm -rf /tmp/CICD	
}
# 获取当前项目所有分支
# git branch -a

# checkout_commit=$(git log -1| grep commit | awk '{print $2}')
function f_write_application(){

	# devops_application表信息填写
	id='DEFAULT'                                        
	project_id=${project_id}             ###战队ID
	name=${project_name}
	code=${project_name}
	type=normal
	is_active=1
	is_synchro=1
	is_failed="Null"
	is_skip_check_permission=1
	gitlab_project_id=${gitlab_project_id}        		###gitlab库代码ID
	hook_id=${hook_id}
	app_template_id=$(mysql -h${mysqlname}  -P${mysqlport}  -u${username} -p${password} -D${dbname} -Nse "select id from devops_app_template where code='${app_template_code}'")								#模板ID
	harbor_config_id=1
	chart_config_id=2
	uuid="Null"
	token=${token}
	object_version_number=1
	created_by=${all_user_id}									###创建项目的用户ID
	creation_date="Null"
	last_updated_by=${all_user_id} 								    ###用户ID
	last_update_date="Null"

	#插入数据
	insert_sql="insert into ${tablename1} values( ${id},'"${project_id}"','"${name}"','"${code}"','${type}','${is_active}','${is_synchro}',${is_failed},'${is_skip_check_permission}','${gitlab_project_id}','${hook_id}','${app_template_id}','${harbor_config_id}','${chart_config_id}',${uuid},'${token}','${object_version_number}','${created_by}',${creation_date},'${last_updated_by}',${last_update_date} )"

	mysql -h${mysqlname}  -P${mysqlport}  -u${username} -p${password} -D${dbname} -e "${insert_sql}"
}

# token=$(cat /proc/sys/kernel/random/uuid)
function f_write_branch(){

	# devops_branch表信息填写
	id='DEFAULT'
	app_id=$(mysql -h${mysqlname}  -P${mysqlport}  -u${username} -p${password} -D${dbname} -Nse "select id from devops_application where name='${project_name}'") 												
	user_id=${all_user_id}								# 用户ID
	issue_id="Null"
	is_deleted=0
	checkout_date="Null"
	last_commit_user=${all_user_id}									#用户ID
	last_commit_date="Null"
	object_version_number=1
	created_by=${all_user_id}										#用户ID
	creation_date="Null"
	last_updated_by=${all_user_id}									#用户ID
	last_update_date="Null"

	cd ${basedir}/new_lab/${project_name}
	all_branch_name=$(git branch -a | egrep -v '\*|HEAD' | sed 's/remotes\/origin\///g' | awk '{print $1}')
	for branch_name in ${all_branch_name}; do
		if [[ ${branch_name} == "master" ]]; then
			origin_branch=null
		else
			origin_branch=master
		fi
		git checkout ${branch_name}
		checkout_commit=$(git log -1 | head -1 | awk '{print $2}')
		last_commit=$(git log -1 | head -1 | awk '{print $2}')
		last_commit_msg=$(git log -1| tail -1)		
		insert_sql="insert into ${tablename2} values (${id},'${app_id}','${user_id}',${issue_id},'"${branch_name}"','"${origin_branch}"','${checkout_commit}','${is_deleted}',${checkout_date},'${last_commit}','${last_commit_msg}','${last_commit_user}',${last_commit_date},'${object_version_number}','${created_by}',${creation_date},'${last_updated_by}',${last_update_date})"
		mysql -h${mysqlname}  -P${mysqlport}  -u${username} -p${password} -D${dbname} -e "${insert_sql}"
	done
}


# 第三阶段gitlab api添加webhook和Token变量,并写入数据库
function f_gitlab_add(){

	# 获取到项目id
	project_info=$(curl --request GET --header "PRIVATE-TOKEN: ${user_token}" ${git_new_url}/api/v4/projects?search=${project_name}) 
	gitlab_project_id=$(echo ${project_info:1:-1} |jq .id )

	#随机uuid
	token=$(cat /proc/sys/kernel/random/uuid)

	# 创建webhooks
	hook_info=$(curl --request POST --header "PRIVATE-TOKEN: ${user_token}" "${git_new_url}/api/v4/projects/${gitlab_project_id}/hooks" \
	--form "id=null" \
	--form "url=http://api.${git_new_url#*.}/devops/webhook" \
	--form "push_events=yes" \
	--form "tag_push_events=yes" \
	--form "merge_requests_events=yes" \
	--form "job_events=yes" \
	--form "pipeline_events=yes" \
	--form "token=${token}" )
	# 得到创建的webhook的id返回值
	hook_id=$(echo $hook_info|jq .id)

	# 创建Token变量
	curl --request POST --header "PRIVATE-TOKEN: ${user_token}" "${git_new_url}/api/v4/projects/${gitlab_project_id}/variables" \
	--form "key=Token"  \
	--form "value=${token}"
}


function main(){
	f_init;
	if [[ -f ${basedir}/git_old_url.txt ]]; then
		git_old_urls=$(cat ${basedir}/git_old_url.txt)
	else
		read -p "请输入导出gitlab库的地址: "  git_old_urls
	fi
	for git_old_url in ${git_old_urls}; do
		f_logs "[info] Start program ${git_old_url}"
		f_mirror_git;
		f_gitlab_add;
		f_clone_c7n;		
		f_write_application;
		f_write_branch;
		f_add_cicd;
		f_logs "[info] End program ${git_old_url}"
		rm -rf /tmp/CICD
	done
}
main;

#2

点赞


#3

点赞


#4

这边平台是不建议通过这种方式导入gitlab应用,因为关联的表不只是devops服务,还有iam服务,其他的服务,导入gitlabproject choerodon界面上在新版本里面已经提供了应用导入功能,会导入应用仓库的代码。并同步所有的分支,且会加入choerodon平台所需要的ci-cd文件


#5

这个只是之前没有全量迁移前的临时解决办法,既然平台支持全量迁移了那当然更好