-
Choerodon平台版本：0.16.0
-
运行环境：自主搭建
Gitlab全量迁移到c7n流程图：
图例说明：
1.git mirror简单来说就是1比1镜像拉取gitlab仓库代码，保存所有的commit记录信息，具体信息请看下面的官方说明。
git clone --mirror官方说明：
git push --mirror官方说明：
devops_service数据库信息简单说明：
涉及到的表：
devops_project 项目表
devops_app_template 模板表
devops_application 应用表
devops_branch 分支表
devops_user 用户表
在数据库中各个表之间没有主外键关联关系，可以随便写入，但是实际上使用过程中，表关联主要关系如下图：
准备工作：
1.c7n平台正常运行
2.expose c7n-mysql数据库的service，添加externalIPs
3.本地localhost主机需要mysql客户端
4.本地localhost主机需要有git客户端
5.执行脚本的用户具有对mysql、git的执行权限
6.拉取CI/CD template文件存放到本地，建议拉取choerodon自带的CICD文件，或者通过拉取修改官方自带的文件来生成达到自己需要的效果，目录格式为：
7.编辑CICD/charts/demo/Chart.yaml文件，删除第三行对于name的描述，并把CICD目录存放于脚本同级目录
可能存在的问题：
用户token不存在，devops_service库里面的gitlab_token表里面，不存在对应用户的token，需要手动在c7n的gitlab里面创建token，并写入到c7n-mysql库里
没有找到上传附件的地方，就直接粘贴在后面吧，仅供参考，请根据自己实际情况调整
脚本内容：
#!/bin/sh
# authored by tay
# time 2019-05-09 ver 0.1
# used for tranfer gitlab to c7n
#######数据库信息配置#########
mysqlname="ipaddr" #c7n数据库地址
mysqlport="port" #数据库端口号
username="username" #数据库用户名
password="password" #数据库密码
dbname="devops_service"
tablename1="devops_application"
tablename2="devops_branch"
#########git配置信息#########
# 需要在脚本对应目录准备CI/CD的demo文件
git_user="username" #gitlab账户
git_passwd="password" #gitlab密码
git_new_user="********" #c7n gitlab账户
git_new_passwd="********" #c7n gitlab密码
git_new_url="http://********" #c7n gitlab地址
# c7n mysql部分参数值配置
app_template_code="****" #模板code,应用创建自哪个模板
basedir=$(cd `dirname $0`; pwd)
# 读取，手动输入要导入的战队信息
# project_id=
#read -p "请输入导入战队(ID):" project_id #战队ID
read -p "请输入导致战队code编码:" project_code
read -p "请输入指定导入的用户名:" all_user_name
read -p "请输入导入的战队gitlab域名地址:" git_project_url #战队gitlab域名 #c7n上的gitlab总路径,不包含项目路径,例如：http://
project_id=$(mysql -h${mysqlname} -P${mysqlport} -u${username} -p${password} -Diam_service -Nse "select ID from fd_project where CODE='${project_code}'")
all_user_id=$(mysql -h${mysqlname} -P${mysqlport} -u${username} -p${password} -D${dbname} -Nse "select gitlab_user_id from devops_user where gitlab_user_name='${all_user_name}'")
user_token=$(mysql -h${mysqlname} -P${mysqlport} -u${username} -p${password} -D${dbname} -Nse "select gitlab_token from devops_user where gitlab_user_name='${all_user_name}'")
# 判断战队、用户ID为数字
expr ${project_id} "+" 10 &> /dev/null
if [ $? -ne 0 ];then
f_logs "[error] 战队ID必须是数字number"
exit -1;
fi
# 初始化
function f_init(){
mkdir -p ${basedir}/logs
mkdir -p ${basedir}/old_lab
mkdir -p ${basedir}/new_lab
mkdir -p /tmp/CICD
}
# 第一阶段clone mirror整库到c7n系统的gitlab上
function f_logs(){
fdate=$(date "+%Y-%m-%d %H:%M:%S")
echo "$fdate : $1"
echo "$fdate : $1" >> $basedir/logs/tranfer.log
}
function f_mirror_git(){
# 读取文件获取git_old_url
cd ${basedir}/old_lab
git_dir=${git_old_url##*/}
expect <<-EOF
set timeout 1000
spawn git clone --mirror ${git_old_url}
expect "Username"
send "${git_user}\r"
expect "Password"
send "${git_passwd}\r"
expect eof
EOF
wait;
#判断是否拉取git库成功
if [[ ! -d ${git_dir} ]]; then
f_logs "[error]: git clone --mirror ${git_old_url} faild\n"
exit -1;
fi
f_logs "[info] git clone mirror ${git_old_url} successful !"
# 推送到c7n库
cd ${basedir}/old_lab/${git_dir}
git_dir=${git_dir,,} #转换为小写
project_name=${git_dir%.*}
expect <<-EOF
set timeout 1000
spawn git push --mirror ${git_project_url}/${git_dir}
expect "Username"
send "${git_new_user}\r"
expect "Password"
send "${git_new_passwd}\r"
expect eof
EOF
wait;
#git push --mirror http://${git_new_user}:${git_new_passwd}@${git_project_url}/${git_dir}
if [[ $? -eq '0' ]]; then
f_logs "[info] git mirror push to ${git_project_url} successful !"
else
f_logs "[error] git mirror push to ${git_project_url} faild !"
exit -1
fi
}
function f_clone_c7n(){
# 拉取c7n gitlab
cd ${basedir}/new_lab/
expect <<-EOF
set timeout 1000
spawn git clone ${git_project_url}/${git_dir}
expect "Username"
send "${git_new_user}\r"
expect "Password"
send "${git_new_passwd}\r"
expect eof
EOF
wait;
if [[ $? -eq '0' ]]; then
f_logs "[info] git clone from ${git_project_url} successful !"
else
f_logs "[error] git clone from ${git_project_url} faild !"
exit -1
fi
}
# 第二阶段clone checkout分支上传CI/CD所需的文件
# 拉取c7n gitlab库
function f_add_cicd(){
#project_name=${git_dir%.*}
# 存放CICD修改后的临时文件
if [[ ! -d /tmp/CICD ]]; then
mkdir -p /tmp/CICD
fi
cp -a ${basedir}/CICD/. /tmp/CICD/.
mv /tmp/CICD/charts/demo /tmp/CICD/charts/${project_name}
sed -i "3a\name\: ${project_name}" /tmp/CICD/charts/${project_name}/Chart.yaml
echo "CICD file is Ready !"
#git clone http://${git_new_user}:${git_new_passwd}@${git_project_url}/${git_dir}
# 添加CI/CD文件,修改对应内容
cd ${basedir}/new_lab/${project_name}
branchs=$(git branch -a|grep -v remotes|awk '{print $NF}')
for git_branch in ${branchs}; do
git checkout ${git_branch}
cp -a /tmp/CICD/. ${basedir}/new_lab/${project_name}/.
git add .
git commit -m "add ci/cd file"
#git push http://${git_new_user}:${git_new_passwd}@${git_project_url}/${git_dir}
expect <<-EOF
set timeout 1000
spawn git push ${git_project_url}/${git_dir}
expect "Username"
send "${git_new_user}\r"
expect "Password"
send "${git_new_passwd}\r"
expect eof
EOF
wait;
if [[ $? -eq '0' ]]; then
f_logs "[info] git push ${git_branch} ci/cd to ${git_project_url} successful !"
else
f_logs "[error] git push ${git_branch} ci/cd to ${git_project_url} faild !"
fi
done
#CICD完成，删除CICD临时文件
rm -rf /tmp/CICD
}
# 获取当前项目所有分支
# git branch -a
# checkout_commit=$(git log -1| grep commit | awk '{print $2}')
function f_write_application(){
# devops_application表信息填写
id='DEFAULT'
project_id=${project_id} ###战队ID
name=${project_name}
code=${project_name}
type=normal
is_active=1
is_synchro=1
is_failed="Null"
is_skip_check_permission=1
gitlab_project_id=${gitlab_project_id} ###gitlab库代码ID
hook_id=${hook_id}
app_template_id=$(mysql -h${mysqlname} -P${mysqlport} -u${username} -p${password} -D${dbname} -Nse "select id from devops_app_template where code='${app_template_code}'") #模板ID
harbor_config_id=1
chart_config_id=2
uuid="Null"
token=${token}
object_version_number=1
created_by=${all_user_id} ###创建项目的用户ID
creation_date="Null"
last_updated_by=${all_user_id} ###用户ID
last_update_date="Null"
#插入数据
insert_sql="insert into ${tablename1} values( ${id},'"${project_id}"','"${name}"','"${code}"','${type}','${is_active}','${is_synchro}',${is_failed},'${is_skip_check_permission}','${gitlab_project_id}','${hook_id}','${app_template_id}','${harbor_config_id}','${chart_config_id}',${uuid},'${token}','${object_version_number}','${created_by}',${creation_date},'${last_updated_by}',${last_update_date} )"
mysql -h${mysqlname} -P${mysqlport} -u${username} -p${password} -D${dbname} -e "${insert_sql}"
}
# token=$(cat /proc/sys/kernel/random/uuid)
function f_write_branch(){
# devops_branch表信息填写
id='DEFAULT'
app_id=$(mysql -h${mysqlname} -P${mysqlport} -u${username} -p${password} -D${dbname} -Nse "select id from devops_application where name='${project_name}'")
user_id=${all_user_id} # 用户ID
issue_id="Null"
is_deleted=0
checkout_date="Null"
last_commit_user=${all_user_id} #用户ID
last_commit_date="Null"
object_version_number=1
created_by=${all_user_id} #用户ID
creation_date="Null"
last_updated_by=${all_user_id} #用户ID
last_update_date="Null"
cd ${basedir}/new_lab/${project_name}
all_branch_name=$(git branch -a | egrep -v '\*|HEAD' | sed 's/remotes\/origin\///g' | awk '{print $1}')
for branch_name in ${all_branch_name}; do
if [[ ${branch_name} == "master" ]]; then
origin_branch=null
else
origin_branch=master
fi
git checkout ${branch_name}
checkout_commit=$(git log -1 | head -1 | awk '{print $2}')
last_commit=$(git log -1 | head -1 | awk '{print $2}')
last_commit_msg=$(git log -1| tail -1)
insert_sql="insert into ${tablename2} values (${id},'${app_id}','${user_id}',${issue_id},'"${branch_name}"','"${origin_branch}"','${checkout_commit}','${is_deleted}',${checkout_date},'${last_commit}','${last_commit_msg}','${last_commit_user}',${last_commit_date},'${object_version_number}','${created_by}',${creation_date},'${last_updated_by}',${last_update_date})"
mysql -h${mysqlname} -P${mysqlport} -u${username} -p${password} -D${dbname} -e "${insert_sql}"
done
}
# 第三阶段gitlab api添加webhook和Token变量，并写入数据库
function f_gitlab_add(){
# 获取到项目id
project_info=$(curl --request GET --header "PRIVATE-TOKEN: ${user_token}" ${git_new_url}/api/v4/projects?search=${project_name})
gitlab_project_id=$(echo ${project_info:1:-1} |jq .id )
#随机uuid
token=$(cat /proc/sys/kernel/random/uuid)
# 创建webhooks
hook_info=$(curl --request POST --header "PRIVATE-TOKEN: ${user_token}" "${git_new_url}/api/v4/projects/${gitlab_project_id}/hooks" \
--form "id=null" \
--form "url=http://api.${git_new_url#*.}/devops/webhook" \
--form "push_events=yes" \
--form "tag_push_events=yes" \
--form "merge_requests_events=yes" \
--form "job_events=yes" \
--form "pipeline_events=yes" \
--form "token=${token}" )
# 得到创建的webhook的id返回值
hook_id=$(echo $hook_info|jq .id)
# 创建Token变量
curl --request POST --header "PRIVATE-TOKEN: ${user_token}" "${git_new_url}/api/v4/projects/${gitlab_project_id}/variables" \
--form "key=Token" \
--form "value=${token}"
}
function main(){
f_init;
if [[ -f ${basedir}/git_old_url.txt ]]; then
git_old_urls=$(cat ${basedir}/git_old_url.txt)
else
read -p "请输入导出gitlab库的地址: " git_old_urls
fi
for git_old_url in ${git_old_urls}; do
f_logs "[info] Start program ${git_old_url}"
f_mirror_git;
f_gitlab_add;
f_clone_c7n;
f_write_application;
f_write_branch;
f_add_cicd;
f_logs "[info] End program ${git_old_url}"
rm -rf /tmp/CICD
done
}
main;