Harbor同步仓库镜像到另一个仓库

Harbor同步仓库镜像到另一个仓库

• 因为近期有业务需求需要将一个Harbor仓库的所有镜像同步到另一个仓库，所以写了点脚本来处理

清单文件-示例

• 提前准备镜像清单（CSV文件头“项目, artifact, tag, 哈希值”）

custom-images,docker-image-tmp,test-20220211171020,sha256:8ea34c3291aa5a28ef7b8de188023037f8ccc3a31287bd5a80d6e58be3714bc7
custom-images,imagetmp,test-20211230154117,sha256:05dbe98097d2a6b489a4b7095a416c838004246b6ebdf33d8389df47bc451d58
custom-images,flaskweb,test-20220510162600,sha256:5e3451fea2a66ba82887f8be8d0d1f421438e5b7cc435aac8475204b02fa8c50
custom-images,flaskweb,test-20211223164921,sha256:31e69778c25ff51bd5d29ad6cfb340a46d86a07212c8f6f4813f557598ac2e8e

• 可以不要最后的哈希值
• 生成该列表的Python样例脚本见文末

同步脚本-示例

• Shell 脚本 transformer.sh

#! /bin/bash

# 镜像清单。CSV格式，头：项目,artifact,Tag,哈希值
images_list_file=image_list.csv

# Harbor镜像源仓库和推送的目标仓库
src_harbor=harbor.myai.com
tgt_harbor=harbor-internal.myai.com:44104
docker login -u admin -p NFW3xDVVa1ki ${src_harbor}
docker login -u aaas -p EML3G0zcV2Pv ${tgt_harbor}

# 本地单独备份镜像的目录，不想保存的话请注释掉下面docker save部分的代码
output_dir=/data/ai/harbor-images

# 开始处理
echo -e "\e[34m Start transmission: ${src_harbor} => ${tgt_harbor} \e[0m"

while read line
do
    # 按行读取文本，开始解析
    echo -e "\e[34m Transfer ===> ${line} \e[0m"
    array=(${line//,/ })
    project=${array[0]}
    artifact=${array[1]}
    tag=${array[2]}
    # digest=${array[3]}

    # 拉取，保存，推送，清理
    image_url="${project}/${artifact}:${tag}"
    docker pull "${src_harbor}/${image_url}"
    docker save "${src_harbor}/${image_url}" -o "${output_dir}/${project}.${artifact}.${tag}.tar"
    docker tag "${src_harbor}/${image_url}" "${tgt_harbor}/${image_url}"
    docker push "${tgt_harbor}/${image_url}"
    docker rmi "${src_harbor}/${image_url}"
    docker rmi "${tgt_harbor}/${image_url}"

done < $images_list_file

# 结束处理
echo -e "\e[34m End transmission: ${src_harbor} => ${tgt_harbor} \e[0m"

• 直接执行 $ bash transformer.sh
• 后台执行 $ nohup ./transformer.sh 1>run.log 2>&1 &

使用Python获取清单文件的示例

• 替换domain_url和 cookie 后，直接执行该Python代码即可

import requests
import json

domain_url = "http://1.1.1.1"
cookie = "rem-username=admin; _gorilla_csrf=MTcxxTA0NDI0N3xJbWcxY2tkREt2Vk1Wa3hIZG5sS01WWmhWVzR5YUU0MWVGQjNVMmtyYldaUWJYcEtWSGhaTkZaUFJsazlJZ289fHY6Uw-UXyL3iCnO_E4zkDoAgYj3QsEki0-dr0wmqb2i; sid=96e1f91f4c157a6f51489ca6962e2eb7"
user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/124.0.0.0"

def req_root_list():
    url = f"{domain_url}/api/v2.0/projects"
    querystring = {"page": "1", "page_size": "25"}
    headers = {
        "cookie": cookie,
        "user-agent": user_agent,
        "accept": "application/json",
        "accept-language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6",
        "cache-control": "no-cache",
        "connection": "keep-alive",
        "content-type": "application/json"
    }
    response = requests.request("GET", url, headers=headers, params=querystring)

    if response.status_code == 200:
        return json.loads(response.text)
    else:
        raise Exception("Error get_root_list")


def req_project_list(project_name: str):
    url = f"{domain_url}/api/v2.0/projects/{project_name}/repositories"
    querystring = {"page_size": "15", "page": "1"}
    headers = {
        "cookie": cookie,
        "user-agent": user_agent,
        "accept": "application/json, text/plain, */*",
        "accept-language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6",
        "connection": "keep-alive",
    }
    response = requests.request("GET", url, headers=headers, params=querystring)

    if response.status_code == 200:
        return json.loads(response.text)
    else:
        raise Exception("Error get_project_list")


def req_artifacts_list(project_name: str, artifacts_name: str):
    url = f"{domain_url}/api/v2.0/projects/{project_name}/repositories/{artifacts_name}/artifacts"
    querystring = {
        "with_tag": "false",
        "with_scan_overview": "true",
        "with_label": "true",
        "page_size": "15",
        "page": "1",
    }
    headers = {
        "cookie": cookie,
        "user-agent": user_agent,
        "accept": "application/json, text/plain, */*",
        "accept-language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6",
        "connection": "keep-alive",
        "x-accept-vulnerabilities": "application/vnd.security.vulnerability.report; version=1.1, application/vnd.scanner.adapter.vuln.report.harbor+json; version=1.0",
    }
    response = requests.request("GET", url, headers=headers, params=querystring)

    if response.status_code == 200:
        return json.loads(response.text)
    else:
        raise Exception("Error get_artifacts_list")


def req_tags_by_digest(project_name: str, artifacts_name: str, digest: str):
    url = f"{domain_url}/api/v2.0/projects/{project_name}/repositories/{artifacts_name}/artifacts/{digest}/tags"
    querystring = {
        "with_signature": "true",
        "with_immutable_status": "true",
        "page_size": "8",
        "page": "1",
    }
    headers = {
        "cookie": cookie,
        "user-agent": user_agent,
        "accept": "application/json, text/plain, */*",
        "accept-language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6",
        "connection": "keep-alive",
    }

    response = requests.request("GET", url, headers=headers, params=querystring)

    if response.status_code == 200:
        return json.loads(response.text)
    else:
        raise Exception("Error get_tag_by_digest")


if __name__ == "__main__":
    # 获取Harbor根列表
    root_list = req_root_list()
    for r_item in root_list:
        project_name = r_item["name"]
        # 获取Harbor根下的项目内的列表
        project_list = req_project_list(project_name)
        for p_item in project_list:
            artifacts_name = p_item["name"].split("/")[1]
            # print(artifacts_name)
            artifacts_list = req_artifacts_list(project_name, artifacts_name)
            for a_item in artifacts_list:
                digest = a_item["digest"]
                tags = req_tags_by_digest(project_name, artifacts_name, digest)
                for tag in tags:
                    tag_name = tag["name"]
                    print(project_name + "/" + artifacts_name + ", " + tag_name + ", " + digest)