container
/
base-images
Archived
10
0
Fork 0
Code Releases Activity
Browse Source

change sync images

main
winloong 3 years ago
parent
dfe8dfc92d
commit
460672a289
14 changed files with 130 additions and 360 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 17
      syncimages/app/Dockerfile
  2. 48
      syncimages/app/app.py
  3. 102
      syncimages/app/dispatch.sh
  4. 24
      syncimages/app/docker-util.py
  5. 38
      syncimages/app/harbor.py
  6. 10
      syncimages/app/hosts.txt
  7. 0
      syncimages/app/images.txt
  8. 39
      syncimages/app/k8s.py
  9. 28
      syncimages/app/logging.ini
  10. 49
      syncimages/app/pull.sh
  11. 16
      syncimages/app/run.sh
  12. 27
      syncimages/app/ssh/id_rsa
  13. 1
      syncimages/app/ssh/id_rsa.pub
  14. 91
      syncimages/app/syncimages.py

17
syncimages/app/Dockerfile
View File

@ -1,15 +1,20 @@
FROM docker:20-dind
FROM docker:20-dind-rootless
USER root
RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.tuna.tsinghua.edu.cn/g' /etc/apk/repositories \
&& apk add --no-cache python3 py3-pip \
&& apk add --no-cache python3 py3-pip shadow \
&& ln -sf python3 /usr/bin/python \
&& python3 -m ensurepip \
&& pip3 install --no-cache --upgrade pip setuptools \
&& pip install kubernetes requests \
&& rm -rf /var/cache/apk/*
&& pip3 install --no-cache --upgrade pip setuptools -i https://pypi.tuna.tsinghua.edu.cn/simple \
&& pip install --no-cache kubernetes requests docker -i https://pypi.tuna.tsinghua.edu.cn/simple \
&& rm -rf /var/cache/apk/* \
&& usermod -aG ping rootless \
&& echo "0 1 * * * run-parts /app/syncimages.py" >> /etc/crontabs/root
WORKDIR /app
COPY . .
RUN chmod 600 -R ssh
RUN chown -R rootless:rootless /app
USER rootless

48
syncimages/app/app.py
View File

@ -1,48 +0,0 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
import k8s, harbor
import os
import argparse
argparser = argparse.ArgumentParser()
argparser.description = '输入一个参数，--is_first '
argparser.add_argument('-f', '--is_first', dest='is_first', default=False, help='是否是第一次执行')
ns_list = ['anxincloud', 'environment', 'smart-city', 'smart-xxx', 'free-sun', 'ops']
def first_get_all():
for ns in ns_list:
image_list = k8s.list_deployment(ns)
out_file(image_list)
def out_file(image_list):
image_url_list = list(
map(lambda i: "repository.anxinyun.cn/{}/{}:{}".format(i['project'], i['name'], i['tag']), image_list))
if os.path.exists('images.txt'):
os.remove("images.txt")
with open("images.txt", "a") as file:
file.writelines(s + '\n' for s in image_url_list)
def get_latest_images():
pre_image_list = []
for ns in ns_list:
image_list = k8s.list_deployment(ns)
for image in image_list:
tag = harbor.get_image_latest_tag(image['project'], image['name'])
if tag != image['tag']:
pre_image_list.append(image)
out_file(pre_image_list)
if __name__ == '__main__':
args = argparser.parse_args()
if args.is_first:
first_get_all()
else:
get_latest_images()
os.system('sh ./pull.sh images.txt')
os.system('sh ./dispatch.sh images.tar.gz hosts.txt')

102
syncimages/app/dispatch.sh
View File

@ -1,102 +0,0 @@
#!/bin/sh
if [ x${1} = x ]; then
echo -e "\033[31m 请在第一个命令行参数指定 *-images.txt 文件 \033[0m"
exit
fi
if [ ! -f "${1}" ]; then
echo -e "\033[31m 文件 ${1} 不存在 \033[0m"
exit
fi
if [ x${2} = x ]; then
echo -e "\033[31m 请在第二个命令行参数指定 target-hosts.txt 文件 \033[0m"
exit
fi
if [ ! -f "${2}" ]; then
echo -e "\033[31m 文件 ${2} 不存在 \033[0m"
exit
fi
read line < ${2}
prvKey=$(echo ${line})
if [ ! -f "${prvKey}" ]; then
echo -e "\033[31m 文件 '${prvKey}' 不存在，请在文件 ${2} 的第一行指定 ssh privateKey 的路径 \033[0m"
exit
fi
while read line
do
let count++
if [ ${count} -gt 1 ]; then
line=$(echo $line)
if [ x${line} = x ]; then
continue
fi
user=$(echo ${line%@*})
ipport=$(echo ${line#*@})
ip=$(echo ${ipport%:*})
port=$(echo ${ipport#*:})
if [ x${user} = x${line} -o x${ip} = x${ipport} -o x${port} = x${ipport} ]; then
echo -e "\033[31m 文件 ${2} 的第 ${count} 行应该符合 user@192.168.2.10:22 的格式，当前该行内容为： \033[0m"
echo ${line}
exit
fi
hostIndex=`expr ${count} - 1`
echo -e "\033[36m>>>>> 开始分发镜像到第 ${hostIndex} 个目标主机 ${ip} >>>>>\033[0m"
scp -P ${port} -i ${prvKey} ${1} ${user}@${ip}:~/
ssh -p ${port} -i ${prvKey} ${user}@${ip} "rm -rf ${1%???????} || true
echo -e \"\033[36mstep ${hostIndex}.1 解压缩\033[0m\"
tar zxvf ${1}
echo -e \"\033[36mstep ${hostIndex}.2 加载镜像\033[0m\"
while read line
do
let c++
line=\$(echo \${line})
if [ x\${line} = x ]; then
echo -e \"Step ${hostIndex}.2.\${c} \033[33m第 \${c} 行为空\033[0m\"
continue
fi
echo -e \"Step ${hostIndex}.2.\${c} docker load < ${1%???????}/\${line//\//_}.tar \\t \\c\"
line=\$(echo \$line)
docker load < ${1%???????}/\${line//\//_}.tar
done < ${1%???????}/images.txt
echo -e \"\033[36m加载到目标主机 ${ip} 的镜像如下\033[0m\"
echo -e \"IMAGE ID\t CREATED\t\tSIZE\t\t REPOSITORY:TAG\"
while read line
do
line=\$(echo \${line})
if [ x\${line} = x ]; then
continue
fi
tag=\$(echo \${line%:*})
version=\$(echo \${line%:*})
docker images \${line} --format \"table {{.ID}}\t{{.CreatedSince}}\t{{.Size}}\t{{.Repository}}:{{.Tag}}\" | grep \${tag} | grep \${version}
done < ${1%???????}/images.txt
echo
echo -e \"\033[36m清理目标主机 ${ip} 上的临时文件\033[0m\"
rm -rf ${1%???????} || true
rm -rf ${1} || true
echo
" < /dev/null
echo -e "\033[32m<<<<< 已结束将镜像分发到第 ${hostIndex} 个目标主机 ${ip} <<<<<\033[0m"
echo ""
fi
done < ${2}
echo -e "\033[32m----- 已结束将镜像分发到 ${2} 文件中定义的所有主机 -----\033[0m"
echo ""

24
syncimages/app/docker-util.py
View File

@ -1,24 +0,0 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
import docker
client = docker.DockerClient(base_url='unix://var/run/docker.sock')
def exist_images(name):
# image = client.images.get('hello-world')
try:
client.images.get(name)
return True
except docker.errors.ImageNotFound:
return False
except docker.errors.APIError as err:
print(err)
if __name__ == '__main__':
print(exist_images('alpine:3.15'))
print(exist_images('alpine:latest'))

38
syncimages/app/harbor.py
View File

@ -1,38 +0,0 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
import requests
from urllib import parse
session = requests.Session()
HEADERS = {'Content-Type': 'application/x-www-form-urlencoded'}
def get_session():
payload = {"principal": 'admin', "password": 'Harbor12345'}
data = parse.urlencode(payload)
r = session.post("https://repository.anxinyun.cn/login", headers=HEADERS, data=data)
print(r.status_code)
def get_all_project():
url = "https://repository.anxinyun.cn/api/projects?page=1&page_size=100"
rs = session.get(url)
if rs.status_code == 200:
projects = dict()
for p in rs.json():
projects[p['name']] = p['project_id']
return projects
def get_image_latest_tag(project, image):
url = "https://repository.anxinyun.cn/api/repositories/{}/{}/tags?detail=1".format(project, image)
r = session.get(url)
if r.status_code == 200:
tags = list(map(lambda t: {'created': t['created'], 'tag': t['name']}, r.json()))
tags.sort(key=lambda x: x['created'], reverse=True)
return tags[0]['tag'] if len(tags) == 1 else tags[0]['tag'] if tags[0]['tag'] != 'latest' else tags[1]['tag']
# if len(tags) >= 2:
# return tags[0]['tag'] if tags[0]['tag'] != 'latest' else tags[1]['tag']
# else:
# return tags[0]['tag']

10
syncimages/app/hosts.txt
View File

@ -1,10 +0,0 @@
/app/ssh/id_rsa
anxinyun@10.8.40.111:22
anxinyun@10.8.40.112:22
anxinyun@10.8.40.113:22
anxinyun@10.8.40.114:22
anxinyun@10.8.40.115:22
anxinyun@10.8.40.116:22
anxinyun@10.8.40.117:22
anxinyun@10.8.40.118:22
anxinyun@10.8.40.122:22

0
syncimages/app/images.txt
View File

39
syncimages/app/k8s.py
View File

@ -1,39 +0,0 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
from kubernetes import client, config
config.load_kube_config("/app/config")
apps_v1 = client.AppsV1Api()
core_v1 = client.CoreV1Api()
def list_deployment(ns):
image_list = []
deploy_list = apps_v1.list_namespaced_deployment(namespace=ns).items
for item in deploy_list:
if item.spec.template.spec.containers[0].image.startswith('repository.anxinyun.cn'):
project_image = get_project(item.spec.template.spec.containers[0].image)
image_tag = get_image_tag(project_image[2])
image_list.append({'project': project_image[1], 'name': image_tag[0], 'tag': image_tag[1]})
return image_list
def get_project(image_url):
return image_url.split("/")
def get_image_tag(image):
return image.split(":")
def list_node():
nodes = core_v1.list_node()
for node in nodes.items:
print(node.metadata.name)
if __name__ == '__main__':
list_deployment()
# print(get_project('repository.anxinyun.cn/anxinyun/actionview-dashboard:11.21-12-09'))

28
syncimages/app/logging.ini
View File

@ -0,0 +1,28 @@
[loggers]
keys=root
[handlers]
keys=fileHandler,stream_handler
[formatters]
keys=formatter
[logger_root]
level=INFO
handlers=fileHandler,stream_handler
[handler_stream_handler]
class=StreamHandler
args=(sys.stdout,)
level=INFO
formatter=formatter
[handler_fileHandler]
class=FileHandler
args=('log.log', 'a')
level=INFO
formatter=formatter
[formatter_formatter]
format=%(asctime)s - %(name)s - %(levelname)s - %(message)s
datefmt=

49
syncimages/app/pull.sh
View File

@ -1,49 +0,0 @@
#!/bin/sh
folder=${1%.*}
echo ${folder}
if [ x"${folder}" = x ]; then
echo -e "\033[31m 请指定 *-images.txt 文件 \033[0m"
exit
fi
echo ""
echo "创建临时文件夹 ${folder}"
echo ""
rm -rf ${folder} || true
mkdir ${folder}
while read line
do
let count++
line=$(echo $line)
if [ x${line} = x ]; then
echo -e "\033[33m第 ${count} 行为空\033[0m"
echo
continue
fi
echo ">>>>> 下载第 ${count} 个镜像 ${line} >>>>>"
docker pull $line;
echo -e "\033[32m<<<<< 保存第 ${count} 个镜像到 ${folder}/${line//\//_}.tar \033[0m";
docker save $line > ${folder}/${line//\//_}.tar;
echo ""
done < ${1}
echo "----- 创建压缩文件 ${folder}.tar.gz -----"
cp ${1} ${folder}/images.txt
tar -zcvf ${folder}.tar.gz ${folder}/*.tar ${folder}/images.txt
echo -e "\033[32m----- 已压缩到文件 ${folder}.tar.gz ----- \033[0m"
echo -e "文件大小为 \c"
ls -hl ${folder}.tar.gz | awk '{print $5}'
echo ""
echo "清除临时文件夹 ${folder}"
rm -rf ${folder}
echo ""
echo "请执行以下指令，将镜像分发到 ./target-hosts.txt 文件中定义的目标主机上。"
echo -e "\033[36m./dispatch.sh ${folder}.tar.gz target-hosts.txt \033[0m"
echo ""

16
syncimages/app/run.sh
View File

@ -1,16 +0,0 @@
#!/bin/sh
#dockerd-entrypoint.sh
#exec nohup "dockerd-entrypoint.sh" > nohup.out 2>&1
nohup "dockerd-entrypoint.sh" &
COUNT=0
while [[ $COUNT -lt 10 ]]
do
echo $COUNT
sleep 1;
COUNT=$((COUNT+1))
done
python app.py -f True

27
syncimages/app/ssh/id_rsa
View File

@ -1,27 +0,0 @@
-----BEGIN RSA PRIVATE KEY-----
MIIEpQIBAAKCAQEA53LaR0IGLj4JIZwEECt7ZtpB2aqqy5y0UmkJujUS/fe11S34
BKLkrSOfycLNkkHQ6etmLFb0IYHXzUkF07AlG7esnndJ0E7mKBtEUv1MfAREYKSy
rCWpsuhXQpRlah9qHpGIoyg69VbYlKFRbdu/ziYt37zDzRiViFHNDZwr69M1pC5m
CCjfANmqAtn7xXPF+/OqBTWSqlCEraRSt2RCH8V4MaIOkyvN8KZDAMPlCZ3s0TvU
mtOi2HbrQWBxNHKl3BUWIxmTYQzZ9KewEZw5Mo7hmS6lpMKZ+0C7u9IJclwvsyBs
yckxQonvOr5VKDrYKb4PwipSq2/SQjsSAjcxNwIDAQABAoIBAQDDggFgsCUIat7L
xT6pahGTkEqP09rypCyucIwG/05LujOfIHWhdPg2SSFxDV0Zbv9Kmc51Jf6TT1s7
zbNeXiz6fO0T7zArBnrk5iOQ9ubk27Xm7TkAsc/nkNwlIbWJL4A00jrZl+I13GaX
Jq3iXv5m5Vla5dmAJoQp4u+Tz5hKW0Qj4hjpqMtZVhOEwCKCTGPVc7vR6KcpAW5B
ypp+zXFHMcEP/yjmNXME/GmxX5zd8+9etFZssvPLmbXQSN2CEdGBaYdZhZ1DGmG4
ML60AAP5EsHF3fLj2w45tDIem07bIJjfRa1pMNrJmoaW7RvhpUKycpjUg/cAg0YZ
EG1rTLIBAoGBAP1SzrWjBavm7AlsvFlre4OhkuvP0brStGkhpHKwFr38/GZyGw/Z
kG1GowaTTp0wD8+sTf71JBpAe4p2EEQeF9mWaGRvPKNqUbWM/0Iz5e6VZiMLSu3A
1Xw1ZzCklEM73GVWQYoNmtJZtedAR05ecodMo3UeF9NFSij750B4pmc7AoGBAOnk
4L1c18hAiynI7c3Ya8z/tQKMO1Rbat8oS83ujc/N+wwd73/iTuelDtu8I4gVQpWC
4JEhDhaofi7hOEm+iJSLPyaXGHd4Zby8Nx748hx6IapQxr+TqIT294coDG+K7CpA
nSATpLvdVeFGlPqV/DxVT4cTN7wMf+WEq2QN4lY1AoGARwShCND0NRYfFCFUyGjW
jreMXem8LXkGtPaGiNSO+6JiDEJvDcl7sPb9m0lO38hqllkC4LhO78EmIVIqCz64
hvqgt49r25Bh6djmcuPj0Tg3ExoGXpMSBqleDYgGPLcaeZpt80sPHWujEHq3wuO9
jerRZHMUUNl7CfRdB3kLhaMCgYEAr5QWNXC1t2jkTui7w3O8cPZfvlrgytGZZ44L
Ybq7kAxzccQjHuAXFYNtpPwVvDkhc7T8uVWUCuRPXQfKxmkWhFQHgwOX5U9nKAgu
ZLzCmyf/j6f1mqjQr4fphvdEZpNl983rZcH5PuHHb0YZ3garg+sSuTZu92Z2uCgg
tQpLJyUCgYEAgPg1OZcClZKaxbaQ3KyQvogKlHW803gTJO/JvbbIE+mIjxsWkKD8
alnJixyHfxK4h6Cd38xgsIc+ju1aAWPfe5CasMisU+2T9p41Mh6YCxxoRBYxtM/9
+iiOXwnlGgF2aN3w7RRMy3HPvI0NYX0KrobpQ2y5Of2EGpRQv2fjPnI=
-----END RSA PRIVATE KEY-----

1
syncimages/app/ssh/id_rsa.pub
View File

@ -1 +0,0 @@
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDnctpHQgYuPgkhnAQQK3tm2kHZqqrLnLRSaQm6NRL997XVLfgEouStI5/Jws2SQdDp62YsVvQhgdfNSQXTsCUbt6yed0nQTuYoG0RS/Ux8BERgpLKsJamy6FdClGVqH2oekYijKDr1VtiUoVFt27/OJi3fvMPNGJWIUc0NnCvr0zWkLmYIKN8A2aoC2fvFc8X786oFNZKqUIStpFK3ZEIfxXgxog6TK83wpkMAw+UJnezRO9Sa06LYdutBYHE0cqXcFRYjGZNhDNn0p7ARnDkyjuGZLqWkwpn7QLu70glyXC+zIGzJyTFCie86vlUoOtgpvg/CKlKrb9JCOxICNzE3 anxinyun@anxinyun-m3

91
syncimages/app/syncimages.py
View File

@ -0,0 +1,91 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
import os
import docker
from docker import errors
from kubernetes import client, config
import requests
import time
import logging.config
import logging
config.load_kube_config("/app/config")
apps_v1 = client.AppsV1Api()
client = None
logging.config.fileConfig("/app/logging.ini")
logger = logging.getLogger('sync-images')
ns_list = ['anxincloud', 'environment', 'smart-city', 'smart-xxx', 'free-sun', 'ops']
def list_deployment(ns):
image_list = []
deploy_list = apps_v1.list_namespaced_deployment(namespace=ns).items
for item in deploy_list:
if item.spec.template.spec.containers[0].image.startswith('repository.anxinyun.cn'):
project_image = get_project(item.spec.template.spec.containers[0].image)
image_tag = get_image_tag(project_image[2])
image_list.append({'project': project_image[1], 'name': image_tag[0], 'tag': image_tag[1]})
return image_list
def get_project(image_url):
return image_url.split("/")
def get_image_tag(image):
return image.split(":")
def get_image_latest_tag(project, image):
url = "https://repository.anxinyun.cn/api/repositories/{}/{}/tags?detail=1".format(project, image)
r = requests.get(url)
if r.status_code == 200:
tags = list(map(lambda t: {'created': t['created'], 'tag': t['name']}, r.json()))
tags.sort(key=lambda x: x['created'], reverse=True)
return tags[0]['tag'] if len(tags) == 1 else tags[0]['tag'] if tags[0]['tag'] != 'latest' else tags[1]['tag']
# if len(tags) >= 2:
# return tags[0]['tag'] if tags[0]['tag'] != 'latest' else tags[1]['tag']
# else:
# return tags[0]['tag']
def exist_images(name):
# image = client.images.get('hello-world')
try:
client.images.get(name)
return True
except errors.ImageNotFound:
return False
except errors.APIError as err:
print(err)
return False
except Exception as ex:
print(ex)
return False
def pull_images():
for ns in ns_list:
image_list = list_deployment(ns)
for image in image_list:
if not exist_images(image):
image_url = "repository.anxinyun.cn/{}/{}".format(image['project'], image['name'])
logger.info("下载镜像 {}:{} ... ...".format(image_url, image['tag']))
client.images.pull(image_url, image['tag'])
logger.info("镜像 {}:{} 下载完成。".format(image_url, image['tag']))
if __name__ == '__main__':
count = 0
while count < 10:
print("wait ... ...")
time.sleep(5)
count += 1
while not os.path.exists('/var/run/docker.sock'):
print("wait ... ...")
time.sleep(5)
print("init complete.")
client = docker.DockerClient(base_url='unix://var/run/docker.sock')
logger.info("start sync images ....")
pull_images()
logger.info("images sync finished.")