diff --git a/jenkins/Dockerfile b/jenkins/Dockerfile new file mode 100644 index 0000000..fbcdb82 --- /dev/null +++ b/jenkins/Dockerfile @@ -0,0 +1,145 @@ +FROM jenkins/inbound-agent:latest-alpine + +USER root + +RUN sed -i "s@https://dl-cdn.alpinelinux.org/@https://mirrors.huaweicloud.com/@g" /etc/apk/repositories; \ + apk add --no-cache \ + ca-certificates \ + libc6-compat \ + openssh-client + +RUN [ ! -e /etc/nsswitch.conf ] && echo 'hosts: files dns' > /etc/nsswitch.conf + +ENV DOCKER_VERSION 20.10.15 + +RUN set -eux; \ + \ + apkArch="$(apk --print-arch)"; \ + case "$apkArch" in \ + 'x86_64') \ + url='https://download.docker.com/linux/static/stable/x86_64/docker-20.10.15.tgz'; \ + ;; \ + 'armhf') \ + url='https://download.docker.com/linux/static/stable/armel/docker-20.10.15.tgz'; \ + ;; \ + 'armv7') \ + url='https://download.docker.com/linux/static/stable/armhf/docker-20.10.15.tgz'; \ + ;; \ + 'aarch64') \ + url='https://download.docker.com/linux/static/stable/aarch64/docker-20.10.15.tgz'; \ + ;; \ + *) echo >&2 "error: unsupported architecture ($apkArch)"; exit 1 ;; \ + esac; \ + \ + wget -O docker.tgz "$url"; \ + \ + tar --extract \ + --file docker.tgz \ + --strip-components 1 \ + --directory /usr/local/bin/ \ + ; \ + rm docker.tgz; \ + \ + dockerd --version; \ + docker --version + +COPY modprobe.sh /usr/local/bin/modprobe +COPY docker-entrypoint.sh /usr/local/bin/ + +ENV DOCKER_TLS_CERTDIR=/certs +RUN mkdir /certs /certs/client && chmod 1777 /certs /certs/client + +#ENTRYPOINT ["docker-entrypoint.sh"] +#CMD ["sh"] + +RUN set -eux; \ + apk add --no-cache \ + btrfs-progs \ + e2fsprogs \ + e2fsprogs-extra \ + ip6tables \ + iptables \ + openssl \ + shadow-uidmap \ + xfsprogs \ + xz \ + pigz \ + ; \ + if zfs="$(apk info --no-cache --quiet zfs)" && [ -n "$zfs" ]; then \ + apk add --no-cache zfs; \ + fi + +ENV DIND_COMMIT 42b1175eda071c0e9121e1d64345928384a93df1 +RUN set -eux; \ + addgroup -S dockremap; \ + adduser -S -G dockremap dockremap; \ + echo 'dockremap:165536:65536' >> /etc/subuid; \ + echo 'dockremap:165536:65536' >> /etc/subgid; \ + wget -O /usr/local/bin/dind "https://raw.githubusercontent.com/docker/docker/${DIND_COMMIT}/hack/dind"; \ + chmod +x /usr/local/bin/dind; \ + sed -i '40i\nohup dockerd-entrypoint.sh > nohup.out 2>&1 &' /usr/local/bin/jenkins-agent; \ + mkdir -p /root/.docker; \ + echo $'{ \n\ + "auths": { \n\ + "repository.anxinyun.cn": { \n\ + "auth": "YWRtaW46SGFyYm9yMTIzNDU=" \n\ + } \n\ + } \n\ +}' >> /root/.docker/config.json + +COPY dockerd-entrypoint.sh /usr/local/bin/ + +VOLUME /var/lib/docker +EXPOSE 2375 2376 + +#ENTRYPOINT ["dockerd-entrypoint.sh"] +#CMD [] + +#RUN apk add --no-cache iproute2 +# +#RUN mkdir /run/user && chmod 1777 /run/user +# +#RUN set -eux; \ +# echo 'jenkins:100000:65536' >> /etc/subuid; \ +# echo 'jenkins:100000:65536' >> /etc/subgid +# +#RUN set -eux; \ +# \ +# apkArch="$(apk --print-arch)"; \ +# case "$apkArch" in \ +# 'x86_64') \ +# url='https://download.docker.com/linux/static/stable/x86_64/docker-rootless-extras-20.10.15.tgz'; \ +# ;; \ +# 'aarch64') \ +# url='https://download.docker.com/linux/static/stable/aarch64/docker-rootless-extras-20.10.15.tgz'; \ +# ;; \ +# *) echo >&2 "error: unsupported architecture ($apkArch)"; exit 1 ;; \ +# esac; \ +# \ +# wget -O rootless.tgz "$url"; \ +# \ +# tar --extract \ +# --file rootless.tgz \ +# --strip-components 1 \ +# --directory /usr/local/bin/ \ +# 'docker-rootless-extras/rootlesskit' \ +# 'docker-rootless-extras/rootlesskit-docker-proxy' \ +# 'docker-rootless-extras/vpnkit' \ +# ; \ +# rm rootless.tgz; \ +# \ +# rootlesskit --version; \ +# vpnkit --version +# +## pre-create "/var/lib/docker" for our rootless user +#RUN set -eux; \ +# mkdir -p /home/jenkins/.local/share/docker; \ +# chown -R jenkins:jenkins /home/jenkins/.local/share/docker; \ +# sed -i '40i\nohup dockerd-entrypoint.sh > nohup.out 2>&1 &' /usr/local/bin/jenkins-agent +#VOLUME /home/jenkins/.local/share/docker + + + +#USER jenkins + + diff --git a/jenkins/docker-entrypoint.sh b/jenkins/docker-entrypoint.sh new file mode 100644 index 0000000..6503a18 --- /dev/null +++ b/jenkins/docker-entrypoint.sh @@ -0,0 +1,61 @@ +#!/bin/sh +set -eu + +# first arg is `-f` or `--some-option` +if [ "${1#-}" != "$1" ]; then + set -- docker "$@" +fi + +# if our command is a valid Docker subcommand, let's invoke it through Docker instead +# (this allows for "docker run docker ps", etc) +if docker help "$1" > /dev/null 2>&1; then + set -- docker "$@" +fi + +_should_tls() { + [ -n "${DOCKER_TLS_CERTDIR:-}" ] \ + && [ -s "$DOCKER_TLS_CERTDIR/client/ca.pem" ] \ + && [ -s "$DOCKER_TLS_CERTDIR/client/cert.pem" ] \ + && [ -s "$DOCKER_TLS_CERTDIR/client/key.pem" ] +} + +# if we have no DOCKER_HOST but we do have the default Unix socket (standard or rootless), use it explicitly +if [ -z "${DOCKER_HOST:-}" ] && [ -S /var/run/docker.sock ]; then + export DOCKER_HOST=unix:///var/run/docker.sock +elif [ -z "${DOCKER_HOST:-}" ] && XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}" && [ -S "$XDG_RUNTIME_DIR/docker.sock" ]; then + export DOCKER_HOST="unix://$XDG_RUNTIME_DIR/docker.sock" +fi + +# if DOCKER_HOST isn't set (no custom setting, no default socket), let's set it to a sane remote value +if [ -z "${DOCKER_HOST:-}" ]; then + if _should_tls || [ -n "${DOCKER_TLS_VERIFY:-}" ]; then + export DOCKER_HOST='tcp://docker:2376' + else + export DOCKER_HOST='tcp://docker:2375' + fi +fi +if [ "${DOCKER_HOST#tcp:}" != "$DOCKER_HOST" ] \ + && [ -z "${DOCKER_TLS_VERIFY:-}" ] \ + && [ -z "${DOCKER_CERT_PATH:-}" ] \ + && _should_tls \ +; then + export DOCKER_TLS_VERIFY=1 + export DOCKER_CERT_PATH="$DOCKER_TLS_CERTDIR/client" +fi + +if [ "$1" = 'dockerd' ]; then + cat >&2 <<-'EOW' + + 📎 Hey there! It looks like you're trying to run a Docker daemon. + + You probably should use the "dind" image variant instead, something like: + + docker run --privileged --name some-docker ... docker:dind ... + + See https://hub.docker.com/_/docker/ for more documentation and usage examples. + + EOW + sleep 3 +fi + +exec "$@" diff --git a/jenkins/dockerd-entrypoint.sh b/jenkins/dockerd-entrypoint.sh new file mode 100644 index 0000000..38bc262 --- /dev/null +++ b/jenkins/dockerd-entrypoint.sh @@ -0,0 +1,198 @@ +#!/bin/sh +set -eu + +_tls_ensure_private() { + local f="$1"; shift + [ -s "$f" ] || openssl genrsa -out "$f" 4096 +} +_tls_san() { + { + ip -oneline address | awk '{ gsub(/\/.+$/, "", $4); print "IP:" $4 }' + { + cat /etc/hostname + echo 'docker' + echo 'localhost' + hostname -f + hostname -s + } | sed 's/^/DNS:/' + [ -z "${DOCKER_TLS_SAN:-}" ] || echo "$DOCKER_TLS_SAN" + } | sort -u | xargs printf '%s,' | sed "s/,\$//" +} +_tls_generate_certs() { + local dir="$1"; shift + + # if ca/key.pem || !ca/cert.pem, generate CA public if necessary + # if ca/key.pem, generate server public + # if ca/key.pem, generate client public + # (regenerating public certs every startup to account for SAN/IP changes and/or expiration) + + # https://github.com/FiloSottile/mkcert/issues/174 + local certValidDays='825' + + if [ -s "$dir/ca/key.pem" ] || [ ! -s "$dir/ca/cert.pem" ]; then + # if we either have a CA private key or do *not* have a CA public key, then we should create/manage the CA + mkdir -p "$dir/ca" + _tls_ensure_private "$dir/ca/key.pem" + openssl req -new -key "$dir/ca/key.pem" \ + -out "$dir/ca/cert.pem" \ + -subj '/CN=docker:dind CA' -x509 -days "$certValidDays" + fi + + if [ -s "$dir/ca/key.pem" ]; then + # if we have a CA private key, we should create/manage a server key + mkdir -p "$dir/server" + _tls_ensure_private "$dir/server/key.pem" + openssl req -new -key "$dir/server/key.pem" \ + -out "$dir/server/csr.pem" \ + -subj '/CN=docker:dind server' + cat > "$dir/server/openssl.cnf" <<-EOF + [ x509_exts ] + subjectAltName = $(_tls_san) + EOF + openssl x509 -req \ + -in "$dir/server/csr.pem" \ + -CA "$dir/ca/cert.pem" \ + -CAkey "$dir/ca/key.pem" \ + -CAcreateserial \ + -out "$dir/server/cert.pem" \ + -days "$certValidDays" \ + -extfile "$dir/server/openssl.cnf" \ + -extensions x509_exts + cp "$dir/ca/cert.pem" "$dir/server/ca.pem" + openssl verify -CAfile "$dir/server/ca.pem" "$dir/server/cert.pem" + fi + + if [ -s "$dir/ca/key.pem" ]; then + # if we have a CA private key, we should create/manage a client key + mkdir -p "$dir/client" + _tls_ensure_private "$dir/client/key.pem" + chmod 0644 "$dir/client/key.pem" # openssl defaults to 0600 for the private key, but this one needs to be shared with arbitrary client contexts + openssl req -new \ + -key "$dir/client/key.pem" \ + -out "$dir/client/csr.pem" \ + -subj '/CN=docker:dind client' + cat > "$dir/client/openssl.cnf" <<-'EOF' + [ x509_exts ] + extendedKeyUsage = clientAuth + EOF + openssl x509 -req \ + -in "$dir/client/csr.pem" \ + -CA "$dir/ca/cert.pem" \ + -CAkey "$dir/ca/key.pem" \ + -CAcreateserial \ + -out "$dir/client/cert.pem" \ + -days "$certValidDays" \ + -extfile "$dir/client/openssl.cnf" \ + -extensions x509_exts + cp "$dir/ca/cert.pem" "$dir/client/ca.pem" + openssl verify -CAfile "$dir/client/ca.pem" "$dir/client/cert.pem" + fi +} + +# no arguments passed +# or first arg is `-f` or `--some-option` +if [ "$#" -eq 0 ] || [ "${1#-}" != "$1" ]; then + # set "dockerSocket" to the default "--host" *unix socket* value (for both standard or rootless) + uid="$(id -u)" + if [ "$uid" = '0' ]; then + dockerSocket='unix:///var/run/docker.sock' + else + # if we're not root, we must be trying to run rootless + : "${XDG_RUNTIME_DIR:=/run/user/$uid}" + dockerSocket="unix://$XDG_RUNTIME_DIR/docker.sock" + fi + case "${DOCKER_HOST:-}" in + unix://*) + dockerSocket="$DOCKER_HOST" + ;; + esac + + # add our default arguments + if [ -n "${DOCKER_TLS_CERTDIR:-}" ] \ + && _tls_generate_certs "$DOCKER_TLS_CERTDIR" \ + && [ -s "$DOCKER_TLS_CERTDIR/server/ca.pem" ] \ + && [ -s "$DOCKER_TLS_CERTDIR/server/cert.pem" ] \ + && [ -s "$DOCKER_TLS_CERTDIR/server/key.pem" ] \ + ; then + # generate certs and use TLS if requested/possible (default in 19.03+) + set -- dockerd \ + --host="$dockerSocket" \ + --host=tcp://0.0.0.0:2376 \ + --tlsverify \ + --tlscacert "$DOCKER_TLS_CERTDIR/server/ca.pem" \ + --tlscert "$DOCKER_TLS_CERTDIR/server/cert.pem" \ + --tlskey "$DOCKER_TLS_CERTDIR/server/key.pem" \ + "$@" + DOCKERD_ROOTLESS_ROOTLESSKIT_FLAGS="${DOCKERD_ROOTLESS_ROOTLESSKIT_FLAGS:-} -p 0.0.0.0:2376:2376/tcp" + else + # TLS disabled (-e DOCKER_TLS_CERTDIR='') or missing certs + set -- dockerd \ + --host="$dockerSocket" \ + --host=tcp://0.0.0.0:2375 \ + "$@" + DOCKERD_ROOTLESS_ROOTLESSKIT_FLAGS="${DOCKERD_ROOTLESS_ROOTLESSKIT_FLAGS:-} -p 0.0.0.0:2375:2375/tcp" + fi +fi + +if [ "$1" = 'dockerd' ]; then + # explicitly remove Docker's default PID file to ensure that it can start properly if it was stopped uncleanly (and thus didn't clean up the PID file) + find /run /var/run -iname 'docker*.pid' -delete || : + + if dockerd --version | grep -qF ' 20.10.'; then + # XXX inject "docker-init" (tini) as pid1 to workaround https://github.com/docker-library/docker/issues/318 (zombie container-shim processes) + set -- docker-init -- "$@" + fi + + if ! iptables -nL > /dev/null 2>&1; then + # if iptables fails to run, chances are high the necessary kernel modules aren't loaded (perhaps the host is using nftables with the translating "iptables" wrappers, for example) + # https://github.com/docker-library/docker/issues/350 + # https://github.com/moby/moby/issues/26824 + modprobe ip_tables || : + fi + + uid="$(id -u)" + if [ "$uid" != '0' ]; then + # if we're not root, we must be trying to run rootless + if ! command -v rootlesskit > /dev/null; then + echo >&2 "error: attempting to run rootless dockerd but missing 'rootlesskit' (perhaps the 'docker:dind-rootless' image variant is intended?)" + exit 1 + fi + user="$(id -un 2>/dev/null || :)" + if ! grep -qE "^($uid${user:+|$user}):" /etc/subuid || ! grep -qE "^($uid${user:+|$user}):" /etc/subgid; then + echo >&2 "error: attempting to run rootless dockerd but missing necessary entries in /etc/subuid and/or /etc/subgid for $uid" + exit 1 + fi + : "${XDG_RUNTIME_DIR:=/run/user/$uid}" + export XDG_RUNTIME_DIR + if ! mkdir -p "$XDG_RUNTIME_DIR" || [ ! -w "$XDG_RUNTIME_DIR" ] || ! mkdir -p "$HOME/.local/share/docker" || [ ! -w "$HOME/.local/share/docker" ]; then + echo >&2 "error: attempting to run rootless dockerd but need writable HOME ($HOME) and XDG_RUNTIME_DIR ($XDG_RUNTIME_DIR) for user $uid" + exit 1 + fi + if [ -f /proc/sys/kernel/unprivileged_userns_clone ] && unprivClone="$(cat /proc/sys/kernel/unprivileged_userns_clone)" && [ "$unprivClone" != '1' ]; then + echo >&2 "error: attempting to run rootless dockerd but need 'kernel.unprivileged_userns_clone' (/proc/sys/kernel/unprivileged_userns_clone) set to 1" + exit 1 + fi + if [ -f /proc/sys/user/max_user_namespaces ] && maxUserns="$(cat /proc/sys/user/max_user_namespaces)" && [ "$maxUserns" = '0' ]; then + echo >&2 "error: attempting to run rootless dockerd but need 'user.max_user_namespaces' (/proc/sys/user/max_user_namespaces) set to a sufficiently large value" + exit 1 + fi + # TODO overlay support detection? + exec rootlesskit \ + --net="${DOCKERD_ROOTLESS_ROOTLESSKIT_NET:-vpnkit}" \ + --mtu="${DOCKERD_ROOTLESS_ROOTLESSKIT_MTU:-1500}" \ + --disable-host-loopback \ + --port-driver=builtin \ + --copy-up=/etc \ + --copy-up=/run \ + ${DOCKERD_ROOTLESS_ROOTLESSKIT_FLAGS:-} \ + "$@" + elif [ -x '/usr/local/bin/dind' ]; then + # if we have the (mostly defunct now) Docker-in-Docker wrapper script, use it + set -- '/usr/local/bin/dind' "$@" + fi +else + # if it isn't `dockerd` we're trying to run, pass it through `docker-entrypoint.sh` so it gets `DOCKER_HOST` set appropriately too + set -- docker-entrypoint.sh "$@" +fi + +exec "$@" diff --git a/jenkins/entrypoint.sh b/jenkins/entrypoint.sh new file mode 100644 index 0000000..5da5d25 --- /dev/null +++ b/jenkins/entrypoint.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +nohup dockerd-entrypoint.sh > nohup.out 2>&1 & +/usr/local/bin/jenkins-agent + diff --git a/jenkins/modprobe.sh b/jenkins/modprobe.sh new file mode 100644 index 0000000..b357d89 --- /dev/null +++ b/jenkins/modprobe.sh @@ -0,0 +1,20 @@ +#!/bin/sh +set -eu + +# "modprobe" without modprobe +# https://twitter.com/lucabruno/status/902934379835662336 + +# this isn't 100% fool-proof, but it'll have a much higher success rate than simply using the "real" modprobe + +# Docker often uses "modprobe -va foo bar baz" +# so we ignore modules that start with "-" +for module; do + if [ "${module#-}" = "$module" ]; then + ip link show "$module" || true + lsmod | grep "$module" || true + fi +done + +# remove /usr/local/... from PATH so we can exec the real modprobe as a last resort +export PATH='/usr/sbin:/usr/bin:/sbin:/bin' +exec modprobe "$@"