import files

This commit is contained in:
Yann Autissier
2021-02-09 17:05:00 +01:00
parent f5c4576411
commit 44a6d37ba5
425 changed files with 23195 additions and 0 deletions
+133
View File
@@ -0,0 +1,133 @@
#!/bin/bash
# Author: Yann Autissier <yann.autissier@gmail.com>
DOCKER_IMAGE_REPOSITORY="centile"
DOCKER_BUILD_DIRECTORY="/etc/docker"
usage() {
echo Usage: $0 [-c cluster] [-f] [-q] [-t] image [image [...]]
echo -e "Build a docker image in the '${DOCKER_IMAGE_REPOSITORY}' repository."
echo
echo -e "image\tis a directory with a Dockerfile, default in '${DOCKER_BUILD_DIRECTORY}/image'."
echo -e "\t'image' can contains a dash. The suffixed part after the dash is taken into account"
echo -e "\tin the image name but not in the name of the directory containing the Dockerfile."
echo -e "\tsuffix will be available in your Dockerfile in the DOCKER_BUILD_SUFFIX build-arg."
echo
echo -e "Options:"
echo -e "\t-c 'cluster'\tAllow to override files in 'image' directory with existing files in"
echo -e "\t\t\tthe 'image/cluster' directory. 'cluster' will be available in your"
echo -e "\t\t\tDockerfile in the DOCKER_BUILD_PREFIX build-arg."
echo -e "\t-f\t\tforce build, do not use cache when building image."
echo -e "\t-q\t\tquiet mode, minimal output."
echo -e "\t-t\t\ttest mode, do nothing but output the command that would haev been launched."
echo
echo -e "EXAMPLES"
echo
echo -e "$0 elk"
echo -e "Build a docker image named '${DOCKER_IMAGE_REPOSITORY}/elk' with Dockerfile ${DOCKER_BUILD_DIRECTORY}/elk/Dockerfile"
echo
echo -e "$0 elk-es01"
echo -e "Build a docker image named '${DOCKER_IMAGE_REPOSITORY}/elk-es01' with Dockerfile ${DOCKER_BUILD_DIRECTORY}/elk/Dockerfile"
echo -e "and build-arg DOCKER_BUILD_SUFFIX=-es01"
echo
echo -e "$0 -c elisa-sdc elk-es01"
echo -e "Build a docker image named '${DOCKER_IMAGE_REPOSITORY}/elk-es01' with Dockerfile ${DOCKER_BUILD_DIRECTORY}/elk/Dockerfile,"
echo -e "build-arg DOCKER_BUILD_PREFIX=elisa-sdc/ and build-arg DOCKER_BUILD_SUFFIX=-es01"
echo
exit 1
}
while [ $# -gt 0 ]; do
case $1 in
-c|--cluster) shift && CLUSTER="$1"
;;
-f|--force) FORCE=1
;;
-t|--test) TEST=1
;;
-q|--quiet) QUIET=1
;;
-h|--help) usage
;;
*) args="${args:-} $1"
esac
shift
args="${args# }"
done
# check args
[ "${args:0:1}" = "-" ] && usage
# grsec/pax on alpine linux with docker < 1.12
[ -f /etc/alpine-release ] && while read major minor patch; do
if [ "${major}" -eq 1 ] && [ "${minor:-0}" -lt 12 ]; then
[ "$(sysctl -n kernel.grsecurity.chroot_deny_chmod 2>/dev/null)" = 1 ] && sysctl -w kernel.grsecurity.chroot_deny_chmod=0 2>/dev/null && grsec_disabled_chmod=1
[ "$(sysctl -n kernel.grsecurity.chroot_deny_mknod 2>/dev/null)" = 1 ] && sysctl -w kernel.grsecurity.chroot_deny_mknod=0 2>/dev/null && grsec_disabled_mknod=1
fi
done <<< $(apk version docker |awk -F '-' '/^docker/ {print $2}' |sed 's/\./ /g')
for arg in $args; do
# extract docker image name
image="$(basename ${arg})"
# keep part before the dash as the directory name
dir="$(dirname ${arg})/${image%-*}"
# keep part after the dash as an image suffix name
[ "${image##*-}" != "${image}" ] && suffix="${image##*-}"
# default to ${DOCKER_BUILD_DIRECTORY}/${dir} if ${dir} does not exists
[ ! -d "${dir}" ] && [ -d "${DOCKER_BUILD_DIRECTORY}/${dir}" ] && dir="${DOCKER_BUILD_DIRECTORY}/${dir#./}"
# directory exists && contains a Dockerfile
[ -d ${dir} ] && [ -f "${dir}/Dockerfile" ] || usage
# cluster directory exists
[ -n "${CLUSTER}" ] && { [ -d ${dir}/${CLUSTER} ] || usage; }
# search for Dockeropts files
files="${dir}/Dockeropts ${dir}/Dockeropts-${suffix}"
[ -n "${CLUSTER}" ] && files="${files} ${dir}/${CLUSTER}/Dockeropts ${dir}/${CLUSTER}/Dockeropts-${suffix}"
# source the Dockeropts files
for dockeropts in ${files}; do
[ -f "${dockeropts}" ] && . ${dockeropts}
done
# quiet build
[ ${QUIET} ] && DOCKER_BUILD_ARGS="--quiet" || DOCKER_BUILD_ARGS=""
# do not use cache
[ ${FORCE} ] && DOCKER_BUILD_ARGS="${DOCKER_BUILD_ARGS} --no-cache"
# extract DOCKER_ARGS
[ -n "${DOCKER_ARGS}" ] && for build_arg in ${DOCKER_ARGS}; do
DOCKER_BUILD_ARGS="${DOCKER_BUILD_ARGS} --build-arg ${build_arg}"
done
# add DOCKER_BUILD_PREFIX and DOCKER_BUILD_SUFFIX
[ -n "${CLUSTER}" ] && DOCKER_BUILD_ARGS="${DOCKER_BUILD_ARGS} --build-arg DOCKER_BUILD_PREFIX=${CLUSTER}/"
[ -n "${suffix}" ] && DOCKER_BUILD_ARGS="${DOCKER_BUILD_ARGS} --build-arg DOCKER_BUILD_SUFFIX=-${suffix}"
# search for Dockerfile
[ -n "${CLUSTER}" ] && files="${dir}/${CLUSTER}/Dockerfile-${suffix} ${dir}/${CLUSTER}/Dockerfile" || files=""
files="${files} ${dir}/Dockerfile-${suffix} ${dir}/Dockerfile"
# build docker image with 1st found Dockerfile
for dockerfile in ${files}; do
[ -f "${dockerfile}" ] || continue
[ ${QUIET} ] && [ ! ${TEST} ] && echo -n "${image} "
[ ! ${QUIET} ] && echo "Building image ${image}"
if [ ${TEST} ]; then
echo docker build ${DOCKER_BUILD_ARGS} -t ${DOCKER_IMAGE_REPOSITORY}/${image} -f ${dockerfile} ${dir}
else
docker build ${DOCKER_BUILD_ARGS} -t ${DOCKER_IMAGE_REPOSITORY}/${image} -f ${dockerfile} ${dir}
result=$?
fi
[ ${result:-0} -ge ${return:-0} ] && return=${result}
break
done
done
# grsec/pax
[ ${grsec_disabled_chmod} ] && sysctl -w kernel.grsecurity.chroot_deny_chmod=1 2>/dev/null
[ ${grsec_disabled_mknod} ] && sysctl -w kernel.grsecurity.chroot_deny_mknod=1 2>/dev/null
exit ${return:-1}
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
# Author: Yann Autissier <yann.autissier@gmail.com>
docker ps -q --no-trunc --filter status=exited,status=created,status=dead |while read docker; do docker rm ${docker}; done
@@ -0,0 +1,4 @@
#!/bin/sh
# Author: Yann Autissier <yann.autissier@gmail.com>
docker images -q --no-trunc --filter dangling=true |while read image; do docker rmi ${image}; done
@@ -0,0 +1,144 @@
#! /bin/bash
set -eou pipefail
#usage: sudo ./docker-cleanup-volumes.sh [--dry-run]
docker_bin="$(which docker.io 2> /dev/null || which docker 2> /dev/null)"
# Default dir
dockerdir=/var/lib/docker
# Look for an alternate docker directory with -g/--graph option
dockerpid=$(ps ax | grep "$docker_bin" | grep -v grep | awk '{print $1; exit}') || :
if [[ -n "$dockerpid" && $dockerpid -gt 0 ]]; then
next_arg_is_dockerdir=false
while read -d $'\0' arg
do
if [[ $arg =~ ^--graph=(.+) ]]; then
dockerdir=${BASH_REMATCH[1]}
break
elif [ $arg = '-g' ]; then
next_arg_is_dockerdir=true
elif [ $next_arg_is_dockerdir = true ]; then
dockerdir=$arg
break
fi
done < /proc/$dockerpid/cmdline
fi
dockerdir=$(readlink -f "$dockerdir")
volumesdir=${dockerdir}/volumes
vfsdir=${dockerdir}/vfs/dir
allvolumes=()
dryrun=false
verbose=false
function log_verbose() {
if [ "${verbose}" = true ]; then
echo "$1"
fi;
}
function delete_volumes() {
local targetdir=$1
echo
if [[ ! -d "${targetdir}" || ! "$(ls -A "${targetdir}")" ]]; then
echo "Directory ${targetdir} does not exist or is empty, skipping."
return
fi
echo "Delete unused volume directories from $targetdir"
local dir
while read -d $'\0' dir
do
dir=$(basename "$dir")
if [[ -d "${targetdir}/${dir}/_data" || "${dir}" =~ [0-9a-f]{64} ]]; then
if [ ${#allvolumes[@]} -gt 0 ] && [[ ${allvolumes[@]} =~ "${dir}" ]]; then
echo "In use ${dir}"
else
if [ "${dryrun}" = false ]; then
echo "Deleting ${dir}"
rm -rf "${targetdir}/${dir}"
else
echo "Would have deleted ${dir}"
fi
fi
else
echo "Not a volume ${dir}"
fi
done < <(find "${targetdir}" -mindepth 1 -maxdepth 1 -type d -print0 2>/dev/null)
}
if [ $UID != 0 ]; then
echo "You need to be root to use this script."
exit 1
fi
if [ -z "$docker_bin" ] ; then
echo "Please install docker. You can install docker by running \"wget -qO- https://get.docker.io/ | sh\"."
exit 1
fi
while [[ $# > 0 ]]
do
key="$1"
case $key in
-n|--dry-run)
dryrun=true
;;
-v|--verbose)
verbose=true
;;
*)
echo "Cleanup docker volumes: remove unused volumes."
echo "Usage: ${0##*/} [--dry-run] [--verbose]"
echo " -n, --dry-run: dry run: display what would get removed."
echo " -v, --verbose: verbose output."
exit 1
;;
esac
shift
done
# Make sure that we can talk to docker daemon. If we cannot, we fail here.
${docker_bin} version >/dev/null
container_ids=$(${docker_bin} ps -a -q --no-trunc)
#All volumes from all containers
SAVEIFS=$IFS
IFS=$(echo -en "\n\b")
for container in $container_ids; do
#add container id to list of volumes, don't think these
#ever exists in the volumesdir but just to be safe
allvolumes+=${container}
#add all volumes from this container to the list of volumes
log_verbose "Inspecting container ${container}"
for volpath in $(
${docker_bin} inspect --format='{{range $key, $val := .}}{{if eq $key "Volumes"}}{{range $vol, $path := .}}{{$path}}{{"\n"}}{{end}}{{end}}{{if eq $key "Mounts"}}{{range $mount := $val}}{{$mount.Source}}{{"\n"}}{{end}}{{end}}{{end}}' ${container} \
); do
log_verbose "Processing volumepath ${volpath}"
#try to get volume id from the volume path
vid=$(echo "${volpath}" | sed 's|.*/\(.*\)/_data$|\1|;s|.*/\([0-9a-f]\{64\}\)$|\1|')
# check for either a 64 character vid or then end of a volumepath containing _data:
if [[ "${vid}" =~ ^[0-9a-f]{64}$ || (${volpath} =~ .*/_data$ && ! "${vid}" =~ "/") ]]; then
log_verbose "Found volume ${vid}"
allvolumes+=("${vid}")
else
#check if it's a bindmount, these have a config.json file in the ${volumesdir} but no files in ${vfsdir} (docker 1.6.2 and below)
for bmv in $(find "${volumesdir}" -name config.json -print | xargs grep -l "\"IsBindMount\":true" | xargs grep -l "\"Path\":\"${volpath}\""); do
bmv="$(basename "$(dirname "${bmv}")")"
log_verbose "Found bindmount ${bmv}"
allvolumes+=("${bmv}")
#there should be only one config for the bindmount, delete any duplicate for the same bindmount.
break
done
fi
done
done
IFS=$SAVEIFS
delete_volumes "${volumesdir}"
delete_volumes "${vfsdir}"
@@ -0,0 +1,8 @@
#!/bin/bash
# Author: Yann Autissier <yann.autissier@gmail.com>
[ -n "$1" ] || exit
for docker in $@; do
docker inspect "${docker}" 2>/dev/null |awk '$1 == "\"Image\":" && $2 ~ /^\"sha/ {gsub(/(^\"|\",$)/, "", $2); print "'${docker}' "$2}'
done
@@ -0,0 +1,8 @@
#!/bin/bash
# Author: Yann Autissier <yann.autissier@gmail.com>
[ -n "$1" ] || exit
for docker in $@; do
docker inspect "${docker}" 2>/dev/null |awk '$1 == "\"Status\":" {gsub(/(^\"|\",$)/, "", $2); print "'${docker}' "$2}'
done
@@ -0,0 +1,4 @@
#!/bin/sh
# Author: Yann Autissier <yann.autissier@gmail.com>
[ -n "$1" ] && :> $(docker inspect $1 | grep '"LogPath": "*"' | sed -e 's/.*"LogPath": "//g' | sed -e 's/",//g')
@@ -0,0 +1,7 @@
#!/bin/sh
# Author: Yann Autissier <yann.autissier@gmail.com>
[ -n "$1" ] && \
docker_log=$(docker inspect $1 2>/dev/null | grep '"LogPath": "*"' | sed -e 's/.*"LogPath": "//g' | sed -e 's/",//g') && \
[ -f "${docker_log}" ] && \
tail -n 100 ${docker_log} > ${docker_log}
+157
View File
@@ -0,0 +1,157 @@
#!/bin/bash
# Author: Yann Autissier <yann.autissier@gmail.com>
DOCKER_IMAGE_REPOSITORY="centile"
DOCKER_BUILD_DIRECTORY="/etc/docker"
usage() {
echo Usage: $0 [ -c cluster] [ -i image ] [-f] [-q] [-t] name [name [...]]
echo -e "Run a docker from an image in the '${DOCKER_IMAGE_REPOSITORY}' repository."
echo
echo -e "name\t is a directory with a Dockerfile, default in '${DOCKER_BUILD_DIRECTORY}/name'."
echo -e "\t'name' can contains a dash. The directory name will be extracted for the first part"
echo -e "\tbefore a dash."
echo
echo -e "Options:"
echo -e "\t-c 'cluster'\tAllow to override files in 'image' directory with existing files in"
echo -e "\t\t\tthe 'image/cluster' directory."
echo -e "\t -i 'image'\tthe docker image to run, default in '${DOCKER_IMAGE_REPOSITORY}' repository."
echo -e "\t -f\t\tforce run, stop and remove existing docker before running a new one."
echo -e "\t -q\t\tquiet mode, minimal output."
echo -e "\t -t\t\ttest mode, do nothing but output the command that would have been launched."
echo
echo -e "EXAMPLES"
echo
echo -e "$0 elk"
echo -e "Run a docker named 'elk' from the '${DOCKER_IMAGE_REPOSITORY}/elk' image"
echo
echo -e "$0 elk-es01"
echo -e "Run a docker named 'elk-es01' from the '${DOCKER_IMAGE_REPOSITORY}/elk-es01' image"
echo
echo -e "$0 -i elk elk-es01"
echo -e "Run a docker named 'elk-es01' from the '${DOCKER_IMAGE_REPOSITORY}/elk' image"
echo
exit 1
}
while [ $# -gt 0 ]; do
case $1 in
-c|--cluster) shift && CLUSTER="$1"
;;
-i|--image) shift && IMAGE="$1"
;;
-h|--help) usage
;;
-f|--force) FORCE=1
;;
-q|--quiet) QUIET=1
;;
-t|--test) TEST=1
;;
*) args="${args:-} $1"
esac
shift
args="${args# }"
done
# check args
[ "${args:0:1}" = "-" ] && usage
for arg in ${args}; do
# reset vars
image=""; DOCKER_OPT=""
# extract docker name
name="$(basename ${arg})"
# keep part before the dash as the directory name
dir="$(dirname ${arg})/${name%-*}"
# keep part after the dash as an image suffix name
[ "${name##*-}" != "${name}" ] && suffix="${name##*-}"
# if provided, set docker image from args
if [ -n "${IMAGE}" ]; then
# if docker image does not contain a /, add our default repository
[ "${IMAGE##*/}" != "${IMAGE}" ] && image="${IMAGE}" || image="${DOCKER_IMAGE_REPOSITORY}/${IMAGE}"
# else try to find an image from the docker name
else
# try docker name, docker name without ending numbers, docker name without suffix
for image in ${name} ${name%%[0-9]*} ${name%-*}; do
# search for image in ${DOCKER_IMAGE_REPOSITORY}
[ -n "$(docker images 2>/dev/null |awk '$1 == "'${DOCKER_IMAGE_REPOSITORY}/${image}'" {print $1}')" ] && image="${DOCKER_IMAGE_REPOSITORY}/${image}" && break
image="${name}"
done
fi
tag="$(docker images |awk '$1 == "'${image}'" {print $2}')"
[ -z "${tag}" ] && echo "ERROR: Cannot find image '${image}'" >2 && exit 2
# default to ${DOCKER_BUILD_DIRECTORY}/${dir} if ${dir} does not exists
[ ! -d "${dir}" ] && [ -d "${DOCKER_BUILD_DIRECTORY}/${dir}" ] && dir="${DOCKER_BUILD_DIRECTORY}/${dir#./}"
# directory exists && contains a Dockerfile
[ -d ${dir} ] && [ -f "${dir}/Dockerfile" ] || usage
# cluster directory exists
[ -n "${CLUSTER}" ] && { [ -d ${dir}/${CLUSTER} ] || usage; }
# search for Dockeropts files
files="${dir}/Dockeropts ${dir}/Dockeropts-${suffix}"
[ -n "${CLUSTER}" ] && files="${files} ${dir}/${CLUSTER}/Dockeropts ${dir}/${CLUSTER}/Dockeropts-${suffix}"
# source the Dockeropts files
for dockeropts in ${files}; do
[ -f "${dockeropts}" ] && . ${dockeropts}
done
# extract SYSCTL
[ -n "${SYSCTL}" ] && for sysctl in ${SYSCTL}; do
sysctl -w ${sysctl} 2>/dev/null
done
# extract DOCKER_OPT
[ -n "${DOCKER_OPT}" ] && DOCKER_OPTS="--${DOCKER_OPT/ / --}" || DOCKER_OPTS=""
# extract DOCKER_ENV
[ -n "${DOCKER_ENV}" ] && DOCKER_OPTS="${DOCKER_OPTS} -e ${DOCKER_ENV//\" /\" -e }"
# extract DOCKER_LINK
[ -n "${DOCKER_LINK}" ] && DOCKER_OPTS="--link ${DOCKER_LINK/ / --link }"
# extract DOCKER_PORT
[ -n "${DOCKER_PORT}" ] && DOCKER_OPTS="${DOCKER_OPTS} -p ${DOCKER_PORT// / -p }"
# extract DOCKER_ULIMIT
[ -n "${DOCKER_ULIMIT}" ] && DOCKER_OPTS="${DOCKER_OPTS} --ulimit ${DOCKER_ULIMIT// / --ulimit }"
# extract DOCKER_VOLUME
[ -n "${DOCKER_VOLUME}" ] && DOCKER_OPTS="${DOCKER_OPTS} -v ${DOCKER_VOLUME// / -v }"
# enable access to host volumes on selinux
for volume in ${HOST_VOLUME}; do
chcon -Rt svirt_sandbox_file_t ${volume} 2>/dev/null
done
# remove current docker
if [ ${FORCE} ]; then
if [ -n "$(docker ps -q --filter status=created,status=restarting,status=running,status=paused,status=exited,status=dead,name=${name})" ]; then
[ ! ${QUIET} ] && echo -n "Removing docker ${name}... "
if [ ${TEST} ]; then
echo docker rm -f ${name}
else
eval docker rm -f ${name} >/dev/null 2>&1
result=$? && [ ${result} -ne 0 ] && echo "ERROR" && { [ ${result:-0} -ge ${return:-0} ] && return=${result}; } && break
[ ! ${QUIET} ] && echo "OK"
fi
fi
fi
# launch docker
[ ${QUIET} ] && [ ! ${TEST} ] && echo -n "${name} "
[ ! ${QUIET} ] && echo -n "Running docker ${name}... "
if [ ${TEST} ]; then
echo docker run --restart=always ${DOCKER_OPTS} -d --name ${name} ${image} ${DOCKER_RUN:-}
else
eval docker run --restart=always ${DOCKER_OPTS} -d --name ${name} ${image} ${DOCKER_RUN:-} 2>/dev/null
result=$? && [ ${result} -ne 0 ] && echo "ERROR"
fi
[ ${result:-0} -ge ${return:-0} ] && return=${result}
done
exit ${return:-1}