import files

2021-02-09 17:05:00 +01:00
parent f5c4576411
commit 44a6d37ba5
425 changed files with 23195 additions and 0 deletions
@@ -0,0 +1,12 @@
+; DO NOT EDIT (unless you know what you are doing)
+;
+; This subdirectory is a git "subrepo", and this file is maintained by the
+; git-subrepo command. See https://github.com/git-commands/git-subrepo#readme
+;
+[subrepo]
+	remote = ssh://git@github.com/1001Pharmacies/docker-vsftpd-s3
+	branch = master
+	commit = 6d5b3310525d8cb1be32f0461a8633aba8641b24
+	parent = 162fd1a1c05971c62996f6be53522c74559f567b
+	method = merge
+	cmdver = 0.4.0
@@ -0,0 +1,10 @@
+# Changelog
+
+## v1.0.0 (April 21, 2017)
+
+Initial release.
+
+* Install vsftpd
+* Install s3fs
+* Create ftp user
+* Mount s3 bucket in ftp user homedirectory
@@ -0,0 +1,69 @@
+FROM alpine:3.11 as dist
+LABEL maintainer "yann.autissier@gmail.com"
+ARG DOCKER_BUILD_DIR
+
+ARG S3FS_VERSION=v1.85
+
+# Install s3fs-fuse and sftpserver
+RUN apk --no-cache upgrade \
+ && apk --no-cache add --virtual .build-deps \
+        alpine-sdk \
+        automake \
+        autoconf \
+        curl-dev \
+        fuse-dev \
+        libressl-dev \
+        libgcrypt-dev \
+        libxml2-dev \
+ && git clone https://github.com/s3fs-fuse/s3fs-fuse \
+ && cd s3fs-fuse \
+ && git checkout tags/${S3FS_VERSION} -b ${S3FS_VERSION} \
+ && ./autogen.sh \
+ && ./configure --prefix=/usr/local \
+ && make install \
+ && cd .. \
+ && rm -rf s3fs-fuse \
+ && runDeps="$( \
+    scanelf --needed --nobanner --recursive /usr/local \
+      | awk '{ gsub(/,/, "\nso:", $2); print "so:" $2 }' \
+      | xargs -r apk info --installed \
+      | sort -u \
+  )" \
+ && apk del .build-deps \
+ && apk add --no-cache --virtual .run-deps $runDeps \
+        curl \
+        lftp \
+        logrotate \
+        openssh \
+        openssl \
+        vsftpd
+
+RUN sed -i 's|/var/log/messages|/var/log/*.log|' /etc/logrotate.conf
+
+RUN ssh-keygen -t ed25519 -f /etc/ssh/ssh_host_ed25519_key -N '' \
+ && ssh-keygen -t rsa -b 4096 -f /etc/ssh/ssh_host_rsa_key -N ''
+
+COPY ${DOCKER_BUILD_DIR}/lftp-sync.sh /usr/local/bin/
+RUN chmod 755 /usr/local/bin/lftp-sync.sh
+
+COPY ${DOCKER_BUILD_DIR}/docker-entrypoint.sh /
+ENTRYPOINT ["/docker-entrypoint.sh"]
+EXPOSE 21/tcp
+EXPOSE 1022/tcp
+EXPOSE 65000/tcp
+VOLUME ["/var/log"]
+
+FROM dist as local
+ARG DOCKER_BUILD_DIR
+
+FROM local as debug
+ARG DOCKER_BUILD_DIR
+
+FROM local as tests
+ARG DOCKER_BUILD_DIR
+
+FROM tests as preprod
+ARG DOCKER_BUILD_DIR
+
+FROM preprod as prod
+ARG DOCKER_BUILD_DIR
@@ -0,0 +1,20 @@
+MIT License
+
+Copyright (c) 2017 1001Pharmacies
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,86 @@
+# docker-vsftpd-s3
+
+Alpine based Dockerfile running a vsftpd server providing secure FTP access to an Amazon S3 bucket.
+This docker image can run in Amazon ECS.
+
+## Usage
+
+Following environment variables can be customized.
+
+```shell
+AWS_ACCESS_KEY_ID=               # acces key of the AWS user, required
+AWS_SECRET_ACCESS_KEY=           # secret key of the AWS user, required
+S3_BUCKET=                       # the S3 bucket name, required
+S3_ACL=private                   # default to private, optional
+FTPD_USER=s3ftp                  # the ftp user, default to s3ftp, optional
+FTPD_PASS=s3ftp                  # the ftp password, default to s3ftp, optional
+FTPD_BANNER=                     # the ftp banner
+CMDS_ALLOWED=                    # the ftp allowed commands, default to upload only, no delete or download, optional
+PASV_ADDRESS=                    # the ftp server external IP address, default to the AWS instance public IP, optional
+PASV_MIN_PORT=65000              # the ftp server pasv_min_port, default to 65000, optional
+PASV_MAX_PORT=65000              # the ftp server pasv_max_port, default to 65000, optional
+FTP_SYNC=                        # enable file synchronisation with a remote ftp server
+FTP_HOST=                        # the remote ftp server to sync with
+FTP_USER=                        # the ftp user to connect to remote ftp server
+FTP_PASS=                        # the ftp password to connect to remote ftp server
+REMOTE_DIR=                      # the directory to sync from on the remote ftp server, default to /
+LOCAL_DIR=                       # the directory to sync to on the local server, default to /home/$FTPD_USER
+```
+
+When you need multiple FTPD_USERs to serve multiple S3_BUCKETs, you have to set all variables at once in a list of double twopoints separated values.
+
+FTPD_USERS="FTPD_USER_1::FTPD_PASS_1::S3_BUCKET_1::AWS_ACCESS_KEY_ID_1::AWS_SECRET_ACCESS_KEY_1 FTPD_USER_2::FTPD_PASS_2::S3_BUCKET_2::AWS_ACCESS_KEY_ID_2::AWS_SECRET_ACCESS_KEY_2 ..."
+
+You can specify values in FTPD_PASS, S3_BUCKET, AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY that will be use as default for FTPD_USERS.
+
+FTPD_USERS="FTPD_USER_1::FTPD_PASS_1::S3_BUCKET_1::AWS_ACCESS_KEY_ID_1::AWS_SECRET_ACCESS_KEY_1 FTPD_USER_2::FTPD_PASS_2::S3_BUCKET_2 FTPD_USER_3 FTPD_USER_4::FTPD_PASS_4"
+
+## AWS Notes
+
+### IAM User
+
+You should create an IAM User with a dedicated access/secret key pair to access your S3 bucket and create a specific strategy attached to this user. 
+
+```json
+{
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": "s3:*",
+            "Resource": [
+                "arn:aws:s3:::my-bucket",
+                "arn:aws:s3:::my-bucket/*"
+            ]
+        }
+    ]
+}
+```
+
+### ELB/ECS Compatibility
+
+You can set an ELB listening on port 21 and forwarding requests to sftpd-s3 dockers running in an ECS cluster.
+
+### Security groups
+
+You should allow access on port 21 and port 65000 at least on the instances running sftpd-s3 dockers and on the attached ELB.
+
+## Example
+
+Build a docker image named "vsftpd-s3".
+
+```shell
+$ docker build -t vsftpd-s3 .
+```
+
+Start a docker from this image.
+
+```shell
+$ docker run -it --device /dev/fuse --cap-add sys_admin --security-opt apparmor:unconfined -p 21:21 -p 65000:65000 -e AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST -e AWS_SECRET_ACCESS_KEY=0123456789ABCDEF0123456789ABCDEF01234567 -e S3_BUCKET="my-s3-bucket" -e FTPD_USER="my_ftp_user" -e FTPD_PASS="my_ftp_password" vsftpd-s3
+```
+
+## Security notes
+
+Current docker image is shipped with FTPS and SFTP support, although SFTP support should be (and will be !) shipped in a separate docker image.
+SFTP is served by openssh listening on port 22. SFTP is not properly configured to chroot users in their homedir.
+This allows an authenticated user to leak the list of your ftp users.
+
@@ -0,0 +1,138 @@
+#!/bin/ash
+set -euo pipefail
+set -o errexit
+
+trap 'kill -SIGQUIT $PID' INT
+
+# VSFTPD PASV configuration
+PASV_ADDRESS=${PASV_ADDRESS:-$(timeout 1 wget -qO- http://169.254.169.254/latest/meta-data/public-ipv4 2>/dev/null ||:)}
+PASV_MIN_PORT=${PASV_MIN_PORT:-65000}
+PASV_MAX_PORT=${PASV_MAX_PORT:-65535}
+
+# VSFTPD Banner
+FTPD_BANNER=${FTPD_BANNER:-1001Pharmacies FTP Server}
+
+# FTP allowed commands
+# full command list : https://blog.vigilcode.com/2011/08/configure-secure-ftp-with-vsftpd/
+CMDS_ALLOWED=${CMDS_ALLOWED:-ABOR,ALLO,APPE,CCC,CDUP,CWD,DELE,FEAT,HELP,LIST,LPSV,MKD,MLST,MODE,NLST,NOOP,OPTS,PASS,PASV,PBSZ,PORT,PWD,QUIT,REIN,REST,RETR,RMD,RNFR,RNTO,SITE,SIZE,STAT,STOR,STRU,SYST,TYPE,USER}
+
+# Configure vsftpd
+echo "anonymous_enable=NO
+seccomp_sandbox=NO
+local_enable=YES
+write_enable=YES
+xferlog_enable=YES
+log_ftp_protocol=YES
+nopriv_user=vsftp
+chroot_local_user=YES
+allow_writeable_chroot=YES
+delete_failed_uploads=YES
+port_enable=YES
+port_promiscuous=YES
+cmds_allowed=$CMDS_ALLOWED
+ftpd_banner=$FTPD_BANNER
+pasv_enable=YES
+pasv_promiscuous=YES
+pasv_min_port=$PASV_MIN_PORT
+pasv_max_port=$PASV_MAX_PORT" > /etc/vsftpd.conf
+[ -n "$PASV_ADDRESS" ] && echo "pasv_address=$PASV_ADDRESS" >> /etc/vsftpd.conf
+
+# SSL certificate
+SSL_CERT_C=${SSL_CERT_C:-FR}
+SSL_CERT_ST=${SSL_CERT_ST:-Herault}
+SSL_CERT_L=${SSL_CERT_L:-Montpellier}
+SSL_CERT_O=${SSL_CERT_O:-1001Pharmacies}
+SSL_CERT_OU=${SSL_CERT_OU:-Hosting}
+SSL_CERT_CN=${SSL_CERT_CN:-ftp.1001pharmacies.com}
+
+# Create SSL certificate
+openssl req -x509 -nodes -days 365 -newkey rsa:2048 -addext extendedKeyUsage=serverAuth -addext subjectAltName=DNS:${SSL_CERT_CN} -subj "/C=${SSL_CERT_C}/ST=${SSL_CERT_ST}/L=${SSL_CERT_L}/O=${SSL_CERT_O}/OU=${SSL_CERT_OU}/CN=${SSL_CERT_CN}" -keyout /etc/ssl/private/vsftpd.pem -out /etc/ssl/private/vsftpd.pem 2>/dev/null && echo "
+rsa_cert_file=/etc/ssl/private/vsftpd.pem
+rsa_private_key_file=/etc/ssl/private/vsftpd.pem
+ssl_enable=YES
+allow_anon_ssl=YES
+force_anon_data_ssl=NO
+force_anon_logins_ssl=NO
+force_local_data_ssl=NO
+force_local_logins_ssl=NO
+ssl_tlsv1=YES
+ssl_sslv2=YES
+ssl_sslv3=YES
+require_cert=NO
+require_ssl_reuse=NO
+ssl_ciphers=HIGH" >> /etc/vsftpd.conf
+
+# Amazon S3 bucket
+S3_ACL=${S3_ACL:-private}
+S3_BUCKET=${S3_BUCKET:-s3bucket}
+
+# Amazon credentials
+AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-aws_access_key_id}
+AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-aws_secret_access_key}
+
+# VSFTPD credentials
+FTPD_USER=${FTPD_USER:-s3ftp}
+FTPD_PASS=${FTPD_PASS:-s3ftp}
+
+# Multi users
+FTPD_USERS=${FTPD_USERS:-${FTPD_USER}::${FTPD_PASS}::${S3_BUCKET}::${AWS_ACCESS_KEY_ID}::${AWS_SECRET_ACCESS_KEY}}
+
+# For each user
+echo "${FTPD_USERS}" |sed 's/ /\n/g' |while read line; do
+  echo ${line//::/ } |while read ftpd_user ftpd_pass s3_bucket aws_access_key_id aws_secret_access_key; do
+
+    # Check if user already exist
+    id ${ftpd_user} >/dev/null 2>&1 && continue
+
+    # Create FTP user
+    adduser -h /home/${ftpd_user} -s /sbin/nologin -D ${ftpd_user}
+    echo "${ftpd_user}:${ftpd_pass:-$FTPD_PASS}" | chpasswd 2> /dev/null
+
+    # Configure s3fs
+    echo "${aws_access_key_id:-$AWS_ACCESS_KEY_ID}:${aws_secret_access_key:-$AWS_SECRET_ACCESS_KEY}" > /home/${ftpd_user}/.passwd-s3fs
+    chmod 0400 /home/${ftpd_user}/.passwd-s3fs
+
+    # Mount s3fs
+    /usr/local/bin/s3fs ${s3_bucket:-$S3_BUCKET} /home/${ftpd_user} -o nosuid,nonempty,nodev,allow_other,complement_stat,mp_umask=027,uid=$(id -u ${ftpd_user}),gid=$(id -g ${ftpd_user}),passwd_file=/home/${ftpd_user}/.passwd-s3fs,default_acl=${S3_ACL},retries=5
+
+    # Exit docker if the s3 filesystem is not reachable anymore
+    ( crontab -l && echo "* * * * * timeout 3 touch /home/${ftpd_user}/.test >/dev/null || kill -KILL -1" ) | crontab -
+
+  done
+done
+
+# Enable SFTP
+echo "Protocol 2
+HostKey /etc/ssh/ssh_host_ed25519_key
+HostKey /etc/ssh/ssh_host_rsa_key
+UseDNS no
+PermitRootLogin no
+X11Forwarding no
+AllowTcpForwarding no
+Subsystem sftp internal-sftp
+ForceCommand internal-sftp -d %u
+ChrootDirectory /home
+Port 1022
+" > /etc/ssh/sshd_config
+
+# FTP sync client
+FTP_SYNC=${FTP_SYNC:-0}
+FTP_HOST=${FTP_HOST:-localhost}
+DIR_REMOTE=${DIR_REMOTE:-/}
+DIR_LOCAL=${DIR_LOCAL:-/home/$FTPD_USER}
+
+# Sync remote FTP every hour (at random time to allow multiple dockers to run)
+[ "$FTP_SYNC" != "0" ] \
+  && MIN=$(awk 'BEGIN { srand(); printf("%d\n",rand()*60)  }') \
+  && ( echo "$MIN * * * * /usr/local/bin/lftp-sync.sh $FTP_HOST $DIR_REMOTE $DIR_LOCAL/retour/\$(/bin/date +%Y/%m/%d) ^8.*$" ) | crontab -u ${FTPD_USER} - \
+  && MIN=$(awk 'BEGIN { srand(); printf("%d\n",rand()*rand()*60)  }') \
+  && ( crontab -u ${FTPD_USER} -l && echo "$MIN * * * * /usr/local/bin/lftp-sync.sh $FTP_HOST $DIR_REMOTE $DIR_LOCAL/facture ^INV.*$" ) | crontab -u ${FTPD_USER} - \
+  && touch /var/log/lftp-sync.log \
+  && chown ${FTPD_USER} /var/log/lftp-sync.log
+
+# Launch crond
+crond -L /var/log/crond.log
+
+# Launch sshd && vsftpd
+[ $# -eq 0 ] && /usr/sbin/sshd -e && /usr/sbin/vsftpd || exec "$@" &
+PID=$! && wait
@@ -0,0 +1,43 @@
+#!/bin/sh
+
+[ -d "/home/$FTPD_USER/log" ] && LOGDIR="/home/$FTPD_USER/log" || LOGDIR="/var/log"
+LOG=$LOGDIR/lftp-sync.log
+
+# lock to prevent multiple sync running together
+LOCK="${TMP}/.lock-${0##*/}"
+set -o noclobber
+(echo "$$" > $LOCK) 2>/dev/null && trap "rm ${LOCK}; exit" HUP INT TERM || exit=255
+set +o noclobber
+[ ${error:-0} -ne 0 ] && echo "ERROR : $(basename $0) is LOCKED on ${HOSTNAME}. Please remove ${LOCK}" |tee -a $LOG && exit ${exit}
+
+# check binaries
+which lftp >/dev/null 2>&1 || exit 1
+
+[ -n "$1" ] && FTP_HOST="$1"
+[ -n "$2" ] && DIR_REMOTE="$2"
+[ -n "$3" ] && DIR_LOCAL="$3"
+[ -n "$4" ] && FILES="$4"
+
+# check variables
+[ -n "$FTP_HOST" ] && [ -n "$FTP_USER" ] && [ -n "$FTP_PASS" ] || exit 2
+
+# check local path
+[ -d ${DIR_LOCAL:-~/} ] || mkdir -p ${DIR_LOCAL:-~/} || exit 3
+
+# Get files from the remote FTP server and remove them
+lftp ftp://$FTP_USER:$FTP_PASS@$FTP_HOST << EOC
+  set ftp:ssl-allow yes
+  set xfer:log-file $LOG
+  mirror \
+    --Remove-source-files \
+    -i "${FILES:-.*}" \
+    ${DIR_REMOTE:-/} \
+    ${DIR_LOCAL:-~/}
+  quit
+EOC
+
+# unlock
+rm -f "${LOCK}" 2>/dev/null && trap - HUP INT TERM
+
+# exit
+exit ${exit:-0}