gggeek / db-3v4l

bin/dbstack

#!/usr/bin/env bash

# Manage the whole set of containers without having to learn Docker

# @todo allow end user to enter pwd for root db accounts on build. If not interactive, generate a random one
# @todo check for ports conflicts before starting the web container
# @todo if there's no docker-compose onboard but there is curl or wget, download and install docker-compose
# @todo add a command to remove the built containers and their anon volumes ? eg. `docker-compose rm -s -v`
# @todo make SETUP_APP_ON_BOOT take effect also on start. Also: make it tri-valued: skip, force and null

# consts
BOOTSTRAP_OK_FILE=/var/run/bootstrap_ok
COMPOSE_DEFAULT_ENV_FILE=.env
WORKER_SERVICE=worker
# vars
COMPOSE_LOCAL_ENV_FILE=${COMPOSE_LOCAL_ENV_FILE:-.env.local}
AS_ROOT=false
BOOTSTRAP_TIMEOUT=300
CLEANUP_UNUSED_IMAGES=false
DOCKER_COMPOSE_CMD=
DOCKER_NO_CACHE=
PARALLEL_BUILD=
PULL_IMAGES=false
REBUILD=false
RECREATE=false
SILENT=false
SETUP_APP_ON_BOOT=true
SILENT=false
VERBOSITY=
WORKER_CONTAINER=
WORKER_USER=

help() {
    printf "Usage: dbstack [OPTIONS] COMMAND [OPTARGS]

Manages the Db3v4l Docker Stack

Commands:
    build               build or rebuild the complete set of containers and set up the app. Leaves the stack running
    cleanup CATEGORY    remove temporary data/logs/caches/etc... CATEGORY can be any of:
                        - databases       NB: this removes all your data! Better done when containers are stopped
                        - docker-images   removes only unused images. Can be quite beneficial to free up space
                        - docker-logs     NB: for this to work, you'll need to run this script as root
                        - logs            removes log files from the databases, webservers, symfony
                        - shared-data     removes every file in the ./shared folder
    images              list container images
    kill [\$svc]         kill containers
    logs [\$svc]         view output from containers
    monitor             starts an interactive console for monitoring containers, images, volumes
    pause [\$svc]        pause the containers
    ps [\$svc]           show the status of running containers
    setup               set up the app without rebuilding the containers first
    run                 execute a single command in the worker container
    shell               start a shell in the worker container as the application user
    services            list docker-compose services
    restart [\$svc]      restart the complete set of containers
    start [\$svc]        start the complete set of containers
    stop [\$svc]         stop the complete set of containers
    top [\$svc]          display the running container processes
    unpause [\$svc]      unpause the containers

Options:
    -c              clean up docker images which have become useless - when running 'build'
    -e ENV_FILE     use an alternative to .env.local as local config file (path relative to the docker folder).
                    You can also use the env var COMPOSE_LOCAL_ENV_FILE for the same purpose.
    -h              print help
    -n              do not set up the app - when running 'build'
    -p              build containers in parallel - when running 'build'
    -r              force containers to rebuild from scratch (this forces a full app set up as well) - when running 'build'
                    log in as root instead of the app user account - when running 'shell' and 'run'
    -s              force app set up (via resetting containers to clean-build status besides updating them if needed) - when running 'build'
    -u              update containers by pulling the base images - when running 'build'
    -v              verbose mode
    -w SECONDS      wait timeout for completion of app and container set up - when running 'build' and 'start'. Defaults to ${BOOTSTRAP_TIMEOUT}
    -z              avoid using docker cache - when running 'build -r'
"
}

check_requirements() {

The mentioned parser error was in this brace group.

    # @todo do proper min-version checking

    which docker >/dev/null 2>&1
    if [ $? -ne 0 ]; then

Did you forget the 'then' for this 'if'?

Couldn't parse this if expression.

Use semicolon or linefeed before 'then' (or quote to make it literal).

        printf "\n\e[31mPlease install docker & add it to \$PATH\e[0m\n\n" >&2
        exit 1
    fi

Expected 'then'.

Unexpected keyword/token. Fix any mentioned problems and try again.

    which docker-compose >/dev/null 2>&1
    if [ $? -ne 0 ]; then
        printf "\n\e[31mPlease install docker-compose & add it to \$PATH\e[0m\n\n" >&2
        exit 1
    fi
}

# @todo have this function looked at by a bash guru to validate it's not a brainf**t
#load_config_value() {
#    local VALUE=
#    if [ -f ${COMPOSE_LOCAL_ENV_FILE} ]; then
#        VALUE=$(grep "^${1}=" ${COMPOSE_LOCAL_ENV_FILE})
#    fi
#    if [ -z "${VALUE}" ]; then
#        VALUE=$(grep "^${1}=" ${COMPOSE_DEFAULT_ENV_FILE})
#    fi
#    VALUE=${VALUE/${1}=/}
#}

load_config() {
    source "${COMPOSE_DEFAULT_ENV_FILE}"

    # @todo check that COMPOSE_LOCAL_ENV_FILE does not override env vars that it should not, eg. COMPOSE_LOCAL_ENV_FILE, BOOTSTRAP_OK_FILE, ...
    if [ -f "${COMPOSE_LOCAL_ENV_FILE}" ]; then
        # these vars we have to export as otherwise they are not taken into account by docker-compose
        set -a
        source "${COMPOSE_LOCAL_ENV_FILE}"
        set +a
    fi

    # @todo run `docker-compose ps` to retrieve the WORKER_CONTAINER id instead of parsing the config
    if [ -z "${COMPOSE_PROJECT_NAME}" ]; then
        printf "\n\e[31mCan not find the name of the composer project name in ${COMPOSE_DEFAULT_ENV_FILE} / ${COMPOSE_LOCAL_ENV_FILE}\e[0m\n\n" >&2
        exit 1
    fi
    # @todo the value for WORKER_USER could be hardcoded instead of being variable...
    if [ -z "${CONTAINER_USER}" ]; then
        printf "\n\e[31mCan not find the name of the container user account in ${COMPOSE_DEFAULT_ENV_FILE} / ${COMPOSE_LOCAL_ENV_FILE}\e[0m\n\n" >&2
        exit 1
    fi

    WORKER_CONTAINER="${COMPOSE_PROJECT_NAME}_${WORKER_SERVICE}"
    WORKER_USER=${CONTAINER_USER}
}

setup_local_config() {
    local CURRENT_USER_UID
    local CURRENT_USER_GID

    CURRENT_USER_UID=$(id -u)
    CURRENT_USER_GID=$(id -g)

    if [ "${CONTAINER_USER_UID}" = "${CURRENT_USER_UID}" -a "${CONTAINER_USER_GID}" = "${CURRENT_USER_GID}" ]; then
        return
    fi

    if [ -f "${COMPOSE_LOCAL_ENV_FILE}" ]; then
        # @todo in case the file already exists and has incorrect values for these vars, replace them instead of skipping...
        printf "\n\e[31mWARNING: current user id and/or group id do not match the ones in config files\e[0m\n\n" >&2
        # Q: why not just use the current values ?
        #export CONTAINER_USER_UID=${CURRENT_USER_UID}
        #export CONTAINER_USER_GID=${CURRENT_USER_GID}
        return
    fi

    echo "[$(date)] Setting up the configuration file '${COMPOSE_LOCAL_ENV_FILE}'..."

    #CURRENT_USER_UID=$(id -u)
    #CURRENT_USER_GID=$(id -g)
    #CONTAINER_USER_UID=$(grep -F CONTAINER_USER_UID ${COMPOSE_DEFAULT_ENV_FILE} | sed 's/CONTAINER_USER_UID=//')
    #CONTAINER_USER_GID=$(grep -F CONTAINER_USER_GID ${COMPOSE_DEFAULT_ENV_FILE} | sed 's/CONTAINER_USER_GID=//')

    touch "${COMPOSE_LOCAL_ENV_FILE}"

    if [ "${CONTAINER_USER_UID}" != "${CURRENT_USER_UID}" ]; then
        echo "CONTAINER_USER_UID=${CURRENT_USER_UID}" >> "${COMPOSE_LOCAL_ENV_FILE}"
        export CONTAINER_USER_UID=${CURRENT_USER_UID}
    fi
    if [ "${CONTAINER_USER_GID}" != "${CURRENT_USER_GID}" ]; then
        echo "CONTAINER_USER_GID=${CURRENT_USER_GID}" >> "${COMPOSE_LOCAL_ENV_FILE}"
        export CONTAINER_USER_GID=${CURRENT_USER_GID}
    fi

    # @todo allow setting up: custom db root account pwd, sf env, etc...
}

create_compose_command() {
    local VENDORS
    DOCKER_COMPOSE_CMD='docker-compose -f docker-compose.yml'
    if [ -n "${COMPOSE_ONLY_VENDORS}" ]; then
        VENDORS=${COMPOSE_ONLY_VENDORS//,/ }
    else
        VENDORS=$(cd compose && ls -- *.yml | tr '\n' ' ')
        VENDORS=${VENDORS//.yml/}
    fi
    if [ -n "${COMPOSE_EXCEPT_VENDORS}" ]; then
        for COMPOSE_EXCEPT_VENDOR in ${COMPOSE_EXCEPT_VENDORS//,/ }; do
            # @bug what if a COMPOSE_EXCEPT_VENDOR is a substring of a VENDOR ?
            VENDORS=${VENDORS/$COMPOSE_EXCEPT_VENDOR/}
        done
    fi
    for DC_CONF_FILE in ${VENDORS}; do
        DOCKER_COMPOSE_CMD="${DOCKER_COMPOSE_CMD} -f compose/${DC_CONF_FILE}.yml"
    done
}

build() {
    local IMAGES

    if [ ${CLEANUP_UNUSED_IMAGES} = 'true' ]; then
        # for good measure, do a bit of hdd disk cleanup ;-)
        echo "[$(date)] Removing unused Docker images from disk..."
        docker rmi $(docker images | grep "<none>" | awk "{print \$3}")
    fi

    echo "[$(date)] Building and Starting all Containers..."

    ${DOCKER_COMPOSE_CMD} ${VERBOSITY} stop
    if [ ${REBUILD} = 'true' ]; then
        ${DOCKER_COMPOSE_CMD} ${VERBOSITY} rm -f
    fi

    if [ ${PULL_IMAGES} = 'true' ]; then
        echo "[$(date)] Pulling base Docker images..."

        for DOCKERFILE in $(find . -name Dockerfile); do
            IMAGE=$(fgrep -h 'FROM' "${DOCKERFILE}" | sed 's/FROM //g')
            if [[ "${IMAGE}" == *'${base_image_version}'* ]]; then
                # @todo resolve the `base_image_version` dockerfile arg by resolving the source env var - run eg. docker-compose config
                DEFAULT_BASE_IMAGE=$(fgrep -h 'ARG base_image_version=' "${DOCKERFILE}" | sed 's/ARG base_image_version=//g')
                IMAGE=${IMAGE/\$\{base_image_version\}/$DEFAULT_BASE_IMAGE}

                printf "\e[31mWARNING: pulling base image ${IMAGE} for container ${DOCKERFILE} which might have been overwritten via env var...\e[0m\n" >&2
            fi

            docker pull "${IMAGE}"
        done
    fi

    ${DOCKER_COMPOSE_CMD} ${VERBOSITY} build ${PARALLEL_BUILD} ${DOCKER_NO_CACHE}

    if [ ${SETUP_APP_ON_BOOT} = 'false' ]; then
        export COMPOSE_SETUP_APP_ON_BOOT=false
    fi

    if [ ${RECREATE} = 'true' ]; then
        ${DOCKER_COMPOSE_CMD} ${VERBOSITY} up -d --force-recreate
        RETCODE=$?
    else
        ${DOCKER_COMPOSE_CMD} ${VERBOSITY} up -d
        RETCODE=$?
    fi
    if [ ${RETCODE} -ne 0 ]; then
        exit ${RETCODE}
    fi

    wait_for_bootstrap all
    RETCODE=$?

    if [ ${CLEANUP_UNUSED_IMAGES} = 'true' ]; then
        echo "[$(date)] Removing unused Docker images from disk, again..."
        docker rmi $(docker images | grep "<none>" | awk "{print \$3}")
    fi

    echo "[$(date)] Build finished"

    exit ${RETCODE}
}


# @todo loop over all args
cleanup() {
    case "${1}" in
        databases)
            if [ ${SILENT} != true ]; then
                echo "Do you really want to delete all database data?"
                select yn in "Yes" "No"; do
                    case $yn in
                        Yes ) break ;;
                        No ) exit 1 ;;
                    esac
                done
            fi

            find ./data/ -type f ! -name .gitkeep -delete
            # leftover sockets happen...
            find ./data/ -type s -delete
            find ./data/ -type d -empty -delete
        ;;
        docker-images)
            # @todo this gives a warning when no images are found to delete
            docker rmi $(docker images | grep "<none>" | awk "{print \$3}")
        ;;
        docker-logs)
            for CONTAINER in $(${DOCKER_COMPOSE_CMD} ps -q)
            do
                LOGFILE=$(docker inspect --format='{{.LogPath}}' ${CONTAINER})
                if [ -n "${LOGFILE}" ]; then
                    echo "" > "${LOGFILE}"
                fi
            done
        ;;
        logs)
            find ./logs/ -type f ! -name .gitkeep -delete
            find ../app/var/log/ -type f ! -name .gitkeep -delete
        ;;
        shared-data)
            if [ ${SILENT} != true ]; then
                echo "Do you really want to delete all data in the 'shared' folder?"
                select yn in "Yes" "No"; do
                    case $yn in
                        Yes ) break ;;
                        No ) exit 1 ;;
                    esac
                done
            fi

            find ../shared/ -type f ! -name .gitkeep -delete
        ;;
        symfony-cache)
            find ../app/var/cache/ -type f ! -name .gitkeep -delete
        ;;
        *)
            printf "\n\e[31mERROR: unknown cleanup target: ${1}\e[0m\n\n" >&2
            help
            exit 1
        ;;
    esac
}

setup_app() {
    echo "[$(date)] Starting the Worker container..."

    # avoid automatic app setup being triggered here
    export COMPOSE_SETUP_APP_ON_BOOT=false

    ${DOCKER_COMPOSE_CMD} ${VERBOSITY} up -d ${WORKER_SERVICE}
    RETCODE=$?
    if [ ${RETCODE} -ne 0 ]; then
        exit ${RETCODE}
    fi

    wait_for_bootstrap worker
    RETCODE=$?
    if [ ${RETCODE} -ne 0 ]; then
        exit ${RETCODE}
    fi

    echo "[$(date)] Setting up the app (from inside the Worker container)..."
    docker exec ${WORKER_CONTAINER} su - "${WORKER_USER}" -c "cd /home/${WORKER_USER}/app && composer install"
    echo "[$(date)] Setup finished"
}

# Wait until containers have fully booted
wait_for_bootstrap() {

    if [ ${BOOTSTRAP_TIMEOUT} -le 0 ]; then
        return 0
    fi

    case "${1}" in
        admin)
            BOOTSTRAP_CONTAINERS=adminer
        ;;
        all)
            # q: check all services or only the running ones?
            #BOOTSTRAP_CONTAINERS=$(${DOCKER_COMPOSE_CMD} config --services)
            BOOTSTRAP_CONTAINERS=$(${DOCKER_COMPOSE_CMD} ps --services | sort | tr '\n' ' ')
        ;;
        app)
            BOOTSTRAP_CONTAINERS='worker web adminer'
        ;;
        #web)
        #    BOOTSTRAP_CONTAINERS=web
        #;;
        #worker)
        #    BOOTSTRAP_CONTAINERS=worker
        #;;
        *)
            #printf "\n\e[31mERROR: unknown booting container: ${1}\e[0m\n\n" >&2
            #help
            #exit 1
            # @todo add check that this service is actually defined
            BOOTSTRAP_CONTAINERS=${1}
        ;;
    esac

    echo "[$(date)] Waiting for containers bootstrap to finish..."

    # @todo speed this up...
    #       - maybe go back to generating and checking files mounted on the host?
    #       - find a way to run commands in parallel while collecting their output/exit-code?
    #         see eg. https://www.golinuxcloud.com/run-shell-scripts-in-parallel-collect-exit-status-process/
    #         or https://gist.github.com/mjambon/79adfc5cf6b11252e78b75df50793f24
    local BOOTSTRAP_OK
    i=0
    while [ $i -le "${BOOTSTRAP_TIMEOUT}" ]; do
        sleep 1
        BOOTSTRAP_OK=''
        for BS_CONTAINER in ${BOOTSTRAP_CONTAINERS}; do
            printf "Waiting for ${BS_CONTAINER} ... "
            ${DOCKER_COMPOSE_CMD} exec ${BS_CONTAINER} cat ${BOOTSTRAP_OK_FILE} >/dev/null 2>/dev/null
            RETCODE=$?
            if [ ${RETCODE} -eq 0 ]; then
                printf "\e[32mdone\e[0m\n"
                BOOTSTRAP_OK="${BOOTSTRAP_OK} ${BS_CONTAINER}"
            else
                echo;
            fi
        done
        if [ -n "${BOOTSTRAP_OK}" ]; then
            for BS_CONTAINER in ${BOOTSTRAP_OK}; do
                BOOTSTRAP_CONTAINERS=${BOOTSTRAP_CONTAINERS//${BS_CONTAINER}/}
            done
            if [ -z  "${BOOTSTRAP_CONTAINERS// /}" ]; then
                break
            fi
        fi
        i=$(( i + 1 ))
    done
    if [ $i -gt 0 ]; then echo; fi

    if [ -n "${BOOTSTRAP_CONTAINERS// /}" ]; then
        printf "\n\e[31mBootstrap process did not finish within ${BOOTSTRAP_TIMEOUT} seconds\e[0m\n\n" >&2
        return 1
    fi

    return 0
}

### Begin live code

# @todo move to a function
while getopts ":ce:hnprsuvw:z" opt
do
    case $opt in
        c)
            CLEANUP_UNUSED_IMAGES=true
        ;;
        e)
            COMPOSE_LOCAL_ENV_FILE=${OPTARG}
        ;;
        h)
            help
            exit 0
        ;;
        n)
            SETUP_APP_ON_BOOT=false
        ;;
        p)
            PARALLEL_BUILD=--parallel
        ;;
        r)
            AS_ROOT=true
            REBUILD=true
        ;;
        s)
            RECREATE=true
        ;;
        u)
            PULL_IMAGES=true
        ;;
        v)
            VERBOSITY=--verbose
        ;;
        w)
            BOOTSTRAP_TIMEOUT=${OPTARG}
        ;;
        z)
            DOCKER_NO_CACHE=--no-cache
        ;;
        \?)
            printf "\n\e[31mERROR: unknown option -${OPTARG}\e[0m\n\n" >&2
            help
            exit 1
        ;;
    esac
done
shift $((OPTIND-1))

COMMAND=$1

check_requirements

cd "$(dirname -- ${BASH_SOURCE[0]})/../docker"

load_config

setup_local_config

create_compose_command

case "${COMMAND}" in
    build)
        build
    ;;

    cleanup)
        # @todo allow to pass many cleanup targets in one go
        cleanup "${2}"
    ;;

    config)
        ${DOCKER_COMPOSE_CMD} ${VERBOSITY} config
    ;;

    dbconsole)
        # @deprecated - left in for courtesy
        shift
        # scary line ? found it at https://stackoverflow.com/questions/12343227/escaping-bash-function-arguments-for-use-by-su-c
        docker exec -ti "${WORKER_CONTAINER}" su - "${WORKER_USER}" -c '"$0" "$@"' -- "/usr/bin/php" "app/bin/dbconsole" "$@"
    ;;

    images)
        ${DOCKER_COMPOSE_CMD} ${VERBOSITY} images ${2}
    ;;

    kill)
        ${DOCKER_COMPOSE_CMD} ${VERBOSITY} kill ${2}
    ;;

    logs)
        ${DOCKER_COMPOSE_CMD} ${VERBOSITY} logs ${2}
    ;;

    monitor)
        docker exec -ti db3v4l_lazydocker lazydocker
    ;;

    pause)
        ${DOCKER_COMPOSE_CMD} ${VERBOSITY} pause ${2}
    ;;

    ps)
        ${DOCKER_COMPOSE_CMD} ${VERBOSITY} ps ${2}
    ;;

    restart)
        ${DOCKER_COMPOSE_CMD} ${VERBOSITY} stop ${2}
        ${DOCKER_COMPOSE_CMD} ${VERBOSITY} up -d ${2}
        RETCODE=$?
        if [ ${RETCODE} -ne 0 ]; then
            exit ${RETCODE}
        fi
        if [ -z "${2}" ]; then
            wait_for_bootstrap all
            exit $?
        else
            wait_for_bootstrap ${2}
            exit $?
        fi
    ;;

    run)
        shift
        if [ ${AS_ROOT} = true ]; then
            docker exec -ti "${WORKER_CONTAINER}" "$@"
        else
            # @todo should we try to start from the 'app' dir ?
            # q: which one is better? test with a command with spaces in options values, and with a composite command such as cd here && do that
            docker exec -ti "${WORKER_CONTAINER}" sudo -iu "${WORKER_USER}" -- "$@"
            #docker exec -ti "${WORKER_CONTAINER}" su - "${WORKER_USER}" -c '"$0" "$@"' -- "$@"
        fi
    ;;

    setup)
        setup_app
    ;;

    services)
        ${DOCKER_COMPOSE_CMD} config --services | sort
    ;;

    shell)
        if [ ${AS_ROOT} = true ]; then
            docker exec -ti "${WORKER_CONTAINER}" bash
        else
            docker exec -ti "${WORKER_CONTAINER}" sudo -iu "${WORKER_USER}"
            #docker exec -ti "${WORKER_CONTAINER}" su - "${WORKER_USER}"
        fi
    ;;

    start)
        ${DOCKER_COMPOSE_CMD} ${VERBOSITY} up -d ${2}
        RETCODE=$?
        if [ ${RETCODE} -ne 0 ]; then
            exit ${RETCODE}
        fi
        if [ -z "${2}" ]; then
            wait_for_bootstrap all
            exit $?
        else
            wait_for_bootstrap ${2}
            exit $?
        fi
    ;;

    stop)
        ${DOCKER_COMPOSE_CMD} ${VERBOSITY} stop ${2}
    ;;

    top)
        ${DOCKER_COMPOSE_CMD} ${VERBOSITY} top ${2}
    ;;

    unpause)
        ${DOCKER_COMPOSE_CMD} ${VERBOSITY} unpause ${2}
    ;;

    # achieved by running `build -u`... could be expanded to also do a git pull?
    #update)
    #;;

    *)
        printf "\n\e[31mERROR: unknown command '${COMMAND}'\e[0m\n\n" >&2
        help
        exit 1
    ;;
esac