From caac8a5c03f2b7ad1acd44de69af41a5f1de5918 Mon Sep 17 00:00:00 2001 From: Felix Bartels Date: Sun, 24 Mar 2019 17:33:41 +0100 Subject: [PATCH] wip: Implement a scheduler to run recurring tasks (such as z-push-gabsync) (#123) * add scheduler container for gabsync Adds a general scheduler container to trigger tasks within containers. Also adds gabsync to zpush image * make scheduler dynamic execute each cron job once at startup to see if they would succeed * remove services scripts from core container (now is part of the scheduler) * add "CRONDELAY" for tasks that should not be executed at startup * add documentation * clear out crontab at startup --- Makefile | 13 +++++++++- README.md | 13 ++++++++++ core/Dockerfile | 1 - core/services/kopano-public-store.sh | 8 ------ core/services/kopano-users.sh | 8 ------ core/start-service.sh | 2 -- docker-compose.yml | 25 ++++++++++++++++++- scheduler/Dockerfile | 17 +++++++++++++ scheduler/start.sh | 37 ++++++++++++++++++++++++++++ setup.sh | 2 +- zpush/Dockerfile | 7 ++++++ zpush/start.sh | 3 +++ 12 files changed, 114 insertions(+), 22 deletions(-) delete mode 100755 core/services/kopano-public-store.sh delete mode 100755 core/services/kopano-users.sh create mode 100644 scheduler/Dockerfile create mode 100755 scheduler/start.sh diff --git a/Makefile b/Makefile index f491106..4ff4745 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ export # convert lowercase componentname to uppercase COMPONENT = $(shell echo $(component) | tr a-z A-Z) -build-all: build-base build-core build-kdav build-konnect build-kwmserver build-ldap-demo build-meet build-playground build-ssl build-utils build-web build-webapp build-zpush +build-all: build-base build-core build-kdav build-konnect build-kwmserver build-ldap-demo build-meet build-playground build-scheduler build-ssl build-utils build-web build-webapp build-zpush .PHONY: build build: component ?= base @@ -79,6 +79,9 @@ build-playground: build-kdav: component=kdav make build +build-scheduler: + component=scheduler make build-simple + build-ssl: component=ssl make build-simple @@ -128,6 +131,11 @@ tag-meet: $(shell docker run --rm $(docker_repo)/kopano_meet cat /kopano/buildversion | grep meet | cut -d- -f2 | cut -d+ -f1)) component=meet make tag-container +tag-scheduler: + $(eval scheduler_version := \ + $(shell docker run --rm $(docker_repo)/kopano_scheduler env | grep SUPERCRONIC_VERSION | cut -d'=' -f2)) + component=scheduler make tag-container + tag-utils: $(eval utils_version := \ $(shell docker run --rm $(docker_repo)/kopano_utils cat /kopano/buildversion | cut -d- -f2)) @@ -182,6 +190,9 @@ publish-kdav: build-kdav #tag-kdav #component=zpush make publish-container docker push $(docker_repo)/kopano_kdav:latest +publish-scheduler: build-scheduler tag-scheduler + component=scheduler make publish-container + publish-ssl: build-ssl docker push $(docker_repo)/kopano_ssl:latest diff --git a/README.md b/README.md index 6281caa..ebf12c5 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,19 @@ You can easily rebuild all images based on the currently available Kopano versio To be able to easily go back to a previous version you can also "tag" you Docker images by running e.g. `make tag-core`. +### Recurring tasks and maintenance tasks within Kopano + +There are certain tasks within Kopano that either need to be executed once (like creating the public store when starting a new environment for the first time) or on a regular base (like syncing the internal user list with and external ldap tree). For convinience this project includes a "schedular" container that will take care of this and that can be dynamically configured through env variables. + +The container knows two kinds of cron jobs (syntax is the same as within crontab): + +- `CRON_ZPUSHGAB=0 22 * * * docker exec kopano_zpush z-push-gabsync -a sync` + - Jobs prefixed with `CRON_` are executed once at container startup (and container startup will fail if one of the jobs fail) and then at the scheduled time. +- `CRONDELAYED_KBACKUP=30 1 * * * docker run --rm -it zokradonh/kopano_utils kopano-backup -h` + - Jobs prefixed with `CRONDELAYED_` are only executed at the scheduled time. + +Instead of using the internal scheduler one can also just use an existing scheduler (cron on the docker host?) to execute these tasks. + ### How to use the project with the official and supported Kopano releases? This project also makes it possible to build Docker images based on the official Kopano releases. For this the following section needs to be modified in `.env`: diff --git a/core/Dockerfile b/core/Dockerfile index b778983..d407fea 100644 --- a/core/Dockerfile +++ b/core/Dockerfile @@ -54,7 +54,6 @@ ENV LANG=en_US.UTF-8 ENV SERVICE_TO_START=server COPY defaultconfigs/ start-service.sh /kopano/ -COPY services /kopano/services WORKDIR /kopano/path diff --git a/core/services/kopano-public-store.sh b/core/services/kopano-public-store.sh deleted file mode 100755 index 6e687dd..0000000 --- a/core/services/kopano-public-store.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -set -eo pipefail - -exec dockerize \ - -wait tcp://localhost:236 \ - -timeout 360s \ - kopano-storeadm -h default: -P diff --git a/core/services/kopano-users.sh b/core/services/kopano-users.sh deleted file mode 100755 index 24dc5be..0000000 --- a/core/services/kopano-users.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -set -eo pipefail - -dockerize \ - -wait tcp://localhost:236 \ - -timeout 360s -while true; do kopano-cli --sync && sleep 3600; done diff --git a/core/start-service.sh b/core/start-service.sh index 3234491..8ac0f47 100755 --- a/core/start-service.sh +++ b/core/start-service.sh @@ -45,8 +45,6 @@ server) else DB_CONN="tcp://$KCCONF_SERVER_MYSQL_HOST:$KCCONF_SERVER_MYSQL_PORT" fi - /kopano/services/kopano-public-store.sh & - /kopano/services/kopano-users.sh & dockerize \ -wait file://"$KCCONF_SERVER_SERVER_SSL_CA_FILE" \ -wait file://"$KCCONF_SERVER_SERVER_SSL_KEY_FILE" \ diff --git a/docker-compose.yml b/docker-compose.yml index 3680790..275b5cc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -195,6 +195,7 @@ services: - db - ldap - kopano_ssl + - kopano_konnect ports: - ${KOPANOPORT:-236}:236 - ${KOPANOSPORT:-237}:237 @@ -309,7 +310,7 @@ services: - web-net kopano_kdav: - image: ${docker_repo:?err}/kopano_kdav:${KDAV_VERSION:-latest} + image: ${docker_repo:-zokradonh}/kopano_kdav:${KDAV_VERSION:-latest} hostname: kopano_kdav container_name: ${COMPOSE_PROJECT_NAME}_kdav depends_on: @@ -441,6 +442,7 @@ services: - kopanosocket/:/run/kopano depends_on: - kopano_ssl + - web environment: - FQDN=${FQDN} - allow_client_guests=yes @@ -493,6 +495,27 @@ services: networks: - web-net + kopano_scheduler: + image: ${docker_repo:-zokradonh}/kopano_scheduler:${SCHEDULER_VERSION:-latest} + container_name: kopano_scheduler + restart: always + networks: + - kopano-net + - ldap-net + - web-net + depends_on: + - kopano_server + - kopano_zpush + environment: + - TZ=${TZ} + - CRON_KOPANOUSERS=10 * * * * docker exec kopano_server kopano-cli --sync + - CRON_ZPUSHGAB=0 22 * * * docker exec kopano_zpush z-push-gabsync -a sync + - CRONDELAYED_KBACKUP=30 1 * * * docker run --rm -it zokradonh/kopano_utils kopano-backup -h + env_file: + - ldap.env + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + volumes: web: ldap: diff --git a/scheduler/Dockerfile b/scheduler/Dockerfile new file mode 100644 index 0000000..5b7f448 --- /dev/null +++ b/scheduler/Dockerfile @@ -0,0 +1,17 @@ +FROM docker:18.09.1 + +RUN apk --no-cache add bash + +ENV SUPERCRONIC_VERSION 0.1.8 +RUN wget https://github.com/aptible/supercronic/releases/download/v${SUPERCRONIC_VERSION}/supercronic-linux-amd64 \ + -O /usr/local/bin/supercronic \ + && chmod +x /usr/local/bin/supercronic + +ENV DOCKERIZE_VERSION v0.6.1 +RUN wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-alpine-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ + && tar -C /usr/local/bin -xzvf dockerize-alpine-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ + && rm dockerize-alpine-linux-amd64-$DOCKERIZE_VERSION.tar.gz + +COPY start.sh /usr/local/bin/ + +CMD ["start.sh"] diff --git a/scheduler/start.sh b/scheduler/start.sh new file mode 100755 index 0000000..5a423df --- /dev/null +++ b/scheduler/start.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +set -eo pipefail + +cronfile=/etc/crontab + +# purge existing entries from crontab +true > "$cronfile" + +for cronvar in ${!CRON_*}; do + cronvalue=${!cronvar} + echo "Adding $cronvalue to crontab" + echo "$cronvalue" >> "$cronfile" +done + +for cronvar in ${!CRONDELAYED_*}; do + cronvalue=${!cronvar} + echo "Adding $cronvalue to crontab" + echo "$cronvalue" >> "$cronfile" +done + +# wait for kopano_server statup to run one-off commands +dockerize \ + -wait tcp://kopano_server:236 \ + -timeout 360s +echo "creating public store" +docker exec kopano_server kopano-storeadm -h default: -P || true + +# run sheduled cron jobs once +for cronvar in ${!CRON_*}; do + cronvalue=${!cronvar} + croncommand=$(echo "$cronvalue" | cut -d ' ' -f 6-) + echo "Running: $croncommand" + $croncommand +done + +exec supercronic /etc/crontab diff --git a/setup.sh b/setup.sh index 3fd619a..7200ed3 100755 --- a/setup.sh +++ b/setup.sh @@ -65,7 +65,7 @@ docker_tag_search () { } echo "Creating individual env files for containers (if they do not exist already)" -for dockerenv in ldap password-self-service mail db kopano_ssl kopano_server kopano_webapp kopano_zpush kopano_grapi kopano_kapi kopano_dagent kopano_spooler kopano_gateway kopano_ical kopano_monitor kopano_search kopano_konnect kopano_kwmserver kopano_meet; do +for dockerenv in ldap password-self-service mail db kopano_ssl kopano_server kopano_webapp kopano_zpush kopano_grapi kopano_kapi kopano_dagent kopano_spooler kopano_gateway kopano_ical kopano_monitor kopano_scheduler kopano_search kopano_konnect kopano_kwmserver kopano_meet; do touch ./"$dockerenv".env done diff --git a/zpush/Dockerfile b/zpush/Dockerfile index 2e0d817..8001090 100644 --- a/zpush/Dockerfile +++ b/zpush/Dockerfile @@ -45,6 +45,7 @@ RUN \ crudini \ z-push-kopano \ z-push-config-apache \ + z-push-kopano-gabsync \ ca-certificates \ ${ADDITIONAL_KOPANO_PACKAGES} \ && rm -rf /var/cache/apt /var/lib/apt/lists @@ -73,6 +74,12 @@ RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ mkdir -p /var/lib/z-push /var/log/z-push && \ chown www-data:www-data /var/lib/z-push /var/log/z-push +# Patch Gabsync to make it work +# See https://jira.z-hub.io/browse/ZP-1463 +# https://forum.kopano.io/topic/1928/8-7-80-missing-php-files-in-php-mapi-deb-package-ubuntu-16-04 +# can be removed once gabsync is fixed - should not hurt +RUN sed -i -e "s/set_include_path(get_include_path() . PATH_SEPARATOR . BASE_PATH_CLI);/define('PATH_TO_ZPUSH', '..\/..\/backend\/kopano\/');\n set_include_path(get_include_path() . PATH_SEPARATOR . BASE_PATH_CLI . PATH_SEPARATOR . BASE_PATH_CLI . PATH_TO_ZPUSH);/" /usr/share/z-push/tools/gab-sync/gab-sync.php + VOLUME /var/lib/z-push/ EXPOSE 80/tcp diff --git a/zpush/start.sh b/zpush/start.sh index 8481527..a085b31 100755 --- a/zpush/start.sh +++ b/zpush/start.sh @@ -30,6 +30,9 @@ echo "Configuring Z-Push for use behind a reverse proxy" sed -e "s#define([\"']USE_CUSTOM_REMOTE_IP_HEADER[\"'],\s*false)#define('USE_CUSTOM_REMOTE_IP_HEADER', 'HTTP_X_FORWARDED_FOR')#" \ -i /etc/z-push/z-push.conf.php +sed -e "s#define([\"']USERNAME[\"'],\s*'')#define('USERNAME', 'SYSTEM')#" \ + -i /etc/z-push/gabsync.conf.php + echo "Ensure config ownership" chown -R www-data:www-data /run/sessions