wip: Implement a scheduler to run recurring tasks (such as z-push-gabsync) (#123)

* add scheduler container for gabsync
Adds a general scheduler container to trigger tasks within containers.
Also adds gabsync to zpush image
* make scheduler dynamic
execute each cron job once at startup to see if they would succeed
* remove services scripts from core container (now is part of the scheduler)
* add "CRONDELAY" for tasks that should not be executed at startup
* add documentation
* clear out crontab at startup

Makefile

@@ -28,7 +28,7 @@ export
 # convert lowercase componentname to uppercase
 COMPONENT = $(shell echo $(component) | tr a-z A-Z)
 
-build-all: build-base build-core build-kdav build-konnect build-kwmserver build-ldap-demo build-meet build-playground build-ssl build-utils build-web build-webapp build-zpush
+build-all: build-base build-core build-kdav build-konnect build-kwmserver build-ldap-demo build-meet build-playground build-scheduler build-ssl build-utils build-web build-webapp build-zpush
 
 .PHONY: build
 build: component ?= base
@@ -79,6 +79,9 @@ build-playground:
 build-kdav:
 	component=kdav make build
 
+build-scheduler:
+	component=scheduler make build-simple
+
 build-ssl:
 	component=ssl make build-simple
 
@@ -128,6 +131,11 @@ tag-meet:
 	$(shell docker run --rm $(docker_repo)/kopano_meet cat /kopano/buildversion | grep meet | cut -d- -f2 | cut -d+ -f1))
 	component=meet make tag-container
 
+tag-scheduler:
+	$(eval scheduler_version := \
+	$(shell docker run --rm $(docker_repo)/kopano_scheduler env | grep SUPERCRONIC_VERSION | cut -d'=' -f2))
+	component=scheduler make tag-container
+
 tag-utils:
 	$(eval utils_version := \
 	$(shell docker run --rm $(docker_repo)/kopano_utils cat /kopano/buildversion | cut -d- -f2))
@@ -182,6 +190,9 @@ publish-kdav: build-kdav #tag-kdav
 	#component=zpush make publish-container
 	docker push $(docker_repo)/kopano_kdav:latest
 
+publish-scheduler: build-scheduler tag-scheduler
+	component=scheduler make publish-container
+
 publish-ssl: build-ssl
 	docker push $(docker_repo)/kopano_ssl:latest
 

README.md

@@ -89,6 +89,19 @@ You can easily rebuild all images based on the currently available Kopano versio
 
 To be able to easily go back to a previous version you can also "tag" you Docker images by running e.g. `make tag-core`.
 
+### Recurring tasks and maintenance tasks within Kopano
+
+There are certain tasks within Kopano that either need to be executed once (like creating the public store when starting a new environment for the first time) or on a regular base (like syncing the internal user list with and external ldap tree). For convinience this project includes a "schedular" container that will take care of this and that can be dynamically configured through env variables.
+
+The container knows two kinds of cron jobs (syntax is the same as within crontab):
+
+- `CRON_ZPUSHGAB=0 22 * * * docker exec kopano_zpush z-push-gabsync -a sync`
+  - Jobs prefixed with `CRON_` are executed once at container startup (and container startup will fail if one of the jobs fail) and then at the scheduled time.
+- `CRONDELAYED_KBACKUP=30 1 * * * docker run --rm -it zokradonh/kopano_utils kopano-backup -h`
+  - Jobs prefixed with `CRONDELAYED_` are only executed at the scheduled time.
+
+Instead of using the internal scheduler one can also just use an existing scheduler (cron on the docker host?) to execute these tasks.
+
 ### How to use the project with the official and supported Kopano releases?
 
 This project also makes it possible to build Docker images based on the official Kopano releases. For this the following section needs to be modified in `.env`:

core/Dockerfile

@@ -54,7 +54,6 @@ ENV LANG=en_US.UTF-8
 ENV SERVICE_TO_START=server
 
 COPY defaultconfigs/ start-service.sh /kopano/
-COPY services /kopano/services
 
 WORKDIR /kopano/path
 

core/services/kopano-public-store.sh

@@ -1,8 +0,0 @@
-#!/bin/bash
-
-set -eo pipefail
-
-exec dockerize \
-	-wait tcp://localhost:236 \
-	-timeout 360s \
-	kopano-storeadm -h default: -P

core/services/kopano-users.sh

@@ -1,8 +0,0 @@
-#!/bin/bash
-
-set -eo pipefail
-
-dockerize \
-	-wait tcp://localhost:236 \
-	-timeout 360s
-while true; do kopano-cli --sync && sleep 3600; done

core/start-service.sh

@@ -45,8 +45,6 @@ server)
 	else
 		DB_CONN="tcp://$KCCONF_SERVER_MYSQL_HOST:$KCCONF_SERVER_MYSQL_PORT"
 	fi
-	/kopano/services/kopano-public-store.sh &
-	/kopano/services/kopano-users.sh &
 	dockerize \
 		-wait file://"$KCCONF_SERVER_SERVER_SSL_CA_FILE" \
 		-wait file://"$KCCONF_SERVER_SERVER_SSL_KEY_FILE" \

docker-compose.yml

@@ -195,6 +195,7 @@ services:
       - db
       - ldap
       - kopano_ssl
+      - kopano_konnect
     ports:
       - ${KOPANOPORT:-236}:236
       - ${KOPANOSPORT:-237}:237
@@ -309,7 +310,7 @@ services:
       - web-net
 
   kopano_kdav:
-    image: ${docker_repo:?err}/kopano_kdav:${KDAV_VERSION:-latest}
+    image: ${docker_repo:-zokradonh}/kopano_kdav:${KDAV_VERSION:-latest}
     hostname: kopano_kdav
     container_name: ${COMPOSE_PROJECT_NAME}_kdav
     depends_on:
@@ -441,6 +442,7 @@ services:
      - kopanosocket/:/run/kopano
     depends_on:
      - kopano_ssl
+     - web
     environment:
      - FQDN=${FQDN}
      - allow_client_guests=yes
@@ -493,6 +495,27 @@ services:
     networks:
      - web-net
 
+  kopano_scheduler:
+    image: ${docker_repo:-zokradonh}/kopano_scheduler:${SCHEDULER_VERSION:-latest}
+    container_name: kopano_scheduler
+    restart: always
+    networks:
+      - kopano-net
+      - ldap-net
+      - web-net
+    depends_on:
+      - kopano_server
+      - kopano_zpush
+    environment:
+      - TZ=${TZ}
+      - CRON_KOPANOUSERS=10 * * * * docker exec kopano_server kopano-cli --sync
+      - CRON_ZPUSHGAB=0 22 * * * docker exec kopano_zpush z-push-gabsync -a sync
+      - CRONDELAYED_KBACKUP=30 1 * * * docker run --rm -it zokradonh/kopano_utils kopano-backup -h
+    env_file:
+      - ldap.env
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+
 volumes:
   web:
   ldap:

scheduler/Dockerfile

@@ -0,0 +1,17 @@
+FROM docker:18.09.1
+
+RUN apk --no-cache add bash
+
+ENV SUPERCRONIC_VERSION 0.1.8
+RUN wget https://github.com/aptible/supercronic/releases/download/v${SUPERCRONIC_VERSION}/supercronic-linux-amd64 \
+    -O /usr/local/bin/supercronic \
+    && chmod +x /usr/local/bin/supercronic
+
+ENV DOCKERIZE_VERSION v0.6.1
+RUN wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-alpine-linux-amd64-$DOCKERIZE_VERSION.tar.gz \
+    && tar -C /usr/local/bin -xzvf dockerize-alpine-linux-amd64-$DOCKERIZE_VERSION.tar.gz \
+    && rm dockerize-alpine-linux-amd64-$DOCKERIZE_VERSION.tar.gz
+
+COPY start.sh /usr/local/bin/
+
+CMD ["start.sh"]

scheduler/start.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+set -eo pipefail
+
+cronfile=/etc/crontab
+
+# purge existing entries from crontab
+true > "$cronfile"
+
+for cronvar in ${!CRON_*}; do
+	cronvalue=${!cronvar}
+	echo "Adding $cronvalue to crontab"
+	echo "$cronvalue" >> "$cronfile"
+done
+
+for cronvar in ${!CRONDELAYED_*}; do
+	cronvalue=${!cronvar}
+	echo "Adding $cronvalue to crontab"
+	echo "$cronvalue" >> "$cronfile"
+done
+
+# wait for kopano_server statup to run one-off commands
+dockerize \
+	-wait tcp://kopano_server:236 \
+	-timeout 360s
+echo "creating public store"
+docker exec kopano_server kopano-storeadm -h default: -P || true
+
+# run sheduled cron jobs once
+for cronvar in ${!CRON_*}; do
+	cronvalue=${!cronvar}
+	croncommand=$(echo "$cronvalue" | cut -d ' ' -f 6-)
+	echo "Running: $croncommand"
+	$croncommand
+done
+
+exec supercronic /etc/crontab

setup.sh

@@ -65,7 +65,7 @@ docker_tag_search () {
 }
 
 echo "Creating individual env files for containers (if they do not exist already)"
-for dockerenv in ldap password-self-service mail db kopano_ssl kopano_server kopano_webapp kopano_zpush kopano_grapi kopano_kapi kopano_dagent kopano_spooler kopano_gateway kopano_ical kopano_monitor kopano_search kopano_konnect kopano_kwmserver kopano_meet; do
+for dockerenv in ldap password-self-service mail db kopano_ssl kopano_server kopano_webapp kopano_zpush kopano_grapi kopano_kapi kopano_dagent kopano_spooler kopano_gateway kopano_ical kopano_monitor kopano_scheduler kopano_search kopano_konnect kopano_kwmserver kopano_meet; do
 	touch ./"$dockerenv".env
 done
 

zpush/Dockerfile

@@ -45,6 +45,7 @@ RUN \
         crudini \
         z-push-kopano \
         z-push-config-apache \
+        z-push-kopano-gabsync \
         ca-certificates \
         ${ADDITIONAL_KOPANO_PACKAGES} \
     && rm -rf /var/cache/apt /var/lib/apt/lists
@@ -73,6 +74,12 @@ RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
     mkdir -p /var/lib/z-push /var/log/z-push && \
     chown www-data:www-data /var/lib/z-push /var/log/z-push
 
+# Patch Gabsync to make it work
+# See https://jira.z-hub.io/browse/ZP-1463
+# https://forum.kopano.io/topic/1928/8-7-80-missing-php-files-in-php-mapi-deb-package-ubuntu-16-04
+# can be removed once gabsync is fixed - should not hurt
+RUN sed -i -e "s/set_include_path(get_include_path() . PATH_SEPARATOR . BASE_PATH_CLI);/define('PATH_TO_ZPUSH', '..\/..\/backend\/kopano\/');\n    set_include_path(get_include_path() . PATH_SEPARATOR . BASE_PATH_CLI . PATH_SEPARATOR . BASE_PATH_CLI . PATH_TO_ZPUSH);/" /usr/share/z-push/tools/gab-sync/gab-sync.php
+
 VOLUME /var/lib/z-push/
 
 EXPOSE 80/tcp

zpush/start.sh

@@ -30,6 +30,9 @@ echo "Configuring Z-Push for use behind a reverse proxy"
 sed -e "s#define([\"']USE_CUSTOM_REMOTE_IP_HEADER[\"'],\s*false)#define('USE_CUSTOM_REMOTE_IP_HEADER', 'HTTP_X_FORWARDED_FOR')#" \
     -i /etc/z-push/z-push.conf.php
 
+sed -e "s#define([\"']USERNAME[\"'],\s*'')#define('USERNAME', 'SYSTEM')#" \
+    -i /etc/z-push/gabsync.conf.php
+
 echo "Ensure config ownership"
 chown -R www-data:www-data /run/sessions