add kweb config for z-push (#419)

* add kweb config for z-push
* clean out apache bits from z-push container and use kopano_php as the base
* fix commander tests for current php version
* globally define log location for php-fpm error_log
* move port in outer kweb config
* rewrite other spellings of the autodiscover url
* fix port in startup test
* force plain output to buildkit
* reorder z-push dockerfile

Makefile

@@ -60,7 +60,7 @@ endif
 ifeq (,$(wildcard ./apt_auth.conf))
 	touch apt_auth.conf
 endif
-	DOCKER_BUILDKIT=1 docker build --rm \
+	BUILDKIT_PROGRESS=plain DOCKER_BUILDKIT=1 docker build --rm \
 		--build-arg VCS_REF=$(vcs_ref) \
 		--build-arg docker_repo=${docker_repo} \
 		--build-arg KOPANO_CORE_VERSION=${core_download_version} \
@@ -99,7 +99,7 @@ ifdef TRAVIS
 	@echo "fetching previous build to warm up build cache (only on travis)"
 	docker pull  $(docker_repo)/kopano_$(component):builder || true
 endif
-	DOCKER_BUILDKIT=1 docker build --rm \
+	BUILDKIT_PROGRESS=plain DOCKER_BUILDKIT=1 docker build --rm \
 		--target builder \
 		--build-arg VCS_REF=$(vcf_ref) \
 		--build-arg docker_repo=${docker_repo} \

docker-compose.yml

@@ -155,10 +155,10 @@ services:
       - kopano-net
       - web-net
     tmpfs:
-      - /run/apache2/
+      - /tmp/
       - /run/sessions/
-      - /tmp
-      - /var/log/z-push/
+      - /run/php/
+      - /var/log/
 
   kopano_grapi:
     image: ${docker_repo:-zokradonh}/kopano_core:${CORE_VERSION:-latest}

php/Dockerfile

@@ -51,6 +51,7 @@ RUN \
 
 EXPOSE 9080/tcp
 
+COPY php-fpm.conf /etc/php/7.3/fpm/pool.d/
 COPY start-helper.sh /kopano/start-helper.sh
 COPY kweb.cfg /etc/kweb.cfg
 

php/php-fpm.conf

@@ -0,0 +1,2 @@
+;output errors on stderr
+error_log = /proc/self/fd/2

tests/startup-test/test.sh

@@ -16,7 +16,7 @@ dockerize \
 	-wait tcp://kopano_server:236 \
 	-wait tcp://kopano_server:237 \
 	-wait tcp://kopano_webapp:9080 \
-	-wait tcp://kopano_zpush:80 \
+	-wait tcp://kopano_zpush:9080 \
 	-wait tcp://web:2015 \
 	-timeout 120s
 

web/kweb.cfg

@@ -168,27 +168,27 @@
 	}
 	folderish /webapp
 
-	proxy /Microsoft-Server-ActiveSync {%KWEBD_DNS_ZPUSH%}:80 {
+	proxy /Microsoft-Server-ActiveSync {%KWEBD_DNS_ZPUSH%}:9080 {
 		transparent
 		keepalive 0
 		timeout 3540s
 	}
 
-	proxy /AutoDiscover/AutoDiscover.xml {%KWEBD_DNS_ZPUSH%}:80 {
+	proxy /AutoDiscover/AutoDiscover.xml {%KWEBD_DNS_ZPUSH%}:9080 {
 		transparent
 		keepalive 0
 		fail_timeout 10s
 		try_duration 30s
 	}
 
-	proxy /Autodiscover/Autodiscover.xml {%KWEBD_DNS_ZPUSH%}:80 {
+	proxy /Autodiscover/Autodiscover.xml {%KWEBD_DNS_ZPUSH%}:9080 {
 		transparent
 		keepalive 0
 		fail_timeout 10s
 		try_duration 30s
 	}
 
-	proxy /autodiscover/autodiscover.xml {%KWEBD_DNS_ZPUSH%}:80 {
+	proxy /autodiscover/autodiscover.xml {%KWEBD_DNS_ZPUSH%}:9080 {
 		transparent
 		keepalive 0
 		fail_timeout 10s

webapp/commander.yaml

@@ -1,12 +1,12 @@
 tests:
   start-service script:
-    command: bash -c "shopt -s expand_aliases; alias exec='echo exec'; alias php-fpm7.0='echo php-fpm7.0'; . /kopano/start.sh"
+    command: bash -c "shopt -s expand_aliases; alias exec='echo exec'; alias php-fpm7.3='echo php-fpm7.3'; . /kopano/start.sh"
     exit-code: 0
     stdout:
       not-contains:
         - "Reading package lists..."
   start-service script (installing new package):
-    command: bash -c "shopt -s expand_aliases; alias exec='echo exec'; alias php-fpm7.0='echo php-fpm7.0'; . /kopano/start.sh"
+    command: bash -c "shopt -s expand_aliases; alias exec='echo exec'; alias php-fpm7.3='echo php-fpm7.3'; . /kopano/start.sh"
     exit-code: 0
     stdout:
       contains:
@@ -15,7 +15,7 @@ tests:
       env:
         ADDITIONAL_KOPANO_PACKAGES: "nano"
   start-service script (installing existing package):
-    command: bash -c "shopt -s expand_aliases; alias exec='echo exec'; alias php-fpm7.0='echo php-fpm7.0'; . /kopano/start.sh"
+    command: bash -c "shopt -s expand_aliases; alias exec='echo exec'; alias php-fpm7.3='echo php-fpm7.3'; . /kopano/start.sh"
     exit-code: 0
     stdout:
       contains:
@@ -24,7 +24,7 @@ tests:
       env:
         ADDITIONAL_KOPANO_PACKAGES: "kopano-webapp"
   start-service script (webapp configuration):
-    command: bash -c "shopt -s expand_aliases; alias exec='echo exec'; alias php-fpm7.0='echo php-fpm7.0'; . /kopano/start.sh" && cat /etc/kopano/webapp/config.php
+    command: bash -c "shopt -s expand_aliases; alias exec='echo exec'; alias php-fpm7.3='echo php-fpm7.3'; . /kopano/start.sh" && cat /etc/kopano/webapp/config.php
     exit-code: 0
     stdout:
       contains:
@@ -33,7 +33,7 @@ tests:
       env:
         KCCONF_WEBAPP_OIDC_CLIENT_ID: "webapp"
   start-service script (installing & configuring webapp mdm plugin):
-    command: bash -c "shopt -s expand_aliases; alias exec='echo exec'; alias php-fpm7.0='echo php-fpm7.0'; . /kopano/start.sh" && cat /etc/kopano/webapp/config-mdm.php
+    command: bash -c "shopt -s expand_aliases; alias exec='echo exec'; alias php-fpm7.3='echo php-fpm7.3'; . /kopano/start.sh" && cat /etc/kopano/webapp/config-mdm.php
     exit-code: 0
     stdout:
       contains:

zpush/Dockerfile

@@ -1,6 +1,6 @@
 # syntax = docker/dockerfile:1.0-experimental
 ARG docker_repo=zokradonh
-FROM ${docker_repo}/kopano_base
+FROM ${docker_repo}/kopano_php
 
 ARG ADDITIONAL_KOPANO_PACKAGES=""
 ARG DOWNLOAD_COMMUNITY_PACKAGES=1
@@ -29,13 +29,12 @@ LABEL maintainer=az@zok.xyz \
     org.label-schema.version=$KOPANO_ZPUSH_VERSION \
     org.label-schema.schema-version="1.0"
 
+VOLUME /var/lib/z-push/
+
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 # install Z-Push
-# TODO secret handling could go away when kopano_php is used as a base image
-# hadolint ignore=SC2215
-RUN --mount=type=secret,id=repocred,target=/etc/apt/auth.conf.d/kopano.conf \
-    echo "deb [${KOPANO_REPOSITORY_FLAGS}] ${KOPANO_CORE_REPOSITORY_URL} ./" > /etc/apt/sources.list.d/kopano.list; \
+RUN \
     # prepare z-push installation
     echo "deb ${KOPANO_ZPUSH_REPOSITORY_URL} /" > /etc/apt/sources.list.d/zpush.list && \
     # this is the same key as for the rest of the Kopano stack, making a separate download anyways as this may not be the case in the future 
@@ -46,10 +45,7 @@ RUN --mount=type=secret,id=repocred,target=/etc/apt/auth.conf.d/kopano.conf \
     # TODO remove php-mbstring once https://jira.z-hub.io/browse/ZP-1541 is resolved
     # TODO remove php-xml once https://jira.z-hub.io/projects/ZP/issues/ZP-1558 is resolved
     apt-get update && apt-get install -y --no-install-recommends \
-        apache2 \
-        ca-certificates \
         crudini \
-        libapache2-mod-php7.3 \
         php-mbstring \
         php-xml \
         z-push-autodiscover \
@@ -60,6 +56,12 @@ RUN --mount=type=secret,id=repocred,target=/etc/apt/auth.conf.d/kopano.conf \
         ${ADDITIONAL_KOPANO_PACKAGES} \
     && rm -rf /var/cache/apt /var/lib/apt/lists
 
+# Patch Gabsync to make it work
+# See https://jira.z-hub.io/browse/ZP-1463
+# https://forum.kopano.io/topic/1928/8-7-80-missing-php-files-in-php-mapi-deb-package-ubuntu-16-04
+# can be removed once gabsync is fixed - should not hurt
+RUN sed -i -e "s/set_include_path(get_include_path() . PATH_SEPARATOR . BASE_PATH_CLI);/define('PATH_TO_ZPUSH', '..\/..\/backend\/kopano\/');\n    set_include_path(get_include_path() . PATH_SEPARATOR . BASE_PATH_CLI . PATH_SEPARATOR . BASE_PATH_CLI . PATH_TO_ZPUSH);/" /usr/share/z-push/tools/gab-sync/gab-sync.php
+
 # tweak to make the container read-only
 RUN mkdir -p /tmp/z-push/ && \
     for i in /etc/z-push/*; do \
@@ -67,37 +69,12 @@ RUN mkdir -p /tmp/z-push/ && \
         ln -s /tmp/z-push/"$(basename "$i")" "$i"; \
     done
 
-COPY apache2-kopano.conf /etc/apache2/sites-available/kopano.conf
-
-# configure basics
+# ensure right permissions of folders (should have been taked care of by the packaging, just for good measure)
 RUN \
-    # configure apache
-    rm /etc/apache2/sites-enabled/* && \
-    sed -e 's,^ErrorLog.*,ErrorLog "|/bin/cat",' -i /etc/apache2/apache2.conf && \
-    sed -e "s,MaxSpareServers[^:].*,MaxSpareServers 5," -i /etc/apache2/mods-available/mpm_prefork.conf && \
-    a2disconf other-vhosts-access-log && \
-    a2ensite kopano && \
-    echo "Listen 80" > /etc/apache2/ports.conf && \
-    # configure mod_php
-    a2enmod rewrite && \
-    crudini --set /etc/php/7.3/apache2/php.ini PHP upload_max_filesize 500M && \
-    crudini --set /etc/php/7.3/apache2/php.ini PHP post_max_size 500M && \
-    crudini --set /etc/php/7.3/apache2/php.ini PHP max_input_vars 1800 && \
-    crudini --set /etc/php/7.3/apache2/php.ini Session session.save_path /run/sessions && \
-    # configure z-push
     mkdir -p /var/lib/z-push /var/log/z-push && \
     chown www-data:www-data /var/lib/z-push /var/log/z-push
 
-# Patch Gabsync to make it work
-# See https://jira.z-hub.io/browse/ZP-1463
-# https://forum.kopano.io/topic/1928/8-7-80-missing-php-files-in-php-mapi-deb-package-ubuntu-16-04
-# can be removed once gabsync is fixed - should not hurt
-RUN sed -i -e "s/set_include_path(get_include_path() . PATH_SEPARATOR . BASE_PATH_CLI);/define('PATH_TO_ZPUSH', '..\/..\/backend\/kopano\/');\n    set_include_path(get_include_path() . PATH_SEPARATOR . BASE_PATH_CLI . PATH_SEPARATOR . BASE_PATH_CLI . PATH_TO_ZPUSH);/" /usr/share/z-push/tools/gab-sync/gab-sync.php
-
-VOLUME /var/lib/z-push/
-
-EXPOSE 80/tcp
-
+COPY kweb.cfg /etc/kweb.cfg
 COPY start.sh /kopano/start.sh
 
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]

zpush/commander.yaml

@@ -1,12 +1,12 @@
 tests:
   start-service script:
-    command: bash -c "shopt -s expand_aliases; alias exec='echo exec'; alias php-fpm7.0='echo php-fpm7.0'; . /kopano/start.sh"
+    command: bash -c "shopt -s expand_aliases; alias exec='echo exec'; alias php-fpm7.3='echo php-fpm7.3'; . /kopano/start.sh"
     exit-code: 0
     stdout:
       not-contains:
         - "Reading package lists..."
   start-service script (installing new package):
-    command: bash -c "shopt -s expand_aliases; alias exec='echo exec'; alias php-fpm7.0='echo php-fpm7.0'; . /kopano/start.sh"
+    command: bash -c "shopt -s expand_aliases; alias exec='echo exec'; alias php-fpm7.3='echo php-fpm7.3'; . /kopano/start.sh"
     exit-code: 0
     stdout:
       contains:
@@ -15,7 +15,7 @@ tests:
       env:
         ADDITIONAL_KOPANO_PACKAGES: "nano"
   start-service script (installing existing package):
-    command: bash -c "shopt -s expand_aliases; alias exec='echo exec'; alias php-fpm7.0='echo php-fpm7.0'; . /kopano/start.sh"
+    command: bash -c "shopt -s expand_aliases; alias exec='echo exec'; alias php-fpm7.3='echo php-fpm7.3'; . /kopano/start.sh"
     exit-code: 0
     stdout:
       contains:
@@ -24,19 +24,19 @@ tests:
       env:
         ADDITIONAL_KOPANO_PACKAGES: "z-push-kopano"
   start-service script (z-push configuration):
-    command: bash -c "shopt -s expand_aliases; alias exec='echo exec'; alias php-fpm7.0='echo php-fpm7.0'; . /kopano/start.sh" && cat /etc/z-push/z-push.conf.php
+    command: bash -c "shopt -s expand_aliases; alias exec='echo exec'; alias php-fpm7.3='echo php-fpm7.3'; . /kopano/start.sh" && cat /etc/z-push/z-push.conf.php
     exit-code: 0
     stdout:
       contains:
         - "define('USE_CUSTOM_REMOTE_IP_HEADER', 'HTTP_X_FORWARDED_FOR');"
   start-service script (configuring gabsync):
-    command: bash -c "shopt -s expand_aliases; alias exec='echo exec'; alias php-fpm7.0='echo php-fpm7.0'; . /kopano/start.sh" && cat /etc/z-push/gabsync.conf.php
+    command: bash -c "shopt -s expand_aliases; alias exec='echo exec'; alias php-fpm7.3='echo php-fpm7.3'; . /kopano/start.sh" && cat /etc/z-push/gabsync.conf.php
     exit-code: 0
     stdout:
       contains:
         - "define('USERNAME', 'SYSTEM');"
   start-service script (no additional folders):
-    command: bash -c "shopt -s expand_aliases; alias exec='echo exec'; alias php-fpm7.0='echo php-fpm7.0'; . /kopano/start.sh" && cat /etc/z-push/z-push.conf.php
+    command: bash -c "shopt -s expand_aliases; alias exec='echo exec'; alias php-fpm7.3='echo php-fpm7.3'; . /kopano/start.sh" && cat /etc/z-push/z-push.conf.php
     exit-code: 0
     stdout:
       contains:
@@ -45,7 +45,7 @@ tests:
         - "\t$additionalFolders = array("
         - "\t);"
   start-service script (empty additional folders):
-    command: bash -c "shopt -s expand_aliases; alias exec='echo exec'; alias php-fpm7.0='echo php-fpm7.0'; . /kopano/start.sh" && cat /etc/z-push/z-push.conf.php
+    command: bash -c "shopt -s expand_aliases; alias exec='echo exec'; alias php-fpm7.3='echo php-fpm7.3'; . /kopano/start.sh" && cat /etc/z-push/z-push.conf.php
     exit-code: 0
     stdout:
       contains:
@@ -57,7 +57,7 @@ tests:
       env:
         ZPUSH_ADDITIONAL_FOLDERS: "[]"
   start-service script (set additional folders):
-    command: bash -c "shopt -s expand_aliases; alias exec='echo exec'; alias php-fpm7.0='echo php-fpm7.0'; . /kopano/start.sh" && cat /etc/z-push/z-push.conf.php
+    command: bash -c "shopt -s expand_aliases; alias exec='echo exec'; alias php-fpm7.3='echo php-fpm7.3'; . /kopano/start.sh" && cat /etc/z-push/z-push.conf.php
     exit-code: 0
     stdout:
       contains:

zpush/kweb.cfg

@@ -0,0 +1,27 @@
+:9080 {
+	log stdout
+	errors stderr
+
+	# healthcheck
+	status 200 /status
+
+	rewrite /Microsoft-Server-ActiveSync /Microsoft-Server-ActiveSync/index.php
+
+	fastcgi2 /Microsoft-Server-ActiveSync /run/php/php7.3-fpm.sock php {
+		without /Microsoft-Server-ActiveSync/
+		root /usr/share/z-push/
+		read_timeout 3605s
+	}
+
+	# Case insensitive path rewrite.
+	rewrite / {
+		regexp (?i)^/Autodiscover/Autodiscover.xml
+		to /Autodiscover/autodiscover.php
+	}
+
+	fastcgi2 /Autodiscover/ /run/php/php7.3-fpm.sock php {
+		without /Autodiscover/
+		root /usr/share/z-push/autodiscover
+	}
+
+}

zpush/start.sh

@@ -148,11 +148,9 @@ tail --pid=$$ -F --lines=0 -q /var/log/z-push/z-push-error.log &
 tail --pid=$$ -F --lines=0 -q /var/log/z-push/autodiscover.log &
 tail --pid=$$ -F --lines=0 -q /var/log/z-push/autodiscover-error.log &
 
-echo "Starting Apache"
-rm -f /run/apache2/apache2.pid
 set +u
-# shellcheck disable=SC1091
-source /etc/apache2/envvars
 # cleaning up env variables
 unset "${!KCCONF_@}"
-exec /usr/sbin/apache2 -DFOREGROUND
+echo "Starting php-fpm"
+php-fpm7.3 -F &
+exec /usr/libexec/kopano/kwebd caddy -conf /etc/kweb.cfg