Run all containers read-only (#314)

* switch containers in compose file to read-only
related to https://github.com/zokradonh/kopano-docker/issues/310
* make scheduler container read-only
* make meet container read-only
* fix konnect for read-only
* make core mostly read-only
* add custom location for ldap.cfg to default config
* sort config option for readability
* update makefile pull workaround
* make kdav read-only
* remove locale generation code from core startup
* fix commander tests
* fix store language tests
* add test for a dutch mailbox
* make it possible to run webapp read-only
* add locale hint to the compose file
* finish read-only mode for z-push

core/commander/server/commander.yaml

@@ -24,7 +24,7 @@ tests:
       env:
         ADDITIONAL_KOPANO_PACKAGES: "kopano-server"
   generate configuration for kopano-server:
-    command: /usr/bin/python3 /kopano/server.py && cat /etc/kopano/server.cfg
+    command: bash -c "shopt -s expand_aliases; alias exec='echo'; . /kopano/start-service.sh" && cat /tmp/kopano/server.cfg
     exit-code: 0
     stdout:
       contains:
@@ -33,7 +33,7 @@ tests:
       not-contains:
         - #server_listen_tls = *:237
   generate ldap configuration openLDAP (default):
-    command: /usr/bin/python3 /kopano/server.py && cat /etc/kopano/ldap.cfg
+    command: bash -c "shopt -s expand_aliases; alias exec='echo'; . /kopano/start-service.sh" && cat /tmp/kopano/ldap.cfg
     exit-code: 0
     stdout:
       contains:
@@ -42,7 +42,7 @@ tests:
       not-contains:
         - "#!include /usr/share/kopano/ldap.openldap.cfg"
   generate ldap configuration for ADS:
-    command: /usr/bin/python3 /kopano/server.py && cat /etc/kopano/ldap.cfg
+    command: bash -c "shopt -s expand_aliases; alias exec='echo'; . /kopano/start-service.sh" && cat /tmp/kopano/ldap.cfg
     exit-code: 0
     stdout:
       contains:
@@ -55,16 +55,6 @@ tests:
         KCCOMMENT_LDAP_1: "!include /usr/share/kopano/ldap.openldap.cfg"
         KCUNCOMMENT_LDAP_1: "!include /usr/share/kopano/ldap.active-directory.cfg"
   start-service script default locale:
-    command: bash -c "shopt -s expand_aliases; alias exec='echo'; . /kopano/start-service.sh && locale -a"
-    exit-code: 0
-    stdout:
-      contains:
-        - C
-        - C.UTF-8
-        - de_DE.utf8
-        - en_US.utf8
-        - POSIX
-  start-service script dutch locale:
     command: bash -c "shopt -s expand_aliases; alias exec='echo'; . /kopano/start-service.sh && locale -a"
     exit-code: 0
     stdout:
@@ -75,38 +65,6 @@ tests:
         - en_US.utf8
         - nl_NL.utf8
         - POSIX
-    config:
-      env:
-        KCCONF_ADMIN_DEFAULT_STORE_LOCALE: "nl_NL.UTF-8"
-  start-service script dutch locale (short):
-    command: bash -c "shopt -s expand_aliases; alias exec='echo'; . /kopano/start-service.sh && locale -a"
-    exit-code: 0
-    stdout:
-      contains:
-        - C
-        - C.UTF-8
-        - de_DE.utf8
-        - en_US.utf8
-        - nl_NL.utf8
-        - POSIX
-    config:
-      env:
-        KCCONF_ADMIN_DEFAULT_STORE_LOCALE: "nl_NL"
-  start-service script unknown locale:
-    command: bash -c "shopt -s expand_aliases; alias exec='echo'; . /kopano/start-service.sh && locale -a"
-    exit-code: 0
-    stdout:
-      contains:
-        - C
-        - C.UTF-8
-        - de_DE.utf8
-        - en_US.utf8
-        - POSIX
-      not-contains:
-        - abc
-    config:
-      env:
-        KCCONF_ADMIN_DEFAULT_STORE_LOCALE: "abc"
   start-service script no dockerize:
     command: bash -c "shopt -s expand_aliases; alias exec='echo'; . /kopano/start-service.sh"
     exit-code: 0

core/defaultconfigs/dagent.py

@@ -3,7 +3,7 @@ import kcconf
 
 # Component specific configurations
 kcconf.configkopano({
-    r"/etc/kopano/dagent.cfg":
+    r"/tmp/kopano/dagent.cfg":
     {
         'lmtp_listen': "0.0.0.0:2003",
         'log_file': "-",
@@ -13,4 +13,4 @@ kcconf.configkopano({
 })
 
 # Override configs from environment variables
-kcconf.configkopano(kcconf.parseenvironmentvariables(r"/etc/kopano/"))
+kcconf.configkopano(kcconf.parseenvironmentvariables(r"/tmp/kopano/"))

core/defaultconfigs/gateway.py

@@ -3,7 +3,7 @@ import kcconf
 
 # Component specific configurations
 kcconf.configkopano({
-    r"/etc/kopano/gateway.cfg":
+    r"/tmp/kopano/gateway.cfg":
     {
         'imap_listen': "0.0.0.0:143",
         'log_file': "-",
@@ -13,4 +13,4 @@ kcconf.configkopano({
 })
 
 # Override configs from environment variables
-kcconf.configkopano(kcconf.parseenvironmentvariables(r"/etc/kopano/"))
+kcconf.configkopano(kcconf.parseenvironmentvariables(r"/tmp/kopano/"))

core/defaultconfigs/grapi.py

@@ -2,4 +2,4 @@ import os
 import kcconf
 
 # Override configs from environment variables
-kcconf.configkopano(kcconf.parseenvironmentvariables(r"/etc/kopano/"))
+kcconf.configkopano(kcconf.parseenvironmentvariables(r"/tmp/kopano/"))

core/defaultconfigs/ical.py

@@ -3,7 +3,7 @@ import kcconf
 
 # Component specific configurations
 kcconf.configkopano({
-    r"/etc/kopano/ical.cfg":
+    r"/tmp/kopano/ical.cfg":
     {
         'ical_listen': "0.0.0.0:8080",
         'log_file': "-",
@@ -12,4 +12,4 @@ kcconf.configkopano({
 })
 
 # Override configs from environment variables
-kcconf.configkopano(kcconf.parseenvironmentvariables(r"/etc/kopano/"))
+kcconf.configkopano(kcconf.parseenvironmentvariables(r"/tmp/kopano/"))

core/defaultconfigs/kapi.py

@@ -3,7 +3,7 @@ import kcconf
 
 # Component specific configurations
 kcconf.configkopano({
-    r"/etc/kopano/kapid.cfg":
+    r"/tmp/kopano/kapid.cfg":
     {
         'log_level': "info",
         'listen': "0.0.0.0:8039",
@@ -15,4 +15,4 @@ kcconf.configkopano({
 })
 
 # Override configs from environment variables
-kcconf.configkopano(kcconf.parseenvironmentvariables(r"/etc/kopano/"))
+kcconf.configkopano(kcconf.parseenvironmentvariables(r"/tmp/kopano/"))

core/defaultconfigs/monitor.py

@@ -3,7 +3,7 @@ import kcconf
 
 # Component specific configurations
 kcconf.configkopano({
-    r"/etc/kopano/monitor.cfg":
+    r"/tmp/kopano/monitor.cfg":
     {
         'log_file': "-",
         'log_level': "4"
@@ -11,4 +11,4 @@ kcconf.configkopano({
 })
 
 # Override configs from environment variables
-kcconf.configkopano(kcconf.parseenvironmentvariables(r"/etc/kopano/"))
+kcconf.configkopano(kcconf.parseenvironmentvariables(r"/tmp/kopano/"))

core/defaultconfigs/search.py

@@ -3,7 +3,7 @@ import kcconf
 
 # Component specific configurations
 kcconf.configkopano({
-    r"/etc/kopano/search.cfg":
+    r"/tmp/kopano/search.cfg":
     {
         'log_file': "-",
         'log_level': "4",
@@ -12,4 +12,4 @@ kcconf.configkopano({
 })
 
 # Override configs from environment variables
-kcconf.configkopano(kcconf.parseenvironmentvariables(r"/etc/kopano/"))
+kcconf.configkopano(kcconf.parseenvironmentvariables(r"/tmp/kopano/"))

core/defaultconfigs/server.py

@@ -3,19 +3,20 @@ import kcconf
 
 # Component specific configurations
 kcconf.configkopano({
-    r"/etc/kopano/server.cfg":
+    r"/tmp/kopano/server.cfg":
     {
+        'attachment_path': "/kopano/data/attachments/",
+        'kcoidc_initialize_timeout': "360",
         'log_file': "-",
         'log_level': "3",
-        'attachment_path': "/kopano/data/attachments/",
-        'user_plugin': "ldap",
-        'server_listen': "0.0.0.0:236",
         'server_listen_tls': "0.0.0.0:237",
-        'sync_gab_realtime': "no",
+        'server_listen': "0.0.0.0:236",
         'softdelete_lifetime': "0",
-        'kcoidc_initialize_timeout': "360"
+        'sync_gab_realtime': "no",
+        'user_plugin_config': "/tmp/kopano/ldap.cfg",
+        'user_plugin': "ldap"
     }
 })
 
 # Override configs from environment variables
-kcconf.configkopano(kcconf.parseenvironmentvariables(r"/etc/kopano/"))
+kcconf.configkopano(kcconf.parseenvironmentvariables(r"/tmp/kopano/"))

core/defaultconfigs/spooler.py

@@ -3,7 +3,7 @@ import kcconf
 
 # Component specific configurations
 kcconf.configkopano({
-    r"/etc/kopano/spooler.cfg":
+    r"/tmp/kopano/spooler.cfg":
     {
         'log_file': "-",
         'log_level': "4",
@@ -12,4 +12,4 @@ kcconf.configkopano({
 })
 
 # Override configs from environment variables
-kcconf.configkopano(kcconf.parseenvironmentvariables(r"/etc/kopano/"))
+kcconf.configkopano(kcconf.parseenvironmentvariables(r"/tmp/kopano/"))

core/start-service.sh

@@ -15,11 +15,17 @@ KCCONF_SEARCH_SERVER_SOCKET=${KCCONF_SEARCH_SERVER_SOCKET:-"file:///var/run/kopa
 KCCONF_SPOOLER_SERVER_SOCKET=${KCCONF_SPOOLER_SERVER_SOCKET:-"file:///var/run/kopano/server.sock"}
 KOPANO_CON=${KOPANO_CON:-"file:///var/run/kopano/server.sock"}
 
+# copy configuration files to /tmp/kopano to prevent modification of mounted config files
+mkdir -p /tmp/kopano
+cp /etc/kopano/*.cfg /tmp/kopano
+
 if [ ! -e /kopano/"$SERVICE_TO_START".py ]; then
 	echo "Invalid service specified: $SERVICE_TO_START" | ts
 	exit 1
 fi
 
+# Hint: this is not compatible with a read-only container.
+# The general recommendation is to already build a container that has all required packages installed.
 ADDITIONAL_KOPANO_PACKAGES=$(echo "$ADDITIONAL_KOPANO_PACKAGES" | tr -d '"')
 [ -n "${ADDITIONAL_KOPANO_PACKAGES// }" ] && apt update
 [ -n "${ADDITIONAL_KOPANO_PACKAGES// }" ] && for installpkg in $ADDITIONAL_KOPANO_PACKAGES; do
@@ -31,8 +37,9 @@ ADDITIONAL_KOPANO_PACKAGES=$(echo "$ADDITIONAL_KOPANO_PACKAGES" | tr -d '"')
 	fi
 done
 
-mkdir -p /kopano/data/attachments /kopano/data/kapi-kvs /tmp/"$SERVICE_TO_START" /var/run/kopano /var/lib/kopano-grapi
+mkdir -p /tmp/"$SERVICE_TO_START" /var/run/kopano
 
+# TODO is this still required now that we won't modify configuration mounted to /etc/kopano?
 if [ "${DISABLE_CONFIG_CHANGES}" == false ]; then
 	echo "Configure core service '$SERVICE_TO_START'" | ts
 	/usr/bin/python3 /kopano/"$SERVICE_TO_START".py
@@ -41,10 +48,6 @@ fi
 # ensure removed pid-file on unclean shutdowns and mounted volumes
 rm -f /var/run/kopano/"$SERVICE_TO_START".pid
 
-echo "Set ownership" | ts
-chown kopano:kopano /kopano/data/ /kopano/data/attachments
-chown kapi:kopano /var/lib/kopano-grapi
-
 coreversion=$(dpkg-query --showformat='${Version}' --show kopano-server)
 echo "Using Kopano Groupware Core: $coreversion"
 
@@ -89,30 +92,10 @@ fi
 # start regular service
 case "$SERVICE_TO_START" in
 server)
-	# TODO this could check if the desired locale already exists before calling sed
-	KCCONF_ADMIN_DEFAULT_STORE_LOCALE=${KCCONF_ADMIN_DEFAULT_STORE_LOCALE:-"en_US.UTF-8"}
-	# get locales from env
-	# shellcheck disable=SC1004
-	sed --regexp-extended --expression='
-
-		1  {
-			i\
-# This file lists locales that you wish to have built. You can find a list\
-# of valid supported locales at /usr/share/i18n/SUPPORTED, and you can add\
-# user defined locales to /usr/local/share/i18n/SUPPORTED. If you change\
-# this file, you need to rerun locale-gen.\
-\
-
-
-			}
-
-		/^('"$KCCONF_ADMIN_DEFAULT_STORE_LOCALE"')(_[[:upper:]]+)?(\.UTF-8)?(@[^[:space:]]+)?[[:space:]]+UTF-8$/!   s/^/# /
-	' /usr/share/i18n/SUPPORTED >  /etc/locale.gen
-	# make sure that en_US and de_DE are always there
-	sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen
-	sed -i -e 's/# de_DE.UTF-8 UTF-8/de_DE.UTF-8 UTF-8/' /etc/locale.gen
-	dpkg-reconfigure --frontend=noninteractive locales
-
+	echo "Set ownership" | ts
+	mkdir -p /kopano/data/attachments
+	chown kopano:kopano /kopano/data/ /kopano/data/attachments
+	
 	if [[ "$DISABLE_CHECKS" == false ]]; then
 		# determine db connection mode (unix vs. network socket)
 		if [ -n "$KCCONF_SERVER_MYSQL_SOCKET" ]; then
@@ -129,11 +112,11 @@ server)
 	fi
 	# pre populate database
 	if dpkg --compare-versions "$coreversion" "gt" "8.7.84"; then
-		kopano-dbadm populate
+		kopano-dbadm -c /tmp/kopano/server.cfg populate
 	fi
 	# cleaning up env variables
 	unset "${!KCCONF_@}"
-	exec /usr/sbin/kopano-server -F
+	exec /usr/sbin/kopano-server --config /tmp/kopano/server.cfg -F
 	;;
 dagent)
 	dockerize \
@@ -141,7 +124,7 @@ dagent)
 		-timeout 360s
 	# cleaning up env variables
 	unset "${!KCCONF_@}"
-	exec /usr/sbin/kopano-dagent -l
+	exec /usr/sbin/kopano-dagent --config /tmp/kopano/dagent.cfg -l
 	;;
 gateway)
 	dockerize \
@@ -149,7 +132,7 @@ gateway)
 		-timeout 360s
 	# cleaning up env variables
 	unset "${!KCCONF_@}"
-	exec /usr/sbin/kopano-gateway -F
+	exec /usr/sbin/kopano-gateway --config /tmp/kopano/gateway.cfg -F
 	;;
 ical)
 	dockerize \
@@ -157,13 +140,15 @@ ical)
 		-timeout 360s
 	# cleaning up env variables
 	unset "${!KCCONF_@}"
-	exec /usr/sbin/kopano-ical -F
+	exec /usr/sbin/kopano-ical --config /tmp/kopano/ical.cfg -F
 	;;
 grapi)
 	LC_CTYPE=en_US.UTF-8
 	export socket_path=/var/run/kopano/grapi
-	mkdir -p "$socket_path"
+	export pid_file="$socket_path/grapi.pid"
+	mkdir -p "$socket_path" /var/lib/kopano-grapi
 	chown -R kapi:kopano "$socket_path"
+	chown kapi:kopano /var/lib/kopano-grapi
 	# TODO there could be a case where multiple backends are desired
 	case $GRAPI_BACKEND in
 	ldap)
@@ -176,7 +161,7 @@ grapi)
 		fi
 		;;
 	esac
-	sed s/\ *=\ */=/g /etc/kopano/grapi.cfg > /tmp/grapi-env
+	sed s/\ *=\ */=/g /tmp/kopano/grapi.cfg > /tmp/grapi-env
 	# shellcheck disable=SC2046
 	export $(grep -v '^#' /tmp/grapi-env | xargs -d '\n')
 	# cleaning up env variables
@@ -191,6 +176,7 @@ grapi)
 	fi
 	;;
 kapi)
+	mkdir -p /kopano/data/kapi-kvs
 	if [ "$KCCONF_KAPID_INSECURE" = "yes" ]; then
 		dockerize \
 		-skip-tls-verify \
@@ -206,7 +192,7 @@ kapi)
 	kapiversion=$(dpkg-query --showformat='${Version}' --show kopano-kapid)
 	echo "Using Kopano Kapi: $kapiversion"
 	LC_CTYPE=en_US.UTF-8
-	sed s/\ *=\ */=/g /etc/kopano/kapid.cfg > /tmp/kapid-env
+	sed s/\ *=\ */=/g /tmp/kopano/kapid.cfg > /tmp/kapid-env
 	# shellcheck disable=SC2046
 	export $(grep -v '^#' /tmp/kapid-env | xargs -d '\n')
 	kopano-kapid setup
@@ -220,7 +206,7 @@ monitor)
 		-timeout 360s
 	# cleaning up env variables
 	unset "${!KCCONF_@}"
-	exec /usr/sbin/kopano-monitor -F
+	exec /usr/sbin/kopano-monitor --config /tmp/kopano/monitor.cfg -F
 	;;
 search)
 	dockerize \
@@ -233,9 +219,9 @@ search)
 	# with commit 702bb3fccb3 search does not need -F any longer
 	searchversion=$(dpkg-query --showformat='${Version}' --show kopano-search)
 	if dpkg --compare-versions "$searchversion" "gt" "8.7.82.165"; then
-		exec /usr/sbin/kopano-search
+		exec /usr/sbin/kopano-search --config /tmp/kopano/search.cfg
 	else
-		exec /usr/bin/python3 /usr/sbin/kopano-search -F
+		exec /usr/bin/python3 /usr/sbin/kopano-search --config /tmp/kopano/search.cfg -F
 	fi
 	;;
 spooler)
@@ -245,7 +231,7 @@ spooler)
 		-timeout 1080s
 	# cleaning up env variables
 	unset "${!KCCONF_@}"
-	exec /usr/sbin/kopano-spooler -F
+	exec /usr/sbin/kopano-spooler --config /tmp/kopano/spooler.cfg -F
 	;;
 *)
 	echo "Failed to start: Unknown service name: '$SERVICE_TO_START'" | ts