Store language (#251)

* clean out unneeded locales
* add test to verify that language is picked up in admin.cfg
* add basic question for language and use variable in compose
* use scriptlet to generate locales before start of kopano-server
* add script to check folders of a given mailbox (for language checks)
* add tests for locale generation

relates to #248

Makefile

@@ -397,6 +397,7 @@ test-commander: ## Test scripts with commander
 	COMMANDER_OPTS="--concurrent 1" COMMANDER_FILES_PATH=core/commander/spooler dccommander run kopano_spooler
 	COMMANDER_OPTS="--concurrent 1" COMMANDER_FILES_PATH=webapp dccommander run kopano_webapp
 	COMMANDER_OPTS="--concurrent 1" COMMANDER_FILES_PATH=zpush dccommander run kopano_zpush
+	COMMANDER_OPTS="--concurrent 1" COMMANDER_FILES_PATH=scheduler dccommander run kopano_scheduler
 	# this test will fail if you are not on a whitelisted ip
 	commander test tests/commander-supported.yaml || true
 

core/Dockerfile

@@ -61,13 +61,12 @@ RUN \
     cp /usr/share/doc/kopano/example-config/*.cfg.gz /etc/kopano/ && \
     gzip -d -f /etc/kopano/*.gz
 
-ENV KOPANO_LOCALE="de_DE.UTF-8"
-ENV KOPANO_USERSCRIPT_LOCALE="de_DE.UTF-8"
 ENV LANG=en_US.UTF-8
 
 ENV SERVICE_TO_START=server
 
 COPY defaultconfigs/ start-service.sh healthcheck.sh /kopano/
+COPY bin/ /usr/local/bin/
 COPY goss/ /kopano/goss
 
 WORKDIR /kopano/path

core/bin/show-folders.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
+
+
+try:
+    import kopano
+except ImportError:
+    import zarafa as kopano
+from MAPI.Util import *
+import binascii
+
+def opt_args():
+    parser = kopano.parser('skpcf')
+    parser.add_option("--user", dest="user", action="store", help="Username")
+    parser.add_option("--public", dest="public", action="store_true", help="Show public folders")
+    parser.add_option("--delete", dest="delete", action="store", help="Delete folder based on entryid")
+    parser.add_option("--extend", dest="extend", action="store_true", help="show more values")
+
+    return parser.parse_args()
+def printprop(typename, item):
+    if typename == 'PT_MV_BINARY':
+        listItem = []
+        for i in item:
+            listItem.append(str(binascii.hexlify(i)).upper())
+        return listItem
+    if typename == 'PT_OBJECT':
+        return None
+    if typename == 'PT_BINARY':
+        return str(binascii.hexlify(item)).upper()
+    if typename == 'PT_UNICODE':
+        try:
+            return item.encode('utf-8').decode()
+        except:
+            return item
+    else:
+        return item
+
+def printmapiprops(folder):
+    props = []
+    for prop in folder.props():
+        if hex(prop.proptag) == "0x10130102L":
+            props.append([prop.id_, prop.idname, hex(prop.proptag), prop.typename, printprop(prop.typename, prop.value), prop.value])
+        else:
+            props.append([prop.id_, prop.idname, hex(prop.proptag), prop.typename, printprop(prop.typename, prop.value)])
+
+    return props
+def main():
+    options, args = opt_args()
+
+    if not  options.user and not options.public:
+        print('Please use\n' \
+            '%s --user <username>  or\n' \
+            '%s --public' % (sys.argv[0], sys.argv[0]))
+        sys.exit(1)
+    if options.user:
+        user = kopano.server(options).user(options.user)
+        store = user.store
+        name = user.name
+    if options.public:
+        name = 'Public'
+        store = kopano.server(options).public_store
+    if not options.delete:
+        print('Store:', name.encode('utf-8').decode())
+        print('{:50} {:50} {:50}'.format('Folder name', 'Parent folder', 'Entryid'))
+
+        for folder in store.root.folders():
+            print('{:50} {:50} {:50}'.format(folder.name.encode('utf8').decode(), folder.parent.name.encode('utf8').decode(), folder.entryid))
+            if options.extend:
+                props = printmapiprops(folder)
+                f = open('%s-%s.prop' % (folder.name, folder.entryid), 'w')
+                for prop in props:
+                    f.write('{0:5}  {1:37}  {2:8}  {3:10}  {4:1}\n'.format(prop[0], prop[1], prop[2], prop[3], prop[4]))
+                f.close()
+
+    else:
+        print('Not in yet')
+
+
+
+
+if __name__ == "__main__":
+    main()

core/commander/server/commander.yaml

@@ -54,6 +54,59 @@ tests:
       env:
         KCCOMMENT_LDAP_1: "!include /usr/share/kopano/ldap.openldap.cfg"
         KCUNCOMMENT_LDAP_1: "!include /usr/share/kopano/ldap.active-directory.cfg"
+  start-service script default locale:
+    command: bash -c "shopt -s expand_aliases; alias exec='echo'; . /kopano/start-service.sh && locale -a"
+    exit-code: 0
+    stdout:
+      contains:
+        - C
+        - C.UTF-8
+        - de_DE.utf8
+        - en_US.utf8
+        - POSIX
+  start-service script dutch locale:
+    command: bash -c "shopt -s expand_aliases; alias exec='echo'; . /kopano/start-service.sh && locale -a"
+    exit-code: 0
+    stdout:
+      contains:
+        - C
+        - C.UTF-8
+        - de_DE.utf8
+        - en_US.utf8
+        - nl_NL.utf8
+        - POSIX
+    config:
+      env:
+        KCCONF_ADMIN_DEFAULT_STORE_LOCALE: "nl_NL.UTF-8"
+  start-service script dutch locale (short):
+    command: bash -c "shopt -s expand_aliases; alias exec='echo'; . /kopano/start-service.sh && locale -a"
+    exit-code: 0
+    stdout:
+      contains:
+        - C
+        - C.UTF-8
+        - de_DE.utf8
+        - en_US.utf8
+        - nl_NL.utf8
+        - POSIX
+    config:
+      env:
+        KCCONF_ADMIN_DEFAULT_STORE_LOCALE: "nl_NL"
+  start-service script unknown locale:
+    command: bash -c "shopt -s expand_aliases; alias exec='echo'; . /kopano/start-service.sh && locale -a"
+    exit-code: 0
+    stdout:
+      contains:
+        - C
+        - C.UTF-8
+        - de_DE.utf8
+        - en_US.utf8
+        - POSIX
+      not-contains:
+        - abc
+    config:
+      env:
+        KCCONF_ADMIN_DEFAULT_STORE_LOCALE: "abc"
 config:
   env:
     DEBUG: ${DEBUG}

core/start-service.sh

@@ -42,6 +42,30 @@ fi
 # start regular service
 case "$SERVICE_TO_START" in
 server)
+	# TODO this could check if the desired locale already exists before calling sed
+	KCCONF_ADMIN_DEFAULT_STORE_LOCALE=${KCCONF_ADMIN_DEFAULT_STORE_LOCALE:-"en_US.UTF-8"}
+	# get locales from env
+	# shellcheck disable=SC1004
+	sed --regexp-extended --expression='
+
+		1  {
+			i\
+# This file lists locales that you wish to have built. You can find a list\
+# of valid supported locales at /usr/share/i18n/SUPPORTED, and you can add\
+# user defined locales to /usr/local/share/i18n/SUPPORTED. If you change\
+# this file, you need to rerun locale-gen.\
+\
+
+
+			}
+
+		/^('"$KCCONF_ADMIN_DEFAULT_STORE_LOCALE"')(_[[:upper:]]+)?(\.UTF-8)?(@[^[:space:]]+)?[[:space:]]+UTF-8$/!   s/^/# /
+	' /usr/share/i18n/SUPPORTED >  /etc/locale.gen
+	# make sure that en_US and de_DE are always there
+	sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen
+	sed -i -e 's/# de_DE.UTF-8 UTF-8/de_DE.UTF-8 UTF-8/' /etc/locale.gen
+	dpkg-reconfigure --frontend=noninteractive locales
+
 	# determine db connection mode (unix vs. network socket)
 	if [ -n "$KCCONF_SERVER_MYSQL_SOCKET" ]; then
 		DB_CONN="file://$KCCONF_SERVER_MYSQL_SOCKET"

docker-compose.yml

@@ -164,6 +164,7 @@ services:
       - KCCONF_SERVER_ENABLE_SSO=yes
       - KCCONF_SERVER_KCOIDC_ISSUER_IDENTIFIER=https://${FQDN}
       - KCCONF_SERVER_KCOIDC_INSECURE_SKIP_VERIFY=${INSECURE}
+      - KCCONF_ADMIN_DEFAULT_STORE_LOCALE=${MAILBOXLANG:-en_US.UTF-8}
     env_file:
       - kopano_server.env
     networks:

meet/Dockerfile

@@ -52,10 +52,6 @@ RUN \
     cp /usr/share/doc/kopano-meet-webapp/config.json.in /usr/share/kopano-kweb/www/config/kopano/meet.json && \
     rm -rf /var/cache/apt /var/lib/apt/lists
 
-ENV KOPANO_LOCALE="de_DE.UTF-8"
-ENV KOPANO_USERSCRIPT_LOCALE="de_DE.UTF-8"
-ENV LANG=en_US.UTF-8
-
 COPY defaultconfigs/ start-service.sh /kopano/
 COPY goss.yaml /goss/
 CMD [ "/kopano/start-service.sh" ]

scheduler/commander.yaml

@@ -0,0 +1,26 @@
+tests:
+  check mailbox language: # this needs a running kopano-server can can therefore not be executed in that container directly
+    command: "docker exec kopano_server kopano-storeadm -C -n user10; docker exec kopano_server show-folders.py --user user10"
+    exit-code: 0
+    stdout:
+      contains:
+        - "Inbox"
+        - "Drafts"
+  check mailbox language german: # this needs a running kopano-server can can therefore not be executed in that container directly
+    command: "docker exec kopano-storeadm -D user 11; docker exec kopano_server kopano-storeadm -C -l de_DE -n user11; docker exec kopano_server show-folders.py --user user11"
+    exit-code: 0
+    stdout:
+      not-contains:
+        - "Inbox"
+        - "Drafts"
+  test renaming of folders:
+    command: docker exec kopano_server env KCCONF_ADMIN_DEFAULT_STORE_LOCALE=de_DE.UTF-8 /usr/bin/python3 /kopano/server.py && docker exec kopano_server kopano-storeadm -Y -n user12
+    exit-code: 0
+    stderr:
+      contains:
+        - The -l option was not specified; "de_DE.UTF-8" will be used as language
+        - Posteingang
+        - Entwürfe
+config:
+  env:
+    PATH: ${PATH}

setup.sh

@@ -29,7 +29,7 @@ plugin_menu() {
 	[[ "$msg" ]] && echo "$msg"; :
 }
 
-docker_tag_search () {
+docker_tag_search() {
 	image="$1"
 	results=$(reg tags "$image" 2> /dev/null)
 	echo "$results" | xargs -n1 | sort --version-sort -ru | xargs
@@ -153,6 +153,11 @@ if [ ! -e ./.env ]; then
 		LDAP_BIND_PW=$(random_string)
 	fi
 
+	# TODO get locale from system
+	value_default="en_US.UTF-8"
+	read -r -p "Language to be used for new mailoboxes (needs to be available as a locale in the container) [$value_default]: " new_value
+	MAILBOXLANG=${new_value:-$value_default}
+
 	if [ -f /etc/timezone ]; then
 		value_default=$(cat /etc/timezone)
 	elif [ -f /etc/localtime ]; then
@@ -294,6 +299,7 @@ MYSQL_DATABASE=$MYSQL_DATABASE
 KCCONF_SERVER_SERVER_NAME=Kopano
 
 POSTMASTER_ADDRESS=$POSTMASTER_ADDRESS
+MAILBOXLANG=$MAILBOXLANG
 TZ=$TZ
 
 # Defines how Kopano can be accessed from the outside world