Smaller optimisations (#137)

* add docker pull to warm up build cache on travis
* explicitly build multi stage build stages
* make create repo script dynamic in the choice of the distribution

Makefile

@@ -33,6 +33,11 @@ build-all: build-base build-core build-kdav build-konnect build-kwmserver build-
 .PHONY: build
 build: component ?= base
 build:
+ifdef TRAVIS
+	@echo "fetching previous build to warm up build cache (only on travis)"
+	docker pull  $(docker_repo)/kopano_$(component) || true
+	docker pull  $(docker_repo)/kopano_$(component):builder || true
+endif
 	docker build \
 		--build-arg docker_repo=${docker_repo} \
 		--build-arg KOPANO_CORE_VERSION=${core_download_version} \
@@ -48,12 +53,49 @@ build:
 		--build-arg DOWNLOAD_COMMUNITY_PACKAGES=$(DOWNLOAD_COMMUNITY_PACKAGES) \
 		--build-arg ADDITIONAL_KOPANO_PACKAGES="$(ADDITIONAL_KOPANO_PACKAGES)" \
 		--build-arg ADDITIONAL_KOPANO_WEBAPP_PLUGINS="$(ADDITIONAL_KOPANO_WEBAPP_PLUGINS)" \
+		--cache-from $(docker_repo)/kopano_$(component) \
+		--cache-from $(docker_repo)/kopano_$(component):builder \
 		-t $(docker_repo)/kopano_$(component) $(component)/
 
 .PHONY: build-simple
 build-simple: component ?= ssl
 build-simple:
-	docker build -t $(docker_repo)/kopano_$(component) $(component)/
+ifdef TRAVIS
+	@echo "fetching previous build to warm up build cache (only on travis)"
+	docker pull  $(docker_repo)/kopano_$(component) || true
+	docker pull  $(docker_repo)/kopano_$(component):builder || true
+endif
+	docker build \
+	--cache-from $(docker_repo)/kopano_$(component) \
+	--cache-from $(docker_repo)/kopano_$(component):builder \
+	--build-arg docker_repo=$(docker_repo) \
+	-t $(docker_repo)/kopano_$(component) $(component)/
+
+.PHONY: build-builder
+build-builder: component ?= kdav
+build-builder:
+ifdef TRAVIS
+	@echo "fetching previous build to warm up build cache (only on travis)"
+	docker pull  $(docker_repo)/kopano_$(component):builder || true
+endif
+	docker build --target builder \
+	--build-arg docker_repo=${docker_repo} \
+	--build-arg KOPANO_CORE_VERSION=${core_download_version} \
+	--build-arg KOPANO_$(COMPONENT)_VERSION=${$(component)_download_version} \
+	--build-arg KOPANO_CORE_REPOSITORY_URL=$(KOPANO_CORE_REPOSITORY_URL) \
+	--build-arg KOPANO_MEET_REPOSITORY_URL=$(KOPANO_MEET_REPOSITORY_URL) \
+	--build-arg KOPANO_WEBAPP_REPOSITORY_URL=$(KOPANO_WEBAPP_REPOSITORY_URL) \
+	--build-arg KOPANO_WEBAPP_FILES_REPOSITORY_URL=$(KOPANO_WEBAPP_FILES_REPOSITORY_URL) \
+	--build-arg KOPANO_WEBAPP_MDM_REPOSITORY_URL=$(KOPANO_WEBAPP_MDM_REPOSITORY_URL) \
+	--build-arg KOPANO_WEBAPP_SMIME_REPOSITORY_URL=$(KOPANO_WEBAPP_SMIME_REPOSITORY_URL) \
+	--build-arg KOPANO_ZPUSH_REPOSITORY_URL=$(KOPANO_ZPUSH_REPOSITORY_URL) \
+	--build-arg RELEASE_KEY_DOWNLOAD=$(RELEASE_KEY_DOWNLOAD) \
+	--build-arg DOWNLOAD_COMMUNITY_PACKAGES=$(DOWNLOAD_COMMUNITY_PACKAGES) \
+	--build-arg ADDITIONAL_KOPANO_PACKAGES="$(ADDITIONAL_KOPANO_PACKAGES)" \
+	--build-arg ADDITIONAL_KOPANO_WEBAPP_PLUGINS="$(ADDITIONAL_KOPANO_WEBAPP_PLUGINS)" \
+	--cache-from $(docker_repo)/kopano_$(component) \
+	--cache-from $(docker_repo)/kopano_$(component):builder \
+	-t $(docker_repo)/kopano_$(component):builder $(component)/
 
 build-base:
 	component=base make build
@@ -74,9 +116,11 @@ build-meet: build-base
 	component=meet make build
 
 build-playground:
+	component=playground make build-builder
 	component=playground make build-simple
 
 build-kdav:
+	component=kdav make build-builder
 	component=kdav make build
 
 build-scheduler:
@@ -185,10 +229,12 @@ publish-meet: build-meet tag-meet
 
 publish-playground: build-playground
 	docker push $(docker_repo)/kopano_playground:latest
+	docker push $(docker_repo)/kopano_playground:builder
 
 publish-kdav: build-kdav #tag-kdav
 	#component=zpush make publish-container
 	docker push $(docker_repo)/kopano_kdav:latest
+	docker push $(docker_repo)/kopano_kdav:builder
 
 publish-scheduler: build-scheduler tag-scheduler
 	component=scheduler make publish-container

base/create-kopano-repo.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 set -eu
+#set -x
 
 function urldecode { : "${*//+/ }"; echo -e "${_//%/\\x}"; }
 
@@ -9,10 +10,12 @@ function version_from_filename {
 }
 
 function h5ai_query {
-    component="$1"
+    component=${1:-core}
+    distribution=${2:-Debian_9.0}
+
     filename=$(curl -s -S -L -d "action=get&items%5Bhref%5D=%2Fcommunity%2F$component%3A%2F&items%5Bwhat%5D=1" -H \
                 "Accept: application/json" https://download.kopano.io/community/ | jq '.items[].href' | \
-                grep 'Debian_9.0-all\|Debian_9.0-amd64' | sed 's#"##g' | sed "s#/community/$component:/##")
+                grep "$distribution-all\|$distribution-amd64" | sed 's#"##g' | sed "s#/community/$component:/##")
 
     if [ -z "${filename// }" ]; then
         echo "unknown component"
@@ -26,9 +29,10 @@ function h5ai_query {
 function dl_and_package_community {
     # take component as first argument and fallback to core if none given
     component=${1:-core}
+    distribution=${2:-Debian_9.0}
 
     # query community server by h5ai API
-    filename=$(h5ai_query "$component")
+    filename=$(h5ai_query "$component" "$distribution")
 
     # download & extract packages
     curl -s -S -L -o "$filename" https://download.kopano.io/community/"$component":/"${filename}"

kdav/Dockerfile

@@ -1,5 +1,5 @@
 ARG docker_repo=zokradonh
-FROM composer:1.8.4 as composer
+FROM composer:1.8.4 as builder
 
 RUN git clone --depth 1 https://stash.kopano.io/scm/kc/kdav.git /usr/share/kdav
 WORKDIR /usr/share/kdav
@@ -68,7 +68,7 @@ RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
     mkdir -p /var/log/kdav && \
     chown www-data:www-data /var/log/kdav
 
-COPY --from=composer /usr/share/kdav /usr/share/kdav
+COPY --from=builder /usr/share/kdav /usr/share/kdav
 
 EXPOSE 80/tcp
 

version.sh

@@ -7,6 +7,7 @@ fi
 source base/create-kopano-repo.sh
 
 component=${1:-core}
+distribution=${2:-Debian_9.0}
 
 if [ -e ./.env ]; then
 	# this is a kind of ugly hack to be able to source the env file
@@ -51,7 +52,7 @@ kdav)
 esac
 
 # query community server by h5ai API
-filename=$(h5ai_query "$component")
+filename=$(h5ai_query "$component" "$distribution")
 
 currentVersion=$(version_from_filename "$filename")