Make repo script more dynamic #232

* also add the ability to download the tar for supported releases
* use env variables for distribution, channel and branch
* add tests for new archive download for supported releases

Makefile

@@ -397,6 +397,8 @@ test-commander: ## Test scripts with commander
 	COMMANDER_OPTS="--concurrent 1" COMMANDER_FILES_PATH=core/commander/spooler dccommander run kopano_spooler
 	COMMANDER_OPTS="--concurrent 1" COMMANDER_FILES_PATH=webapp dccommander run kopano_webapp
 	COMMANDER_OPTS="--concurrent 1" COMMANDER_FILES_PATH=zpush dccommander run kopano_zpush
+	# this test will fail if you are not on a whitelisted ip
+	commander test tests/commander-supported.yaml || true
 
 test-security: ## Scan containers with Trivy for known security risks (not part of CI workflow for now).
 	cat $(TAG_FILE) | xargs -I % sh -c 'trivy --exit-code 0 --severity HIGH --quiet --auto-refresh %'

base/create-kopano-repo.sh

@@ -1,21 +1,23 @@
 #!/bin/bash
 
 set -eu
-#set -x
+[ "$DEBUG" ] && set -x
 
 function urldecode { : "${*//+/ }"; echo -e "${_//%/\\x}"; }
 
 function version_from_filename {
-	echo "$1" | awk -F"-" '{print $2}'
+	echo $(basename "$1") | awk -F"-" '{print $2}'
 }
 
 function h5ai_query {
 	component=${1:-core}
 	distribution=${2:-Debian_9.0}
+	channel=${3:-community} # could either be community, supported or limited
+	branch=${4:-""} # could either be empty, "master/tarballs/", "pre-final/tarballs/" or "final/tarballs/"
 
-	filename=$(curl -s -S -L -d "action=get&items%5Bhref%5D=%2Fcommunity%2F$component%3A%2F&items%5Bwhat%5D=1" -H \
+	filename=$(curl -s -XPOST "https://download.kopano.io/$channel/?action=get&items\[href\]=/$channel/$component:/$branch&items\[what\]=1" | \
-				"Accept: application/json" https://download.kopano.io/community/ | jq -r '.items[].href' | \
+			jq -r '.items[].href' | \
-				grep "$distribution-all\|$distribution-amd64" | sed "s#/community/$component:/##")
+			grep "$distribution-all\|$distribution-amd64" | sed "s#/$channel/$component:/##")
 
 	if [ -z "${filename// }" ]; then
 		echo "unknown component"
@@ -30,22 +32,25 @@ function dl_and_package_community {
 	# take component as first argument and fallback to core if none given
 	component=${1:-core}
 	distribution=${2:-Debian_9.0}
+	channel=${3:-community}
+	branch=${4:-""}
 
 	# query community server by h5ai API
-	filename=$(h5ai_query "$component" "$distribution")
+	filename=$(h5ai_query "$component" "$distribution" "$channel" "$branch")
+	filename2=$(basename "$filename")
 
 	# download & extract packages
-	curl -s -S -L -o "$filename" https://download.kopano.io/community/"$component":/"${filename}"
+	curl -s -S -L -o "$filename2" https://download.kopano.io/"$channel"/"$component":/"${filename}"
-	tar xf "$filename"
+	tar xf "$filename2"
 
 	# save buildversion
 	currentVersion=$(version_from_filename "$filename")
 	echo "$component-$currentVersion" >> /kopano/buildversion
 
 	# save disk space
-	rm "$filename"
+	rm "$filename2"
 
-	mv "${filename%.tar.gz}" "$component"
+	mv "${filename2%.tar.gz}" "$component"
 
 	# prepare directory to be apt source
 	cd "$component"

core/Dockerfile

@@ -13,6 +13,9 @@ ARG KOPANO_CORE_VERSION=newest
 ENV \
     ADDITIONAL_KOPANO_PACKAGES=$ADDITIONAL_KOPANO_PACKAGES \
     DOWNLOAD_COMMUNITY_PACKAGES=$DOWNLOAD_COMMUNITY_PACKAGES \
+    DOWNLOAD_DISTRIBUTION="Debian_9.0" \
+    DOWNLOAD_CHANNEL="community" \
+    DOWNLOAD_BRANCH="" \
     KOPANO_CORE_REPOSITORY_URL=$KOPANO_CORE_REPOSITORY_URL \
     KOPANO_CORE_VERSION=$KOPANO_CORE_VERSION \
     KOPANO_REPOSITORY_FLAGS=$KOPANO_REPOSITORY_FLAGS \
@@ -39,7 +42,7 @@ RUN \
     # community download and package as apt source repository
     . /kopano/helper/create-kopano-repo.sh && \
     if [ ${DOWNLOAD_COMMUNITY_PACKAGES} -eq 1 ]; then \
-        dl_and_package_community "core"; \
+        dl_and_package_community "core" "$DOWNLOAD_DISTRIBUTION" "$DOWNLOAD_CHANNEL" "$DOWNLOAD_BRANCH"; \
     fi; \
     # apt key has already been installed in base
     echo "deb [${KOPANO_REPOSITORY_FLAGS}] ${KOPANO_CORE_REPOSITORY_URL} ./" > /etc/apt/sources.list.d/kopano.list; \

tests/commander-supported.yaml

@@ -0,0 +1,37 @@
+tests:
+  ./version.sh core Debian_9.0 supported final/tarballs/:
+    exit-code: 0
+    stdout:
+      not-contains:
+        - core-
+        - Debian
+        - amd64
+        - tar.gz
+        - tarballs
+  ./version.sh core Debian_9.0 supported pre-final/tarballs/:
+    exit-code: 0
+    stdout:
+      not-contains:
+        - core-
+        - Debian
+        - amd64
+        - tar.gz
+        - tarballs
+  ./version.sh core Debian_9.0 supported master/tarballs/:
+    exit-code: 0
+    stdout:
+      not-contains:
+        - core-
+        - Debian
+        - amd64
+        - tar.gz
+        - tarballs
+  ./version.sh webapp Debian_9.0 supported final/tarballs/:
+    exit-code: 0
+    stdout:
+      not-contains:
+        - webapp-
+        - Debian
+        - amd64
+        - tar.gz
+        - tarballs

version.sh

@@ -1,4 +1,8 @@
 #!/bin/bash
+
+DEBUG=${DEBUG:-""}
+tmpfile=${tmpfile:-""}
+
 if ! command -v jq > /dev/null; then
 	echo "Please install jq in order to run this build script."
 	exit 1
@@ -15,6 +19,8 @@ source base/create-kopano-repo.sh
 
 component=${1:-core}
 distribution=${2:-Debian_9.0}
+channel=${3:-community}
+branch=${4:-""}
 
 if [ -e ./.env ]; then
 	# this is a kind of ugly hack to be able to source the env file
@@ -62,7 +68,7 @@ kdav)
 esac
 
 # query community server by h5ai API
-filename=$(h5ai_query "$component" "$distribution")
+filename=$(h5ai_query "$component" "$distribution" "$channel" "$branch")
 
 currentVersion=$(version_from_filename "$filename")