From e8f86f188b95d4d0327758ecdb74ffaeff2b208c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20S=C3=A1nchez?= Date: Tue, 26 Jul 2016 18:37:38 +0200 Subject: [PATCH] Upgrade zookeeper and traefik packages (#185) * Upgrade zookeeper to version 3.4.8 * Updated traefik version to v1.0.0. Added new features: ssl, multidomain, import key/cert from metadata, run as traefik user. * Upgrade kafka to version 0.10.0 * Updated traefik version to v1.0.1. ssl key and crt set as required. * Updated traefik version to v1.0.1. ssl working modes available. --- templates/kafka/1/README.md | 28 +++++++++ templates/kafka/1/docker-compose.yml | 43 +++++++++++++ templates/kafka/1/rancher-compose.yml | 70 +++++++++++++++++++++ templates/kafka/config.yml | 4 +- templates/traefik/0/docker-compose.yml | 5 +- templates/traefik/0/rancher-compose.yml | 15 ++++- templates/traefik/1/README.md | 47 ++++++++++++++ templates/traefik/1/docker-compose.yml | 32 ++++++++++ templates/traefik/1/rancher-compose.yml | 77 +++++++++++++++++++++++ templates/traefik/config.yml | 2 +- templates/zookeeper/1/README.md | 27 ++++++++ templates/zookeeper/1/docker-compose.yml | 40 ++++++++++++ templates/zookeeper/1/rancher-compose.yml | 67 ++++++++++++++++++++ templates/zookeeper/config.yml | 4 +- 14 files changed, 454 insertions(+), 7 deletions(-) create mode 100644 templates/kafka/1/README.md create mode 100644 templates/kafka/1/docker-compose.yml create mode 100644 templates/kafka/1/rancher-compose.yml create mode 100644 templates/traefik/1/README.md create mode 100644 templates/traefik/1/docker-compose.yml create mode 100644 templates/traefik/1/rancher-compose.yml create mode 100644 templates/zookeeper/1/README.md create mode 100644 templates/zookeeper/1/docker-compose.yml create mode 100644 templates/zookeeper/1/rancher-compose.yml diff --git a/templates/kafka/1/README.md b/templates/kafka/1/README.md new file mode 100644 index 0000000..8477dbe --- /dev/null +++ b/templates/kafka/1/README.md @@ -0,0 +1,28 @@ +# Apache Kafka (Experimental) + +### Info: + + This template creates, scale in and scale out a multinodes kafka broker cluster on top of Rancher. The configuration is generated with confd from Rancher metadata. + Cluster size are variable after deployment, and get reconfigured after refresh interval. + + +### Usage: + + Select Apache Kafka from catalog. + + Enter the number of nodes, mem and refresh interval for the kafka cluster. + + Change the following kafka default parameters, if you need: + +- KAFKA_LOG_DIRS="/opt/kafka/logs" +- KAFKA_LOG_RETENTION_HOURS="168" +- KAFKA_NUM_PARTITIONS="1" +- ADVERTISE_PUB_IP= < true | false > + + Select zookeeper stack/service to connect to. + + Click deploy. + + Kafka can now be accessed over the Rancher network. + + Note: When you scale the cluster, zero downtime is not guaranteed..yet.. diff --git a/templates/kafka/1/docker-compose.yml b/templates/kafka/1/docker-compose.yml new file mode 100644 index 0000000..a775193 --- /dev/null +++ b/templates/kafka/1/docker-compose.yml @@ -0,0 +1,43 @@ +broker: + tty: true + image: rawmind/alpine-kafka:0.10.0.0-2 + volumes_from: + - broker-volume + - broker-conf + environment: + - JVMFLAGS=-Xmx${kafka_mem}m -Xms${kafka_mem}m + - CONFD_INTERVAL=${kafka_interval} + - ZK_SERVICE=${zk_link} + - KAFKA_LOG_DIRS=${kafka_log_dir} + - KAFKA_LOG_RETENTION_HOURS=${kafka_log_retention} + - KAFKA_NUM_PARTITIONS=${kafka_num_partitions} + - ADVERTISE_PUB_IP=${kafka_pub_ip} + external_links: + - ${zk_link}:zk + labels: + io.rancher.scheduler.affinity:container_label_soft_ne: io.rancher.stack_service.name=$${stack_name}/$${service_name} + io.rancher.container.hostname_override: container_name + io.rancher.sidekicks: broker-volume, broker-conf +broker-conf: + net: none + labels: + io.rancher.scheduler.affinity:container_label_soft_ne: io.rancher.stack_service.name=$${stack_name}/$${service_name} + io.rancher.container.hostname_override: container_name + io.rancher.container.start_once: true + image: rawmind/rancher-kafka:0.10.0.0 +broker-volume: + net: none + labels: + io.rancher.scheduler.affinity:container_label_soft_ne: io.rancher.stack_service.name=$${stack_name}/$${service_name} + io.rancher.container.hostname_override: container_name + io.rancher.container.start_once: true + environment: + - SERVICE_USER=kafka + - SERVICE_UID=10003 + - SERVICE_GROUP=kafka + - SERVICE_GID=10003 + - SERVICE_VOLUME=${kafka_log_dir} + volumes: + - ${kafka_log_dir} + volume_driver: local + image: rawmind/alpine-volume:0.0.1-1 diff --git a/templates/kafka/1/rancher-compose.yml b/templates/kafka/1/rancher-compose.yml new file mode 100644 index 0000000..f97ded0 --- /dev/null +++ b/templates/kafka/1/rancher-compose.yml @@ -0,0 +1,70 @@ +.catalog: + name: Kafka + version: 0.10.0.0-rancher1 + description: | + (Experimental) Apache Kafka cluster. + minimum_rancher_version: v0.59.0 + maintainer: "Raul Sanchez " + uuid: kafka-0 + questions: + - variable: "kafka_scale" + description: "Number of brokers nodes. Note: Recommended an odd number" + label: "Broker Nodes:" + required: true + default: 3 + type: "int" + - variable: "kafka_mem" + description: "Amount of memory to config brokers." + label: "Broker Memory (mb):" + required: true + default: 1024 + type: "string" + - variable: "kafka_log_dir" + description: "Broker volume to log data" + label: "Broker log volume:" + required: true + default: "/opt/kafka/logs" + type: "string" + - variable: "kafka_log_retention" + description: "Broker log retention in hours" + label: "Broker log retention (h):" + required: true + default: "168" + type: "int" + - variable: "kafka_num_partitions" + description: "Number of broker partitions" + label: "Broker partitions:" + required: true + default: "1" + type: "int" + - variable: "kafka_interval" + description: "Interval to poll/apply configuration changes. 0 to disable" + label: "Broker Interval (s):" + required: true + default: 60 + type: "int" + - variable: "kafka_pub_ip" + description: "Advertise host ip in zookeeper" + label: "Advertise host ip:" + required: false + default: false + type: "boolean" + - variable: "zk_link" + description: | + Choose the Zookeeper service to use. + It's really recommended a dedicated zookeeper service just for Kafka. + label: "Zookeeper stack/service" + default: "kafka-zk/zk" + required: true + type: "service" +broker: + scale: ${kafka_scale} + retain_ip: true + health_check: + port: 9092 + interval: 5000 + unhealthy_threshold: 3 + request_line: '' + healthy_threshold: 2 + response_timeout: 5000 + diff --git a/templates/kafka/config.yml b/templates/kafka/config.yml index c93f189..706e7c0 100644 --- a/templates/kafka/config.yml +++ b/templates/kafka/config.yml @@ -1,8 +1,8 @@ name: Apache Kafka description: | (Experimental) Kafka cluster -version: 0.9.0-rancher1 +version: 0.10.0.0-rancher1 category: Clustering maintainer: "Raul Sanchez " -minimum_rancher_version: v0.56.0 +minimum_rancher_version: v0.59.0 license: diff --git a/templates/traefik/0/docker-compose.yml b/templates/traefik/0/docker-compose.yml index 6cb1550..1a3154c 100644 --- a/templates/traefik/0/docker-compose.yml +++ b/templates/traefik/0/docker-compose.yml @@ -1,7 +1,9 @@ + traefik: ports: - ${admin_port}:${admin_port}/tcp - ${http_port}:${http_port}/tcp + - ${https_port}:${https_port}/tcp log_driver: '' labels: io.rancher.scheduler.global: 'true' @@ -11,9 +13,10 @@ traefik: io.rancher.container.hostname_override: container_name tty: true log_opt: {} - image: rawmind/alpine-traefik:1.0.0-beta.555-6 + image: rawmind/alpine-traefik:1.0.0-beta.771 environment: - TRAEFIK_HTTP_PORT=${http_port} + - TRAEFIK_HTTPS_PORT=${https_port} - TRAEFIK_ADMIN_PORT=${admin_port} - CONF_INTERVAL=${refresh_interval} volumes_from: diff --git a/templates/traefik/0/rancher-compose.yml b/templates/traefik/0/rancher-compose.yml index 4d23846..d134d5c 100644 --- a/templates/traefik/0/rancher-compose.yml +++ b/templates/traefik/0/rancher-compose.yml @@ -13,20 +13,33 @@ required: true default: 8080 type: "int" + - variable: "https_port" + description: "Traefik https public port to listen." + label: "Https port:" + required: true + default: 8443 + type: "int" - variable: "admin_port" description: "Traefik admin public port to listen." label: "Admin port:" required: true default: 8000 - type: "string" + type: "int" - variable: "refresh_interval" description: "Interval to poll/apply configuration changes." label: "Refresh Interval (s):" required: true default: 60 type: "int" + - variable: "ssl_cert" + description: "SSL certificate" + label: "SSL certificate:" + required: false + default: "" + type: "certificate" traefik: retain_ip: true + default_cert: ${ssl_cert} health_check: port: ${admin_port} interval: 5000 diff --git a/templates/traefik/1/README.md b/templates/traefik/1/README.md new file mode 100644 index 0000000..34fa7e7 --- /dev/null +++ b/templates/traefik/1/README.md @@ -0,0 +1,47 @@ +# Traefik active load balancer (Experimental) + +### Info: + + This template deploys traefik active load balancers on top of Rancher. The configuration is generated and updated with confd from Rancher metadata. + It would be deployed in hosts with label traefik_lb=true. + +### Config: + +- host_label = "traefik_lb=true" # Host label where to run traefik service. +- http_port = 8080 # Port exposed to get access to the published services. +- https_port = 8443 # Port exposed to get secured access to the published services. +- admin_port = 8000 # Port exposed to get admin access to the traefik service. +- https_enable = + - false: Enable http enpoints and disable https ones. + - true: Enable http and https endpoints. + - only: Disable http endpoints and enable https ones. +- ssl_key # Paste your ssl key. Defaul value a test one +- ssl_crt # Paste your ssl crt. Defaul value a test one +- refresh_interval = 60s # Interval to refresh traefik rules.toml from rancher-metadata. + +### Service configuration labels: + +Traefik labels has to be added in your services, in order to get included in traefik dynamic config. + +- traefik.enable = + - true: the service will be published as *service_name.stack_name.traefik_domain* + - stack: the service will be published as *stack_name.domain*. WARNING: You can have collisions inside services within yout stack + - false: the service will not be published +- traefik.domain = < domain names to route rule. Multiple values separated by "," > +- traefik.port = < port to expose throught traefik > + +### Usage: + + Select Traefik from catalog. + + Set the params. + + Click deploy. + + Services will be accessed throught hosts whith traefik_lb=true at: + - http://${service_name}.${stack_name}.${traefik.domain}:${http_port} + - https://${service_name}.${stack_name}.${traefik.domain}:${https_port} + or + - http://${stack_name}.${traefik.domain}:${http_port} + - https://${stack_name}.${traefik.domain}:${https_port} + diff --git a/templates/traefik/1/docker-compose.yml b/templates/traefik/1/docker-compose.yml new file mode 100644 index 0000000..769ed7c --- /dev/null +++ b/templates/traefik/1/docker-compose.yml @@ -0,0 +1,32 @@ +traefik: + ports: + - ${admin_port}:8000/tcp + - ${http_port}:8080/tcp + - ${https_port}:8443/tcp + log_driver: '' + labels: + io.rancher.scheduler.global: 'true' + io.rancher.scheduler.affinity:host_label: ${host_label} + io.rancher.scheduler.affinity:container_label_ne: io.rancher.stack_service.name=$${stack_name}/$${service_name} + io.rancher.sidekicks: traefik-conf + io.rancher.container.hostname_override: container_name + tty: true + log_opt: {} + image: rawmind/alpine-traefik:1.0.1-1 + environment: + - CONF_INTERVAL=${refresh_interval} + - TRAEFIK_HTTPS_ENABLE=${https_enable} + volumes_from: + - traefik-conf +traefik-conf: + log_driver: '' + labels: + io.rancher.scheduler.global: 'true' + io.rancher.scheduler.affinity:host_label: ${host_label} + io.rancher.scheduler.affinity:container_label_ne: io.rancher.stack_service.name=$${stack_name}/$${service_name} + io.rancher.container.start_once: 'true' + tty: true + log_opt: {} + image: rawmind/rancher-traefik:0.3.4 + net: none + diff --git a/templates/traefik/1/rancher-compose.yml b/templates/traefik/1/rancher-compose.yml new file mode 100644 index 0000000..82c2036 --- /dev/null +++ b/templates/traefik/1/rancher-compose.yml @@ -0,0 +1,77 @@ +.catalog: + name: traefik + version: v1.0.1-rancher1 + description: | + (Experimental) Traefik load balancer. + minimum_rancher_version: v0.59.0 + maintainer: "Raul Sanchez " + uuid: traefik-0 + questions: + - variable: "host_label" + description: "Host label where to run traefik service." + label: "Host label:" + required: true + default: "traefik_lb=true" + type: "string" + - variable: "http_port" + description: "Traefik http public port to listen." + label: "Http port:" + required: true + default: 8080 + type: "int" + - variable: "https_port" + description: "Traefik https public port to listen." + label: "Https port:" + required: true + default: 8443 + type: "int" + - variable: "admin_port" + description: "Traefik admin public port to listen." + label: "Admin port:" + required: true + default: 8000 + type: "int" + - variable: "https_enable" + label: "Enable HTTPS:" + description: | + Enable https working mode. If you activate, you need to fill SSL key and SSL crt in order to work. + default: false + required: true + type: enum + options: + - false + - true + - only + - variable: "ssl_key" + description: "SSL key to secure the service" + label: "SSL key" + type: "multiline" + required: false + default: "" + - variable: "ssl_crt" + description: "SSL cert to secure the service" + label: "SSL crt" + type: "multiline" + required: false + default: "" + - variable: "refresh_interval" + description: "Interval to poll/apply configuration changes." + label: "Refresh Interval (s):" + required: true + default: 60 + type: "int" +traefik: + retain_ip: true + health_check: + port: 8000 + interval: 5000 + unhealthy_threshold: 3 + request_line: 'GET /dashboard/# HTTP/1.0' + healthy_threshold: 2 + response_timeout: 5000 + metadata: + traefik: + ssl_key: | + ${ssl_key} + ssl_crt: | + ${ssl_crt} \ No newline at end of file diff --git a/templates/traefik/config.yml b/templates/traefik/config.yml index 3617525..726db15 100644 --- a/templates/traefik/config.yml +++ b/templates/traefik/config.yml @@ -1,7 +1,7 @@ name: Traefik description: | (Experimental) Traefik active load balancer -version: v1.0.0-beta.555-rancher1 +version: v1.0.1-rancher1 category: Load Balancing maintainer: "Raul Sanchez " minimum_rancher_version: v0.59.0 diff --git a/templates/zookeeper/1/README.md b/templates/zookeeper/1/README.md new file mode 100644 index 0000000..7672d64 --- /dev/null +++ b/templates/zookeeper/1/README.md @@ -0,0 +1,27 @@ +# Apache Zookeeper (Experimental) + +### Info: + + This template creates, scale in and scale out a multinodes zk (zookeeper) cluster on top of Rancher. The configuration is generated with confd from Rancher metadata. + Cluster size are variable after deployment, and get reconfigured if refresh interval > 0. + + +### Usage: + + Select Apache Zookeeper from catalog. + + Enter the number of nodes, mem and refresh interval for the zk cluster. (set refresh data to 0 to disable dinamic config) + + Change the following zookeeper default parameters, if you need: + +- ZK_DATA_DIR="/opt/zk/data" +- ZK_INIT_LIMIT="10" +- ZK_MAX_CLIENT_CXNS="500" +- ZK_SYNC_LIMIT="5" +- ZK_TICK_TIME="2000" + + Click deploy. + + Zookeeper can now be accessed over the Rancher network. + + Note: When you scale the cluster, zero downtime is not guaranteed..yet.. diff --git a/templates/zookeeper/1/docker-compose.yml b/templates/zookeeper/1/docker-compose.yml new file mode 100644 index 0000000..006ca7d --- /dev/null +++ b/templates/zookeeper/1/docker-compose.yml @@ -0,0 +1,40 @@ +zk: + tty: true + image: rawmind/alpine-zk:3.4.8-4 + volumes_from: + - zk-volume + - zk-conf + environment: + - JVMFLAGS=-Xmx${zk_mem}m -Xms${zk_mem}m + - ZK_DATA_DIR=${zk_data_dir} + - ZK_INIT_LIMIT=${zk_init_limit} + - ZK_MAX_CLIENT_CXNS=${zk_max_client_cxns} + - ZK_SYNC_LIMIT=${zk_sync_limit} + - ZK_TICK_TIME=${zk_tick_time} + labels: + io.rancher.scheduler.affinity:container_label_soft_ne: io.rancher.stack_service.name=$${stack_name}/$${service_name} + io.rancher.container.hostname_override: container_name + io.rancher.sidekicks: zk-volume, zk-conf +zk-conf: + net: none + labels: + io.rancher.scheduler.affinity:container_label_soft_ne: io.rancher.stack_service.name=$${stack_name}/$${service_name} + io.rancher.container.hostname_override: container_name + io.rancher.container.start_once: true + image: rawmind/rancher-zk:0.3.3 +zk-volume: + net: none + labels: + io.rancher.scheduler.affinity:container_label_soft_ne: io.rancher.stack_service.name=$${stack_name}/$${service_name} + io.rancher.container.hostname_override: container_name + io.rancher.container.start_once: true + environment: + - SERVICE_USER=zookeeper + - SERVICE_UID=10002 + - SERVICE_GROUP=zookeeper + - SERVICE_GID=10002 + - SERVICE_VOLUME=${zk_data_dir} + volumes: + - ${zk_data_dir} + volume_driver: local + image: rawmind/alpine-volume:0.0.1-1 diff --git a/templates/zookeeper/1/rancher-compose.yml b/templates/zookeeper/1/rancher-compose.yml new file mode 100644 index 0000000..0ca9081 --- /dev/null +++ b/templates/zookeeper/1/rancher-compose.yml @@ -0,0 +1,67 @@ +.catalog: + name: Zookeeper + version: 3.4.8-rancher1 + description: | + (Experimental) Apache Zookeeper cluster. + minimum_rancher_version: v0.59.0 + maintainer: "Raul Sanchez " + uuid: zk-0 + questions: + - variable: "zk_scale" + description: "Number of zk nodes. Note: Recommended an odd number" + label: "Zk Nodes:" + required: true + default: 3 + type: "int" + - variable: "zk_mem" + description: "Amount of memory to config zk." + label: "Zk Memory (mb):" + required: true + default: 512 + type: "int" + - variable: "zk_init_limit" + description: "Time to allow followers to connect and sync with leader" + label: "Zk init limit (ticks):" + required: true + default: 10 + type: "int" + - variable: "zk_data_dir" + description: "Directory where zookeeper store data" + label: "Zk data dir:" + required: true + default: "/opt/zk/data" + type: "string" + - variable: "zk_max_client_cxns" + description: "Max client concurrent connections" + label: "Zk max client cxns:" + required: true + default: 500 + type: "int" + - variable: "zk_sync_limit" + description: "Time to allow followers to sync with leader" + label: "Zk sync limit (ticks):" + required: true + default: 5 + type: "int" + - variable: "zk_tick_time" + description: "Tick time length" + label: "Zk tick time (ms):" + required: true + default: 2000 + type: "int" + - variable: "zk_interval" + description: "Interval to poll/apply configuration changes. 0 to disable, reconfiguration will be done when you restart zk nodes" + label: "Zk Interval (s):" + required: true + default: 60 + type: "int" +zk: + scale: ${zk_scale} + retain_ip: true + health_check: + port: 2181 + interval: 5000 + unhealthy_threshold: 3 + request_line: '' + healthy_threshold: 2 + response_timeout: 5000 diff --git a/templates/zookeeper/config.yml b/templates/zookeeper/config.yml index 14ce016..5d83443 100644 --- a/templates/zookeeper/config.yml +++ b/templates/zookeeper/config.yml @@ -1,8 +1,8 @@ name: Apache Zookeeper description: | (Experimental) Zookeeper cluster -version: 3.4.6-rancher1 +version: 3.4.8-rancher1 category: Clustering maintainer: "Raul Sanchez " -minimum_rancher_version: v0.56.0 +minimum_rancher_version: v0.59.0 license: