Upgrade zookeeper and traefik packages (#185)

* Upgrade zookeeper to version 3.4.8 * Updated traefik version to v1.0.0. Added new features: ssl, multidomain, import key/cert from metadata, run as traefik user. * Upgrade kafka to version 0.10.0 * Updated traefik version to v1.0.1. ssl key and crt set as required. * Updated traefik version to v1.0.1. ssl working modes available.
2016-07-26 18:37:38 +02:00 · 2016-07-26 18:37:38 +02:00 · e8f86f188b
commit e8f86f188b
parent 2881ae2839
14 changed files with 454 additions and 7 deletions
--- a/templates/kafka/1/README.md
+++ b/templates/kafka/1/README.md
@ -0,0 +1,28 @@
+# Apache Kafka (Experimental)
+
+### Info:
+
+ This template creates, scale in and scale out a multinodes kafka broker cluster on top of Rancher. The configuration is generated with confd from Rancher metadata. 
+ Cluster size are variable after deployment, and get reconfigured after refresh interval.
+ 
+ 
+### Usage:
+
+ Select Apache Kafka from catalog. 
+ 
+ Enter the number of nodes, mem and refresh interval for the kafka cluster.
+ 
+ Change the following kafka default parameters, if you need:
+
+- KAFKA_LOG_DIRS="/opt/kafka/logs"
+- KAFKA_LOG_RETENTION_HOURS="168"
+- KAFKA_NUM_PARTITIONS="1"
+- ADVERTISE_PUB_IP= < true | false >
+
+ Select zookeeper stack/service to connect to.
+ 
+ Click deploy.
+ 
+ Kafka can now be accessed over the Rancher network. 
+
+ Note: When you scale the cluster, zero downtime is not guaranteed..yet..
--- a/templates/kafka/1/docker-compose.yml
+++ b/templates/kafka/1/docker-compose.yml
@ -0,0 +1,43 @@
+broker:
+  tty: true
+  image: rawmind/alpine-kafka:0.10.0.0-2
+  volumes_from:
+    - broker-volume
+    - broker-conf
+  environment:
+    - JVMFLAGS=-Xmx${kafka_mem}m -Xms${kafka_mem}m
+    - CONFD_INTERVAL=${kafka_interval}
+    - ZK_SERVICE=${zk_link}
+    - KAFKA_LOG_DIRS=${kafka_log_dir}
+    - KAFKA_LOG_RETENTION_HOURS=${kafka_log_retention}
+    - KAFKA_NUM_PARTITIONS=${kafka_num_partitions}
+    - ADVERTISE_PUB_IP=${kafka_pub_ip}
+  external_links:
+    - ${zk_link}:zk
+  labels: 
+    io.rancher.scheduler.affinity:container_label_soft_ne: io.rancher.stack_service.name=$${stack_name}/$${service_name}
+    io.rancher.container.hostname_override: container_name
+    io.rancher.sidekicks: broker-volume, broker-conf
+broker-conf:
+  net: none
+  labels:
+    io.rancher.scheduler.affinity:container_label_soft_ne: io.rancher.stack_service.name=$${stack_name}/$${service_name}
+    io.rancher.container.hostname_override: container_name
+    io.rancher.container.start_once: true
+  image: rawmind/rancher-kafka:0.10.0.0
+broker-volume:
+  net: none
+  labels:
+    io.rancher.scheduler.affinity:container_label_soft_ne: io.rancher.stack_service.name=$${stack_name}/$${service_name}
+    io.rancher.container.hostname_override: container_name
+    io.rancher.container.start_once: true
+  environment:
+    - SERVICE_USER=kafka
+    - SERVICE_UID=10003
+    - SERVICE_GROUP=kafka 
+    - SERVICE_GID=10003
+    - SERVICE_VOLUME=${kafka_log_dir}
+  volumes:
+    - ${kafka_log_dir}
+  volume_driver: local
+  image: rawmind/alpine-volume:0.0.1-1
--- a/templates/kafka/1/rancher-compose.yml
+++ b/templates/kafka/1/rancher-compose.yml
@ -0,0 +1,70 @@
+.catalog:
+  name: Kafka
+  version: 0.10.0.0-rancher1
+  description: |
+    (Experimental) Apache Kafka cluster.
+  minimum_rancher_version: v0.59.0
+  maintainer: "Raul Sanchez <rawmind@gmail.com>"
+  uuid: kafka-0
+  questions:
+    - variable: "kafka_scale"
+      description: "Number of brokers nodes. Note: Recommended an odd number"
+      label: "Broker Nodes:"
+      required: true
+      default: 3
+      type: "int"
+    - variable: "kafka_mem"
+      description: "Amount of memory to config brokers."
+      label: "Broker Memory (mb):"
+      required: true
+      default: 1024 
+      type: "string"
+    - variable: "kafka_log_dir"
+      description: "Broker volume to log data"
+      label: "Broker log volume:"
+      required: true
+      default: "/opt/kafka/logs"
+      type: "string"
+    - variable: "kafka_log_retention"
+      description: "Broker log retention in hours"
+      label: "Broker log retention (h):"
+      required: true
+      default: "168"
+      type: "int"
+    - variable: "kafka_num_partitions"
+      description: "Number of broker partitions"
+      label: "Broker partitions:"
+      required: true
+      default: "1"
+      type: "int"
+    - variable: "kafka_interval"
+      description: "Interval to poll/apply configuration changes. 0 to disable" 
+      label: "Broker Interval (s):"
+      required: true
+      default: 60 
+      type: "int"
+    - variable: "kafka_pub_ip"
+      description: "Advertise host ip in zookeeper" 
+      label: "Advertise host ip:"
+      required: false
+      default: false 
+      type: "boolean"
+    - variable: "zk_link"
+      description: |
+        Choose the Zookeeper service to use. 
+        It's really recommended a dedicated zookeeper service just for Kafka.
+      label: "Zookeeper stack/service"
+      default: "kafka-zk/zk"
+      required: true
+      type: "service"
+broker:
+  scale: ${kafka_scale}
+  retain_ip: true
+  health_check:
+    port: 9092
+    interval: 5000
+    unhealthy_threshold: 3
+    request_line: ''
+    healthy_threshold: 2
+    response_timeout: 5000
+
--- a/templates/kafka/config.yml
+++ b/templates/kafka/config.yml
@ -1,8 +1,8 @@
 name: Apache Kafka
 description: |
  (Experimental) Kafka cluster
-version: 0.9.0-rancher1
+version: 0.10.0.0-rancher1
 category: Clustering
 maintainer: "Raul Sanchez <rawmind@gmail.com>"
-minimum_rancher_version: v0.56.0
+minimum_rancher_version: v0.59.0
 license: 
--- a/templates/traefik/0/docker-compose.yml
+++ b/templates/traefik/0/docker-compose.yml
@ -1,7 +1,9 @@
+
 traefik:
  ports:
  - ${admin_port}:${admin_port}/tcp
  - ${http_port}:${http_port}/tcp
+  - ${https_port}:${https_port}/tcp
  log_driver: ''
  labels:
    io.rancher.scheduler.global: 'true'
@ -11,9 +13,10 @@ traefik:
    io.rancher.container.hostname_override: container_name
  tty: true
  log_opt: {}
-  image: rawmind/alpine-traefik:1.0.0-beta.555-6
+  image: rawmind/alpine-traefik:1.0.0-beta.771
  environment:
  - TRAEFIK_HTTP_PORT=${http_port}
+  - TRAEFIK_HTTPS_PORT=${https_port}
  - TRAEFIK_ADMIN_PORT=${admin_port}
  - CONF_INTERVAL=${refresh_interval}
  volumes_from:
--- a/templates/traefik/0/rancher-compose.yml
+++ b/templates/traefik/0/rancher-compose.yml
@ -13,20 +13,33 @@
      required: true
      default: 8080
      type: "int"
+    - variable: "https_port"
+      description: "Traefik https public port to listen."
+      label: "Https port:"
+      required: true
+      default: 8443
+      type: "int"
    - variable: "admin_port"
      description: "Traefik admin public port to listen."
      label: "Admin port:"
      required: true
      default: 8000 
-      type: "string"
+      type: "int"
    - variable: "refresh_interval"
      description: "Interval to poll/apply configuration changes." 
      label: "Refresh Interval (s):"
      required: true
      default: 60 
      type: "int"
+    - variable: "ssl_cert"
+      description: "SSL certificate" 
+      label: "SSL certificate:"
+      required: false
+      default: ""
+      type: "certificate"
 traefik:
  retain_ip: true
+  default_cert: ${ssl_cert}
  health_check:
    port: ${admin_port}
    interval: 5000
--- a/templates/traefik/1/README.md
+++ b/templates/traefik/1/README.md
@ -0,0 +1,47 @@
+# Traefik active load balancer (Experimental)
+
+### Info:
+
+ This template deploys traefik active load balancers on top of Rancher. The configuration is generated and updated with confd from Rancher metadata. 
+ It would be deployed in hosts with label traefik_lb=true.
+
+### Config:
+
+- host_label = "traefik_lb=true" # Host label where to run traefik service.
+- http_port = 8080  # Port exposed to get access to the published services.
+- https_port = 8443  # Port exposed to get secured access to the published services.
+- admin_port = 8000  # Port exposed to get admin access to the traefik service.
+- https_enable = <false | true | only>
+  - false: Enable http enpoints and disable https ones.
+  - true: Enable http and https endpoints.
+  - only: Disable http endpoints and enable https ones.
+- ssl_key # Paste your ssl key. Defaul value a test one
+- ssl_crt # Paste your ssl crt. Defaul value a test one
+- refresh_interval = 60s  # Interval to refresh traefik rules.toml from rancher-metadata.
+
+### Service configuration labels:
+
+Traefik labels has to be added in your services, in order to get included in traefik dynamic config.
+
+- traefik.enable = <true | false> 
+  - true: the service will be published as *service_name.stack_name.traefik_domain*
+  - stack: the service will be published as *stack_name.domain*. WARNING: You can have collisions inside services within yout stack
+  - false: the service will not be published
+- traefik.domain = < domain names to route rule. Multiple values separated by "," > 
+- traefik.port = < port to expose throught traefik >  
+ 
+### Usage:
+
+ Select Traefik from catalog. 
+ 
+ Set the params.
+
+ Click deploy.
+
+ Services will be accessed throught hosts whith traefik_lb=true at: 
+ - http://${service_name}.${stack_name}.${traefik.domain}:${http_port}
+ - https://${service_name}.${stack_name}.${traefik.domain}:${https_port}
+ or 
+ - http://${stack_name}.${traefik.domain}:${http_port}
+ - https://${stack_name}.${traefik.domain}:${https_port}
+
--- a/templates/traefik/1/docker-compose.yml
+++ b/templates/traefik/1/docker-compose.yml
@ -0,0 +1,32 @@
+traefik:
+  ports:
+  - ${admin_port}:8000/tcp
+  - ${http_port}:8080/tcp
+  - ${https_port}:8443/tcp
+  log_driver: ''
+  labels:
+    io.rancher.scheduler.global: 'true'
+    io.rancher.scheduler.affinity:host_label: ${host_label}
+    io.rancher.scheduler.affinity:container_label_ne: io.rancher.stack_service.name=$${stack_name}/$${service_name}
+    io.rancher.sidekicks: traefik-conf
+    io.rancher.container.hostname_override: container_name
+  tty: true
+  log_opt: {}
+  image: rawmind/alpine-traefik:1.0.1-1
+  environment:
+  - CONF_INTERVAL=${refresh_interval}
+  - TRAEFIK_HTTPS_ENABLE=${https_enable}
+  volumes_from:
+  - traefik-conf
+traefik-conf:
+  log_driver: ''
+  labels:
+    io.rancher.scheduler.global: 'true'
+    io.rancher.scheduler.affinity:host_label: ${host_label}
+    io.rancher.scheduler.affinity:container_label_ne: io.rancher.stack_service.name=$${stack_name}/$${service_name}
+    io.rancher.container.start_once: 'true'
+  tty: true
+  log_opt: {}
+  image: rawmind/rancher-traefik:0.3.4
+  net: none
+
--- a/templates/traefik/1/rancher-compose.yml
+++ b/templates/traefik/1/rancher-compose.yml
@ -0,0 +1,77 @@
+.catalog:
+  name: traefik
+  version: v1.0.1-rancher1
+  description: |
+    (Experimental) Traefik load balancer.
+  minimum_rancher_version: v0.59.0
+  maintainer: "Raul Sanchez <rawmind@gmail.com>"
+  uuid: traefik-0
+  questions:
+    - variable: "host_label"
+      description: "Host label where to run traefik service."
+      label: "Host label:"
+      required: true
+      default: "traefik_lb=true" 
+      type: "string"
+    - variable: "http_port"
+      description: "Traefik http public port to listen."
+      label: "Http port:"
+      required: true
+      default: 8080
+      type: "int"
+    - variable: "https_port"
+      description: "Traefik https public port to listen."
+      label: "Https port:"
+      required: true
+      default: 8443
+      type: "int"
+    - variable: "admin_port"
+      description: "Traefik admin public port to listen."
+      label: "Admin port:"
+      required: true
+      default: 8000 
+      type: "int"
+    - variable: "https_enable"
+      label: "Enable HTTPS:"
+      description: |
+        Enable https working mode. If you activate, you need to fill SSL key and SSL crt in order to work.
+      default: false
+      required: true
+      type: enum
+      options:
+        - false
+        - true
+        - only
+    - variable: "ssl_key"
+      description: "SSL key to secure the service"
+      label: "SSL key"
+      type: "multiline"
+      required: false
+      default: ""
+    - variable: "ssl_crt"
+      description: "SSL cert to secure the service"
+      label: "SSL crt"
+      type: "multiline"
+      required: false
+      default: ""
+    - variable: "refresh_interval"
+      description: "Interval to poll/apply configuration changes." 
+      label: "Refresh Interval (s):"
+      required: true
+      default: 60 
+      type: "int"
+traefik:
+  retain_ip: true
+  health_check:
+    port: 8000
+    interval: 5000
+    unhealthy_threshold: 3
+    request_line: 'GET /dashboard/# HTTP/1.0'
+    healthy_threshold: 2
+    response_timeout: 5000
+  metadata:
+    traefik:
+      ssl_key: |
+        ${ssl_key}
+      ssl_crt: |
+        ${ssl_crt}
--- a/templates/traefik/config.yml
+++ b/templates/traefik/config.yml
@ -1,7 +1,7 @@
 name: Traefik
 description: |
  (Experimental) Traefik active load balancer
-version: v1.0.0-beta.555-rancher1
+version: v1.0.1-rancher1
 category: Load Balancing
 maintainer: "Raul Sanchez <rawmind@gmail.com>"
 minimum_rancher_version: v0.59.0
--- a/templates/zookeeper/1/README.md
+++ b/templates/zookeeper/1/README.md
@ -0,0 +1,27 @@
+# Apache Zookeeper (Experimental)
+
+### Info:
+
+ This template creates, scale in and scale out a multinodes zk (zookeeper) cluster on top of Rancher. The configuration is generated with confd from Rancher metadata. 
+ Cluster size are variable after deployment, and get reconfigured if refresh interval > 0.
+ 
+ 
+### Usage:
+
+ Select Apache Zookeeper from catalog. 
+ 
+ Enter the number of nodes, mem and refresh interval for the zk cluster. (set refresh data to 0 to disable dinamic config)
+
+ Change the following zookeeper default parameters, if you need:
+
+- ZK_DATA_DIR="/opt/zk/data"
+- ZK_INIT_LIMIT="10"
+- ZK_MAX_CLIENT_CXNS="500"
+- ZK_SYNC_LIMIT="5"
+- ZK_TICK_TIME="2000"
+ 
+ Click deploy.
+ 
+ Zookeeper can now be accessed over the Rancher network. 
+
+ Note: When you scale the cluster, zero downtime is not guaranteed..yet..
--- a/templates/zookeeper/1/docker-compose.yml
+++ b/templates/zookeeper/1/docker-compose.yml
@ -0,0 +1,40 @@
+zk:
+  tty: true
+  image: rawmind/alpine-zk:3.4.8-4
+  volumes_from:
+    - zk-volume
+    - zk-conf
+  environment:
+    - JVMFLAGS=-Xmx${zk_mem}m -Xms${zk_mem}m
+    - ZK_DATA_DIR=${zk_data_dir}
+    - ZK_INIT_LIMIT=${zk_init_limit}
+    - ZK_MAX_CLIENT_CXNS=${zk_max_client_cxns}
+    - ZK_SYNC_LIMIT=${zk_sync_limit}
+    - ZK_TICK_TIME=${zk_tick_time}
+  labels: 
+    io.rancher.scheduler.affinity:container_label_soft_ne: io.rancher.stack_service.name=$${stack_name}/$${service_name}
+    io.rancher.container.hostname_override: container_name
+    io.rancher.sidekicks: zk-volume, zk-conf
+zk-conf:
+  net: none
+  labels:
+    io.rancher.scheduler.affinity:container_label_soft_ne: io.rancher.stack_service.name=$${stack_name}/$${service_name}
+    io.rancher.container.hostname_override: container_name
+    io.rancher.container.start_once: true
+  image: rawmind/rancher-zk:0.3.3
+zk-volume:
+  net: none
+  labels:
+    io.rancher.scheduler.affinity:container_label_soft_ne: io.rancher.stack_service.name=$${stack_name}/$${service_name}
+    io.rancher.container.hostname_override: container_name
+    io.rancher.container.start_once: true
+  environment:
+    - SERVICE_USER=zookeeper
+    - SERVICE_UID=10002
+    - SERVICE_GROUP=zookeeper 
+    - SERVICE_GID=10002
+    - SERVICE_VOLUME=${zk_data_dir}
+  volumes:
+    - ${zk_data_dir}
+  volume_driver: local
+  image: rawmind/alpine-volume:0.0.1-1
--- a/templates/zookeeper/1/rancher-compose.yml
+++ b/templates/zookeeper/1/rancher-compose.yml
@ -0,0 +1,67 @@
+.catalog:
+  name: Zookeeper
+  version: 3.4.8-rancher1
+  description: |
+    (Experimental) Apache Zookeeper cluster.
+  minimum_rancher_version: v0.59.0
+  maintainer: "Raul Sanchez <rawmind@gmail.com>"
+  uuid: zk-0
+  questions:
+    - variable: "zk_scale"
+      description: "Number of zk nodes. Note: Recommended an odd number"
+      label: "Zk Nodes:"
+      required: true
+      default: 3
+      type: "int"
+    - variable: "zk_mem"
+      description: "Amount of memory to config zk."
+      label: "Zk Memory (mb):"
+      required: true
+      default: 512 
+      type: "int"
+    - variable: "zk_init_limit"
+      description: "Time to allow followers to connect and sync with leader"
+      label: "Zk init limit (ticks):"
+      required: true
+      default: 10
+      type: "int"
+    - variable: "zk_data_dir"
+      description: "Directory where zookeeper store data"
+      label: "Zk data dir:"
+      required: true
+      default: "/opt/zk/data"
+      type: "string"
+    - variable: "zk_max_client_cxns"
+      description: "Max client concurrent connections"
+      label: "Zk max client cxns:"
+      required: true
+      default: 500
+      type: "int"
+    - variable: "zk_sync_limit"
+      description: "Time to allow followers to sync with leader"
+      label: "Zk sync limit (ticks):"
+      required: true
+      default: 5
+      type: "int"
+    - variable: "zk_tick_time"
+      description: "Tick time length"
+      label: "Zk tick time (ms):"
+      required: true
+      default: 2000
+      type: "int"
+    - variable: "zk_interval"
+      description: "Interval to poll/apply configuration changes. 0 to disable, reconfiguration will be done when you restart zk nodes"
+      label: "Zk Interval (s):"
+      required: true
+      default: 60
+      type: "int"
+zk:
+  scale: ${zk_scale}
+  retain_ip: true
+  health_check:
+    port: 2181
+    interval: 5000
+    unhealthy_threshold: 3
+    request_line: ''
+    healthy_threshold: 2
+    response_timeout: 5000
--- a/templates/zookeeper/config.yml
+++ b/templates/zookeeper/config.yml
@ -1,8 +1,8 @@
 name: Apache Zookeeper
 description: |
  (Experimental) Zookeeper cluster
-version: 3.4.6-rancher1
+version: 3.4.8-rancher1
 category: Clustering
 maintainer: "Raul Sanchez <rawmind@gmail.com>"
-minimum_rancher_version: v0.56.0
+minimum_rancher_version: v0.59.0
 license: