Upgraded kafka package to 0.11.0.0

templates/kafka/3/README.md

@@ -0,0 +1,32 @@
+# Apache Kafka (Experimental)
+
+### Info:
+
+ This template creates, scale in and scale out a multinodes kafka broker cluster on top of Rancher. The configuration is generated with confd from Rancher metadata. 
+ Cluster size are variable after deployment, and get reconfigured after refresh interval.
+ 
+ 
+### Usage:
+
+ Select Apache Kafka from catalog. 
+ 
+ Enter the number of nodes, mem and refresh interval for the kafka cluster.
+ 
+ Change the following kafka default parameters, if you need:
+
+- kafka_scale=3							# kafka scale to deploy.
+- kafka_mem=512							# kafka broker memory.
+- kafka_log_dir="/opt/kafka/logs"		# Kafka log dir.
+- kafka_log_retention="168"				# kafka log retention. 
+- kafka_num_partitions="1"				# Kafka partitions number
+- kafka_delete_topics="false"			# kafka delete topics
+- kafka_auto_create_topics="true"		# kafka auto create topics
+- kafka_replication_factor=1 			# Kafka replication factor
+- kafka_pub_ip= < true | false >		# Advertise public ip to zookeeper.
+- zk_link="kafka-zk/zk" 				# zookeeper stack/service to connect to.
+ 
+ Click deploy.
+ 
+ Kafka can now be accessed over the Rancher network. 
+
+ Note: When you scale the cluster, zero downtime is not guaranteed..yet..

templates/kafka/3/docker-compose.yml

@@ -0,0 +1,46 @@
+broker:
+  tty: true
+  image: rawmind/alpine-kafka:0.11.0.0-1
+  volumes_from:
+    - broker-volume
+    - broker-conf
+  environment:
+    - JVMFLAGS=-Xmx${kafka_mem}m -Xms${kafka_mem}m
+    - CONFD_INTERVAL=${kafka_interval}
+    - ZK_SERVICE=${zk_link}
+    - KAFKA_DELETE_TOPICS=${kafka_delete_topics}
+    - KAFKA_LOG_DIRS=${kafka_log_dir}
+    - KAFKA_LOG_RETENTION_HOURS=${kafka_log_retention}
+    - KAFKA_NUM_PARTITIONS=${kafka_num_partitions}
+    - ADVERTISE_PUB_IP=${kafka_pub_ip}
+    - KAFKA_AUTO_CREATE_TOPICS=${kafka_auto_create_topics}
+    - KAFKA_REPLICATION_FACTOR=${kafka_replication_factor}
+  external_links:
+    - ${zk_link}:zk
+  labels: 
+    io.rancher.scheduler.affinity:container_label_soft_ne: io.rancher.stack_service.name=$${stack_name}/$${service_name}
+    io.rancher.container.hostname_override: container_name
+    io.rancher.sidekicks: broker-volume, broker-conf
+broker-conf:
+  net: none
+  labels:
+    io.rancher.scheduler.affinity:container_label_soft_ne: io.rancher.stack_service.name=$${stack_name}/$${service_name}
+    io.rancher.container.hostname_override: container_name
+    io.rancher.container.start_once: true
+  image: rawmind/rancher-kafka:0.11.0.0-1
+  volumes:
+    - /opt/tools
+broker-volume:
+  net: none
+  labels:
+    io.rancher.scheduler.affinity:container_label_soft_ne: io.rancher.stack_service.name=$${stack_name}/$${service_name}
+    io.rancher.container.hostname_override: container_name
+    io.rancher.container.start_once: true
+  environment:
+    - SERVICE_UID=10003
+    - SERVICE_GID=10003
+    - SERVICE_VOLUME=${kafka_log_dir}
+  volumes:
+    - ${kafka_log_dir}
+  volume_driver: local
+  image: rawmind/alpine-volume:0.0.2-1

templates/kafka/3/rancher-compose.yml

@@ -0,0 +1,96 @@
+.catalog:
+  name: Kafka
+  version: 0.11.0.0-rancher1
+  description: |
+    (Experimental) Apache Kafka cluster.
+  minimum_rancher_version: v0.59.0
+  maintainer: "Raul Sanchez <rawmind@gmail.com>"
+  uuid: kafka-0
+  questions:
+    - variable: "kafka_scale"
+      description: "Number of brokers nodes. Note: Recommended an odd number"
+      label: "Broker Nodes:"
+      required: true
+      default: 3
+      type: "int"
+    - variable: "kafka_mem"
+      description: "Amount of memory to config brokers."
+      label: "Broker Memory (mb):"
+      required: true
+      default: 1024 
+      type: "string"
+    - variable: "kafka_log_dir"
+      description: "Broker volume to log data"
+      label: "Broker log volume:"
+      required: true
+      default: "/opt/kafka/logs"
+      type: "string"
+    - variable: "kafka_log_retention"
+      description: "Broker log retention in hours"
+      label: "Broker log retention (h):"
+      required: true
+      default: "168"
+      type: "int"
+    - variable: "kafka_num_partitions"
+      description: "Number of broker partitions"
+      label: "Broker partitions:"
+      required: true
+      default: "1"
+      type: "int"
+    - variable: "kafka_delete_topics"
+      label: "Delete topics:"
+      description: |
+        Enable delete topics in kafka.
+      default: false
+      required: true
+      type: enum
+      options:
+        - false
+        - true 
+    - variable: "kafka_auto_create_topics"
+      label: "Auto create topics:"
+      description: |
+        Enable auto create topics in kafka.
+      default: true
+      required: true
+      type: enum
+      options:
+        - false
+        - true
+    - variable: "kafka_replication_factor"
+      description: "Topics replication factor" 
+      label: "Replication factor:"
+      required: true
+      default: 1 
+      type: "int"
+    - variable: "kafka_interval"
+      description: "Interval to poll/apply configuration changes. 0 to disable" 
+      label: "Broker Interval (s):"
+      required: true
+      default: 60 
+      type: "int"
+    - variable: "kafka_pub_ip"
+      description: "Advertise local and host public ip" 
+      label: "Broker public ip:"
+      required: false
+      default: false 
+      type: "boolean"
+    - variable: "zk_link"
+      description: |
+        Choose the Zookeeper service to use. 
+        It's really recommended a dedicated zookeeper service just for Kafka.
+      label: "Zookeeper stack/service"
+      default: "kafka-zk/zk"
+      required: true
+      type: "service"
+broker:
+  scale: ${kafka_scale}
+  retain_ip: true
+  health_check:
+    port: 9092
+    interval: 5000
+    unhealthy_threshold: 3
+    request_line: ''
+    healthy_threshold: 2
+    response_timeout: 5000
+

templates/kafka/config.yml

@@ -1,7 +1,7 @@
 name: Apache Kafka
 description: |
   (Experimental) Kafka cluster
-version: 0.10.2.1-rancher1
+version: 0.11.0.0-rancher1
 category: Clustering
 maintainer: "Raul Sanchez <rawmind@gmail.com>"
 minimum_rancher_version: v0.59.0