diff --git a/templates/kafka/3/README.md b/templates/kafka/3/README.md new file mode 100644 index 0000000..43dac9e --- /dev/null +++ b/templates/kafka/3/README.md @@ -0,0 +1,32 @@ +# Apache Kafka (Experimental) + +### Info: + + This template creates, scale in and scale out a multinodes kafka broker cluster on top of Rancher. The configuration is generated with confd from Rancher metadata. + Cluster size are variable after deployment, and get reconfigured after refresh interval. + + +### Usage: + + Select Apache Kafka from catalog. + + Enter the number of nodes, mem and refresh interval for the kafka cluster. + + Change the following kafka default parameters, if you need: + +- kafka_scale=3 # kafka scale to deploy. +- kafka_mem=512 # kafka broker memory. +- kafka_log_dir="/opt/kafka/logs" # Kafka log dir. +- kafka_log_retention="168" # kafka log retention. +- kafka_num_partitions="1" # Kafka partitions number +- kafka_delete_topics="false" # kafka delete topics +- kafka_auto_create_topics="true" # kafka auto create topics +- kafka_replication_factor=1 # Kafka replication factor +- kafka_pub_ip= < true | false > # Advertise public ip to zookeeper. +- zk_link="kafka-zk/zk" # zookeeper stack/service to connect to. + + Click deploy. + + Kafka can now be accessed over the Rancher network. + + Note: When you scale the cluster, zero downtime is not guaranteed..yet.. diff --git a/templates/kafka/3/docker-compose.yml b/templates/kafka/3/docker-compose.yml new file mode 100644 index 0000000..8e4d841 --- /dev/null +++ b/templates/kafka/3/docker-compose.yml @@ -0,0 +1,46 @@ +broker: + tty: true + image: rawmind/alpine-kafka:0.11.0.0-1 + volumes_from: + - broker-volume + - broker-conf + environment: + - JVMFLAGS=-Xmx${kafka_mem}m -Xms${kafka_mem}m + - CONFD_INTERVAL=${kafka_interval} + - ZK_SERVICE=${zk_link} + - KAFKA_DELETE_TOPICS=${kafka_delete_topics} + - KAFKA_LOG_DIRS=${kafka_log_dir} + - KAFKA_LOG_RETENTION_HOURS=${kafka_log_retention} + - KAFKA_NUM_PARTITIONS=${kafka_num_partitions} + - ADVERTISE_PUB_IP=${kafka_pub_ip} + - KAFKA_AUTO_CREATE_TOPICS=${kafka_auto_create_topics} + - KAFKA_REPLICATION_FACTOR=${kafka_replication_factor} + external_links: + - ${zk_link}:zk + labels: + io.rancher.scheduler.affinity:container_label_soft_ne: io.rancher.stack_service.name=$${stack_name}/$${service_name} + io.rancher.container.hostname_override: container_name + io.rancher.sidekicks: broker-volume, broker-conf +broker-conf: + net: none + labels: + io.rancher.scheduler.affinity:container_label_soft_ne: io.rancher.stack_service.name=$${stack_name}/$${service_name} + io.rancher.container.hostname_override: container_name + io.rancher.container.start_once: true + image: rawmind/rancher-kafka:0.11.0.0-1 + volumes: + - /opt/tools +broker-volume: + net: none + labels: + io.rancher.scheduler.affinity:container_label_soft_ne: io.rancher.stack_service.name=$${stack_name}/$${service_name} + io.rancher.container.hostname_override: container_name + io.rancher.container.start_once: true + environment: + - SERVICE_UID=10003 + - SERVICE_GID=10003 + - SERVICE_VOLUME=${kafka_log_dir} + volumes: + - ${kafka_log_dir} + volume_driver: local + image: rawmind/alpine-volume:0.0.2-1 diff --git a/templates/kafka/3/rancher-compose.yml b/templates/kafka/3/rancher-compose.yml new file mode 100644 index 0000000..69c89bf --- /dev/null +++ b/templates/kafka/3/rancher-compose.yml @@ -0,0 +1,96 @@ +.catalog: + name: Kafka + version: 0.11.0.0-rancher1 + description: | + (Experimental) Apache Kafka cluster. + minimum_rancher_version: v0.59.0 + maintainer: "Raul Sanchez " + uuid: kafka-0 + questions: + - variable: "kafka_scale" + description: "Number of brokers nodes. Note: Recommended an odd number" + label: "Broker Nodes:" + required: true + default: 3 + type: "int" + - variable: "kafka_mem" + description: "Amount of memory to config brokers." + label: "Broker Memory (mb):" + required: true + default: 1024 + type: "string" + - variable: "kafka_log_dir" + description: "Broker volume to log data" + label: "Broker log volume:" + required: true + default: "/opt/kafka/logs" + type: "string" + - variable: "kafka_log_retention" + description: "Broker log retention in hours" + label: "Broker log retention (h):" + required: true + default: "168" + type: "int" + - variable: "kafka_num_partitions" + description: "Number of broker partitions" + label: "Broker partitions:" + required: true + default: "1" + type: "int" + - variable: "kafka_delete_topics" + label: "Delete topics:" + description: | + Enable delete topics in kafka. + default: false + required: true + type: enum + options: + - false + - true + - variable: "kafka_auto_create_topics" + label: "Auto create topics:" + description: | + Enable auto create topics in kafka. + default: true + required: true + type: enum + options: + - false + - true + - variable: "kafka_replication_factor" + description: "Topics replication factor" + label: "Replication factor:" + required: true + default: 1 + type: "int" + - variable: "kafka_interval" + description: "Interval to poll/apply configuration changes. 0 to disable" + label: "Broker Interval (s):" + required: true + default: 60 + type: "int" + - variable: "kafka_pub_ip" + description: "Advertise local and host public ip" + label: "Broker public ip:" + required: false + default: false + type: "boolean" + - variable: "zk_link" + description: | + Choose the Zookeeper service to use. + It's really recommended a dedicated zookeeper service just for Kafka. + label: "Zookeeper stack/service" + default: "kafka-zk/zk" + required: true + type: "service" +broker: + scale: ${kafka_scale} + retain_ip: true + health_check: + port: 9092 + interval: 5000 + unhealthy_threshold: 3 + request_line: '' + healthy_threshold: 2 + response_timeout: 5000 + diff --git a/templates/kafka/config.yml b/templates/kafka/config.yml index 72c9555..af07587 100644 --- a/templates/kafka/config.yml +++ b/templates/kafka/config.yml @@ -1,7 +1,7 @@ name: Apache Kafka description: | (Experimental) Kafka cluster -version: 0.10.2.1-rancher1 +version: 0.11.0.0-rancher1 category: Clustering maintainer: "Raul Sanchez " minimum_rancher_version: v0.59.0