Merge pull request #522 from rawmind0/master
Updated kafka to 0.10.2.1 and fix for zookeeper. cattle and k8s
This commit is contained in:
Raúl Sánchez
GitHub
commit
115e3ee471
33
kubernetes-templates/kafka/1/README.md
Normal file
33
kubernetes-templates/kafka/1/README.md
Normal file
@ -0,0 +1,33 @@
|
|||||||
|
# Apache Kafka (Experimental)
|
||||||
|
|
||||||
|
### Info:
|
||||||
|
|
||||||
|
This template creates, scale in and scale out a multinodes kafka broker cluster on top of k8s. The configuration is generated with confd from k8s.
|
||||||
|
Cluster size are variable after deployment, and get reconfigured.
|
||||||
|
|
||||||
|
|
||||||
|
### Usage:
|
||||||
|
|
||||||
|
Select Apache Kafka from catalog.
|
||||||
|
|
||||||
|
Enter the number of nodes, mem and refresh interval for the kafka cluster.
|
||||||
|
|
||||||
|
Change the following kafka default parameters, if you need:
|
||||||
|
|
||||||
|
- kafka_name="kafka" # kafka k8s service and rc name.
|
||||||
|
- kafka_namespace="kafka" # kafka k8s namespace
|
||||||
|
- kafka_scale=3 # kafka scale to deploy.
|
||||||
|
- kafka_mem=512 # kafka broker memory.
|
||||||
|
- kafka_log_dir="/opt/kafka/logs" # Kafka log dir.
|
||||||
|
- kafka_log_retention="168" # kafka log retention.
|
||||||
|
- kafka_num_partitions="1" # Kafka partitions number
|
||||||
|
- kafka_delete_topics="false" # kafka delete topics
|
||||||
|
- kafka_pub_ip= < true | false > # Advertise public ip to zookeeper.
|
||||||
|
- zk_link="kafka-zk/zk" # zookeeper namespace/service to connect to.
|
||||||
|
|
||||||
|
Click deploy.
|
||||||
|
|
||||||
|
Kafka can now be accessed over the Rancher network.
|
||||||
|
|
||||||
|
Note: When you scale the cluster, zero downtime is not guaranteed..yet..
|
||||||
|
|
||||||
77
kubernetes-templates/kafka/1/kafka-rc.yml
Normal file
77
kubernetes-templates/kafka/1/kafka-rc.yml
Normal file
@ -0,0 +1,77 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ReplicationController
|
||||||
|
metadata:
|
||||||
|
name: "${kafka_name}"
|
||||||
|
namespace: "${kafka_namespace}"
|
||||||
|
labels:
|
||||||
|
name: "${kafka_name}"
|
||||||
|
spec:
|
||||||
|
replicas: ${kafka_scale}
|
||||||
|
selector:
|
||||||
|
name: "${kafka_name}"
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
name: "${kafka_name}"
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: "broker-conf"
|
||||||
|
image: "rawmind/k8s-kafka:0.10.2.0-1"
|
||||||
|
volumeMounts:
|
||||||
|
- mountPath: "/opt/tools"
|
||||||
|
name: "broker-conf"
|
||||||
|
ports:
|
||||||
|
imagePullPolicy: "IfNotPresent"
|
||||||
|
- name: "broker-volume"
|
||||||
|
image: "rawmind/alpine-volume:0.0.2-1"
|
||||||
|
env:
|
||||||
|
- name: SERVICE_UID
|
||||||
|
value: "10003"
|
||||||
|
- name: SERVICE_GID
|
||||||
|
value: "10003"
|
||||||
|
- name: SERVICE_VOLUME
|
||||||
|
value: "${kafka_log_dir}"
|
||||||
|
- name: KEEP_ALIVE
|
||||||
|
value: "1"
|
||||||
|
volumeMounts:
|
||||||
|
- mountPath: "${kafka_log_dir}"
|
||||||
|
name: "broker-data"
|
||||||
|
ports:
|
||||||
|
imagePullPolicy: "IfNotPresent"
|
||||||
|
- name: "broker-service"
|
||||||
|
image: "rawmind/alpine-kafka:0.10.2.0-2"
|
||||||
|
env:
|
||||||
|
- name: JVMFLAGS
|
||||||
|
value: "-Xmx${kafka_mem}m -Xms${kafka_mem}m"
|
||||||
|
- name: CONFD_INTERVAL
|
||||||
|
value: "${kafka_interval}"
|
||||||
|
- name: KAFKA_ZK_SERVICE
|
||||||
|
value: "${zk_link}"
|
||||||
|
- name: KAFKA_DELETE_TOPICS
|
||||||
|
value: "${kafka_delete_topics}"
|
||||||
|
- name: KAFKA_LOG_DIRS
|
||||||
|
value: "${kafka_log_dir}"
|
||||||
|
- name: KAFKA_LOG_RETENTION_HOURS
|
||||||
|
value: "${kafka_log_retention}"
|
||||||
|
- name: KAFKA_NUM_PARTITIONS
|
||||||
|
value: "${kafka_num_partitions}"
|
||||||
|
- name: ADVERTISE_PUB_IP
|
||||||
|
value: "${kafka_pub_ip}"
|
||||||
|
- name: POD_NAMESPACE
|
||||||
|
value: "${kafka_namespace}"
|
||||||
|
volumeMounts:
|
||||||
|
- mountPath: "${kafka_log_dir}"
|
||||||
|
name: "broker-data"
|
||||||
|
- mountPath: "/opt/tools"
|
||||||
|
name: "broker-conf"
|
||||||
|
ports:
|
||||||
|
- containerPort: 9092
|
||||||
|
name: "broker-client"
|
||||||
|
protocol: "TCP"
|
||||||
|
imagePullPolicy: "IfNotPresent"
|
||||||
|
restartPolicy: "Always"
|
||||||
|
volumes:
|
||||||
|
- name: "broker-data"
|
||||||
|
emptyDir: {}
|
||||||
|
- name: "broker-conf"
|
||||||
|
emptyDir: {}
|
||||||
15
kubernetes-templates/kafka/1/kafka-service.yml
Normal file
15
kubernetes-templates/kafka/1/kafka-service.yml
Normal file
@ -0,0 +1,15 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: "${kafka_name}"
|
||||||
|
labels:
|
||||||
|
name: "${kafka_name}"
|
||||||
|
spec:
|
||||||
|
ports:
|
||||||
|
- port: 9092
|
||||||
|
targetPort: 9092
|
||||||
|
name: "broker-client"
|
||||||
|
protocol: "TCP"
|
||||||
|
selector:
|
||||||
|
name: "${kafka_name}"
|
||||||
|
clusterIP: "None"
|
||||||
95
kubernetes-templates/kafka/1/rancher-compose.yml
Normal file
95
kubernetes-templates/kafka/1/rancher-compose.yml
Normal file
@ -0,0 +1,95 @@
|
|||||||
|
.catalog:
|
||||||
|
name: Kafka
|
||||||
|
version: 0.10.2.0-k8s
|
||||||
|
description: |
|
||||||
|
(Experimental) Apache Kafka cluster.
|
||||||
|
minimum_rancher_version: v0.59.0
|
||||||
|
maintainer: "Raul Sanchez <rawmind@gmail.com>"
|
||||||
|
uuid: kafka-0
|
||||||
|
questions:
|
||||||
|
- variable: "kafka_name"
|
||||||
|
description: "Name of kafka rc and service."
|
||||||
|
label: "K8s rc name:"
|
||||||
|
required: true
|
||||||
|
default: "kafka"
|
||||||
|
type: "string"
|
||||||
|
- variable: "kafka_namespace"
|
||||||
|
description: "Name of kafka namespace."
|
||||||
|
label: "K8s namespace:"
|
||||||
|
required: true
|
||||||
|
default: "default"
|
||||||
|
type: "string"
|
||||||
|
- variable: "kafka_scale"
|
||||||
|
description: "Number of brokers nodes. Note: Recommended an odd number"
|
||||||
|
label: "Broker Nodes:"
|
||||||
|
required: true
|
||||||
|
default: 3
|
||||||
|
type: "int"
|
||||||
|
- variable: "kafka_mem"
|
||||||
|
description: "Amount of memory to config brokers."
|
||||||
|
label: "Broker Memory (mb):"
|
||||||
|
required: true
|
||||||
|
default: 1024
|
||||||
|
type: "string"
|
||||||
|
- variable: "kafka_log_dir"
|
||||||
|
description: "Broker volume to log data"
|
||||||
|
label: "Broker log volume:"
|
||||||
|
required: true
|
||||||
|
default: "/opt/kafka/logs"
|
||||||
|
type: "string"
|
||||||
|
- variable: "kafka_log_retention"
|
||||||
|
description: "Broker log retention in hours"
|
||||||
|
label: "Broker log retention (h):"
|
||||||
|
required: true
|
||||||
|
default: "168"
|
||||||
|
type: "int"
|
||||||
|
- variable: "kafka_num_partitions"
|
||||||
|
description: "Number of broker partitions"
|
||||||
|
label: "Broker partitions:"
|
||||||
|
required: true
|
||||||
|
default: "1"
|
||||||
|
type: "int"
|
||||||
|
- variable: "kafka_delete_topics"
|
||||||
|
label: "Kafka delete topics:"
|
||||||
|
description: |
|
||||||
|
Enable delete topics in kafka.
|
||||||
|
default: false
|
||||||
|
required: true
|
||||||
|
type: enum
|
||||||
|
options:
|
||||||
|
- false
|
||||||
|
- true
|
||||||
|
- variable: "kafka_interval"
|
||||||
|
description: "Interval to poll/apply configuration changes. 0 to disable"
|
||||||
|
label: "Broker Interval (s):"
|
||||||
|
required: true
|
||||||
|
default: 60
|
||||||
|
type: "int"
|
||||||
|
- variable: "kafka_pub_ip"
|
||||||
|
description: "Advertise local and host public ip"
|
||||||
|
label: "Broker public ip:"
|
||||||
|
default: false
|
||||||
|
required: true
|
||||||
|
type: enum
|
||||||
|
options:
|
||||||
|
- false
|
||||||
|
- true
|
||||||
|
- variable: "zk_link"
|
||||||
|
description: |
|
||||||
|
Choose the Zookeeper service to use.
|
||||||
|
It's really recommended a dedicated zookeeper service just for Kafka.
|
||||||
|
label: "Zookeeper stack/service"
|
||||||
|
default: "default/zookeeper"
|
||||||
|
required: true
|
||||||
|
type: "service"
|
||||||
|
broker:
|
||||||
|
scale: ${kafka_scale}
|
||||||
|
retain_ip: true
|
||||||
|
health_check:
|
||||||
|
port: 9092
|
||||||
|
interval: 5000
|
||||||
|
unhealthy_threshold: 3
|
||||||
|
request_line: ''
|
||||||
|
healthy_threshold: 2
|
||||||
|
response_timeout: 5000
|
||||||
|
|
||||||
33
kubernetes-templates/kafka/2/README.md
Normal file
33
kubernetes-templates/kafka/2/README.md
Normal file
@ -0,0 +1,33 @@
|
|||||||
|
# Apache Kafka (Experimental)
|
||||||
|
|
||||||
|
### Info:
|
||||||
|
|
||||||
|
This template creates, scale in and scale out a multinodes kafka broker cluster on top of k8s. The configuration is generated with confd from k8s.
|
||||||
|
Cluster size are variable after deployment, and get reconfigured.
|
||||||
|
|
||||||
|
|
||||||
|
### Usage:
|
||||||
|
|
||||||
|
Select Apache Kafka from catalog.
|
||||||
|
|
||||||
|
Enter the number of nodes, mem and refresh interval for the kafka cluster.
|
||||||
|
|
||||||
|
Change the following kafka default parameters, if you need:
|
||||||
|
|
||||||
|
- kafka_name="kafka" # kafka k8s service and rc name.
|
||||||
|
- kafka_namespace="kafka" # kafka k8s namespace
|
||||||
|
- kafka_scale=3 # kafka scale to deploy.
|
||||||
|
- kafka_mem=512 # kafka broker memory.
|
||||||
|
- kafka_log_dir="/opt/kafka/logs" # Kafka log dir.
|
||||||
|
- kafka_log_retention="168" # kafka log retention.
|
||||||
|
- kafka_num_partitions="1" # Kafka partitions number
|
||||||
|
- kafka_delete_topics="false" # kafka delete topics
|
||||||
|
- kafka_pub_ip= < true | false > # Advertise public ip to zookeeper.
|
||||||
|
- zk_link="kafka-zk/zk" # zookeeper namespace/service to connect to.
|
||||||
|
|
||||||
|
Click deploy.
|
||||||
|
|
||||||
|
Kafka can now be accessed over the Rancher network.
|
||||||
|
|
||||||
|
Note: When you scale the cluster, zero downtime is not guaranteed..yet..
|
||||||
|
|
||||||
77
kubernetes-templates/kafka/2/kafka-rc.yml
Normal file
77
kubernetes-templates/kafka/2/kafka-rc.yml
Normal file
@ -0,0 +1,77 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ReplicationController
|
||||||
|
metadata:
|
||||||
|
name: "${kafka_name}"
|
||||||
|
namespace: "${kafka_namespace}"
|
||||||
|
labels:
|
||||||
|
name: "${kafka_name}"
|
||||||
|
spec:
|
||||||
|
replicas: ${kafka_scale}
|
||||||
|
selector:
|
||||||
|
name: "${kafka_name}"
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
name: "${kafka_name}"
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: "broker-conf"
|
||||||
|
image: "rawmind/k8s-kafka:0.10.2.0-1"
|
||||||
|
volumeMounts:
|
||||||
|
- mountPath: "/opt/tools"
|
||||||
|
name: "broker-conf"
|
||||||
|
ports:
|
||||||
|
imagePullPolicy: "IfNotPresent"
|
||||||
|
- name: "broker-volume"
|
||||||
|
image: "rawmind/alpine-volume:0.0.2-1"
|
||||||
|
env:
|
||||||
|
- name: SERVICE_UID
|
||||||
|
value: "10003"
|
||||||
|
- name: SERVICE_GID
|
||||||
|
value: "10003"
|
||||||
|
- name: SERVICE_VOLUME
|
||||||
|
value: "${kafka_log_dir}"
|
||||||
|
- name: KEEP_ALIVE
|
||||||
|
value: "1"
|
||||||
|
volumeMounts:
|
||||||
|
- mountPath: "${kafka_log_dir}"
|
||||||
|
name: "broker-data"
|
||||||
|
ports:
|
||||||
|
imagePullPolicy: "IfNotPresent"
|
||||||
|
- name: "broker-service"
|
||||||
|
image: "rawmind/alpine-kafka:0.10.2.1"
|
||||||
|
env:
|
||||||
|
- name: JVMFLAGS
|
||||||
|
value: "-Xmx${kafka_mem}m -Xms${kafka_mem}m"
|
||||||
|
- name: CONFD_INTERVAL
|
||||||
|
value: "${kafka_interval}"
|
||||||
|
- name: KAFKA_ZK_SERVICE
|
||||||
|
value: "${zk_link}"
|
||||||
|
- name: KAFKA_DELETE_TOPICS
|
||||||
|
value: "${kafka_delete_topics}"
|
||||||
|
- name: KAFKA_LOG_DIRS
|
||||||
|
value: "${kafka_log_dir}"
|
||||||
|
- name: KAFKA_LOG_RETENTION_HOURS
|
||||||
|
value: "${kafka_log_retention}"
|
||||||
|
- name: KAFKA_NUM_PARTITIONS
|
||||||
|
value: "${kafka_num_partitions}"
|
||||||
|
- name: ADVERTISE_PUB_IP
|
||||||
|
value: "${kafka_pub_ip}"
|
||||||
|
- name: POD_NAMESPACE
|
||||||
|
value: "${kafka_namespace}"
|
||||||
|
volumeMounts:
|
||||||
|
- mountPath: "${kafka_log_dir}"
|
||||||
|
name: "broker-data"
|
||||||
|
- mountPath: "/opt/tools"
|
||||||
|
name: "broker-conf"
|
||||||
|
ports:
|
||||||
|
- containerPort: 9092
|
||||||
|
name: "broker-client"
|
||||||
|
protocol: "TCP"
|
||||||
|
imagePullPolicy: "IfNotPresent"
|
||||||
|
restartPolicy: "Always"
|
||||||
|
volumes:
|
||||||
|
- name: "broker-data"
|
||||||
|
emptyDir: {}
|
||||||
|
- name: "broker-conf"
|
||||||
|
emptyDir: {}
|
||||||
15
kubernetes-templates/kafka/2/kafka-service.yml
Normal file
15
kubernetes-templates/kafka/2/kafka-service.yml
Normal file
@ -0,0 +1,15 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: "${kafka_name}"
|
||||||
|
labels:
|
||||||
|
name: "${kafka_name}"
|
||||||
|
spec:
|
||||||
|
ports:
|
||||||
|
- port: 9092
|
||||||
|
targetPort: 9092
|
||||||
|
name: "broker-client"
|
||||||
|
protocol: "TCP"
|
||||||
|
selector:
|
||||||
|
name: "${kafka_name}"
|
||||||
|
clusterIP: "None"
|
||||||
95
kubernetes-templates/kafka/2/rancher-compose.yml
Normal file
95
kubernetes-templates/kafka/2/rancher-compose.yml
Normal file
@ -0,0 +1,95 @@
|
|||||||
|
.catalog:
|
||||||
|
name: Kafka
|
||||||
|
version: 0.10.2.1-k8s
|
||||||
|
description: |
|
||||||
|
(Experimental) Apache Kafka cluster.
|
||||||
|
minimum_rancher_version: v0.59.0
|
||||||
|
maintainer: "Raul Sanchez <rawmind@gmail.com>"
|
||||||
|
uuid: kafka-0
|
||||||
|
questions:
|
||||||
|
- variable: "kafka_name"
|
||||||
|
description: "Name of kafka rc and service."
|
||||||
|
label: "K8s rc name:"
|
||||||
|
required: true
|
||||||
|
default: "kafka"
|
||||||
|
type: "string"
|
||||||
|
- variable: "kafka_namespace"
|
||||||
|
description: "Name of kafka namespace."
|
||||||
|
label: "K8s namespace:"
|
||||||
|
required: true
|
||||||
|
default: "default"
|
||||||
|
type: "string"
|
||||||
|
- variable: "kafka_scale"
|
||||||
|
description: "Number of brokers nodes. Note: Recommended an odd number"
|
||||||
|
label: "Broker Nodes:"
|
||||||
|
required: true
|
||||||
|
default: 3
|
||||||
|
type: "int"
|
||||||
|
- variable: "kafka_mem"
|
||||||
|
description: "Amount of memory to config brokers."
|
||||||
|
label: "Broker Memory (mb):"
|
||||||
|
required: true
|
||||||
|
default: 1024
|
||||||
|
type: "string"
|
||||||
|
- variable: "kafka_log_dir"
|
||||||
|
description: "Broker volume to log data"
|
||||||
|
label: "Broker log volume:"
|
||||||
|
required: true
|
||||||
|
default: "/opt/kafka/logs"
|
||||||
|
type: "string"
|
||||||
|
- variable: "kafka_log_retention"
|
||||||
|
description: "Broker log retention in hours"
|
||||||
|
label: "Broker log retention (h):"
|
||||||
|
required: true
|
||||||
|
default: "168"
|
||||||
|
type: "int"
|
||||||
|
- variable: "kafka_num_partitions"
|
||||||
|
description: "Number of broker partitions"
|
||||||
|
label: "Broker partitions:"
|
||||||
|
required: true
|
||||||
|
default: "1"
|
||||||
|
type: "int"
|
||||||
|
- variable: "kafka_delete_topics"
|
||||||
|
label: "Kafka delete topics:"
|
||||||
|
description: |
|
||||||
|
Enable delete topics in kafka.
|
||||||
|
default: false
|
||||||
|
required: true
|
||||||
|
type: enum
|
||||||
|
options:
|
||||||
|
- false
|
||||||
|
- true
|
||||||
|
- variable: "kafka_interval"
|
||||||
|
description: "Interval to poll/apply configuration changes. 0 to disable"
|
||||||
|
label: "Broker Interval (s):"
|
||||||
|
required: true
|
||||||
|
default: 60
|
||||||
|
type: "int"
|
||||||
|
- variable: "kafka_pub_ip"
|
||||||
|
description: "Advertise local and host public ip"
|
||||||
|
label: "Broker public ip:"
|
||||||
|
default: false
|
||||||
|
required: true
|
||||||
|
type: enum
|
||||||
|
options:
|
||||||
|
- false
|
||||||
|
- true
|
||||||
|
- variable: "zk_link"
|
||||||
|
description: |
|
||||||
|
Choose the Zookeeper service to use.
|
||||||
|
It's really recommended a dedicated zookeeper service just for Kafka.
|
||||||
|
label: "Zookeeper stack/service"
|
||||||
|
default: "default/zookeeper"
|
||||||
|
required: true
|
||||||
|
type: "service"
|
||||||
|
broker:
|
||||||
|
scale: ${kafka_scale}
|
||||||
|
retain_ip: true
|
||||||
|
health_check:
|
||||||
|
port: 9092
|
||||||
|
interval: 5000
|
||||||
|
unhealthy_threshold: 3
|
||||||
|
request_line: ''
|
||||||
|
healthy_threshold: 2
|
||||||
|
response_timeout: 5000
|
||||||
|
|
||||||
@ -1,7 +1,7 @@
|
|||||||
name: Apache Kafka
|
name: Apache Kafka
|
||||||
description: |
|
description: |
|
||||||
(Experimental) Kafka cluster
|
(Experimental) Kafka cluster
|
||||||
version: 0.10.0.1-k8s
|
version: 0.10.2.1-k8s
|
||||||
category: Clustering
|
category: Clustering
|
||||||
maintainer: "Raul Sanchez <rawmind@gmail.com>"
|
maintainer: "Raul Sanchez <rawmind@gmail.com>"
|
||||||
minimum_rancher_version: v0.59.0
|
minimum_rancher_version: v0.59.0
|
||||||
|
|||||||
@ -17,7 +17,7 @@ spec:
|
|||||||
spec:
|
spec:
|
||||||
containers:
|
containers:
|
||||||
- name: "zk-conf"
|
- name: "zk-conf"
|
||||||
image: "rawmind/k8s-zk:3.4.8-10"
|
image: "rawmind/k8s-zk:3.4.9"
|
||||||
volumeMounts:
|
volumeMounts:
|
||||||
- mountPath: "/opt/tools"
|
- mountPath: "/opt/tools"
|
||||||
name: "zk-conf"
|
name: "zk-conf"
|
||||||
@ -40,7 +40,7 @@ spec:
|
|||||||
ports:
|
ports:
|
||||||
imagePullPolicy: "IfNotPresent"
|
imagePullPolicy: "IfNotPresent"
|
||||||
- name: "zk-service"
|
- name: "zk-service"
|
||||||
image: "rawmind/alpine-zk:3.4.9"
|
image: "rawmind/alpine-zk:3.4.9-3"
|
||||||
env:
|
env:
|
||||||
- name: JVMFLAGS
|
- name: JVMFLAGS
|
||||||
value: "-Xmx${zk_mem}m -Xms${zk_mem}m"
|
value: "-Xmx${zk_mem}m -Xms${zk_mem}m"
|
||||||
|
|||||||
@ -21,3 +21,4 @@ spec:
|
|||||||
protocol: "TCP"
|
protocol: "TCP"
|
||||||
selector:
|
selector:
|
||||||
name: "${zk_name}"
|
name: "${zk_name}"
|
||||||
|
|
||||||
|
|||||||
30
templates/kafka/2/README.md
Normal file
30
templates/kafka/2/README.md
Normal file
@ -0,0 +1,30 @@
|
|||||||
|
# Apache Kafka (Experimental)
|
||||||
|
|
||||||
|
### Info:
|
||||||
|
|
||||||
|
This template creates, scale in and scale out a multinodes kafka broker cluster on top of Rancher. The configuration is generated with confd from Rancher metadata.
|
||||||
|
Cluster size are variable after deployment, and get reconfigured after refresh interval.
|
||||||
|
|
||||||
|
|
||||||
|
### Usage:
|
||||||
|
|
||||||
|
Select Apache Kafka from catalog.
|
||||||
|
|
||||||
|
Enter the number of nodes, mem and refresh interval for the kafka cluster.
|
||||||
|
|
||||||
|
Change the following kafka default parameters, if you need:
|
||||||
|
|
||||||
|
- kafka_scale=3 # kafka scale to deploy.
|
||||||
|
- kafka_mem=512 # kafka broker memory.
|
||||||
|
- kafka_log_dir="/opt/kafka/logs" # Kafka log dir.
|
||||||
|
- kafka_log_retention="168" # kafka log retention.
|
||||||
|
- kafka_num_partitions="1" # Kafka partitions number
|
||||||
|
- kafka_delete_topics="false" # kafka delete topics
|
||||||
|
- kafka_pub_ip= < true | false > # Advertise public ip to zookeeper.
|
||||||
|
- zk_link="kafka-zk/zk" # zookeeper stack/service to connect to.
|
||||||
|
|
||||||
|
Click deploy.
|
||||||
|
|
||||||
|
Kafka can now be accessed over the Rancher network.
|
||||||
|
|
||||||
|
Note: When you scale the cluster, zero downtime is not guaranteed..yet..
|
||||||
44
templates/kafka/2/docker-compose.yml
Normal file
44
templates/kafka/2/docker-compose.yml
Normal file
@ -0,0 +1,44 @@
|
|||||||
|
broker:
|
||||||
|
tty: true
|
||||||
|
image: rawmind/alpine-kafka:0.10.2.1
|
||||||
|
volumes_from:
|
||||||
|
- broker-volume
|
||||||
|
- broker-conf
|
||||||
|
environment:
|
||||||
|
- JVMFLAGS=-Xmx${kafka_mem}m -Xms${kafka_mem}m
|
||||||
|
- CONFD_INTERVAL=${kafka_interval}
|
||||||
|
- ZK_SERVICE=${zk_link}
|
||||||
|
- KAFKA_DELETE_TOPICS=${kafka_delete_topics}
|
||||||
|
- KAFKA_LOG_DIRS=${kafka_log_dir}
|
||||||
|
- KAFKA_LOG_RETENTION_HOURS=${kafka_log_retention}
|
||||||
|
- KAFKA_NUM_PARTITIONS=${kafka_num_partitions}
|
||||||
|
- ADVERTISE_PUB_IP=${kafka_pub_ip}
|
||||||
|
external_links:
|
||||||
|
- ${zk_link}:zk
|
||||||
|
labels:
|
||||||
|
io.rancher.scheduler.affinity:container_label_soft_ne: io.rancher.stack_service.name=$${stack_name}/$${service_name}
|
||||||
|
io.rancher.container.hostname_override: container_name
|
||||||
|
io.rancher.sidekicks: broker-volume, broker-conf
|
||||||
|
broker-conf:
|
||||||
|
net: none
|
||||||
|
labels:
|
||||||
|
io.rancher.scheduler.affinity:container_label_soft_ne: io.rancher.stack_service.name=$${stack_name}/$${service_name}
|
||||||
|
io.rancher.container.hostname_override: container_name
|
||||||
|
io.rancher.container.start_once: true
|
||||||
|
image: rawmind/rancher-kafka:0.10.2.0
|
||||||
|
volumes:
|
||||||
|
- /opt/tools
|
||||||
|
broker-volume:
|
||||||
|
net: none
|
||||||
|
labels:
|
||||||
|
io.rancher.scheduler.affinity:container_label_soft_ne: io.rancher.stack_service.name=$${stack_name}/$${service_name}
|
||||||
|
io.rancher.container.hostname_override: container_name
|
||||||
|
io.rancher.container.start_once: true
|
||||||
|
environment:
|
||||||
|
- SERVICE_UID=10003
|
||||||
|
- SERVICE_GID=10003
|
||||||
|
- SERVICE_VOLUME=${kafka_log_dir}
|
||||||
|
volumes:
|
||||||
|
- ${kafka_log_dir}
|
||||||
|
volume_driver: local
|
||||||
|
image: rawmind/alpine-volume:0.0.2-1
|
||||||
80
templates/kafka/2/rancher-compose.yml
Normal file
80
templates/kafka/2/rancher-compose.yml
Normal file
@ -0,0 +1,80 @@
|
|||||||
|
.catalog:
|
||||||
|
name: Kafka
|
||||||
|
version: 0.10.2.1-rancher1
|
||||||
|
description: |
|
||||||
|
(Experimental) Apache Kafka cluster.
|
||||||
|
minimum_rancher_version: v0.59.0
|
||||||
|
maintainer: "Raul Sanchez <rawmind@gmail.com>"
|
||||||
|
uuid: kafka-0
|
||||||
|
questions:
|
||||||
|
- variable: "kafka_scale"
|
||||||
|
description: "Number of brokers nodes. Note: Recommended an odd number"
|
||||||
|
label: "Broker Nodes:"
|
||||||
|
required: true
|
||||||
|
default: 3
|
||||||
|
type: "int"
|
||||||
|
- variable: "kafka_mem"
|
||||||
|
description: "Amount of memory to config brokers."
|
||||||
|
label: "Broker Memory (mb):"
|
||||||
|
required: true
|
||||||
|
default: 1024
|
||||||
|
type: "string"
|
||||||
|
- variable: "kafka_log_dir"
|
||||||
|
description: "Broker volume to log data"
|
||||||
|
label: "Broker log volume:"
|
||||||
|
required: true
|
||||||
|
default: "/opt/kafka/logs"
|
||||||
|
type: "string"
|
||||||
|
- variable: "kafka_log_retention"
|
||||||
|
description: "Broker log retention in hours"
|
||||||
|
label: "Broker log retention (h):"
|
||||||
|
required: true
|
||||||
|
default: "168"
|
||||||
|
type: "int"
|
||||||
|
- variable: "kafka_num_partitions"
|
||||||
|
description: "Number of broker partitions"
|
||||||
|
label: "Broker partitions:"
|
||||||
|
required: true
|
||||||
|
default: "1"
|
||||||
|
type: "int"
|
||||||
|
- variable: "kafka_delete_topics"
|
||||||
|
label: "Kafka delete topics:"
|
||||||
|
description: |
|
||||||
|
Enable delete topics in kafka.
|
||||||
|
default: false
|
||||||
|
required: true
|
||||||
|
type: enum
|
||||||
|
options:
|
||||||
|
- false
|
||||||
|
- true
|
||||||
|
- variable: "kafka_interval"
|
||||||
|
description: "Interval to poll/apply configuration changes. 0 to disable"
|
||||||
|
label: "Broker Interval (s):"
|
||||||
|
required: true
|
||||||
|
default: 60
|
||||||
|
type: "int"
|
||||||
|
- variable: "kafka_pub_ip"
|
||||||
|
description: "Advertise local and host public ip"
|
||||||
|
label: "Broker public ip:"
|
||||||
|
required: false
|
||||||
|
default: false
|
||||||
|
type: "boolean"
|
||||||
|
- variable: "zk_link"
|
||||||
|
description: |
|
||||||
|
Choose the Zookeeper service to use.
|
||||||
|
It's really recommended a dedicated zookeeper service just for Kafka.
|
||||||
|
label: "Zookeeper stack/service"
|
||||||
|
default: "kafka-zk/zk"
|
||||||
|
required: true
|
||||||
|
type: "service"
|
||||||
|
broker:
|
||||||
|
scale: ${kafka_scale}
|
||||||
|
retain_ip: true
|
||||||
|
health_check:
|
||||||
|
port: 9092
|
||||||
|
interval: 5000
|
||||||
|
unhealthy_threshold: 3
|
||||||
|
request_line: ''
|
||||||
|
healthy_threshold: 2
|
||||||
|
response_timeout: 5000
|
||||||
|
|
||||||
@ -1,7 +1,7 @@
|
|||||||
name: Apache Kafka
|
name: Apache Kafka
|
||||||
description: |
|
description: |
|
||||||
(Experimental) Kafka cluster
|
(Experimental) Kafka cluster
|
||||||
version: 0.10.0.1-rancher1
|
version: 0.10.2.1-rancher1
|
||||||
category: Clustering
|
category: Clustering
|
||||||
maintainer: "Raul Sanchez <rawmind@gmail.com>"
|
maintainer: "Raul Sanchez <rawmind@gmail.com>"
|
||||||
minimum_rancher_version: v0.59.0
|
minimum_rancher_version: v0.59.0
|
||||||
|
|||||||
@ -1,6 +1,6 @@
|
|||||||
zk:
|
zk:
|
||||||
tty: true
|
tty: true
|
||||||
image: rawmind/alpine-zk:3.4.9
|
image: rawmind/alpine-zk:3.4.9-3
|
||||||
volumes_from:
|
volumes_from:
|
||||||
- zk-volume
|
- zk-volume
|
||||||
- zk-conf
|
- zk-conf
|
||||||
@ -21,7 +21,7 @@ zk-conf:
|
|||||||
io.rancher.scheduler.affinity:container_label_soft_ne: io.rancher.stack_service.name=$${stack_name}/$${service_name}
|
io.rancher.scheduler.affinity:container_label_soft_ne: io.rancher.stack_service.name=$${stack_name}/$${service_name}
|
||||||
io.rancher.container.hostname_override: container_name
|
io.rancher.container.hostname_override: container_name
|
||||||
io.rancher.container.start_once: true
|
io.rancher.container.start_once: true
|
||||||
image: rawmind/rancher-zk:3.4.8-5
|
image: rawmind/rancher-zk:3.4.9
|
||||||
volumes:
|
volumes:
|
||||||
- /opt/tools
|
- /opt/tools
|
||||||
zk-volume:
|
zk-volume:
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user