Added kafka package for kubernetes. Some bugfixes for cattle packages
This commit is contained in:
Raul Sanchez
@@ -0,0 +1,33 @@
|
||||
# Apache Kafka (Experimental)
|
||||
|
||||
### Info:
|
||||
|
||||
This template creates, scale in and scale out a multinodes kafka broker cluster on top of k8s. The configuration is generated with confd from k8s.
|
||||
Cluster size are variable after deployment, and get reconfigured.
|
||||
|
||||
|
||||
### Usage:
|
||||
|
||||
Select Apache Kafka from catalog.
|
||||
|
||||
Enter the number of nodes, mem and refresh interval for the kafka cluster.
|
||||
|
||||
Change the following kafka default parameters, if you need:
|
||||
|
||||
- kafka_name="kafka" # kafka k8s service and rc name.
|
||||
- kafka_namespace="kafka" # kafka k8s namespace
|
||||
- kafka_scale=3 # kafka scale to deploy.
|
||||
- kafka_mem=512 # kafka broker memory.
|
||||
- kafka_log_dir="/opt/kafka/logs" # Kafka log dir.
|
||||
- kafka_log_retention="168" # kafka log retention.
|
||||
- kafka_num_partitions="1" # Kafka partitions number
|
||||
- kafka_delete_topics="false" # kafka delete topics
|
||||
- kafka_pub_ip= < true | false > # Advertise public ip to zookeeper.
|
||||
- zk_link="kafka-zk/zk" # zookeeper namespace/service to connect to.
|
||||
|
||||
Click deploy.
|
||||
|
||||
Kafka can now be accessed over the Rancher network.
|
||||
|
||||
Note: When you scale the cluster, zero downtime is not guaranteed..yet..
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
.catalog:
|
||||
name: Kafka
|
||||
version: 0.10.0.1-k8s
|
||||
description: |
|
||||
(Experimental) Apache Kafka cluster.
|
||||
minimum_rancher_version: v0.59.0
|
||||
maintainer: "Raul Sanchez <rawmind@gmail.com>"
|
||||
uuid: kafka-0
|
||||
questions:
|
||||
- variable: "kafka_name"
|
||||
description: "Name of kafka rc and service."
|
||||
label: "K8s rc name:"
|
||||
required: true
|
||||
default: "kafka"
|
||||
type: "string"
|
||||
- variable: "kafka_namespace"
|
||||
description: "Name of kafka namespace."
|
||||
label: "K8s namespace:"
|
||||
required: true
|
||||
default: "default"
|
||||
type: "string"
|
||||
- variable: "kafka_scale"
|
||||
description: "Number of brokers nodes. Note: Recommended an odd number"
|
||||
label: "Broker Nodes:"
|
||||
required: true
|
||||
default: 3
|
||||
type: "int"
|
||||
- variable: "kafka_mem"
|
||||
description: "Amount of memory to config brokers."
|
||||
label: "Broker Memory (mb):"
|
||||
required: true
|
||||
default: 1024
|
||||
type: "string"
|
||||
- variable: "kafka_log_dir"
|
||||
description: "Broker volume to log data"
|
||||
label: "Broker log volume:"
|
||||
required: true
|
||||
default: "/opt/kafka/logs"
|
||||
type: "string"
|
||||
- variable: "kafka_log_retention"
|
||||
description: "Broker log retention in hours"
|
||||
label: "Broker log retention (h):"
|
||||
required: true
|
||||
default: "168"
|
||||
type: "int"
|
||||
- variable: "kafka_num_partitions"
|
||||
description: "Number of broker partitions"
|
||||
label: "Broker partitions:"
|
||||
required: true
|
||||
default: "1"
|
||||
type: "int"
|
||||
- variable: "kafka_delete_topics"
|
||||
label: "Kafka delete topics:"
|
||||
description: |
|
||||
Enable delete topics in kafka.
|
||||
default: false
|
||||
required: true
|
||||
type: enum
|
||||
options:
|
||||
- false
|
||||
- true
|
||||
- variable: "kafka_interval"
|
||||
description: "Interval to poll/apply configuration changes. 0 to disable"
|
||||
label: "Broker Interval (s):"
|
||||
required: true
|
||||
default: 60
|
||||
type: "int"
|
||||
- variable: "kafka_pub_ip"
|
||||
description: "Advertise local and host public ip"
|
||||
label: "Broker public ip:"
|
||||
default: false
|
||||
required: true
|
||||
type: enum
|
||||
options:
|
||||
- false
|
||||
- true
|
||||
- variable: "zk_link"
|
||||
description: |
|
||||
Choose the Zookeeper service to use.
|
||||
It's really recommended a dedicated zookeeper service just for Kafka.
|
||||
label: "Zookeeper stack/service"
|
||||
default: "default/zookeeper"
|
||||
required: true
|
||||
type: "service"
|
||||
broker:
|
||||
scale: ${kafka_scale}
|
||||
retain_ip: true
|
||||
health_check:
|
||||
port: 9092
|
||||
interval: 5000
|
||||
unhealthy_threshold: 3
|
||||
request_line: ''
|
||||
healthy_threshold: 2
|
||||
response_timeout: 5000
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
apiVersion: v1
|
||||
kind: ReplicationController
|
||||
metadata:
|
||||
name: "${kafka_name}"
|
||||
namespace: "${kafka_namespace}"
|
||||
labels:
|
||||
name: "${kafka_name}"
|
||||
spec:
|
||||
replicas: ${kafka_scale}
|
||||
selector:
|
||||
name: "${kafka_name}"
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
name: "${kafka_name}"
|
||||
spec:
|
||||
containers:
|
||||
- name: "broker-conf"
|
||||
image: "rawmind/k8s-kafka:0.10.0.0-4"
|
||||
volumeMounts:
|
||||
- mountPath: "/opt/tools"
|
||||
name: "broker-conf"
|
||||
ports:
|
||||
imagePullPolicy: "IfNotPresent"
|
||||
- name: "broker-volume"
|
||||
image: "rawmind/alpine-volume:0.0.2-1"
|
||||
env:
|
||||
- name: SERVICE_UID
|
||||
value: "10003"
|
||||
- name: SERVICE_GID
|
||||
value: "10003"
|
||||
- name: SERVICE_VOLUME
|
||||
value: "${kafka_log_dir}"
|
||||
- name: KEEP_ALIVE
|
||||
value: "1"
|
||||
volumeMounts:
|
||||
- mountPath: "${kafka_log_dir}"
|
||||
name: "broker-data"
|
||||
ports:
|
||||
imagePullPolicy: "IfNotPresent"
|
||||
- name: "broker-service"
|
||||
image: "rawmind/alpine-kafka:0.10.0.1"
|
||||
env:
|
||||
- name: JVMFLAGS
|
||||
value: "-Xmx${kafka_mem}m -Xms${kafka_mem}m"
|
||||
- name: CONFD_INTERVAL
|
||||
value: "${kafka_interval}"
|
||||
- name: KAFKA_ZK_SERVICE
|
||||
value: "${zk_link}"
|
||||
- name: KAFKA_DELETE_TOPICS
|
||||
value: "${kafka_delete_topics}"
|
||||
- name: KAFKA_LOG_DIRS
|
||||
value: "${kafka_log_dir}"
|
||||
- name: KAFKA_LOG_RETENTION_HOURS
|
||||
value: "${kafka_log_retention}"
|
||||
- name: KAFKA_NUM_PARTITIONS
|
||||
value: "${kafka_num_partitions}"
|
||||
- name: ADVERTISE_PUB_IP
|
||||
value: "${kafka_pub_ip}"
|
||||
- name: POD_NAMESPACE
|
||||
value: "${kafka_namespace}"
|
||||
volumeMounts:
|
||||
- mountPath: "${kafka_log_dir}"
|
||||
name: "broker-data"
|
||||
- mountPath: "/opt/tools"
|
||||
name: "broker-conf"
|
||||
ports:
|
||||
- containerPort: 9092
|
||||
name: "broker-client"
|
||||
protocol: "TCP"
|
||||
imagePullPolicy: "IfNotPresent"
|
||||
restartPolicy: "Always"
|
||||
volumes:
|
||||
- name: "broker-data"
|
||||
emptyDir: {}
|
||||
- name: "broker-conf"
|
||||
emptyDir: {}
|
||||
@@ -0,0 +1,15 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: "${kafka_name}"
|
||||
labels:
|
||||
name: "${kafka_name}"
|
||||
spec:
|
||||
ports:
|
||||
- port: 9092
|
||||
targetPort: 9092
|
||||
name: "broker-client"
|
||||
protocol: "TCP"
|
||||
selector:
|
||||
name: "${kafka_name}"
|
||||
clusterIP: "None"
|
||||
Reference in New Issue
Block a user