Added kafka package for kubernetes. Some bugfixes for cattle packages
This commit is contained in:
Raul Sanchez
parent
37dba8043b
commit
7bd642ced5
33
kubernetes-templates/kafka/0/README.md
Normal file
33
kubernetes-templates/kafka/0/README.md
Normal file
@ -0,0 +1,33 @@
|
||||
# Apache Kafka (Experimental)
|
||||
|
||||
### Info:
|
||||
|
||||
This template creates, scale in and scale out a multinodes kafka broker cluster on top of k8s. The configuration is generated with confd from k8s.
|
||||
Cluster size are variable after deployment, and get reconfigured.
|
||||
|
||||
|
||||
### Usage:
|
||||
|
||||
Select Apache Kafka from catalog.
|
||||
|
||||
Enter the number of nodes, mem and refresh interval for the kafka cluster.
|
||||
|
||||
Change the following kafka default parameters, if you need:
|
||||
|
||||
- kafka_name="kafka" # kafka k8s service and rc name.
|
||||
- kafka_namespace="kafka" # kafka k8s namespace
|
||||
- kafka_scale=3 # kafka scale to deploy.
|
||||
- kafka_mem=512 # kafka broker memory.
|
||||
- kafka_log_dir="/opt/kafka/logs" # Kafka log dir.
|
||||
- kafka_log_retention="168" # kafka log retention.
|
||||
- kafka_num_partitions="1" # Kafka partitions number
|
||||
- kafka_delete_topics="false" # kafka delete topics
|
||||
- kafka_pub_ip= < true | false > # Advertise public ip to zookeeper.
|
||||
- zk_link="kafka-zk/zk" # zookeeper namespace/service to connect to.
|
||||
|
||||
Click deploy.
|
||||
|
||||
Kafka can now be accessed over the Rancher network.
|
||||
|
||||
Note: When you scale the cluster, zero downtime is not guaranteed..yet..
|
||||
|
||||
95
kubernetes-templates/kafka/0/rancher-compose.yml
Normal file
95
kubernetes-templates/kafka/0/rancher-compose.yml
Normal file
@ -0,0 +1,95 @@
|
||||
.catalog:
|
||||
name: Kafka
|
||||
version: 0.10.0.1-k8s
|
||||
description: |
|
||||
(Experimental) Apache Kafka cluster.
|
||||
minimum_rancher_version: v0.59.0
|
||||
maintainer: "Raul Sanchez <rawmind@gmail.com>"
|
||||
uuid: kafka-0
|
||||
questions:
|
||||
- variable: "kafka_name"
|
||||
description: "Name of kafka rc and service."
|
||||
label: "K8s rc name:"
|
||||
required: true
|
||||
default: "kafka"
|
||||
type: "string"
|
||||
- variable: "kafka_namespace"
|
||||
description: "Name of kafka namespace."
|
||||
label: "K8s namespace:"
|
||||
required: true
|
||||
default: "default"
|
||||
type: "string"
|
||||
- variable: "kafka_scale"
|
||||
description: "Number of brokers nodes. Note: Recommended an odd number"
|
||||
label: "Broker Nodes:"
|
||||
required: true
|
||||
default: 3
|
||||
type: "int"
|
||||
- variable: "kafka_mem"
|
||||
description: "Amount of memory to config brokers."
|
||||
label: "Broker Memory (mb):"
|
||||
required: true
|
||||
default: 1024
|
||||
type: "string"
|
||||
- variable: "kafka_log_dir"
|
||||
description: "Broker volume to log data"
|
||||
label: "Broker log volume:"
|
||||
required: true
|
||||
default: "/opt/kafka/logs"
|
||||
type: "string"
|
||||
- variable: "kafka_log_retention"
|
||||
description: "Broker log retention in hours"
|
||||
label: "Broker log retention (h):"
|
||||
required: true
|
||||
default: "168"
|
||||
type: "int"
|
||||
- variable: "kafka_num_partitions"
|
||||
description: "Number of broker partitions"
|
||||
label: "Broker partitions:"
|
||||
required: true
|
||||
default: "1"
|
||||
type: "int"
|
||||
- variable: "kafka_delete_topics"
|
||||
label: "Kafka delete topics:"
|
||||
description: |
|
||||
Enable delete topics in kafka.
|
||||
default: false
|
||||
required: true
|
||||
type: enum
|
||||
options:
|
||||
- false
|
||||
- true
|
||||
- variable: "kafka_interval"
|
||||
description: "Interval to poll/apply configuration changes. 0 to disable"
|
||||
label: "Broker Interval (s):"
|
||||
required: true
|
||||
default: 60
|
||||
type: "int"
|
||||
- variable: "kafka_pub_ip"
|
||||
description: "Advertise local and host public ip"
|
||||
label: "Broker public ip:"
|
||||
default: false
|
||||
required: true
|
||||
type: enum
|
||||
options:
|
||||
- false
|
||||
- true
|
||||
- variable: "zk_link"
|
||||
description: |
|
||||
Choose the Zookeeper service to use.
|
||||
It's really recommended a dedicated zookeeper service just for Kafka.
|
||||
label: "Zookeeper stack/service"
|
||||
default: "default/zookeeper"
|
||||
required: true
|
||||
type: "service"
|
||||
broker:
|
||||
scale: ${kafka_scale}
|
||||
retain_ip: true
|
||||
health_check:
|
||||
port: 9092
|
||||
interval: 5000
|
||||
unhealthy_threshold: 3
|
||||
request_line: ''
|
||||
healthy_threshold: 2
|
||||
response_timeout: 5000
|
||||
|
||||
77
kubernetes-templates/kafka/0/zk-rc.yml
Normal file
77
kubernetes-templates/kafka/0/zk-rc.yml
Normal file
@ -0,0 +1,77 @@
|
||||
apiVersion: v1
|
||||
kind: ReplicationController
|
||||
metadata:
|
||||
name: "${kafka_name}"
|
||||
namespace: "${kafka_namespace}"
|
||||
labels:
|
||||
name: "${kafka_name}"
|
||||
spec:
|
||||
replicas: ${kafka_scale}
|
||||
selector:
|
||||
name: "${kafka_name}"
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
name: "${kafka_name}"
|
||||
spec:
|
||||
containers:
|
||||
- name: "broker-conf"
|
||||
image: "rawmind/k8s-kafka:0.10.0.0-4"
|
||||
volumeMounts:
|
||||
- mountPath: "/opt/tools"
|
||||
name: "broker-conf"
|
||||
ports:
|
||||
imagePullPolicy: "IfNotPresent"
|
||||
- name: "broker-volume"
|
||||
image: "rawmind/alpine-volume:0.0.2-1"
|
||||
env:
|
||||
- name: SERVICE_UID
|
||||
value: "10003"
|
||||
- name: SERVICE_GID
|
||||
value: "10003"
|
||||
- name: SERVICE_VOLUME
|
||||
value: "${kafka_log_dir}"
|
||||
- name: KEEP_ALIVE
|
||||
value: "1"
|
||||
volumeMounts:
|
||||
- mountPath: "${kafka_log_dir}"
|
||||
name: "broker-data"
|
||||
ports:
|
||||
imagePullPolicy: "IfNotPresent"
|
||||
- name: "broker-service"
|
||||
image: "rawmind/alpine-kafka:0.10.0.1"
|
||||
env:
|
||||
- name: JVMFLAGS
|
||||
value: "-Xmx${kafka_mem}m -Xms${kafka_mem}m"
|
||||
- name: CONFD_INTERVAL
|
||||
value: "${kafka_interval}"
|
||||
- name: KAFKA_ZK_SERVICE
|
||||
value: "${zk_link}"
|
||||
- name: KAFKA_DELETE_TOPICS
|
||||
value: "${kafka_delete_topics}"
|
||||
- name: KAFKA_LOG_DIRS
|
||||
value: "${kafka_log_dir}"
|
||||
- name: KAFKA_LOG_RETENTION_HOURS
|
||||
value: "${kafka_log_retention}"
|
||||
- name: KAFKA_NUM_PARTITIONS
|
||||
value: "${kafka_num_partitions}"
|
||||
- name: ADVERTISE_PUB_IP
|
||||
value: "${kafka_pub_ip}"
|
||||
- name: POD_NAMESPACE
|
||||
value: "${kafka_namespace}"
|
||||
volumeMounts:
|
||||
- mountPath: "${kafka_log_dir}"
|
||||
name: "broker-data"
|
||||
- mountPath: "/opt/tools"
|
||||
name: "broker-conf"
|
||||
ports:
|
||||
- containerPort: 9092
|
||||
name: "broker-client"
|
||||
protocol: "TCP"
|
||||
imagePullPolicy: "IfNotPresent"
|
||||
restartPolicy: "Always"
|
||||
volumes:
|
||||
- name: "broker-data"
|
||||
emptyDir: {}
|
||||
- name: "broker-conf"
|
||||
emptyDir: {}
|
||||
15
kubernetes-templates/kafka/0/zk-service.yml
Normal file
15
kubernetes-templates/kafka/0/zk-service.yml
Normal file
@ -0,0 +1,15 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: "${kafka_name}"
|
||||
labels:
|
||||
name: "${kafka_name}"
|
||||
spec:
|
||||
ports:
|
||||
- port: 9092
|
||||
targetPort: 9092
|
||||
name: "broker-client"
|
||||
protocol: "TCP"
|
||||
selector:
|
||||
name: "${kafka_name}"
|
||||
clusterIP: "None"
|
||||
28
kubernetes-templates/kafka/catalogIcon-kafka.svg
Normal file
28
kubernetes-templates/kafka/catalogIcon-kafka.svg
Normal file
@ -0,0 +1,28 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!-- Generator: Adobe Illustrator 18.1.1, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
|
||||
<svg version="1.1" id="Typo" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
|
||||
viewBox="0 0 93.5 148.8" enable-background="new 0 0 93.5 148.8" xml:space="preserve">
|
||||
<g>
|
||||
<path d="M80.9,43l-0.1-0.1c-1.2-2.1-3.2-3.5-5.4-4.1c-2.3-0.6-4.7-0.3-6.8,0.9h0c-2.2,1.2-3.6,3.2-4.2,5.5
|
||||
c-0.6,2.2-0.3,4.7,0.9,6.8l0,0.1c1.2,2.1,3.2,3.5,5.4,4.1c2.2,0.6,4.7,0.4,6.8-0.9l0.1-0.1c2.1-1.2,3.5-3.2,4.1-5.4
|
||||
C82.4,47.6,82.1,45.1,80.9,43L80.9,43z M30.6,49.8L30.6,49.8c5.2,0.9,9.9,3.4,13.5,7h0c0.6,0.6,1.1,1.2,1.6,1.8l9-5.2
|
||||
c-1.2-3.6-1.3-7.4-0.3-11c1.3-4.8,4.4-9.1,9-11.7l0.2-0.1c4.6-2.6,9.8-3.1,14.5-1.9c4.8,1.3,9.1,4.4,11.8,9v0
|
||||
c2.7,4.6,3.2,9.9,1.9,14.7c-1.3,4.8-4.4,9.1-9,11.7l-1.2,0.7h-0.1c-4.3,2-9,2.4-13.3,1.2c-3.6-0.9-6.8-2.9-9.4-5.7l-8.9,5.2
|
||||
c1,2.7,1.6,5.7,1.6,8.7c0,3.1-0.6,6.1-1.6,8.8l8.9,5.1c2.6-2.9,5.8-4.8,9.4-5.7c4.8-1.3,10.1-0.8,14.7,1.9l0.3,0.1v0
|
||||
c4.5,2.7,7.5,6.9,8.7,11.6c1.3,4.7,0.7,10-1.9,14.6l-0.2,0.3l0,0l0,0c-2.7,4.5-6.9,7.5-11.6,8.8c-4.8,1.3-10.1,0.7-14.7-1.9v0
|
||||
c-4.6-2.7-7.7-7-9-11.7c-1-3.6-0.9-7.4,0.3-11l-9-5.2c-0.5,0.6-1.1,1.2-1.6,1.8l0,0c-3.6,3.6-8.3,6.1-13.5,6.9v10.4
|
||||
c3.7,0.8,7.1,2.6,9.7,5.2l0,0l0,0c3.5,3.5,5.7,8.3,5.7,13.7c0,5.3-2.2,10.1-5.7,13.6l0,0.1l0,0c-3.5,3.5-8.4,5.7-13.7,5.7
|
||||
c-5.3,0-10.2-2.2-13.7-5.7h0v-0.1c-3.5-3.5-5.7-8.3-5.7-13.6c0-5.3,2.2-10.2,5.7-13.7v0h0c2.6-2.6,5.9-4.5,9.7-5.2V98.8
|
||||
c-5.2-0.8-9.9-3.3-13.5-6.9l0,0c-4.5-4.5-7.3-10.7-7.3-17.5c0-6.8,2.8-13,7.3-17.5h0c3.6-3.6,8.3-6.1,13.5-7V39.5
|
||||
c-3.7-0.8-7-2.7-9.7-5.3h0v0c-3.5-3.5-5.7-8.3-5.7-13.7c0-5.3,2.2-10.2,5.7-13.7l0,0l0,0c3.5-3.5,8.4-5.7,13.7-5.7
|
||||
c5.3,0,10.2,2.2,13.7,5.7v0h0c3.5,3.5,5.7,8.4,5.7,13.7c0,5.3-2.2,10.1-5.7,13.7l0,0c-2.6,2.6-5.9,4.5-9.7,5.3V49.8L30.6,49.8z
|
||||
M35.8,65.1L35.8,65.1c-2.4-2.4-5.6-3.8-9.2-3.8c-3.6,0-6.8,1.5-9.2,3.8c-2.3,2.3-3.8,5.6-3.8,9.2c0,3.6,1.4,6.9,3.8,9.2
|
||||
c2.4,2.3,5.6,3.8,9.2,3.8c3.6,0,6.8-1.5,9.2-3.8c2.3-2.4,3.8-5.6,3.8-9.2C39.5,70.7,38.1,67.5,35.8,65.1L35.8,65.1z M75.4,109.9
|
||||
L75.4,109.9c2.2-0.6,4.2-2,5.5-4.2l0.2-0.3l0,0c1.1-2,1.3-4.4,0.7-6.5c-0.6-2.2-2-4.2-4.2-5.5l-0.1-0.1c-2.1-1.1-4.5-1.4-6.7-0.8
|
||||
c-2.2,0.6-4.2,2-5.5,4.2c-1.2,2.1-1.5,4.6-0.9,6.8c0.6,2.2,2,4.2,4.2,5.5h0C70.8,110.2,73.2,110.5,75.4,109.9L75.4,109.9z
|
||||
M26.6,137.1L26.6,137.1c2.5,0,4.7-1,6.4-2.6l0,0c1.6-1.6,2.6-3.8,2.6-6.3c0-2.5-1-4.7-2.6-6.4h0c-1.6-1.6-3.9-2.6-6.4-2.6
|
||||
c-2.5,0-4.7,1-6.3,2.6h0c-1.6,1.6-2.6,3.9-2.6,6.4c0,2.5,1,4.7,2.6,6.3l0,0C21.8,136,24.1,137.1,26.6,137.1L26.6,137.1z M32.9,14.2
|
||||
L32.9,14.2c-1.6-1.6-3.9-2.6-6.4-2.6c-2.5,0-4.7,1-6.3,2.6h0c-1.6,1.6-2.6,3.9-2.6,6.4c0,2.5,1,4.7,2.6,6.3l0,0
|
||||
c1.6,1.6,3.8,2.6,6.3,2.6c2.5,0,4.7-1,6.4-2.6l0,0c1.6-1.6,2.6-3.8,2.6-6.3C35.5,18.1,34.5,15.8,32.9,14.2L32.9,14.2z"/>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 2.8 KiB |
9
kubernetes-templates/kafka/config.yml
Normal file
9
kubernetes-templates/kafka/config.yml
Normal file
@ -0,0 +1,9 @@
|
||||
name: Apache Kafka
|
||||
description: |
|
||||
(Experimental) Kafka cluster
|
||||
version: 0.10.0.1-k8s
|
||||
category: Clustering
|
||||
maintainer: "Raul Sanchez <rawmind@gmail.com>"
|
||||
minimum_rancher_version: v0.59.0
|
||||
license:
|
||||
projectURL: https://github.com/rawmind0/alpine-kafka
|
||||
@ -14,12 +14,14 @@
|
||||
|
||||
Change the following kafka default parameters, if you need:
|
||||
|
||||
- KAFKA_LOG_DIRS="/opt/kafka/logs"
|
||||
- KAFKA_LOG_RETENTION_HOURS="168"
|
||||
- KAFKA_NUM_PARTITIONS="1"
|
||||
- ADVERTISE_PUB_IP= < true | false >
|
||||
|
||||
Select zookeeper stack/service to connect to.
|
||||
- kafka_scale=3 # kafka scale to deploy.
|
||||
- kafka_mem=512 # kafka broker memory.
|
||||
- kafka_log_dir="/opt/kafka/logs" # Kafka log dir.
|
||||
- kafka_log_retention="168" # kafka log retention.
|
||||
- kafka_num_partitions="1" # Kafka partitions number
|
||||
- kafka_delete_topics="false" # kafka delete topics
|
||||
- kafka_pub_ip= < true | false > # Advertise public ip to zookeeper.
|
||||
- zk_link="kafka-zk/zk" # zookeeper stack/service to connect to.
|
||||
|
||||
Click deploy.
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
broker:
|
||||
tty: true
|
||||
image: rawmind/alpine-kafka:0.10.0.0-2
|
||||
image: rawmind/alpine-kafka:0.10.0.1
|
||||
volumes_from:
|
||||
- broker-volume
|
||||
- broker-conf
|
||||
@ -8,6 +8,7 @@ broker:
|
||||
- JVMFLAGS=-Xmx${kafka_mem}m -Xms${kafka_mem}m
|
||||
- CONFD_INTERVAL=${kafka_interval}
|
||||
- ZK_SERVICE=${zk_link}
|
||||
- KAFKA_DELETE_TOPICS=${kafka_delete_topics}
|
||||
- KAFKA_LOG_DIRS=${kafka_log_dir}
|
||||
- KAFKA_LOG_RETENTION_HOURS=${kafka_log_retention}
|
||||
- KAFKA_NUM_PARTITIONS=${kafka_num_partitions}
|
||||
@ -24,7 +25,9 @@ broker-conf:
|
||||
io.rancher.scheduler.affinity:container_label_soft_ne: io.rancher.stack_service.name=$${stack_name}/$${service_name}
|
||||
io.rancher.container.hostname_override: container_name
|
||||
io.rancher.container.start_once: true
|
||||
image: rawmind/rancher-kafka:0.10.0.0
|
||||
image: rawmind/rancher-kafka:0.10.0.0-3
|
||||
volumes:
|
||||
- /opt/tools
|
||||
broker-volume:
|
||||
net: none
|
||||
labels:
|
||||
@ -32,12 +35,10 @@ broker-volume:
|
||||
io.rancher.container.hostname_override: container_name
|
||||
io.rancher.container.start_once: true
|
||||
environment:
|
||||
- SERVICE_USER=kafka
|
||||
- SERVICE_UID=10003
|
||||
- SERVICE_GROUP=kafka
|
||||
- SERVICE_GID=10003
|
||||
- SERVICE_VOLUME=${kafka_log_dir}
|
||||
volumes:
|
||||
- ${kafka_log_dir}
|
||||
volume_driver: local
|
||||
image: rawmind/alpine-volume:0.0.1-1
|
||||
image: rawmind/alpine-volume:0.0.2-1
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
.catalog:
|
||||
name: Kafka
|
||||
version: 0.10.0.0-rancher1
|
||||
version: 0.10.0.1-rancher1
|
||||
description: |
|
||||
(Experimental) Apache Kafka cluster.
|
||||
minimum_rancher_version: v0.59.0
|
||||
@ -37,6 +37,16 @@
|
||||
required: true
|
||||
default: "1"
|
||||
type: "int"
|
||||
- variable: "kafka_delete_topics"
|
||||
label: "Kafka delete topics:"
|
||||
description: |
|
||||
Enable delete topics in kafka.
|
||||
default: false
|
||||
required: true
|
||||
type: enum
|
||||
options:
|
||||
- false
|
||||
- true
|
||||
- variable: "kafka_interval"
|
||||
description: "Interval to poll/apply configuration changes. 0 to disable"
|
||||
label: "Broker Interval (s):"
|
||||
@ -44,11 +54,14 @@
|
||||
default: 60
|
||||
type: "int"
|
||||
- variable: "kafka_pub_ip"
|
||||
description: "Advertise host ip in zookeeper"
|
||||
label: "Advertise host ip:"
|
||||
required: false
|
||||
description: "Advertise local and host public ip"
|
||||
label: "Broker public ip:"
|
||||
default: false
|
||||
type: "boolean"
|
||||
required: true
|
||||
type: enum
|
||||
options:
|
||||
- false
|
||||
- true
|
||||
- variable: "zk_link"
|
||||
description: |
|
||||
Choose the Zookeeper service to use.
|
||||
|
||||
@ -1,8 +1,9 @@
|
||||
name: Apache Kafka
|
||||
description: |
|
||||
(Experimental) Kafka cluster
|
||||
version: 0.10.0.0-rancher1
|
||||
version: 0.10.0.1-rancher1
|
||||
category: Clustering
|
||||
maintainer: "Raul Sanchez <rawmind@gmail.com>"
|
||||
minimum_rancher_version: v0.59.0
|
||||
license:
|
||||
projectURL: https://github.com/rawmind0/alpine-kafka
|
||||
Loading…
x
Reference in New Issue
Block a user