flywithu/community-catalog
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #300 from rawmind0/master

Browse Source 
Updates for traefik, kafka and zookeeper packages
This commit is contained in:
Raúl Sánchez
2016-11-03 11:53:11 +01:00
committed by GitHub
parent 1f7edb1b4f 7bd642ced5
commit 147ff3b67d
26 changed files with 893 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kubernetes-templates/kafka/0/README.md
+33
View File
@@ -0,0 +1,33 @@
# Apache Kafka (Experimental)
### Info:
This template creates, scale in and scale out a multinodes kafka broker cluster on top of k8s. The configuration is generated with confd from k8s.
Cluster size are variable after deployment, and get reconfigured.
### Usage:
Select Apache Kafka from catalog.
Enter the number of nodes, mem and refresh interval for the kafka cluster.
Change the following kafka default parameters, if you need:
- kafka_name="kafka" # kafka k8s service and rc name.
- kafka_namespace="kafka" # kafka k8s namespace
- kafka_scale=3 # kafka scale to deploy.
- kafka_mem=512 # kafka broker memory.
- kafka_log_dir="/opt/kafka/logs" # Kafka log dir.
- kafka_log_retention="168" # kafka log retention.
- kafka_num_partitions="1" # Kafka partitions number
- kafka_delete_topics="false" # kafka delete topics
- kafka_pub_ip= < true | false > # Advertise public ip to zookeeper.
- zk_link="kafka-zk/zk" # zookeeper namespace/service to connect to.
Click deploy.
Kafka can now be accessed over the Rancher network.
Note: When you scale the cluster, zero downtime is not guaranteed..yet..
kubernetes-templates/kafka/0/rancher-compose.yml
+95
View File
@@ -0,0 +1,95 @@
.catalog:
name: Kafka
version: 0.10.0.1-k8s
description: |
(Experimental) Apache Kafka cluster.
minimum_rancher_version: v0.59.0
maintainer: "Raul Sanchez <rawmind@gmail.com>"
uuid: kafka-0
questions:
- variable: "kafka_name"
description: "Name of kafka rc and service."
label: "K8s rc name:"
required: true
default: "kafka"
type: "string"
- variable: "kafka_namespace"
description: "Name of kafka namespace."
label: "K8s namespace:"
required: true
default: "default"
type: "string"
- variable: "kafka_scale"
description: "Number of brokers nodes. Note: Recommended an odd number"
label: "Broker Nodes:"
required: true
default: 3
type: "int"
- variable: "kafka_mem"
description: "Amount of memory to config brokers."
label: "Broker Memory (mb):"
required: true
default: 1024
type: "string"
- variable: "kafka_log_dir"
description: "Broker volume to log data"
label: "Broker log volume:"
required: true
default: "/opt/kafka/logs"
type: "string"
- variable: "kafka_log_retention"
description: "Broker log retention in hours"
label: "Broker log retention (h):"
required: true
default: "168"
type: "int"
- variable: "kafka_num_partitions"
description: "Number of broker partitions"
label: "Broker partitions:"
required: true
default: "1"
type: "int"
- variable: "kafka_delete_topics"
label: "Kafka delete topics:"
description: |
Enable delete topics in kafka.
default: false
required: true
type: enum
options:
- false
- true
- variable: "kafka_interval"
description: "Interval to poll/apply configuration changes. 0 to disable"
label: "Broker Interval (s):"
required: true
default: 60
type: "int"
- variable: "kafka_pub_ip"
description: "Advertise local and host public ip"
label: "Broker public ip:"
default: false
required: true
type: enum
options:
- false
- true
- variable: "zk_link"
description: |
Choose the Zookeeper service to use.
It's really recommended a dedicated zookeeper service just for Kafka.
label: "Zookeeper stack/service"
default: "default/zookeeper"
required: true
type: "service"
broker:
scale: ${kafka_scale}
retain_ip: true
health_check:
port: 9092
interval: 5000
unhealthy_threshold: 3
request_line: ''
healthy_threshold: 2
response_timeout: 5000
kubernetes-templates/kafka/0/zk-rc.yml
+77
View File
@@ -0,0 +1,77 @@
apiVersion: v1
kind: ReplicationController
metadata:
name: "${kafka_name}"
namespace: "${kafka_namespace}"
labels:
name: "${kafka_name}"
spec:
replicas: ${kafka_scale}
selector:
name: "${kafka_name}"
template:
metadata:
labels:
name: "${kafka_name}"
spec:
containers:
- name: "broker-conf"
image: "rawmind/k8s-kafka:0.10.0.0-4"
volumeMounts:
- mountPath: "/opt/tools"
name: "broker-conf"
ports:
imagePullPolicy: "IfNotPresent"
- name: "broker-volume"
image: "rawmind/alpine-volume:0.0.2-1"
env:
- name: SERVICE_UID
value: "10003"
- name: SERVICE_GID
value: "10003"
- name: SERVICE_VOLUME
value: "${kafka_log_dir}"
- name: KEEP_ALIVE
value: "1"
volumeMounts:
- mountPath: "${kafka_log_dir}"
name: "broker-data"
ports:
imagePullPolicy: "IfNotPresent"
- name: "broker-service"
image: "rawmind/alpine-kafka:0.10.0.1"
env:
- name: JVMFLAGS
value: "-Xmx${kafka_mem}m -Xms${kafka_mem}m"
- name: CONFD_INTERVAL
value: "${kafka_interval}"
- name: KAFKA_ZK_SERVICE
value: "${zk_link}"
- name: KAFKA_DELETE_TOPICS
value: "${kafka_delete_topics}"
- name: KAFKA_LOG_DIRS
value: "${kafka_log_dir}"
- name: KAFKA_LOG_RETENTION_HOURS
value: "${kafka_log_retention}"
- name: KAFKA_NUM_PARTITIONS
value: "${kafka_num_partitions}"
- name: ADVERTISE_PUB_IP
value: "${kafka_pub_ip}"
- name: POD_NAMESPACE
value: "${kafka_namespace}"
volumeMounts:
- mountPath: "${kafka_log_dir}"
name: "broker-data"
- mountPath: "/opt/tools"
name: "broker-conf"
ports:
- containerPort: 9092
name: "broker-client"
protocol: "TCP"
imagePullPolicy: "IfNotPresent"
restartPolicy: "Always"
volumes:
- name: "broker-data"
emptyDir: {}
- name: "broker-conf"
emptyDir: {}
kubernetes-templates/kafka/0/zk-service.yml
+15
View File
@@ -0,0 +1,15 @@
apiVersion: v1
kind: Service
metadata:
name: "${kafka_name}"
labels:
name: "${kafka_name}"
spec:
ports:
- port: 9092
targetPort: 9092
name: "broker-client"
protocol: "TCP"
selector:
name: "${kafka_name}"
clusterIP: "None"
kubernetes-templates/kafka/catalogIcon-kafka.svg
+28
View File
@@ -0,0 +1,28 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Generator: Adobe Illustrator 18.1.1, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
<svg version="1.1" id="Typo" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
viewBox="0 0 93.5 148.8" enable-background="new 0 0 93.5 148.8" xml:space="preserve">
<g>
<path d="M80.9,43l-0.1-0.1c-1.2-2.1-3.2-3.5-5.4-4.1c-2.3-0.6-4.7-0.3-6.8,0.9h0c-2.2,1.2-3.6,3.2-4.2,5.5
c-0.6,2.2-0.3,4.7,0.9,6.8l0,0.1c1.2,2.1,3.2,3.5,5.4,4.1c2.2,0.6,4.7,0.4,6.8-0.9l0.1-0.1c2.1-1.2,3.5-3.2,4.1-5.4
C82.4,47.6,82.1,45.1,80.9,43L80.9,43z M30.6,49.8L30.6,49.8c5.2,0.9,9.9,3.4,13.5,7h0c0.6,0.6,1.1,1.2,1.6,1.8l9-5.2
c-1.2-3.6-1.3-7.4-0.3-11c1.3-4.8,4.4-9.1,9-11.7l0.2-0.1c4.6-2.6,9.8-3.1,14.5-1.9c4.8,1.3,9.1,4.4,11.8,9v0
c2.7,4.6,3.2,9.9,1.9,14.7c-1.3,4.8-4.4,9.1-9,11.7l-1.2,0.7h-0.1c-4.3,2-9,2.4-13.3,1.2c-3.6-0.9-6.8-2.9-9.4-5.7l-8.9,5.2
c1,2.7,1.6,5.7,1.6,8.7c0,3.1-0.6,6.1-1.6,8.8l8.9,5.1c2.6-2.9,5.8-4.8,9.4-5.7c4.8-1.3,10.1-0.8,14.7,1.9l0.3,0.1v0
c4.5,2.7,7.5,6.9,8.7,11.6c1.3,4.7,0.7,10-1.9,14.6l-0.2,0.3l0,0l0,0c-2.7,4.5-6.9,7.5-11.6,8.8c-4.8,1.3-10.1,0.7-14.7-1.9v0
c-4.6-2.7-7.7-7-9-11.7c-1-3.6-0.9-7.4,0.3-11l-9-5.2c-0.5,0.6-1.1,1.2-1.6,1.8l0,0c-3.6,3.6-8.3,6.1-13.5,6.9v10.4
c3.7,0.8,7.1,2.6,9.7,5.2l0,0l0,0c3.5,3.5,5.7,8.3,5.7,13.7c0,5.3-2.2,10.1-5.7,13.6l0,0.1l0,0c-3.5,3.5-8.4,5.7-13.7,5.7
c-5.3,0-10.2-2.2-13.7-5.7h0v-0.1c-3.5-3.5-5.7-8.3-5.7-13.6c0-5.3,2.2-10.2,5.7-13.7v0h0c2.6-2.6,5.9-4.5,9.7-5.2V98.8
c-5.2-0.8-9.9-3.3-13.5-6.9l0,0c-4.5-4.5-7.3-10.7-7.3-17.5c0-6.8,2.8-13,7.3-17.5h0c3.6-3.6,8.3-6.1,13.5-7V39.5
c-3.7-0.8-7-2.7-9.7-5.3h0v0c-3.5-3.5-5.7-8.3-5.7-13.7c0-5.3,2.2-10.2,5.7-13.7l0,0l0,0c3.5-3.5,8.4-5.7,13.7-5.7
c5.3,0,10.2,2.2,13.7,5.7v0h0c3.5,3.5,5.7,8.4,5.7,13.7c0,5.3-2.2,10.1-5.7,13.7l0,0c-2.6,2.6-5.9,4.5-9.7,5.3V49.8L30.6,49.8z
M35.8,65.1L35.8,65.1c-2.4-2.4-5.6-3.8-9.2-3.8c-3.6,0-6.8,1.5-9.2,3.8c-2.3,2.3-3.8,5.6-3.8,9.2c0,3.6,1.4,6.9,3.8,9.2
c2.4,2.3,5.6,3.8,9.2,3.8c3.6,0,6.8-1.5,9.2-3.8c2.3-2.4,3.8-5.6,3.8-9.2C39.5,70.7,38.1,67.5,35.8,65.1L35.8,65.1z M75.4,109.9
L75.4,109.9c2.2-0.6,4.2-2,5.5-4.2l0.2-0.3l0,0c1.1-2,1.3-4.4,0.7-6.5c-0.6-2.2-2-4.2-4.2-5.5l-0.1-0.1c-2.1-1.1-4.5-1.4-6.7-0.8
c-2.2,0.6-4.2,2-5.5,4.2c-1.2,2.1-1.5,4.6-0.9,6.8c0.6,2.2,2,4.2,4.2,5.5h0C70.8,110.2,73.2,110.5,75.4,109.9L75.4,109.9z
M26.6,137.1L26.6,137.1c2.5,0,4.7-1,6.4-2.6l0,0c1.6-1.6,2.6-3.8,2.6-6.3c0-2.5-1-4.7-2.6-6.4h0c-1.6-1.6-3.9-2.6-6.4-2.6
c-2.5,0-4.7,1-6.3,2.6h0c-1.6,1.6-2.6,3.9-2.6,6.4c0,2.5,1,4.7,2.6,6.3l0,0C21.8,136,24.1,137.1,26.6,137.1L26.6,137.1z M32.9,14.2
L32.9,14.2c-1.6-1.6-3.9-2.6-6.4-2.6c-2.5,0-4.7,1-6.3,2.6h0c-1.6,1.6-2.6,3.9-2.6,6.4c0,2.5,1,4.7,2.6,6.3l0,0
c1.6,1.6,3.8,2.6,6.3,2.6c2.5,0,4.7-1,6.4-2.6l0,0c1.6-1.6,2.6-3.8,2.6-6.3C35.5,18.1,34.5,15.8,32.9,14.2L32.9,14.2z"/>
</g>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 2.8 KiB

kubernetes-templates/kafka/config.yml
+9
View File
@@ -0,0 +1,9 @@
name: Apache Kafka
description: |
(Experimental) Kafka cluster
version: 0.10.0.1-k8s
category: Clustering
maintainer: "Raul Sanchez <rawmind@gmail.com>"
minimum_rancher_version: v0.59.0
license:
projectURL: https://github.com/rawmind0/alpine-kafka
kubernetes-templates/zookeeper/0/README.md
+31
View File
@@ -0,0 +1,31 @@
# Apache Zookeeper (Experimental)
### Info:
This template creates, scale in and scale out a multinodes zk (zookeeper) cluster on top of k8s. The configuration is generated with confd watching k8s metadata.
Cluster size are variable after deployment, and get reconfigured if you scale.
The services generates ZKID's (saving them as pod labels) and start the zookeeper cluster.
### Usage:
Select Apache Zookeeper from catalog.
Change the following zookeeper default parameters, if you need:
- zk_name="zookeeper" # Name of the k8s rc and service.
- zk_namespace="default" # Name of the k8s namespace
- zk_scale="3" # Number of zk replicas
- zk_mem="512" # Mem to configure zk.
- zk_data_dir="/opt/zk/data" # Zk dataDir param value
- zk_init_limit="10" # zk initLimit param value
- zk_max_client_cxns="500" # zk maxClientCnxns param value
- zk_sync_limit="5" # zk syncLimit param value
- zk_tick_time="2000" # zk tickTime param value
Click deploy.
Zookeeper can now be accessed over the Rancher network.
Testing Feature: When you scale the cluster, zero downtime is expected...
kubernetes-templates/zookeeper/0/rancher-compose.yml
+73
View File
@@ -0,0 +1,73 @@
.catalog:
name: Zookeeper
version: 3.4.8-k8s1
description: |
(Experimental) Apache Zookeeper cluster.
minimum_rancher_version: v0.59.0
maintainer: "Raul Sanchez <rawmind@gmail.com>"
uuid: zk-0
questions:
- variable: "zk_name"
description: "Name of zookeeper rc and service."
label: "K8s rc name:"
required: true
default: "zookeeper"
type: "string"
- variable: "zk_namespace"
description: "Name of zookeeper namespace."
label: "K8s namespace:"
required: true
default: "default"
type: "string"
- variable: "zk_scale"
description: "Number of zk nodes. Note: Recommended an odd number"
label: "Zk Nodes:"
required: true
default: 3
type: "int"
- variable: "zk_mem"
description: "Amount of memory to config zk."
label: "Zk Memory (mb):"
required: true
default: 512
type: "int"
- variable: "zk_init_limit"
description: "Time to allow followers to connect and sync with leader"
label: "Zk init limit (ticks):"
required: true
default: 10
type: "int"
- variable: "zk_data_dir"
description: "Directory where zookeeper store data"
label: "Zk data dir:"
required: true
default: "/opt/zk/data"
type: "string"
- variable: "zk_max_client_cxns"
description: "Max client concurrent connections"
label: "Zk max client cxns:"
required: true
default: 500
type: "int"
- variable: "zk_sync_limit"
description: "Time to allow followers to sync with leader"
label: "Zk sync limit (ticks):"
required: true
default: 5
type: "int"
- variable: "zk_tick_time"
description: "Tick time length"
label: "Zk tick time (ms):"
required: true
default: 2000
type: "int"
zk:
scale: ${zk_scale}
retain_ip: true
health_check:
port: 2181
interval: 5000
unhealthy_threshold: 3
request_line: ''
healthy_threshold: 2
response_timeout: 5000
kubernetes-templates/zookeeper/0/zk-rc.yml
+80
View File
@@ -0,0 +1,80 @@
apiVersion: v1
kind: ReplicationController
metadata:
name: "${zk_name}"
namespace: "${zk_namespace}"
labels:
name: "${zk_name}"
spec:
replicas: ${zk_scale}
selector:
name: "${zk_name}"
template:
metadata:
labels:
name: "${zk_name}"
annotations:
spec:
containers:
- name: "zk-conf"
image: "rawmind/k8s-zk:3.4.8-10"
volumeMounts:
- mountPath: "/opt/tools"
name: "zk-conf"
ports:
imagePullPolicy: "IfNotPresent"
- name: "zk-volume"
image: "rawmind/alpine-volume:0.0.2-1"
env:
- name: SERVICE_UID
value: "10002"
- name: SERVICE_GID
value: "10002"
- name: SERVICE_VOLUME
value: "${kafka_log_dir}"
- name: KEEP_ALIVE
value: "1"
volumeMounts:
- mountPath: "${zk_data_dir}"
name: "zk-data"
ports:
imagePullPolicy: "IfNotPresent"
- name: "zk-service"
image: "rawmind/alpine-zk:3.4.8-16"
env:
- name: JVMFLAGS
value: "-Xmx${zk_mem}m -Xms${zk_mem}m"
- name: ZK_DATA_DIR
value: "${zk_data_dir}"
- name: ZK_INIT_LIMIT
value: "${zk_init_limit}"
- name: ZK_MAX_CLIENT_CXNS
value: "${zk_max_client_cxns}"
- name: ZK_SYNC_LIMIT
value: "${zk_sync_limit}"
- name: ZK_TICK_TIME
value: "${zk_tick_time}"
- name: POD_NAMESPACE
value: "${zk_namespace}"
volumeMounts:
- mountPath: "${zk_data_dir}"
name: "zk-data"
- mountPath: "/opt/tools"
name: "zk-conf"
ports:
- containerPort: 2181
name: "zk-client"
protocol: "TCP"
- containerPort: 2888
name: "zk-server"
protocol: "TCP"
- containerPort: 3888
name: "zk-leader"
protocol: "TCP"
imagePullPolicy: "IfNotPresent"
restartPolicy: "Always"
volumes:
- name: "zk-data"
emptyDir: {}
- name: "zk-conf"
emptyDir: {}
kubernetes-templates/zookeeper/0/zk-service.yml
+23
View File
@@ -0,0 +1,23 @@
apiVersion: v1
kind: Service
metadata:
name: "${zk_name}"
namespace: "${zk_namespace}"
labels:
name: "${zk_name}"
spec:
ports:
- port: 2181
targetPort: 2181
name: "zk-client"
protocol: "TCP"
- port: 2888
targetPort: 2888
name: "zk-server"
protocol: "TCP"
- port: 3888
targetPort: 3888
name: "zk-leader"
protocol: "TCP"
selector:
name: "${zk_name}"
kubernetes-templates/zookeeper/catalogIcon-zookeeper.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 42 KiB

kubernetes-templates/zookeeper/config.yml
+7
View File
@@ -0,0 +1,7 @@
name: Apache Zookeeper
description: |
(Experimental) Zookeeper cluster
version: 3.4.8-k8s1
category: Clustering
maintainer: "Raul Sanchez <rawmind@gmail.com>"
projectURL: https://github.com/rawmind0/alpine-zk