diff --git a/kubernetes-templates/kafka/0/zk-rc.yml b/kubernetes-templates/kafka/0/kafka-rc.yml similarity index 100% rename from kubernetes-templates/kafka/0/zk-rc.yml rename to kubernetes-templates/kafka/0/kafka-rc.yml diff --git a/kubernetes-templates/kafka/0/zk-service.yml b/kubernetes-templates/kafka/0/kafka-service.yml similarity index 100% rename from kubernetes-templates/kafka/0/zk-service.yml rename to kubernetes-templates/kafka/0/kafka-service.yml diff --git a/kubernetes-templates/kafka/1/README.md b/kubernetes-templates/kafka/1/README.md new file mode 100644 index 0000000..1bab12c --- /dev/null +++ b/kubernetes-templates/kafka/1/README.md @@ -0,0 +1,33 @@ +# Apache Kafka (Experimental) + +### Info: + + This template creates, scale in and scale out a multinodes kafka broker cluster on top of k8s. The configuration is generated with confd from k8s. + Cluster size are variable after deployment, and get reconfigured. + + +### Usage: + + Select Apache Kafka from catalog. + + Enter the number of nodes, mem and refresh interval for the kafka cluster. + + Change the following kafka default parameters, if you need: + +- kafka_name="kafka" # kafka k8s service and rc name. +- kafka_namespace="kafka" # kafka k8s namespace +- kafka_scale=3 # kafka scale to deploy. +- kafka_mem=512 # kafka broker memory. +- kafka_log_dir="/opt/kafka/logs" # Kafka log dir. +- kafka_log_retention="168" # kafka log retention. +- kafka_num_partitions="1" # Kafka partitions number +- kafka_delete_topics="false" # kafka delete topics +- kafka_pub_ip= < true | false > # Advertise public ip to zookeeper. +- zk_link="kafka-zk/zk" # zookeeper namespace/service to connect to. + + Click deploy. + + Kafka can now be accessed over the Rancher network. + + Note: When you scale the cluster, zero downtime is not guaranteed..yet.. + diff --git a/kubernetes-templates/kafka/1/kafka-rc.yml b/kubernetes-templates/kafka/1/kafka-rc.yml new file mode 100644 index 0000000..25d6139 --- /dev/null +++ b/kubernetes-templates/kafka/1/kafka-rc.yml @@ -0,0 +1,77 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: "${kafka_name}" + namespace: "${kafka_namespace}" + labels: + name: "${kafka_name}" +spec: + replicas: ${kafka_scale} + selector: + name: "${kafka_name}" + template: + metadata: + labels: + name: "${kafka_name}" + spec: + containers: + - name: "broker-conf" + image: "rawmind/k8s-kafka:0.10.2.0-1" + volumeMounts: + - mountPath: "/opt/tools" + name: "broker-conf" + ports: + imagePullPolicy: "IfNotPresent" + - name: "broker-volume" + image: "rawmind/alpine-volume:0.0.2-1" + env: + - name: SERVICE_UID + value: "10003" + - name: SERVICE_GID + value: "10003" + - name: SERVICE_VOLUME + value: "${kafka_log_dir}" + - name: KEEP_ALIVE + value: "1" + volumeMounts: + - mountPath: "${kafka_log_dir}" + name: "broker-data" + ports: + imagePullPolicy: "IfNotPresent" + - name: "broker-service" + image: "rawmind/alpine-kafka:0.10.2.0-2" + env: + - name: JVMFLAGS + value: "-Xmx${kafka_mem}m -Xms${kafka_mem}m" + - name: CONFD_INTERVAL + value: "${kafka_interval}" + - name: KAFKA_ZK_SERVICE + value: "${zk_link}" + - name: KAFKA_DELETE_TOPICS + value: "${kafka_delete_topics}" + - name: KAFKA_LOG_DIRS + value: "${kafka_log_dir}" + - name: KAFKA_LOG_RETENTION_HOURS + value: "${kafka_log_retention}" + - name: KAFKA_NUM_PARTITIONS + value: "${kafka_num_partitions}" + - name: ADVERTISE_PUB_IP + value: "${kafka_pub_ip}" + - name: POD_NAMESPACE + value: "${kafka_namespace}" + volumeMounts: + - mountPath: "${kafka_log_dir}" + name: "broker-data" + - mountPath: "/opt/tools" + name: "broker-conf" + ports: + - containerPort: 9092 + name: "broker-client" + protocol: "TCP" + imagePullPolicy: "IfNotPresent" + restartPolicy: "Always" + volumes: + - name: "broker-data" + emptyDir: {} + - name: "broker-conf" + emptyDir: {} \ No newline at end of file diff --git a/kubernetes-templates/kafka/1/kafka-service.yml b/kubernetes-templates/kafka/1/kafka-service.yml new file mode 100644 index 0000000..0a6371a --- /dev/null +++ b/kubernetes-templates/kafka/1/kafka-service.yml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: "${kafka_name}" + labels: + name: "${kafka_name}" +spec: + ports: + - port: 9092 + targetPort: 9092 + name: "broker-client" + protocol: "TCP" + selector: + name: "${kafka_name}" + clusterIP: "None" diff --git a/kubernetes-templates/kafka/1/rancher-compose.yml b/kubernetes-templates/kafka/1/rancher-compose.yml new file mode 100644 index 0000000..1dda912 --- /dev/null +++ b/kubernetes-templates/kafka/1/rancher-compose.yml @@ -0,0 +1,95 @@ +.catalog: + name: Kafka + version: 0.10.2.0-k8s + description: | + (Experimental) Apache Kafka cluster. + minimum_rancher_version: v0.59.0 + maintainer: "Raul Sanchez " + uuid: kafka-0 + questions: + - variable: "kafka_name" + description: "Name of kafka rc and service." + label: "K8s rc name:" + required: true + default: "kafka" + type: "string" + - variable: "kafka_namespace" + description: "Name of kafka namespace." + label: "K8s namespace:" + required: true + default: "default" + type: "string" + - variable: "kafka_scale" + description: "Number of brokers nodes. Note: Recommended an odd number" + label: "Broker Nodes:" + required: true + default: 3 + type: "int" + - variable: "kafka_mem" + description: "Amount of memory to config brokers." + label: "Broker Memory (mb):" + required: true + default: 1024 + type: "string" + - variable: "kafka_log_dir" + description: "Broker volume to log data" + label: "Broker log volume:" + required: true + default: "/opt/kafka/logs" + type: "string" + - variable: "kafka_log_retention" + description: "Broker log retention in hours" + label: "Broker log retention (h):" + required: true + default: "168" + type: "int" + - variable: "kafka_num_partitions" + description: "Number of broker partitions" + label: "Broker partitions:" + required: true + default: "1" + type: "int" + - variable: "kafka_delete_topics" + label: "Kafka delete topics:" + description: | + Enable delete topics in kafka. + default: false + required: true + type: enum + options: + - false + - true + - variable: "kafka_interval" + description: "Interval to poll/apply configuration changes. 0 to disable" + label: "Broker Interval (s):" + required: true + default: 60 + type: "int" + - variable: "kafka_pub_ip" + description: "Advertise local and host public ip" + label: "Broker public ip:" + default: false + required: true + type: enum + options: + - false + - true + - variable: "zk_link" + description: | + Choose the Zookeeper service to use. + It's really recommended a dedicated zookeeper service just for Kafka. + label: "Zookeeper stack/service" + default: "default/zookeeper" + required: true + type: "service" +broker: + scale: ${kafka_scale} + retain_ip: true + health_check: + port: 9092 + interval: 5000 + unhealthy_threshold: 3 + request_line: '' + healthy_threshold: 2 + response_timeout: 5000 + diff --git a/kubernetes-templates/kafka/2/README.md b/kubernetes-templates/kafka/2/README.md new file mode 100644 index 0000000..1bab12c --- /dev/null +++ b/kubernetes-templates/kafka/2/README.md @@ -0,0 +1,33 @@ +# Apache Kafka (Experimental) + +### Info: + + This template creates, scale in and scale out a multinodes kafka broker cluster on top of k8s. The configuration is generated with confd from k8s. + Cluster size are variable after deployment, and get reconfigured. + + +### Usage: + + Select Apache Kafka from catalog. + + Enter the number of nodes, mem and refresh interval for the kafka cluster. + + Change the following kafka default parameters, if you need: + +- kafka_name="kafka" # kafka k8s service and rc name. +- kafka_namespace="kafka" # kafka k8s namespace +- kafka_scale=3 # kafka scale to deploy. +- kafka_mem=512 # kafka broker memory. +- kafka_log_dir="/opt/kafka/logs" # Kafka log dir. +- kafka_log_retention="168" # kafka log retention. +- kafka_num_partitions="1" # Kafka partitions number +- kafka_delete_topics="false" # kafka delete topics +- kafka_pub_ip= < true | false > # Advertise public ip to zookeeper. +- zk_link="kafka-zk/zk" # zookeeper namespace/service to connect to. + + Click deploy. + + Kafka can now be accessed over the Rancher network. + + Note: When you scale the cluster, zero downtime is not guaranteed..yet.. + diff --git a/kubernetes-templates/kafka/2/kafka-rc.yml b/kubernetes-templates/kafka/2/kafka-rc.yml new file mode 100644 index 0000000..306fd1a --- /dev/null +++ b/kubernetes-templates/kafka/2/kafka-rc.yml @@ -0,0 +1,77 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: "${kafka_name}" + namespace: "${kafka_namespace}" + labels: + name: "${kafka_name}" +spec: + replicas: ${kafka_scale} + selector: + name: "${kafka_name}" + template: + metadata: + labels: + name: "${kafka_name}" + spec: + containers: + - name: "broker-conf" + image: "rawmind/k8s-kafka:0.10.2.0-1" + volumeMounts: + - mountPath: "/opt/tools" + name: "broker-conf" + ports: + imagePullPolicy: "IfNotPresent" + - name: "broker-volume" + image: "rawmind/alpine-volume:0.0.2-1" + env: + - name: SERVICE_UID + value: "10003" + - name: SERVICE_GID + value: "10003" + - name: SERVICE_VOLUME + value: "${kafka_log_dir}" + - name: KEEP_ALIVE + value: "1" + volumeMounts: + - mountPath: "${kafka_log_dir}" + name: "broker-data" + ports: + imagePullPolicy: "IfNotPresent" + - name: "broker-service" + image: "rawmind/alpine-kafka:0.10.2.1" + env: + - name: JVMFLAGS + value: "-Xmx${kafka_mem}m -Xms${kafka_mem}m" + - name: CONFD_INTERVAL + value: "${kafka_interval}" + - name: KAFKA_ZK_SERVICE + value: "${zk_link}" + - name: KAFKA_DELETE_TOPICS + value: "${kafka_delete_topics}" + - name: KAFKA_LOG_DIRS + value: "${kafka_log_dir}" + - name: KAFKA_LOG_RETENTION_HOURS + value: "${kafka_log_retention}" + - name: KAFKA_NUM_PARTITIONS + value: "${kafka_num_partitions}" + - name: ADVERTISE_PUB_IP + value: "${kafka_pub_ip}" + - name: POD_NAMESPACE + value: "${kafka_namespace}" + volumeMounts: + - mountPath: "${kafka_log_dir}" + name: "broker-data" + - mountPath: "/opt/tools" + name: "broker-conf" + ports: + - containerPort: 9092 + name: "broker-client" + protocol: "TCP" + imagePullPolicy: "IfNotPresent" + restartPolicy: "Always" + volumes: + - name: "broker-data" + emptyDir: {} + - name: "broker-conf" + emptyDir: {} \ No newline at end of file diff --git a/kubernetes-templates/kafka/2/kafka-service.yml b/kubernetes-templates/kafka/2/kafka-service.yml new file mode 100644 index 0000000..0a6371a --- /dev/null +++ b/kubernetes-templates/kafka/2/kafka-service.yml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: "${kafka_name}" + labels: + name: "${kafka_name}" +spec: + ports: + - port: 9092 + targetPort: 9092 + name: "broker-client" + protocol: "TCP" + selector: + name: "${kafka_name}" + clusterIP: "None" diff --git a/kubernetes-templates/kafka/2/rancher-compose.yml b/kubernetes-templates/kafka/2/rancher-compose.yml new file mode 100644 index 0000000..46d835c --- /dev/null +++ b/kubernetes-templates/kafka/2/rancher-compose.yml @@ -0,0 +1,95 @@ +.catalog: + name: Kafka + version: 0.10.2.1-k8s + description: | + (Experimental) Apache Kafka cluster. + minimum_rancher_version: v0.59.0 + maintainer: "Raul Sanchez " + uuid: kafka-0 + questions: + - variable: "kafka_name" + description: "Name of kafka rc and service." + label: "K8s rc name:" + required: true + default: "kafka" + type: "string" + - variable: "kafka_namespace" + description: "Name of kafka namespace." + label: "K8s namespace:" + required: true + default: "default" + type: "string" + - variable: "kafka_scale" + description: "Number of brokers nodes. Note: Recommended an odd number" + label: "Broker Nodes:" + required: true + default: 3 + type: "int" + - variable: "kafka_mem" + description: "Amount of memory to config brokers." + label: "Broker Memory (mb):" + required: true + default: 1024 + type: "string" + - variable: "kafka_log_dir" + description: "Broker volume to log data" + label: "Broker log volume:" + required: true + default: "/opt/kafka/logs" + type: "string" + - variable: "kafka_log_retention" + description: "Broker log retention in hours" + label: "Broker log retention (h):" + required: true + default: "168" + type: "int" + - variable: "kafka_num_partitions" + description: "Number of broker partitions" + label: "Broker partitions:" + required: true + default: "1" + type: "int" + - variable: "kafka_delete_topics" + label: "Kafka delete topics:" + description: | + Enable delete topics in kafka. + default: false + required: true + type: enum + options: + - false + - true + - variable: "kafka_interval" + description: "Interval to poll/apply configuration changes. 0 to disable" + label: "Broker Interval (s):" + required: true + default: 60 + type: "int" + - variable: "kafka_pub_ip" + description: "Advertise local and host public ip" + label: "Broker public ip:" + default: false + required: true + type: enum + options: + - false + - true + - variable: "zk_link" + description: | + Choose the Zookeeper service to use. + It's really recommended a dedicated zookeeper service just for Kafka. + label: "Zookeeper stack/service" + default: "default/zookeeper" + required: true + type: "service" +broker: + scale: ${kafka_scale} + retain_ip: true + health_check: + port: 9092 + interval: 5000 + unhealthy_threshold: 3 + request_line: '' + healthy_threshold: 2 + response_timeout: 5000 + diff --git a/kubernetes-templates/kafka/config.yml b/kubernetes-templates/kafka/config.yml index 061b88b..adc60a9 100644 --- a/kubernetes-templates/kafka/config.yml +++ b/kubernetes-templates/kafka/config.yml @@ -1,7 +1,7 @@ name: Apache Kafka description: | (Experimental) Kafka cluster -version: 0.10.0.1-k8s +version: 0.10.2.1-k8s category: Clustering maintainer: "Raul Sanchez " minimum_rancher_version: v0.59.0 diff --git a/kubernetes-templates/zookeeper/1/zk-rc.yml b/kubernetes-templates/zookeeper/1/zk-rc.yml index 739d5a6..3bbaa4f 100644 --- a/kubernetes-templates/zookeeper/1/zk-rc.yml +++ b/kubernetes-templates/zookeeper/1/zk-rc.yml @@ -17,7 +17,7 @@ spec: spec: containers: - name: "zk-conf" - image: "rawmind/k8s-zk:3.4.8-10" + image: "rawmind/k8s-zk:3.4.9" volumeMounts: - mountPath: "/opt/tools" name: "zk-conf" @@ -40,7 +40,7 @@ spec: ports: imagePullPolicy: "IfNotPresent" - name: "zk-service" - image: "rawmind/alpine-zk:3.4.9" + image: "rawmind/alpine-zk:3.4.9-3" env: - name: JVMFLAGS value: "-Xmx${zk_mem}m -Xms${zk_mem}m" diff --git a/kubernetes-templates/zookeeper/1/zk-service.yml b/kubernetes-templates/zookeeper/1/zk-service.yml index 438bb3c..2ce431c 100644 --- a/kubernetes-templates/zookeeper/1/zk-service.yml +++ b/kubernetes-templates/zookeeper/1/zk-service.yml @@ -21,3 +21,4 @@ spec: protocol: "TCP" selector: name: "${zk_name}" + diff --git a/kubernetes-templates/zookeeper/config.yml b/kubernetes-templates/zookeeper/config.yml index a33c707..4aa9875 100644 --- a/kubernetes-templates/zookeeper/config.yml +++ b/kubernetes-templates/zookeeper/config.yml @@ -4,4 +4,4 @@ description: | version: 3.4.9-k8s1 category: Clustering maintainer: "Raul Sanchez " -projectURL: https://github.com/rawmind0/alpine-zk +projectURL: https://github.com/rawmind0/alpine-zk \ No newline at end of file diff --git a/templates/kafka/2/README.md b/templates/kafka/2/README.md new file mode 100644 index 0000000..df396ad --- /dev/null +++ b/templates/kafka/2/README.md @@ -0,0 +1,30 @@ +# Apache Kafka (Experimental) + +### Info: + + This template creates, scale in and scale out a multinodes kafka broker cluster on top of Rancher. The configuration is generated with confd from Rancher metadata. + Cluster size are variable after deployment, and get reconfigured after refresh interval. + + +### Usage: + + Select Apache Kafka from catalog. + + Enter the number of nodes, mem and refresh interval for the kafka cluster. + + Change the following kafka default parameters, if you need: + +- kafka_scale=3 # kafka scale to deploy. +- kafka_mem=512 # kafka broker memory. +- kafka_log_dir="/opt/kafka/logs" # Kafka log dir. +- kafka_log_retention="168" # kafka log retention. +- kafka_num_partitions="1" # Kafka partitions number +- kafka_delete_topics="false" # kafka delete topics +- kafka_pub_ip= < true | false > # Advertise public ip to zookeeper. +- zk_link="kafka-zk/zk" # zookeeper stack/service to connect to. + + Click deploy. + + Kafka can now be accessed over the Rancher network. + + Note: When you scale the cluster, zero downtime is not guaranteed..yet.. diff --git a/templates/kafka/2/docker-compose.yml b/templates/kafka/2/docker-compose.yml new file mode 100644 index 0000000..373ffc0 --- /dev/null +++ b/templates/kafka/2/docker-compose.yml @@ -0,0 +1,44 @@ +broker: + tty: true + image: rawmind/alpine-kafka:0.10.2.1 + volumes_from: + - broker-volume + - broker-conf + environment: + - JVMFLAGS=-Xmx${kafka_mem}m -Xms${kafka_mem}m + - CONFD_INTERVAL=${kafka_interval} + - ZK_SERVICE=${zk_link} + - KAFKA_DELETE_TOPICS=${kafka_delete_topics} + - KAFKA_LOG_DIRS=${kafka_log_dir} + - KAFKA_LOG_RETENTION_HOURS=${kafka_log_retention} + - KAFKA_NUM_PARTITIONS=${kafka_num_partitions} + - ADVERTISE_PUB_IP=${kafka_pub_ip} + external_links: + - ${zk_link}:zk + labels: + io.rancher.scheduler.affinity:container_label_soft_ne: io.rancher.stack_service.name=$${stack_name}/$${service_name} + io.rancher.container.hostname_override: container_name + io.rancher.sidekicks: broker-volume, broker-conf +broker-conf: + net: none + labels: + io.rancher.scheduler.affinity:container_label_soft_ne: io.rancher.stack_service.name=$${stack_name}/$${service_name} + io.rancher.container.hostname_override: container_name + io.rancher.container.start_once: true + image: rawmind/rancher-kafka:0.10.2.0 + volumes: + - /opt/tools +broker-volume: + net: none + labels: + io.rancher.scheduler.affinity:container_label_soft_ne: io.rancher.stack_service.name=$${stack_name}/$${service_name} + io.rancher.container.hostname_override: container_name + io.rancher.container.start_once: true + environment: + - SERVICE_UID=10003 + - SERVICE_GID=10003 + - SERVICE_VOLUME=${kafka_log_dir} + volumes: + - ${kafka_log_dir} + volume_driver: local + image: rawmind/alpine-volume:0.0.2-1 diff --git a/templates/kafka/2/rancher-compose.yml b/templates/kafka/2/rancher-compose.yml new file mode 100644 index 0000000..9c02325 --- /dev/null +++ b/templates/kafka/2/rancher-compose.yml @@ -0,0 +1,80 @@ +.catalog: + name: Kafka + version: 0.10.2.1-rancher1 + description: | + (Experimental) Apache Kafka cluster. + minimum_rancher_version: v0.59.0 + maintainer: "Raul Sanchez " + uuid: kafka-0 + questions: + - variable: "kafka_scale" + description: "Number of brokers nodes. Note: Recommended an odd number" + label: "Broker Nodes:" + required: true + default: 3 + type: "int" + - variable: "kafka_mem" + description: "Amount of memory to config brokers." + label: "Broker Memory (mb):" + required: true + default: 1024 + type: "string" + - variable: "kafka_log_dir" + description: "Broker volume to log data" + label: "Broker log volume:" + required: true + default: "/opt/kafka/logs" + type: "string" + - variable: "kafka_log_retention" + description: "Broker log retention in hours" + label: "Broker log retention (h):" + required: true + default: "168" + type: "int" + - variable: "kafka_num_partitions" + description: "Number of broker partitions" + label: "Broker partitions:" + required: true + default: "1" + type: "int" + - variable: "kafka_delete_topics" + label: "Kafka delete topics:" + description: | + Enable delete topics in kafka. + default: false + required: true + type: enum + options: + - false + - true + - variable: "kafka_interval" + description: "Interval to poll/apply configuration changes. 0 to disable" + label: "Broker Interval (s):" + required: true + default: 60 + type: "int" + - variable: "kafka_pub_ip" + description: "Advertise local and host public ip" + label: "Broker public ip:" + required: false + default: false + type: "boolean" + - variable: "zk_link" + description: | + Choose the Zookeeper service to use. + It's really recommended a dedicated zookeeper service just for Kafka. + label: "Zookeeper stack/service" + default: "kafka-zk/zk" + required: true + type: "service" +broker: + scale: ${kafka_scale} + retain_ip: true + health_check: + port: 9092 + interval: 5000 + unhealthy_threshold: 3 + request_line: '' + healthy_threshold: 2 + response_timeout: 5000 + diff --git a/templates/kafka/config.yml b/templates/kafka/config.yml index 1bee24d..72c9555 100644 --- a/templates/kafka/config.yml +++ b/templates/kafka/config.yml @@ -1,7 +1,7 @@ name: Apache Kafka description: | (Experimental) Kafka cluster -version: 0.10.0.1-rancher1 +version: 0.10.2.1-rancher1 category: Clustering maintainer: "Raul Sanchez " minimum_rancher_version: v0.59.0 diff --git a/templates/zookeeper/2/docker-compose.yml b/templates/zookeeper/2/docker-compose.yml index 67c4da6..795cb29 100644 --- a/templates/zookeeper/2/docker-compose.yml +++ b/templates/zookeeper/2/docker-compose.yml @@ -1,6 +1,6 @@ zk: tty: true - image: rawmind/alpine-zk:3.4.9 + image: rawmind/alpine-zk:3.4.9-3 volumes_from: - zk-volume - zk-conf @@ -21,7 +21,7 @@ zk-conf: io.rancher.scheduler.affinity:container_label_soft_ne: io.rancher.stack_service.name=$${stack_name}/$${service_name} io.rancher.container.hostname_override: container_name io.rancher.container.start_once: true - image: rawmind/rancher-zk:3.4.8-5 + image: rawmind/rancher-zk:3.4.9 volumes: - /opt/tools zk-volume: