Skip to content

Commit

Permalink
operator: automatic registration for the Pika service (OpenAtomFounda…
Browse files Browse the repository at this point in the history
…tion#1931)

* helm: auto register server

* helm: remove unnecessary PVC and make PVC configurable independently

* helm: make resource configurable independently

* helm: wait for codis dashboard success
  • Loading branch information
machinly authored Aug 25, 2023
1 parent cacb5f1 commit 409271e
Show file tree
Hide file tree
Showing 7 changed files with 200 additions and 68 deletions.
5 changes: 4 additions & 1 deletion codis/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,10 @@ RUN if [ "$ENABLE_PROXY" = "true" ] ; \
fi

RUN apt-get update && \
apt-get install -y ca-certificates && \
apt-get install -y ca-certificates \
jq \
netcat \
&& \
rm -rf /var/lib/apt/lists/*

COPY --from=builder /codis/bin /codis/bin
Expand Down
69 changes: 23 additions & 46 deletions tools/kubeblocks_helm/pika-cluster/templates/cluster.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,16 @@ spec:
tolerations: {{ . | toYaml | nindent 4 }}
{{- end }}
componentSpecs:
- name: pika # user-defined
componentDefRef: pika # ref clusterDefinition componentDefs.name
monitor: {{ .Values.monitor.enabled | default false }}
enabledLogs: {{ .Values.enabledLogs | toJson | indent 4 }}
replicas: {{ .Values.replicaCount | default 2 }}
serviceAccountName: {{ include "pika-cluster.serviceAccountName" . }}
{{- range $i := until (int .Values.replicaCount) }}
- name: pika-group-{{ add ($i) 1 }} # user-defined
componentDefRef: pika-group # ref clusterDefinition componentDefs.name
monitor: {{ $.Values.monitor.enabled | default false }}
enabledLogs: {{ $.Values.enabledLogs | toJson | indent 4 }}
replicas: {{ add (int $.Values.slaveCount) 1 | default 2 }}
serviceAccountName: {{ include "pika-cluster.serviceAccountName" $ }}
switchPolicy:
type: {{ .Values.switchPolicy.type}}
{{- with .Values.resources }}
type: {{ $.Values.switchPolicy.type}}
{{- with $.Values.resources.pikaGroup }}
resources:
limits:
cpu: {{ .limits.cpu | quote }}
Expand All @@ -32,21 +33,25 @@ spec:
cpu: {{ .requests.cpu | quote }}
memory: {{ .requests.memory | quote }}
{{- end }}
{{- if .Values.persistence.enabled }}
{{- if $.Values.persistence.enabled }}
volumeClaimTemplates:
{{- with $.Values.persistence.pikaData }}
- name: data # ref clusterdefinition components.containers.volumeMounts.name
spec:
storageClassName: {{ .storageClassName }}
accessModes:
- ReadWriteOnce
resources:
requests:
storage: {{ .Values.persistence.data.size }}
storage: {{ .size }}
{{- end }}
{{- end }}
{{- end }}
- name: etcd # user-defined
componentDefRef: etcd # ref clusterdefinition components.name
monitor: {{ .Values.monitor.enabled | default false }}
replicas: {{ .Values.etcdReplicaCount| default 3 }}
{{- with .Values.resources }}
{{- with .Values.resources.etcd }}
resources:
{{- with .limits }}
limits:
Expand All @@ -61,19 +66,21 @@ spec:
{{- end }}
{{- if .Values.persistence.enabled }}
volumeClaimTemplates:
{{- with $.Values.persistence.pikaData }}
- name: data # ref clusterdefinition components.containers.volumeMounts.name
spec:
storageClassName: {{ .Values.persistence.data.storageClassName }}
storageClassName: {{ .storageClassName }}
accessModes:
- ReadWriteOnce
resources:
requests:
storage: {{ .Values.persistence.data.size }}
storage: {{ .size }}
{{- end }}
{{- end }}
- name: codis-proxy
componentDefRef: codis-proxy # ref clusterDefinition componentDefs.name
replicas: {{ .Values.codisProxyReplicaCount | default 2 }}
{{- with .Values.resources }}
{{- with .Values.resources.codisProxy }}
resources:
limits:
cpu: {{ .limits.cpu | quote }}
Expand All @@ -82,20 +89,10 @@ spec:
cpu: {{ .requests.cpu | quote }}
memory: {{ .requests.memory | quote }}
{{- end }}
{{- if .Values.persistence.enabled }}
volumeClaimTemplates:
- name: data # ref clusterdefinition components.containers.volumeMounts.name
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: {{ .Values.persistence.data.size }}
{{- end }}
- name: codis-fe
componentDefRef: codis-fe # ref clusterDefinition componentDefs.name
replicas: {{ .Values.codisFeReplicaCount | default 1 }}
{{- with .Values.resources }}
{{- with .Values.resources.codisFe }}
resources:
limits:
cpu: {{ .limits.cpu | quote }}
Expand All @@ -104,20 +101,10 @@ spec:
cpu: {{ .requests.cpu | quote }}
memory: {{ .requests.memory | quote }}
{{- end }}
{{- if .Values.persistence.enabled }}
volumeClaimTemplates:
- name: data # ref clusterdefinition components.containers.volumeMounts.name
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: {{ .Values.persistence.data.size }}
{{- end }}
- name: codis-dashboard
componentDefRef: codis-dashboard # ref clusterDefinition componentDefs.name
replicas: 1
{{- with .Values.resources }}
{{- with .Values.resources.codisFe }}
resources:
limits:
cpu: {{ .limits.cpu | quote }}
Expand All @@ -126,13 +113,3 @@ spec:
cpu: {{ .requests.cpu | quote }}
memory: {{ .requests.memory | quote }}
{{- end }}
{{- if .Values.persistence.enabled }}
volumeClaimTemplates:
- name: data # ref clusterdefinition components.containers.volumeMounts.name
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: {{ .Values.persistence.data.size }}
{{- end }}
48 changes: 41 additions & 7 deletions tools/kubeblocks_helm/pika-cluster/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ fullnameOverride: ""

replicaCount: 2

slaveCount: 1

etcdReplicaCount: 3

codisProxyReplicaCount: 2
Expand All @@ -30,16 +32,48 @@ resources:
# choice for the user. This also increases chances charts run on environments with little
# resources, such as Minikube. If you do want to specify resources, uncomment the following
# lines, adjust them as necessary, and remove the curly braces after 'resources:'.
limits:
cpu: 500m
memory: 3Gi
requests:
cpu: 500m
memory: 1Gi
pikaGroup:
limits:
cpu: 500m
memory: 3Gi
requests:
cpu: 500m
memory: 1Gi
etcd:
limits:
cpu: 500m
memory: 3Gi
requests:
cpu: 500m
memory: 1Gi
codisProxy:
limits:
cpu: 500m
memory: 3Gi
requests:
cpu: 500m
memory: 1Gi
codisFe:
limits:
cpu: 500m
memory: 3Gi
requests:
cpu: 500m
memory: 1Gi
codisDashboard:
limits:
cpu: 500m
memory: 3Gi
requests:
cpu: 500m
memory: 1Gi

persistence:
enabled: true
data:
pikaData:
storageClassName:
size: 10Gi
etcdData:
storageClassName:
size: 1Gi

Expand Down
101 changes: 101 additions & 0 deletions tools/kubeblocks_helm/pika/script/admin.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
#! /bin/bash
set -x

# set instance role
set_instance_role() {
POD_ID=${HOSTNAME##*-}
echo "POD_ID: "${POD_ID}
}

# set group id
set_group_id() {
GROUP_ID=${KB_CLUSTER_COMP_NAME##*-}
echo "GROUP_ID: "${GROUP_ID}
}

# set codis dashboard
set_codis_dashboard() {
CODIS_DASHBOARD="${KB_CLUSTER_NAME}-codis-dashboard"
echo "CODIS_DASHBOARD: "${CODIS_DASHBOARD}
CODIS_ADMIN="/codis/bin/codis-admin --dashboard=${CODIS_DASHBOARD}:18080"
echo "CODIS_ADMIN: "${CODIS_ADMIN}
}

wait_server_running() {
until nc -z 127.0.0.1 9221; do
echo waiting for pika
sleep 2
done
}

wait_dashboard_running() {
until nc -z ${CODIS_DASHBOARD} 18080; do
echo waiting for codis dashboard
sleep 2
done
}

wait_master_registered() {
until $CODIS_ADMIN --list-group | jq -r '.[] | select(.id == '${GROUP_ID}') | .servers[] | select(.role == "master") | .server'; do
echo waiting for master registered
sleep 2
done
}

reload_until_success() {
until $CODIS_ADMIN --reload 1>/dev/null 2>&1; do
echo waiting for reload success
sleep 2
done
}

register_server() {
reload_until_success
if [ ${POD_ID} -gt 0 ]; then wait_master_registered; fi
$CODIS_ADMIN --create-group --gid=${GROUP_ID} 1>/dev/null 2>&1
$CODIS_ADMIN --group-add --gid=${GROUP_ID} --addr=${KB_POD_FQDN}:9221
$CODIS_ADMIN --sync-action --create --addr=${KB_POD_FQDN}:9221 1>/dev/null 2>&1
}

remove_server() {
$CODIS_ADMIN --reload
if [ $? != 0 ]; then exit 1; fi
gid=${GROUP_ID}
sleep 5
$CODIS_ADMIN --group-del --gid=${GROUP_ID} --addr=${KB_POD_FQDN}:9221
}

reblance() {
$CODIS_ADMIN --rebalance --confirm
}

set_group_id
set_instance_role
set_codis_dashboard

if [ $# -eq 1 ]; then
case $1 in
--help)
echo "Usage: $0 [options]"
echo "Options:"
echo " --help show help information"
echo " --register-server register server to dashboard"
echo " --remove-server remove server from dashboard"
exit 0
;;
--register-server)
wait_dashboard_running
wait_server_running
register_server
exit 0
;;
--remove-server)
remove_server
exit 0
;;
*)
echo "Error: invalid option '$1'"
exit 1
;;
esac
fi
31 changes: 18 additions & 13 deletions tools/kubeblocks_helm/pika/templates/clusterdefinition.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ spec:
host: "$(SVC_FQDN)"
port: "$(SVC_PORT_pika)"
componentDefs:
- name: pika
workloadType: Replication
- name: pika-group
workloadType: Stateful
characterType: pika
service:
ports:
Expand All @@ -26,25 +26,35 @@ spec:
templateRef: pika-conf-template
namespace: {{ .Release.Namespace }}
volumeName: config
volumeTypes:
- name: data
type: data
scriptSpecs:
- name: pika-script
templateRef: pika-script-template
namespace: {{ .Release.Namespace }}
volumeName: script
defaultMode: 0555
podSpec:
containers:
- name: pika
ports:
- name: pika
containerPort: 9221
volumeMounts:
- name: data
mountPath: /data
- name: config
mountPath: /etc/pika
command:
- "/pika/bin/pika"
args:
- "-c"
- "/etc/pika/pika.conf"
- name: codis-admin
volumeMounts:
- name: script
mountPath: /script
command:
- "/bin/bash"
args:
- "-c"
- "/script/admin.sh --register-server;tail -f /dev/null"
- name: etcd
workloadType: Stateful
characterType: etcd
Expand Down Expand Up @@ -137,7 +147,7 @@ spec:
- name: ETCD_ENABLE_V2
value: "true"
- name: codis-proxy
workloadType: Stateless
workloadType: Stateful
characterType: pika
service:
ports:
Expand Down Expand Up @@ -275,9 +285,6 @@ spec:
templateRef: pika-conf-template
namespace: {{ .Release.Namespace }}
volumeName: config
volumeTypes:
- name: data
type: data
podSpec:
initContainers:
- name: wait-etcd
Expand All @@ -296,8 +303,6 @@ spec:
- containerPort: 18080
name: dashboard
volumeMounts:
- name: data
mountPath: /data
- name: config
mountPath: /etc/codis
env:
Expand Down
Loading

0 comments on commit 409271e

Please sign in to comment.