1apiVersion: v1
 2kind: Namespace
 3metadata:
 4  name: mysql-ha
 5
 6---
 7apiVersion: v1
 8kind: ServiceAccount
 9metadata:
10  name: cluster-helper
11  namespace: mysql-ha
12
13---
14kind: Role
15apiVersion: rbac.authorization.k8s.io/v1
16metadata:
17  name: cluster-helper
18  namespace: mysql-ha
19rules:
20  - apiGroups: ["apps"]
21    resources: ["statefulsets"]
22    verbs: ["get", "list"]
23
24---
25kind: RoleBinding
26apiVersion: rbac.authorization.k8s.io/v1
27metadata:
28  name: cluster-helper
29  namespace: mysql-ha
30subjects:
31  - kind: ServiceAccount
32    name: cluster-helper
33roleRef:
34  kind: Role
35  name: cluster-helper
36  apiGroup: rbac.authorization.k8s.io

  1---
  2apiVersion: v1
  3kind: ConfigMap
  4metadata:
  5  name: mysql-scripts
  6  namespace: mysql-ha
  7data:
  8  common.sh: |
  9    #!/bin/bash
 10    update_data() {
 11      local svc=${HOSTNAME%-*}
 12      local token=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
 13      local ns=${POD_NAMESPACE}
 14      local url=https://${KUBERNETES_SERVICE_HOST}/apis/apps/v1/namespaces/${ns}/statefulsets/${svc}
 15      curl -kfsSL -H "Authorization: Bearer ${token}" ${url} > /ha-info/state.json
 16    }
 17    get_data() {
 18      if [ ! -f /ha-info/state.json ]; then
 19        return
 20      fi
 21      cat /ha-info/state.json
 22    }
 23    get_replicas() {
 24      get_data | grep '"replicas":' | head -n 1 | awk '{print $2}' | tr -d ","
 25    }
 26    get_ready_replicas() {
 27      get_data | grep '"readyReplicas":' | awk '{print $2}' | tr -d ","
 28    }
 29    get_current_replicas() {
 30      get_data | grep '"currentReplicas":' | awk '{print $2}' | tr -d ","
 31    }
 32    sync_done() {
 33      echo "Sync has been done."
 34      # Infinite blocking
 35      while true; do
 36        sleep 3600
 37      done
 38    }
 39  init-cluster.sh: |
 40    #!/bin/sh
 41    if [[ -f /var/lib/mysql/CLUSTER_WAS_INIT ]]; then
 42      exit 0
 43    fi
 44    echo Initialize cluster configuration...
 45    source /scripts/common.sh
 46    SERVER_ID=$(hostname | grep -oE '[^-]+$')
 47    cat > /etc/mysql/conf.d/master-to-master.cnf <<EOF
 48    [mysql]
 49    no-auto-rehash
 50    [mysqld]
 51    skip-host-cache
 52    skip-name-resolve
 53    server_id=1${SERVER_ID}
 54    log-bin=mysql-bin
 55    auto_increment_increment=2
 56    auto_increment_offset=$((${SERVER_ID}+1))
 57    EOF
 58    echo Initialize cluster configuration successful.
 59  sync-ha.sh: |
 60    #!/bin/bash
 61    set -e
 62    if [ -f /var/lib/mysql/CLUSTER_WAS_INIT ]; then
 63      sync_done
 64    fi
 65    source /scripts/common.sh
 66    echo "Waiting all replicas up..."
 67    while [ `get_current_replicas` -ne `get_ready_replicas` ]
 68    do
 69      sleep 5
 70    done
 71    get_another_host() {
 72      local svc=${HOSTNAME%-*}
 73      local server_id=$(hostname | grep -oE '[^-]+$')
 74      local another_id
 75      if [[ $server_id == 0 ]]; then
 76        another_id=1
 77      else
 78        another_id=0
 79      fi
 80      echo "${svc}-${another_id}"
 81    }
 82    get_binlog_file() {
 83      local host=$1
 84      mysql -h $host -p"${MYSQL_ROOT_PASSWORD}" -e 'show master status\G;' | grep 'File' | awk '{print $2}'
 85    }
 86    get_binlog_pos() {
 87      local host=$1
 88      mysql -h $host -p"${MYSQL_ROOT_PASSWORD}" -e 'show master status\G;' | grep 'Position' | awk '{print $2}'
 89    }
 90    ensure_db() {
 91      while ! mysql -p"${MYSQL_ROOT_PASSWORD}" -e 'SELECT 1' > /dev/null; do
 92        sleep 5
 93      done
 94      mysql -p"${MYSQL_ROOT_PASSWORD}" -e "GRANT RELOAD, SUPER, REPLICATION SLAVE ON *.* TO 'root'@'%' identified by '${MYSQL_ROOT_PASSWORD}'"
 95    }
 96    wait_db() {
 97      local host=$1
 98      while ! mysql -h $host -p"${MYSQL_ROOT_PASSWORD}" -e 'SELECT 1' > /dev/null; do
 99        sleep 5
100      done
101    }
102    sync_core() {
103      local mainHost=$1
104      local targetHost=$2
105      local file=$3
106      local pos=$4
107      echo Synchionize "${mainHost}" "file=${file}" "pos=${pos}"
108      mysql -h "${mainHost}" -p"${MYSQL_ROOT_PASSWORD}" -e "CHANGE MASTER TO MASTER_HOST='${targetHost}', 
109        MASTER_USER='root',
110        MASTER_PASSWORD='${MYSQL_ROOT_PASSWORD}',
111        MASTER_LOG_FILE='${file}',
112        MASTER_LOG_POS=${pos};"
113      mysql -h "${mainHost}" -p"${MYSQL_ROOT_PASSWORD}" -e 'start slave;'
114    }
115    sync_db() {
116      local svc=${HOSTNAME%-*}
117      local right_fqdn=`get_another_host`.${svc}.${POD_NAMESPACE}.svc.cluster.local
118      local left_fqdn=${HOSTNAME}.${svc}.${POD_NAMESPACE}.svc.cluster.local
119      echo "Wait db $right_fqdn ..."
120      wait_db $right_fqdn
121      local left_file=`get_binlog_file ${left_fqdn}`
122      local left_pos=`get_binlog_pos ${left_fqdn}`
123      local right_file=`get_binlog_file ${right_fqdn}`
124      local right_pos=`get_binlog_pos ${right_fqdn}`
125      echo Sync right -> left ...
126      sync_core "$left_fqdn" "$right_fqdn" "$right_file" "$right_pos"
127      echo Sync left -> right ...
128      sync_core "$right_fqdn" "$left_fqdn" "$left_file" "$left_pos"
129    }
130    echo "Ensuring db..."
131    ensure_db
132    server_id=$(hostname | grep -oE '[^-]+$')
133    if [[ $server_id -eq 0 ]]; then
134      sync_db
135      echo "master-$server_id synchionize suffessful."
136    else
137      echo "master-$server_id are ready."
138    fi
139    touch /var/lib/mysql/CLUSTER_WAS_INIT
140    sync_done
141  sync-ha-metadata.sh: |
142    #!/bin/sh
143    if [ -f /var/lib/mysql/CLUSTER_WAS_INIT ]; then
144      sync_done
145    fi
146    apk add curl
147    source /scripts/common.sh
148    while [ ! -f /var/lib/mysql/CLUSTER_WAS_INIT ]
149    do
150      update_data
151      sleep 3
152    done
153    sync_done
154
155---
156apiVersion: apps/v1beta2
157kind: StatefulSet
158metadata:
159  name: mysql-mm
160  namespace: mysql-ha
161spec:
162  serviceName: mysql-mm
163  replicas: 2
164  selector:
165    matchLabels:
166      app: mysql-mm
167  template:
168    metadata:
169      labels:
170        app: mysql-mm
171    spec:
172      initContainers:
173        - name: init-cluster
174          image: alpine
175          command:
176            - /scripts/init-cluster.sh
177          volumeMounts:
178          - name: scripts
179            mountPath: /scripts
180            readOnly: true
181          - name: mysql-conf
182            mountPath: /etc/mysql/conf.d
183          - name: ha-metadata
184            mountPath: /ha-info
185            readOnly: true
186      containers:
187      - name: mysql
188        image: mysql:5.7
189        env:
190        - name: "MYSQL_ROOT_PASSWORD"
191          value: "123456!!"
192        - name: "MYSQL_ROOT_HOST"
193          value: "%"
194        volumeMounts:
195        - name: scripts
196          mountPath: /scripts
197          readOnly: true
198        - name: mysql-conf
199          mountPath: /etc/mysql/conf.d
200        - name: mysql-sock
201          mountPath: /var/run/mysqld
202        - name: mysql-data
203          mountPath: /var/lib/mysql
204        ports:
205        - containerPort: 3306
206      - name: sync-ha
207        image: mysql:5.7
208        command:
209          - /scripts/sync-ha.sh
210        env:
211        - name:  "MYSQL_ROOT_PASSWORD"
212          value: "123456!!"
213        - name: POD_NAMESPACE
214          valueFrom:
215            fieldRef:
216              fieldPath: metadata.namespace
217        volumeMounts:
218        - name: scripts
219          mountPath: /scripts
220          readOnly: true
221        - name: mysql-sock
222          mountPath: /var/run/mysqld
223        - name: ha-metadata
224          mountPath: /ha-info
225        - name: mysql-data
226          mountPath: /var/lib/mysql
227      - name: sync-ha-metadata
228        image: alpine
229        command:
230          - /scripts/sync-ha-metadata.sh
231        env:
232          - name: POD_NAMESPACE
233            valueFrom:
234              fieldRef:
235                fieldPath: metadata.namespace
236        volumeMounts:
237        - name: scripts
238          mountPath: /scripts
239          readOnly: true
240        - name: ha-metadata
241          mountPath: /ha-info
242        - name: mysql-data
243          mountPath: /var/lib/mysql
244      volumes:
245        - name: scripts
246          configMap:
247            name: mysql-scripts
248            defaultMode: 0755
249        - name: ha-metadata
250          emptyDir: {}
251        - name: mysql-conf
252          emptyDir: {}
253        - name: mysql-sock
254          emptyDir: {}
255        - name: mysql-data
256          emptyDir: {}
257      serviceAccountName: cluster-helper
258  # volumeClaimTemplates:
259  # - metadata:
260  #     name: pvc-mysql-data
261  #   spec:
262  #     accessModes: [ "ReadWriteOnce" ]
263  #     resources:
264  #       requests:
265  #         storage: 10Gi
266
267---
268# Service
269apiVersion: v1
270kind: Service
271metadata:
272  name: mysql-mm
273  namespace: mysql-ha
274spec:
275  ports:
276  - port: 3306
277    targetPort: 3306
278  selector:
279    app: mysql-mm

 1apt-get update
 2apt-get install -yq \
 3    apt-transport-https \
 4    ca-certificates \
 5    curl \
 6    gnupg2
 7apt-get update
 8
 9curl -fsSL  https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
10
11cat <<EOF >/etc/apt/sources.list.d/kubernetes.list
12deb https://apt.kubernetes.io/ kubernetes-xenial main
13EOF
14
15apt-get update
16apt-get install -y \
17    ipvsadm ipset \ # for ipvs mode
18    kubelet kubeadm kubectl
19apt-mark hold kubelet kubeadm kubectl

1NETWORK_PLUGIN=https://raw.githubusercontent.com/coreos/flannel/a70459be0084506e4ec919aa1c114638878db11b/Documentation/kube-flannel.yml
2NETWORK_CIDR=10.64.0.0/16
3
4kubeadm init --pod-network-cidr=$NETWORK_CIDR
5
6export KUBECONFIG=/etc/kubernetes/admin.conf
7curl -sSL $NETWORK_PLUGIN | sed "s|10.244.0.0/16|${NETWORK_CIDR}|g" |
8  kubectl apply -f -

1kubeadm join K8S-MASTER-LAVA:6443 --token <YOUR_TOKEN> \
2  --discovery-token-ca-cert-hash sha256:<YOUR_CA_CERT_HASH>

1nameserver 10.96.0.10
2search default.svc.cluster.local svc.cluster.local cluster.local lan
3options ndots:5

1example.com.example.com.default.svc.cluster.local.
2example.com.svc.cluster.local.
3example.com.cluster.local.
4example.com.lan.
5example.com.

 1apiVersion: v1
 2kind: Pod
 3metadata:
 4  namespace: default
 5  name: custom-pod-dns
 6spec:
 7  containers:
 8    - name: custom-pod-dns
 9      image: alpine
10      command:
11        - sleep
12        - "3600"
13  dnsPolicy: "None"
14  dnsConfig:
15    nameservers:
16      - 8.8.8.8
17    searches:
18      - lan
19    options:
20      - name: ndots
21        value: "2"

1...
2networks:
3  default:
4    driver: bridge
5    driver_opts:
6      com.docker.network.driver.mtu: 1450

1/ip firewall filter
2add action=drop chain=forward connection-state=invalid

1/ip firewall nat
2add action=netmap chain=dstnat comment=DNS dst-address=10.0.0.1 dst-port=53 \
3    protocol=udp src-address=!10.0.1.1 to-addresses=10.0.1.1 to-ports=53
4add action=masquerade chain=srcnat dst-address=10.0.1.1 dst-port=53 protocol=\
5    udp src-address=!10.0.1.1
6add action=netmap chain=dstnat comment=HijackDNS dst-address=8.8.8.8 dst-port=\
7    53 protocol=udp src-address=!10.0.1.1 to-addresses=10.0.1.1 to-ports=53

 1:local ttl "00:05:00"
 2:local zone "lan"
 3:local hostname
 4:local ip
 5:local dnsip
 6:local dhcpip
 7:local dnsnode
 8:local dhcpnode
 9 
10/ip dns static;
11:foreach i in=[find where name ~ (".*\\.".$zone) ] do={
12  :local nodettl
13  :set hostname [ get $i name ];
14  :set hostname [ :pick $hostname 0 ( [ :len $hostname ] - ( [ :len $zone ] + 1 ) ) ];
15  :set nodettl [get $i ttl];
16  /ip dhcp-server lease;
17  :set dhcpnode [ find where host-name=$hostname ];
18  :if ( [ :len $dhcpnode ] > 0) do={
19    :log debug ("Lease for ".$hostname." still exists. Not deleting.");
20  } else={
21# there's no lease by that name. Maybe this mac has a static name.
22    :local found false
23    /system script environment
24    :foreach n in=[ find where name ~ "shost[0-9A-F]+" ] do={
25       :if ( [ get $n value ] = $hostname ) do={
26         :set found true;
27       }
28    }
29    :if ( found ) do={
30      :log debug ("Hostname ".$hostname." is static");
31    } else={
32      :if ( $nodettl = $ttl ) do={
33        :log info ("Lease expired for ".$hostname.", deleting DNS entry.");
34        /ip dns static remove $i;
35      }
36    }
37  }
38}
39 
40/ip dhcp-server lease;
41:foreach i in=[find] do={
42  :set hostname ""
43  :local mac
44  :set dhcpip [ get $i address ];
45  :set mac [ get $i mac-address ];
46  :while ($mac ~ ":") do={
47    :local pos [ :find $mac ":" ];
48    :set mac ( [ :pick $mac 0 $pos ] . [ :pick $mac ($pos + 1) 999 ] );
49  };
50  :foreach n in=[ /system script environment find where name=("shost" . $mac) ] do={
51    :set hostname [ /system script environment get $n value ];
52  }
53  :if ( [ :len $hostname ] = 0) do={
54    :set hostname [ get $i host-name ];
55  }
56  :if ( [ :len $hostname ] > 0) do={
57    :set hostname ( $hostname . "." . $zone );
58    /ip dns static;
59    :set dnsnode [ find where name=$hostname ];
60    :if ( [ :len $dnsnode ] > 0 ) do={
61# it exists. Is its IP the same
62      :set dnsip [ get $dnsnode address ];
63      :if ( $dnsip = $dhcpip ) do={
64        :log debug ("DNS entry for " . $hostname . " does not need updating.");
65      } else={
66        :log info ("Replacing DNS entry for " . $hostname);
67        /ip dns static remove $dnsnode;
68        /ip dns static add name=$hostname address=$dhcpip ttl=$ttl;
69      }
70    } else={
71# it doesn't exist. Add it
72      :log info ("Adding new DNS entry for " . $hostname);
73      /ip dns static add name=$hostname address=$dhcpip ttl=$ttl;
74    }
75  }
76}