BlueXIII's Blog

热爱技术,持续学习

0%

Posted on Edited on

参考

常规操作

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
# 生成骨架
helm create mychart

# 修改并安装
helm install full-coral ./mychart

# 查看实际加载的模板
helm get manifest full-coral

## 安装
helm install clunky-serval ./mychart --namespace mynamespace

# 测试渲染
helm install --debug --dry-run goodly-guppy ./mychart
helm install solid-vulture ./mychart --dry-run --debug --set favoriteDrink=slurm

使用HarborHelm仓库

https://juejin.cn/post/6844903795827146759

1
2
3
4
5
6
7
8
9
# 安装cm-push插件，3.8以上虽然自带push命令，但无法实现向http仓库推送
helm plugin install https://github.com/chartmuseum/helm-push

# 添加HarborHelm仓库
helm repo add dubhe http://harbor.dubhe:30002/chartrepo/dubhe
helm repo update

# 上传chart
helm cm-push ./gogs.tgz dubhe --username admin --password yourpass

简易脚手架

Chart.yaml

1
2
3
4
5
6
apiVersion: v2
name: mysql
description: Dubhe MySQL Helm Chart for Kubernetes
type: application
version: 1.0.0
appVersion: "5.7.42-debian"

values.yaml

1
2
3
4
5
6
7
mysql:
  repository: mysql
  tag: 5.7.42-debian
  imagePullPolicy: Always
  storageClassName: longhorn-retain
  storageSize: 20Gi
  nodePort: 30353

templates/deployment.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
apiVersion: apps/v1
kind: Deployment
metadata:
  name: mysql
  labels:
    app: mysql
spec:
  selector:
    matchLabels:
      app: mysql
  strategy:
    type: Recreate
  template:
    metadata:
      labels:
        app: mysql
    spec:
      containers:
      - name: mysql
        image: "{{ .Values.mysql.repository }}:{{ .Values.mysql.tag }}"
        imagePullPolicy: {{ .Values.mysql.imagePullPolicy }}
        env:
        - name: MYSQL_ROOT_PASSWORD
          valueFrom:
            secretKeyRef:
              name: dubhe-secret
              key: mysql-password
        ports:
        - containerPort: 3306
          name: mysql
        volumeMounts:
        - name: mysql-persistent-storage
          mountPath: /var/lib/mysql
        - name: mysql-config
          mountPath: /etc/mysql/mysql.conf.d/mysqld.cnf
          subPath: mysqld.cnf
      volumes:
      - name: mysql-persistent-storage
        persistentVolumeClaim:
          claimName: mysql
      - name: mysql-config
        configMap:
          name: mysql-configmap
          items:
            - key: mysql-conf
              path: mysqld.cnf

templates/configmap.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
apiVersion: v1
kind: ConfigMap
metadata:
  name: mysql-configmap
data:
  mysql-conf: |
    [mysqld]
    server_id       = 1
    log-bin         = mysql-bin
    pid-file        = /var/run/mysqld/mysqld.pid
    socket          = /var/run/mysqld/mysqld.sock
    datadir         = /var/lib/mysql
    #log-error      = /var/log/mysql/error.log
    # By default we only accept connections from localhost
    #bind-address   = 127.0.0.1
    # Disabling symbolic-links is recommended to prevent assorted security risks
    symbolic-links=0
    
    lower_case_table_names=1
    sql_mode=STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION
    character-set-server=utf8
    group_concat_max_len=102400
    max_connections=2000
    max_allowed_packet=524288000
    
    default-time-zone='+08:00'
    # 忽略挂载目录
    ignore-db-dir=lost+found

templates/pvc.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: mysql
spec:
  accessModes:
    - ReadWriteOnce
  {{- if not ( empty .Values.mysql.storageClassName ) }}
  storageClassName: {{ .Values.mysql.storageClassName }}
  {{- end }}
  resources:
    requests:
      storage: {{ .Values.mysql.storageSize }}

templates/service.yaml

1
2
3
4
5
6
7
8
9
10
11
apiVersion: v1
kind: Service
metadata:
  name: mysql
spec:
  selector:
    app: mysql
  ports:
    - protocol: TCP
      port: 3306
  clusterIP: None

templates/service-exposed.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
apiVersion: v1
kind: Service
metadata:
  name: mysql-exposed
spec:
  selector:
    app: mysql
  ports:
    - protocol: TCP
      port: 3306
      targetPort: 3306
      {{- if not ( empty .Values.mysql.nodePort ) }}
      nodePort: {{ .Values.mysql.nodePort }}
      {{- end }}
  type: NodePort

Posted on Edited on

官方文档

其它参考文档

离线镜像清单

安装时使用

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
gcr.io/tekton-releases/github.com/tektoncd/pipeline/cmd/controller:v0.48.0
gcr.io/tekton-releases/github.com/tektoncd/pipeline/cmd/events:v0.48.0
gcr.io/tekton-releases/github.com/tektoncd/pipeline/cmd/resolvers:v0.48.0
gcr.io/tekton-releases/github.com/tektoncd/pipeline/cmd/webhook:v0.48.0
gcr.io/tekton-releases/github.com/tektoncd/triggers/cmd/controller:v0.24.0
gcr.io/tekton-releases/github.com/tektoncd/triggers/cmd/webhook:v0.24.0
gcr.io/tekton-releases/github.com/tektoncd/triggers/cmd/interceptors:v0.24.0
gcr.io/tekton-releases/github.com/tektoncd/dashboard/cmd/dashboard:v0.35.0
gcr.io/tekton-releases/github.com/tektoncd/pipeline/cmd/entrypoint:v0.48.0
gcr.io/tekton-releases/github.com/tektoncd/pipeline/cmd/nop:v0.48.0
gcr.io/tekton-releases/github.com/tektoncd/pipeline/cmd/sidecarlogresults:v0.48.0
gcr.io/tekton-releases/github.com/tektoncd/pipeline/cmd/workingdirinit:v0.48.0
gcr.io/tekton-releases/github.com/tektoncd/triggers/cmd/eventlistenersink:v0.24.0
cgr.dev/chainguard/busybox:latest
mcr.microsoft.com/powershell:latest

构建时使用

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
# 官方镜像
alpine:latest
busybox:latest
gcr.io/tekton-releases/github.com/tektoncd/pipeline/cmd/git-init:v0.40.2
alpine/git:latest
docker:latest
docker:dind
curlimages/curl:latest
gcr.io/kaniko-project/executor:v1.5.1
gcr.io/cloud-builders/mvn:latest
docker.io/library/node:12-alpine
# 自制镜像
harbor.dubhe:30002/dubhe-base/alpine:3.16.3
harbor.dubhe:30002/dubhe-base/dubhe-buildkit:1.2.0
harbor.dubhe:30002/dubhe-base/chart-update:latest

部署

下载常用组件

1
2
3
4
5
6
curl -o operator.yaml https://storage.googleapis.com/tekton-releases/operator/latest/release.yaml
curl -o pipeline.yaml https://storage.googleapis.com/tekton-releases/pipeline/latest/release.yaml
curl -o triggers.yaml https://storage.googleapis.com/tekton-releases/triggers/latest/release.yaml
curl -o interceptors.yaml https://storage.googleapis.com/tekton-releases/triggers/latest/interceptors.yaml
curl -o dashboard.yaml https://storage.googleapis.com/tekton-releases/dashboard/latest/release.yaml
curl -o chains.yaml https://storage.googleapis.com/tekton-releases/chains/latest/release.yaml

部署常用组件

1
2
3
4
5
6
kubectl apply -f ./operator.yaml
kubectl apply -f ./pipeline.yaml
kubectl apply -f ./triggers.yaml
kubectl apply -f ./interceptors.yaml
kubectl apply -f ./dashboard.yaml
kubectl apply -f ./chains.yaml

DashBoard开放访问

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
apiVersion: v1
kind: Service
metadata:
  name: tekton-dashboard-exposed
  namespace: tekton-pipelines
spec:
  selector:
    app.kubernetes.io/component: dashboard
    app.kubernetes.io/instance: default
    app.kubernetes.io/name: dashboard
    app.kubernetes.io/part-of: tekton-dashboard
  ports:
    - protocol: TCP
      port: 9097
      targetPort: 9097
      nodePort: 30014
  type: NodePort
1
2
kubectl apply -f nodeport.yaml
open http://10.193.35.21:30014

代理服务器配置(可选)

vi operator.yaml

1
2
3
4
5
6
7
8
containers:
- env:
  - name: HTTP_PROXY
    value: "socks5://10.193.36.33:7890/"
  - name: HTTPS_PROXY
    value: "socks5://10.193.36.33:7890/"
  - name: NO_PROXY
    value: "127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,harbor.dubhe,10.193.36.252"

关闭亲和性助手(可选)

https://github.com/tektoncd/pipeline/blob/main/config/config-feature-flags.yaml

1
2
kubectl edit configmap feature-flags -n tekton-pipelines
disable-affinity-assistant: "true"

Posted on Edited on

参考

下载

配置hosts

在本机及所有节点配置hosts:
192.168.101.244 harbor.dubhe

Compose方式安装

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
# 解压
tar -zxvf harbor-offline-installer-v2.8.0.tgz && cd harbor

# 配置harbor.yml
cp harbor.yml.tmpl harbor.yml
vi harbor.yml 
1. 注释掉https
2. hostname改为域名 harbor.dubhe或IP
3. 端口改为 5000
4. 修改数据目录/data(可选)

# 安装
./install.sh

# 启停
docker-compose down
docker-compose up -d

# 重新生成配置
./prepare

Helm方式安装

镜像清单

1
2
3
4
5
6
7
8
9
10
11
12
goharbor/harbor-core:v2.8.0
goharbor/harbor-db:v2.8.0
goharbor/harbor-exporter:v2.8.0
goharbor/harbor-jobservice:v2.8.0
goharbor/harbor-portal:v2.8.0
goharbor/harbor-registryctl:v2.8.0
goharbor/nginx-photon:v2.8.0
goharbor/notary-server-photon:v2.8.0
goharbor/notary-signer-photon:v2.8.0
goharbor/redis-photon:v2.8.0
goharbor/registry-photon:v2.8.0
goharbor/trivy-adapter-photon:v2.8.0

安装过程

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
# 使用dockemon工具将镜像导入到每个节点

# 下载chart并修改
helm repo add harbor https://helm.goharbor.io
helm pull harbor/harbor
修改values.yaml: 
1、type: nodePort
2、tls.enabled: false
3、externalURL: http://harbor.dubhe
4、storageClass: "local-path" 
5、size: 50Gi

# 使用Helm安装
helm install harbor ./harbor_chart --namespace harbor --create-namespace

# 更新
helm upgrade harbor ./harbor_chart --namespace harbor

helm uninstall harbor --namespace harbor

Console

http://harbor.dubhe:30002
admin/yourpass

自签名证书(可选)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
openssl genrsa -aes256 -out ca-key.pem 4096
openssl req -new -x509 -days 365 -key ca-key.pem -sha256 -subj "/CN=harbor.dubhe" -out ca.pem
openssl genrsa -out server-key.pem 4096
openssl req -new -sha256 \
    -key server-key.pem \
    -subj "/C=CN/OU=sdas/O=evayinfo/CN=dubhe" \
    -reqexts SAN \
    -config <(cat /etc/ssl/openssl.cnf \
        <(printf "\n[SAN]\nsubjectAltName=DNS:harbor.dubhe")) \
    -out server.csr
openssl x509 -req -days 365 \
    -in server.csr -out server-cert.pem \
    -CA ca.pem -CAkey ca-key.pem -CAcreateserial \
    -extensions SAN \
    -extfile <(cat /etc/ssl/openssl.cnf <(printf "[SAN]\nsubjectAltName=DNS:harbor.dubhe"))

镜像仓库配置

docker配置http方式私有仓库

1
2
3
4
5
6
7
8
# 配置
mkdir -p /etc/docker
vi /etc/docker/daemon.json
{ "insecure-registries":["harbor.dubhe:30002"] } 

# 重启
killall dockerd
dockerd&

containerd(k3s)配置http方式私有仓库

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
# 配置DNS
echo "10.193.36.252  harbor.dubhe">/etc/hosts
# 配置
mkdir -p /etc/rancher/k3s
cat << EOF >/etc/rancher/k3s/registries.yaml
mirrors:
  "harbor.dubhe:30002":
    endpoint:
      - "http://harbor.dubhe:30002"
configs:
  "harbor.dubhe:30002":
    auth:
      username: admin
      password: yourpass
EOF

# 重启k3s，注意要修改并重启每个节点
systemctl restart k3s
systemctl restart k3s-agent

# 查看自动生成的配置
cat /var/lib/rancher/k3s/agent/etc/containerd/config.toml

containerd(k3s)配置镜像仓库

https://docs.k3s.io/zh/installation/private-registry

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
# 配置DNS
echo "10.193.36.50  harbor.dubhe">>/etc/hosts
# 配置
mkdir -p /etc/rancher/k3s
cat << EOF >/etc/rancher/k3s/registries.yaml
mirrors:
  "docker.io":
    endpoint:
      - "http://harbor.dubhe:30002"
  "quay.io":
    endpoint:
      - "http://harbor.dubhe:30002"
  "ghcr.io":
    endpoint:
      - "http://harbor.dubhe:30002"
  "k8s.gcr.io":
    endpoint:
      - "http://harbor.dubhe:30002"
  "registry.k8s.io":
    endpoint:
      - "http://harbor.dubhe:30002"
  "gcr.io":
    endpoint:
      - "http://harbor.dubhe:30002"
  "harbor.dubhe:30002":
    endpoint:
      - "http://harbor.dubhe:30002"
configs:
  "harbor.dubhe:30002":
    auth:
      username: admin
      password: yourpass
EOF

      
# 重启k3s，注意要修改并重启每个节点
systemctl restart k3s
systemctl restart k3s-agent

# 查看自动生成的配置
cat /var/lib/rancher/k3s/agent/etc/containerd/config.toml

Posted on Edited on

官网

https://github.com/juewuy/ShellClash/blob/master/README_CN.md

跳板机

ssh root@10.193.36.33

配置

1
2
3
4
5
6
7
8
9
10
11
# 临时开启反向SSH隧道
ssh -R 7890:127.0.0.1:7890 root@10.193.36.33
export https_proxy=http://127.0.0.1:7890 http_proxy=http://127.0.0.1:7890 all_proxy=http://127.0.0.1:7890

# 在线安装ShellClash
export url='https://fastly.jsdelivr.net/gh/juewuy/ShellClash@master' && wget -q --no-check-certificate -O /tmp/install.sh $url/install.sh  && bash /tmp/install.sh && source /etc/profile &> /dev/null

# 导入机场订阅

# UI面板
http://10.193.36.33:9999/ui

常规代理

1
export https_proxy=http://10.193.36.33:7890 http_proxy=http://10.193.36.33:7890 all_proxy=http://10.193.36.33:7890

docker配置http代理

1
2
3
4
5
6
7
8
9
10
11
12
# 创建配置文件
mkdir -p /etc/systemd/system/docker.service.d
# 编辑配置文件
cat << EOF >/etc/systemd/system/docker.service.d/http-proxy.conf
[Service]
Environment="HTTP_PROXY=socks5://10.193.36.33:7890/"
Environment="HTTPS_PROXY=socks5://10.193.36.33:7890/"
Environment="NO_PROXY=localhost,127.0.0.1,harbor.dubhe,10.193.36.252"
EOF
# 重启 Docker
systemctl daemon-reload && systemctl restart docker
systemctl show --property=Environment docker

containerd(k3s)配置http代理

https://docs.k3s.io/zh/advanced

1
2
3
4
5
6
7
8
9
10
11
12
13
vi /etc/systemd/system/k3s.service.env        # 在master节点
vi /etc/systemd/system/k3s-agent.service.env  # 在agent节点

HTTP_PROXY=socks5://10.193.36.33:7890/
HTTPS_PROXY=socks5://10.193.36.33:7890/
NO_PROXY=127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,harbor.dubhe,10.193.36.252,10.193.35.11

# 重启k3s，注意要修改并重启每个节点
systemctl restart k3s
systemctl restart k3s-agent

# 查看自动生成的配置
cat /var/lib/rancher/k3s/agent/etc/containerd/config.toml

Posted on Edited on

参考

k3s离线部署

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
# 拷贝文件
sudo mkdir -p /var/lib/rancher/k3s/agent/images/
gunzip k3s-airgap-images-amd64.tar.gz
sudo cp ./k3s-airgap-images-amd64.tar /var/lib/rancher/k3s/agent/images/
sudo chmod +x k3s
sudo cp ./k3s /usr/local/bin/
chmod +x install.sh

# server端安装
INSTALL_K3S_SKIP_DOWNLOAD=true ./install.sh

# 查看安装状态
kubectl get pods --all-namespaces

# 查看token
cat /var/lib/rancher/k3s/server/token

# client端安装
INSTALL_K3S_SKIP_DOWNLOAD=true K3S_URL=https://10.193.36.61:6443 \
K3S_TOKEN=K1010e7f60c6242eaa23cdbfc4ac1da1476c281fb740e871e8639b049aad3a8aa8d::server:467f87f6a531a63cd8a9cb74c1caa890 \
./install.sh

# 配置local-storage路径
vi /etc/systemd/system/k3s.service
ExecStart=/usr/local/bin/k3s server --default-local-storage-path /dubhe
systemctl daemon-reload && systemctl restart k3s
cat /var/lib/rancher/k3s/server/manifests/local-storage.yaml # 验证

常用运维操作

启停

1
2
3
4
5
6
7
8
9
10
11
12
13
14
# 启停
systemctl restart k3s
systemctl restart k3s-agent

# 强杀
/usr/local/bin/k3s-killall.sh

# 卸载
/usr/local/bin/k3s-uninstall.sh
/usr/local/bin/k3s-agent-uninstall.sh

# 查看日志
journalctl -u k3s -f -n100
journalctl -u containerd

server节点访问

1
2
3
4
5
6
7
8
9
# 配置.profile
alias ctr="ctr --address /run/k3s/containerd/containerd.sock --namespace k8s.io"
export KUBECONFIG=/etc/rancher/k3s/k3s.yaml

# 测试
kubectl get pods --all-namespaces
helm ls --all-namespaces
ctr c ls
crictl ps

本机访问

将/etc/rancher/k3s/k3s.yaml复制到本机的 ~/.kube/config，并将server字段的值替换为Server的IP
接下来可使用kubectl或k9s进行远程管理

管理工具安装

nerdctl离线安装

https://github.com/containerd/nerdctl

1
2
3
4
5
# 安装
tar -zxvf nerdctl-1.3.1-linux-amd64.tar.gz -C /usr/local/bin

# 配置alias
alias nerdctl="nerdctl --host=/run/k3s/containerd/containerd.sock --namespace k8s.io --insecure-registry"

k9s离线安装

https://i.cloudnative.to/toolkits/kubernetes/k9s
https://k9scli.io/topics/install/

1
2
tar -zxvf k9s_Linux_amd64.tar.gz
cp ./k9s /usr/local/bin/

helm离线安装

https://github.com/helm/helm/releases

1
2
tar -zxvf helm-v3.11.3-linux-amd64.tar.gz
mv linux-amd64/helm /usr/local/bin/helm

宿主机联网

本机开启HTTP代理端口

方式有多种，可以:
1、使用ClashX在本机开启一个7890的Socks5端口
2、使用Privoxy将其转为HTTP端口8118

SSH反向代理

1
2
3
4
5
6
7
8
9
10
# 开启GatewayPorts(使局域网中其它机器也可访问代理端口)
vi /etc/ssh/sshd_config
GatewayPorts yes
systemctl restart sshd
# SSH接入其中一台服务器
ssh -R 7890:127.0.0.1:8118 root@10.193.35.11
# 环境变量(每台机器)
export https_proxy=http://10.193.35.11:7890 http_proxy=http://10.193.35.11:7890 all_proxy=http://10.193.35.11:7890
# 测试
curl cip.cc

配置k3s

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
# 编辑
vi /etc/systemd/system/k3s.service.env  # server节点
vi /etc/systemd/system/k3s-agent.service.env  # agent节点

# 添加
HTTP_PROXY=http://10.193.35.11:7890
HTTPS_PROXY=http://10.193.35.11:7890
NO_PROXY=127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16

# 重启
systemctl restart k3s  # server节点
systemctl restart k3s-agent  # agent节点

# 测试
ctr i pull docker.io/library/nginx:alpine
crictl pull nginx:alpine

Posted on Edited on

参考

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
# 安装registry
docker run -d --name registry -p 5000:5000 -v /dubhe/registry:/var/lib/registry  registry:2
docker run -d --name registry --network host -v /dubhe/registry:/var/lib/registry registry:2

# 配置daemon.json并测试
docker image tag ubuntu 10.193.36.33:5000/myfirstimage
docker push 10.193.36.33:5000/myfirstimage
docker pull 10.193.36.33:5000/myfirstimage

# 安装UI
docker run -d --name registry-ui \
-e ENV_DOCKER_REGISTRY_HOST=10.193.36.33 \
-e ENV_DOCKER_REGISTRY_PORT=5000 \
-p 8080:80 \
konradkleine/docker-registry-frontend:v2

http://10.193.36.33:8080

Posted on Edited on

节点亲和性

1
2
kubectl get nodes --show-labels
kubectl label nodes worker-s001 node-role.kubernetes.io/worker=ci

构建所需基础镜像

1
2
3
busybox:1.36.0
java:8u111-jre-alpine
openjdk:8-jre-slim-buster

自定义容器

  1. 编辑kubesphere-devops-system命名空间下的名为jenkins-casc-config的configmap，
  2. 重启名为devops-jenkins-xxx的deployment
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    49
    50
    51
    52
    53
    54
    55
    56
    57
    58
    59
    60
    61
    62
    63
    64
    65
    66
    67
    68
    69
    70
    71
    72
    73
    74
    75
    76
    77
    78
    79
    80
    81
    82
    83
    84
    85
    86
    87
    88
    89
    90
    91
    92
    93
    94
    95
    96
    97
    98
    99
    100
    101
    - name: "dubhe"
      namespace: "kubesphere-devops-worker"
      label: "dubhe"
      nodeUsageMode: "EXCLUSIVE"
      idleMinutes: 0
      containers:
      - name: "maven"
        image: "kubesphere/builder-maven:v3.2.0-podman"
        command: "cat"
        args: ""
        ttyEnabled: true
        privileged: true
        resourceRequestCpu: "100m"
        resourceLimitCpu: "4000m"
        resourceRequestMemory: "100Mi"
        resourceLimitMemory: "8192Mi"
      - name: "nodejs"
        image: "kubesphere/builder-nodejs:v3.2.0-podman"
        command: "cat"
        args: ""
        ttyEnabled: true
        privileged: true
        resourceRequestCpu: "100m"
        resourceLimitCpu: "4000m"
        resourceRequestMemory: "100Mi"
        resourceLimitMemory: "8192Mi"
      - name: "jnlp"
        image: "jenkins/inbound-agent:4.10-2"
        args: "^${computer.jnlpmac} ^${computer.name}"
        resourceRequestCpu: "50m"
        resourceLimitCpu: "500m"
        resourceRequestMemory: "400Mi"
        resourceLimitMemory: "1536Mi"
      workspaceVolume:
        emptyDirWorkspaceVolume:
          memory: false
      volumes:
      - hostPathVolume:
          hostPath: "/run/k3s/containerd/containerd.sock"
          mountPath: "/run/containerd/containerd.sock"
      - hostPathVolume:
          hostPath: "/var/data/jenkins_maven_cache"
          mountPath: "/root/.m2"
      - hostPathVolume:
          hostPath: "/var/data/jenkins_sonar_cache"
          mountPath: "/root/.sonar/cache"
      - hostPathVolume:
          hostPath: "/var/data/jenkins_nodejs_yarn_cache"
          mountPath: "/root/.yarn"
      - hostPathVolume:
          hostPath: "/var/data/jenkins_nodejs_npm_cache"
          mountPath: "/root/.npm"
      yaml: |
        spec:
          hostAliases:
          - ip: "10.193.35.21"
            hostnames:
            - "harbor.dubhe"
          affinity:
            nodeAffinity:
              preferredDuringSchedulingIgnoredDuringExecution:
              - weight: 1
                preference:
                  matchExpressions:
                  - key: node-role.kubernetes.io/worker
                    operator: In
                    values:
                    - ci
          tolerations:
          - key: "node.kubernetes.io/ci"
            operator: "Exists"
            effect: "NoSchedule"
          - key: "node.kubernetes.io/ci"
            operator: "Exists"
            effect: "PreferNoSchedule"
          containers:
          - name: "maven"
            resources:
              requests:
                ephemeral-storage: "1Gi"
              limits:
                ephemeral-storage: "10Gi"
            volumeMounts:
            - name: config-volume
              mountPath: /opt/apache-maven-3.5.3/conf/settings.xml
              subPath: settings.xml
          - name: "nodejs"
            resources:
              requests:
                ephemeral-storage: "1Gi"
              limits:
                ephemeral-storage: "10Gi"
          volumes:
            - name: config-volume
              configMap:
                name: ks-devops-agent
                items:
                - key: MavenSetting
                  path: settings.xml
          securityContext:
            fsGroup: 1000

自定义maven settings.xml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
<?xml version="1.0" encoding="UTF-8"?>
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
    <pluginGroups>
    </pluginGroups>
    <proxies>
    </proxies>
    <servers>
    </servers>
    <mirrors>
        <mirror>
            <id>nexus-aliyun</id>
            <mirrorOf>*,!jitpack.io,!elasticsearch-releases,!dtstack-github,!nexus-public,!elastic.co</mirrorOf>
            <name>Nexus aliyun</name>
            <url>http://maven.aliyun.com/nexus/content/groups/public</url>
        </mirror>
    </mirrors>
    <profiles>
    </profiles>
</settings>

Posted on Edited on

网络配置

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
# centos/ubuntu传统配置
nmtui
或
vi /etc/network/interfaces
auto eth0
iface eth0 inet static
    address 192.0.2.7
    netmask 255.255.255.0
    gateway 192.0.2.254
systemctl restart networking

# ubuntu netplan方式配置
vim /etc/netplan/00-installer-config.yaml
network:
  version: 2
  renderer: networkd
  ethernets:
    enp0s9:
      dhcp4: no
      addresses:
        - 192.168.121.221/24
      gateway4: 192.168.121.1
      nameservers:
          addresses: [8.8.8.8, 1.1.1.1]
netplan apply

开启root用户ssh登录

1
2
3
4
5
6
# ubuntu下设置root密码
sudo passwd

# 开启SSH
vi /etc/ssh/sshd_config
PermitRootLogin yes

配置hostname

1
hostnamectl set-hostname k3s11

磁盘挂载

1
2
3
4
5
6
7
8
9
10
11
12
13
# 格盘
df -h
lsblk -l
mkfs.ext4 /dev/vdc

# 挂载
lsblk -f
mkdir /dubhe
vi /etc/fstab
UUID=6fbe9169-6432-4186-bfb9-6ec9e3d309eb  /dubhe  ext4  defaults  0  0
# tidb需要添加nodelalloc,noatime参数
UUID=356e7933-6f43-4d02-97f2-213ecdc9499f  /dubhe  ext4  defaults,nodelalloc,noatime  0  0
mount -a

通用调整

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
# Ulimit设置
cat <<EOF >>  /etc/security/limits.conf
root        soft        nofile        1048576
root        hard        nofile        1048576
root        soft        stack         10240
EOF
sysctl --system

# 关闭Swap
swapoff -a
sed -i 's/^\(.*swap.*\)$/#\1/' /etc/fstab

# 关闭防火墙(centos)
systemctl stop firewalld
systemctl disable firewalld

# 关闭防火墙(ubuntu)
ufw status verbose
ufw disable

# 禁用selinux(only centos)
setenforce 0
sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config

K8S专属调整

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
# 配置IP转发
iptables -P FORWARD ACCEPT

# 内核参数调优
modprobe br_netfilter
cat <<EOF >  /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-arptables = 1
net.core.somaxconn = 32768
vm.swappiness = 0
net.ipv4.tcp_syncookies = 0
net.ipv4.ip_forward = 1
fs.file-max = 1000000
fs.inotify.max_user_watches = 1048576
fs.inotify.max_user_instances = 1024
net.ipv4.conf.all.rp_filter = 1
net.ipv4.neigh.default.gc_thresh1 = 80000
net.ipv4.neigh.default.gc_thresh2 = 90000
net.ipv4.neigh.default.gc_thresh3 = 100000
EOF
sysctl --system

TiDB专属调整

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
# ulimit
cat << EOF >>/etc/security/limits.conf
tidb           soft    nofile          1000000
tidb           hard    nofile          1000000
tidb           soft    stack          32768
tidb           hard    stack          32768
EOF

# TIDB配置irqbalance服务
systemctl status irqbalance
systemctl enable irqbalance
systemctl start irqbalance

# 内核参数调整
echo never > /sys/kernel/mm/transparent_hugepage/enabled
echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo 'export GODEBUG=madvdontneed=1' >> /root/.bash_profile && source /root/.bash_profile

Docker配置

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
# Docker的数据保存到一块单独的盘上
vi /etc/docker/daemon.json
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m"
  },
  "storage-driver": "overlay2",
  "storage-opts": [
    "overlay2.override_kernel_check=true"
  ],
  "data-root": "/data1/docker"
}

# Docker配置http仓库
vi /etc/docker/daemon.json
{
  "insecure-registries" : ["10.193.34.4:9010"]
}


# 设置 Docker daemon 的 ulimit
## 1、创建 docker service 的 systemd drop-in 目录 /etc/systemd/system/docker.service.d
mkdir -p /etc/systemd/system/docker.service.d
## 2、创建 /etc/systemd/system/docker.service.d/limit-nofile.conf 文件，并配置 LimitNOFILE 参数的值，取值范围为大于等于 1048576 的数字即可
cat > /etc/systemd/system/docker.service.d/limit-nofile.conf <<EOF
[Service]
LimitNOFILE=1048576
EOF
## 3、重新加载配置
systemctl daemon-reload && systemctl restart docker

配置软件源

ubuntu在线阿里源

1
2
3
4
5
6
7
8
9
10
11
12
13
# 参考: https://developer.aliyun.com/mirror/ubuntu/
cp /etc/apt/sources.list /etc/apt/sources.list.bak
cat << EOF >/etc/apt/sources.list
deb https://mirrors.aliyun.com/ubuntu/ focal main restricted universe multiverse
deb-src https://mirrors.aliyun.com/ubuntu/ focal main restricted universe multiverse
deb https://mirrors.aliyun.com/ubuntu/ focal-security main restricted universe multiverse
deb-src https://mirrors.aliyun.com/ubuntu/ focal-security main restricted universe multiverse
deb https://mirrors.aliyun.com/ubuntu/ focal-updates main restricted universe multiverse
deb-src https://mirrors.aliyun.com/ubuntu/ focal-updates main restricted universe multiverse
deb https://mirrors.aliyun.com/ubuntu/ focal-backports main restricted universe multiverse
deb-src https://mirrors.aliyun.com/ubuntu/ focal-backports main restricted universe multiverse
EOF
apt update

centos在线阿里源

1
2
3
4
5
mv /etc/yum.repos.d /etc/yum.repos.d.bak
mkdir /etc/yum.repos.d
curl -o /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
curl -o /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo
yum clean all && yum makecache

centos本地源

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
# 挂载镜像
mount -t iso9660 -o loop,utf8 your.iso /mnt/yum

# 清空Yum配置目录
mkdir -p /etc/yum.repos.d.bak
mv /etc/yum.repos.d/* /etc/yum.repos.d.bak

# 添加本地Yum源配置
cat << EOF >/etc/yum.repos.d/CentOS-Local.repo
[local]
name=local
baseurl=file:///mnt/yum
enabled=1
gpgcheck=1
gpgkey=file:///mnt/yum
EOF

# 重建Yum缓存
yum clean all && yum makecache

开启ntp

1
2
3
4
yum install ntp ntpdate -y
systemctl start ntpd.service
systemctl enable ntpd.service
ntpstat

Posted on Edited on

亲和性

https://tachingchen.com/tw/blog/kubernetes-assigning-pod-to-nodes/

1
2
kubectl get nodes --show-labels
kubectl label nodes worker-s001 dubhe/role=middleware

deploy.yaml: 

1
2
3
4
5
6
7
8
9
10
11
12
13
spec:
  template:
    spec:
      affinity:
        nodeAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
            - weight: 1
              preference:
                matchExpressions:
                  - key: dubhe/role
                    operator: In
                    values:
                      - middleware

coredns自定义解析

1
2
3
4
5
6
7
8
9
10
11
12
13
# 编辑configmap
kubectl -n kube-system edit configmap coredns

# 添加自定义解析
        hosts /etc/coredns/NodeHosts {
          10.193.36.252 harbor.dubhe
          10.193.36.33  gogs.dubhe
          ttl 60
          reload 15s
          fallthrough
        }

# 重启coredns的deployment

Posted on Edited on

参考

配置自动补全

1
2
3
4
5
6
7
8
9
# bash
source <(kubectl completion bash) # 在 bash 中设置当前 shell 的自动补全，要先安装 bash-completion 包。
echo "source <(kubectl completion bash)" >> ~/.bashrc # 在你的 bash shell 中永久地添加自动补全
# zsh
source <(kubectl completion zsh)  # 在 zsh 中设置当前 shell 的自动补全
echo '[[ $commands[kubectl] ]] && source <(kubectl completion zsh)' >> ~/.zshrc # 在你的 zsh shell 中永久地添加自动补全
# alias
alias k=kubectl
complete -o default -F __start_kubectl k

docker转kubectl操作

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
# 测试容器
kubectl run test-postfix --restart='Never' --image postfix:1.1 --command -- sleep infinity
kubectl exec -it test-postfix -- bash
kubectl delete pod test-postfix --force

# run
docker run -d --restart=always -e DOMAIN=cluster --name nginx-app -p 80:80 nginx
kubectl run --image=nginx nginx-app --port=80 --env="DOMAIN=cluster"

# exec
docker exec nginx-app cat /etc/hostname
kubectl exec nginx-app -- cat /etc/hostname

# exec shell
docker exec -it nginx-app /bin/sh
kubectl exec -it nginx-app -- /bin/sh

# log
docker logs -f --tail 100 nginx-app
kubectl logs -f --tail 100 nginx-app

# delete
docker stop nginx-app && docker rm -f nginx-app
kubectl delete pod nginx-app --force

# 复杂run
docker run -itd --name redis \
-p 6379:6379 \
-v /tmp/data:/data \
--env foo=bar \
redis:5.0.14-alpine --requirepass "123456"

kubectl run test-redis \
--image=redis:5.0.14-alpine \
--restart=Never \
--port=6379 \
--env="foo=bar" \
-- --requirepass "123456"

运行

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
# 命令行
kubectl run myterminal --image=busybox:1.28 --restart=Never -- bash

# 生成pod
kubectl run test-pod --image=nginx --restart=Never # Always/OnFailure/Never
# 创建Job
kubectl create job test-job --image=busybox:1.28 -- echo "Hello World" 
# 创建Cronjob
kubectl create cronjob test-cron --image=busybox:1.28 --schedule="*/1 * * * *" -- echo "Hello World"   
# 生成deploy
kubectl create deployment test-dp --image=nginx --replicas=2
# 生成svc
kubectl expose deployment test-dp --type=ClusterIP --port=80 --target-port=8000 --name=test-svc
kubectl delete svc test-svc
# 生成名称空间
kubectl create ns test-ns
# 生成configmap
kubectl create configmap test-cm --from-literal=key1=value1 --from-literal=key2=value2

查看

1
2
3
4
5
6
7
8
9
10
11
12
13
# 查看所有namespace的pod
kubectl get pods -A  # 或--all-namespaces
kubectl get pods test-pod -o wide  # 展示更多列
# 获取所有deployment
kubectl get deployment
# 查看deployment和servers
kubectl get deployment,services
# 查看pod详情
kubectl describe pods test-pod --namespace=default
# 查看pod日志
kubectl logs -f --tail 100 test-pod
# 查看pod环境变量
kubectl exec test-pod -- printenv

集群

1
2
3
4
5
6
7
8
9
10
11
12
# 集群核心组件运行情况
kubectl cluster-info
# 命名空间
kubectl get namespaces
# 版本
kubectl version
# api
kubectl api-versions
# 查看事件
kubectl get events
# 获取全部节点
kubectl get nodes

创建

1
2
3
4
5
6
7
8
9
10
11
12
13
# 创建资源
kubectl create -f ./nginx.yaml
# 创建当前目录下的所有yaml资源
kubectl create -f ./
# 使用多个文件创建资源
kubectl create -f ./nginx1.yaml -f ./mysql2.yaml
# 使用目录下的所有文件夹的yaml来创建资源
kubectl create -f ./dir
# 使用 url 来创建资源
kubectl create -f https://xxxx/xxx
# 获取api文档
kubectl explain pods
kubectl explain service

伸缩

1
2
3
4
5
6
# 自动伸缩
kubectl autoscale deployment test-dp --min=2 --max=5
# 修改副本数
kubectl scale --replicas=3 deployment/test-dp
# 变更多个控制器的数量
kubectl scale --replicas=5 rc/foo rc/bar rc/baz

删除

1
2
3
4
# 根据yaml删除
kubectl delete -f ./pod.yaml
# 根据名称删除，强制删除
kubectl delete pods test-pod --force

交互

1
2
3
4
5
6
7
8
9
10
# 交互式 shell 的方式运行 pod
kubectl run -it busybox --image=busybox -- sh
kubectl run -it busybox --image=busybox:1.28 --namespace=dubhe-uat -- sh
kubectl exec -it busybox --namespace=dubhe-uat -- sh
# 在已存在的pod单容器中执行命令
kubectl exec nginx-pod -- ls /
# 在已存在的pod中的容器执行命令
kubectl exec nginx-pod -c my-container -- ls /
## 端口转发，本地监听5000
kubectl port-forward test-pod 5000:80

复制文件

1
2
3
4
kubectl cp /tmp/foo_dir my-pod:/tmp/bar_dir            # 将 /tmp/foo_dir 本地目录复制到远程当前命名空间中 Pod 中的 /tmp/bar_dir
kubectl cp /tmp/foo my-pod:/tmp/bar -c my-container    # 将 /tmp/foo 本地文件复制到远程 Pod 中特定容器的 /tmp/bar 下
kubectl cp /tmp/foo my-namespace/my-pod:/tmp/bar       # 将 /tmp/foo 本地文件复制到远程 “my-namespace” 命名空间内指定 Pod 中的 /tmp/bar
kubectl cp my-namespace/my-pod:/tmp/foo /tmp/bar       # 将 /tmp/foo 从远程 Pod 复制到本地 /tmp/bar

调度配置

1
2
3
4
5
6
7
# 标记my-node不可调度
kubectl cordon k8s-node
# 驱逐my-node上的pod以待维护
kubectl drain k8s-node
# 标记my-node可调度
kubectl uncordon k8s-node

DEBUG

1
2
3
4
5
kubectl run -it debug-pod --image=bluexiii/ubuntu:22.04 --image-pull-policy='Always' -n dubhe-dev  -- bash
kubectl run -it chart-update --image=bluexiii/chart-update:latest --image-pull-policy='Always' -n dubhe-dev  -- bash

kubectl exec -it chart-update --namespace=dubhe-dev -- bash
kubectl delete pod chart-update --namespace=dubhe-dev --force