# 異常疊加場(chǎng)景
-**業(yè)務(wù)層**：10%用戶支付失?。℉TTP500），錯(cuò)誤集中在訂單提交接口`/api/v1/pay`
-**中間件層**：v2PodMySQL連接池達(dá)到上限（100連接），日志報(bào)錯(cuò)`CannotacquireJDBCconnection`
-**網(wǎng)絡(luò)層**：IngressGateway出現(xiàn)0.5%的`NO_ROUTE`錯(cuò)誤，部分請(qǐng)求繞過(guò)Sidecar直連Pod IP

# 檢查Envoy實(shí)際生效配置（對(duì)比聲明式配置）
istioctl proxy-config routes $(kubectl -n istio-system get pod -l app=istio-ingressgateway -o name) \
--name payment-service -o json | jq '.routes[0].route.weightedClusters'

# 繪制服務(wù)依賴圖譜（需安裝kubectl-neat）
kubectl get svc,deploy,pod -l app=payment-service -o json | kubectl-neat | jq '.items[] | {name:.metadata.name, labels:.metadata.labels}'

{
  "name":"payment-service-v2",
"labels":{
    "app":"payment-service",
    "version":"v2",
    "istio.io/rev":"istio-120"http:// 確認(rèn)Sidecar注入版本
}
}

kubectl patch virtualservice payment -type=merge -p \
'{"spec":{"http":[{"route":[{"destination":{"host":"payment-service","subset":"v1"},"weight":100}]}]}}'

watch -n 1 'kubectl exec -n istio-system deploy/istio-ingressgateway -- curl -s http://localhost:15000/stats | grep v2.upstream_rq_active'
# 預(yù)期輸出：v2.upstream_rq_active 0

# 通過(guò)標(biāo)簽驅(qū)逐v2 Pod
kubectl label pods -l version=v2 version=quarantine --overwrite
kubectl scale deploy/payment-service-v2 --replicas=0

# 清理殘留Endpoint
kubectl get endpoints payment-service -o json | jq '.subsets[].addresses |= map(select(.targetRef.resourceVersion != "v2"))' | kubectl apply -f -

# 強(qiáng)制所有流量經(jīng)過(guò)Sidecar（NetworkPolicy+AuthorizationPolicy雙保險(xiǎn)）
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: payment-service-strict
spec:
  podSelector:
    matchLabels:
      app: payment-service
  policyTypes: [Ingress, Egress]
  ingress:
  - from:
    - podSelector:
        matchLabels:
          istio: ingressgateway
---
apiVersion: security.istio.io/v1beta1
kind: AuthorizationPolicy
metadata:
  name: payment-service-mesh-only
spec:
  selector:
    matchLabels:
      app: payment-service
  rules:
  - from:
    - source:
        principals: ["cluster.local/ns/istio-system/sa/istio-ingressgateway-service-account"]

# 容器級(jí)網(wǎng)絡(luò)隔離（基于nsenter）
kubectl exec payment-service-v2-xxxxx -c istio-proxy -- nsenter -t 1 -n iptables -A OUTPUT -p tcp --dport 3306 -j DROP

# 抓取容器啟動(dòng)參數(shù)（分析資源限制）
kubectl get pod payment-service-v2-xxxxx -o jsonpath='{.spec.containers[*].resources}' | jq

# 采集內(nèi)核日志（定位OOM等底層問(wèn)題）
kubectl exec payment-service-v2-xxxxx -- dmesg --time-format iso > dmesg.log

# 導(dǎo)出Envoy全量配置（含動(dòng)態(tài)更新歷史）
istioctl proxy-config all payment-service-v2-xxxxx --file envoy_config

# 錄制故障時(shí)間窗的訪問(wèn)日志（JSON格式）
kubectl exec payment-service-v2-xxxxx -c istio-proxy -- curl -X POST http://localhost:15000/logging?level=trace
kubectl logs payment-service-v2-xxxxx -c istio-proxy --since=10m > envoy_access.log

# Java應(yīng)用連續(xù)線程快照（間隔5秒）
for i in {0..5}; do
  kubectl exec payment-service-v2-xxxxx -- pgrep -f payment-service | xargs -I {} jstack {} > jstack_$i.log
  sleep 5
done

# 內(nèi)存泄漏追蹤（結(jié)合jemalloc）
kubectl exec payment-service-v2-xxxxx -- env MALLOC_CONF=prof:true,lg_prof_interval:30 java -jar app.jar
kubectl cp payment-service-v2-xxxxx:/tmp/heap.hprof .

# 時(shí)間軸對(duì)齊工具（示例）
import pandas as pd

logs = pd.read_csv('envoy_access.log', parse_dates=['timestamp'])
metrics = pd.read_csv('prometheus_metrics.csv', parse_dates=['timestamp'])
joined = pd.merge_asof(logs, metrics, on='timestamp', tolerance=pd.Timedelta('1s'))
joined[joined['status_code'] == 500].plot(x='timestamp', y=['cpu_usage', 'active_connections'])

# 動(dòng)態(tài)追蹤數(shù)據(jù)庫(kù)連接打開(kāi)/關(guān)閉（基于eBPF）
kubectl debug payment-service-v2-xxxxx -it --image=nicolaka/netshoot \
-- bash -c "bpftrace -e 'tracepoint:syscalls:sys_enter_close { printf(\"Closed FD: %d\n\", args->fd); }'"

-- 連接使用熱點(diǎn)分析
SELECT 
  user, host, command, 
  COUNT(*) as total_conn,
  SUM(state='Sleep') as idle_conn,
  SUM(state='Query') as active_conn 
FROM information_schema.processlist 
GROUP BY user, host, command;

// Hikari連接池泄漏檢測(cè)（擴(kuò)展配置）
HikariConfig config = new HikariConfig();
config.setLeakDetectionThreshold(5000);  // 5秒未歸還連接即報(bào)泄漏
config.setRegisterMbeans(true);

# 復(fù)合灰度策略模板
http:
- match:
    - headers:
        x-env: ["canary"]
    - sourceLabels: [request.auth.claims["group"]]
      values: ["premium"]
  route:
    - destination:
        host: payment-service
        subset: v2
      weight: 25
    - destination:
        host: payment-service
        subset: v1
      weight: 75
  mirror:
    host: payment-service-shadow
    percentage: 
      value: 100

// Jenkins Pipeline集成測(cè)試
pipeline {
  stages {
    stage('Deploy Canary') {
      steps {
        sh 'kubectl apply -f virtualservice-canary.yaml'
      }
    }
    stage('Validation') {
      steps {
        sh 'fortio load -c 10 -qps 100 -t 300s -H "X-Env: canary" http://payment-service/api/v1/pay'
        sh '''
          ERROR_RATE=$(curl -s http://prometheus/api/v1/query?query=rate(http_requests_total{status=~"5.."}[1m]) | jq '.data.result[0].value[1]')
          if [ $(echo "$ERROR_RATE > 0.01" | bc -l) -eq 1 ]; then
            exit 1
          fi
        '''
      }
    }
  }
}

apiVersion: chaos-mesh.org/v1alpha1
kind: MySQLChaos
metadata:
  name: mysql-connection-pool-failure
spec:
  action: delay
  mode: one
  selector:
    namespaces: ["default"]
  delay:
    latency: "2s"
    correlation: "100"
  duration: "10m"

apiVersion: chaos-mesh.org/v1alpha1
kind: NetworkChaos
metadata:
  name: payment-service-partition
spec:
  action: partition
  direction: both
  mode: all
  selector:
    namespaces: ["default"]
    labelSelectors:
      app: "payment-service"
      version: "v2"
  duration: "5m"

# 灰度權(quán)重動(dòng)態(tài)調(diào)整算法（偽代碼）
classGrayScaleController:
    def__init__(self):
        self.model = load_ai_model()
        
    defadjust_weights(self, metrics):
        success_rate = metrics['http_success_rate']
        latency_p99 = metrics['latency_p99']
        conn_usage = metrics['db_connection_usage']
        
        # AI模型輸出權(quán)重調(diào)整建議
        action = self.model.predict(success_rate, latency_p99, conn_usage)
        new_weight = action * 10# 按10%步長(zhǎng)調(diào)整
        
        patch_virtual_service(new_weight)

# 基于實(shí)際流量錄制回放
kubectl exec -n istio-system deploy/istio-ingressgateway -- \
curl -X POST http://localhost:15019/debug/tcpdump \
-d '{
  "duration": "60s",
  "interface": "eth0",
  "filter": "tcp port 8080",
  "outputPath": "/tmp/capture.pcap"
}'

# 使用GoReplay進(jìn)行流量復(fù)制
goreplay --input-file capture.pcap --output-http="http://payment-service-canary" --rate 200%

# 多集群灰度路由策略
apiVersion: networking.istio.io/v1alpha3
kind: ServiceEntry
metadata:
  name: cross-cluster-payment
spec:
  hosts:
  - payment-service.global
  ports:
  - number: 80
    name: http
    protocol: HTTP
  resolution: DNS
  addresses:
  - 240.0.0.1
  location: MESH_INTERNAL
  endpoints:
  - address: payment-service-v1.cluster-1.svc.cluster.local
    labels:
      version: v1
  - address: payment-service-v2.cluster-2.svc.cluster.local
    labels:
      version: v2