kubectl apply -f samples/addons

$ kubectl get svc prometheus -n istio-system
NAME         TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)    AGE
prometheus   ClusterIP   10.106.228.196   <none>        9090/TCP   25d
$ kubectl get pods -n istio-system -l app=prometheus
NAME                         READY   STATUS    RESTARTS       AGE
prometheus-5d5d6d6fc-2gtxm   2/2     Running   0              25d

istioctl dashboard prometheus
# 也可以創(chuàng)建 Ingress 或者 Gateway 來(lái)訪問(wèn) Prometheus UI

istio_requests_total{destination_service="productpage.default.svc.cluster.local"}

istio_requests_total{destination_service="reviews.default.svc.cluster.local", destination_versinotallow="v3"}

rate(istio_requests_total{destination_service=~"productpage.*", response_code="200"}[5m])

kubectl apply -f samples/addons

$ kubectl -n istio-system get svc grafana
NAME      TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)    AGE
grafana   ClusterIP   10.96.197.74   <none>        3000/TCP   25d
$ kubectl -n istio-system get pods -l app=grafana
NAME                       READY   STATUS    RESTARTS       AGE
grafana-5f9b8c6c5d-jv65v   1/1     Running   0              25d

istioctl dashboard grafana
# 也可以創(chuàng)建 Ingress 或者 Gateway 來(lái)訪問(wèn) Grafana

for i in $(seq 1 100); do curl -s -o /dev/null "http://$GATEWAY_URL/productpage"; done

# Source: prometheus/templates/service.yaml
apiVersion: v1
kind: Service
metadata:
  labels:
    component: "server"
    app: prometheus
    release: prometheus
    chart: prometheus-19.6.1
    heritage: Helm
  name: prometheus
  namespace: istio-system
spec:
  ports:
    - name: http
      port: 9090
      protocol: TCP
      targetPort: 9090
  selector:
    component: "server"
    app: prometheus
    release: prometheus
  sessionAffinity: None
  type: "ClusterIP"
---
# Source: prometheus/templates/deploy.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    component: "server"
    app: prometheus
    release: prometheus
    chart: prometheus-19.6.1
    heritage: Helm
  name: prometheus
  namespace: istio-system
spec:
  selector:
    matchLabels:
      component: "server"
      app: prometheus
      release: prometheus
  replicas: 1
  strategy:
    type: Recreate
    rollingUpdate: null
  template:
    metadata:
      labels:
        component: "server"
        app: prometheus
        release: prometheus
        chart: prometheus-19.6.1
        heritage: Helm
        sidecar.istio.io/inject: "false"
    spec:
      enableServiceLinks: true
      serviceAccountName: prometheus
      containers:
        - name: prometheus-server-configmap-reload
          image: "jimmidyson/configmap-reload:v0.8.0"
          imagePullPolicy: "IfNotPresent"
          args:
            - --volume-dir=/etc/config
            - --webhook-url=http://127.0.0.1:9090/-/reload
          resources: {}
          volumeMounts:
            - name: config-volume
              mountPath: /etc/config
              readOnly: true
        - name: prometheus-server
          image: "prom/prometheus:v2.41.0"
          imagePullPolicy: "IfNotPresent"
          args:
            - --storage.tsdb.retention.time=15d
            - --config.file=/etc/config/prometheus.yml
            - --storage.tsdb.path=/data
            - --web.console.libraries=/etc/prometheus/console_libraries
            - --web.console.templates=/etc/prometheus/consoles
            - --web.enable-lifecycle
          ports:
            - containerPort: 9090
          readinessProbe:
            httpGet:
              path: /-/ready
              port: 9090
              scheme: HTTP
            initialDelaySeconds: 0
            periodSeconds: 5
            timeoutSeconds: 4
            failureThreshold: 3
            successThreshold: 1
          livenessProbe:
            httpGet:
              path: /-/healthy
              port: 9090
              scheme: HTTP
            initialDelaySeconds: 30
            periodSeconds: 15
            timeoutSeconds: 10
            failureThreshold: 3
            successThreshold: 1
          resources: {}
          volumeMounts:
            - name: config-volume
              mountPath: /etc/config
            - name: storage-volume
              mountPath: /data
              subPath: ""
      dnsPolicy: ClusterFirst
      securityContext:
        fsGroup: 65534
        runAsGroup: 65534
        runAsNonRoot: true
        runAsUser: 65534
      terminationGracePeriodSeconds: 300
      volumes:
        - name: config-volume
          configMap:
            name: prometheus
        - name: storage-volume
          emptyDir: {}
# 省略了部分配置

# Source: prometheus/templates/cm.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  labels:
    component: "server"
    app: prometheus
    release: prometheus
    chart: prometheus-19.6.1
    heritage: Helm
  name: prometheus
  namespace: istio-system
data:
  allow-snippet-annotations: "false"
  alerting_rules.yml: |
    {}
  alerts: |
    {}
  prometheus.yml: |
    global:
      evaluation_interval: 1m
      scrape_interval: 15s
      scrape_timeout: 10s
    rule_files:
    - /etc/config/recording_rules.yml
    - /etc/config/alerting_rules.yml
    - /etc/config/rules
    - /etc/config/alerts
    scrape_configs:
    - job_name: prometheus
      static_configs:
      - targets:
        - localhost:9090
    - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      job_name: kubernetes-apiservers
      kubernetes_sd_configs:
      - role: endpoints
      relabel_configs:
      - action: keep
        regex: default;kubernetes;https
        source_labels:
        - __meta_kubernetes_namespace
        - __meta_kubernetes_service_name
        - __meta_kubernetes_endpoint_port_name
      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        insecure_skip_verify: true
    - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      job_name: kubernetes-nodes
      kubernetes_sd_configs:
      - role: node
      relabel_configs:
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)
      - replacement: kubernetes.default.svc:443
        target_label: __address__
      - regex: (.+)
        replacement: /api/v1/nodes/$1/proxy/metrics
        source_labels:
        - __meta_kubernetes_node_name
        target_label: __metrics_path__
      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        insecure_skip_verify: true
    - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      job_name: kubernetes-nodes-cadvisor
      kubernetes_sd_configs:
      - role: node
      relabel_configs:
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)
      - replacement: kubernetes.default.svc:443
        target_label: __address__
      - regex: (.+)
        replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor
        source_labels:
        - __meta_kubernetes_node_name
        target_label: __metrics_path__
      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        insecure_skip_verify: true
    - honor_labels: true
      job_name: kubernetes-service-endpoints
      kubernetes_sd_configs:
      - role: endpoints
      relabel_configs:
      - action: keep
        regex: true
        source_labels:
        - __meta_kubernetes_service_annotation_prometheus_io_scrape
      - action: drop
        regex: true
        source_labels:
        - __meta_kubernetes_service_annotation_prometheus_io_scrape_slow
      - action: replace
        regex: (https?)
        source_labels:
        - __meta_kubernetes_service_annotation_prometheus_io_scheme
        target_label: __scheme__
      - action: replace
        regex: (.+)
        source_labels:
        - __meta_kubernetes_service_annotation_prometheus_io_path
        target_label: __metrics_path__
      - action: replace
        regex: (.+?)(?::\d+)?;(\d+)
        replacement: $1:$2
        source_labels:
        - __address__
        - __meta_kubernetes_service_annotation_prometheus_io_port
        target_label: __address__
      - action: labelmap
        regex: __meta_kubernetes_service_annotation_prometheus_io_param_(.+)
        replacement: __param_$1
      - action: labelmap
        regex: __meta_kubernetes_service_label_(.+)
      - action: replace
        source_labels:
        - __meta_kubernetes_namespace
        target_label: namespace
      - action: replace
        source_labels:
        - __meta_kubernetes_service_name
        target_label: service
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_node_name
        target_label: node
    - honor_labels: true
      job_name: kubernetes-service-endpoints-slow
      kubernetes_sd_configs:
      - role: endpoints
      relabel_configs:
      - action: keep
        regex: true
        source_labels:
        - __meta_kubernetes_service_annotation_prometheus_io_scrape_slow
      - action: replace
        regex: (https?)
        source_labels:
        - __meta_kubernetes_service_annotation_prometheus_io_scheme
        target_label: __scheme__
      - action: replace
        regex: (.+)
        source_labels:
        - __meta_kubernetes_service_annotation_prometheus_io_path
        target_label: __metrics_path__
      - action: replace
        regex: (.+?)(?::\d+)?;(\d+)
        replacement: $1:$2
        source_labels:
        - __address__
        - __meta_kubernetes_service_annotation_prometheus_io_port
        target_label: __address__
      - action: labelmap
        regex: __meta_kubernetes_service_annotation_prometheus_io_param_(.+)
        replacement: __param_$1
      - action: labelmap
        regex: __meta_kubernetes_service_label_(.+)
      - action: replace
        source_labels:
        - __meta_kubernetes_namespace
        target_label: namespace
      - action: replace
        source_labels:
        - __meta_kubernetes_service_name
        target_label: service
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_node_name
        target_label: node
      scrape_interval: 5m
      scrape_timeout: 30s
    - honor_labels: true
      job_name: prometheus-pushgateway
      kubernetes_sd_configs:
      - role: service
      relabel_configs:
      - action: keep
        regex: pushgateway
        source_labels:
        - __meta_kubernetes_service_annotation_prometheus_io_probe
    - honor_labels: true
      job_name: kubernetes-services
      kubernetes_sd_configs:
      - role: service
      metrics_path: /probe
      params:
        module:
        - http_2xx
      relabel_configs:
      - action: keep
        regex: true
        source_labels:
        - __meta_kubernetes_service_annotation_prometheus_io_probe
      - source_labels:
        - __address__
        target_label: __param_target
      - replacement: blackbox
        target_label: __address__
      - source_labels:
        - __param_target
        target_label: instance
      - action: labelmap
        regex: __meta_kubernetes_service_label_(.+)
      - source_labels:
        - __meta_kubernetes_namespace
        target_label: namespace
      - source_labels:
        - __meta_kubernetes_service_name
        target_label: service
    - honor_labels: true
      job_name: kubernetes-pods
      kubernetes_sd_configs:
      - role: pod
      relabel_configs:
      - action: keep
        regex: true
        source_labels:
        - __meta_kubernetes_pod_annotation_prometheus_io_scrape
      - action: drop
        regex: true
        source_labels:
        - __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow
      - action: replace
        regex: (https?)
        source_labels:
        - __meta_kubernetes_pod_annotation_prometheus_io_scheme
        target_label: __scheme__
      - action: replace
        regex: (.+)
        source_labels:
        - __meta_kubernetes_pod_annotation_prometheus_io_path
        target_label: __metrics_path__
      - action: replace
        regex: (\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})
        replacement: '[$2]:$1'
        source_labels:
        - __meta_kubernetes_pod_annotation_prometheus_io_port
        - __meta_kubernetes_pod_ip
        target_label: __address__
      - action: replace
        regex: (\d+);((([0-9]+?)(\.|$)){4})
        replacement: $2:$1
        source_labels:
        - __meta_kubernetes_pod_annotation_prometheus_io_port
        - __meta_kubernetes_pod_ip
        target_label: __address__
      - action: labelmap
        regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+)
        replacement: __param_$1
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+)
      - action: replace
        source_labels:
        - __meta_kubernetes_namespace
        target_label: namespace
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_name
        target_label: pod
      - action: drop
        regex: Pending|Succeeded|Failed|Completed
        source_labels:
        - __meta_kubernetes_pod_phase
    - honor_labels: true
      job_name: kubernetes-pods-slow
      kubernetes_sd_configs:
      - role: pod
      relabel_configs:
      - action: keep
        regex: true
        source_labels:
        - __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow
      - action: replace
        regex: (https?)
        source_labels:
        - __meta_kubernetes_pod_annotation_prometheus_io_scheme
        target_label: __scheme__
      - action: replace
        regex: (.+)
        source_labels:
        - __meta_kubernetes_pod_annotation_prometheus_io_path
        target_label: __metrics_path__
      - action: replace
        regex: (\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})
        replacement: '[$2]:$1'
        source_labels:
        - __meta_kubernetes_pod_annotation_prometheus_io_port
        - __meta_kubernetes_pod_ip
        target_label: __address__
      - action: replace
        regex: (\d+);((([0-9]+?)(\.|$)){4})
        replacement: $2:$1
        source_labels:
        - __meta_kubernetes_pod_annotation_prometheus_io_port
        - __meta_kubernetes_pod_ip
        target_label: __address__
      - action: labelmap
        regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+)
        replacement: __param_$1
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+)
      - action: replace
        source_labels:
        - __meta_kubernetes_namespace
        target_label: namespace
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_name
        target_label: pod
      - action: drop
        regex: Pending|Succeeded|Failed|Completed
        source_labels:
        - __meta_kubernetes_pod_phase
      scrape_interval: 5m
      scrape_timeout: 30s
  recording_rules.yml: |
    {}
  rules: |
    {}
---

istio_requests_total{
    app="details",
    connection_security_policy="mutual_tls",
    destination_app="details",
    destination_canonical_revisinotallow="v1",
    destination_canonical_service="details",
    destination_cluster="Kubernetes",
    destination_principal="spiffe://cluster.local/ns/default/sa/bookinfo-details",
    destination_service="details.default.svc.cluster.local",
    destination_service_name="details",
    destination_service_namespace="default",
    destination_versinotallow="v1",
    destination_workload="details-v1",
    destination_workload_namespace="default",
    instance="10.244.2.74:15020",
    job="kubernetes-pods",
    namespace="default",
    pod="details-v1-5f4d584748-9fflw",
    pod_template_hash="5f4d584748",
    reporter="destination",
    request_protocol="http",
    response_code="200",
    response_flags="-",
    security_istio_io_tlsMode="istio",
    service_istio_io_canonical_name="details",
    service_istio_io_canonical_revisinotallow="v1",
    source_app="productpage",
    source_canonical_revisinotallow="v1",
    source_canonical_service="productpage",
    source_cluster="Kubernetes",
    source_principal="spiffe://cluster.local/ns/default/sa/bookinfo-productpage",
    source_versinotallow="v1",
    source_workload="productpage-v1",
    source_workload_namespace="default",
    versinotallow="v1"
}  362

$ kubectl get pods productpage-v1-564d4686f-l8kxr -oyaml
apiVersion: v1
kind: Pod
metadata:
  annotations:
    istio.io/rev: default
    kubectl.kubernetes.io/default-container: productpage
    kubectl.kubernetes.io/default-logs-container: productpage
    prometheus.io/path: /stats/prometheus
    prometheus.io/port: "15020"
    prometheus.io/scrape: "true"
    sidecar.istio.io/status: '{"initContainers":["istio-init"],"containers":["istio-proxy"],"volumes":["workload-socket","credential-socket","workload-certs","istio-envoy","istio-data","istio-podinfo","istio-token","istiod-ca-cert"],"imagePullSecrets":null,"revision":"default"}'
  labels:
    app: productpage
    pod-template-hash: 564d4686f
    security.istio.io/tlsMode: istio
    service.istio.io/canonical-name: productpage
    service.istio.io/canonical-revision: v1
    version: v1
  name: productpage-v1-564d4686f-l8kxr
  namespace: default
spec:
  containers:
  - image: docker.io/istio/examples-bookinfo-productpage-v1:1.18.0
    imagePullPolicy: IfNotPresent
# ......

apiVersion: install.istio.io/v1alpha1
kind: IstioOperator
spec:
  values:
    telemetry:
      enabled: true
      v2:
        enabled: false

$ kubectl explain Telemetry.spec
GROUP:      telemetry.istio.io
KIND:       Telemetry
VERSION:    v1alpha1

FIELD: spec <Object>

DESCRIPTION:
    Telemetry configuration for workloads. See more details at:
    https://istio.io/docs/reference/config/telemetry.html

FIELDS:
  accessLogging <[]Object>
    Optional.

  metrics       <[]Object>
    Optional.

  selector      <Object>
    Optional.

  tracing       <[]Object>
    Optional.

$ kubectl explain Telemetry.spec.metrics
GROUP:      telemetry.istio.io
KIND:       Telemetry
VERSION:    v1alpha1

FIELD: metrics <[]Object>

DESCRIPTION:
    Optional.

FIELDS:
  overrides     <[]Object>
    Optional.

  providers     <[]Object>
    Optional.

  reportingInterval     <string>
    Optional.

# istiooperator.yaml
telemetry:
  enabled: true
  v2:
    enabled: true
    prometheus:
      enabled: true
      configOverride:
        outboundSidecar:
          debug: false
          stat_prefix: istio
          metrics:
            - tags_to_remove:
                - destination_canonical_service
                  ...

apiVersion: telemetry.istio.io/v1alpha1
kind: Telemetry
metadata:
  name: remove-tags
  namespace: istio-system
spec:
  metrics:
    - providers:
        - name: prometheus # 指定指標(biāo)數(shù)據(jù)的提供者
      overrides:
        - match: # 提供覆蓋的范圍，可用于選擇個(gè)別指標(biāo)，以及生成指標(biāo)的工作負(fù)載模式（服務(wù)器和/或客戶(hù)端）。如果未指定，則overrides 將應(yīng)用于兩種操作模式（客戶(hù)端和服務(wù)器）的所有指標(biāo)。
            metric: ALL_METRICS # Istio 標(biāo)準(zhǔn)指標(biāo)之一
            mode: CLIENT_AND_SERVER # 控制選擇的指標(biāo)生成模式：客戶(hù)端和/或服務(wù)端。
          tagOverrides: # 要覆蓋的標(biāo)簽列表
            destination_canonical_service:
              operation: REMOVE
          # disabled: true  # 是否禁用指標(biāo)

apiVersion: telemetry.istio.io/v1alpha1
kind: Telemetry
metadata:
  name: add-tags
spec:
  metrics:
    - overrides:
        - match:
            metric: REQUEST_COUNT
            mode: CLIENT
          tagOverrides:
            destination_x:
              operation: UPSERT
              value: "upstream_peer.labels['app'].value" # 必須加上雙引號(hào)
        - match:
            metric: REQUEST_COUNT
          tagOverrides:
            destination_port:
              value: "string(destination.port)"
            request_host:
              value: "request.host"
      providers:
        - name: prometheus

destination_x:
  operation: UPSERT
  value: "upstream_peer.labels['app'].value"

apiVersion: telemetry.istio.io/v1alpha1
kind: Telemetry
metadata:
  name: remove-all-metrics
  namespace: istio-system
spec:
  metrics:
    - providers:
        - name: prometheus
      overrides:
        - disabled: true
          match:
            mode: CLIENT_AND_SERVER
            metric: ALL_METRICS

apiVersion: telemetry.istio.io/v1alpha1
kind: Telemetry
metadata:
  name: remove-request-count
  namespace: istio-system
spec:
  metrics:
    - providers:
        - name: prometheus
      overrides:
        - disabled: true
          match:
            mode: CLIENT_AND_SERVER
            metric: REQUEST_COUNT

apiVersion: telemetry.istio.io/v1alpha1
kind: Telemetry
metadata:
  name: remove-client
  namespace: istio-system
spec:
  metrics:
    - providers:
        - name: prometheus
      overrides:
        - disabled: true
          match:
            mode: CLIENT
            metric: REQUEST_COUNT

apiVersion: telemetry.istio.io/v1alpha1
kind: Telemetry
metadata:
  name: remove-server
  namespace: istio-system
spec:
  metrics:
    - providers:
        - name: prometheus
      overrides:
        - disabled: true
          match:
            mode: SERVER
            metric: REQUEST_COUNT