LuxyAI Flawless SRE 控制面

16ACTION_CATALOG: dict[str, dict[str, Any]] = {
17    "create_workload": {
18        "risk": "high", "auto_allowed": False, "rollback": "delete the newly created workload after approval",
19        "description": "Create one validated apps/v1 Deployment, StatefulSet or DaemonSet from the release gate.",
20    },
21    "patch_workload": {
22        "risk": "medium", "auto_allowed": True, "rollback": "restore the previous workload template",
23        "description": "Patch Deployment, StatefulSet or DaemonSet pod template/replicas.",
24    },
25    "restart": {
26        "risk": "medium", "auto_allowed": True, "rollback": "not applicable; rollout is convergent",
27        "description": "Trigger a controlled rolling restart.",
28    },
29    "scale_out": {
30        "risk": "medium", "auto_allowed": True, "rollback": "restore the previous replica count",
31        "description": "Increase workload replicas within the configured cap.",
32    },
33    "recreate_pod": {
34        "risk": "medium", "auto_allowed": True, "rollback": "controller recreates the pod from the unchanged template",
35        "description": "Delete one controller-owned unhealthy pod for clean rescheduling.",
36    },
37    "patch_hpa": {
38        "risk": "medium", "auto_allowed": True, "rollback": "restore previous HPA min/max replicas",
39        "description": "Adjust HPA bounds without changing its metric semantics.",
40    },
41    "expand_pvc": {
42        "risk": "high", "auto_allowed": False, "rollback": "volume expansion is normally irreversible",
43        "description": "Expand a bound PVC when its StorageClass supports expansion.",
44    },
45    "create_pvc": {

415def score_root_causes(alert: dict, diagnosis: dict, context: dict) -> list[dict[str, Any]]:
416    """Rank runbooks with a transparent weighted evidence score.
417
418    Alert/user text is weak evidence, status/Events are medium evidence, and
419    logs/last termination details are strong evidence. The sigmoid keeps the
420    reported confidence stable as the amount of evidence grows.
421    """
422    sources = [
423        ("alert", _flatten_text(alert), 0.7),
424        ("diagnosis", _flatten_text(diagnosis.get("root_cause"), diagnosis.get("signals")), 0.9),
425        ("pod_status", _flatten_text(context.get("pods"), context.get("pod")), 1.25),
426        ("events", _flatten_text(context.get("events")), 1.45),
427        ("logs", _flatten_text(context.get("logs"), context.get("previous_logs"), context.get("diagnostics")), 1.8),
428    ]
429    decisive_text = _flatten_text(
430        alert,
431        diagnosis,
432        context.get("events"),
433        context.get("storage"),
434        context.get("pod"),
435        context.get("pods"),
436        context.get("logs"),
437        context.get("previous_logs"),
438    )
439    storage_binding_proven = any(term in decisive_text for term in (
440        "unbound immediate persistentvolumeclaims",
441        "no persistent volumes available",
442        "persistentvolumeclaim is not bound",
443        "persistentvolumeclaim pending",
444    ))
445    oom_proven = any(term in decisive_text for term in (
446        "oomkilled",
447        "exit code 137",
448        "exitcode': 137",
449        '"exitcode": 137',
450        "last_terminated_reason': 'oomkilled",
451        '"last_terminated_reason": "oomkilled',
452    ))
453    image_arch_proven = any(term in decisive_text for term in (
454        "exec format error",
455        "no matching manifest for linux/amd64",
456        "no matching manifest for linux/arm64",
457        "image architecture",
458        "platform mismatch",

489        # FailedScheduling is only the symptom here. An unbound PVC is a
490        # deterministic storage root cause and must outrank generic scheduling.
491        if runbook_id == "storage_mount" and storage_binding_proven:
492            raw_score += 3.2
493            matches.append({
494                "source": "kubernetes_storage_state",
495                "terms": ["pvc_unbound"],
496                "weight": 3.2,
497            })
498        if runbook_id == "oom" and oom_proven:
499            raw_score += 2.8
500            matches.append({
501                "source": "container_last_state",
502                "terms": ["oomkilled_or_exit_137"],
503                "weight": 2.8,
504            })
505        if runbook_id == "crash_unknown" and oom_proven:
506            raw_score = max(0.0, raw_score - 1.4)
507            matches.append({
508                "source": "container_last_state",
509                "terms": ["demoted_by_oom_evidence"],
510                "weight": -1.4,
511            })
512        if runbook_id == "image_architecture" and image_arch_proven:
513            raw_score += 2.8
514            matches.append({
515                "source": "runtime_platform_state",
516                "terms": ["exec_format_or_manifest_platform_mismatch"],
517                "weight": 2.8,
518            })
519        if runbook_id == "crash_unknown" and image_arch_proven:
520            raw_score = max(0.0, raw_score - 1.2)
521            matches.append({
522                "source": "runtime_platform_state",
523                "terms": ["demoted_by_architecture_evidence"],
524                "weight": -1.2,
525            })
526        if runbook_id == "storage_permission" and storage_permission_proven:
527            raw_score += 2.6
528            matches.append({
529                "source": "container_logs",
530                "terms": ["write_permission_denied"],
531                "weight": 2.6,
532            })
533        if runbook_id == "crash_unknown" and storage_permission_proven:
534            raw_score = max(0.0, raw_score - 1.2)
535            matches.append({
536                "source": "container_logs",
537                "terms": ["demoted_by_storage_permission_evidence"],
538                "weight": -1.2,
539            })
540        if runbook_id == "crash_unknown" and log_unavailable_proven and not storage_binding_proven:
541            raw_score += 1.4
542            matches.append({
543                "source": "log_probe",
544                "terms": ["logs_unavailable_need_diagnostic_recreate"],
545                "weight": 1.4,
546            })
547        if matches:
548            confidence = 1.0 / (1.0 + math.exp(-(raw_score - 1.65)))
549            ranked.append({
550                "id": runbook_id,
551                "title": runbook["title"],
552                "score": round(raw_score, 3),
553                "confidence": round(min(0.99, confidence), 3),
554                "matched_evidence": matches,
555                "diagnostics": list(runbook["diagnostics"]),
556                "success_criteria": list(runbook["success"]),
557            })
558    return sorted(ranked, key=lambda item: (-item["score"], item["id"]))

961def build_remediation_plan(alert: dict, diagnosis: dict, context: dict) -> dict[str, Any]:
962    hypotheses = score_root_causes(alert, diagnosis, context)
963    namespace, workload_type, workload_name, pod_name, pod = _target(alert, diagnosis, context)
964    primary = hypotheses[0] if hypotheses else None
965    runbook_id = primary["id"] if primary else "unknown"
966    container = _first_container(pod)
967    container_name = container.get("name", "")
968    changes: list[dict[str, Any]] = []
969    patchable = workload_name and str(workload_type).lower() in {"deployment", "statefulset", "daemonset"}
970    confidence = float((primary or {}).get("confidence") or 0.0)
971
972    def workload_patch(patch: dict, reason: str):
973        changes.append({
974            "type": "patch_workload", "namespace": namespace, "workload_type": workload_type,
975            "workload_name": workload_name, "patch": patch, "reason": reason,
976            "runbook_id": runbook_id, **ACTION_CATALOG["patch_workload"],
977        })
978
979    if runbook_id == "storage_mount":
980        issue = _first_storage_issue(context)
981        issue_text = _flatten_text(issue)
982        pvc_name = str(issue.get("pvc") or issue.get("pvc_name") or issue.get("claim") or "").strip()
983        missing_pvc = bool(
984            pvc_name
985            and (
986                issue.get("missing") is True
987                or "not found" in issue_text
988                or "404" in issue_text
989                or "does not exist" in issue_text
990            )
991        )
992        phase = str(issue.get("pvc_phase") or issue.get("phase") or "").lower()
993        if missing_pvc:
994            changes.append({
995                "type": "create_pvc", "namespace": namespace, "pvc_name": pvc_name,
996                "manifest": _pvc_manifest(namespace, pvc_name, issue),
997                "reason": "Kubernetes API 已确认 Pod 引用的 PVC 不存在；创建受存储策略约束的 PVC，并等待动态供卷后验证 Pod。",
998                "runbook_id": runbook_id, **ACTION_CATALOG["create_pvc"],
999            })
1000        elif pvc_name and phase in {"pending", "lost"}:
1001            pv_manifest = _static_pv_manifest(namespace, pvc_name, issue)
1002            if pv_manifest:
1003                changes.append({
1004                    "type": "create_pv", "namespace": namespace, "pvc_name": pvc_name,
1005                    "manifest": pv_manifest,
1006                    "reason": "Kubernetes API 已确认 PVC 未绑定，且平台配置了批准的静态存储模板；创建预绑定 PV 并验证 PVC=Bound。",
1007                    "runbook_id": runbook_id, **ACTION_CATALOG["create_pv"],
1008                })
1009
1010    if primary and confidence >= 0.62 and patchable:
1011        if runbook_id == "oom" and container_name:
1012            resources = container.get("resources") or {}
1013            requests, limits = resources.get("requests") or {}, resources.get("limits") or {}
1014            workload_patch({"spec": {"template": {"spec": {"containers": [{
1015                **_container_patch_base(container_name, container),
1016                "resources": {
1017                    "requests": {"cpu": requests.get("cpu") or "100m", "memory": requests.get("memory") or "256Mi"},
1018                    "limits": {"cpu": limits.get("cpu") or "1", "memory": _memory_growth(limits.get("memory"))},
1019                },
1020            }]}}}}, "OOM 证据达到执行阈值：提高内存上限并保留 CPU 约束，滚动后验证 OOM 是否消失。")
1021        elif runbook_id == "probe" and container_name:
1022            workload_patch({"spec": {"template": {"spec": {"containers": [{
1023                **_container_patch_base(container_name, container), **_startup_probe_patch(container)
1024            }]}}}}, "探针/慢启动证据达到执行阈值：增加 startupProbe 容错窗口并验证 Endpoint Ready。")
1025        elif runbook_id == "storage_permission":
1026            fs_group = _security_group_from_pod(pod)
1027            workload_patch({"spec": {"template": {"spec": {"securityContext": {
1028                "fsGroup": fs_group, "fsGroupChangePolicy": "OnRootMismatch"
1029            }}}}}, f"卷写入权限证据达到执行阈值：按容器运行用户/组选择 fsGroup={fs_group}，并在 rollout 后复查挂载与写入错误。")
1030        elif runbook_id == "storage_mount" and not changes:
1031            issue = _first_storage_issue(context)
1032            issue_text = _flatten_text(issue)
1033            pvc_name = str(issue.get("pvc") or issue.get("pvc_name") or issue.get("claim") or "").strip()
1034            if pvc_name and ("not found" in issue_text or "404" in issue_text or issue.get("missing") is True):
1035                changes.append({
1036                    "type": "create_pvc", "namespace": namespace, "pvc_name": pvc_name,
1037                    "manifest": _pvc_manifest(namespace, pvc_name, issue),
1038                    "reason": "Pod 引用了不存在的 PVC；先创建受策略约束的 PVC，再等待存储插件动态供卷并验证 Pod 挂载。",
1039                    "runbook_id": runbook_id, **ACTION_CATALOG["create_pvc"],

40def _load_kubeconfig_access() -> tuple[str, str, ssl.SSLContext]:
41    """加载本地 kubeconfig，用于演示机/开发机直接运行 MCP。
42
43    生产容器内优先使用 ServiceAccount。只有不存在集群内 token 时，才会进入
44    kubeconfig 兼容模式，避免影响正式 K8s 部署。
45    """
46    import yaml
47
48    kubeconfig = os.getenv("KUBECONFIG") or os.path.expanduser("~/.kube/config")
49    with open(kubeconfig, encoding="utf-8") as f:
50        config = yaml.safe_load(f) or {}
51
52    current_context = os.getenv("KUBE_CONTEXT") or config.get("current-context")
53    contexts = {item.get("name"): item.get("context") or {} for item in config.get("contexts", [])}
54    clusters = {item.get("name"): item.get("cluster") or {} for item in config.get("clusters", [])}
55    users = {item.get("name"): item.get("user") or {} for item in config.get("users", [])}
56    context = contexts.get(current_context) or {}
57    cluster = clusters.get(context.get("cluster")) or {}
58    user = users.get(context.get("user")) or {}
59    host = str(cluster.get("server") or "").rstrip("/")
60    if not host:
61        raise RuntimeError(f"kubeconfig {kubeconfig} does not contain a server for context {current_context}")
62
63    if cluster.get("insecure-skip-tls-verify"):
64        ctx = ssl._create_unverified_context()
65    else:
66        ca_data = _decode_kubeconfig_blob(cluster.get("certificate-authority-data"))
67        ca_file = cluster.get("certificate-authority")
68        if ca_data:
69            ca_file = _write_temp_pem(ca_data, ".ca.crt")
70        ctx = ssl.create_default_context(cafile=ca_file)
71
72    cert_data = _decode_kubeconfig_blob(user.get("client-certificate-data"))
73    key_data = _decode_kubeconfig_blob(user.get("client-key-data"))
74    cert_file = user.get("client-certificate")
75    key_file = user.get("client-key")
76    if cert_data:
77        cert_file = _write_temp_pem(cert_data, ".client.crt")
78    if key_data:
79        key_file = _write_temp_pem(key_data, ".client.key")
80    if cert_file and key_file:
81        ctx.load_cert_chain(certfile=cert_file, keyfile=key_file)
82
83    return host, str(user.get("token") or ""), ctx
84
85
86if os.path.exists(_TOKEN_FILE):
87    with open(_TOKEN_FILE) as f:
88        _TOKEN = f.read().strip()
89    _CTX = ssl.create_default_context(cafile=_CA_FILE)
90else:
91    _HOST, _TOKEN, _CTX = _load_kubeconfig_access()

427def classify_pod(pod: dict) -> dict:
428    namespace = pod.get("namespace", "")
429    name = pod.get("name", "")
430    labels = pod.get("labels", {}) or {}
431    owners = pod.get("owner_references") or pod.get("ownerReferences") or []
432    containers = pod.get("containers", []) or []
433    phase = str(pod.get("phase") or "")
434    text = " ".join([
435        namespace,
436        name,
437        " ".join(labels.values()),
438        " ".join(o.get("kind", "") + " " + o.get("name", "") for o in owners),
439        " ".join(c.get("name", "") + " " + c.get("image", "") for c in containers),
440    ]).lower()
441
442    is_job_like = any(k in text for k in JOB_KEYWORDS) or any(o.get("kind") in ["Job", "CronJob"] for o in owners)
443    if phase in {"Succeeded", "Completed"} and is_job_like:
444        return {
445            "class": "batch",
446            "label": "批处理已完成",
447            "confidence": 0.94,
448            "reason": "Job/CronJob 一次性 Pod 已成功完成，不计入服务可用性风险",
449        }
450    if namespace in INFRA_NAMESPACES or any(k in text for k in INFRA_KEYWORDS):
451        return {
452            "class": "infrastructure",
453            "label": "基础服务",
454            "confidence": 0.9,
455            "reason": "系统命名空间或基础设施组件特征",
456        }
457    if any(k in text for k in DATA_KEYWORDS):
458        return {
459            "class": "data",
460            "label": "数据服务",
461            "confidence": 0.86,
462            "reason": "数据库/消息队列/缓存等有状态组件特征",
463        }
464    if is_job_like:
465        return {
466            "class": "batch",
467            "label": "批处理",
468            "confidence": 0.8,
469            "reason": "Job/CronJob/worker/batch 特征",
470        }
471    return {
472        "class": "application",
473        "label": "应用服务",
474        "confidence": 0.72,
475        "reason": "默认业务应用工作负载",
476    }

620@mcp.tool()
621def list_pods(namespace: str = "default") -> dict:
622    """List pods in a Kubernetes namespace."""
623    try:
624        data = _k8s_get(f"/api/v1/namespaces/{namespace}/pods")
625    except Exception as e:
626        return {"error": f"Failed to list pods in namespace {namespace}: {str(e)}"}
627
628    pods = []
629    for item in data["items"]:
630        pod = _pod_identity(item)
631        pod["classification"] = classify_pod(pod)
632        pod["workload"] = _workload_from_pod(pod)
633        pods.append(pod)
634    return {"namespace": namespace, "pods": pods}
635
636@mcp.tool()
637def get_pod_events(namespace: str, pod_name: str) -> dict:
638    """Get Kubernetes events for a specific pod."""
639    try:
640        field_selector = f"involvedObject.name={pod_name}"
641        data = _k8s_get(
642            f"/api/v1/namespaces/{namespace}/events?fieldSelector={field_selector}"
643        )
644    except Exception as e:
645        return {"error": str(e)}
646
647    return {
648        "namespace": namespace,
649        "pod_name": pod_name,
650        "events": [
651            {
652                "reason": e["reason"],
653                "message": e["message"],
654                "type": e["type"],
655                "count": e.get("count"),
656                "first_timestamp": e.get("firstTimestamp"),
657                "last_timestamp": e.get("lastTimestamp"),
658            }
659            for e in data.get("items", [])
660        ],
661    }
662
663@mcp.tool()
664def get_pod_logs(
665    namespace: str,
666    pod_name: str,
667    container: Optional[str] = None,
668    tail_lines: int = 200,
669    previous: bool = False,
670) -> dict:
671    """Get recent pod logs."""
672    try:
673        query = {"tailLines": max(1, min(int(tail_lines), 500)), "previous": str(bool(previous)).lower()}
674        if container:
675            query["container"] = container
676        path = f"/api/v1/namespaces/{quote(namespace, safe='')}/pods/{quote(pod_name, safe='')}/log?{urlencode(query)}"

7623async def _evidence_based_replan(
7624    plan: dict,
7625    steps: list[dict],
7626    attempted_actions: set[str] | None = None,
7627    *,
7628    include_llm: bool = True,
7629) -> list[dict]:
7630    deep = plan.get("_runtime_evidence") or {}
7631    failed_context = plan.get("_last_failure") or {}
7632    blocked_change_fingerprints = {
7633        str(value) for value in (plan.get("_attempted_change_fingerprints") or []) if value
7634    }
7635    blocked_change_fingerprints.update(
7636        _change_item_fingerprint(change)
7637        for change in (plan.get("changes") or [])
7638        if isinstance(change, dict)
7639    )
7640    alert = {
7641        "alert_name": plan.get("strategy_class") or plan.get("runbook_id") or "runtime_incident",
7642        "summary": plan.get("summary", ""),
7643        "namespace": plan.get("namespace", "default"),
7644        "workload_type": _workload_identity_from_plan(plan)[1],
7645        "workload_name": _workload_identity_from_plan(plan)[2],
7646        "pod": _target_pod_from_plan(plan),
7647    }
7648    diagnosis = {"root_cause": plan.get("summary", ""), "signals": deep.get("events", [])}
7649    engine_plan = build_remediation_plan(alert, diagnosis, {
7650        **deep,
7651        "pod": deep.get("pod") or ((plan.get("evidence") or {}).get("pod") or {}),
7652        "pods": [deep.get("pod")] if deep.get("pod") else [],
7653        "events": {"events": deep.get("events", [])},
7654    })
7655    primary = (engine_plan.get("hypotheses") or [{}])[0]
7656    plan["_runtime_replan"] = {
7657        "runbook_id": engine_plan.get("runbook_id"),
7658        "confidence": float(primary.get("confidence") or 0.0),
7659        "reason": engine_plan.get("reason"),
7660        "evidence_gap": engine_plan.get("evidence_gap"),
7661        "success_criteria": engine_plan.get("success_criteria") or [],
7662    }
7663    candidates = list(engine_plan.get("changes") or [])

7705    normalized: list[dict] = []
7706    rejected: list[str] = []
7707    seen = set()
7708    constrained_actions = {
7709        "storage_mount": {"create_pvc", "create_pv", "expand_pvc", "patch_workload_volume"},
7710        "storage_permission": {"patch_workload", "patch_workload_runtime_security"},
7711        "config_missing": {"create_configmap"},
7712        "image_auth": {"patch_workload", "patch_service_account", "rollback_workload"},
7713        "image_architecture": {"rollback_workload", "patch_workload"},
7714        "oom": {"patch_workload"},
7715        "probe": {"patch_workload"},
7716        "node_pressure": {"cordon_node", "evict_pod"},
7717    }.get(str(engine_plan.get("runbook_id") or "")) if float(primary.get("confidence") or 0.0) >= 0.72 else None
7718    for raw in candidates:
7719        if constrained_actions and str((raw or {}).get("type") or "") not in constrained_actions:
7720            rejected.append(
7721                f"{(raw or {}).get('type') or 'unknown'} conflicts with proven {engine_plan.get('runbook_id')} evidence"
7722            )
7723            continue
7724        change, reason = _normalize_planner_change(raw, plan)
7725        if not change:
7726            rejected.append(reason)
7727            continue
7728        fingerprint = _change_item_fingerprint(change)
7729        if fingerprint in blocked_change_fingerprints:
7730            rejected.append(f"{change.get('type') or 'unknown'} repeats a previously failed action/target/parameter set")
7731            continue
7732        if fingerprint in seen:
7733            continue
7734        seen.add(fingerprint)
7735        normalized.append(change)
7736    if not normalized:
7737        return []
7738    namespace, workload_type, workload_name = _workload_identity_from_plan(plan)
7739    return [{
7740        "id": f"evidence-replan-{uuid.uuid4().hex[:8]}",
7741        "title": f"证据重规划：{engine_plan.get('runbook_id', 'expert-runbook')}",
7742        "namespace": namespace,
7743        "target": f"{workload_type}/{workload_name}" if workload_name else plan.get("target", namespace),
7744        "pod_name": _target_pod_from_plan(plan),
7745        "cluster": plan.get("cluster"),
7746        "cluster_id": plan.get("cluster_id"),
7747        "source": plan.get("source"),
7748        "evidence": plan.get("evidence") or {},
7749        "summary": planner_meta.get("reason") or engine_plan.get("reason") or "根据新证据生成差异化修复策略。",
7750        "steps": engine_plan.get("steps") or [],
7751        "changes": normalized[:2],
7752        "requires_confirmation": True,
7753        "requires_high_risk_confirmation": any(change.get("risk") == "high" for change in normalized[:2]),
7754        "strategy_source": "evidence_replan",
7755        "planning": planner_meta,
7756        "root_cause_hypotheses": engine_plan.get("hypotheses", []),
7757        "success_criteria": engine_plan.get("success_criteria", []),
7758        "verification_plan": _next_attempt_verification_plan(f"{workload_type}/{workload_name}" if workload_name else plan.get("target", "")),
7759        "rejected_candidates": rejected,
7760    }]

37    def test_root_cause_families_are_classified(self):
38        cases = {
39            "OOMKilled exit code 137": "oom",
40            "startup probe failed connection refused": "probe",
41            "volume permission denied": "storage_permission",
42            "ImagePullBackOff unauthorized": "image_auth",
43            "exec format error no matching manifest for linux/amd64": "image_architecture",
44            "FailedScheduling insufficient cpu": "scheduling_capacity",
45            "service has no endpoints": "network_service",
46            "FailedCreatePodSandBox CNI error": "dns_cni",
47            "high CPU usage": "cpu_saturation",
48        }
49        for evidence, expected in cases.items():
50            with self.subTest(evidence=evidence):
51                self.assertEqual(self.plan(evidence)["runbook_id"], expected)
52
53    def test_mkdir_permission_denied_generates_storage_permission_patch(self):
54        plan = build_remediation_plan(
55            self.alert,
56            {"root_cause": "CrashLoopBackOff"},
57            {
58                "pod": self.pod,
59                "pods": [self.pod],
60                "logs": {
61                    "api": {
62                        "current": "mkdir: can't create directory '/data/cache': Permission denied",
63                        "previous": "",
64                    }
65                },
66            },
67        )
68        self.assertEqual(plan["runbook_id"], "storage_permission")
69        self.assertEqual(plan["changes"][0]["type"], "patch_workload")
70        pod_spec = plan["changes"][0]["patch"]["spec"]["template"]["spec"]
71        self.assertEqual(pod_spec["securityContext"]["fsGroupChangePolicy"], "OnRootMismatch")
72

186    def test_high_risk_action_requires_second_confirmation(self):
187        result = asyncio.run(server._execute_change(
188            {"type": "cordon_node", "node_name": "worker-1", "risk": "high"},
189            {"cluster_id": "local", "namespace": "default", "high_risk_confirmed": False},
190        ))
191        self.assertEqual(result["status"], "blocked")
192        self.assertTrue(result["result"]["requires_high_risk_confirmation"])
193
194    def test_permission_guidance_is_action_specific(self):
195        guidance = server._permission_guidance(
196            {"error": "HTTP Error 403: Forbidden"},
197            {"cluster": "nonprod", "namespace": "k8s-agent", "source": "rancher"},
198            {"type": "recreate_pod"},
199        )
200        self.assertEqual(guidance["minimal_verbs"], ["get", "delete"])
201        self.assertEqual(guidance["minimal_resources"], ["pods"])
202        self.assertIn("Rancher Token", guidance["summary"])
203