Judge JSON 解析与容错 深度解析

  28def parse_judge_output(raw_output: str, response_tag: str) -> Dict[str, Any]:
  29    """Parse LLM Judge output and extract scores.
  30    
  31    Args:
  32        raw_output: The raw text output from the judge model.
  33        response_tag: The tag wrapping the final JSON output (default: "final_evaluation").
  34    """
  35    try:
  36        cleaned = raw_output.strip() if raw_output else ""
  37        if not cleaned:
  38            logger.warning("LLM Judge returned empty output")
  39            return _get_error_result("Empty output", raw_output)
  40
  41        # Remove thinking tags
  42        cleaned = _remove_thinking_tags(cleaned, response_tag=response_tag)
  43
  44        # Extract content from response tag if present
  45        json_content = _extract_tagged_content(cleaned, response_tag)
  46        
  47        # If no tagged content found, use the whole cleaned text
  48        text_to_parse = json_content if json_content else cleaned
  49
  50        # Try extraction strategies in order on the target text
  51        json_str = (
  52            _extract_from_markdown(text_to_parse) or
  53            _extract_best_json_object(text_to_parse) or
  54            _extract_last_json(text_to_parse)
  55        )
  56        
  57        # If that failed, and we had a tag, maybe the tag content itself IS the JSON (without markdown blocks)
  58        if not json_str and json_content:
  59             json_str = (
  60                _extract_best_json_object(json_content) or
  61                _extract_last_json(json_content)
  62            )
  63

  64        if not json_str:
  65            logger.warning("No valid JSON object found in LLM Judge output")
  66            logger.debug(f"Raw output (first 500 chars): {raw_output[:500] if raw_output else ''}")
  67            return _get_error_result("No JSON object found", raw_output)
  68
  69        # Repair and parse JSON
  70        json_str = _repair_json(json_str)
  71        try:
  72            result = json.loads(json_str)
  73        except json.JSONDecodeError as e:
  74            repaired = _extract_scores_only(json_str)
  75            if repaired:
  76                result = json.loads(repaired)
  77            else:
  78                raise e
  79
  80        # Extract and validate scores
  81        scores = _extract_scores(result)
  82        if not scores:
  83            logger.warning("Missing or invalid 'scores' field in LLM Judge output")
  84            logger.debug(f"Available keys: {list(result.keys())}")
  85            return _get_error_result(f"Invalid scores. Keys: {list(result.keys())}", raw_output)
  86
  87        valid_scores = [v for v in scores.values() if v > 0]
  88        return {
  89            "strengths": _to_list(result.get("strengths", [])),
  90            "weaknesses": _to_list(result.get("weaknesses", [])),
  91            "scores": scores,
  92            "total_score": sum(valid_scores) / len(valid_scores) if valid_scores else 0,
  93            "success": True,
  94            "judge_raw_output": raw_output,
  95        }
  96
  97    except json.JSONDecodeError as e:
  98        logger.warning(f"JSON decode error parsing LLM Judge output: {e}")
  99        logger.debug(f"Raw output (first 500 chars):\n{raw_output[:500] if raw_output else ''}")
 100        return _get_error_result(f"JSON error: {e}", raw_output)
 101    except Exception as e:
 102        logger.warning(f"Unexpected error parsing LLM Judge output: {e}")
 103        logger.debug(f"Raw output (first 500 chars):\n{raw_output[:500] if raw_output else ''}")

 145    for m in reversed(matches):
 146        c = m.group(1).strip()
 147        if c.startswith('{') and c.endswith('}') and '"scores"' in c:
 148            return c
 149    
 150    # Fallback to last valid JSON block
 151    for m in reversed(matches):
 152        c = m.group(1).strip()
 153        if c.startswith('{') and c.endswith('}'):
 154            return c
 155    return None
 156
 157
 158def _extract_best_json_object(text: str) -> Optional[str]:
 159    """Extract all JSON objects, return last one with 'scores'."""
 160    objects = []
 161    pos = 0
 162    while True:
 163        start = text.find('{', pos)
 164        if start == -1:
 165            break
 166        obj = _match_braces_forward(text, start)
 167        if obj:
 168            objects.append(obj)
 169            pos = start + len(obj)
 170        else:
 171            pos = start + 1
 172    
 173    if not objects:
 174        return None
 175    
 176    # Prefer object with "scores"
 177    for obj in reversed(objects):
 178        if '"scores"' in obj:
 179            return obj
 180    return objects[-1]