TF-IDF + TextRank 本地摘要器 深度解析

  54def summarize(text, k=8):
  55    sentences = split_sentences(text)
  56    if len(sentences) <= k:
  57        return sentences
  58    if len(sentences) > _MAX_RANK:
  59        sentences = sentences[-_MAX_RANK:]  # recency-biased; keeps it fast
  60
  61    tokens = [_tokenize(s) for s in sentences]
  62    if not any(tokens):
  63        return sentences[:k]
  64
  65    idf = _idf(tokens)
  66    try:
  67        import numpy  # noqa: F401
  68        scores = _textrank_numpy(tokens, idf)
  69    except Exception:
  70        scores = _textrank_python(tokens, idf)
  71
  72    return _select(sentences, scores, k)
  73
  74
  75def _select(sentences, scores, k):
  76    """Top-k sentences by score, returned in original order. Scores are rounded
  77    before ranking and ties are broken by position, so the sub-epsilon numerical
  78    difference between the numpy and pure-Python cores can never change which
  79    sentences are chosen — both backends always produce the same summary (a
  80    committed context.md must not depend on whether numpy happens to be present)."""
  81    order = sorted(range(len(sentences)),
  82                   key=lambda i: (round(scores[i], 10), -i), reverse=True)
  83    return [sentences[i] for i in sorted(order[:k])]
  84
  85
  86def backend_name():
  87    try:
  88        import numpy  # noqa: F401
  89        return "TextRank (vendored, numpy-accelerated)"

  90    except ImportError:
  91        return "TextRank (vendored, pure Python)"
  92
  93
  94# --- shared TF-IDF ----------------------------------------------------------
  95
  96def _idf(tokens):
  97    n = len(tokens)
  98    df = Counter()
  99    for toks in tokens:
 100        for w in set(toks):
 101            df[w] += 1
 102    return {w: math.log((1.0 + n) / (1.0 + d)) + 1.0 for w, d in df.items()}
 103
 104
 105# --- pure-Python core -------------------------------------------------------
 106
 107def _tfidf_sparse(tokens, idf):
 108    vecs = []
 109    for toks in tokens:
 110        if not toks:
 111            vecs.append({})
 112            continue
 113        tf = Counter(toks)
 114        length = len(toks)
 115        vec = {w: (c / length) * idf[w] for w, c in tf.items()}
 116        norm = math.sqrt(sum(v * v for v in vec.values())) or 1.0
 117        vecs.append({w: v / norm for w, v in vec.items()})
 118    return vecs
 119
 120
 121def _cosine(a, b):
 122    if len(a) > len(b):
 123        a, b = b, a
 124    return sum(val * b.get(w, 0.0) for w, val in a.items())
 125
 126
 127def _textrank_python(tokens, idf):
 128    vecs = _tfidf_sparse(tokens, idf)
 129    n = len(vecs)

 120
 121def _cosine(a, b):
 122    if len(a) > len(b):
 123        a, b = b, a
 124    return sum(val * b.get(w, 0.0) for w, val in a.items())
 125
 126
 127def _textrank_python(tokens, idf):
 128    vecs = _tfidf_sparse(tokens, idf)
 129    n = len(vecs)
 130    sim = [[0.0] * n for _ in range(n)]
 131    for i in range(n):
 132        for j in range(i + 1, n):
 133            c = _cosine(vecs[i], vecs[j])
 134            if c:
 135                sim[i][j] = sim[j][i] = c
 136    row_sum = [sum(row) for row in sim]
 137
 138    scores = [1.0 / n] * n
 139    base = (1.0 - _DAMPING) / n
 140    for _ in range(_MAX_ITERS):
 141        new = [base] * n
 142        for i in range(n):
 143            if row_sum[i] == 0.0:
 144                continue
 145            share = _DAMPING * scores[i] / row_sum[i]
 146            row = sim[i]
 147            for j in range(n):
 148                w = row[j]
 149                if w:
 150                    new[j] += share * w
 151        if sum(abs(new[i] - scores[i]) for i in range(n)) < _TOL:
 152            scores = new
 153            break
 154        scores = new
 155    return scores