Nested detail page
索引与检索 深度解析
这一页拆 deja-vu 如何把多 harness 会话变成本地倒排索引，并在查询时用 postings、substring fallback 和 record scan 控制性能与准确性。
§ 00 · IMPORT CHAIN
入口契约
搜索路径从 CLI 的 cmd/deja/main.go:214 开始，先解析 Options，再由 cmd/deja/main.go:218 触发 EnsureForSearch；索引侧 internal/index/index.go:152 会检查 manifest freshness，查询侧 internal/index/index.go:179 则读取 records 与 postings。这个链条说明搜索不是直接扫源日志，而是先把日志转成缓存结构。
214 o, err := parseSearch(filtered) 215 if err != nil { 216 return err 217 } 218 if err := index.EnsureForSearch(index.DefaultDir(), o, force, os.Stderr); err != nil { 219 return fmt.Errorf("ensure: %w", err) 220 } 221 maybeFirstIndexGreeting() 222 ss, err := index.SearchWithRecovery(index.DefaultDir(), o, os.Stderr) 223 if err != nil { 224 return fmt.Errorf("search: %w", err) 225 }
§ 01 · CORE
缓存失效与自愈
▸freshness 判断
internal/index/index.go:161 收集源文件状态，internal/index/index.go:163 读取 manifest，internal/index/index.go:164 同时判断 manifestFresh 和 recordsIntact。只要版本、scope 或 records 尾部异常，internal/index/index.go:167 会切换到 rebuildForSearch，避免用坏缓存返回假结果。
152func EnsureForSearch(dir string, o search.Options, force bool, progress io.Writer) error { 153 if dir == "" { 154 dir = DefaultDir() 155 } 156 unlock, err := lockDir(dir) 157 if err != nil { 158 return err 159 } 160 defer unlock() 161 want := currentFiles("") 162 scope := "" 163 m, err := readManifest(dir) 164 if !force && err == nil && manifestFresh(m, want, scope) && recordsIntact(dir, m) { 165 return nil 166 } 167 if force || err != nil || m.Version != version || m.Scope != scope || !recordsIntact(dir, m) { 168 if progress != nil { 169 fmt.Fprintf(progress, "deja: indexing sessions into %s ...\n", dir) 170 } 171 return rebuildForSearch(dir, o, scope, want, progress) 172 } 173 if err := updateIndex(dir, o.Harness, scope, want, force, progress); err != nil { 174 return fmt.Errorf("update: %w", err) 175 } 176 return nil 177}
▸postings 优先
查询时 internal/index/index.go:197 跳过 regex 的 postings 优化，internal/index/index.go:198 先把 query 转 key，internal/index/index.go:200 做 postings 交集；如果完全没结果，internal/index/index.go:208 用 substring postings 做 grep-like fallback。只有 regex 或无 postings 场景才回到 internal/index/index.go:219 的 scanRecords。
179func Search(dir string, o search.Options) ([]model.Session, error) { 180 if dir == "" { 181 dir = DefaultDir() 182 } 183 unlock, err := lockDir(dir) 184 if err != nil { 185 return nil, err 186 } 187 defer unlock() 188 m, err := readManifest(dir) 189 if err != nil { 190 return nil, fmt.Errorf("manifest: %w", err) 191 } 192 if !recordsIntact(dir, m) { 193 return nil, fmt.Errorf("%w: records.bin truncated", errCorruptIndex) 194 } 195 var posts []posting 196 usedPostings := false 197 if !o.Regex { 198 if keys := queryKeys(o.Query); len(keys) > 0 { 199 usedPostings = true 200 posts, err = intersectPostings(dir, retrievalKeys(keys)) 201 if err != nil { 202 return nil, fmt.Errorf("postings: %w", err) 203 } 204 if len(posts) == 0 { 205 // grep expectation: "code" should find "opencode". Expand each query 206 // token to all indexed tokens containing it (bucket directories only, 207 // no record scan), then intersect. 208 posts, err = intersectSubstringPostings(dir, tokens(o.Query)) 209 if err != nil { 210 return nil, fmt.Errorf("substr postings: %w", err) 211 } 212 } 213 } 214 } 215 if len(posts) == 0 { 216 if usedPostings { 217 return nil, nil 218 } 219 return scanRecords(dir, m, o, nil) 220 } 221 posts = cutPostingsBySession(posts, m, o) 222 if len(posts) == 0 { 223 return nil, nil 224 } 225 return scanRecords(dir, m, o, postingOffsets(posts)) 226}
▸重建写入
重建管线在 internal/index/index.go:356 开始，internal/index/index.go:366 加载 harness，internal/index/index.go:380 准备消息去重，internal/index/index.go:412 写入前脱敏，internal/index/index.go:430 并发落 bucket。这个流程将隐私、安全和性能放在同一个 ingest 点。
380 seenMsgs := msgSeen{} 381 buckets, err := indexTextParallel(func(push func(tokenJob)) error { 382 for _, s := range ss { 383 key := s.Harness + ":" + s.ID 384 ord := uint32(0) 385 if old, ok := m.Sessions[key]; ok { 386 ord = old.Ord 387 if s.Started.IsZero() || (!old.Started.IsZero() && old.Started.Before(s.Started)) { 388 s.Started = old.Started 389 } 390 if old.Updated.After(s.Updated) { 391 s.Updated = old.Updated 392 } 393 if s.Project == "history" && old.Project != "" && old.Project != "history" { 394 s.Project = old.Project 395 } 396 if s.Title == "" { 397 s.Title = old.Title 398 } 399 } 400 if ord == 0 { 401 ord = nextSessionOrd(m.Sessions) 402 } 403 m.Sessions[key] = metaWithOrd(metaForSession(s), ord) 404 for _, msg := range s.Messages { 405 if seenMsgs.dup(key, msg.Role, msg.Time, msg.Text) { 406 continue 407 } 408 text := msg.Text 409 if len(text) > maxIndexedText { 410 text = text[:maxIndexedText] 411 } 412 text = redactForIngest(&m, s.Path, text) 413 off, err := rw.write(Record{Key: key, SourcePath: s.Path, Role: msg.Role, Text: text, Time: msg.Time}) 414 if err != nil { 415 return err 416 } 417 writtenMessages++ 418 push(tokenJob{text: text, offset: off, sid: m.Sessions[key].Ord}) 419 } 420 } 421 return nil 422 }) 423 if err != nil { 424 _ = rw.Close() 425 return err 426 } 427 if err := rw.Close(); err != nil { 428 return err 429 } 430 if err := writeBucketsConcurrent(filepath.Join(tmp, "buckets"), buckets); err != nil { 431 return err 432 } 433 setOpencodeLastUpdated(m.Files, m.Sessions) 434 if err := writeManifest(tmp, m); err != nil { 435 return err 436 } 437 _ = os.RemoveAll(dir) 438 if err := os.Rename(tmp, dir); err != nil { 439 return err 440 } 441 summarizeBuild(initialBuild, len(m.Sessions), writtenMessages, ss) 442 return nil 443}
§ 02 · FLOW
关键流程
CLI query -> EnsureForSearch -> currentFiles/manifestFresh -> rebuild/update -> Search -> postings intersection -> search.Run scoring -> Print
排序发生在 search 包：internal/search/search.go:57 合并 session，internal/search/search.go:68 逐消息计数，internal/search/search.go:93 用 age 衰减 score，internal/search/search.go:98 进行排序，最终 internal/search/search.go:104 截断默认结果。检索和展示分离，便于 MCP 和 CLI 共用同一层。
57 for _, s := range mergeSessions(ss) { 58 if o.Harness != "" && s.Harness != o.Harness { 59 continue 60 } 61 if o.Project != "" && !strings.Contains(strings.ToLower(s.Project), strings.ToLower(o.Project)) { 62 continue 63 } 64 if !cut.IsZero() && s.Updated.Before(cut) { 65 continue 66 } 67 h := Hit{Session: s} 68 for _, m := range s.Messages { 69 if o.Role != "" && m.Role != o.Role { 70 continue 71 } 72 c := 0 73 if re != nil { 74 c = len(re.FindAllStringIndex(m.Text, -1)) 75 } else { 76 low := strings.ToLower(m.Text) 77 if len(qtoks) <= 1 { 78 if strings.Contains(low, qlow) { 79 c = strings.Count(low, qlow) 80 } 81 } else { 82 c = countAllTokens(low, qtoks) 83 } 84 } 85 if c > 0 { 86 h.Count += c 87 if len(h.Snippets) < 3 { 88 h.Snippets = append(h.Snippets, snippet(m.Text, o.Query, re)) 89 } 90 } 91 } 92 if h.Count > 0 { 93 age := time.Since(s.Updated).Hours() / 24 94 h.Score = float64(h.Count) * 1000 / (1 + age) 95 hits = append(hits, h) 96 } 97 } 98 sort.Slice(hits, func(i, j int) bool { 99 if hits[i].Score == hits[j].Score { 100 return hits[i].Session.Updated.After(hits[j].Session.Updated) 101 } 102 return hits[i].Score > hits[j].Score 103 }) 104 if !o.All && len(hits) > 15 { 105 hits = hits[:15] 106 } 107 return hits, nil
§ 03 · EDGE
边界
internal/index/index.go:228的 SearchWithRecovery 能处理 bucket corruption，但如果原始日志本身包含错误内容，索引不会纠正语义。internal/index/index.go:409的 maxIndexedText 截断保护索引体积，超长消息会损失尾部上下文。internal/search/search.go:77的单词查询与internal/search/search.go:82的多词 AND 更偏精确检索，不是 embedding 语义召回。