deja-vu agent memory layer

487var harnessLoaders = []struct {
488	name string
489	load func() []model.Session
490}{
491	{"claude", sources.LoadClaude},
492	{"codex", sources.LoadCodex},
493	{"opencode", sources.LoadOpencode},
494	{"aider", sources.LoadAider},
495	{"gemini", sources.LoadGemini},
496	{"cursor", sources.LoadCursor},
497	{"antigravity", sources.LoadAntigravity},
498	{"grok", sources.LoadGrok},
499}
500
501func load(h string) []model.Session { return loadProgress(h, nil) }
502
503// loadProgress narrates a full rebuild per harness: a cold pass over a large
504// corpus takes seconds and used to look hung.
505func loadProgress(h string, progress io.Writer) []model.Session {
506	var ss []model.Session
507	for _, hl := range harnessLoaders {
508		if h != "" && h != hl.name {
509			continue
510		}
511		got := hl.load()
512		ss = append(ss, got...)
513		if progress != nil && len(got) > 0 {
514			msgs := 0
515			for _, s := range got {
516				msgs += len(s.Messages)
517			}
518			fmt.Fprintf(progress, "deja: %s: %d sessions, %d messages\n", hl.name, len(got), msgs)
519		}
520	}
521	return ss

356func rebuild(dir string, harness string, scope string, files map[string]FileState, progress io.Writer) error {
357	lastIngestFiles = len(files)
358	initialBuild := !HasManifest(dir)
359	writtenMessages := 0
360	imported := importedSessions(dir)
361	tmp := dir + ".tmp"
362	_ = os.RemoveAll(tmp)
363	if err := os.MkdirAll(filepath.Join(tmp, "buckets"), 0o755); err != nil {
364		return err
365	}
366	ss := loadProgress(harness, progress)
367	ss = append(ss, imported.sessions...)
368	m := Manifest{Version: version, Files: files, Sessions: map[string]SessionMeta{}, BuiltAt: time.Now(), Scope: scope,
369		ExportWatermarks: imported.watermarks, ImportedRecords: imported.dedupe}
370	recPath := filepath.Join(tmp, "records.bin")
371	rf, err := os.Create(recPath)
372	if err != nil {
373		return err
374	}
375	rw, err := newRecordWriter(rf)
376	if err != nil {
377		_ = rf.Close()
378		return err
379	}
380	seenMsgs := msgSeen{}
381	buckets, err := indexTextParallel(func(push func(tokenJob)) error {
382		for _, s := range ss {
383			key := s.Harness + ":" + s.ID
384			ord := uint32(0)
385			if old, ok := m.Sessions[key]; ok {
386				ord = old.Ord
387				if s.Started.IsZero() || (!old.Started.IsZero() && old.Started.Before(s.Started)) {
388					s.Started = old.Started
389				}
390				if old.Updated.After(s.Updated) {
391					s.Updated = old.Updated
392				}
393				if s.Project == "history" && old.Project != "" && old.Project != "history" {
394					s.Project = old.Project
395				}
396				if s.Title == "" {
397					s.Title = old.Title
398				}
399			}
400			if ord == 0 {
401				ord = nextSessionOrd(m.Sessions)
402			}
403			m.Sessions[key] = metaWithOrd(metaForSession(s), ord)
404			for _, msg := range s.Messages {
405				if seenMsgs.dup(key, msg.Role, msg.Time, msg.Text) {
406					continue
407				}
408				text := msg.Text
409				if len(text) > maxIndexedText {
410					text = text[:maxIndexedText]
411				}
412				text = redactForIngest(&m, s.Path, text)
413				off, err := rw.write(Record{Key: key, SourcePath: s.Path, Role: msg.Role, Text: text, Time: msg.Time})
414				if err != nil {
415					return err
416				}
417				writtenMessages++
418				push(tokenJob{text: text, offset: off, sid: m.Sessions[key].Ord})
419			}
420		}
421		return nil
422	})
423	if err != nil {
424		_ = rw.Close()
425		return err
426	}
427	if err := rw.Close(); err != nil {
428		return err
429	}
430	if err := writeBucketsConcurrent(filepath.Join(tmp, "buckets"), buckets); err != nil {
431		return err
432	}
433	setOpencodeLastUpdated(m.Files, m.Sessions)
434	if err := writeManifest(tmp, m); err != nil {
435		return err
436	}
437	_ = os.RemoveAll(dir)
438	if err := os.Rename(tmp, dir); err != nil {
439		return err
440	}
441	summarizeBuild(initialBuild, len(m.Sessions), writtenMessages, ss)
442	return nil
443}

41func Run(ss []model.Session, o Options) ([]Hit, error) {
42	var re *regexp.Regexp
43	qlow := strings.ToLower(o.Query)
44	qtoks := queryTokens(o.Query)
45	if o.Regex {
46		var err error
47		re, err = regexp.Compile("(?i)" + o.Query)
48		if err != nil {
49			return nil, err
50		}
51	}
52	cut := time.Time{}
53	if o.Since > 0 {
54		cut = time.Now().Add(-o.Since)
55	}
56	var hits []Hit
57	for _, s := range mergeSessions(ss) {
58		if o.Harness != "" && s.Harness != o.Harness {
59			continue
60		}
61		if o.Project != "" && !strings.Contains(strings.ToLower(s.Project), strings.ToLower(o.Project)) {
62			continue
63		}
64		if !cut.IsZero() && s.Updated.Before(cut) {
65			continue
66		}
67		h := Hit{Session: s}
68		for _, m := range s.Messages {
69			if o.Role != "" && m.Role != o.Role {
70				continue
71			}
72			c := 0
73			if re != nil {
74				c = len(re.FindAllStringIndex(m.Text, -1))
75			} else {
76				low := strings.ToLower(m.Text)
77				if len(qtoks) <= 1 {
78					if strings.Contains(low, qlow) {
79						c = strings.Count(low, qlow)
80					}
81				} else {
82					c = countAllTokens(low, qtoks)
83				}
84			}
85			if c > 0 {
86				h.Count += c
87				if len(h.Snippets) < 3 {
88					h.Snippets = append(h.Snippets, snippet(m.Text, o.Query, re))
89				}
90			}
91		}
92		if h.Count > 0 {
93			age := time.Since(s.Updated).Hours() / 24
94			h.Score = float64(h.Count) * 1000 / (1 + age)
95			hits = append(hits, h)
96		}
97	}
98	sort.Slice(hits, func(i, j int) bool {
99		if hits[i].Score == hits[j].Score {
100			return hits[i].Session.Updated.After(hits[j].Session.Updated)
101		}
102		return hits[i].Score > hits[j].Score
103	})
104	if !o.All && len(hits) > 15 {
105		hits = hits[:15]
106	}
107	return hits, nil

66func handleMCP(req rpcRequest) (any, int, string) {
67	switch req.Method {
68	case "initialize":
69		return map[string]any{
70			"protocolVersion": mcpProtocolVersion,
71			"capabilities":    map[string]any{"tools": map[string]any{}},
72			"serverInfo":      map[string]any{"name": "deja", "version": version},
73		}, 0, ""
74	case "tools/list":
75		return map[string]any{"tools": []map[string]any{
76			{
77				"name":        "recall",
78				"description": "Search past coding-agent sessions (Claude Code, Codex, opencode, aider, Gemini CLI, Cursor, Antigravity, Grok Build) indexed on this machine and return the best matches as dense text under ~4KB. Use before debugging or re-implementing something: prior sessions often contain the exact fix, error message, or command. Query works best with specific tokens — an error string, function name, or flag.",
79				"inputSchema": map[string]any{"type": "object", "properties": map[string]any{"query": map[string]any{"type": "string", "description": "Search terms; specific tokens (error strings, function names, flags) match best. Multiple words are ANDed."}, "harness": map[string]any{"type": "string", "description": "Optional filter: claude, codex, opencode, aider, gemini, cursor, antigravity or grok."}, "limit": map[string]any{"type": "number", "description": "Max sessions to return (default 5)."}}, "required": []string{"query"}},
80			},
81			{
82				"name":        "recall_context",
83				"description": "Return a markdown digest (~8KB) of the single best-matching prior session — the full problem/solution arc, not just snippets. Use after recall points at a session and more detail is needed.",
84				"inputSchema": map[string]any{"type": "object", "properties": map[string]any{"query": map[string]any{"type": "string", "description": "Search terms identifying the session to digest."}}, "required": []string{"query"}},
85			},
86		}}, 0, ""
87	case "tools/call":
88		var p struct {
89			Name      string          `json:"name"`
90			Arguments json.RawMessage `json:"arguments"`
91		}
92		if err := json.Unmarshal(req.Params, &p); err != nil {
93			return nil, -32602, "invalid params"
94		}
95		text, err := callMCPTool(p.Name, p.Arguments)
96		if err != nil {
97			return nil, -32602, err.Error()
98		}
99		return toolText(text), 0, ""

25var (
26	awsAccessKeyRE = regexp.MustCompile(`A(?:KIA|SIA)[0-9A-Z]{16}`)
27	awsSecretRE    = regexp.MustCompile(`(?i)\b(aws[_-]?secret[_-]?access[_-]?key)(\s*['"]?\s*[:=]\s*)(['"]?)([A-Za-z0-9/+=_-]{32,})(['"]?)`)
28	// The key may be embedded in a larger identifier (ANTHROPIC_API_KEY,
29	// x-api-key) and, in JSON, a closing quote can sit between the key and the
30	// delimiter ("api_key": "..."). Tolerate both so env-var and JSON forms are
31	// caught, not just a bare `api_key=`.
32	genericKVRE  = regexp.MustCompile(`(?i)\b([\w.-]{0,64}?(?:api[_-]?key|secret|token|passwd|password|authorization))(\s*['"]?\s*[:=]\s*)(['"]?)([A-Za-z0-9/+=._-]{16,})(['"]?)`)
33	bearerRE     = regexp.MustCompile(`(?i)\b(Bearer)(\s+)([A-Za-z0-9._~+/=-]{16,})`)
34	pemPrivateRE = regexp.MustCompile(`(?s)-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----.*?-----END [A-Z0-9 ]*PRIVATE KEY-----`)
35	// Provider prefixes. sk- allows internal hyphens/underscores so modern
36	// hyphenated formats (sk-ant-…, sk-proj-…) are covered, not just legacy
37	// sk-<alnum> keys.
38	providerRE = regexp.MustCompile(`\b(gh[opsur]_[A-Za-z0-9_]{20,}|github_pat_[A-Za-z0-9_]{20,}|glpat-[A-Za-z0-9_-]{20,}|(?:sk|rk)_(?:live|test)_[A-Za-z0-9]{16,}|sk-[A-Za-z0-9_-]{20,}|gsk_[A-Za-z0-9]{20,}|xai-[A-Za-z0-9-]{20,}|hf_[A-Za-z0-9]{20,}|npm_[A-Za-z0-9]{30,}|xox[bpcs]-[A-Za-z0-9-]{10,}|AIza[0-9A-Za-z_-]{30,})\b`)
39	jwtRE      = regexp.MustCompile(`\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{4,}\.[A-Za-z0-9_-]{4,}\b`)
40	// Password is greedy so a password containing '@' (user:p@ss@host) splits on
41	// the last '@' and is redacted whole, not just up to the first '@'.
42	connURLRE = regexp.MustCompile(`\b([A-Za-z][A-Za-z0-9+.-]*://)([^\s/@:]+):([^\s]+)@([^\s]+)`) // scheme://user:pass@host
43)
44
45func Disabled() bool { return os.Getenv("DEJA_NO_REDACT") == "1" }
46
47// kvHints are the substrings genericKVRE can anchor on; providerHints the
48// literal prefixes of providerRE. Checking them first keeps the regexes off
49// the vast majority of messages, which contain no credentials at all.
50var kvHints = []string{"key", "secret", "token", "passw", "authorization"}
51
52// "github_pat_" is listed on its own: "gh" is not a substring of "github".
53var providerHints = []string{"gh", "github_pat_", "glpat-", "sk_", "rk_", "sk-", "gsk_", "xai-", "hf_", "npm_", "xox", "AIza"}
54
55func containsAnyFold(s string, hints []string) bool {
56	for _, h := range hints {
57		if strings.Contains(s, h) {
58			return true
59		}
60	}
61	return false
62}
63
64func Text(s string) (string, Counts) {
65	counts := Counts{}
66	if Disabled() || s == "" {
67		return s, counts
68	}
69	lower := strings.ToLower(s)
70	if strings.Contains(s, "-----BEGIN") {
71		s = replaceWhole(s, pemPrivateRE, "private-key", counts)
72	}
73	if strings.Contains(s, "://") {
74		s = replaceSubmatch(s, connURLRE, "url-credentials", counts, func(m []string) string {
75			return m[1] + m[2] + ":[redacted:url-credentials]@" + m[4]
76		})
77	}
78	if strings.Contains(lower, "aws") {
79		s = replaceSubmatch(s, awsSecretRE, "aws-secret", counts, func(m []string) string {
80			return m[1] + m[2] + m[3] + "[redacted:aws-secret]" + closingQuote(m[3], m[5])
81		})
82	}
83	if strings.Contains(s, "AKIA") || strings.Contains(s, "ASIA") {
84		s = replaceWhole(s, awsAccessKeyRE, "aws-access-key", counts)
85	}
86	if strings.Contains(lower, "bearer") {
87		s = replaceSubmatch(s, bearerRE, "bearer-token", counts, func(m []string) string {
88			return m[1] + m[2] + "[redacted:bearer-token]"
89		})
90	}
91	if strings.Contains(s, "eyJ") {
92		s = replaceWhole(s, jwtRE, "jwt", counts)
93	}
94	if containsAnyFold(lower, kvHints) {
95		s = replaceSubmatch(s, genericKVRE, "credential", counts, func(m []string) string {
96			return m[1] + m[2] + m[3] + "[redacted:credential]" + closingQuote(m[3], m[5])
97		})
98	}
99	if containsAnyFold(s, providerHints) {
100		s = replaceProvider(s, counts)
101	}
102	return s, counts
103}

41func exportRecords(dir, outDir string, full bool) (int, error) {
42	if dir == "" {
43		dir = DefaultDir()
44	}
45	unlock, err := lockDir(dir)
46	if err != nil {
47		return 0, err
48	}
49	defer unlock()
50	m, err := readManifest(dir)
51	if err != nil {
52		return 0, err
53	}
54	if err := os.MkdirAll(outDir, 0o755); err != nil {
55		return 0, err
56	}
57	if m.ExportWatermarks == nil {
58		m.ExportWatermarks = map[string]int64{}
59	}
60	bySource := map[string][]SyncRecord{}
61	nextWatermarks := map[string]int64{}
62	for k, v := range m.ExportWatermarks {
63		nextWatermarks[k] = v
64	}
65	err = eachRecord(filepath.Join(dir, "records.bin"), func(r Record) {
66		if r.SourcePath == syncImportPath {
67			return
68		}
69		source := r.SourcePath
70		if source == "" {
71			source = r.Key
72		}
73		if !full && !r.Time.IsZero() && r.Time.UnixNano() <= m.ExportWatermarks[source] {
74			return
75		}
76		meta, ok := m.Sessions[r.Key]
77		if !ok {
78			return
79		}
80		text, _ := redact.Text(r.Text)
81		rec := SyncRecord{Harness: meta.Harness, SessionID: meta.ID, Project: meta.Project, Role: r.Role, Text: text, Time: r.Time}
82		bySource[source] = append(bySource[source], rec)
83		if r.Time.UnixNano() > nextWatermarks[source] {
84			nextWatermarks[source] = r.Time.UnixNano()
85		}
86	})
87	if err != nil {
88		return 0, err
89	}
90	total := 0
91	for source, recs := range bySource {
92		if len(recs) == 0 {
93			continue
94		}
95		name := fmt.Sprintf("deja-sync-%s-%d.jsonl", shortHash(source), time.Now().UnixNano())
96		f, err := os.Create(filepath.Join(outDir, name))
97		if err != nil {
98			return total, err
99		}
100		enc := json.NewEncoder(f)
101		for _, rec := range recs {
102			if err := enc.Encode(rec); err != nil {
103				_ = f.Close()
104				return total, err
105			}
106			total++
107		}
108		if err := f.Close(); err != nil {
109			return total, err
110		}
111	}
112	for source, wm := range nextWatermarks {
113		m.ExportWatermarks[source] = wm
114	}
115	if err := writeManifest(dir, m); err != nil {
116		return total, err
117	}
118	return total, nil
119}