同步与安全 深度解析

25var (
26	awsAccessKeyRE = regexp.MustCompile(`A(?:KIA|SIA)[0-9A-Z]{16}`)
27	awsSecretRE    = regexp.MustCompile(`(?i)\b(aws[_-]?secret[_-]?access[_-]?key)(\s*['"]?\s*[:=]\s*)(['"]?)([A-Za-z0-9/+=_-]{32,})(['"]?)`)
28	// The key may be embedded in a larger identifier (ANTHROPIC_API_KEY,
29	// x-api-key) and, in JSON, a closing quote can sit between the key and the
30	// delimiter ("api_key": "..."). Tolerate both so env-var and JSON forms are
31	// caught, not just a bare `api_key=`.
32	genericKVRE  = regexp.MustCompile(`(?i)\b([\w.-]{0,64}?(?:api[_-]?key|secret|token|passwd|password|authorization))(\s*['"]?\s*[:=]\s*)(['"]?)([A-Za-z0-9/+=._-]{16,})(['"]?)`)
33	bearerRE     = regexp.MustCompile(`(?i)\b(Bearer)(\s+)([A-Za-z0-9._~+/=-]{16,})`)
34	pemPrivateRE = regexp.MustCompile(`(?s)-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----.*?-----END [A-Z0-9 ]*PRIVATE KEY-----`)
35	// Provider prefixes. sk- allows internal hyphens/underscores so modern
36	// hyphenated formats (sk-ant-…, sk-proj-…) are covered, not just legacy
37	// sk-<alnum> keys.
38	providerRE = regexp.MustCompile(`\b(gh[opsur]_[A-Za-z0-9_]{20,}|github_pat_[A-Za-z0-9_]{20,}|glpat-[A-Za-z0-9_-]{20,}|(?:sk|rk)_(?:live|test)_[A-Za-z0-9]{16,}|sk-[A-Za-z0-9_-]{20,}|gsk_[A-Za-z0-9]{20,}|xai-[A-Za-z0-9-]{20,}|hf_[A-Za-z0-9]{20,}|npm_[A-Za-z0-9]{30,}|xox[bpcs]-[A-Za-z0-9-]{10,}|AIza[0-9A-Za-z_-]{30,})\b`)
39	jwtRE      = regexp.MustCompile(`\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{4,}\.[A-Za-z0-9_-]{4,}\b`)
40	// Password is greedy so a password containing '@' (user:p@ss@host) splits on
41	// the last '@' and is redacted whole, not just up to the first '@'.
42	connURLRE = regexp.MustCompile(`\b([A-Za-z][A-Za-z0-9+.-]*://)([^\s/@:]+):([^\s]+)@([^\s]+)`) // scheme://user:pass@host
43)
44
45func Disabled() bool { return os.Getenv("DEJA_NO_REDACT") == "1" }
46
47// kvHints are the substrings genericKVRE can anchor on; providerHints the
48// literal prefixes of providerRE. Checking them first keeps the regexes off
49// the vast majority of messages, which contain no credentials at all.
50var kvHints = []string{"key", "secret", "token", "passw", "authorization"}
51
52// "github_pat_" is listed on its own: "gh" is not a substring of "github".
53var providerHints = []string{"gh", "github_pat_", "glpat-", "sk_", "rk_", "sk-", "gsk_", "xai-", "hf_", "npm_", "xox", "AIza"}
54
55func containsAnyFold(s string, hints []string) bool {
56	for _, h := range hints {
57		if strings.Contains(s, h) {
58			return true
59		}
60	}
61	return false
62}
63
64func Text(s string) (string, Counts) {

64func Text(s string) (string, Counts) {
65	counts := Counts{}
66	if Disabled() || s == "" {
67		return s, counts
68	}
69	lower := strings.ToLower(s)
70	if strings.Contains(s, "-----BEGIN") {
71		s = replaceWhole(s, pemPrivateRE, "private-key", counts)
72	}
73	if strings.Contains(s, "://") {
74		s = replaceSubmatch(s, connURLRE, "url-credentials", counts, func(m []string) string {
75			return m[1] + m[2] + ":[redacted:url-credentials]@" + m[4]
76		})
77	}
78	if strings.Contains(lower, "aws") {
79		s = replaceSubmatch(s, awsSecretRE, "aws-secret", counts, func(m []string) string {
80			return m[1] + m[2] + m[3] + "[redacted:aws-secret]" + closingQuote(m[3], m[5])
81		})
82	}
83	if strings.Contains(s, "AKIA") || strings.Contains(s, "ASIA") {
84		s = replaceWhole(s, awsAccessKeyRE, "aws-access-key", counts)
85	}
86	if strings.Contains(lower, "bearer") {
87		s = replaceSubmatch(s, bearerRE, "bearer-token", counts, func(m []string) string {
88			return m[1] + m[2] + "[redacted:bearer-token]"
89		})
90	}
91	if strings.Contains(s, "eyJ") {
92		s = replaceWhole(s, jwtRE, "jwt", counts)
93	}
94	if containsAnyFold(lower, kvHints) {
95		s = replaceSubmatch(s, genericKVRE, "credential", counts, func(m []string) string {
96			return m[1] + m[2] + m[3] + "[redacted:credential]" + closingQuote(m[3], m[5])
97		})
98	}
99	if containsAnyFold(s, providerHints) {
100		s = replaceProvider(s, counts)
101	}
102	return s, counts
103}

17func runShare(args []string, w io.Writer) error {
18	if len(args) < 1 {
19		return fmt.Errorf("share needs id-prefix")
20	}
21	s, ok, err := findByPrefix(args[0])
22	if err != nil {
23		return err
24	}
25	if !ok {
26		return fmt.Errorf("no session matches %q", args[0])
27	}
28	printSanitized(w, shareDigest(s, shareBudget))
29	return nil
30}
31
32func shareDigest(s model.Session, budget int) string {
33	if budget <= 0 {
34		budget = shareBudget
35	}
36	var b strings.Builder
37	date := "unknown"
38	if !s.Updated.IsZero() {
39		date = s.Updated.Format(time.RFC3339)
40	}
41	fmt.Fprintf(&b, "# deja share: %s\n\n", s.ID)
42	fmt.Fprintf(&b, "- Project: %s\n", s.Project)
43	fmt.Fprintf(&b, "- Harness: %s\n", s.Harness)
44	fmt.Fprintf(&b, "- Date: %s\n\n", date)
45	appendSection := func(title string, messages []model.Message) {
46		if len(messages) == 0 || b.Len() >= budget {
47			return
48		}
49		fmt.Fprintf(&b, "## %s\n\n", title)
50		for _, m := range messages {
51			if b.Len() >= budget {
52				break
53			}
54			text := shareMessageText(m.Text)
55			if text == "" {
56				continue
57			}
58			chunk := fmt.Sprintf("%s\n\n", text)
59			if b.Len()+len(chunk) > budget {
60				chunk = utf8SafeCut(chunk, budget-b.Len())
61			}
62			b.WriteString(chunk)
63		}
64	}
65	var users, assistants []model.Message
66	for _, m := range s.Messages {
67		if noisyShareMessage(m.Text) {
68			continue
69		}
70		switch m.Role {
71		case "user":
72			users = append(users, m)
73		case "assistant":
74			assistants = append(assistants, m)
75		}
76	}
77	appendSection("User problem statement(s)", users)
78	appendSection("Key assistant conclusions / code blocks", assistants)
79	return strings.TrimSpace(b.String()) + "\n"
80}
81
82func printSanitized(w io.Writer, text string) {
83	// Redact the whole document at once: multiline secrets (PEM private key
84	// blocks) never match when scanned line-by-line.
85	redacted, _ := redact.Text(text)
86	fmt.Fprint(w, redacted)
87	if !strings.HasSuffix(redacted, "\n") {
88		fmt.Fprintln(w)
89	}
90}

41func exportRecords(dir, outDir string, full bool) (int, error) {
42	if dir == "" {
43		dir = DefaultDir()
44	}
45	unlock, err := lockDir(dir)
46	if err != nil {
47		return 0, err
48	}
49	defer unlock()
50	m, err := readManifest(dir)
51	if err != nil {
52		return 0, err
53	}
54	if err := os.MkdirAll(outDir, 0o755); err != nil {
55		return 0, err
56	}
57	if m.ExportWatermarks == nil {
58		m.ExportWatermarks = map[string]int64{}
59	}
60	bySource := map[string][]SyncRecord{}
61	nextWatermarks := map[string]int64{}
62	for k, v := range m.ExportWatermarks {
63		nextWatermarks[k] = v
64	}
65	err = eachRecord(filepath.Join(dir, "records.bin"), func(r Record) {
66		if r.SourcePath == syncImportPath {
67			return
68		}
69		source := r.SourcePath
70		if source == "" {
71			source = r.Key
72		}
73		if !full && !r.Time.IsZero() && r.Time.UnixNano() <= m.ExportWatermarks[source] {
74			return
75		}
76		meta, ok := m.Sessions[r.Key]
77		if !ok {
78			return
79		}
80		text, _ := redact.Text(r.Text)
81		rec := SyncRecord{Harness: meta.Harness, SessionID: meta.ID, Project: meta.Project, Role: r.Role, Text: text, Time: r.Time}
82		bySource[source] = append(bySource[source], rec)
83		if r.Time.UnixNano() > nextWatermarks[source] {
84			nextWatermarks[source] = r.Time.UnixNano()
85		}
86	})
87	if err != nil {
88		return 0, err
89	}
90	total := 0
91	for source, recs := range bySource {
92		if len(recs) == 0 {
93			continue
94		}
95		name := fmt.Sprintf("deja-sync-%s-%d.jsonl", shortHash(source), time.Now().UnixNano())
96		f, err := os.Create(filepath.Join(outDir, name))
97		if err != nil {
98			return total, err
99		}
100		enc := json.NewEncoder(f)
101		for _, rec := range recs {
102			if err := enc.Encode(rec); err != nil {
103				_ = f.Close()
104				return total, err
105			}
106			total++
107		}
108		if err := f.Close(); err != nil {
109			return total, err
110		}
111	}
112	for source, wm := range nextWatermarks {
113		m.ExportWatermarks[source] = wm
114	}
115	if err := writeManifest(dir, m); err != nil {
116		return total, err
117	}
118	return total, nil
119}

121func Import(dir, inDir string) (int, error) {
122	if dir == "" {
123		dir = DefaultDir()
124	}
125	unlock, err := lockDir(dir)
126	if err != nil {
127		return 0, err
128	}
129	defer unlock()
130	if !HasManifest(dir) {
131		if err := initEmptyIndex(dir); err != nil {
132			return 0, err
133		}
134	}
135	m, err := readManifest(dir)
136	if err != nil {
137		return 0, err
138	}
139	if m.ImportedRecords == nil {
140		m.ImportedRecords = map[string]bool{}
141	}
142	paths, err := filepath.Glob(filepath.Join(inDir, "*.jsonl"))
143	if err != nil {
144		return 0, err
145	}
146	sort.Strings(paths)
147	recsByKey := map[string][]Record{}
148	metas := map[string]SessionMeta{}
149	added := 0
150	for _, p := range paths {
151		if err := readSyncFile(p, func(sr SyncRecord) error {
152			origID := sr.SessionID
153			if sr.Harness == "" || origID == "" {
154				return nil
155			}
156			// Key includes role and a text hash: two messages can legally share
157			// a timestamp (aider stamps a whole session with its start time).
158			// The legacy time-only key is still honored so batches imported by
159			// older versions stay idempotent.
160			legacy := sr.Harness + ":" + origID + ":" + sr.Time.UTC().Format(time.RFC3339Nano)
161			th := fnv.New64a()
162			_, _ = th.Write([]byte(sr.Text))
163			dedupe := legacy + ":" + sr.Role + ":" + strconv.FormatUint(th.Sum64(), 16)
164			if m.ImportedRecords[dedupe] || m.ImportedRecords[legacy] {
165				return nil
166			}
167			importID := ImportedSessionID(sr.Harness, origID)
168			key := sr.Harness + ":" + importID
169			text, _ := redact.Text(sr.Text)
170			recsByKey[key] = append(recsByKey[key], Record{Key: key, Role: sr.Role, Text: text, Time: sr.Time, SourcePath: syncImportPath})
171			meta := metas[key]
172			if meta.ID == "" {
173				meta = SessionMeta{ID: importID, Harness: sr.Harness, Project: "imported:" + sr.Project, Path: syncImportPath}
174			}
175			if meta.Started.IsZero() || (!sr.Time.IsZero() && sr.Time.Before(meta.Started)) {
176				meta.Started = sr.Time
177			}
178			if sr.Time.After(meta.Updated) {
179				meta.Updated = sr.Time
180			}
181			metas[key] = meta
182			m.ImportedRecords[dedupe] = true
183			added++
184			return nil
185		}); err != nil {
186			return added, err
187		}
188	}
189	if added == 0 {
190		return 0, writeManifest(dir, m)
191	}
192	if err := appendImportedRecords(dir, &m, recsByKey, metas); err != nil {
193		return added, err
194	}
195	return added, nil

48func syncSSHPush(host string, full bool) error {
49	if err := index.EnsureForSearch(index.DefaultDir(), search.Options{All: true}, false, os.Stderr); err != nil {
50		return err
51	}
52	tmp, err := os.MkdirTemp("", "deja-sync-")
53	if err != nil {
54		return err
55	}
56	defer os.RemoveAll(tmp)
57	n, err := exportBatches(tmp, full)
58	if err != nil {
59		return err
60	}
61	fmt.Fprintf(os.Stdout, "deja: exported %d records\n", n)
62	if n == 0 {
63		fmt.Fprintln(os.Stdout, "deja: nothing new to push")
64		return nil
65	}
66	batches, err := filepath.Glob(filepath.Join(tmp, "*.jsonl"))
67	if err != nil || len(batches) == 0 {
68		return fmt.Errorf("no batches produced")
69	}
70	rtmp, err := sshCapture(host, "mktemp -d")
71	if err != nil {
72		return err
73	}
74	scpArgs := append([]string{"-q"}, batches...)
75	scpArgs = append(scpArgs, host+":"+rtmp+"/")
76	if out, err := sshRunner("scp", scpArgs...); err != nil {
77		return fmt.Errorf("scp: %v: %s", err, strings.TrimSpace(out))
78	}
79	remote := fmt.Sprintf(`d=$(command -v deja || echo "$HOME/.local/bin/deja"); "$d" sync import %s; rc=$?; rm -rf %s; exit $rc`,
80		shellQuote(rtmp), shellQuote(rtmp))
81	out, err := sshRunner("ssh", host, "sh -lc "+shellQuote(remote))
82	out = strings.TrimSpace(out)
83	if err != nil {
84		return fmt.Errorf("remote import: %v: %s", err, out)
85	}
86	if out != "" {
87		fmt.Fprintf(os.Stdout, "%s: %s\n", host, out)
88	}
89	return nil
90}