package syncer import ( "context" "encoding/base64" "encoding/json" "fmt" "net/http" "net/http/httptest" "os" "path/filepath" "strings" "sync" "testing" "time" ) // fakeBucket 是一个内存版 HF Bucket:只实现同步用到的两个只读端点。 type fakeBucket struct { mu sync.Mutex files map[string][]byte mtime map[string]time.Time } func newFakeBucket() *fakeBucket { return &fakeBucket{files: map[string][]byte{}, mtime: map[string]time.Time{}} } func (f *fakeBucket) put(path string, content []byte, mtime time.Time) { f.mu.Lock() defer f.mu.Unlock() f.files[path] = content f.mtime[path] = mtime } func (f *fakeBucket) get(path string) ([]byte, bool) { f.mu.Lock() defer f.mu.Unlock() b, ok := f.files[path] return b, ok } func (f *fakeBucket) handler(bucket string) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { case strings.HasPrefix(r.URL.Path, "/api/buckets/"): f.mu.Lock() type entry struct { Type string `json:"type"` Path string `json:"path"` Size int64 `json:"size"` Mtime string `json:"mtime"` XetHash string `json:"xetHash"` } out := []entry{} for p, b := range f.files { out = append(out, entry{ Type: "file", Path: p, Size: int64(len(b)), Mtime: f.mtime[p].UTC().Format(time.RFC3339Nano), XetHash: "xet-" + p, }) } f.mu.Unlock() _ = json.NewEncoder(w).Encode(out) case strings.HasPrefix(r.URL.Path, "/buckets/"+bucket+"/resolve/"): // Go 已把 %2F 解码回 '/',直接剥前缀取路径。 p := strings.TrimPrefix(r.URL.Path, "/buckets/"+bucket+"/resolve/") b, ok := f.get(p) if !ok { http.Error(w, "not found", http.StatusNotFound) return } _, _ = w.Write(b) default: http.NotFound(w, r) } }) } // fakeWriter 记录写入,避免测试真的碰网络/子进程。 type fakeWriter struct { mu sync.Mutex put map[string][]byte batches int failOn string } func (f *fakeWriter) Name() string { return "fake" } func (f *fakeWriter) Put(_ context.Context, path string, content []byte) error { f.mu.Lock() defer f.mu.Unlock() if f.failOn == path { return fmt.Errorf("boom") } if f.put == nil { f.put = map[string][]byte{} } f.put[path] = content return nil } func (f *fakeWriter) PutBatch(_ context.Context, files []PushFile, _ bool) error { f.mu.Lock() defer f.mu.Unlock() f.batches++ if f.put == nil { f.put = map[string][]byte{} } for _, fl := range files { f.put[fl.Path] = fl.Content } return nil } func (f *fakeWriter) Delete(context.Context, string) error { return nil } func (f *fakeWriter) written(path string) ([]byte, bool) { f.mu.Lock() defer f.mu.Unlock() b, ok := f.put[path] return b, ok } // authDoc 造一个嵌套形态的凭证文件(网关 auth.Parse 认的形态)。 func authDoc(uid, nick, at, rt string, exp int64) []byte { return []byte(fmt.Sprintf( `{"account":{"uid":%q,"nickname":%q},"auth":{"accessToken":%q,"refreshToken":%q,"expiresAt":%d}}`, uid, nick, at, rt, exp)) } // newTestEngine 起一个假桶 + 假写入通道的 Engine。 func newTestEngine(t *testing.T, fb *fakeBucket, local map[string][]byte) (*Engine, string, string) { t.Helper() dir := t.TempDir() authDir := filepath.Join(dir, "auths") dataDir := filepath.Join(dir, "data") for rel, content := range local { fp := filepath.Join(dir, rel) if err := os.MkdirAll(filepath.Dir(fp), 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(fp, content, 0o600); err != nil { t.Fatal(err) } } ts := httptest.NewServer(fb.handler("u/b")) t.Cleanup(ts.Close) e, err := New(Config{ Bucket: "u/b", Endpoint: ts.URL, AuthDir: authDir, DataDir: dataDir, IncludeData: []string{"state.json"}, Writer: "relay", RelayURL: "http://127.0.0.1:1", // 占位,测试里换成 fakeWriter Timeout: "10s", }) if err != nil { t.Fatalf("New: %v", err) } return e, authDir, dataDir } // TestPullDownloadsRemoteOnly 本地首次启动:桶里独有的凭证要落到本地 auths/。 func TestPullDownloadsRemoteOnly(t *testing.T) { fb := newFakeBucket() fb.put("auths/workbuddy-u1.json", authDoc("u1", "n1", "at1", "rt1", 5000), time.Now()) e, authDir, _ := newTestEngine(t, fb, nil) fw := &fakeWriter{} e.writer = fw p, res, err := e.Sync(context.Background(), ModePull, false) if err != nil { t.Fatalf("Sync: %v", err) } if res.Downloaded != 1 || res.Failed != 0 { t.Fatalf("res=%+v plan=%+v", res, p.Actions) } raw, err := os.ReadFile(filepath.Join(authDir, "workbuddy-u1.json")) if err != nil { t.Fatalf("下载后本地文件不存在: %v", err) } if string(raw) != string(authDoc("u1", "n1", "at1", "rt1", 5000)) { t.Errorf("内容不一致: %s", raw) } } // TestPushUploadsLocalOnly 本地新增的账号要推到远端。 func TestPushUploadsLocalOnly(t *testing.T) { fb := newFakeBucket() e, _, _ := newTestEngine(t, fb, map[string][]byte{ "auths/workbuddy-u2.json": authDoc("u2", "n2", "at2", "rt2", 9000), }) fw := &fakeWriter{} e.writer = fw _, res, err := e.Sync(context.Background(), ModePush, false) if err != nil { t.Fatalf("Sync: %v", err) } if res.Uploaded != 1 || res.Failed != 0 { t.Fatalf("res=%+v", res) } got, ok := fw.written("auths/workbuddy-u2.json") if !ok { t.Fatal("远端没收到上传") } if string(got) != string(authDoc("u2", "n2", "at2", "rt2", 9000)) { t.Errorf("上传内容不一致: %s", got) } } // TestConflictNewerAccessTokenWins 两端都有同一账号且都刷新过 → 采用 expiresAt 更晚的一方, // 并把赢家回灌到输家。这是"防止凭证不同步"的核心行为。 func TestConflictNewerAccessTokenWins(t *testing.T) { fb := newFakeBucket() // 远端更新(expiresAt 9000),本地陈旧(expiresAt 1000)。 fb.put("auths/workbuddy-u3.json", authDoc("u3", "n3", "remote-at", "remote-rt", 9000), time.Now()) e, authDir, _ := newTestEngine(t, fb, map[string][]byte{ "auths/workbuddy-u3.json": authDoc("u3", "n3", "local-at", "local-rt", 1000), }) fw := &fakeWriter{} e.writer = fw p, res, err := e.Sync(context.Background(), ModeSync, false) if err != nil { t.Fatalf("Sync: %v", err) } if len(p.Conflicts()) != 1 { t.Fatalf("应识别出 1 个冲突: %+v", p.Actions) } if res.Downloaded != 1 || res.Uploaded != 0 { t.Fatalf("应由远端覆盖本地: res=%+v", res) } raw, _ := os.ReadFile(filepath.Join(authDir, "workbuddy-u3.json")) if !strings.Contains(string(raw), "remote-at") { t.Errorf("本地未被远端覆盖: %s", raw) } } // TestConflictLocalNewerPushes 反向:本地更新 → 推到远端。 func TestConflictLocalNewerPushes(t *testing.T) { fb := newFakeBucket() fb.put("auths/workbuddy-u4.json", authDoc("u4", "n4", "remote-at", "remote-rt", 1000), time.Now()) e, authDir, _ := newTestEngine(t, fb, map[string][]byte{ "auths/workbuddy-u4.json": authDoc("u4", "n4", "local-at", "local-rt", 9000), }) fw := &fakeWriter{} e.writer = fw _, res, err := e.Sync(context.Background(), ModeSync, false) if err != nil { t.Fatalf("Sync: %v", err) } if res.Uploaded != 1 || res.Downloaded != 0 { t.Fatalf("应由本地覆盖远端: res=%+v", res) } got, _ := fw.written("auths/workbuddy-u4.json") if !strings.Contains(string(got), "local-at") { t.Errorf("远端未被本地覆盖: %s", got) } raw, _ := os.ReadFile(filepath.Join(authDir, "workbuddy-u4.json")) if !strings.Contains(string(raw), "local-at") { t.Errorf("本地被误改: %s", raw) } } // TestIdenticalIsNoop 两端一致时不应产生任何读写。 func TestIdenticalIsNoop(t *testing.T) { doc := authDoc("u5", "n5", "at", "rt", 5000) fb := newFakeBucket() fb.put("auths/workbuddy-u5.json", doc, time.Now()) e, _, _ := newTestEngine(t, fb, map[string][]byte{"auths/workbuddy-u5.json": doc}) fw := &fakeWriter{} e.writer = fw p, res, err := e.Sync(context.Background(), ModeSync, false) if err != nil { t.Fatalf("Sync: %v", err) } if p.Changes() != 0 || res.Uploaded != 0 || res.Downloaded != 0 { t.Fatalf("一致时应无动作: plan=%+v res=%+v", p.Actions, res) } } // TestExcludedPathsNeverSynced config.json 与 request-logs 绝不能进同步范围。 func TestExcludedPathsNeverSynced(t *testing.T) { fb := newFakeBucket() fb.put("config.json", []byte(`{"listen":":7861","api_key":"relay-secret"}`), time.Now()) fb.put("data/request-logs/requests-2026-10-02.jsonl", []byte("{}\n"), time.Now()) fb.put("data/state.json", []byte(`{"accounts":{}}`), time.Now()) e, authDir, dataDir := newTestEngine(t, fb, nil) fw := &fakeWriter{} e.writer = fw p, res, err := e.Sync(context.Background(), ModeSync, false) if err != nil { t.Fatalf("Sync: %v", err) } if res.Downloaded != 1 { // 只有 data/state.json t.Fatalf("应只下载 state.json: res=%+v plan=%+v", res, p.Actions) } for _, a := range p.Actions { if a.Path == "config.json" || strings.HasPrefix(a.Path, "data/request-logs/") { t.Errorf("被排除的路径出现在计划里: %+v", a) } } if _, err := os.Stat(filepath.Join(authDir, "..", "config.json")); err == nil { t.Error("config.json 被写到了本地") } if _, err := os.Stat(filepath.Join(dataDir, "request-logs")); err == nil { t.Error("request-logs 被同步了") } } // TestDryRunNeverWrites dry-run(status)不得产生任何落盘或上传, // 但**必须**照常把差异算出来——漏算差异会让 status 骗人。 func TestDryRunNeverWrites(t *testing.T) { fb := newFakeBucket() fb.put("auths/workbuddy-u6.json", authDoc("u6", "n6", "at", "rt", 9000), time.Now()) e, authDir, _ := newTestEngine(t, fb, map[string][]byte{ "auths/workbuddy-u6.json": authDoc("u6", "n6", "old", "old", 1), }) fw := &fakeWriter{} e.writer = fw plan, res, err := e.Sync(context.Background(), ModeSync, true) if err != nil { t.Fatalf("Sync: %v", err) } if plan.Changes() != 1 { t.Fatalf("dry-run 也必须报出这 1 处差异: %+v", plan.Actions) } if res.Downloaded != 1 { // 计入"将会做",但没真做 t.Fatalf("dry-run 应计入计划动作: %+v", res) } if len(fw.put) != 0 || fw.batches != 0 { t.Errorf("dry-run 不应上传: %+v", fw.put) } raw, _ := os.ReadFile(filepath.Join(authDir, "workbuddy-u6.json")) if !strings.Contains(string(raw), "old") { t.Errorf("dry-run 不应改本地: %s", raw) } } // TestApplyNoWriterFailsLoudly 没有写入通道时必须明确失败,不能假装成功。 func TestApplyNoWriterFailsLoudly(t *testing.T) { fb := newFakeBucket() e, _, _ := newTestEngine(t, fb, map[string][]byte{ "auths/workbuddy-u7.json": authDoc("u7", "n7", "at", "rt", 9000), }) e.writer = nil _, res, err := e.Sync(context.Background(), ModePush, false) if err != nil { t.Fatalf("Sync 本身不应报错: %v", err) } if res.Failed != 1 || res.Uploaded != 0 { t.Fatalf("应报告 1 个失败: %+v", res) } } // TestNextLink 分页 Link 头解析(含相对 URL 与多段)。 func TestNextLink(t *testing.T) { const ep = "https://huggingface.co" cases := []struct{ in, want string }{ {"", ""}, {`; rel="next"`, "https://huggingface.co/api/buckets/u/b/tree?cursor=2"}, {`; rel="next"`, ep + "/api/buckets/u/b/tree?cursor=3"}, {`; rel="prev", ; rel="next"`, ep + "/api/buckets/u/b/tree?c=9"}, {`; rel="prev"`, ""}, {"garbage", ""}, } for _, tc := range cases { if got := nextLink(tc.in, ep); got != tc.want { t.Errorf("nextLink(%q) = %q, want %q", tc.in, got, tc.want) } } } // TestLocalPathRejectsTraversal localPath 必须挡住越界路径。 func TestLocalPathRejectsTraversal(t *testing.T) { e := &Engine{cfg: Config{AuthDir: "/tmp/a", DataDir: "/tmp/d", IncludeData: []string{"state.json"}}} for _, bad := range []string{"config.json", "../evil.json", "auths/../../evil.json", "other/x.json"} { if _, err := e.localPath(bad); err == nil { t.Errorf("localPath(%q) 应报错", bad) } } if fp, err := e.localPath("auths/workbuddy-1.json"); err != nil || !strings.HasSuffix(fp, "workbuddy-1.json") { t.Errorf("localPath 正常路径失败: %q %v", fp, err) } } // TestJSONOutputNeverLeaksCredentialMaterial 安全回归:FileState.Content 绝不能被序列化出去。 // // Action 会被 `wb2api-sync -json` 整个打印,Content 一旦进 JSON,`status -json` // 就会把 accessToken / refreshToken 明文(原始与 base64 两种形态)灌进终端、 // CI 日志或 issue 附件——等于自己把凭证泄露出去。实测早期版本正是这样泄露的。 func TestJSONOutputNeverLeaksCredentialMaterial(t *testing.T) { const secret = "SUPER-SECRET-REFRESH-TOKEN-VALUE" doc := authDoc("u1", "nick", "access-"+secret, "refresh-"+secret, 9000) st := newFileState("auths/workbuddy-u1.json", doc, time.Now(), catAuth) if st.Content == nil { t.Fatal("夹具前提不成立:Content 应为原始字节") } if !strings.Contains(string(st.Content), secret) { t.Fatal("夹具前提不成立:内容里应含 secret") } raw, err := json.Marshal(Action{Op: OpDownload, Path: st.Path, Category: catAuth, Local: st, Remote: st}) if err != nil { t.Fatal(err) } got := string(raw) if strings.Contains(got, secret) { t.Fatalf("JSON 里出现了凭证明文: %s", got) } if strings.Contains(strings.ToLower(got), `"content"`) { t.Errorf("JSON 里出现了 content 字段: %s", got) } b64 := base64.StdEncoding.EncodeToString(doc) if len(b64) > 24 && strings.Contains(got, b64[:24]) { t.Errorf("JSON 里出现了 content 的 base64 前缀: %s", got) } // 该有的诊断字段不能因为脱敏一起丢掉。 if !strings.Contains(got, st.SHA256) || !strings.Contains(got, "u1") { t.Errorf("SHA256/UID 等诊断字段应保留: %s", got) } }