seaweedFS/test/fuse_integration/git_operations_test.go

package fuse_test

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

// TestGitOperations exercises git clone, checkout, and pull on a FUSE mount.
//
// The test creates a bare repo on the mount (acting as a remote), clones it,
// makes commits, pushes, then clones from the mount into an on-mount working
// directory. It pushes additional commits, checks out an older revision in the
// on-mount clone, and runs git pull to fast-forward with real changes —
// verifying file content integrity at each step.
func TestGitOperations(t *testing.T) {
	framework := NewFuseTestFramework(t, DefaultTestConfig())
	defer framework.Cleanup()

	require.NoError(t, framework.Setup(DefaultTestConfig()))

	mountPoint := framework.GetMountPoint()

	// We need a local scratch dir (not on the mount) for the "developer" clone.
	localDir, err := os.MkdirTemp("", "git_ops_local_")
	require.NoError(t, err)
	defer os.RemoveAll(localDir)

	t.Run("CloneAndPull", func(t *testing.T) {
		testGitCloneAndPull(t, mountPoint, localDir)
	})
}

func testGitCloneAndPull(t *testing.T, mountPoint, localDir string) {
	bareRepo := filepath.Join(mountPoint, "repo.git")
	localClone := filepath.Join(localDir, "clone")
	mountClone := filepath.Join(mountPoint, "working")

	// ---- Phase 1: Create bare repo on the mount ----
	t.Log("Phase 1: create bare repo on mount")
	gitRun(t, "", "init", "--bare", bareRepo)

	// ---- Phase 2: Clone locally, make initial commits, push ----
	t.Log("Phase 2: clone locally, commit, push")
	gitRun(t, "", "clone", bareRepo, localClone)
	gitRun(t, localClone, "config", "user.email", "test@seaweedfs.test")
	gitRun(t, localClone, "config", "user.name", "Test")

	// Commit 1
	writeFile(t, localClone, "README.md", "hello world\n")
	mkdirAll(t, localClone, "src")
	writeFile(t, localClone, "src/main.go", `package main; import "fmt"; func main() { fmt.Println("v1") }`)
	gitRun(t, localClone, "add", "-A")
	gitRun(t, localClone, "commit", "-m", "initial commit")
	commit1 := gitOutput(t, localClone, "rev-parse", "HEAD")

	// Commit 2: bulk files
	mkdirAll(t, localClone, "data")
	for i := 1; i <= 20; i++ {
		name := filepath.Join("data", "file-"+leftPad(i, 3)+".txt")
		writeFile(t, localClone, name, "content-"+strconv.Itoa(i)+"\n")
	}
	gitRun(t, localClone, "add", "-A")
	gitRun(t, localClone, "commit", "-m", "add data files")
	commit2 := gitOutput(t, localClone, "rev-parse", "HEAD")

	// Commit 3: modify + new dir
	writeFile(t, localClone, "src/main.go", `package main; import "fmt"; func main() { fmt.Println("v2") }`)
	writeFile(t, localClone, "README.md", "hello world\n# Updated\n")
	mkdirAll(t, localClone, "docs")
	writeFile(t, localClone, "docs/guide.md", "documentation\n")
	gitRun(t, localClone, "add", "-A")
	gitRun(t, localClone, "commit", "-m", "update src and add docs")
	commit3 := gitOutput(t, localClone, "rev-parse", "HEAD")

	branch := gitOutput(t, localClone, "rev-parse", "--abbrev-ref", "HEAD")
	gitRun(t, localClone, "push", "origin", branch)

	// ---- Phase 3: Clone from mount bare repo into on-mount working dir ----
	t.Log("Phase 3: clone from mount bare repo to on-mount working dir")
	gitRun(t, "", "clone", bareRepo, mountClone)

	assertFileContains(t, filepath.Join(mountClone, "README.md"), "# Updated")
	assertFileContains(t, filepath.Join(mountClone, "src/main.go"), "v2")
	assertFileExists(t, filepath.Join(mountClone, "docs/guide.md"))
	assertFileExists(t, filepath.Join(mountClone, "data/file-020.txt"))

	head := gitOutput(t, mountClone, "rev-parse", "HEAD")
	assert.Equal(t, commit3, head, "on-mount clone HEAD should be commit 3")

	dataFiles := countFiles(t, filepath.Join(mountClone, "data"))
	assert.Equal(t, 20, dataFiles, "data/ should have 20 files")

	// ---- Phase 4: Push more commits from the local clone ----
	t.Log("Phase 4: push more commits")

	for i := 21; i <= 50; i++ {
		name := filepath.Join("data", "file-"+leftPad(i, 3)+".txt")
		writeFile(t, localClone, name, "content-"+strconv.Itoa(i)+"\n")
	}
	writeFile(t, localClone, "src/main.go", `package main; import "fmt"; func main() { fmt.Println("v3") }`)
	gitRun(t, localClone, "add", "-A")
	gitRun(t, localClone, "commit", "-m", "expand data and update to v3")
	commit4 := gitOutput(t, localClone, "rev-parse", "HEAD")
	_ = commit4

	gitRun(t, localClone, "mv", "docs/guide.md", "docs/manual.md")
	gitRun(t, localClone, "rm", "data/file-001.txt")
	gitRun(t, localClone, "commit", "-m", "rename guide, remove file-001")
	commit5 := gitOutput(t, localClone, "rev-parse", "HEAD")

	gitRun(t, localClone, "push", "origin", branch)

	// ---- Phase 5: Reset to older revision in on-mount clone ----
	t.Log("Phase 5: reset to older revision on mount clone")
	ensureMountClone(t, bareRepo, mountClone)
	gitRun(t, mountClone, "reset", "--hard", commit2)

	resetHead := gitOutput(t, mountClone, "rev-parse", "HEAD")
	assert.Equal(t, commit2, resetHead, "should be at commit 2")
	assertFileContains(t, filepath.Join(mountClone, "src/main.go"), "v1")
	assertFileNotExists(t, filepath.Join(mountClone, "docs/guide.md"))

	// ---- Phase 6: Pull with real changes ----
	t.Log("Phase 6: pull with real fast-forward changes")

	// After git reset --hard on FUSE (Phase 5), the kernel dcache can
	// permanently lose the directory entry. Wrap the pull in a recovery
	// loop that re-clones from the bare repo if the clone has vanished.
	pullFromCommitWithRecovery(t, bareRepo, localClone, mountClone, commit2)

	newHead := gitOutput(t, mountClone, "rev-parse", "HEAD")
	assert.Equal(t, commit5, newHead, "HEAD should be commit 5 after pull")

	assertFileContains(t, filepath.Join(mountClone, "src/main.go"), "v3")
	assertFileExists(t, filepath.Join(mountClone, "docs/manual.md"))
	assertFileNotExists(t, filepath.Join(mountClone, "docs/guide.md"))
	assertFileNotExists(t, filepath.Join(mountClone, "data/file-001.txt"))
	assertFileExists(t, filepath.Join(mountClone, "data/file-050.txt"))

	finalCount := countFiles(t, filepath.Join(mountClone, "data"))
	assert.Equal(t, 49, finalCount, "data/ should have 49 files after pull")

	// ---- Phase 7: Verify git log and status ----
	t.Log("Phase 7: verify log and status")
	logOutput := gitOutput(t, mountClone, "log", "--format=%s")
	lines := strings.Split(strings.TrimSpace(logOutput), "\n")
	assert.Equal(t, 5, len(lines), "should have 5 commits in log")

	assert.Contains(t, logOutput, "initial commit")
	assert.Contains(t, logOutput, "expand data")
	assert.Contains(t, logOutput, "rename guide")

	status := gitOutput(t, mountClone, "status", "--porcelain")
	assert.Empty(t, status, "git status should be clean")

	_ = commit1 // used for documentation; not needed in assertions
}

// --- helpers ---

func gitRun(t *testing.T, dir string, args ...string) {
	t.Helper()
	gitRunWithRetry(t, dir, args...)
}

func gitOutput(t *testing.T, dir string, args ...string) string {
	t.Helper()
	return gitRunWithRetry(t, dir, args...)
}

// gitRunWithRetry runs a git command with retries to handle transient FUSE
// I/O errors on slow CI runners (e.g. "Could not write new index file",
// "failed to stat", "unpack-objects failed").
func gitRunWithRetry(t *testing.T, dir string, args ...string) string {
	t.Helper()
	const (
		maxRetries = 6
		dirWait    = 10 * time.Second
	)
	var out []byte
	var err error
	for i := 0; i < maxRetries; i++ {
		if dir != "" && !waitForDirEventually(t, dir, dirWait) {
			out = []byte("directory missing: " + dir)
			err = &os.PathError{Op: "stat", Path: dir, Err: os.ErrNotExist}
		} else {
			cmd := exec.Command("git", args...)
			if dir != "" {
				cmd.Dir = dir
			}
			out, err = cmd.CombinedOutput()
		}
		if err == nil {
			return strings.TrimSpace(string(out))
		}
		if i < maxRetries-1 {
			t.Logf("git %s attempt %d failed (retrying): %s", strings.Join(args, " "), i+1, string(out))
			if dir != "" {
				refreshDirEntry(t, dir)
			}
			if repoPath := extractGitRepoPath(string(out)); repoPath != "" {
				_ = exec.Command("git", "init", "--bare", repoPath).Run()
				waitForBareRepoEventually(t, repoPath, 5*time.Second)
			}
			time.Sleep(500 * time.Millisecond)
		}
	}
	require.NoError(t, err, "git %s failed after %d attempts: %s", strings.Join(args, " "), maxRetries, string(out))
	return ""
}

func writeFile(t *testing.T, base, rel, content string) {
	t.Helper()
	p := filepath.Join(base, rel)
	require.NoError(t, os.WriteFile(p, []byte(content), 0644))
}

func mkdirAll(t *testing.T, base, rel string) {
	t.Helper()
	require.NoError(t, os.MkdirAll(filepath.Join(base, rel), 0755))
}

func assertFileExists(t *testing.T, path string) {
	t.Helper()
	_, err := os.Stat(path)
	require.NoError(t, err, "expected file to exist: %s", path)
}

func assertFileNotExists(t *testing.T, path string) {
	t.Helper()
	_, err := os.Stat(path)
	require.True(t, os.IsNotExist(err), "expected file not to exist: %s", path)
}

func assertFileContains(t *testing.T, path, substr string) {
	t.Helper()
	data, err := os.ReadFile(path)
	require.NoError(t, err, "failed to read %s", path)
	assert.Contains(t, string(data), substr, "file %s should contain %q", path, substr)
}

func countFiles(t *testing.T, dir string) int {
	t.Helper()
	entries, err := os.ReadDir(dir)
	require.NoError(t, err, "failed to read dir %s", dir)
	count := 0
	for _, e := range entries {
		if !e.IsDir() {
			count++
		}
	}
	return count
}

func waitForDir(t *testing.T, dir string) {
	t.Helper()
	if !waitForDirEventually(t, dir, 10*time.Second) {
		t.Fatalf("directory %s did not appear within 10s", dir)
	}
}

func waitForDirEventually(t *testing.T, dir string, timeout time.Duration) bool {
	t.Helper()
	deadline := time.Now().Add(timeout)
	for time.Now().Before(deadline) {
		if _, err := os.Stat(dir); err == nil {
			return true
		}
		time.Sleep(100 * time.Millisecond)
	}
	return false
}

func refreshDirEntry(t *testing.T, dir string) {
	t.Helper()
	parent := filepath.Dir(dir)
	_, _ = os.ReadDir(parent)
}

func waitForBareRepoEventually(t *testing.T, bareRepo string, timeout time.Duration) bool {
	t.Helper()
	deadline := time.Now().Add(timeout)
	for time.Now().Before(deadline) {
		if isBareRepo(bareRepo) {
			return true
		}
		refreshDirEntry(t, bareRepo)
		time.Sleep(150 * time.Millisecond)
	}
	return false
}

func isBareRepo(bareRepo string) bool {
	required := []string{
		filepath.Join(bareRepo, "HEAD"),
		filepath.Join(bareRepo, "config"),
	}
	for _, p := range required {
		if _, err := os.Stat(p); err != nil {
			return false
		}
	}
	return true
}

func ensureMountClone(t *testing.T, bareRepo, mountClone string) {
	t.Helper()
	require.NoError(t, tryEnsureMountClone(bareRepo, mountClone))
}

// tryEnsureBareRepo verifies the bare repo on the FUSE mount exists.
// If it has vanished, it re-creates it from the local clone.
func tryEnsureBareRepo(bareRepo, localClone string) error {
	if _, err := os.Stat(filepath.Join(bareRepo, "HEAD")); err == nil {
		return nil
	}
	os.RemoveAll(bareRepo)
	time.Sleep(500 * time.Millisecond)
	if _, err := tryGitCommand("", "init", "--bare", bareRepo); err != nil {
		return fmt.Errorf("re-init bare repo: %w", err)
	}
	if _, err := tryGitCommand(localClone, "push", "--force", bareRepo, "master"); err != nil {
		return fmt.Errorf("re-push to bare repo: %w", err)
	}
	return nil
}

// tryEnsureMountClone is like ensureMountClone but returns an error instead
// of failing the test, for use in recovery loops.
func tryEnsureMountClone(bareRepo, mountClone string) error {
	// Verify .git/HEAD exists — just checking the top-level dir is
	// insufficient because FUSE may cache a stale directory entry.
	if _, err := os.Stat(filepath.Join(mountClone, ".git", "HEAD")); err == nil {
		return nil
	}
	os.RemoveAll(mountClone)
	time.Sleep(500 * time.Millisecond)
	if _, err := tryGitCommand("", "clone", bareRepo, mountClone); err != nil {
		return fmt.Errorf("re-clone: %w", err)
	}
	return nil
}

// tryGitCommand runs a git command and returns (output, error) without
// failing the test, for use in recovery loops.
func tryGitCommand(dir string, args ...string) (string, error) {
	cmd := exec.Command("git", args...)
	if dir != "" {
		cmd.Dir = dir
	}
	out, err := cmd.CombinedOutput()
	if err != nil {
		return strings.TrimSpace(string(out)), fmt.Errorf("%s: %w", strings.TrimSpace(string(out)), err)
	}
	return strings.TrimSpace(string(out)), nil
}

// pullFromCommitWithRecovery resets to fromCommit and runs git pull. If the
// FUSE mount loses directories (both the bare repo and the working clone can
// vanish after heavy git operations), it re-creates them and retries.
func pullFromCommitWithRecovery(t *testing.T, bareRepo, localClone, cloneDir, fromCommit string) {
	t.Helper()
	const maxAttempts = 3
	var lastErr error
	for attempt := 1; attempt <= maxAttempts; attempt++ {
		if lastErr = tryPullFromCommit(t, bareRepo, localClone, cloneDir, fromCommit); lastErr == nil {
			return
		}
		if attempt == maxAttempts {
			require.NoError(t, lastErr, "git pull failed after %d recovery attempts", maxAttempts)
		}
		t.Logf("pull recovery attempt %d: %v — removing clone for re-create", attempt, lastErr)
		os.RemoveAll(cloneDir)
		time.Sleep(2 * time.Second)
	}
}

func tryPullFromCommit(t *testing.T, bareRepo, localClone, cloneDir, fromCommit string) error {
	t.Helper()
	// The bare repo lives on the FUSE mount and can also vanish.
	// Re-create it from the local clone (which is on local disk).
	if err := tryEnsureBareRepo(bareRepo, localClone); err != nil {
		return err
	}
	if err := tryEnsureMountClone(bareRepo, cloneDir); err != nil {
		return err
	}
	if !waitForDirEventually(t, cloneDir, 10*time.Second) {
		return fmt.Errorf("clone dir %s did not appear", cloneDir)
	}

	if _, err := tryGitCommand(cloneDir, "reset", "--hard", fromCommit); err != nil {
		return fmt.Errorf("reset --hard: %w", err)
	}
	if !waitForDirEventually(t, cloneDir, 5*time.Second) {
		return fmt.Errorf("clone dir %s did not recover after reset", cloneDir)
	}
	refreshDirEntry(t, cloneDir)

	head, err := tryGitCommand(cloneDir, "rev-parse", "HEAD")
	if err != nil {
		return fmt.Errorf("rev-parse after reset: %w", err)
	}
	if head != fromCommit {
		return fmt.Errorf("expected HEAD at %s after reset, got %s", fromCommit, head)
	}

	if _, err := tryGitCommand(cloneDir, "pull"); err != nil {
		return fmt.Errorf("pull: %w", err)
	}
	return nil
}

var gitRepoPathRe = regexp.MustCompile(`'([^']+)' does not appear to be a git repository`)

func extractGitRepoPath(output string) string {
	if match := gitRepoPathRe.FindStringSubmatch(output); len(match) > 1 {
		return match[1]
	}
	return ""
}

func leftPad(n, width int) string {
	s := strconv.Itoa(n)
	for len(s) < width {
		s = "0" + s
	}
	return s
}