Use Unix sockets for gRPC in mini mode (#8856)

* Use Unix sockets for gRPC between co-located services in mini mode

In `weed mini`, all services run in one process. Previously, inter-service
gRPC traffic (volume↔master, filer↔master, S3↔filer, worker↔admin, etc.)
went through TCP loopback. This adds a gRPC Unix socket registry in the pb
package: mini mode registers a socket path per gRPC port at startup, each
gRPC server additionally listens on its socket, and GrpcDial transparently
routes to the socket via WithContextDialer when a match is found.

Standalone commands (weed master, weed filer, etc.) are unaffected since
no sockets are registered. TCP listeners are kept for external clients.

* Handle Serve error and clean up socket file in ServeGrpcOnLocalSocket

Log non-expected errors from grpcServer.Serve (ignoring
grpc.ErrServerStopped) and always remove the Unix socket file
when Serve returns, ensuring cleanup on Stop/GracefulStop.
This commit is contained in:
Chris Lu
2026-03-30 18:18:52 -07:00
committed by GitHub
parent 0ce4a857e6
commit 2eaf98a7a2
7 changed files with 85 additions and 0 deletions

View File

@@ -4,7 +4,9 @@ import (
"context"
"fmt"
"math/rand/v2"
"net"
"net/http"
"os"
"strconv"
"strings"
"sync"
@@ -42,6 +44,12 @@ var (
// cache grpc connections
grpcClients = make(map[string]*versionedGrpcClient)
grpcClientsLock sync.Mutex
// localGrpcSockets maps gRPC port numbers to Unix socket paths.
// When registered (by mini mode), gRPC clients connect via Unix socket
// instead of TCP for local services.
localGrpcSockets = make(map[int]string)
localGrpcSocketsLock sync.RWMutex
)
type versionedGrpcClient struct {
@@ -55,6 +63,59 @@ func init() {
http.DefaultTransport.(*http.Transport).MaxIdleConns = 1024
}
// RegisterLocalGrpcSocket registers a Unix socket path for a gRPC port.
// When a gRPC client dials an address on this port, it uses the Unix socket.
func RegisterLocalGrpcSocket(grpcPort int, socketPath string) {
localGrpcSocketsLock.Lock()
defer localGrpcSocketsLock.Unlock()
localGrpcSockets[grpcPort] = socketPath
}
// GetLocalGrpcSocket returns the Unix socket path for a gRPC port, or empty if not registered.
func GetLocalGrpcSocket(grpcPort int) string {
localGrpcSocketsLock.RLock()
defer localGrpcSocketsLock.RUnlock()
return localGrpcSockets[grpcPort]
}
// resolveLocalGrpcSocket extracts the port from a gRPC address and returns
// the registered Unix socket path, if any.
func resolveLocalGrpcSocket(address string) string {
_, portStr, err := net.SplitHostPort(address)
if err != nil {
return ""
}
port, err := strconv.Atoi(portStr)
if err != nil {
return ""
}
return GetLocalGrpcSocket(port)
}
// ServeGrpcOnLocalSocket starts serving a gRPC server on a Unix socket
// if one is registered for the given port.
func ServeGrpcOnLocalSocket(grpcServer *grpc.Server, grpcPort int) {
socketPath := GetLocalGrpcSocket(grpcPort)
if socketPath == "" {
return
}
if err := os.Remove(socketPath); err != nil && !os.IsNotExist(err) {
glog.Warningf("Failed to remove old gRPC socket %s: %v", socketPath, err)
}
listener, err := net.Listen("unix", socketPath)
if err != nil {
glog.Errorf("Failed to listen on gRPC Unix socket %s: %v", socketPath, err)
return
}
glog.V(0).Infof("gRPC also listening on Unix socket %s", socketPath)
go func() {
if err := grpcServer.Serve(listener); err != nil && err != grpc.ErrServerStopped {
glog.Errorf("gRPC Unix socket server error on %s: %v", socketPath, err)
}
os.Remove(socketPath)
}()
}
func NewGrpcServer(opts ...grpc.ServerOption) *grpc.Server {
var options []grpc.ServerOption
options = append(options,
@@ -97,6 +158,14 @@ func NewGrpcServer(opts ...grpc.ServerOption) *grpc.Server {
func GrpcDial(ctx context.Context, address string, waitForReady bool, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
var options []grpc.DialOption
// Route through Unix socket if one is registered for this address's port
if socketPath := resolveLocalGrpcSocket(address); socketPath != "" {
options = append(options, grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) {
var d net.Dialer
return d.DialContext(ctx, "unix", socketPath)
}))
}
options = append(options,
grpc.WithDefaultCallOptions(
grpc.MaxCallSendMsgSize(Max_Message_Size),