gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement RPC skeleton for regular/EC volumes scrubbing. (#8187)

Browse Source 
* Implement RPC skeleton for regular/EC volumes scrubbing.

See https://github.com/seaweedfs/seaweedfs/issues/8018 for details.

* Minor proto improvements for `ScrubVolume()`, `ScrubEcVolume()`:

  - Add fields for scrubbing details in `ScrubVolumeResponse` and `ScrubEcVolumeResponse`,
    instead of reporting these through RPC errors.
  - Return a list of broken shards when scrubbing EC volumes, via `EcShardInfo'.
This commit is contained in:
Lisandro Pin
2026-02-03 02:55:04 +01:00
committed by GitHub
parent 345ac950b6
commit ff5a8f0579
7 changed files with 950 additions and 296 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
weed/pb/volume_server.proto
View File

@@ -120,6 +120,12 @@ service VolumeServer {
rpc FetchAndWriteNeedle (FetchAndWriteNeedleRequest) returns (FetchAndWriteNeedleResponse) {
}
// scrubbing
rpc ScrubVolume (ScrubVolumeRequest) returns (ScrubVolumeResponse) {
}
rpc ScrubEcVolume (ScrubEcVolumeRequest) returns (ScrubEcVolumeResponse) {
}
// <experimental> query
rpc Query (QueryRequest) returns (stream QueriedStripe) {
}
@@ -496,6 +502,7 @@ message EcShardInfo {
uint32 shard_id = 1;
int64 size = 2;
string collection = 3;
uint32 volume_id = 4;
}
message ReadVolumeFileStatusRequest {
@@ -632,6 +639,37 @@ message FetchAndWriteNeedleResponse {
string e_tag = 1;
}
enum VolumeScrubMode {
UNKNOWN = 0;
INDEX = 1;
FULL = 2;
}
message ScrubVolumeRequest {
VolumeScrubMode mode = 1;
// optional list of volume IDs to scrub. if empty, all volumes for the server are scrubbed.
repeated uint32 volume_ids = 2;
}
message ScrubVolumeResponse {
uint64 total_volumes = 1;
uint64 total_files = 2;
repeated uint32 broken_volume_ids = 3;
repeated string details = 4;
}
message ScrubEcVolumeRequest {
VolumeScrubMode mode = 1;
// optional list of volume IDs to scrub. if empty, all EC volumes for the server are scrubbed.
repeated uint32 volume_ids = 2;
}
message ScrubEcVolumeResponse {
uint64 total_volumes = 1;
uint64 total_files = 2;
repeated uint32 broken_volume_ids = 3;
repeated EcShardInfo broken_shard_infos = 4;
repeated string details = 5;
}
// select on volume servers
message QueryRequest {
repeated string selections = 1;

938
weed/pb/volume_server_pb/volume_server.pb.go
View File

File diff suppressed because it is too large Load Diff

78
weed/pb/volume_server_pb/volume_server_grpc.pb.go
View File

@@ -62,6 +62,8 @@ const (
VolumeServer_VolumeServerStatus_FullMethodName = "/volume_server_pb.VolumeServer/VolumeServerStatus"
VolumeServer_VolumeServerLeave_FullMethodName = "/volume_server_pb.VolumeServer/VolumeServerLeave"
VolumeServer_FetchAndWriteNeedle_FullMethodName = "/volume_server_pb.VolumeServer/FetchAndWriteNeedle"
VolumeServer_ScrubVolume_FullMethodName = "/volume_server_pb.VolumeServer/ScrubVolume"
VolumeServer_ScrubEcVolume_FullMethodName = "/volume_server_pb.VolumeServer/ScrubEcVolume"
VolumeServer_Query_FullMethodName = "/volume_server_pb.VolumeServer/Query"
VolumeServer_VolumeNeedleStatus_FullMethodName = "/volume_server_pb.VolumeServer/VolumeNeedleStatus"
VolumeServer_Ping_FullMethodName = "/volume_server_pb.VolumeServer/Ping"
@@ -119,6 +121,9 @@ type VolumeServerClient interface {
VolumeServerLeave(ctx context.Context, in *VolumeServerLeaveRequest, opts ...grpc.CallOption) (*VolumeServerLeaveResponse, error)
// remote storage
FetchAndWriteNeedle(ctx context.Context, in *FetchAndWriteNeedleRequest, opts ...grpc.CallOption) (*FetchAndWriteNeedleResponse, error)
// scrubbing
ScrubVolume(ctx context.Context, in *ScrubVolumeRequest, opts ...grpc.CallOption) (*ScrubVolumeResponse, error)
ScrubEcVolume(ctx context.Context, in *ScrubEcVolumeRequest, opts ...grpc.CallOption) (*ScrubEcVolumeResponse, error)
// <experimental> query
Query(ctx context.Context, in *QueryRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[QueriedStripe], error)
VolumeNeedleStatus(ctx context.Context, in *VolumeNeedleStatusRequest, opts ...grpc.CallOption) (*VolumeNeedleStatusResponse, error)
@@ -647,6 +652,26 @@ func (c *volumeServerClient) FetchAndWriteNeedle(ctx context.Context, in *FetchA
return out, nil
}
func (c *volumeServerClient) ScrubVolume(ctx context.Context, in *ScrubVolumeRequest, opts ...grpc.CallOption) (*ScrubVolumeResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(ScrubVolumeResponse)
err := c.cc.Invoke(ctx, VolumeServer_ScrubVolume_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *volumeServerClient) ScrubEcVolume(ctx context.Context, in *ScrubEcVolumeRequest, opts ...grpc.CallOption) (*ScrubEcVolumeResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(ScrubEcVolumeResponse)
err := c.cc.Invoke(ctx, VolumeServer_ScrubEcVolume_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *volumeServerClient) Query(ctx context.Context, in *QueryRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[QueriedStripe], error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
stream, err := c.cc.NewStream(ctx, &VolumeServer_ServiceDesc.Streams[10], VolumeServer_Query_FullMethodName, cOpts...)
@@ -738,6 +763,9 @@ type VolumeServerServer interface {
VolumeServerLeave(context.Context, *VolumeServerLeaveRequest) (*VolumeServerLeaveResponse, error)
// remote storage
FetchAndWriteNeedle(context.Context, *FetchAndWriteNeedleRequest) (*FetchAndWriteNeedleResponse, error)
// scrubbing
ScrubVolume(context.Context, *ScrubVolumeRequest) (*ScrubVolumeResponse, error)
ScrubEcVolume(context.Context, *ScrubEcVolumeRequest) (*ScrubEcVolumeResponse, error)
// <experimental> query
Query(*QueryRequest, grpc.ServerStreamingServer[QueriedStripe]) error
VolumeNeedleStatus(context.Context, *VolumeNeedleStatusRequest) (*VolumeNeedleStatusResponse, error)
@@ -881,6 +909,12 @@ func (UnimplementedVolumeServerServer) VolumeServerLeave(context.Context, *Volum
func (UnimplementedVolumeServerServer) FetchAndWriteNeedle(context.Context, *FetchAndWriteNeedleRequest) (*FetchAndWriteNeedleResponse, error) {
return nil, status.Error(codes.Unimplemented, "method FetchAndWriteNeedle not implemented")
}
func (UnimplementedVolumeServerServer) ScrubVolume(context.Context, *ScrubVolumeRequest) (*ScrubVolumeResponse, error) {
return nil, status.Error(codes.Unimplemented, "method ScrubVolume not implemented")
}
func (UnimplementedVolumeServerServer) ScrubEcVolume(context.Context, *ScrubEcVolumeRequest) (*ScrubEcVolumeResponse, error) {
return nil, status.Error(codes.Unimplemented, "method ScrubEcVolume not implemented")
}
func (UnimplementedVolumeServerServer) Query(*QueryRequest, grpc.ServerStreamingServer[QueriedStripe]) error {
return status.Error(codes.Unimplemented, "method Query not implemented")
}
@@ -1611,6 +1645,42 @@ func _VolumeServer_FetchAndWriteNeedle_Handler(srv interface{}, ctx context.Cont
return interceptor(ctx, in, info, handler)
}
func _VolumeServer_ScrubVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ScrubVolumeRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(VolumeServerServer).ScrubVolume(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: VolumeServer_ScrubVolume_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(VolumeServerServer).ScrubVolume(ctx, req.(*ScrubVolumeRequest))
}
return interceptor(ctx, in, info, handler)
}
func _VolumeServer_ScrubEcVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ScrubEcVolumeRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(VolumeServerServer).ScrubEcVolume(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: VolumeServer_ScrubEcVolume_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(VolumeServerServer).ScrubEcVolume(ctx, req.(*ScrubEcVolumeRequest))
}
return interceptor(ctx, in, info, handler)
}
func _VolumeServer_Query_Handler(srv interface{}, stream grpc.ServerStream) error {
m := new(QueryRequest)
if err := stream.RecvMsg(m); err != nil {
@@ -1797,6 +1867,14 @@ var VolumeServer_ServiceDesc = grpc.ServiceDesc{
MethodName: "FetchAndWriteNeedle",
Handler: _VolumeServer_FetchAndWriteNeedle_Handler,
},
{
MethodName: "ScrubVolume",
Handler: _VolumeServer_ScrubVolume_Handler,
},
{
MethodName: "ScrubEcVolume",
Handler: _VolumeServer_ScrubEcVolume_Handler,
},
{
MethodName: "VolumeNeedleStatus",
Handler: _VolumeServer_VolumeNeedleStatus_Handler,