seaweedFS/weed/pb/filer.proto

syntax = "proto3";

package filer_pb;

option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb";
option java_package = "seaweedfs.client";
option java_outer_classname = "FilerProto";

//////////////////////////////////////////////////

service SeaweedFiler {

    rpc LookupDirectoryEntry (LookupDirectoryEntryRequest) returns (LookupDirectoryEntryResponse) {
    }

    rpc ListEntries (ListEntriesRequest) returns (stream ListEntriesResponse) {
    }

    rpc CreateEntry (CreateEntryRequest) returns (CreateEntryResponse) {
    }

    rpc UpdateEntry (UpdateEntryRequest) returns (UpdateEntryResponse) {
    }

    rpc AppendToEntry (AppendToEntryRequest) returns (AppendToEntryResponse) {
    }

    rpc DeleteEntry (DeleteEntryRequest) returns (DeleteEntryResponse) {
    }

    rpc AtomicRenameEntry (AtomicRenameEntryRequest) returns (AtomicRenameEntryResponse) {
    }
    rpc StreamRenameEntry (StreamRenameEntryRequest) returns (stream StreamRenameEntryResponse) {
    }

    rpc AssignVolume (AssignVolumeRequest) returns (AssignVolumeResponse) {
    }

    rpc LookupVolume (LookupVolumeRequest) returns (LookupVolumeResponse) {
    }

    rpc CollectionList (CollectionListRequest) returns (CollectionListResponse) {
    }

    rpc DeleteCollection (DeleteCollectionRequest) returns (DeleteCollectionResponse) {
    }

    rpc Statistics (StatisticsRequest) returns (StatisticsResponse) {
    }

    rpc Ping (PingRequest) returns (PingResponse) {
    }

    rpc GetFilerConfiguration (GetFilerConfigurationRequest) returns (GetFilerConfigurationResponse) {
    }

    rpc TraverseBfsMetadata (TraverseBfsMetadataRequest) returns (stream TraverseBfsMetadataResponse) {
    }

    rpc SubscribeMetadata (SubscribeMetadataRequest) returns (stream SubscribeMetadataResponse) {
    }

    rpc SubscribeLocalMetadata (SubscribeMetadataRequest) returns (stream SubscribeMetadataResponse) {
    }

    rpc KvGet (KvGetRequest) returns (KvGetResponse) {
    }

    rpc KvPut (KvPutRequest) returns (KvPutResponse) {
    }

    rpc CacheRemoteObjectToLocalCluster (CacheRemoteObjectToLocalClusterRequest) returns (CacheRemoteObjectToLocalClusterResponse) {
    }

    rpc DistributedLock(LockRequest) returns (LockResponse) {
    }
    rpc DistributedUnlock(UnlockRequest) returns (UnlockResponse) {
    }
    rpc FindLockOwner(FindLockOwnerRequest) returns (FindLockOwnerResponse) {
    }
    // distributed lock management internal use only
    rpc TransferLocks(TransferLocksRequest) returns (TransferLocksResponse) {
    }
}

//////////////////////////////////////////////////

message LookupDirectoryEntryRequest {
    string directory = 1;
    string name = 2;
}

message LookupDirectoryEntryResponse {
    Entry entry = 1;
}

message ListEntriesRequest {
    string directory = 1;
    string prefix = 2;
    string startFromFileName = 3;
    bool inclusiveStartFrom = 4;
    uint32 limit = 5;
    int64 snapshot_ts_ns = 6;
}

message ListEntriesResponse {
    Entry entry = 1;
    int64 snapshot_ts_ns = 2;
}

message RemoteEntry {
    string storage_name = 1;
    int64 last_local_sync_ts_ns = 2;
    string remote_e_tag = 3;
    int64 remote_mtime = 4;
    int64 remote_size = 5;
}
message Entry {
    string name = 1;
    bool is_directory = 2;
    repeated FileChunk chunks = 3;
    FuseAttributes attributes = 4;
    map<string, bytes> extended = 5;
    bytes hard_link_id = 7;
    int32 hard_link_counter = 8; // only exists in hard link meta data
    bytes content = 9; // if not empty, the file content

    RemoteEntry remote_entry = 10;
    int64 quota = 11; // for bucket only. Positive/Negative means enabled/disabled.
    int64 worm_enforced_at_ts_ns = 12;
}

message FullEntry {
    string dir = 1;
    Entry entry = 2;
}

message EventNotification {
    Entry old_entry = 1;
    Entry new_entry = 2;
    bool delete_chunks = 3;
    string new_parent_path = 4;
    bool is_from_other_cluster = 5;
    repeated int32 signatures = 6;
}

enum SSEType {
    NONE = 0;    // No server-side encryption
    SSE_C = 1;   // Server-Side Encryption with Customer-Provided Keys  
    SSE_KMS = 2; // Server-Side Encryption with KMS-Managed Keys
    SSE_S3 = 3;  // Server-Side Encryption with S3-Managed Keys
}

message FileChunk {
    string file_id = 1; // to be deprecated
    int64 offset = 2;
    uint64 size = 3;
    int64 modified_ts_ns = 4;
    string e_tag = 5;
    string source_file_id = 6; // to be deprecated
    FileId fid = 7;
    FileId source_fid = 8;
    bytes cipher_key = 9;
    bool is_compressed = 10;
    bool is_chunk_manifest = 11; // content is a list of FileChunks
    SSEType sse_type = 12;           // Server-side encryption type
    bytes sse_metadata = 13;         // Serialized SSE metadata for this chunk (SSE-C, SSE-KMS, or SSE-S3)
}

message FileChunkManifest {
    repeated FileChunk chunks = 1;
}

message FileId {
    uint32 volume_id = 1;
    uint64 file_key = 2;
    fixed32 cookie = 3;
}

message FuseAttributes {
    uint64 file_size = 1;
    int64 mtime = 2; // unix time in seconds
    uint32 file_mode = 3;
    uint32 uid = 4;
    uint32 gid = 5;
    int64 crtime = 6; // unix time in seconds
    string mime = 7;
    int32 ttl_sec = 10;
    string user_name = 11; // for hdfs
    repeated string group_name = 12; // for hdfs
    string symlink_target = 13;
    bytes md5 = 14;
    uint32 rdev = 16;
    uint64 inode = 17;
}

message CreateEntryRequest {
    string directory = 1;
    Entry entry = 2;
    bool o_excl = 3;
    bool is_from_other_cluster = 4;
    repeated int32 signatures = 5;
    bool skip_check_parent_directory = 6;
}

message CreateEntryResponse {
    string error = 1;
    SubscribeMetadataResponse metadata_event = 2;
}

message UpdateEntryRequest {
    string directory = 1;
    Entry entry = 2;
    bool is_from_other_cluster = 3;
    repeated int32 signatures = 4;
}
message UpdateEntryResponse {
    SubscribeMetadataResponse metadata_event = 1;
}

message AppendToEntryRequest {
    string directory = 1;
    string entry_name = 2;
    repeated FileChunk chunks = 3;
}
message AppendToEntryResponse {
}

message DeleteEntryRequest {
    string directory = 1;
    string name = 2;
    // bool is_directory = 3;
    bool is_delete_data = 4;
    bool is_recursive = 5;
    bool ignore_recursive_error = 6;
    bool is_from_other_cluster = 7;
    repeated int32 signatures = 8;
    int64 if_not_modified_after = 9;
}

message DeleteEntryResponse {
    string error = 1;
    SubscribeMetadataResponse metadata_event = 2;
}

message AtomicRenameEntryRequest {
    string old_directory = 1;
    string old_name = 2;
    string new_directory = 3;
    string new_name = 4;
    repeated int32 signatures = 5;
}

message AtomicRenameEntryResponse {
}

message StreamRenameEntryRequest {
    string old_directory = 1;
    string old_name = 2;
    string new_directory = 3;
    string new_name = 4;
    repeated int32 signatures = 5;
}
message StreamRenameEntryResponse {
    string directory = 1;
    EventNotification event_notification = 2;
    int64 ts_ns = 3;
}
message AssignVolumeRequest {
    int32 count = 1;
    string collection = 2;
    string replication = 3;
    int32 ttl_sec = 4;
    string data_center = 5;
    string path = 6;
    string rack = 7;
    string data_node = 9;
    string disk_type = 8;
}

message AssignVolumeResponse {
    string file_id = 1;
    int32 count = 4;
    string auth = 5;
    string collection = 6;
    string replication = 7;
    string error = 8;
    Location location = 9;
}

message LookupVolumeRequest {
    repeated string volume_ids = 1;
}

message Locations {
    repeated Location locations = 1;
}

message Location {
    string url = 1;
    string public_url = 2;
    uint32 grpc_port = 3;
    string data_center = 4;
}
message LookupVolumeResponse {
    map<string, Locations> locations_map = 1;
}

message Collection {
    string name = 1;
}
message CollectionListRequest {
    bool include_normal_volumes = 1;
    bool include_ec_volumes = 2;
}
message CollectionListResponse {
    repeated Collection collections = 1;
}
message DeleteCollectionRequest {
    string collection = 1;
}

message DeleteCollectionResponse {
}

message StatisticsRequest {
    string replication = 1;
    string collection = 2;
    string ttl = 3;
    string disk_type = 4;
}
message StatisticsResponse {
    uint64 total_size = 4;
    uint64 used_size = 5;
    uint64 file_count = 6;
}

message PingRequest {
    string target = 1; // default to ping itself
    string target_type = 2;
}
message PingResponse {
    int64 start_time_ns = 1;
    int64 remote_time_ns = 2;
    int64 stop_time_ns = 3;
}

message GetFilerConfigurationRequest {
}
message GetFilerConfigurationResponse {
    repeated string masters = 1;
    string replication = 2;
    string collection = 3;
    uint32 max_mb = 4;
    string dir_buckets = 5;
    bool cipher = 7;
    int32 signature = 8;
    string metrics_address = 9;
    int32 metrics_interval_sec = 10;
    string version = 11;
    string cluster_id = 12;
    string filer_group = 13;
    int32 major_version = 14;
    int32 minor_version = 15;
}

message SubscribeMetadataRequest {
    string client_name = 1;
    string path_prefix = 2;
    int64 since_ns = 3;
    int32 signature = 4;
    repeated string path_prefixes = 6;
    int32 client_id = 7;
    int64 until_ns = 8;
    int32 client_epoch = 9;
    repeated string directories = 10;  // exact directory to watch
}
message SubscribeMetadataResponse {
    string directory = 1;
    EventNotification event_notification = 2;
    int64 ts_ns = 3;
}

message TraverseBfsMetadataRequest {
    string directory = 1;
    repeated string excluded_prefixes = 2;
}
message TraverseBfsMetadataResponse {
    string directory = 1;
    Entry entry = 2;
}

message LogEntry {
    int64 ts_ns = 1;
    int32 partition_key_hash = 2;
    bytes data = 3;
    bytes key = 4;
    int64 offset = 5;  // Sequential offset within partition
}

message KeepConnectedRequest {
    string name = 1;
    uint32 grpc_port = 2;
    repeated string resources = 3;
}
message KeepConnectedResponse {
}

message LocateBrokerRequest {
    string resource = 1;
}

message LocateBrokerResponse {
    bool found = 1;
    // if found, send the exact address
    // if not found, send the full list of existing brokers
    message Resource {
        string grpc_addresses = 1;
        int32 resource_count = 2;
    }
    repeated Resource resources = 2;
}

/////////////////////////
// Key-Value operations
/////////////////////////
message KvGetRequest {
    bytes key = 1;
}
message KvGetResponse {
    bytes value = 1;
    string error = 2;
}
message KvPutRequest {
    bytes key = 1;
    bytes value = 2;
}
message KvPutResponse {
    string error = 1;
}

/////////////////////////
// path-based configurations
/////////////////////////
message FilerConf {
    int32 version = 1;
    message PathConf {
        string location_prefix = 1;
        string collection = 2;
        string replication = 3;
        string ttl = 4;
        string disk_type = 5;
        bool fsync = 6;
        uint32 volume_growth_count = 7;
        bool read_only = 8;
        string data_center = 9;
        string rack = 10;
        string data_node = 11;
        uint32 max_file_name_length = 12;
        bool disable_chunk_deletion = 13;
        bool worm = 14;
        uint64 worm_grace_period_seconds = 15;
        uint64 worm_retention_time_seconds = 16;
    }
    repeated PathConf locations = 2;
}

/////////////////////////
// Remote Storage related
/////////////////////////
message CacheRemoteObjectToLocalClusterRequest {
    string directory = 1;
    string name = 2;
}
message CacheRemoteObjectToLocalClusterResponse {
    Entry entry = 1;
    SubscribeMetadataResponse metadata_event = 2;
}

/////////////////////////
// distributed lock management
/////////////////////////
message LockRequest {
    string name = 1;
    int64 seconds_to_lock = 2;
    string renew_token = 3;
    bool is_moved = 4;
    string owner = 5;
}
message LockResponse {
    string renew_token = 1;
    string lock_owner = 2;
    string lock_host_moved_to = 3;
    string error = 4;
}
message UnlockRequest {
    string name = 1;
    string renew_token = 2;
    bool is_moved = 3;
}
message UnlockResponse {
    string error = 1;
    string moved_to = 2;
}
message FindLockOwnerRequest {
    string name = 1;
    bool is_moved = 2;
}
message FindLockOwnerResponse {
    string owner = 1;
}
message Lock {
    string name = 1;
    string renew_token = 2;
    int64 expired_at_ns = 3;
    string owner = 4;
}
message TransferLocksRequest {
    repeated Lock locks = 1;
}
message TransferLocksResponse {
}