gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

before writing and after reading file chunks

Browse Source
This commit is contained in:
Chris Lu
2020-07-20 18:26:17 -07:00
parent 6debe3c3ec
commit aee6d89350
2 changed files with 61 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
other/java/client/src/main/java/seaweedfs/client/FileChunkManifest.java
View File

@@ -110,6 +110,7 @@ public class FileChunkManifest {
private static FilerProto.FileChunk mergeIntoManifest(final FilerGrpcClient filerGrpcClient, List<FilerProto.FileChunk> dataChunks) throws IOException { private static FilerProto.FileChunk mergeIntoManifest(final FilerGrpcClient filerGrpcClient, List<FilerProto.FileChunk> dataChunks) throws IOException {
// create and serialize the manifest // create and serialize the manifest
dataChunks = FilerClient.beforeEntrySerialization(dataChunks);
FilerProto.FileChunkManifest.Builder m = FilerProto.FileChunkManifest.newBuilder().addAllChunks(dataChunks); FilerProto.FileChunkManifest.Builder m = FilerProto.FileChunkManifest.newBuilder().addAllChunks(dataChunks);
byte[] data = m.build().toByteArray(); byte[] data = m.build().toByteArray();

81
other/java/client/src/main/java/seaweedfs/client/FilerClient.java
View File

@@ -25,7 +25,64 @@ public class FilerClient {
} }
public static String toFileId(FilerProto.FileId fid) { public static String toFileId(FilerProto.FileId fid) {
return String.format("%d,%d%x", fid.getVolumeId(), fid.getFileKey(), fid.getCookie()); if (fid == null) {
return null;
}
return String.format("%d,%x%08x", fid.getVolumeId(), fid.getFileKey(), fid.getCookie());
}
public static FilerProto.FileId toFileIdObject(String fileIdStr) {
if (fileIdStr == null || fileIdStr.length() == 0) {
return null;
}
int commaIndex = fileIdStr.lastIndexOf(',');
String volumeIdStr = fileIdStr.substring(0, commaIndex);
String fileKeyStr = fileIdStr.substring(commaIndex + 1, fileIdStr.length() - 8);
String cookieStr = fileIdStr.substring(fileIdStr.length() - 8);
return FilerProto.FileId.newBuilder()
.setVolumeId(Integer.parseInt(volumeIdStr))
.setFileKey(Long.parseLong(fileKeyStr, 16))
.setCookie((int) Long.parseLong(cookieStr, 16))
.build();
}
public static List<FilerProto.FileChunk> beforeEntrySerialization(List<FilerProto.FileChunk> chunks) {
List<FilerProto.FileChunk> cleanedChunks = new ArrayList<>();
for (FilerProto.FileChunk chunk : chunks) {
FilerProto.FileChunk.Builder chunkBuilder = chunk.toBuilder();
chunkBuilder.clearFileId();
chunkBuilder.clearSourceFileId();
chunkBuilder.setFid(toFileIdObject(chunk.getFileId()));
FilerProto.FileId sourceFid = toFileIdObject(chunk.getSourceFileId());
if (sourceFid != null) {
chunkBuilder.setSourceFid(sourceFid);
}
cleanedChunks.add(chunkBuilder.build());
}
return cleanedChunks;
}
public static FilerProto.Entry afterEntryDeserialization(FilerProto.Entry entry) {
if (entry.getChunksList().size() <= 0) {
return entry;
}
String fileId = entry.getChunks(0).getFileId();
if (fileId != null && fileId.length() != 0) {
return entry;
}
FilerProto.Entry.Builder entryBuilder = entry.toBuilder();
entryBuilder.clearChunks();
for (FilerProto.FileChunk chunk : entry.getChunksList()) {
FilerProto.FileChunk.Builder chunkBuilder = chunk.toBuilder();
chunkBuilder.setFileId(toFileId(chunk.getFid()));
String sourceFileId = toFileId(chunk.getSourceFid());
if (sourceFileId != null) {
chunkBuilder.setSourceFileId(sourceFileId);
}
entryBuilder.addChunks(chunkBuilder);
}
return entryBuilder.build();
} }
public boolean mkdirs(String path, int mode) { public boolean mkdirs(String path, int mode) {
@@ -188,7 +245,7 @@ public class FilerClient {
List<FilerProto.Entry> entries = new ArrayList<>(); List<FilerProto.Entry> entries = new ArrayList<>();
while (iter.hasNext()) { while (iter.hasNext()) {
FilerProto.ListEntriesResponse resp = iter.next(); FilerProto.ListEntriesResponse resp = iter.next();
entries.add(fixEntryAfterReading(resp.getEntry())); entries.add(afterEntryDeserialization(resp.getEntry()));
} }
return entries; return entries;
} }
@@ -203,7 +260,7 @@ public class FilerClient {
if (entry == null) { if (entry == null) {
return null; return null;
} }
return fixEntryAfterReading(entry); return afterEntryDeserialization(entry);
} catch (Exception e) { } catch (Exception e) {
if (e.getMessage().indexOf("filer: no entry is found in filer store") > 0) { if (e.getMessage().indexOf("filer: no entry is found in filer store") > 0) {
return null; return null;
@@ -270,22 +327,4 @@ public class FilerClient {
return true; return true;
} }
private FilerProto.Entry fixEntryAfterReading(FilerProto.Entry entry) {
if (entry.getChunksList().size() <= 0) {
return entry;
}
String fileId = entry.getChunks(0).getFileId();
if (fileId != null && fileId.length() != 0) {
return entry;
}
FilerProto.Entry.Builder entryBuilder = entry.toBuilder();
entryBuilder.clearChunks();
for (FilerProto.FileChunk chunk : entry.getChunksList()) {
FilerProto.FileChunk.Builder chunkBuilder = chunk.toBuilder();
chunkBuilder.setFileId(toFileId(chunk.getFid()));
entryBuilder.addChunks(chunkBuilder);
}
return entryBuilder.build();
}
} }