gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

filer: cache small file to filer store

Browse Source
This commit is contained in:
Chris Lu
2020-11-30 04:34:04 -08:00
parent a9c6be5fc3
commit f4abd01adf
17 changed files with 531 additions and 472 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
other/java/client/src/main/proto/filer.proto
View File

@@ -100,6 +100,7 @@ message Entry {
map<string, bytes> extended = 5;
bytes hard_link_id = 7;
int32 hard_link_counter = 8; // only exists in hard link meta data
bytes content = 9; // if not empty, the file content
}
message FullEntry {

1
other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java
View File

@@ -193,6 +193,7 @@ public class SeaweedFileSystemStore {
if (existingEntry != null) {
entry = FilerProto.Entry.newBuilder();
entry.mergeFrom(existingEntry);
entry.clearContent();
entry.getAttributesBuilder().setMtime(now);
LOG.debug("createFile merged entry path:{} entry:{} from:{}", path, entry, existingEntry);
writePosition = SeaweedRead.fileSize(existingEntry);

8
other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedInputStream.java
View File

@@ -84,7 +84,13 @@ public class SeaweedInputStream extends FSInputStream {
throw new IllegalArgumentException("requested read length is more than will fit after requested offset in buffer");
}
long bytesRead = SeaweedRead.read(this.filerGrpcClient, this.visibleIntervalList, this.position, b, off, len, SeaweedRead.fileSize(entry));
long bytesRead = 0;
if (position+len < entry.getContent().size()) {
entry.getContent().copyTo(b, (int) position, (int) off, len);
} else {
bytesRead = SeaweedRead.read(this.filerGrpcClient, this.visibleIntervalList, this.position, b, off, len, SeaweedRead.fileSize(entry));
}
if (bytesRead > Integer.MAX_VALUE) {
throw new IOException("Unexpected Content-Length");
}

1
other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java
View File

@@ -193,6 +193,7 @@ public class SeaweedFileSystemStore {
if (existingEntry != null) {
entry = FilerProto.Entry.newBuilder();
entry.mergeFrom(existingEntry);
entry.clearContent();
entry.getAttributesBuilder().setMtime(now);
LOG.debug("createFile merged entry path:{} entry:{} from:{}", path, entry, existingEntry);
writePosition = SeaweedRead.fileSize(existingEntry);

8
other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedInputStream.java
View File

@@ -84,7 +84,13 @@ public class SeaweedInputStream extends FSInputStream {
throw new IllegalArgumentException("requested read length is more than will fit after requested offset in buffer");
}
long bytesRead = SeaweedRead.read(this.filerGrpcClient, this.visibleIntervalList, this.position, b, off, len, SeaweedRead.fileSize(entry));
long bytesRead = 0;
if (position+len < entry.getContent().size()) {
entry.getContent().copyTo(b, (int) position, (int) off, len);
} else {
bytesRead = SeaweedRead.read(this.filerGrpcClient, this.visibleIntervalList, this.position, b, off, len, SeaweedRead.fileSize(entry));
}
if (bytesRead > Integer.MAX_VALUE) {
throw new IOException("Unexpected Content-Length");
}