chore(deps): bump org.apache.hadoop:hadoop-common from 3.2.4 to 3.4.0 in /other/java/hdfs3 (#7512)
* chore(deps): bump org.apache.hadoop:hadoop-common in /other/java/hdfs3 Bumps org.apache.hadoop:hadoop-common from 3.2.4 to 3.4.0. --- updated-dependencies: - dependency-name: org.apache.hadoop:hadoop-common dependency-version: 3.4.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] <support@github.com> * add java client unit tests * Update dependency-reduced-pom.xml * add java integration tests * fix * fix buffer --------- Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: chrislu <chris.lu@gmail.com>
This commit is contained in:
dependabot[bot]
170
.github/workflows/java_integration_tests.yml
vendored
Normal file
170
.github/workflows/java_integration_tests.yml
vendored
Normal file
@@ -0,0 +1,170 @@
|
|||||||
|
name: Java Client Integration Tests
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [ master ]
|
||||||
|
paths:
|
||||||
|
- 'other/java/**'
|
||||||
|
- 'weed/**'
|
||||||
|
- '.github/workflows/java_integration_tests.yml'
|
||||||
|
pull_request:
|
||||||
|
branches: [ master ]
|
||||||
|
paths:
|
||||||
|
- 'other/java/**'
|
||||||
|
- 'weed/**'
|
||||||
|
- '.github/workflows/java_integration_tests.yml'
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
test:
|
||||||
|
name: Java Integration Tests
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
|
||||||
|
strategy:
|
||||||
|
matrix:
|
||||||
|
java: ['11', '17']
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Checkout code
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Set up Go
|
||||||
|
uses: actions/setup-go@v6
|
||||||
|
with:
|
||||||
|
go-version-file: 'go.mod'
|
||||||
|
id: go
|
||||||
|
|
||||||
|
- name: Set up Java
|
||||||
|
uses: actions/setup-java@v4
|
||||||
|
with:
|
||||||
|
java-version: ${{ matrix.java }}
|
||||||
|
distribution: 'temurin'
|
||||||
|
cache: 'maven'
|
||||||
|
|
||||||
|
- name: Build SeaweedFS
|
||||||
|
run: |
|
||||||
|
cd weed
|
||||||
|
go install -buildvcs=false
|
||||||
|
weed version
|
||||||
|
|
||||||
|
- name: Start SeaweedFS Server
|
||||||
|
run: |
|
||||||
|
# Create clean data directory
|
||||||
|
export WEED_DATA_DIR="/tmp/seaweedfs-java-tests-$(date +%s)"
|
||||||
|
mkdir -p "$WEED_DATA_DIR"
|
||||||
|
|
||||||
|
# Start SeaweedFS with optimized settings for CI
|
||||||
|
weed server -dir="$WEED_DATA_DIR" \
|
||||||
|
-master.raftHashicorp \
|
||||||
|
-master.electionTimeout=1s \
|
||||||
|
-master.volumeSizeLimitMB=100 \
|
||||||
|
-volume.max=100 \
|
||||||
|
-volume.preStopSeconds=1 \
|
||||||
|
-master.peers=none \
|
||||||
|
-filer -filer.maxMB=64 \
|
||||||
|
-master.port=9333 \
|
||||||
|
-volume.port=8080 \
|
||||||
|
-filer.port=8888 \
|
||||||
|
-metricsPort=9324 > seaweedfs.log 2>&1 &
|
||||||
|
|
||||||
|
SERVER_PID=$!
|
||||||
|
echo "SERVER_PID=$SERVER_PID" >> $GITHUB_ENV
|
||||||
|
echo "WEED_DATA_DIR=$WEED_DATA_DIR" >> $GITHUB_ENV
|
||||||
|
echo "SeaweedFS server started with PID: $SERVER_PID"
|
||||||
|
|
||||||
|
- name: Wait for SeaweedFS Components
|
||||||
|
run: |
|
||||||
|
echo "Waiting for SeaweedFS components to start..."
|
||||||
|
|
||||||
|
# Wait for master
|
||||||
|
for i in {1..30}; do
|
||||||
|
if curl -s http://localhost:9333/cluster/status > /dev/null 2>&1; then
|
||||||
|
echo "✓ Master server is ready"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
echo "Waiting for master server... ($i/30)"
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
|
|
||||||
|
# Wait for volume
|
||||||
|
for i in {1..30}; do
|
||||||
|
if curl -s http://localhost:8080/status > /dev/null 2>&1; then
|
||||||
|
echo "✓ Volume server is ready"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
echo "Waiting for volume server... ($i/30)"
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
|
|
||||||
|
# Wait for filer
|
||||||
|
for i in {1..30}; do
|
||||||
|
if curl -s http://localhost:8888/ > /dev/null 2>&1; then
|
||||||
|
echo "✓ Filer is ready"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
echo "Waiting for filer... ($i/30)"
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "✓ All SeaweedFS components are ready!"
|
||||||
|
|
||||||
|
# Display cluster status
|
||||||
|
echo "Cluster status:"
|
||||||
|
curl -s http://localhost:9333/cluster/status | head -20
|
||||||
|
|
||||||
|
- name: Build and Install SeaweedFS Client
|
||||||
|
working-directory: other/java/client
|
||||||
|
run: |
|
||||||
|
mvn clean install -DskipTests -Dmaven.javadoc.skip=true -Dgpg.skip=true
|
||||||
|
|
||||||
|
- name: Run Client Unit Tests
|
||||||
|
working-directory: other/java/client
|
||||||
|
run: |
|
||||||
|
mvn test -Dtest=SeaweedReadTest,SeaweedCipherTest
|
||||||
|
|
||||||
|
- name: Run Client Integration Tests
|
||||||
|
working-directory: other/java/client
|
||||||
|
env:
|
||||||
|
SEAWEEDFS_TEST_ENABLED: true
|
||||||
|
run: |
|
||||||
|
mvn test -Dtest=*IntegrationTest
|
||||||
|
|
||||||
|
- name: Run HDFS2 Configuration Tests
|
||||||
|
working-directory: other/java/hdfs2
|
||||||
|
run: |
|
||||||
|
mvn test -Dtest=SeaweedFileSystemConfigTest -Dmaven.javadoc.skip=true -Dgpg.skip=true
|
||||||
|
|
||||||
|
- name: Run HDFS3 Configuration Tests
|
||||||
|
working-directory: other/java/hdfs3
|
||||||
|
run: |
|
||||||
|
mvn test -Dtest=SeaweedFileSystemConfigTest -Dmaven.javadoc.skip=true -Dgpg.skip=true
|
||||||
|
|
||||||
|
- name: Display logs on failure
|
||||||
|
if: failure()
|
||||||
|
run: |
|
||||||
|
echo "=== SeaweedFS Server Log ==="
|
||||||
|
tail -100 seaweedfs.log || echo "No server log"
|
||||||
|
echo ""
|
||||||
|
echo "=== Cluster Status ==="
|
||||||
|
curl -s http://localhost:9333/cluster/status || echo "Cannot reach cluster"
|
||||||
|
echo ""
|
||||||
|
echo "=== Process Status ==="
|
||||||
|
ps aux | grep weed || echo "No weed processes"
|
||||||
|
|
||||||
|
- name: Cleanup
|
||||||
|
if: always()
|
||||||
|
run: |
|
||||||
|
# Stop server using stored PID
|
||||||
|
if [ -n "$SERVER_PID" ]; then
|
||||||
|
echo "Stopping SeaweedFS server (PID: $SERVER_PID)"
|
||||||
|
kill -9 $SERVER_PID 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Fallback: kill any remaining weed processes
|
||||||
|
pkill -f "weed server" || true
|
||||||
|
|
||||||
|
# Clean up data directory
|
||||||
|
if [ -n "$WEED_DATA_DIR" ]; then
|
||||||
|
echo "Cleaning up data directory: $WEED_DATA_DIR"
|
||||||
|
rm -rf "$WEED_DATA_DIR" || true
|
||||||
|
fi
|
||||||
|
|
||||||
64
.github/workflows/java_unit_tests.yml
vendored
Normal file
64
.github/workflows/java_unit_tests.yml
vendored
Normal file
@@ -0,0 +1,64 @@
|
|||||||
|
name: Java Client Unit Tests
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [ master ]
|
||||||
|
paths:
|
||||||
|
- 'other/java/**'
|
||||||
|
- '.github/workflows/java_unit_tests.yml'
|
||||||
|
pull_request:
|
||||||
|
branches: [ master ]
|
||||||
|
paths:
|
||||||
|
- 'other/java/**'
|
||||||
|
- '.github/workflows/java_unit_tests.yml'
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
test:
|
||||||
|
name: Java Unit Tests
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
|
||||||
|
strategy:
|
||||||
|
matrix:
|
||||||
|
java: ['8', '11', '17', '21']
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Checkout code
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Set up Java
|
||||||
|
uses: actions/setup-java@v4
|
||||||
|
with:
|
||||||
|
java-version: ${{ matrix.java }}
|
||||||
|
distribution: 'temurin'
|
||||||
|
cache: 'maven'
|
||||||
|
|
||||||
|
- name: Build and Install SeaweedFS Client
|
||||||
|
working-directory: other/java/client
|
||||||
|
run: |
|
||||||
|
mvn clean install -DskipTests -Dmaven.javadoc.skip=true -Dgpg.skip=true
|
||||||
|
|
||||||
|
- name: Run Client Unit Tests
|
||||||
|
working-directory: other/java/client
|
||||||
|
run: |
|
||||||
|
mvn test -Dtest=SeaweedReadTest,SeaweedCipherTest
|
||||||
|
|
||||||
|
- name: Run HDFS2 Configuration Tests
|
||||||
|
working-directory: other/java/hdfs2
|
||||||
|
run: |
|
||||||
|
mvn test -Dtest=SeaweedFileSystemConfigTest -Dmaven.javadoc.skip=true -Dgpg.skip=true
|
||||||
|
|
||||||
|
- name: Run HDFS3 Configuration Tests
|
||||||
|
working-directory: other/java/hdfs3
|
||||||
|
run: |
|
||||||
|
mvn test -Dtest=SeaweedFileSystemConfigTest -Dmaven.javadoc.skip=true -Dgpg.skip=true
|
||||||
|
|
||||||
|
- name: Upload Test Reports
|
||||||
|
if: always()
|
||||||
|
uses: actions/upload-artifact@v5
|
||||||
|
with:
|
||||||
|
name: test-reports-java-${{ matrix.java }}
|
||||||
|
path: |
|
||||||
|
other/java/client/target/surefire-reports/
|
||||||
|
other/java/hdfs2/target/surefire-reports/
|
||||||
|
other/java/hdfs3/target/surefire-reports/
|
||||||
|
|
||||||
@@ -0,0 +1,323 @@
|
|||||||
|
package seaweedfs.client;
|
||||||
|
|
||||||
|
import org.junit.After;
|
||||||
|
import org.junit.Before;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import static org.junit.Assert.*;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Integration tests for FilerClient.
|
||||||
|
*
|
||||||
|
* These tests verify FilerClient operations against a running SeaweedFS filer
|
||||||
|
* instance.
|
||||||
|
*
|
||||||
|
* Prerequisites:
|
||||||
|
* - SeaweedFS master, volume server, and filer must be running
|
||||||
|
* - Default ports: filer HTTP 8888, filer gRPC 18888
|
||||||
|
*
|
||||||
|
* To run tests:
|
||||||
|
* export SEAWEEDFS_TEST_ENABLED=true
|
||||||
|
* mvn test -Dtest=FilerClientIntegrationTest
|
||||||
|
*/
|
||||||
|
public class FilerClientIntegrationTest {
|
||||||
|
|
||||||
|
private FilerClient filerClient;
|
||||||
|
private static final String TEST_ROOT = "/test-client-integration";
|
||||||
|
private static final boolean TESTS_ENABLED = "true".equalsIgnoreCase(System.getenv("SEAWEEDFS_TEST_ENABLED"));
|
||||||
|
|
||||||
|
@Before
|
||||||
|
public void setUp() throws Exception {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
filerClient = new FilerClient("localhost", 18888);
|
||||||
|
|
||||||
|
// Clean up any existing test directory
|
||||||
|
if (filerClient.exists(TEST_ROOT)) {
|
||||||
|
filerClient.rm(TEST_ROOT, true, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create test root directory
|
||||||
|
filerClient.mkdirs(TEST_ROOT, 0755);
|
||||||
|
}
|
||||||
|
|
||||||
|
@After
|
||||||
|
public void tearDown() throws Exception {
|
||||||
|
if (!TESTS_ENABLED || filerClient == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Clean up test directory
|
||||||
|
if (filerClient.exists(TEST_ROOT)) {
|
||||||
|
filerClient.rm(TEST_ROOT, true, true);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
filerClient.shutdown();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testMkdirs() {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
String testDir = TEST_ROOT + "/testdir";
|
||||||
|
boolean success = filerClient.mkdirs(testDir, 0755);
|
||||||
|
|
||||||
|
assertTrue("Directory creation should succeed", success);
|
||||||
|
assertTrue("Directory should exist", filerClient.exists(testDir));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testTouch() {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
String testFile = TEST_ROOT + "/testfile.txt";
|
||||||
|
boolean success = filerClient.touch(testFile, 0644);
|
||||||
|
|
||||||
|
assertTrue("Touch should succeed", success);
|
||||||
|
assertTrue("File should exist", filerClient.exists(testFile));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testExists() {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
assertTrue("Root should exist", filerClient.exists("/"));
|
||||||
|
assertTrue("Test root should exist", filerClient.exists(TEST_ROOT));
|
||||||
|
assertFalse("Non-existent path should not exist",
|
||||||
|
filerClient.exists(TEST_ROOT + "/nonexistent"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testListEntries() {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create some test files and directories
|
||||||
|
filerClient.touch(TEST_ROOT + "/file1.txt", 0644);
|
||||||
|
filerClient.touch(TEST_ROOT + "/file2.txt", 0644);
|
||||||
|
filerClient.mkdirs(TEST_ROOT + "/subdir", 0755);
|
||||||
|
|
||||||
|
List<FilerProto.Entry> entries = filerClient.listEntries(TEST_ROOT);
|
||||||
|
|
||||||
|
assertNotNull("Entries should not be null", entries);
|
||||||
|
assertEquals("Should have 3 entries", 3, entries.size());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testListEntriesWithPrefix() {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create test files
|
||||||
|
filerClient.touch(TEST_ROOT + "/test1.txt", 0644);
|
||||||
|
filerClient.touch(TEST_ROOT + "/test2.txt", 0644);
|
||||||
|
filerClient.touch(TEST_ROOT + "/other.txt", 0644);
|
||||||
|
|
||||||
|
List<FilerProto.Entry> entries = filerClient.listEntries(TEST_ROOT, "test", "", 100, false);
|
||||||
|
|
||||||
|
assertNotNull("Entries should not be null", entries);
|
||||||
|
assertEquals("Should have 2 entries starting with 'test'", 2, entries.size());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testDeleteFile() {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
String testFile = TEST_ROOT + "/deleteme.txt";
|
||||||
|
filerClient.touch(testFile, 0644);
|
||||||
|
|
||||||
|
assertTrue("File should exist before delete", filerClient.exists(testFile));
|
||||||
|
|
||||||
|
boolean success = filerClient.rm(testFile, false, true);
|
||||||
|
|
||||||
|
assertTrue("Delete should succeed", success);
|
||||||
|
assertFalse("File should not exist after delete", filerClient.exists(testFile));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testDeleteDirectoryRecursive() {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
String testDir = TEST_ROOT + "/deletedir";
|
||||||
|
filerClient.mkdirs(testDir, 0755);
|
||||||
|
filerClient.touch(testDir + "/file.txt", 0644);
|
||||||
|
|
||||||
|
assertTrue("Directory should exist", filerClient.exists(testDir));
|
||||||
|
assertTrue("File should exist", filerClient.exists(testDir + "/file.txt"));
|
||||||
|
|
||||||
|
boolean success = filerClient.rm(testDir, true, true);
|
||||||
|
|
||||||
|
assertTrue("Delete should succeed", success);
|
||||||
|
assertFalse("Directory should not exist after delete", filerClient.exists(testDir));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testRename() {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
String srcFile = TEST_ROOT + "/source.txt";
|
||||||
|
String dstFile = TEST_ROOT + "/destination.txt";
|
||||||
|
|
||||||
|
filerClient.touch(srcFile, 0644);
|
||||||
|
assertTrue("Source file should exist", filerClient.exists(srcFile));
|
||||||
|
|
||||||
|
boolean success = filerClient.mv(srcFile, dstFile);
|
||||||
|
|
||||||
|
assertTrue("Rename should succeed", success);
|
||||||
|
assertFalse("Source file should not exist after rename", filerClient.exists(srcFile));
|
||||||
|
assertTrue("Destination file should exist after rename", filerClient.exists(dstFile));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGetEntry() {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
String testFile = TEST_ROOT + "/getentry.txt";
|
||||||
|
filerClient.touch(testFile, 0644);
|
||||||
|
|
||||||
|
FilerProto.Entry entry = filerClient.lookupEntry(TEST_ROOT, "getentry.txt");
|
||||||
|
|
||||||
|
assertNotNull("Entry should not be null", entry);
|
||||||
|
assertEquals("Entry name should match", "getentry.txt", entry.getName());
|
||||||
|
assertFalse("Entry should not be a directory", entry.getIsDirectory());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGetEntryForDirectory() {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
String testDir = TEST_ROOT + "/testsubdir";
|
||||||
|
filerClient.mkdirs(testDir, 0755);
|
||||||
|
|
||||||
|
FilerProto.Entry entry = filerClient.lookupEntry(TEST_ROOT, "testsubdir");
|
||||||
|
|
||||||
|
assertNotNull("Entry should not be null", entry);
|
||||||
|
assertEquals("Entry name should match", "testsubdir", entry.getName());
|
||||||
|
assertTrue("Entry should be a directory", entry.getIsDirectory());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testCreateAndListNestedDirectories() {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
String nestedPath = TEST_ROOT + "/level1/level2/level3";
|
||||||
|
boolean success = filerClient.mkdirs(nestedPath, 0755);
|
||||||
|
|
||||||
|
assertTrue("Nested directory creation should succeed", success);
|
||||||
|
assertTrue("Nested directory should exist", filerClient.exists(nestedPath));
|
||||||
|
|
||||||
|
// Verify each level exists
|
||||||
|
assertTrue("Level 1 should exist", filerClient.exists(TEST_ROOT + "/level1"));
|
||||||
|
assertTrue("Level 2 should exist", filerClient.exists(TEST_ROOT + "/level1/level2"));
|
||||||
|
assertTrue("Level 3 should exist", filerClient.exists(nestedPath));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testMultipleFilesInDirectory() {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
String testDir = TEST_ROOT + "/multifiles";
|
||||||
|
filerClient.mkdirs(testDir, 0755);
|
||||||
|
|
||||||
|
// Create 10 files
|
||||||
|
for (int i = 0; i < 10; i++) {
|
||||||
|
filerClient.touch(testDir + "/file" + i + ".txt", 0644);
|
||||||
|
}
|
||||||
|
|
||||||
|
List<FilerProto.Entry> entries = filerClient.listEntries(testDir);
|
||||||
|
|
||||||
|
assertNotNull("Entries should not be null", entries);
|
||||||
|
assertEquals("Should have 10 files", 10, entries.size());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testRenameDirectory() {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
String srcDir = TEST_ROOT + "/sourcedir";
|
||||||
|
String dstDir = TEST_ROOT + "/destdir";
|
||||||
|
|
||||||
|
filerClient.mkdirs(srcDir, 0755);
|
||||||
|
filerClient.touch(srcDir + "/file.txt", 0644);
|
||||||
|
|
||||||
|
boolean success = filerClient.mv(srcDir, dstDir);
|
||||||
|
|
||||||
|
assertTrue("Directory rename should succeed", success);
|
||||||
|
assertFalse("Source directory should not exist", filerClient.exists(srcDir));
|
||||||
|
assertTrue("Destination directory should exist", filerClient.exists(dstDir));
|
||||||
|
assertTrue("File should exist in destination", filerClient.exists(dstDir + "/file.txt"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testLookupNonExistentEntry() {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
FilerProto.Entry entry = filerClient.lookupEntry(TEST_ROOT, "nonexistent.txt");
|
||||||
|
|
||||||
|
assertNull("Entry for non-existent file should be null", entry);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testEmptyDirectory() {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
String emptyDir = TEST_ROOT + "/emptydir";
|
||||||
|
filerClient.mkdirs(emptyDir, 0755);
|
||||||
|
|
||||||
|
List<FilerProto.Entry> entries = filerClient.listEntries(emptyDir);
|
||||||
|
|
||||||
|
assertNotNull("Entries should not be null", entries);
|
||||||
|
assertTrue("Empty directory should have no entries", entries.isEmpty());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,417 @@
|
|||||||
|
package seaweedfs.client;
|
||||||
|
|
||||||
|
import org.junit.After;
|
||||||
|
import org.junit.Before;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Random;
|
||||||
|
|
||||||
|
import static org.junit.Assert.*;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Integration tests for SeaweedInputStream and SeaweedOutputStream.
|
||||||
|
*
|
||||||
|
* These tests verify stream operations against a running SeaweedFS instance.
|
||||||
|
*
|
||||||
|
* Prerequisites:
|
||||||
|
* - SeaweedFS master, volume server, and filer must be running
|
||||||
|
* - Default ports: filer HTTP 8888, filer gRPC 18888
|
||||||
|
*
|
||||||
|
* To run tests:
|
||||||
|
* export SEAWEEDFS_TEST_ENABLED=true
|
||||||
|
* mvn test -Dtest=SeaweedStreamIntegrationTest
|
||||||
|
*/
|
||||||
|
public class SeaweedStreamIntegrationTest {
|
||||||
|
|
||||||
|
private FilerClient filerClient;
|
||||||
|
private static final String TEST_ROOT = "/test-stream-integration";
|
||||||
|
private static final boolean TESTS_ENABLED =
|
||||||
|
"true".equalsIgnoreCase(System.getenv("SEAWEEDFS_TEST_ENABLED"));
|
||||||
|
|
||||||
|
@Before
|
||||||
|
public void setUp() throws Exception {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
filerClient = new FilerClient("localhost", 18888);
|
||||||
|
|
||||||
|
// Clean up any existing test directory
|
||||||
|
if (filerClient.exists(TEST_ROOT)) {
|
||||||
|
filerClient.rm(TEST_ROOT, true, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create test root directory
|
||||||
|
filerClient.mkdirs(TEST_ROOT, 0755);
|
||||||
|
}
|
||||||
|
|
||||||
|
@After
|
||||||
|
public void tearDown() throws Exception {
|
||||||
|
if (!TESTS_ENABLED || filerClient == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Clean up test directory
|
||||||
|
if (filerClient.exists(TEST_ROOT)) {
|
||||||
|
filerClient.rm(TEST_ROOT, true, true);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
filerClient.shutdown();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testWriteAndReadSmallFile() throws IOException {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
String testPath = TEST_ROOT + "/small.txt";
|
||||||
|
String testContent = "Hello, SeaweedFS!";
|
||||||
|
|
||||||
|
// Write file
|
||||||
|
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath);
|
||||||
|
outputStream.write(testContent.getBytes(StandardCharsets.UTF_8));
|
||||||
|
outputStream.close();
|
||||||
|
|
||||||
|
// Verify file exists
|
||||||
|
assertTrue("File should exist", filerClient.exists(testPath));
|
||||||
|
|
||||||
|
// Read file
|
||||||
|
FilerProto.Entry entry = filerClient.lookupEntry(
|
||||||
|
SeaweedOutputStream.getParentDirectory(testPath),
|
||||||
|
SeaweedOutputStream.getFileName(testPath)
|
||||||
|
);
|
||||||
|
assertNotNull("Entry should not be null", entry);
|
||||||
|
|
||||||
|
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry);
|
||||||
|
byte[] buffer = new byte[testContent.length()];
|
||||||
|
int bytesRead = inputStream.read(buffer);
|
||||||
|
inputStream.close();
|
||||||
|
|
||||||
|
assertEquals("Should read all bytes", testContent.length(), bytesRead);
|
||||||
|
assertEquals("Content should match", testContent, new String(buffer, StandardCharsets.UTF_8));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testWriteAndReadLargeFile() throws IOException {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
String testPath = TEST_ROOT + "/large.bin";
|
||||||
|
int fileSize = 10 * 1024 * 1024; // 10 MB
|
||||||
|
|
||||||
|
// Generate random data
|
||||||
|
byte[] originalData = new byte[fileSize];
|
||||||
|
new Random(42).nextBytes(originalData); // Use seed for reproducibility
|
||||||
|
|
||||||
|
// Write file
|
||||||
|
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath);
|
||||||
|
outputStream.write(originalData);
|
||||||
|
outputStream.close();
|
||||||
|
|
||||||
|
// Verify file exists
|
||||||
|
assertTrue("File should exist", filerClient.exists(testPath));
|
||||||
|
|
||||||
|
// Read file
|
||||||
|
FilerProto.Entry entry = filerClient.lookupEntry(
|
||||||
|
SeaweedOutputStream.getParentDirectory(testPath),
|
||||||
|
SeaweedOutputStream.getFileName(testPath)
|
||||||
|
);
|
||||||
|
assertNotNull("Entry should not be null", entry);
|
||||||
|
|
||||||
|
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry);
|
||||||
|
|
||||||
|
// Read file in chunks to handle large files properly
|
||||||
|
byte[] readData = new byte[fileSize];
|
||||||
|
int totalRead = 0;
|
||||||
|
int bytesRead;
|
||||||
|
byte[] buffer = new byte[8192]; // Read in 8KB chunks
|
||||||
|
|
||||||
|
while ((bytesRead = inputStream.read(buffer)) > 0) {
|
||||||
|
System.arraycopy(buffer, 0, readData, totalRead, bytesRead);
|
||||||
|
totalRead += bytesRead;
|
||||||
|
}
|
||||||
|
inputStream.close();
|
||||||
|
|
||||||
|
assertEquals("Should read all bytes", fileSize, totalRead);
|
||||||
|
assertArrayEquals("Content should match", originalData, readData);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testWriteInChunks() throws IOException {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
String testPath = TEST_ROOT + "/chunked.txt";
|
||||||
|
String[] chunks = {"First chunk. ", "Second chunk. ", "Third chunk."};
|
||||||
|
|
||||||
|
// Write file in chunks
|
||||||
|
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath);
|
||||||
|
for (String chunk : chunks) {
|
||||||
|
outputStream.write(chunk.getBytes(StandardCharsets.UTF_8));
|
||||||
|
}
|
||||||
|
outputStream.close();
|
||||||
|
|
||||||
|
// Read and verify
|
||||||
|
FilerProto.Entry entry = filerClient.lookupEntry(
|
||||||
|
SeaweedOutputStream.getParentDirectory(testPath),
|
||||||
|
SeaweedOutputStream.getFileName(testPath)
|
||||||
|
);
|
||||||
|
|
||||||
|
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry);
|
||||||
|
byte[] buffer = new byte[1024];
|
||||||
|
int bytesRead = inputStream.read(buffer);
|
||||||
|
inputStream.close();
|
||||||
|
|
||||||
|
String expected = String.join("", chunks);
|
||||||
|
String actual = new String(buffer, 0, bytesRead, StandardCharsets.UTF_8);
|
||||||
|
|
||||||
|
assertEquals("Content should match", expected, actual);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testReadWithOffset() throws IOException {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
String testPath = TEST_ROOT + "/offset.txt";
|
||||||
|
String testContent = "0123456789ABCDEFGHIJ";
|
||||||
|
|
||||||
|
// Write file
|
||||||
|
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath);
|
||||||
|
outputStream.write(testContent.getBytes(StandardCharsets.UTF_8));
|
||||||
|
outputStream.close();
|
||||||
|
|
||||||
|
// Read with offset
|
||||||
|
FilerProto.Entry entry = filerClient.lookupEntry(
|
||||||
|
SeaweedOutputStream.getParentDirectory(testPath),
|
||||||
|
SeaweedOutputStream.getFileName(testPath)
|
||||||
|
);
|
||||||
|
|
||||||
|
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry);
|
||||||
|
inputStream.seek(10); // Skip first 10 bytes
|
||||||
|
|
||||||
|
byte[] buffer = new byte[10];
|
||||||
|
int bytesRead = inputStream.read(buffer);
|
||||||
|
inputStream.close();
|
||||||
|
|
||||||
|
assertEquals("Should read 10 bytes", 10, bytesRead);
|
||||||
|
assertEquals("Should read from offset", "ABCDEFGHIJ",
|
||||||
|
new String(buffer, StandardCharsets.UTF_8));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testReadPartial() throws IOException {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
String testPath = TEST_ROOT + "/partial.txt";
|
||||||
|
String testContent = "The quick brown fox jumps over the lazy dog";
|
||||||
|
|
||||||
|
// Write file
|
||||||
|
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath);
|
||||||
|
outputStream.write(testContent.getBytes(StandardCharsets.UTF_8));
|
||||||
|
outputStream.close();
|
||||||
|
|
||||||
|
// Read partial
|
||||||
|
FilerProto.Entry entry = filerClient.lookupEntry(
|
||||||
|
SeaweedOutputStream.getParentDirectory(testPath),
|
||||||
|
SeaweedOutputStream.getFileName(testPath)
|
||||||
|
);
|
||||||
|
|
||||||
|
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry);
|
||||||
|
|
||||||
|
// Read only "quick brown"
|
||||||
|
inputStream.seek(4);
|
||||||
|
byte[] buffer = new byte[11];
|
||||||
|
int bytesRead = inputStream.read(buffer);
|
||||||
|
inputStream.close();
|
||||||
|
|
||||||
|
assertEquals("Should read 11 bytes", 11, bytesRead);
|
||||||
|
assertEquals("Should read partial content", "quick brown",
|
||||||
|
new String(buffer, StandardCharsets.UTF_8));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testEmptyFile() throws IOException {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
String testPath = TEST_ROOT + "/empty.txt";
|
||||||
|
|
||||||
|
// Write empty file
|
||||||
|
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath);
|
||||||
|
outputStream.close();
|
||||||
|
|
||||||
|
// Verify file exists
|
||||||
|
assertTrue("File should exist", filerClient.exists(testPath));
|
||||||
|
|
||||||
|
// Read empty file
|
||||||
|
FilerProto.Entry entry = filerClient.lookupEntry(
|
||||||
|
SeaweedOutputStream.getParentDirectory(testPath),
|
||||||
|
SeaweedOutputStream.getFileName(testPath)
|
||||||
|
);
|
||||||
|
assertNotNull("Entry should not be null", entry);
|
||||||
|
|
||||||
|
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry);
|
||||||
|
byte[] buffer = new byte[100];
|
||||||
|
int bytesRead = inputStream.read(buffer);
|
||||||
|
inputStream.close();
|
||||||
|
|
||||||
|
assertEquals("Should read 0 bytes from empty file", -1, bytesRead);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testOverwriteFile() throws IOException {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
String testPath = TEST_ROOT + "/overwrite.txt";
|
||||||
|
String originalContent = "Original content";
|
||||||
|
String newContent = "New content that overwrites the original";
|
||||||
|
|
||||||
|
// Write original file
|
||||||
|
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath);
|
||||||
|
outputStream.write(originalContent.getBytes(StandardCharsets.UTF_8));
|
||||||
|
outputStream.close();
|
||||||
|
|
||||||
|
// Overwrite file
|
||||||
|
outputStream = new SeaweedOutputStream(filerClient, testPath);
|
||||||
|
outputStream.write(newContent.getBytes(StandardCharsets.UTF_8));
|
||||||
|
outputStream.close();
|
||||||
|
|
||||||
|
// Read and verify
|
||||||
|
FilerProto.Entry entry = filerClient.lookupEntry(
|
||||||
|
SeaweedOutputStream.getParentDirectory(testPath),
|
||||||
|
SeaweedOutputStream.getFileName(testPath)
|
||||||
|
);
|
||||||
|
|
||||||
|
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry);
|
||||||
|
byte[] buffer = new byte[1024];
|
||||||
|
int bytesRead = inputStream.read(buffer);
|
||||||
|
inputStream.close();
|
||||||
|
|
||||||
|
String actual = new String(buffer, 0, bytesRead, StandardCharsets.UTF_8);
|
||||||
|
assertEquals("Should have new content", newContent, actual);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testMultipleReads() throws IOException {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
String testPath = TEST_ROOT + "/multireads.txt";
|
||||||
|
String testContent = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
|
||||||
|
|
||||||
|
// Write file
|
||||||
|
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath);
|
||||||
|
outputStream.write(testContent.getBytes(StandardCharsets.UTF_8));
|
||||||
|
outputStream.close();
|
||||||
|
|
||||||
|
// Read in multiple small chunks
|
||||||
|
FilerProto.Entry entry = filerClient.lookupEntry(
|
||||||
|
SeaweedOutputStream.getParentDirectory(testPath),
|
||||||
|
SeaweedOutputStream.getFileName(testPath)
|
||||||
|
);
|
||||||
|
|
||||||
|
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry);
|
||||||
|
|
||||||
|
StringBuilder result = new StringBuilder();
|
||||||
|
byte[] buffer = new byte[5];
|
||||||
|
int bytesRead;
|
||||||
|
while ((bytesRead = inputStream.read(buffer)) > 0) {
|
||||||
|
result.append(new String(buffer, 0, bytesRead, StandardCharsets.UTF_8));
|
||||||
|
}
|
||||||
|
inputStream.close();
|
||||||
|
|
||||||
|
assertEquals("Should read entire content", testContent, result.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testBinaryData() throws IOException {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
String testPath = TEST_ROOT + "/binary.bin";
|
||||||
|
byte[] binaryData = new byte[256];
|
||||||
|
for (int i = 0; i < 256; i++) {
|
||||||
|
binaryData[i] = (byte) i;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write binary file
|
||||||
|
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath);
|
||||||
|
outputStream.write(binaryData);
|
||||||
|
outputStream.close();
|
||||||
|
|
||||||
|
// Read and verify
|
||||||
|
FilerProto.Entry entry = filerClient.lookupEntry(
|
||||||
|
SeaweedOutputStream.getParentDirectory(testPath),
|
||||||
|
SeaweedOutputStream.getFileName(testPath)
|
||||||
|
);
|
||||||
|
|
||||||
|
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry);
|
||||||
|
byte[] readData = new byte[256];
|
||||||
|
int bytesRead = inputStream.read(readData);
|
||||||
|
inputStream.close();
|
||||||
|
|
||||||
|
assertEquals("Should read all bytes", 256, bytesRead);
|
||||||
|
assertArrayEquals("Binary data should match", binaryData, readData);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testFlush() throws IOException {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
String testPath = TEST_ROOT + "/flush.txt";
|
||||||
|
String testContent = "Content to flush";
|
||||||
|
|
||||||
|
// Write file with flush
|
||||||
|
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath);
|
||||||
|
outputStream.write(testContent.getBytes(StandardCharsets.UTF_8));
|
||||||
|
outputStream.flush(); // Explicitly flush
|
||||||
|
outputStream.close();
|
||||||
|
|
||||||
|
// Verify file was written
|
||||||
|
assertTrue("File should exist after flush", filerClient.exists(testPath));
|
||||||
|
|
||||||
|
// Read and verify
|
||||||
|
FilerProto.Entry entry = filerClient.lookupEntry(
|
||||||
|
SeaweedOutputStream.getParentDirectory(testPath),
|
||||||
|
SeaweedOutputStream.getFileName(testPath)
|
||||||
|
);
|
||||||
|
|
||||||
|
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry);
|
||||||
|
byte[] buffer = new byte[testContent.length()];
|
||||||
|
int bytesRead = inputStream.read(buffer);
|
||||||
|
inputStream.close();
|
||||||
|
|
||||||
|
assertEquals("Content should match", testContent,
|
||||||
|
new String(buffer, 0, bytesRead, StandardCharsets.UTF_8));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
190
other/java/hdfs2/README.md
Normal file
190
other/java/hdfs2/README.md
Normal file
@@ -0,0 +1,190 @@
|
|||||||
|
# SeaweedFS Hadoop2 Client
|
||||||
|
|
||||||
|
Hadoop FileSystem implementation for SeaweedFS, compatible with Hadoop 2.x/3.x.
|
||||||
|
|
||||||
|
## Building
|
||||||
|
|
||||||
|
```bash
|
||||||
|
mvn clean install
|
||||||
|
```
|
||||||
|
|
||||||
|
## Testing
|
||||||
|
|
||||||
|
This project includes two types of tests:
|
||||||
|
|
||||||
|
### 1. Configuration Tests (No SeaweedFS Required)
|
||||||
|
|
||||||
|
These tests verify configuration handling and initialization logic without requiring a running SeaweedFS instance:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
mvn test -Dtest=SeaweedFileSystemConfigTest
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2. Integration Tests (Requires SeaweedFS)
|
||||||
|
|
||||||
|
These tests verify actual FileSystem operations against a running SeaweedFS instance.
|
||||||
|
|
||||||
|
#### Prerequisites
|
||||||
|
|
||||||
|
1. Start SeaweedFS with default ports:
|
||||||
|
```bash
|
||||||
|
# Terminal 1: Start master
|
||||||
|
weed master
|
||||||
|
|
||||||
|
# Terminal 2: Start volume server
|
||||||
|
weed volume -mserver=localhost:9333
|
||||||
|
|
||||||
|
# Terminal 3: Start filer
|
||||||
|
weed filer -master=localhost:9333
|
||||||
|
```
|
||||||
|
|
||||||
|
2. Verify services are running:
|
||||||
|
- Master: http://localhost:9333
|
||||||
|
- Filer HTTP: http://localhost:8888
|
||||||
|
- Filer gRPC: localhost:18888
|
||||||
|
|
||||||
|
#### Running Integration Tests
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Enable integration tests
|
||||||
|
export SEAWEEDFS_TEST_ENABLED=true
|
||||||
|
|
||||||
|
# Run all tests
|
||||||
|
mvn test
|
||||||
|
|
||||||
|
# Run specific test
|
||||||
|
mvn test -Dtest=SeaweedFileSystemTest
|
||||||
|
```
|
||||||
|
|
||||||
|
### Test Configuration
|
||||||
|
|
||||||
|
Integration tests can be configured via environment variables or system properties:
|
||||||
|
|
||||||
|
- `SEAWEEDFS_TEST_ENABLED`: Set to `true` to enable integration tests (default: false)
|
||||||
|
- Tests use these default connection settings:
|
||||||
|
- Filer Host: localhost
|
||||||
|
- Filer HTTP Port: 8888
|
||||||
|
- Filer gRPC Port: 18888
|
||||||
|
|
||||||
|
### Running Tests with Custom Configuration
|
||||||
|
|
||||||
|
To test against a different SeaweedFS instance, modify the test code or use Hadoop configuration:
|
||||||
|
|
||||||
|
```java
|
||||||
|
conf.set("fs.seaweed.filer.host", "your-host");
|
||||||
|
conf.setInt("fs.seaweed.filer.port", 8888);
|
||||||
|
conf.setInt("fs.seaweed.filer.port.grpc", 18888);
|
||||||
|
```
|
||||||
|
|
||||||
|
## Test Coverage
|
||||||
|
|
||||||
|
The test suite covers:
|
||||||
|
|
||||||
|
- **Configuration & Initialization**
|
||||||
|
- URI parsing and configuration
|
||||||
|
- Default values
|
||||||
|
- Configuration overrides
|
||||||
|
- Working directory management
|
||||||
|
|
||||||
|
- **File Operations**
|
||||||
|
- Create files
|
||||||
|
- Read files
|
||||||
|
- Write files
|
||||||
|
- Append to files
|
||||||
|
- Delete files
|
||||||
|
|
||||||
|
- **Directory Operations**
|
||||||
|
- Create directories
|
||||||
|
- List directory contents
|
||||||
|
- Delete directories (recursive and non-recursive)
|
||||||
|
|
||||||
|
- **Metadata Operations**
|
||||||
|
- Get file status
|
||||||
|
- Set permissions
|
||||||
|
- Set owner/group
|
||||||
|
- Rename files and directories
|
||||||
|
|
||||||
|
## Usage in Hadoop
|
||||||
|
|
||||||
|
1. Copy the built JAR to your Hadoop classpath:
|
||||||
|
```bash
|
||||||
|
cp target/seaweedfs-hadoop2-client-*.jar $HADOOP_HOME/share/hadoop/common/lib/
|
||||||
|
```
|
||||||
|
|
||||||
|
2. Configure `core-site.xml`:
|
||||||
|
```xml
|
||||||
|
<configuration>
|
||||||
|
<property>
|
||||||
|
<name>fs.seaweedfs.impl</name>
|
||||||
|
<value>seaweed.hdfs.SeaweedFileSystem</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>fs.seaweed.filer.host</name>
|
||||||
|
<value>localhost</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>fs.seaweed.filer.port</name>
|
||||||
|
<value>8888</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>fs.seaweed.filer.port.grpc</name>
|
||||||
|
<value>18888</value>
|
||||||
|
</property>
|
||||||
|
</configuration>
|
||||||
|
```
|
||||||
|
|
||||||
|
3. Use SeaweedFS with Hadoop commands:
|
||||||
|
```bash
|
||||||
|
hadoop fs -ls seaweedfs://localhost:8888/
|
||||||
|
hadoop fs -mkdir seaweedfs://localhost:8888/test
|
||||||
|
hadoop fs -put local.txt seaweedfs://localhost:8888/test/
|
||||||
|
```
|
||||||
|
|
||||||
|
## Continuous Integration
|
||||||
|
|
||||||
|
For CI environments, tests can be run in two modes:
|
||||||
|
|
||||||
|
1. **Configuration Tests Only** (default, no SeaweedFS required):
|
||||||
|
```bash
|
||||||
|
mvn test -Dtest=SeaweedFileSystemConfigTest
|
||||||
|
```
|
||||||
|
|
||||||
|
2. **Full Integration Tests** (requires SeaweedFS):
|
||||||
|
```bash
|
||||||
|
# Start SeaweedFS in CI environment
|
||||||
|
# Then run:
|
||||||
|
export SEAWEEDFS_TEST_ENABLED=true
|
||||||
|
mvn test
|
||||||
|
```
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
### Tests are skipped
|
||||||
|
|
||||||
|
If you see "Skipping test - SEAWEEDFS_TEST_ENABLED not set":
|
||||||
|
```bash
|
||||||
|
export SEAWEEDFS_TEST_ENABLED=true
|
||||||
|
```
|
||||||
|
|
||||||
|
### Connection refused errors
|
||||||
|
|
||||||
|
Ensure SeaweedFS is running and accessible:
|
||||||
|
```bash
|
||||||
|
curl http://localhost:8888/
|
||||||
|
```
|
||||||
|
|
||||||
|
### gRPC errors
|
||||||
|
|
||||||
|
Verify the gRPC port is accessible:
|
||||||
|
```bash
|
||||||
|
# Should show the port is listening
|
||||||
|
netstat -an | grep 18888
|
||||||
|
```
|
||||||
|
|
||||||
|
## Contributing
|
||||||
|
|
||||||
|
When adding new features, please include:
|
||||||
|
1. Configuration tests (no SeaweedFS required)
|
||||||
|
2. Integration tests (with SEAWEEDFS_TEST_ENABLED guard)
|
||||||
|
3. Documentation updates
|
||||||
|
|
||||||
@@ -171,6 +171,25 @@
|
|||||||
<version>${hadoop.version}</version>
|
<version>${hadoop.version}</version>
|
||||||
<scope>provided</scope>
|
<scope>provided</scope>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>junit</groupId>
|
||||||
|
<artifactId>junit</artifactId>
|
||||||
|
<version>4.13.1</version>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.mockito</groupId>
|
||||||
|
<artifactId>mockito-core</artifactId>
|
||||||
|
<version>3.12.4</version>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.hadoop</groupId>
|
||||||
|
<artifactId>hadoop-common</artifactId>
|
||||||
|
<version>${hadoop.version}</version>
|
||||||
|
<scope>test</scope>
|
||||||
|
<type>test-jar</type>
|
||||||
|
</dependency>
|
||||||
</dependencies>
|
</dependencies>
|
||||||
|
|
||||||
</project>
|
</project>
|
||||||
|
|||||||
@@ -0,0 +1,90 @@
|
|||||||
|
package seaweed.hdfs;
|
||||||
|
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.fs.Path;
|
||||||
|
import org.junit.Before;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
import static org.junit.Assert.*;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Unit tests for SeaweedFileSystem configuration that don't require a running SeaweedFS instance.
|
||||||
|
*
|
||||||
|
* These tests verify basic properties and constants.
|
||||||
|
*/
|
||||||
|
public class SeaweedFileSystemConfigTest {
|
||||||
|
|
||||||
|
private SeaweedFileSystem fs;
|
||||||
|
private Configuration conf;
|
||||||
|
|
||||||
|
@Before
|
||||||
|
public void setUp() {
|
||||||
|
fs = new SeaweedFileSystem();
|
||||||
|
conf = new Configuration();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testScheme() {
|
||||||
|
assertEquals("seaweedfs", fs.getScheme());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testConstants() {
|
||||||
|
// Test that constants are defined correctly
|
||||||
|
assertEquals("fs.seaweed.filer.host", SeaweedFileSystem.FS_SEAWEED_FILER_HOST);
|
||||||
|
assertEquals("fs.seaweed.filer.port", SeaweedFileSystem.FS_SEAWEED_FILER_PORT);
|
||||||
|
assertEquals("fs.seaweed.filer.port.grpc", SeaweedFileSystem.FS_SEAWEED_FILER_PORT_GRPC);
|
||||||
|
assertEquals(8888, SeaweedFileSystem.FS_SEAWEED_DEFAULT_PORT);
|
||||||
|
assertEquals("fs.seaweed.buffer.size", SeaweedFileSystem.FS_SEAWEED_BUFFER_SIZE);
|
||||||
|
assertEquals(4 * 1024 * 1024, SeaweedFileSystem.FS_SEAWEED_DEFAULT_BUFFER_SIZE);
|
||||||
|
assertEquals("fs.seaweed.replication", SeaweedFileSystem.FS_SEAWEED_REPLICATION);
|
||||||
|
assertEquals("fs.seaweed.volume.server.access", SeaweedFileSystem.FS_SEAWEED_VOLUME_SERVER_ACCESS);
|
||||||
|
assertEquals("fs.seaweed.filer.cn", SeaweedFileSystem.FS_SEAWEED_FILER_CN);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testWorkingDirectoryPathOperations() {
|
||||||
|
// Test path operations that don't require initialization
|
||||||
|
Path testPath = new Path("/test/path");
|
||||||
|
assertTrue("Path should be absolute", testPath.isAbsolute());
|
||||||
|
assertEquals("/test/path", testPath.toUri().getPath());
|
||||||
|
|
||||||
|
Path childPath = new Path(testPath, "child");
|
||||||
|
assertEquals("/test/path/child", childPath.toUri().getPath());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testConfigurationProperties() {
|
||||||
|
// Test that configuration can be set and read
|
||||||
|
conf.set(SeaweedFileSystem.FS_SEAWEED_FILER_HOST, "testhost");
|
||||||
|
assertEquals("testhost", conf.get(SeaweedFileSystem.FS_SEAWEED_FILER_HOST));
|
||||||
|
|
||||||
|
conf.setInt(SeaweedFileSystem.FS_SEAWEED_FILER_PORT, 9999);
|
||||||
|
assertEquals(9999, conf.getInt(SeaweedFileSystem.FS_SEAWEED_FILER_PORT, 0));
|
||||||
|
|
||||||
|
conf.setInt(SeaweedFileSystem.FS_SEAWEED_BUFFER_SIZE, 8 * 1024 * 1024);
|
||||||
|
assertEquals(8 * 1024 * 1024, conf.getInt(SeaweedFileSystem.FS_SEAWEED_BUFFER_SIZE, 0));
|
||||||
|
|
||||||
|
conf.set(SeaweedFileSystem.FS_SEAWEED_REPLICATION, "001");
|
||||||
|
assertEquals("001", conf.get(SeaweedFileSystem.FS_SEAWEED_REPLICATION));
|
||||||
|
|
||||||
|
conf.set(SeaweedFileSystem.FS_SEAWEED_VOLUME_SERVER_ACCESS, "publicUrl");
|
||||||
|
assertEquals("publicUrl", conf.get(SeaweedFileSystem.FS_SEAWEED_VOLUME_SERVER_ACCESS));
|
||||||
|
|
||||||
|
conf.set(SeaweedFileSystem.FS_SEAWEED_FILER_CN, "test-cn");
|
||||||
|
assertEquals("test-cn", conf.get(SeaweedFileSystem.FS_SEAWEED_FILER_CN));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testDefaultBufferSize() {
|
||||||
|
// Test default buffer size constant
|
||||||
|
int expected = 4 * 1024 * 1024; // 4MB
|
||||||
|
assertEquals(expected, SeaweedFileSystem.FS_SEAWEED_DEFAULT_BUFFER_SIZE);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testDefaultPort() {
|
||||||
|
// Test default port constant
|
||||||
|
assertEquals(8888, SeaweedFileSystem.FS_SEAWEED_DEFAULT_PORT);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,379 @@
|
|||||||
|
package seaweed.hdfs;
|
||||||
|
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.fs.FSDataInputStream;
|
||||||
|
import org.apache.hadoop.fs.FSDataOutputStream;
|
||||||
|
import org.apache.hadoop.fs.FileStatus;
|
||||||
|
import org.apache.hadoop.fs.Path;
|
||||||
|
import org.apache.hadoop.fs.permission.FsPermission;
|
||||||
|
import org.junit.After;
|
||||||
|
import org.junit.Before;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.net.URI;
|
||||||
|
|
||||||
|
import static org.junit.Assert.*;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Unit tests for SeaweedFileSystem.
|
||||||
|
*
|
||||||
|
* These tests verify basic FileSystem operations against a SeaweedFS backend.
|
||||||
|
* Note: These tests require a running SeaweedFS filer instance.
|
||||||
|
*
|
||||||
|
* To run tests, ensure SeaweedFS is running with default ports:
|
||||||
|
* - Filer HTTP: 8888
|
||||||
|
* - Filer gRPC: 18888
|
||||||
|
*
|
||||||
|
* Set environment variable SEAWEEDFS_TEST_ENABLED=true to enable these tests.
|
||||||
|
*/
|
||||||
|
public class SeaweedFileSystemTest {
|
||||||
|
|
||||||
|
private SeaweedFileSystem fs;
|
||||||
|
private Configuration conf;
|
||||||
|
private static final String TEST_ROOT = "/test-hdfs2";
|
||||||
|
private static final boolean TESTS_ENABLED =
|
||||||
|
"true".equalsIgnoreCase(System.getenv("SEAWEEDFS_TEST_ENABLED"));
|
||||||
|
|
||||||
|
@Before
|
||||||
|
public void setUp() throws Exception {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
conf = new Configuration();
|
||||||
|
conf.set("fs.seaweed.filer.host", "localhost");
|
||||||
|
conf.setInt("fs.seaweed.filer.port", 8888);
|
||||||
|
conf.setInt("fs.seaweed.filer.port.grpc", 18888);
|
||||||
|
|
||||||
|
fs = new SeaweedFileSystem();
|
||||||
|
URI uri = new URI("seaweedfs://localhost:8888/");
|
||||||
|
fs.initialize(uri, conf);
|
||||||
|
|
||||||
|
// Clean up any existing test directory
|
||||||
|
Path testPath = new Path(TEST_ROOT);
|
||||||
|
if (fs.exists(testPath)) {
|
||||||
|
fs.delete(testPath, true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@After
|
||||||
|
public void tearDown() throws Exception {
|
||||||
|
if (!TESTS_ENABLED || fs == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clean up test directory
|
||||||
|
Path testPath = new Path(TEST_ROOT);
|
||||||
|
if (fs.exists(testPath)) {
|
||||||
|
fs.delete(testPath, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
fs.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testInitialization() throws Exception {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
assertNotNull(fs);
|
||||||
|
assertEquals("seaweedfs", fs.getScheme());
|
||||||
|
assertNotNull(fs.getUri());
|
||||||
|
assertEquals("/", fs.getWorkingDirectory().toUri().getPath());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testMkdirs() throws Exception {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
Path testDir = new Path(TEST_ROOT + "/testdir");
|
||||||
|
assertTrue("Failed to create directory", fs.mkdirs(testDir));
|
||||||
|
assertTrue("Directory should exist", fs.exists(testDir));
|
||||||
|
|
||||||
|
FileStatus status = fs.getFileStatus(testDir);
|
||||||
|
assertTrue("Path should be a directory", status.isDirectory());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testCreateAndReadFile() throws Exception {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
Path testFile = new Path(TEST_ROOT + "/testfile.txt");
|
||||||
|
String testContent = "Hello, SeaweedFS!";
|
||||||
|
|
||||||
|
// Create and write to file
|
||||||
|
FSDataOutputStream out = fs.create(testFile, FsPermission.getDefault(),
|
||||||
|
false, 4096, (short) 1, 4 * 1024 * 1024, null);
|
||||||
|
assertNotNull("Output stream should not be null", out);
|
||||||
|
out.write(testContent.getBytes());
|
||||||
|
out.close();
|
||||||
|
|
||||||
|
// Verify file exists
|
||||||
|
assertTrue("File should exist", fs.exists(testFile));
|
||||||
|
|
||||||
|
// Read and verify content
|
||||||
|
FSDataInputStream in = fs.open(testFile, 4096);
|
||||||
|
assertNotNull("Input stream should not be null", in);
|
||||||
|
byte[] buffer = new byte[testContent.length()];
|
||||||
|
int bytesRead = in.read(buffer);
|
||||||
|
in.close();
|
||||||
|
|
||||||
|
assertEquals("Should read all bytes", testContent.length(), bytesRead);
|
||||||
|
assertEquals("Content should match", testContent, new String(buffer));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testFileStatus() throws Exception {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
Path testFile = new Path(TEST_ROOT + "/statustest.txt");
|
||||||
|
String content = "test content";
|
||||||
|
|
||||||
|
FSDataOutputStream out = fs.create(testFile);
|
||||||
|
out.write(content.getBytes());
|
||||||
|
out.close();
|
||||||
|
|
||||||
|
FileStatus status = fs.getFileStatus(testFile);
|
||||||
|
assertNotNull("FileStatus should not be null", status);
|
||||||
|
assertFalse("Should not be a directory", status.isDirectory());
|
||||||
|
assertTrue("Should be a file", status.isFile());
|
||||||
|
assertEquals("File length should match", content.length(), status.getLen());
|
||||||
|
assertNotNull("Path should not be null", status.getPath());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testListStatus() throws Exception {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
Path testDir = new Path(TEST_ROOT + "/listtest");
|
||||||
|
fs.mkdirs(testDir);
|
||||||
|
|
||||||
|
// Create multiple files
|
||||||
|
for (int i = 0; i < 3; i++) {
|
||||||
|
Path file = new Path(testDir, "file" + i + ".txt");
|
||||||
|
FSDataOutputStream out = fs.create(file);
|
||||||
|
out.write(("content" + i).getBytes());
|
||||||
|
out.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
FileStatus[] statuses = fs.listStatus(testDir);
|
||||||
|
assertNotNull("List should not be null", statuses);
|
||||||
|
assertEquals("Should have 3 files", 3, statuses.length);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testRename() throws Exception {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
Path srcFile = new Path(TEST_ROOT + "/source.txt");
|
||||||
|
Path dstFile = new Path(TEST_ROOT + "/destination.txt");
|
||||||
|
String content = "rename test";
|
||||||
|
|
||||||
|
// Create source file
|
||||||
|
FSDataOutputStream out = fs.create(srcFile);
|
||||||
|
out.write(content.getBytes());
|
||||||
|
out.close();
|
||||||
|
|
||||||
|
assertTrue("Source file should exist", fs.exists(srcFile));
|
||||||
|
|
||||||
|
// Rename
|
||||||
|
assertTrue("Rename should succeed", fs.rename(srcFile, dstFile));
|
||||||
|
|
||||||
|
// Verify
|
||||||
|
assertFalse("Source file should not exist", fs.exists(srcFile));
|
||||||
|
assertTrue("Destination file should exist", fs.exists(dstFile));
|
||||||
|
|
||||||
|
// Verify content preserved
|
||||||
|
FSDataInputStream in = fs.open(dstFile);
|
||||||
|
byte[] buffer = new byte[content.length()];
|
||||||
|
in.read(buffer);
|
||||||
|
in.close();
|
||||||
|
assertEquals("Content should be preserved", content, new String(buffer));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testDelete() throws Exception {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
Path testFile = new Path(TEST_ROOT + "/deletetest.txt");
|
||||||
|
|
||||||
|
// Create file
|
||||||
|
FSDataOutputStream out = fs.create(testFile);
|
||||||
|
out.write("delete me".getBytes());
|
||||||
|
out.close();
|
||||||
|
|
||||||
|
assertTrue("File should exist before delete", fs.exists(testFile));
|
||||||
|
|
||||||
|
// Delete
|
||||||
|
assertTrue("Delete should succeed", fs.delete(testFile, false));
|
||||||
|
assertFalse("File should not exist after delete", fs.exists(testFile));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testDeleteDirectory() throws Exception {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
Path testDir = new Path(TEST_ROOT + "/deletedir");
|
||||||
|
Path testFile = new Path(testDir, "file.txt");
|
||||||
|
|
||||||
|
// Create directory with file
|
||||||
|
fs.mkdirs(testDir);
|
||||||
|
FSDataOutputStream out = fs.create(testFile);
|
||||||
|
out.write("content".getBytes());
|
||||||
|
out.close();
|
||||||
|
|
||||||
|
assertTrue("Directory should exist", fs.exists(testDir));
|
||||||
|
assertTrue("File should exist", fs.exists(testFile));
|
||||||
|
|
||||||
|
// Recursive delete
|
||||||
|
assertTrue("Recursive delete should succeed", fs.delete(testDir, true));
|
||||||
|
assertFalse("Directory should not exist after delete", fs.exists(testDir));
|
||||||
|
assertFalse("File should not exist after delete", fs.exists(testFile));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testAppend() throws Exception {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
Path testFile = new Path(TEST_ROOT + "/appendtest.txt");
|
||||||
|
String initialContent = "initial";
|
||||||
|
String appendContent = " appended";
|
||||||
|
|
||||||
|
// Create initial file
|
||||||
|
FSDataOutputStream out = fs.create(testFile);
|
||||||
|
out.write(initialContent.getBytes());
|
||||||
|
out.close();
|
||||||
|
|
||||||
|
// Append
|
||||||
|
FSDataOutputStream appendOut = fs.append(testFile, 4096, null);
|
||||||
|
assertNotNull("Append stream should not be null", appendOut);
|
||||||
|
appendOut.write(appendContent.getBytes());
|
||||||
|
appendOut.close();
|
||||||
|
|
||||||
|
// Verify combined content
|
||||||
|
FSDataInputStream in = fs.open(testFile);
|
||||||
|
byte[] buffer = new byte[initialContent.length() + appendContent.length()];
|
||||||
|
int bytesRead = in.read(buffer);
|
||||||
|
in.close();
|
||||||
|
|
||||||
|
String expected = initialContent + appendContent;
|
||||||
|
assertEquals("Should read all bytes", expected.length(), bytesRead);
|
||||||
|
assertEquals("Content should match", expected, new String(buffer));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testSetWorkingDirectory() throws Exception {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
Path originalWd = fs.getWorkingDirectory();
|
||||||
|
assertEquals("Original working directory should be /", "/", originalWd.toUri().getPath());
|
||||||
|
|
||||||
|
Path newWd = new Path(TEST_ROOT);
|
||||||
|
fs.mkdirs(newWd);
|
||||||
|
fs.setWorkingDirectory(newWd);
|
||||||
|
|
||||||
|
Path currentWd = fs.getWorkingDirectory();
|
||||||
|
assertTrue("Working directory should be updated",
|
||||||
|
currentWd.toUri().getPath().contains(TEST_ROOT));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testSetPermission() throws Exception {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
Path testFile = new Path(TEST_ROOT + "/permtest.txt");
|
||||||
|
|
||||||
|
// Create file
|
||||||
|
FSDataOutputStream out = fs.create(testFile);
|
||||||
|
out.write("permission test".getBytes());
|
||||||
|
out.close();
|
||||||
|
|
||||||
|
// Set permission
|
||||||
|
FsPermission newPerm = new FsPermission((short) 0644);
|
||||||
|
fs.setPermission(testFile, newPerm);
|
||||||
|
|
||||||
|
FileStatus status = fs.getFileStatus(testFile);
|
||||||
|
assertNotNull("Permission should not be null", status.getPermission());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testSetOwner() throws Exception {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
Path testFile = new Path(TEST_ROOT + "/ownertest.txt");
|
||||||
|
|
||||||
|
// Create file
|
||||||
|
FSDataOutputStream out = fs.create(testFile);
|
||||||
|
out.write("owner test".getBytes());
|
||||||
|
out.close();
|
||||||
|
|
||||||
|
// Set owner - this may not fail even if not fully implemented
|
||||||
|
fs.setOwner(testFile, "testuser", "testgroup");
|
||||||
|
|
||||||
|
// Just verify the call doesn't throw an exception
|
||||||
|
FileStatus status = fs.getFileStatus(testFile);
|
||||||
|
assertNotNull("FileStatus should not be null", status);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testRenameToExistingDirectory() throws Exception {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
Path srcFile = new Path(TEST_ROOT + "/movefile.txt");
|
||||||
|
Path dstDir = new Path(TEST_ROOT + "/movedir");
|
||||||
|
|
||||||
|
// Create source file and destination directory
|
||||||
|
FSDataOutputStream out = fs.create(srcFile);
|
||||||
|
out.write("move test".getBytes());
|
||||||
|
out.close();
|
||||||
|
fs.mkdirs(dstDir);
|
||||||
|
|
||||||
|
// Rename file to existing directory (should move file into directory)
|
||||||
|
assertTrue("Rename to directory should succeed", fs.rename(srcFile, dstDir));
|
||||||
|
|
||||||
|
// File should be moved into the directory
|
||||||
|
Path expectedLocation = new Path(dstDir, srcFile.getName());
|
||||||
|
assertTrue("File should exist in destination directory", fs.exists(expectedLocation));
|
||||||
|
assertFalse("Source file should not exist", fs.exists(srcFile));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
190
other/java/hdfs3/README.md
Normal file
190
other/java/hdfs3/README.md
Normal file
@@ -0,0 +1,190 @@
|
|||||||
|
# SeaweedFS Hadoop3 Client
|
||||||
|
|
||||||
|
Hadoop FileSystem implementation for SeaweedFS, compatible with Hadoop 3.x.
|
||||||
|
|
||||||
|
## Building
|
||||||
|
|
||||||
|
```bash
|
||||||
|
mvn clean install
|
||||||
|
```
|
||||||
|
|
||||||
|
## Testing
|
||||||
|
|
||||||
|
This project includes two types of tests:
|
||||||
|
|
||||||
|
### 1. Configuration Tests (No SeaweedFS Required)
|
||||||
|
|
||||||
|
These tests verify configuration handling and initialization logic without requiring a running SeaweedFS instance:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
mvn test -Dtest=SeaweedFileSystemConfigTest
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2. Integration Tests (Requires SeaweedFS)
|
||||||
|
|
||||||
|
These tests verify actual FileSystem operations against a running SeaweedFS instance.
|
||||||
|
|
||||||
|
#### Prerequisites
|
||||||
|
|
||||||
|
1. Start SeaweedFS with default ports:
|
||||||
|
```bash
|
||||||
|
# Terminal 1: Start master
|
||||||
|
weed master
|
||||||
|
|
||||||
|
# Terminal 2: Start volume server
|
||||||
|
weed volume -mserver=localhost:9333
|
||||||
|
|
||||||
|
# Terminal 3: Start filer
|
||||||
|
weed filer -master=localhost:9333
|
||||||
|
```
|
||||||
|
|
||||||
|
2. Verify services are running:
|
||||||
|
- Master: http://localhost:9333
|
||||||
|
- Filer HTTP: http://localhost:8888
|
||||||
|
- Filer gRPC: localhost:18888
|
||||||
|
|
||||||
|
#### Running Integration Tests
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Enable integration tests
|
||||||
|
export SEAWEEDFS_TEST_ENABLED=true
|
||||||
|
|
||||||
|
# Run all tests
|
||||||
|
mvn test
|
||||||
|
|
||||||
|
# Run specific test
|
||||||
|
mvn test -Dtest=SeaweedFileSystemTest
|
||||||
|
```
|
||||||
|
|
||||||
|
### Test Configuration
|
||||||
|
|
||||||
|
Integration tests can be configured via environment variables or system properties:
|
||||||
|
|
||||||
|
- `SEAWEEDFS_TEST_ENABLED`: Set to `true` to enable integration tests (default: false)
|
||||||
|
- Tests use these default connection settings:
|
||||||
|
- Filer Host: localhost
|
||||||
|
- Filer HTTP Port: 8888
|
||||||
|
- Filer gRPC Port: 18888
|
||||||
|
|
||||||
|
### Running Tests with Custom Configuration
|
||||||
|
|
||||||
|
To test against a different SeaweedFS instance, modify the test code or use Hadoop configuration:
|
||||||
|
|
||||||
|
```java
|
||||||
|
conf.set("fs.seaweed.filer.host", "your-host");
|
||||||
|
conf.setInt("fs.seaweed.filer.port", 8888);
|
||||||
|
conf.setInt("fs.seaweed.filer.port.grpc", 18888);
|
||||||
|
```
|
||||||
|
|
||||||
|
## Test Coverage
|
||||||
|
|
||||||
|
The test suite covers:
|
||||||
|
|
||||||
|
- **Configuration & Initialization**
|
||||||
|
- URI parsing and configuration
|
||||||
|
- Default values
|
||||||
|
- Configuration overrides
|
||||||
|
- Working directory management
|
||||||
|
|
||||||
|
- **File Operations**
|
||||||
|
- Create files
|
||||||
|
- Read files
|
||||||
|
- Write files
|
||||||
|
- Append to files
|
||||||
|
- Delete files
|
||||||
|
|
||||||
|
- **Directory Operations**
|
||||||
|
- Create directories
|
||||||
|
- List directory contents
|
||||||
|
- Delete directories (recursive and non-recursive)
|
||||||
|
|
||||||
|
- **Metadata Operations**
|
||||||
|
- Get file status
|
||||||
|
- Set permissions
|
||||||
|
- Set owner/group
|
||||||
|
- Rename files and directories
|
||||||
|
|
||||||
|
## Usage in Hadoop
|
||||||
|
|
||||||
|
1. Copy the built JAR to your Hadoop classpath:
|
||||||
|
```bash
|
||||||
|
cp target/seaweedfs-hadoop3-client-*.jar $HADOOP_HOME/share/hadoop/common/lib/
|
||||||
|
```
|
||||||
|
|
||||||
|
2. Configure `core-site.xml`:
|
||||||
|
```xml
|
||||||
|
<configuration>
|
||||||
|
<property>
|
||||||
|
<name>fs.seaweedfs.impl</name>
|
||||||
|
<value>seaweed.hdfs.SeaweedFileSystem</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>fs.seaweed.filer.host</name>
|
||||||
|
<value>localhost</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>fs.seaweed.filer.port</name>
|
||||||
|
<value>8888</value>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>fs.seaweed.filer.port.grpc</name>
|
||||||
|
<value>18888</value>
|
||||||
|
</property>
|
||||||
|
</configuration>
|
||||||
|
```
|
||||||
|
|
||||||
|
3. Use SeaweedFS with Hadoop commands:
|
||||||
|
```bash
|
||||||
|
hadoop fs -ls seaweedfs://localhost:8888/
|
||||||
|
hadoop fs -mkdir seaweedfs://localhost:8888/test
|
||||||
|
hadoop fs -put local.txt seaweedfs://localhost:8888/test/
|
||||||
|
```
|
||||||
|
|
||||||
|
## Continuous Integration
|
||||||
|
|
||||||
|
For CI environments, tests can be run in two modes:
|
||||||
|
|
||||||
|
1. **Configuration Tests Only** (default, no SeaweedFS required):
|
||||||
|
```bash
|
||||||
|
mvn test -Dtest=SeaweedFileSystemConfigTest
|
||||||
|
```
|
||||||
|
|
||||||
|
2. **Full Integration Tests** (requires SeaweedFS):
|
||||||
|
```bash
|
||||||
|
# Start SeaweedFS in CI environment
|
||||||
|
# Then run:
|
||||||
|
export SEAWEEDFS_TEST_ENABLED=true
|
||||||
|
mvn test
|
||||||
|
```
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
### Tests are skipped
|
||||||
|
|
||||||
|
If you see "Skipping test - SEAWEEDFS_TEST_ENABLED not set":
|
||||||
|
```bash
|
||||||
|
export SEAWEEDFS_TEST_ENABLED=true
|
||||||
|
```
|
||||||
|
|
||||||
|
### Connection refused errors
|
||||||
|
|
||||||
|
Ensure SeaweedFS is running and accessible:
|
||||||
|
```bash
|
||||||
|
curl http://localhost:8888/
|
||||||
|
```
|
||||||
|
|
||||||
|
### gRPC errors
|
||||||
|
|
||||||
|
Verify the gRPC port is accessible:
|
||||||
|
```bash
|
||||||
|
# Should show the port is listening
|
||||||
|
netstat -an | grep 18888
|
||||||
|
```
|
||||||
|
|
||||||
|
## Contributing
|
||||||
|
|
||||||
|
When adding new features, please include:
|
||||||
|
1. Configuration tests (no SeaweedFS required)
|
||||||
|
2. Integration tests (with SEAWEEDFS_TEST_ENABLED guard)
|
||||||
|
3. Documentation updates
|
||||||
|
|
||||||
@@ -140,7 +140,7 @@
|
|||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.apache.hadoop</groupId>
|
<groupId>org.apache.hadoop</groupId>
|
||||||
<artifactId>hadoop-client</artifactId>
|
<artifactId>hadoop-client</artifactId>
|
||||||
<version>3.2.4</version>
|
<version>3.4.0</version>
|
||||||
<scope>provided</scope>
|
<scope>provided</scope>
|
||||||
<exclusions>
|
<exclusions>
|
||||||
<exclusion>
|
<exclusion>
|
||||||
@@ -172,9 +172,17 @@
|
|||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.apache.hadoop</groupId>
|
<groupId>org.apache.hadoop</groupId>
|
||||||
<artifactId>hadoop-common</artifactId>
|
<artifactId>hadoop-common</artifactId>
|
||||||
<version>3.2.4</version>
|
<version>3.4.0</version>
|
||||||
<scope>provided</scope>
|
<scope>provided</scope>
|
||||||
<exclusions>
|
<exclusions>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>hadoop-shaded-protobuf_3_21</artifactId>
|
||||||
|
<groupId>org.apache.hadoop.thirdparty</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>hadoop-shaded-guava</artifactId>
|
||||||
|
<groupId>org.apache.hadoop.thirdparty</groupId>
|
||||||
|
</exclusion>
|
||||||
<exclusion>
|
<exclusion>
|
||||||
<artifactId>commons-cli</artifactId>
|
<artifactId>commons-cli</artifactId>
|
||||||
<groupId>commons-cli</groupId>
|
<groupId>commons-cli</groupId>
|
||||||
@@ -200,8 +208,8 @@
|
|||||||
<groupId>javax.servlet</groupId>
|
<groupId>javax.servlet</groupId>
|
||||||
</exclusion>
|
</exclusion>
|
||||||
<exclusion>
|
<exclusion>
|
||||||
<artifactId>javax.activation-api</artifactId>
|
<artifactId>jakarta.activation-api</artifactId>
|
||||||
<groupId>javax.activation</groupId>
|
<groupId>jakarta.activation</groupId>
|
||||||
</exclusion>
|
</exclusion>
|
||||||
<exclusion>
|
<exclusion>
|
||||||
<artifactId>jetty-server</artifactId>
|
<artifactId>jetty-server</artifactId>
|
||||||
@@ -233,7 +241,11 @@
|
|||||||
</exclusion>
|
</exclusion>
|
||||||
<exclusion>
|
<exclusion>
|
||||||
<artifactId>jersey-json</artifactId>
|
<artifactId>jersey-json</artifactId>
|
||||||
<groupId>com.sun.jersey</groupId>
|
<groupId>com.github.pjfanning</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>jettison</artifactId>
|
||||||
|
<groupId>org.codehaus.jettison</groupId>
|
||||||
</exclusion>
|
</exclusion>
|
||||||
<exclusion>
|
<exclusion>
|
||||||
<artifactId>jersey-server</artifactId>
|
<artifactId>jersey-server</artifactId>
|
||||||
@@ -287,20 +299,32 @@
|
|||||||
<artifactId>curator-recipes</artifactId>
|
<artifactId>curator-recipes</artifactId>
|
||||||
<groupId>org.apache.curator</groupId>
|
<groupId>org.apache.curator</groupId>
|
||||||
</exclusion>
|
</exclusion>
|
||||||
<exclusion>
|
|
||||||
<artifactId>htrace-core4</artifactId>
|
|
||||||
<groupId>org.apache.htrace</groupId>
|
|
||||||
</exclusion>
|
|
||||||
<exclusion>
|
<exclusion>
|
||||||
<artifactId>zookeeper</artifactId>
|
<artifactId>zookeeper</artifactId>
|
||||||
<groupId>org.apache.zookeeper</groupId>
|
<groupId>org.apache.zookeeper</groupId>
|
||||||
</exclusion>
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>netty-handler</artifactId>
|
||||||
|
<groupId>io.netty</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>netty-transport-native-epoll</artifactId>
|
||||||
|
<groupId>io.netty</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>metrics-core</artifactId>
|
||||||
|
<groupId>io.dropwizard.metrics</groupId>
|
||||||
|
</exclusion>
|
||||||
<exclusion>
|
<exclusion>
|
||||||
<artifactId>commons-compress</artifactId>
|
<artifactId>commons-compress</artifactId>
|
||||||
<groupId>org.apache.commons</groupId>
|
<groupId>org.apache.commons</groupId>
|
||||||
</exclusion>
|
</exclusion>
|
||||||
<exclusion>
|
<exclusion>
|
||||||
<artifactId>kerb-simplekdc</artifactId>
|
<artifactId>bcprov-jdk15on</artifactId>
|
||||||
|
<groupId>org.bouncycastle</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>kerb-core</artifactId>
|
||||||
<groupId>org.apache.kerby</groupId>
|
<groupId>org.apache.kerby</groupId>
|
||||||
</exclusion>
|
</exclusion>
|
||||||
<exclusion>
|
<exclusion>
|
||||||
@@ -319,6 +343,227 @@
|
|||||||
<artifactId>dnsjava</artifactId>
|
<artifactId>dnsjava</artifactId>
|
||||||
<groupId>dnsjava</groupId>
|
<groupId>dnsjava</groupId>
|
||||||
</exclusion>
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>snappy-java</artifactId>
|
||||||
|
<groupId>org.xerial.snappy</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>hadoop-annotations</artifactId>
|
||||||
|
<groupId>org.apache.hadoop</groupId>
|
||||||
|
</exclusion>
|
||||||
|
</exclusions>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>junit</groupId>
|
||||||
|
<artifactId>junit</artifactId>
|
||||||
|
<version>4.13.1</version>
|
||||||
|
<scope>test</scope>
|
||||||
|
<exclusions>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>hamcrest-core</artifactId>
|
||||||
|
<groupId>org.hamcrest</groupId>
|
||||||
|
</exclusion>
|
||||||
|
</exclusions>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.mockito</groupId>
|
||||||
|
<artifactId>mockito-core</artifactId>
|
||||||
|
<version>3.12.4</version>
|
||||||
|
<scope>test</scope>
|
||||||
|
<exclusions>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>byte-buddy</artifactId>
|
||||||
|
<groupId>net.bytebuddy</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>byte-buddy-agent</artifactId>
|
||||||
|
<groupId>net.bytebuddy</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>objenesis</artifactId>
|
||||||
|
<groupId>org.objenesis</groupId>
|
||||||
|
</exclusion>
|
||||||
|
</exclusions>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.hadoop</groupId>
|
||||||
|
<artifactId>hadoop-common</artifactId>
|
||||||
|
<version>3.4.0</version>
|
||||||
|
<type>test-jar</type>
|
||||||
|
<scope>test</scope>
|
||||||
|
<exclusions>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>hadoop-shaded-protobuf_3_21</artifactId>
|
||||||
|
<groupId>org.apache.hadoop.thirdparty</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>hadoop-shaded-guava</artifactId>
|
||||||
|
<groupId>org.apache.hadoop.thirdparty</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>commons-cli</artifactId>
|
||||||
|
<groupId>commons-cli</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>commons-math3</artifactId>
|
||||||
|
<groupId>org.apache.commons</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>commons-io</artifactId>
|
||||||
|
<groupId>commons-io</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>commons-net</artifactId>
|
||||||
|
<groupId>commons-net</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>commons-collections</artifactId>
|
||||||
|
<groupId>commons-collections</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>javax.servlet-api</artifactId>
|
||||||
|
<groupId>javax.servlet</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>jakarta.activation-api</artifactId>
|
||||||
|
<groupId>jakarta.activation</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>jetty-server</artifactId>
|
||||||
|
<groupId>org.eclipse.jetty</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>jetty-util</artifactId>
|
||||||
|
<groupId>org.eclipse.jetty</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>jetty-servlet</artifactId>
|
||||||
|
<groupId>org.eclipse.jetty</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>jetty-webapp</artifactId>
|
||||||
|
<groupId>org.eclipse.jetty</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>jsp-api</artifactId>
|
||||||
|
<groupId>javax.servlet.jsp</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>jersey-core</artifactId>
|
||||||
|
<groupId>com.sun.jersey</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>jersey-servlet</artifactId>
|
||||||
|
<groupId>com.sun.jersey</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>jersey-json</artifactId>
|
||||||
|
<groupId>com.github.pjfanning</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>jettison</artifactId>
|
||||||
|
<groupId>org.codehaus.jettison</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>jersey-server</artifactId>
|
||||||
|
<groupId>com.sun.jersey</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>reload4j</artifactId>
|
||||||
|
<groupId>ch.qos.reload4j</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>commons-beanutils</artifactId>
|
||||||
|
<groupId>commons-beanutils</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>commons-configuration2</artifactId>
|
||||||
|
<groupId>org.apache.commons</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>commons-lang3</artifactId>
|
||||||
|
<groupId>org.apache.commons</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>commons-text</artifactId>
|
||||||
|
<groupId>org.apache.commons</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>slf4j-reload4j</artifactId>
|
||||||
|
<groupId>org.slf4j</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>avro</artifactId>
|
||||||
|
<groupId>org.apache.avro</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>re2j</artifactId>
|
||||||
|
<groupId>com.google.re2j</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>hadoop-auth</artifactId>
|
||||||
|
<groupId>org.apache.hadoop</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>jsch</artifactId>
|
||||||
|
<groupId>com.jcraft</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>curator-client</artifactId>
|
||||||
|
<groupId>org.apache.curator</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>curator-recipes</artifactId>
|
||||||
|
<groupId>org.apache.curator</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>zookeeper</artifactId>
|
||||||
|
<groupId>org.apache.zookeeper</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>netty-handler</artifactId>
|
||||||
|
<groupId>io.netty</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>netty-transport-native-epoll</artifactId>
|
||||||
|
<groupId>io.netty</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>metrics-core</artifactId>
|
||||||
|
<groupId>io.dropwizard.metrics</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>commons-compress</artifactId>
|
||||||
|
<groupId>org.apache.commons</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>bcprov-jdk15on</artifactId>
|
||||||
|
<groupId>org.bouncycastle</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>kerb-core</artifactId>
|
||||||
|
<groupId>org.apache.kerby</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>jackson-databind</artifactId>
|
||||||
|
<groupId>com.fasterxml.jackson.core</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>stax2-api</artifactId>
|
||||||
|
<groupId>org.codehaus.woodstox</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>woodstox-core</artifactId>
|
||||||
|
<groupId>com.fasterxml.woodstox</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>dnsjava</artifactId>
|
||||||
|
<groupId>dnsjava</groupId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<artifactId>snappy-java</artifactId>
|
||||||
|
<groupId>org.xerial.snappy</groupId>
|
||||||
|
</exclusion>
|
||||||
<exclusion>
|
<exclusion>
|
||||||
<artifactId>hadoop-annotations</artifactId>
|
<artifactId>hadoop-annotations</artifactId>
|
||||||
<groupId>org.apache.hadoop</groupId>
|
<groupId>org.apache.hadoop</groupId>
|
||||||
@@ -328,6 +573,6 @@
|
|||||||
</dependencies>
|
</dependencies>
|
||||||
<properties>
|
<properties>
|
||||||
<seaweedfs.client.version>3.80</seaweedfs.client.version>
|
<seaweedfs.client.version>3.80</seaweedfs.client.version>
|
||||||
<hadoop.version>3.2.4</hadoop.version>
|
<hadoop.version>3.4.0</hadoop.version>
|
||||||
</properties>
|
</properties>
|
||||||
</project>
|
</project>
|
||||||
|
|||||||
@@ -6,7 +6,7 @@
|
|||||||
|
|
||||||
<properties>
|
<properties>
|
||||||
<seaweedfs.client.version>3.80</seaweedfs.client.version>
|
<seaweedfs.client.version>3.80</seaweedfs.client.version>
|
||||||
<hadoop.version>3.2.4</hadoop.version>
|
<hadoop.version>3.4.0</hadoop.version>
|
||||||
</properties>
|
</properties>
|
||||||
|
|
||||||
<groupId>com.seaweedfs</groupId>
|
<groupId>com.seaweedfs</groupId>
|
||||||
@@ -171,6 +171,25 @@
|
|||||||
<version>${hadoop.version}</version>
|
<version>${hadoop.version}</version>
|
||||||
<scope>provided</scope>
|
<scope>provided</scope>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>junit</groupId>
|
||||||
|
<artifactId>junit</artifactId>
|
||||||
|
<version>4.13.1</version>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.mockito</groupId>
|
||||||
|
<artifactId>mockito-core</artifactId>
|
||||||
|
<version>3.12.4</version>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.hadoop</groupId>
|
||||||
|
<artifactId>hadoop-common</artifactId>
|
||||||
|
<version>${hadoop.version}</version>
|
||||||
|
<scope>test</scope>
|
||||||
|
<type>test-jar</type>
|
||||||
|
</dependency>
|
||||||
</dependencies>
|
</dependencies>
|
||||||
|
|
||||||
</project>
|
</project>
|
||||||
|
|||||||
@@ -0,0 +1,90 @@
|
|||||||
|
package seaweed.hdfs;
|
||||||
|
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.fs.Path;
|
||||||
|
import org.junit.Before;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
import static org.junit.Assert.*;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Unit tests for SeaweedFileSystem configuration that don't require a running SeaweedFS instance.
|
||||||
|
*
|
||||||
|
* These tests verify basic properties and constants.
|
||||||
|
*/
|
||||||
|
public class SeaweedFileSystemConfigTest {
|
||||||
|
|
||||||
|
private SeaweedFileSystem fs;
|
||||||
|
private Configuration conf;
|
||||||
|
|
||||||
|
@Before
|
||||||
|
public void setUp() {
|
||||||
|
fs = new SeaweedFileSystem();
|
||||||
|
conf = new Configuration();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testScheme() {
|
||||||
|
assertEquals("seaweedfs", fs.getScheme());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testConstants() {
|
||||||
|
// Test that constants are defined correctly
|
||||||
|
assertEquals("fs.seaweed.filer.host", SeaweedFileSystem.FS_SEAWEED_FILER_HOST);
|
||||||
|
assertEquals("fs.seaweed.filer.port", SeaweedFileSystem.FS_SEAWEED_FILER_PORT);
|
||||||
|
assertEquals("fs.seaweed.filer.port.grpc", SeaweedFileSystem.FS_SEAWEED_FILER_PORT_GRPC);
|
||||||
|
assertEquals(8888, SeaweedFileSystem.FS_SEAWEED_DEFAULT_PORT);
|
||||||
|
assertEquals("fs.seaweed.buffer.size", SeaweedFileSystem.FS_SEAWEED_BUFFER_SIZE);
|
||||||
|
assertEquals(4 * 1024 * 1024, SeaweedFileSystem.FS_SEAWEED_DEFAULT_BUFFER_SIZE);
|
||||||
|
assertEquals("fs.seaweed.replication", SeaweedFileSystem.FS_SEAWEED_REPLICATION);
|
||||||
|
assertEquals("fs.seaweed.volume.server.access", SeaweedFileSystem.FS_SEAWEED_VOLUME_SERVER_ACCESS);
|
||||||
|
assertEquals("fs.seaweed.filer.cn", SeaweedFileSystem.FS_SEAWEED_FILER_CN);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testWorkingDirectoryPathOperations() {
|
||||||
|
// Test path operations that don't require initialization
|
||||||
|
Path testPath = new Path("/test/path");
|
||||||
|
assertTrue("Path should be absolute", testPath.isAbsolute());
|
||||||
|
assertEquals("/test/path", testPath.toUri().getPath());
|
||||||
|
|
||||||
|
Path childPath = new Path(testPath, "child");
|
||||||
|
assertEquals("/test/path/child", childPath.toUri().getPath());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testConfigurationProperties() {
|
||||||
|
// Test that configuration can be set and read
|
||||||
|
conf.set(SeaweedFileSystem.FS_SEAWEED_FILER_HOST, "testhost");
|
||||||
|
assertEquals("testhost", conf.get(SeaweedFileSystem.FS_SEAWEED_FILER_HOST));
|
||||||
|
|
||||||
|
conf.setInt(SeaweedFileSystem.FS_SEAWEED_FILER_PORT, 9999);
|
||||||
|
assertEquals(9999, conf.getInt(SeaweedFileSystem.FS_SEAWEED_FILER_PORT, 0));
|
||||||
|
|
||||||
|
conf.setInt(SeaweedFileSystem.FS_SEAWEED_BUFFER_SIZE, 8 * 1024 * 1024);
|
||||||
|
assertEquals(8 * 1024 * 1024, conf.getInt(SeaweedFileSystem.FS_SEAWEED_BUFFER_SIZE, 0));
|
||||||
|
|
||||||
|
conf.set(SeaweedFileSystem.FS_SEAWEED_REPLICATION, "001");
|
||||||
|
assertEquals("001", conf.get(SeaweedFileSystem.FS_SEAWEED_REPLICATION));
|
||||||
|
|
||||||
|
conf.set(SeaweedFileSystem.FS_SEAWEED_VOLUME_SERVER_ACCESS, "publicUrl");
|
||||||
|
assertEquals("publicUrl", conf.get(SeaweedFileSystem.FS_SEAWEED_VOLUME_SERVER_ACCESS));
|
||||||
|
|
||||||
|
conf.set(SeaweedFileSystem.FS_SEAWEED_FILER_CN, "test-cn");
|
||||||
|
assertEquals("test-cn", conf.get(SeaweedFileSystem.FS_SEAWEED_FILER_CN));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testDefaultBufferSize() {
|
||||||
|
// Test default buffer size constant
|
||||||
|
int expected = 4 * 1024 * 1024; // 4MB
|
||||||
|
assertEquals(expected, SeaweedFileSystem.FS_SEAWEED_DEFAULT_BUFFER_SIZE);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testDefaultPort() {
|
||||||
|
// Test default port constant
|
||||||
|
assertEquals(8888, SeaweedFileSystem.FS_SEAWEED_DEFAULT_PORT);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,379 @@
|
|||||||
|
package seaweed.hdfs;
|
||||||
|
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.fs.FSDataInputStream;
|
||||||
|
import org.apache.hadoop.fs.FSDataOutputStream;
|
||||||
|
import org.apache.hadoop.fs.FileStatus;
|
||||||
|
import org.apache.hadoop.fs.Path;
|
||||||
|
import org.apache.hadoop.fs.permission.FsPermission;
|
||||||
|
import org.junit.After;
|
||||||
|
import org.junit.Before;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.net.URI;
|
||||||
|
|
||||||
|
import static org.junit.Assert.*;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Unit tests for SeaweedFileSystem.
|
||||||
|
*
|
||||||
|
* These tests verify basic FileSystem operations against a SeaweedFS backend.
|
||||||
|
* Note: These tests require a running SeaweedFS filer instance.
|
||||||
|
*
|
||||||
|
* To run tests, ensure SeaweedFS is running with default ports:
|
||||||
|
* - Filer HTTP: 8888
|
||||||
|
* - Filer gRPC: 18888
|
||||||
|
*
|
||||||
|
* Set environment variable SEAWEEDFS_TEST_ENABLED=true to enable these tests.
|
||||||
|
*/
|
||||||
|
public class SeaweedFileSystemTest {
|
||||||
|
|
||||||
|
private SeaweedFileSystem fs;
|
||||||
|
private Configuration conf;
|
||||||
|
private static final String TEST_ROOT = "/test-hdfs3";
|
||||||
|
private static final boolean TESTS_ENABLED =
|
||||||
|
"true".equalsIgnoreCase(System.getenv("SEAWEEDFS_TEST_ENABLED"));
|
||||||
|
|
||||||
|
@Before
|
||||||
|
public void setUp() throws Exception {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
conf = new Configuration();
|
||||||
|
conf.set("fs.seaweed.filer.host", "localhost");
|
||||||
|
conf.setInt("fs.seaweed.filer.port", 8888);
|
||||||
|
conf.setInt("fs.seaweed.filer.port.grpc", 18888);
|
||||||
|
|
||||||
|
fs = new SeaweedFileSystem();
|
||||||
|
URI uri = new URI("seaweedfs://localhost:8888/");
|
||||||
|
fs.initialize(uri, conf);
|
||||||
|
|
||||||
|
// Clean up any existing test directory
|
||||||
|
Path testPath = new Path(TEST_ROOT);
|
||||||
|
if (fs.exists(testPath)) {
|
||||||
|
fs.delete(testPath, true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@After
|
||||||
|
public void tearDown() throws Exception {
|
||||||
|
if (!TESTS_ENABLED || fs == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clean up test directory
|
||||||
|
Path testPath = new Path(TEST_ROOT);
|
||||||
|
if (fs.exists(testPath)) {
|
||||||
|
fs.delete(testPath, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
fs.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testInitialization() throws Exception {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
assertNotNull(fs);
|
||||||
|
assertEquals("seaweedfs", fs.getScheme());
|
||||||
|
assertNotNull(fs.getUri());
|
||||||
|
assertEquals("/", fs.getWorkingDirectory().toUri().getPath());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testMkdirs() throws Exception {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
Path testDir = new Path(TEST_ROOT + "/testdir");
|
||||||
|
assertTrue("Failed to create directory", fs.mkdirs(testDir));
|
||||||
|
assertTrue("Directory should exist", fs.exists(testDir));
|
||||||
|
|
||||||
|
FileStatus status = fs.getFileStatus(testDir);
|
||||||
|
assertTrue("Path should be a directory", status.isDirectory());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testCreateAndReadFile() throws Exception {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
Path testFile = new Path(TEST_ROOT + "/testfile.txt");
|
||||||
|
String testContent = "Hello, SeaweedFS!";
|
||||||
|
|
||||||
|
// Create and write to file
|
||||||
|
FSDataOutputStream out = fs.create(testFile, FsPermission.getDefault(),
|
||||||
|
false, 4096, (short) 1, 4 * 1024 * 1024, null);
|
||||||
|
assertNotNull("Output stream should not be null", out);
|
||||||
|
out.write(testContent.getBytes());
|
||||||
|
out.close();
|
||||||
|
|
||||||
|
// Verify file exists
|
||||||
|
assertTrue("File should exist", fs.exists(testFile));
|
||||||
|
|
||||||
|
// Read and verify content
|
||||||
|
FSDataInputStream in = fs.open(testFile, 4096);
|
||||||
|
assertNotNull("Input stream should not be null", in);
|
||||||
|
byte[] buffer = new byte[testContent.length()];
|
||||||
|
int bytesRead = in.read(buffer);
|
||||||
|
in.close();
|
||||||
|
|
||||||
|
assertEquals("Should read all bytes", testContent.length(), bytesRead);
|
||||||
|
assertEquals("Content should match", testContent, new String(buffer));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testFileStatus() throws Exception {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
Path testFile = new Path(TEST_ROOT + "/statustest.txt");
|
||||||
|
String content = "test content";
|
||||||
|
|
||||||
|
FSDataOutputStream out = fs.create(testFile);
|
||||||
|
out.write(content.getBytes());
|
||||||
|
out.close();
|
||||||
|
|
||||||
|
FileStatus status = fs.getFileStatus(testFile);
|
||||||
|
assertNotNull("FileStatus should not be null", status);
|
||||||
|
assertFalse("Should not be a directory", status.isDirectory());
|
||||||
|
assertTrue("Should be a file", status.isFile());
|
||||||
|
assertEquals("File length should match", content.length(), status.getLen());
|
||||||
|
assertNotNull("Path should not be null", status.getPath());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testListStatus() throws Exception {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
Path testDir = new Path(TEST_ROOT + "/listtest");
|
||||||
|
fs.mkdirs(testDir);
|
||||||
|
|
||||||
|
// Create multiple files
|
||||||
|
for (int i = 0; i < 3; i++) {
|
||||||
|
Path file = new Path(testDir, "file" + i + ".txt");
|
||||||
|
FSDataOutputStream out = fs.create(file);
|
||||||
|
out.write(("content" + i).getBytes());
|
||||||
|
out.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
FileStatus[] statuses = fs.listStatus(testDir);
|
||||||
|
assertNotNull("List should not be null", statuses);
|
||||||
|
assertEquals("Should have 3 files", 3, statuses.length);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testRename() throws Exception {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
Path srcFile = new Path(TEST_ROOT + "/source.txt");
|
||||||
|
Path dstFile = new Path(TEST_ROOT + "/destination.txt");
|
||||||
|
String content = "rename test";
|
||||||
|
|
||||||
|
// Create source file
|
||||||
|
FSDataOutputStream out = fs.create(srcFile);
|
||||||
|
out.write(content.getBytes());
|
||||||
|
out.close();
|
||||||
|
|
||||||
|
assertTrue("Source file should exist", fs.exists(srcFile));
|
||||||
|
|
||||||
|
// Rename
|
||||||
|
assertTrue("Rename should succeed", fs.rename(srcFile, dstFile));
|
||||||
|
|
||||||
|
// Verify
|
||||||
|
assertFalse("Source file should not exist", fs.exists(srcFile));
|
||||||
|
assertTrue("Destination file should exist", fs.exists(dstFile));
|
||||||
|
|
||||||
|
// Verify content preserved
|
||||||
|
FSDataInputStream in = fs.open(dstFile);
|
||||||
|
byte[] buffer = new byte[content.length()];
|
||||||
|
in.read(buffer);
|
||||||
|
in.close();
|
||||||
|
assertEquals("Content should be preserved", content, new String(buffer));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testDelete() throws Exception {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
Path testFile = new Path(TEST_ROOT + "/deletetest.txt");
|
||||||
|
|
||||||
|
// Create file
|
||||||
|
FSDataOutputStream out = fs.create(testFile);
|
||||||
|
out.write("delete me".getBytes());
|
||||||
|
out.close();
|
||||||
|
|
||||||
|
assertTrue("File should exist before delete", fs.exists(testFile));
|
||||||
|
|
||||||
|
// Delete
|
||||||
|
assertTrue("Delete should succeed", fs.delete(testFile, false));
|
||||||
|
assertFalse("File should not exist after delete", fs.exists(testFile));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testDeleteDirectory() throws Exception {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
Path testDir = new Path(TEST_ROOT + "/deletedir");
|
||||||
|
Path testFile = new Path(testDir, "file.txt");
|
||||||
|
|
||||||
|
// Create directory with file
|
||||||
|
fs.mkdirs(testDir);
|
||||||
|
FSDataOutputStream out = fs.create(testFile);
|
||||||
|
out.write("content".getBytes());
|
||||||
|
out.close();
|
||||||
|
|
||||||
|
assertTrue("Directory should exist", fs.exists(testDir));
|
||||||
|
assertTrue("File should exist", fs.exists(testFile));
|
||||||
|
|
||||||
|
// Recursive delete
|
||||||
|
assertTrue("Recursive delete should succeed", fs.delete(testDir, true));
|
||||||
|
assertFalse("Directory should not exist after delete", fs.exists(testDir));
|
||||||
|
assertFalse("File should not exist after delete", fs.exists(testFile));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testAppend() throws Exception {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
Path testFile = new Path(TEST_ROOT + "/appendtest.txt");
|
||||||
|
String initialContent = "initial";
|
||||||
|
String appendContent = " appended";
|
||||||
|
|
||||||
|
// Create initial file
|
||||||
|
FSDataOutputStream out = fs.create(testFile);
|
||||||
|
out.write(initialContent.getBytes());
|
||||||
|
out.close();
|
||||||
|
|
||||||
|
// Append
|
||||||
|
FSDataOutputStream appendOut = fs.append(testFile, 4096, null);
|
||||||
|
assertNotNull("Append stream should not be null", appendOut);
|
||||||
|
appendOut.write(appendContent.getBytes());
|
||||||
|
appendOut.close();
|
||||||
|
|
||||||
|
// Verify combined content
|
||||||
|
FSDataInputStream in = fs.open(testFile);
|
||||||
|
byte[] buffer = new byte[initialContent.length() + appendContent.length()];
|
||||||
|
int bytesRead = in.read(buffer);
|
||||||
|
in.close();
|
||||||
|
|
||||||
|
String expected = initialContent + appendContent;
|
||||||
|
assertEquals("Should read all bytes", expected.length(), bytesRead);
|
||||||
|
assertEquals("Content should match", expected, new String(buffer));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testSetWorkingDirectory() throws Exception {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
Path originalWd = fs.getWorkingDirectory();
|
||||||
|
assertEquals("Original working directory should be /", "/", originalWd.toUri().getPath());
|
||||||
|
|
||||||
|
Path newWd = new Path(TEST_ROOT);
|
||||||
|
fs.mkdirs(newWd);
|
||||||
|
fs.setWorkingDirectory(newWd);
|
||||||
|
|
||||||
|
Path currentWd = fs.getWorkingDirectory();
|
||||||
|
assertTrue("Working directory should be updated",
|
||||||
|
currentWd.toUri().getPath().contains(TEST_ROOT));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testSetPermission() throws Exception {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
Path testFile = new Path(TEST_ROOT + "/permtest.txt");
|
||||||
|
|
||||||
|
// Create file
|
||||||
|
FSDataOutputStream out = fs.create(testFile);
|
||||||
|
out.write("permission test".getBytes());
|
||||||
|
out.close();
|
||||||
|
|
||||||
|
// Set permission
|
||||||
|
FsPermission newPerm = new FsPermission((short) 0644);
|
||||||
|
fs.setPermission(testFile, newPerm);
|
||||||
|
|
||||||
|
FileStatus status = fs.getFileStatus(testFile);
|
||||||
|
assertNotNull("Permission should not be null", status.getPermission());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testSetOwner() throws Exception {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
Path testFile = new Path(TEST_ROOT + "/ownertest.txt");
|
||||||
|
|
||||||
|
// Create file
|
||||||
|
FSDataOutputStream out = fs.create(testFile);
|
||||||
|
out.write("owner test".getBytes());
|
||||||
|
out.close();
|
||||||
|
|
||||||
|
// Set owner - this may not fail even if not fully implemented
|
||||||
|
fs.setOwner(testFile, "testuser", "testgroup");
|
||||||
|
|
||||||
|
// Just verify the call doesn't throw an exception
|
||||||
|
FileStatus status = fs.getFileStatus(testFile);
|
||||||
|
assertNotNull("FileStatus should not be null", status);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testRenameToExistingDirectory() throws Exception {
|
||||||
|
if (!TESTS_ENABLED) {
|
||||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
Path srcFile = new Path(TEST_ROOT + "/movefile.txt");
|
||||||
|
Path dstDir = new Path(TEST_ROOT + "/movedir");
|
||||||
|
|
||||||
|
// Create source file and destination directory
|
||||||
|
FSDataOutputStream out = fs.create(srcFile);
|
||||||
|
out.write("move test".getBytes());
|
||||||
|
out.close();
|
||||||
|
fs.mkdirs(dstDir);
|
||||||
|
|
||||||
|
// Rename file to existing directory (should move file into directory)
|
||||||
|
assertTrue("Rename to directory should succeed", fs.rename(srcFile, dstDir));
|
||||||
|
|
||||||
|
// File should be moved into the directory
|
||||||
|
Path expectedLocation = new Path(dstDir, srcFile.getName());
|
||||||
|
assertTrue("File should exist in destination directory", fs.exists(expectedLocation));
|
||||||
|
assertFalse("Source file should not exist", fs.exists(srcFile));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
Reference in New Issue
Block a user