From 43cbec8f11a2750693ba6128f6a8808da13ba318 Mon Sep 17 00:00:00 2001 From: G-XD Date: Tue, 19 Sep 2023 00:58:53 +0800 Subject: [PATCH 01/24] test(binding/java): add behavior test --- .../org/apache/opendal/BaseOperatorTest.java | 109 +++++++++++++++ .../apache/opendal/services/RedisTest.java | 28 ++++ .../org/apache/opendal/services/S3Test.java | 17 +++ .../java/org/apache/opendal/utils/Utils.java | 126 ++++++++++++++++++ 4 files changed, 280 insertions(+) create mode 100644 bindings/java/src/test/java/org/apache/opendal/BaseOperatorTest.java create mode 100644 bindings/java/src/test/java/org/apache/opendal/services/RedisTest.java create mode 100644 bindings/java/src/test/java/org/apache/opendal/services/S3Test.java create mode 100644 bindings/java/src/test/java/org/apache/opendal/utils/Utils.java diff --git a/bindings/java/src/test/java/org/apache/opendal/BaseOperatorTest.java b/bindings/java/src/test/java/org/apache/opendal/BaseOperatorTest.java new file mode 100644 index 000000000000..c6e82b6728d0 --- /dev/null +++ b/bindings/java/src/test/java/org/apache/opendal/BaseOperatorTest.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.opendal; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import java.util.Arrays; +import java.util.UUID; +import java.util.stream.Collectors; +import org.apache.opendal.condition.OpenDALExceptionCondition; +import org.apache.opendal.utils.Utils; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public abstract class BaseOperatorTest { + + protected Operator op; + + protected abstract void initOp(); + + @BeforeAll + public void init() { + this.initOp(); + } + + @AfterAll + public void destroy() { + if (op != null) { + op.close(); + } + } + + @Test + public final void testWrite() throws Exception { + String path = UUID.randomUUID().toString(); + byte[] content = Utils.generateBytes(); + op.write(path, content).join(); + + Metadata metadata = op.stat(path).get(); + + assertEquals(content.length, metadata.getContentLength()); + + op.delete(path).join(); + assertThatThrownBy(() -> op.stat(path).join()) + .is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.NotFound)); + } + + @Test + public final void testRead() throws Exception { + Metadata metadata = op.stat("").get(); + assertTrue(!metadata.isFile()); + + String path = UUID.randomUUID().toString(); + assertThatThrownBy(() -> op.stat(path).join()) + .is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.NotFound)); + + String content = Utils.generateRandomString(); + op.write(path, content).join(); + + assertThat(op.read(path).join()).isEqualTo(content); + + op.delete(path).join(); + assertThatThrownBy(() -> op.stat(path).join()) + .is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.NotFound)); + } + + @Test + public void testAppend() { + String[] trunks = new String[] {"first trunk", "second trunk", "third trunk"}; + + for (int i = 0; i < trunks.length; i++) { + op.append("testAppendManyTimes", trunks[i]).join(); + String expected = Arrays.stream(trunks).limit(i + 1).collect(Collectors.joining()); + assertThat(op.read("testAppendManyTimes").join()).isEqualTo(expected); + } + + // write overwrite existing content + op.write("testAppendManyTimes", "new attempt").join(); + assertThat(op.read("testAppendManyTimes1").join()).isEqualTo("new attempt"); + + for (int i = 0; i < trunks.length; i++) { + op.append("testAppendManyTimes", trunks[i]).join(); + String expected = Arrays.stream(trunks).limit(i + 1).collect(Collectors.joining()); + assertThat(op.read("testAppendManyTimes").join()).isEqualTo("new attempt" + expected); + } + } +} diff --git a/bindings/java/src/test/java/org/apache/opendal/services/RedisTest.java b/bindings/java/src/test/java/org/apache/opendal/services/RedisTest.java new file mode 100644 index 000000000000..58b657dc42f8 --- /dev/null +++ b/bindings/java/src/test/java/org/apache/opendal/services/RedisTest.java @@ -0,0 +1,28 @@ +package org.apache.opendal.services; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.Assert.assertTrue; +import java.util.Optional; +import org.apache.opendal.BaseOperatorTest; +import org.apache.opendal.OpenDALException; +import org.apache.opendal.Operator; +import org.apache.opendal.condition.OpenDALExceptionCondition; +import org.apache.opendal.utils.Utils; +import org.junit.jupiter.api.Test; + +public class RedisTest extends BaseOperatorTest { + + @Override + public void initOp() { + Optional optional = Utils.init("redis"); + assertTrue(optional.isPresent()); + op = optional.get(); + } + + @Test + @Override + public void testAppend() { + assertThatThrownBy(() -> super.testAppend()) + .is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.Unsupported)); + } +} diff --git a/bindings/java/src/test/java/org/apache/opendal/services/S3Test.java b/bindings/java/src/test/java/org/apache/opendal/services/S3Test.java new file mode 100644 index 000000000000..a4fe3b73d5d7 --- /dev/null +++ b/bindings/java/src/test/java/org/apache/opendal/services/S3Test.java @@ -0,0 +1,17 @@ +package org.apache.opendal.services; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import java.util.Optional; +import org.apache.opendal.BaseOperatorTest; +import org.apache.opendal.Operator; +import org.apache.opendal.utils.Utils; + +public class S3Test extends BaseOperatorTest { + + @Override + public void initOp() { + Optional optional = Utils.init("S3"); + assertTrue(optional.isPresent()); + op = optional.get(); + } +} diff --git a/bindings/java/src/test/java/org/apache/opendal/utils/Utils.java b/bindings/java/src/test/java/org/apache/opendal/utils/Utils.java new file mode 100644 index 000000000000..b38aa84bba19 --- /dev/null +++ b/bindings/java/src/test/java/org/apache/opendal/utils/Utils.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.opendal.utils; + +import java.io.File; +import java.io.InputStream; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Properties; +import java.util.Random; +import java.util.UUID; +import java.util.stream.Collectors; +import org.apache.opendal.Operator; + +public class Utils { + + public static final String ENV_NAME = ".env"; + public static final String CONF_PREFIX = "opendal_"; + public static final String CONF_TURN_ON_TEST = "test"; + public static final String CONF_ROOT = "root"; + public static final String CONF_RANDOM_ROOT_FLAG = "OPENDAL_DISABLE_RANDOM_ROOT"; + + /** + * Initializes the Service with the given schema. + * + * @param schema the schema to initialize the Operator service + * @return If `opendal_{schema}_test` is on, construct a new Operator with given root. + * Else, returns a `Empty` to represent no valid config for operator. + */ + public static Optional init(String schema) { + final Properties properties = new Properties(); + + try (InputStream is = Utils.class.getClassLoader().getResourceAsStream(ENV_NAME)) { + properties.load(is); + } catch (Exception ignore) { + } + for (Map.Entry entry : System.getenv().entrySet()) { + properties.setProperty(entry.getKey(), entry.getValue()); + } + + final String confPrefix = (CONF_PREFIX + schema).toLowerCase(); + final Map conf = properties.entrySet().stream() + .filter(Objects::nonNull) + .filter(entry -> Optional.ofNullable(entry.getKey()) + .map(Object::toString) + .orElse("") + .toLowerCase() + .startsWith(confPrefix)) + .collect(Collectors.toMap( + entry -> { + String key = entry.getKey().toString().toLowerCase(); + return key.replace(confPrefix + "_", ""); + }, + entry -> Optional.ofNullable(entry.getValue()) + .map(Object::toString) + .orElse(""), + (existing, replacement) -> existing)); + + final String turnOnTest = conf.get(CONF_TURN_ON_TEST); + if (!isTurnOn(turnOnTest)) { + return Optional.empty(); + } + if (!Boolean.parseBoolean(properties.getProperty(CONF_RANDOM_ROOT_FLAG))) { + String root = conf.getOrDefault(CONF_ROOT, File.separator); + if (!root.endsWith(File.separator)) { + root = root + File.separator; + } + root = root + UUID.randomUUID() + File.separator; + conf.put(CONF_ROOT, root); + } + Operator op = new Operator(schema, conf); + return Optional.of(op); + } + + /** + * Determines if the given value is turn on. + * + * @param val the value to be checked + * @return true if the value is "on" or "true", false otherwise + */ + public static boolean isTurnOn(String val) { + return "on".equalsIgnoreCase(val) || "true".equalsIgnoreCase(val); + } + + public static byte[] generateBytes() { + Random random = new Random(); + + int size = random.nextInt(4 * 1024 * 1024) + 1; + byte[] content = new byte[size]; + random.nextBytes(content); + + return content; + } + + public static String generateRandomString() { + Random random = new Random(); + + int length = random.nextInt(256) + 1; + StringBuilder stringBuilder = new StringBuilder(); + + for (int i = 0; i < length; i++) { + int randomChar = random.nextInt(26); + stringBuilder.append((char) ('a' + randomChar)); + } + + return stringBuilder.toString(); + } +} From c51bbabc85cc0b27145c2b69db4914a98b30e00b Mon Sep 17 00:00:00 2001 From: G-XD Date: Tue, 19 Sep 2023 01:19:27 +0800 Subject: [PATCH 02/24] test(binding/java): override append test for s3 --- .../org/apache/opendal/services/RedisTest.java | 12 ++---------- .../java/org/apache/opendal/services/S3Test.java | 15 ++++++++++++++- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/bindings/java/src/test/java/org/apache/opendal/services/RedisTest.java b/bindings/java/src/test/java/org/apache/opendal/services/RedisTest.java index 58b657dc42f8..7240af169d93 100644 --- a/bindings/java/src/test/java/org/apache/opendal/services/RedisTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/services/RedisTest.java @@ -1,14 +1,12 @@ package org.apache.opendal.services; -import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.Assert.assertTrue; + import java.util.Optional; + import org.apache.opendal.BaseOperatorTest; -import org.apache.opendal.OpenDALException; import org.apache.opendal.Operator; -import org.apache.opendal.condition.OpenDALExceptionCondition; import org.apache.opendal.utils.Utils; -import org.junit.jupiter.api.Test; public class RedisTest extends BaseOperatorTest { @@ -19,10 +17,4 @@ public void initOp() { op = optional.get(); } - @Test - @Override - public void testAppend() { - assertThatThrownBy(() -> super.testAppend()) - .is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.Unsupported)); - } } diff --git a/bindings/java/src/test/java/org/apache/opendal/services/S3Test.java b/bindings/java/src/test/java/org/apache/opendal/services/S3Test.java index a4fe3b73d5d7..4eb058b646f3 100644 --- a/bindings/java/src/test/java/org/apache/opendal/services/S3Test.java +++ b/bindings/java/src/test/java/org/apache/opendal/services/S3Test.java @@ -1,17 +1,30 @@ package org.apache.opendal.services; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertTrue; + import java.util.Optional; + import org.apache.opendal.BaseOperatorTest; +import org.apache.opendal.OpenDALException; import org.apache.opendal.Operator; +import org.apache.opendal.condition.OpenDALExceptionCondition; import org.apache.opendal.utils.Utils; +import org.junit.jupiter.api.Test; public class S3Test extends BaseOperatorTest { @Override public void initOp() { - Optional optional = Utils.init("S3"); + Optional optional = Utils.init("s3"); assertTrue(optional.isPresent()); op = optional.get(); } + + @Test + @Override + public void testAppend() { + assertThatThrownBy(() -> super.testAppend()) + .is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.Unsupported)); + } } From a878d0d90d19a755447c37ae31952afc9bcffe44 Mon Sep 17 00:00:00 2001 From: G-XD Date: Tue, 19 Sep 2023 01:20:22 +0800 Subject: [PATCH 03/24] ci(services/redis): add behavior test for binding java --- .github/workflows/service_test_redis.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/service_test_redis.yml b/.github/workflows/service_test_redis.yml index 85e6b48ef369..fdfad34f77b1 100644 --- a/.github/workflows/service_test_redis.yml +++ b/.github/workflows/service_test_redis.yml @@ -192,3 +192,21 @@ jobs: OPENDAL_REDIS_ENDPOINT: tcp://127.0.0.1:6379 OPENDAL_REDIS_ROOT: / OPENDAL_REDIS_DB: 0 + + binding_java_redis: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Setup Rust toolchain + uses: ./.github/actions/setup + + - name: Test + shell: bash + working-directory: bindings/java + run: ./mvnw test -Dcargo-build.features=services-redis -Dtest=org.apache.opendal.services.RedisTest + env: + OPENDAL_REDIS_TEST: on + OPENDAL_REDIS_ENDPOINT: tcp://127.0.0.1:6379 + OPENDAL_REDIS_ROOT: / + OPENDAL_REDIS_DB: 0 From e72e7423802e8ad268a66f4c3e26d48805a03483 Mon Sep 17 00:00:00 2001 From: G-XD Date: Tue, 19 Sep 2023 01:20:34 +0800 Subject: [PATCH 04/24] ci(services/s3): add behavior test for binding java --- .github/workflows/service_test_s3.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.github/workflows/service_test_s3.yml b/.github/workflows/service_test_s3.yml index b9c44f1dc91b..10ae271398d7 100644 --- a/.github/workflows/service_test_s3.yml +++ b/.github/workflows/service_test_s3.yml @@ -210,3 +210,24 @@ jobs: # This is the R2's limitation # Refer to https://opendal.apache.org/docs/services/s3#compatible-services for more information OPENDAL_S3_ENABLE_EXACT_BUF_WRITE: true + + binding_java_s3: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Setup Rust toolchain + uses: ./.github/actions/setup + + - name: Test + shell: bash + working-directory: bindings/java + run: ./mvnw test -Dtest=org.apache.opendal.services.S3Test + env: + OPENDAL_S3_TEST: ${{ secrets.OPENDAL_S3_TEST }} + OPENDAL_S3_ROOT: ${{ secrets.OPENDAL_S3_ROOT }} + OPENDAL_S3_BUCKET: ${{ secrets.OPENDAL_S3_BUCKET }} + OPENDAL_S3_ENDPOINT: ${{ secrets.OPENDAL_S3_ENDPOINT }} + OPENDAL_S3_ACCESS_KEY_ID: ${{ secrets.OPENDAL_S3_ACCESS_KEY_ID }} + OPENDAL_S3_SECRET_ACCESS_KEY: ${{ secrets.OPENDAL_S3_SECRET_ACCESS_KEY }} + OPENDAL_S3_REGION: ap-northeast-1 From 2a165e4254dfdbfe523522a373a41b030bc1c966 Mon Sep 17 00:00:00 2001 From: G-XD Date: Tue, 19 Sep 2023 01:21:29 +0800 Subject: [PATCH 05/24] test(binding/java): del redis test --- .../org/apache/opendal/RedisServiceTest.java | 74 ------------------- 1 file changed, 74 deletions(-) delete mode 100644 bindings/java/src/test/java/org/apache/opendal/RedisServiceTest.java diff --git a/bindings/java/src/test/java/org/apache/opendal/RedisServiceTest.java b/bindings/java/src/test/java/org/apache/opendal/RedisServiceTest.java deleted file mode 100644 index a344f2c93f68..000000000000 --- a/bindings/java/src/test/java/org/apache/opendal/RedisServiceTest.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.opendal; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatExceptionOfType; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import java.util.HashMap; -import java.util.Map; -import lombok.Cleanup; -import org.apache.opendal.condition.OpenDALExceptionCondition; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.condition.DisabledIfEnvironmentVariable; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.junit.jupiter.Container; -import org.testcontainers.junit.jupiter.Testcontainers; - -@Testcontainers -@DisabledIfEnvironmentVariable(named = "NO_DOCKER", matches = "true") -public class RedisServiceTest { - @Container - private final GenericContainer redisContainer = new GenericContainer<>("redis:7.2.1").withExposedPorts(6379); - - @Test - public void testAccessRedisService() { - assertThat(redisContainer.isRunning()).isTrue(); - - final Map params = new HashMap<>(); - params.put("root", "/tmp"); - params.put("endpoint", "tcp://127.0.0.1:" + redisContainer.getMappedPort(6379)); - @Cleanup final Operator op = new Operator("Redis", params); - - op.write("testAccessRedisService", "Odin").join(); - assertThat(op.read("testAccessRedisService").join()).isEqualTo("Odin"); - op.delete("testAccessRedisService").join(); - assertThatThrownBy(() -> op.stat("testAccessRedisService").join()) - .is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.NotFound)); - } - - @Test - public void testAccessRedisServiceBlocking() { - assertThat(redisContainer.isRunning()).isTrue(); - - final Map params = new HashMap<>(); - params.put("root", "/tmp"); - params.put("endpoint", "tcp://127.0.0.1:" + redisContainer.getMappedPort(6379)); - @Cleanup final BlockingOperator op = new BlockingOperator("Redis", params); - - op.write("testAccessRedisServiceBlocking", "Odin"); - assertThat(op.read("testAccessRedisServiceBlocking")).isEqualTo("Odin"); - op.delete("testAccessRedisServiceBlocking"); - assertThatExceptionOfType(OpenDALException.class) - .isThrownBy(() -> op.stat("testAccessRedisServiceBlocking")) - .extracting(OpenDALException::getCode) - .isEqualTo(OpenDALException.Code.NotFound); - } -} From 396c019500b762c3c724cf5a6398e7e2cff12d15 Mon Sep 17 00:00:00 2001 From: G-XD Date: Tue, 19 Sep 2023 01:48:37 +0800 Subject: [PATCH 06/24] test(binding/java): add behavior test for memory & fs --- .../org/apache/opendal/BaseOperatorTest.java | 10 +- .../java/org/apache/opendal/OperatorTest.java | 75 -------- .../org/apache/opendal/services/FsTest.java | 12 ++ .../apache/opendal/services/MemoryTest.java | 23 +++ .../apache/opendal/services/RedisTest.java | 22 ++- .../org/apache/opendal/services/S3Test.java | 11 +- bindings/java/src/test/resources/.env.example | 166 ++++++++++++++++++ 7 files changed, 223 insertions(+), 96 deletions(-) delete mode 100644 bindings/java/src/test/java/org/apache/opendal/OperatorTest.java create mode 100644 bindings/java/src/test/java/org/apache/opendal/services/FsTest.java create mode 100644 bindings/java/src/test/java/org/apache/opendal/services/MemoryTest.java create mode 100644 bindings/java/src/test/resources/.env.example diff --git a/bindings/java/src/test/java/org/apache/opendal/BaseOperatorTest.java b/bindings/java/src/test/java/org/apache/opendal/BaseOperatorTest.java index c6e82b6728d0..c479408b45c3 100644 --- a/bindings/java/src/test/java/org/apache/opendal/BaseOperatorTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/BaseOperatorTest.java @@ -24,6 +24,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.Arrays; +import java.util.Optional; import java.util.UUID; import java.util.stream.Collectors; import org.apache.opendal.condition.OpenDALExceptionCondition; @@ -38,11 +39,14 @@ public abstract class BaseOperatorTest { protected Operator op; - protected abstract void initOp(); + protected abstract String schema(); @BeforeAll public void init() { - this.initOp(); + String schema = this.schema();; + Optional optional = Utils.init(schema); + assertTrue(optional.isPresent()); + op = optional.get(); } @AfterAll @@ -98,7 +102,7 @@ public void testAppend() { // write overwrite existing content op.write("testAppendManyTimes", "new attempt").join(); - assertThat(op.read("testAppendManyTimes1").join()).isEqualTo("new attempt"); + assertThat(op.read("testAppendManyTimes").join()).isEqualTo("new attempt"); for (int i = 0; i < trunks.length; i++) { op.append("testAppendManyTimes", trunks[i]).join(); diff --git a/bindings/java/src/test/java/org/apache/opendal/OperatorTest.java b/bindings/java/src/test/java/org/apache/opendal/OperatorTest.java deleted file mode 100644 index bcde1c1810e7..000000000000 --- a/bindings/java/src/test/java/org/apache/opendal/OperatorTest.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.opendal; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import java.nio.file.Path; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.stream.Collectors; -import lombok.Cleanup; -import org.apache.opendal.condition.OpenDALExceptionCondition; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -public class OperatorTest { - @TempDir - private static Path tempDir; - - @Test - public void testCreateAndDelete() { - Map params = new HashMap<>(); - params.put("root", "/tmp"); - @Cleanup Operator op = new Operator("Memory", params); - - op.write("testCreateAndDelete", "Odin").join(); - assertThat(op.read("testCreateAndDelete").join()).isEqualTo("Odin"); - op.delete("testCreateAndDelete").join(); - assertThatThrownBy(() -> op.stat("testCreateAndDelete").join()) - .is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.NotFound)); - } - - @Test - public void testAppendManyTimes() { - Map params = new HashMap<>(); - params.put("root", tempDir.toString()); - @Cleanup Operator op = new Operator("fs", params); - - String[] trunks = new String[] {"first trunk", "second trunk", "third trunk"}; - - for (int i = 0; i < trunks.length; i++) { - op.append("testAppendManyTimes", trunks[i]).join(); - String expected = Arrays.stream(trunks).limit(i + 1).collect(Collectors.joining()); - assertThat(op.read("testAppendManyTimes").join()).isEqualTo(expected); - } - - // write overwrite existing content - op.write("testAppendManyTimes", "new attempt").join(); - assertThat(op.read("testAppendManyTimes").join()).isEqualTo("new attempt"); - - for (int i = 0; i < trunks.length; i++) { - op.append("testAppendManyTimes", trunks[i]).join(); - String expected = Arrays.stream(trunks).limit(i + 1).collect(Collectors.joining()); - assertThat(op.read("testAppendManyTimes").join()).isEqualTo("new attempt" + expected); - } - } -} diff --git a/bindings/java/src/test/java/org/apache/opendal/services/FsTest.java b/bindings/java/src/test/java/org/apache/opendal/services/FsTest.java new file mode 100644 index 000000000000..059ffdc088b2 --- /dev/null +++ b/bindings/java/src/test/java/org/apache/opendal/services/FsTest.java @@ -0,0 +1,12 @@ +package org.apache.opendal.services; + +import org.apache.opendal.BaseOperatorTest; + +public class FsTest extends BaseOperatorTest { + + @Override + public String schema() { + return "fs"; + } + +} diff --git a/bindings/java/src/test/java/org/apache/opendal/services/MemoryTest.java b/bindings/java/src/test/java/org/apache/opendal/services/MemoryTest.java new file mode 100644 index 000000000000..b7195e17bc5b --- /dev/null +++ b/bindings/java/src/test/java/org/apache/opendal/services/MemoryTest.java @@ -0,0 +1,23 @@ +package org.apache.opendal.services; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import org.apache.opendal.BaseOperatorTest; +import org.apache.opendal.OpenDALException; +import org.apache.opendal.condition.OpenDALExceptionCondition; +import org.junit.jupiter.api.Test; + +public class MemoryTest extends BaseOperatorTest { + + @Override + public String schema() { + return "Memory"; + } + + @Test + @Override + public void testAppend() { + assertThatThrownBy(() -> super.testAppend()) + .is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.Unsupported)); + } +} diff --git a/bindings/java/src/test/java/org/apache/opendal/services/RedisTest.java b/bindings/java/src/test/java/org/apache/opendal/services/RedisTest.java index 7240af169d93..7366112bbbe3 100644 --- a/bindings/java/src/test/java/org/apache/opendal/services/RedisTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/services/RedisTest.java @@ -1,20 +1,24 @@ package org.apache.opendal.services; -import static org.junit.Assert.assertTrue; - -import java.util.Optional; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import org.apache.opendal.BaseOperatorTest; -import org.apache.opendal.Operator; -import org.apache.opendal.utils.Utils; +import org.apache.opendal.OpenDALException; +import org.apache.opendal.condition.OpenDALExceptionCondition; +import org.junit.jupiter.api.Test; public class RedisTest extends BaseOperatorTest { @Override - public void initOp() { - Optional optional = Utils.init("redis"); - assertTrue(optional.isPresent()); - op = optional.get(); + public String schema() { + return "redis"; + } + + @Test + @Override + public void testAppend() { + assertThatThrownBy(() -> super.testAppend()) + .is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.Unsupported)); } } diff --git a/bindings/java/src/test/java/org/apache/opendal/services/S3Test.java b/bindings/java/src/test/java/org/apache/opendal/services/S3Test.java index 4eb058b646f3..0be0e83b192d 100644 --- a/bindings/java/src/test/java/org/apache/opendal/services/S3Test.java +++ b/bindings/java/src/test/java/org/apache/opendal/services/S3Test.java @@ -1,24 +1,17 @@ package org.apache.opendal.services; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.util.Optional; import org.apache.opendal.BaseOperatorTest; import org.apache.opendal.OpenDALException; -import org.apache.opendal.Operator; import org.apache.opendal.condition.OpenDALExceptionCondition; -import org.apache.opendal.utils.Utils; import org.junit.jupiter.api.Test; public class S3Test extends BaseOperatorTest { @Override - public void initOp() { - Optional optional = Utils.init("s3"); - assertTrue(optional.isPresent()); - op = optional.get(); + public String schema() { + return "s3"; } @Test diff --git a/bindings/java/src/test/resources/.env.example b/bindings/java/src/test/resources/.env.example new file mode 100644 index 000000000000..2a5b0344876b --- /dev/null +++ b/bindings/java/src/test/resources/.env.example @@ -0,0 +1,166 @@ +# memory +OPENDAL_MEMORY_TEST=on +# fs +OPENDAL_FS_TEST=false +OPENDAL_FS_ROOT=/path/to/dir +OPENDAL_FS_ATOMIC_WRITE_DIR=/path/to/tempdir +# cos +OPENDAL_COS_TEST=false +OPENDAL_COS_BUCKET=opendal-testing-1318209832 +OPENDAL_COS_ENDPOINT=https://cos.ap-singapore.myqcloud.com +OPENDAL_COS_SECRET_ID= +OPENDAL_COS_SECRET_KEY= +# s3 +OPENDAL_S3_TEST=false +OPENDAL_S3_BUCKET= +OPENDAL_S3_ENDPOINT= +OPENDAL_S3_REGION= +OPENDAL_S3_ACCESS_KEY_ID= +OPENDAL_S3_SECRET_ACCESS_KEY= +# azblob +OPENDAL_AZBLOB_TEST=false +OPENDAL_AZBLOB_ROOT=/path/to/dir +OPENDAL_AZBLOB_CONTAINER= +OPENDAL_AZBLOB_ENDPOINT= +OPENDAL_AZBLOB_ACCOUNT_NAME= +OPENDAL_AZBLOB_ACCOUNT_KEY= +# hdfs +OPENDAL_HDFS_TEST=false +OPENDAL_HDFS_ROOT=/path/to/dir +OPENDAL_HDFS_NAME_NODE= +# gcs +OPENDAL_GCS_TEST=false +OPENDAL_GCS_ROOT=/path/to/dir +OPENDAL_GCS_BUCKET= +OPENDAL_GCS_CREDENTIAL= +# obs +OPENDAL_OBS_TEST=false +OPENDAL_OBS_BUCKET= +OPENDAL_OBS_ENDPOINT= +OPENDAL_OBS_ACCESS_KEY_ID= +OPENDAL_OBS_SECRET_ACCESS_KEY= +# oss +OPENDAL_OSS_TEST=false +OPENDAL_OSS_BUCKET= +OPENDAL_OSS_ENDPOINT= +OPENDAL_OSS_ACCESS_KEY_ID= +OPENDAL_OSS_ACCESS_KEY_SECRET= +# ipmfs +OPENDAL_IPMFS_TEST=false +OPENDAL_IPMFS_ROOT=/path/to/dir +OPENDAL_IPMFS_ENDPOINT=http://localhost:5001 +# ftp +OPENDAL_FTP_TEST=false +OPENDAL_FTP_ENDPOINT=ftp:// +OPENDAL_FTP_ROOT=/path/to/dir +OPENDAL_FTP_USER= +OPENDAL_FTP_PASSWORD= +# ipfs +OPENDAL_IPFS_TEST=false +OPENDAL_IPFS_ROOT=/ipfs/Qmxxxxxxxx +OPENDAL_IPFS_ENDPOINT=http://localhost:8080 +# redis +OPENDAL_REDIS_TEST=false +OPENDAL_REDIS_ENDPOINT=tcp://127.0.0.1:6379 +# OPENDAL_REDIS_CLUSTER_ENDPOINTS=rediss://127.0.0.1:6380,rediss://127.0.0.1:6381,rediss://127.0.0.1:6382,rediss://127.0.0.1:6383,rediss://127.0.0.1:6384,rediss://127.0.0.1:6385 +OPENDAL_REDIS_ROOT=/ +OPENDAL_REDIS_DB=0 +# rocksdb +OPENDAL_ROCKSDB_TEST=false +OPENDAL_ROCKSDB_DATADIR=/path/to/database +OPENDAL_ROCKSDB_ROOT=/path/to/root +# sftp +OPENDAL_SFTP_TEST=false +OPENDAL_SFTP_ENDPOINT=ssh:// +OPENDAL_SFTP_ROOT=/path/to/dir +OPENDAL_SFTP_USER= +OPENDAL_SFTP_KEY= +OPENDAL_SFTP_KNOWN_HOSTS_STRATEGY= +# sled +OPENDAL_SLED_TEST=false +OPENDAL_SLED_DATADIR=/path/to/database +OPENDAL_SLED_TREE=sled-tree +# mini-moka +OPENDAL_MINI_MOKA_TEST=false +# moka +OPENDAL_MOKA_TEST=false +# ghac +OPENDAL_GHAC_TEST=false +# memcached +OPENDAL_MEMCACHED_TEST=false +OPENDAL_MEMCACHED_ENDPOINT=tcp://127.0.0.1:11211 +OPENDAL_MEMCACHED_ROOT=/ +# webdav +OPENDAL_WEBDAV_TEST=false +OPENDAL_WEBDAV_ROOT=/tmp/opendal/ +OPENDAL_WEBDAV_ENDPOINT=http://127.0.0.1:8080 +# webhfds +OPENDAL_WEBHDFS_TEST=false +OPENDAL_WEBHDFS_ROOT=/tmp/opendal/ +OPENDAL_WEBHDFS_ENDPOINT=http://127.0.0.1:9870 +OPENDAL_WEBHDFS_DELEGATION= +OPENDAL_WEBHDFS_DISABLE_LIST_BATCH=false +# supbase +OPENDAL_SUPABASE_TEST=false +OPENDAL_SUPABASE_BUCKET= +OPENDAL_SUPABASE_ENDPOINT= +OPENDAL_SUPABASE_KEY= +# vercel artifacts +OPENDAL_VERCEL_ARTIFACTS_TEST=false +OPENDAL_VERCEL_ARTIFACTS_ACCESS_TOKEN= +# onedrive +OPENDAL_ONEDRIVE_TEST=false +OPENDAL_ONEDRIVE_ACCESS_TOKEN= +# wasabi +OPENDAL_WASABI_TEST=false +OPENDAL_WASABI_REGION= +OPENDAL_WASABI_BUCKET= +OPENDAL_WASABI_ENDPOINT= +OPENDAL_WASABI_ACCESS_KEY_ID= +OPENDAL_WASABI_SECRET_ACCESS_KEY= +# atomicserver +OPENDAL_ATOMICSERVER_TEST=false +OPENDAL_ATOMICSERVER_ROOT=/path/to/dir +OPENDAL_ATOMICSERVER_ENDPOINT=http://localhost:9883 +OPENDAL_ATOMICSERVER_PRIVATE_KEY= +OPENDAL_ATOMICSERVER_PUBLIC_KEY= +OPENDAL_ATOMICSERVER_PARENT_RESOURCE_ID=http://localhost:9883 +# foundationdb +OPENDAL_FOUNDATIONDB_TEST=false +OPENDAL_FOUNDATIONDB_ROOT=/path/to/dir +OPENDAL_FOUNDATIONDB_CONFIG_PATH=/tmp/opendal/foundationdb.conf +# redb +OPENDAL_REDB_TEST=false +OPENDAL_REDB_DATADIR=/tmp/redb +OPENDAL_REDB_TABLE=redb-table +# cacache +OPENDAL_CACACHE_TEST=false +OPENDAL_CACACHE_DATADIR=/tmp/opendal/cacache/ +# persy +OPENDAL_PERSY_TEST=false +OPENDAL_PERSY_DATAFILE=/tmp/opendal/test.persy +OPENDAL_PERSY_SEGMENT=data +OPENDAL_PERSY_INDEX=index +#dropbox +OPENDAL_DROPBOX_TEST=false +OPENDAL_DROPBOX_ROOT=/tmp/opendal/ +OPENDAL_DROPBOX_ACCESS_TOKEN= +OPENDAL_DROPBOX_REFRESH_TOKEN= +OPENDAL_DROPBOX_CLIENT_ID= +OPENDAL_DROPBOX_CLIENT_SECRET= +# etcd +OPENDAL_ETCD_TEST=false +OPENDAL_ETCD_ENDPOINTS=127.0.0.1:2379 +OPENDAL_ETCD_ROOT=/tmp/opendal/ +OPENDAL_ETCD_USERNAME= +OPENDAL_ETCD_PASSWORD= +OPENDAL_ETCD_CA_PATH= +OPENDAL_ETCD_CERT_PATH= +OPENDAL_ETCD_KEY_PATH= +# google drive +OPENDAL_GDRIVE_TEST=false +OPENDAL_GDRIVE_ROOT=/tmp/opendal/ +OPENDAL_GDRIVE_ACCESS_TOKEN= +OPENDAL_GDRIVE_REFRESH_TOKEN= +OPENDAL_GDRIVE_CLIENT_ID= +OPENDAL_GDRIVE_CLIENT_SECRET= From 418d5e6a6484604e3b3019ac68bfd6bba571810c Mon Sep 17 00:00:00 2001 From: G-XD Date: Tue, 19 Sep 2023 01:51:38 +0800 Subject: [PATCH 07/24] style(binding/java): fix code style --- .../org/apache/opendal/BaseOperatorTest.java | 2 +- .../org/apache/opendal/services/FsTest.java | 20 ++++++++++++++++- .../apache/opendal/services/MemoryTest.java | 22 +++++++++++++++++-- .../apache/opendal/services/RedisTest.java | 21 ++++++++++++++++-- .../org/apache/opendal/services/S3Test.java | 20 ++++++++++++++++- 5 files changed, 78 insertions(+), 7 deletions(-) diff --git a/bindings/java/src/test/java/org/apache/opendal/BaseOperatorTest.java b/bindings/java/src/test/java/org/apache/opendal/BaseOperatorTest.java index c479408b45c3..24a12410bfd3 100644 --- a/bindings/java/src/test/java/org/apache/opendal/BaseOperatorTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/BaseOperatorTest.java @@ -43,7 +43,7 @@ public abstract class BaseOperatorTest { @BeforeAll public void init() { - String schema = this.schema();; + String schema = this.schema(); Optional optional = Utils.init(schema); assertTrue(optional.isPresent()); op = optional.get(); diff --git a/bindings/java/src/test/java/org/apache/opendal/services/FsTest.java b/bindings/java/src/test/java/org/apache/opendal/services/FsTest.java index 059ffdc088b2..53255e148433 100644 --- a/bindings/java/src/test/java/org/apache/opendal/services/FsTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/services/FsTest.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.opendal.services; import org.apache.opendal.BaseOperatorTest; @@ -8,5 +27,4 @@ public class FsTest extends BaseOperatorTest { public String schema() { return "fs"; } - } diff --git a/bindings/java/src/test/java/org/apache/opendal/services/MemoryTest.java b/bindings/java/src/test/java/org/apache/opendal/services/MemoryTest.java index b7195e17bc5b..713055bcf164 100644 --- a/bindings/java/src/test/java/org/apache/opendal/services/MemoryTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/services/MemoryTest.java @@ -1,7 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.opendal.services; import static org.assertj.core.api.Assertions.assertThatThrownBy; - import org.apache.opendal.BaseOperatorTest; import org.apache.opendal.OpenDALException; import org.apache.opendal.condition.OpenDALExceptionCondition; @@ -13,7 +31,7 @@ public class MemoryTest extends BaseOperatorTest { public String schema() { return "Memory"; } - + @Test @Override public void testAppend() { diff --git a/bindings/java/src/test/java/org/apache/opendal/services/RedisTest.java b/bindings/java/src/test/java/org/apache/opendal/services/RedisTest.java index 7366112bbbe3..a49186fc771c 100644 --- a/bindings/java/src/test/java/org/apache/opendal/services/RedisTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/services/RedisTest.java @@ -1,7 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.opendal.services; import static org.assertj.core.api.Assertions.assertThatThrownBy; - import org.apache.opendal.BaseOperatorTest; import org.apache.opendal.OpenDALException; import org.apache.opendal.condition.OpenDALExceptionCondition; @@ -20,5 +38,4 @@ public void testAppend() { assertThatThrownBy(() -> super.testAppend()) .is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.Unsupported)); } - } diff --git a/bindings/java/src/test/java/org/apache/opendal/services/S3Test.java b/bindings/java/src/test/java/org/apache/opendal/services/S3Test.java index 0be0e83b192d..05a475e6a50d 100644 --- a/bindings/java/src/test/java/org/apache/opendal/services/S3Test.java +++ b/bindings/java/src/test/java/org/apache/opendal/services/S3Test.java @@ -1,7 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.opendal.services; import static org.assertj.core.api.Assertions.assertThatThrownBy; - import org.apache.opendal.BaseOperatorTest; import org.apache.opendal.OpenDALException; import org.apache.opendal.condition.OpenDALExceptionCondition; From d5d8dd8ac898960e124a932800b27db0709a2e30 Mon Sep 17 00:00:00 2001 From: G-XD Date: Tue, 19 Sep 2023 02:01:57 +0800 Subject: [PATCH 08/24] test(binding/java): del cucumber test --- .../org/apache/opendal/AsyncStepsTest.java | 86 ------------------- .../java/org/apache/opendal/CucumberTest.java | 32 ------- .../java/org/apache/opendal/StepsTest.java | 70 --------------- 3 files changed, 188 deletions(-) delete mode 100644 bindings/java/src/test/java/org/apache/opendal/AsyncStepsTest.java delete mode 100644 bindings/java/src/test/java/org/apache/opendal/CucumberTest.java delete mode 100644 bindings/java/src/test/java/org/apache/opendal/StepsTest.java diff --git a/bindings/java/src/test/java/org/apache/opendal/AsyncStepsTest.java b/bindings/java/src/test/java/org/apache/opendal/AsyncStepsTest.java deleted file mode 100644 index 5955d6c3266e..000000000000 --- a/bindings/java/src/test/java/org/apache/opendal/AsyncStepsTest.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.opendal; - -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import io.cucumber.java.en.Given; -import io.cucumber.java.en.Then; -import io.cucumber.java.en.When; -import java.time.Duration; -import java.util.HashMap; -import java.util.Map; -import lombok.Cleanup; -import org.apache.opendal.condition.OpenDALExceptionCondition; - -public class AsyncStepsTest { - Operator op; - - @Given("A new OpenDAL Async Operator") - public void a_new_open_dal_async_operator() { - Map params = new HashMap<>(); - params.put("root", "/tmp"); - op = new Operator("Memory", params); - } - - @When("Async write path {string} with content {string}") - public void async_write_path_test_with_content_hello_world(String path, String content) { - op.write(path, content).join(); - } - - @Then("The async file {string} should exist") - public void the_async_file_test_should_exist(String path) { - @Cleanup Metadata metadata = op.stat(path).join(); - assertNotNull(metadata); - } - - @Then("The async file {string} entry mode must be file") - public void the_async_file_test_entry_mode_must_be_file(String path) { - @Cleanup Metadata metadata = op.stat(path).join(); - assertTrue(metadata.isFile()); - } - - @Then("The async file {string} content length must be {int}") - public void the_async_file_test_content_length_must_be_13(String path, int length) { - @Cleanup Metadata metadata = op.stat(path).join(); - assertEquals(metadata.getContentLength(), length); - } - - @Then("The async file {string} must have content {string}") - public void the_async_file_test_must_have_content_hello_world(String path, String content) { - String readContent = op.read(path).join(); - assertEquals(content, readContent); - } - - @Then("The presign operation should success or raise exception Unsupported") - public void the_presign_operation_should_success_or_raise_exception_unsupported() { - assertThatThrownBy( - () -> op.presignStat("test.txt", Duration.ofSeconds(10)).join()) - .is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.Unsupported)); - assertThatThrownBy( - () -> op.presignRead("test.txt", Duration.ofSeconds(10)).join()) - .is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.Unsupported)); - assertThatThrownBy(() -> - op.presignWrite("test.txt", Duration.ofSeconds(10)).join()) - .is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.Unsupported)); - } -} diff --git a/bindings/java/src/test/java/org/apache/opendal/CucumberTest.java b/bindings/java/src/test/java/org/apache/opendal/CucumberTest.java deleted file mode 100644 index ab8ae9c31f69..000000000000 --- a/bindings/java/src/test/java/org/apache/opendal/CucumberTest.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.opendal; - -import static io.cucumber.junit.platform.engine.Constants.GLUE_PROPERTY_NAME; -import org.junit.platform.suite.api.ConfigurationParameter; -import org.junit.platform.suite.api.IncludeEngines; -import org.junit.platform.suite.api.SelectClasspathResource; -import org.junit.platform.suite.api.Suite; - -@Suite -@IncludeEngines("cucumber") -@SelectClasspathResource("features") -@ConfigurationParameter(key = GLUE_PROPERTY_NAME, value = "org.apache.opendal") -public class CucumberTest {} diff --git a/bindings/java/src/test/java/org/apache/opendal/StepsTest.java b/bindings/java/src/test/java/org/apache/opendal/StepsTest.java deleted file mode 100644 index c72a0619d5ec..000000000000 --- a/bindings/java/src/test/java/org/apache/opendal/StepsTest.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.opendal; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import io.cucumber.java.en.Given; -import io.cucumber.java.en.Then; -import io.cucumber.java.en.When; -import java.util.HashMap; -import java.util.Map; -import lombok.Cleanup; - -public class StepsTest { - BlockingOperator op; - - @Given("A new OpenDAL Blocking Operator") - public void a_new_open_dal_blocking_operator() { - Map params = new HashMap<>(); - params.put("root", "/tmp"); - op = new BlockingOperator("Memory", params); - } - - @When("Blocking write path {string} with content {string}") - public void blocking_write_path_test_with_content_hello_world(String path, String content) { - op.write(path, content); - } - - @Then("The blocking file {string} should exist") - public void the_blocking_file_test_should_exist(String path) { - @Cleanup Metadata metadata = op.stat(path); - assertNotNull(metadata); - } - - @Then("The blocking file {string} entry mode must be file") - public void the_blocking_file_test_entry_mode_must_be_file(String path) { - @Cleanup Metadata metadata = op.stat(path); - assertTrue(metadata.isFile()); - } - - @Then("The blocking file {string} content length must be {int}") - public void the_blocking_file_test_content_length_must_be_13(String path, int length) { - @Cleanup Metadata metadata = op.stat(path); - assertEquals(metadata.getContentLength(), length); - } - - @Then("The blocking file {string} must have content {string}") - public void the_blocking_file_test_must_have_content_hello_world(String path, String content) { - String readContent = op.read(path); - assertEquals(content, readContent); - } -} From 60e2b4df961a44007b70d89ab9ce1c81302170a4 Mon Sep 17 00:00:00 2001 From: G-XD Date: Tue, 19 Sep 2023 02:17:30 +0800 Subject: [PATCH 09/24] test(binding/java): refactor append test --- .../org/apache/opendal/BaseOperatorTest.java | 18 ++++++++++-------- .../java/org/apache/opendal/utils/Utils.java | 2 +- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/bindings/java/src/test/java/org/apache/opendal/BaseOperatorTest.java b/bindings/java/src/test/java/org/apache/opendal/BaseOperatorTest.java index 24a12410bfd3..f29d32dff513 100644 --- a/bindings/java/src/test/java/org/apache/opendal/BaseOperatorTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/BaseOperatorTest.java @@ -80,7 +80,7 @@ public final void testRead() throws Exception { assertThatThrownBy(() -> op.stat(path).join()) .is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.NotFound)); - String content = Utils.generateRandomString(); + String content = Utils.generateString(); op.write(path, content).join(); assertThat(op.read(path).join()).isEqualTo(content); @@ -92,22 +92,24 @@ public final void testRead() throws Exception { @Test public void testAppend() { - String[] trunks = new String[] {"first trunk", "second trunk", "third trunk"}; + String path = UUID.randomUUID().toString(); + String[] trunks = new String[] {Utils.generateString(), Utils.generateString(), Utils.generateString()}; for (int i = 0; i < trunks.length; i++) { - op.append("testAppendManyTimes", trunks[i]).join(); + op.append(path, trunks[i]).join(); String expected = Arrays.stream(trunks).limit(i + 1).collect(Collectors.joining()); - assertThat(op.read("testAppendManyTimes").join()).isEqualTo(expected); + assertThat(op.read(path).join()).isEqualTo(expected); } // write overwrite existing content - op.write("testAppendManyTimes", "new attempt").join(); - assertThat(op.read("testAppendManyTimes").join()).isEqualTo("new attempt"); + String newAttempt = Utils.generateString(); + op.write(path, newAttempt).join(); + assertThat(op.read(path).join()).isEqualTo(newAttempt); for (int i = 0; i < trunks.length; i++) { - op.append("testAppendManyTimes", trunks[i]).join(); + op.append(path, trunks[i]).join(); String expected = Arrays.stream(trunks).limit(i + 1).collect(Collectors.joining()); - assertThat(op.read("testAppendManyTimes").join()).isEqualTo("new attempt" + expected); + assertThat(op.read(path).join()).isEqualTo(newAttempt + expected); } } } diff --git a/bindings/java/src/test/java/org/apache/opendal/utils/Utils.java b/bindings/java/src/test/java/org/apache/opendal/utils/Utils.java index b38aa84bba19..7dd02fc296d2 100644 --- a/bindings/java/src/test/java/org/apache/opendal/utils/Utils.java +++ b/bindings/java/src/test/java/org/apache/opendal/utils/Utils.java @@ -110,7 +110,7 @@ public static byte[] generateBytes() { return content; } - public static String generateRandomString() { + public static String generateString() { Random random = new Random(); int length = random.nextInt(256) + 1; From bd9518177a2ae18fb209c629da4645b5740b48f7 Mon Sep 17 00:00:00 2001 From: G-XD Date: Tue, 19 Sep 2023 11:53:36 +0800 Subject: [PATCH 10/24] test(binding/java): refactor blocking test --- ...torTest.java => AbstractOperatorTest.java} | 44 ++++++++++++- .../apache/opendal/BlockingOperatorTest.java | 63 ------------------- .../org/apache/opendal/services/FsTest.java | 4 +- .../apache/opendal/services/MemoryTest.java | 4 +- .../apache/opendal/services/RedisTest.java | 4 +- .../org/apache/opendal/services/S3Test.java | 4 +- .../java/org/apache/opendal/utils/Utils.java | 58 ++++++++++++++--- 7 files changed, 100 insertions(+), 81 deletions(-) rename bindings/java/src/test/java/org/apache/opendal/{BaseOperatorTest.java => AbstractOperatorTest.java} (73%) delete mode 100644 bindings/java/src/test/java/org/apache/opendal/BlockingOperatorTest.java diff --git a/bindings/java/src/test/java/org/apache/opendal/BaseOperatorTest.java b/bindings/java/src/test/java/org/apache/opendal/AbstractOperatorTest.java similarity index 73% rename from bindings/java/src/test/java/org/apache/opendal/BaseOperatorTest.java rename to bindings/java/src/test/java/org/apache/opendal/AbstractOperatorTest.java index f29d32dff513..c7df98d53b0d 100644 --- a/bindings/java/src/test/java/org/apache/opendal/BaseOperatorTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/AbstractOperatorTest.java @@ -35,10 +35,12 @@ import org.junit.jupiter.api.TestInstance; @TestInstance(TestInstance.Lifecycle.PER_CLASS) -public abstract class BaseOperatorTest { +public abstract class AbstractOperatorTest { protected Operator op; + protected BlockingOperator blockingOp; + protected abstract String schema(); @BeforeAll @@ -47,15 +49,53 @@ public void init() { Optional optional = Utils.init(schema); assertTrue(optional.isPresent()); op = optional.get(); + + Optional optionalOfBlocking = Utils.initBlockingOp(schema); + assertTrue(optionalOfBlocking.isPresent()); + blockingOp = optionalOfBlocking.get(); } @AfterAll - public void destroy() { + public void clean() { if (op != null) { op.close(); } } + @Test + public void testBlockingWrite() { + String path = UUID.randomUUID().toString(); + byte[] content = Utils.generateBytes(); + blockingOp.write(path, content); + + Metadata metadata = blockingOp.stat(path); + + assertEquals(content.length, metadata.getContentLength()); + + blockingOp.delete(path); + assertThatThrownBy(() -> blockingOp.stat(path)) + .is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.NotFound)); + } + + @Test + public void testBlockingRead() { + Metadata metadata = blockingOp.stat(""); + assertTrue(!metadata.isFile()); + + String path = UUID.randomUUID().toString(); + assertThatThrownBy(() -> blockingOp.stat(path)) + .is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.NotFound)); + + String content = Utils.generateString(); + blockingOp.write(path, content); + + assertThat(blockingOp.read(path)).isEqualTo(content); + + blockingOp.delete(path); + assertThatThrownBy(() -> blockingOp.stat(path)) + .is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.NotFound)); + } + @Test public final void testWrite() throws Exception { String path = UUID.randomUUID().toString(); diff --git a/bindings/java/src/test/java/org/apache/opendal/BlockingOperatorTest.java b/bindings/java/src/test/java/org/apache/opendal/BlockingOperatorTest.java deleted file mode 100644 index d25db9a8fa09..000000000000 --- a/bindings/java/src/test/java/org/apache/opendal/BlockingOperatorTest.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.opendal; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatExceptionOfType; -import java.util.HashMap; -import java.util.Map; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -public class BlockingOperatorTest { - private BlockingOperator op; - - @BeforeEach - public void init() { - Map params = new HashMap<>(); - params.put("root", "/tmp"); - this.op = new BlockingOperator("Memory", params); - } - - @AfterEach - public void clean() { - this.op.close(); - } - - @Test - public void testStatNotExistFile() { - assertThatExceptionOfType(OpenDALException.class) - .isThrownBy(() -> op.stat("nonexistence")) - .extracting(OpenDALException::getCode) - .isEqualTo(OpenDALException.Code.NotFound); - } - - @Test - public void testCreateAndDelete() { - op.write("testCreateAndDelete", "Odin"); - assertThat(op.read("testCreateAndDelete")).isEqualTo("Odin"); - op.delete("testCreateAndDelete"); - assertThatExceptionOfType(OpenDALException.class) - .isThrownBy(() -> op.stat("testCreateAndDelete")) - .extracting(OpenDALException::getCode) - .isEqualTo(OpenDALException.Code.NotFound); - } -} diff --git a/bindings/java/src/test/java/org/apache/opendal/services/FsTest.java b/bindings/java/src/test/java/org/apache/opendal/services/FsTest.java index 53255e148433..a5f1b024cfec 100644 --- a/bindings/java/src/test/java/org/apache/opendal/services/FsTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/services/FsTest.java @@ -19,9 +19,9 @@ package org.apache.opendal.services; -import org.apache.opendal.BaseOperatorTest; +import org.apache.opendal.AbstractOperatorTest; -public class FsTest extends BaseOperatorTest { +public class FsTest extends AbstractOperatorTest { @Override public String schema() { diff --git a/bindings/java/src/test/java/org/apache/opendal/services/MemoryTest.java b/bindings/java/src/test/java/org/apache/opendal/services/MemoryTest.java index 713055bcf164..bc627ac9c9bf 100644 --- a/bindings/java/src/test/java/org/apache/opendal/services/MemoryTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/services/MemoryTest.java @@ -20,12 +20,12 @@ package org.apache.opendal.services; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import org.apache.opendal.BaseOperatorTest; +import org.apache.opendal.AbstractOperatorTest; import org.apache.opendal.OpenDALException; import org.apache.opendal.condition.OpenDALExceptionCondition; import org.junit.jupiter.api.Test; -public class MemoryTest extends BaseOperatorTest { +public class MemoryTest extends AbstractOperatorTest { @Override public String schema() { diff --git a/bindings/java/src/test/java/org/apache/opendal/services/RedisTest.java b/bindings/java/src/test/java/org/apache/opendal/services/RedisTest.java index a49186fc771c..b33e0673bf7a 100644 --- a/bindings/java/src/test/java/org/apache/opendal/services/RedisTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/services/RedisTest.java @@ -20,12 +20,12 @@ package org.apache.opendal.services; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import org.apache.opendal.BaseOperatorTest; +import org.apache.opendal.AbstractOperatorTest; import org.apache.opendal.OpenDALException; import org.apache.opendal.condition.OpenDALExceptionCondition; import org.junit.jupiter.api.Test; -public class RedisTest extends BaseOperatorTest { +public class RedisTest extends AbstractOperatorTest { @Override public String schema() { diff --git a/bindings/java/src/test/java/org/apache/opendal/services/S3Test.java b/bindings/java/src/test/java/org/apache/opendal/services/S3Test.java index 05a475e6a50d..480ebf27241d 100644 --- a/bindings/java/src/test/java/org/apache/opendal/services/S3Test.java +++ b/bindings/java/src/test/java/org/apache/opendal/services/S3Test.java @@ -20,12 +20,12 @@ package org.apache.opendal.services; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import org.apache.opendal.BaseOperatorTest; +import org.apache.opendal.AbstractOperatorTest; import org.apache.opendal.OpenDALException; import org.apache.opendal.condition.OpenDALExceptionCondition; import org.junit.jupiter.api.Test; -public class S3Test extends BaseOperatorTest { +public class S3Test extends AbstractOperatorTest { @Override public String schema() { diff --git a/bindings/java/src/test/java/org/apache/opendal/utils/Utils.java b/bindings/java/src/test/java/org/apache/opendal/utils/Utils.java index 7dd02fc296d2..19f01a80a66b 100644 --- a/bindings/java/src/test/java/org/apache/opendal/utils/Utils.java +++ b/bindings/java/src/test/java/org/apache/opendal/utils/Utils.java @@ -28,6 +28,7 @@ import java.util.Random; import java.util.UUID; import java.util.stream.Collectors; +import org.apache.opendal.BlockingOperator; import org.apache.opendal.Operator; public class Utils { @@ -42,10 +43,46 @@ public class Utils { * Initializes the Service with the given schema. * * @param schema the schema to initialize the Operator service - * @return If `opendal_{schema}_test` is on, construct a new Operator with given root. - * Else, returns a `Empty` to represent no valid config for operator. + * @return If `opendal_{schema}_test` is on, construct a new Operator with given root. + * Else, returns a `Empty` to represent no valid config for operator. */ public static Optional init(String schema) { + Map conf = readEnv(schema); + + final String turnOnTest = conf.get(CONF_TURN_ON_TEST); + if (!isTurnOn(turnOnTest)) { + return Optional.empty(); + } + Operator op = new Operator(schema, conf); + return Optional.of(op); + } + + /** + * Initializes a blocking operator using the provided schema. + * + * @param schema the schema to be used for initializing the blocking operator + * @return If `opendal_{schema}_test` is on, construct a new BlockingOperator with given root. + * Else, returns a `Empty` to represent no valid config for operator. + */ + public static Optional initBlockingOp(String schema) { + Map conf = readEnv(schema); + + final String turnOnTest = conf.get(CONF_TURN_ON_TEST); + if (!isTurnOn(turnOnTest)) { + return Optional.empty(); + } + BlockingOperator op = new BlockingOperator(schema, conf); + return Optional.of(op); + } + + /** + * Reads the environment variables and system properties and returns a map + * containing the configuration settings for the given schema. + * + * @param schema the schema for which to retrieve the configuration settings + * @return a map containing the configuration settings + */ + private static Map readEnv(String schema) { final Properties properties = new Properties(); try (InputStream is = Utils.class.getClassLoader().getResourceAsStream(ENV_NAME)) { @@ -74,10 +111,6 @@ public static Optional init(String schema) { .orElse(""), (existing, replacement) -> existing)); - final String turnOnTest = conf.get(CONF_TURN_ON_TEST); - if (!isTurnOn(turnOnTest)) { - return Optional.empty(); - } if (!Boolean.parseBoolean(properties.getProperty(CONF_RANDOM_ROOT_FLAG))) { String root = conf.getOrDefault(CONF_ROOT, File.separator); if (!root.endsWith(File.separator)) { @@ -86,8 +119,7 @@ public static Optional init(String schema) { root = root + UUID.randomUUID() + File.separator; conf.put(CONF_ROOT, root); } - Operator op = new Operator(schema, conf); - return Optional.of(op); + return conf; } /** @@ -100,6 +132,11 @@ public static boolean isTurnOn(String val) { return "on".equalsIgnoreCase(val) || "true".equalsIgnoreCase(val); } + /** + * Generates a byte array of random content. + * + * @return the generated byte array + */ public static byte[] generateBytes() { Random random = new Random(); @@ -110,6 +147,11 @@ public static byte[] generateBytes() { return content; } + /** + * Generates a random string of lowercase letters with a length between 1 and 256. + * + * @return the randomly generated string + */ public static String generateString() { Random random = new Random(); From f14775a34b780d5ea4e3cb2c2c272e18c80c8bb0 Mon Sep 17 00:00:00 2001 From: G-XD Date: Tue, 19 Sep 2023 12:13:54 +0800 Subject: [PATCH 11/24] test(binding/java): skip tests if the env is not set --- .../{src/test/resources => }/.env.example | 0 bindings/java/README.md | 36 +++++++++++++++- .../apache/opendal/AbstractOperatorTest.java | 42 ++++++++++++++----- .../apache/opendal/services/MemoryTest.java | 3 ++ .../apache/opendal/services/RedisTest.java | 3 ++ .../org/apache/opendal/services/S3Test.java | 3 ++ 6 files changed, 74 insertions(+), 13 deletions(-) rename bindings/java/{src/test/resources => }/.env.example (100%) diff --git a/bindings/java/src/test/resources/.env.example b/bindings/java/.env.example similarity index 100% rename from bindings/java/src/test/resources/.env.example rename to bindings/java/.env.example diff --git a/bindings/java/README.md b/bindings/java/README.md index 6523b00d47cc..c63da4606594 100644 --- a/bindings/java/README.md +++ b/bindings/java/README.md @@ -62,16 +62,48 @@ You can use Maven to build both Rust dynamic lib and JAR files with one command ./mvnw clean package -DskipTests=true ``` +## Setup Tests + +Please copy `.env.example` to `src/test/resources/.env` and change the values on need. + +Take `fs` for example, we need to enable bench on `fs` on `/tmp`. + +```dotenv +OPENDAL_FS_TEST=false +OPENDAL_FS_ROOT=/path/to/dir +``` + +into + +```dotenv +OPENDAL_FS_TEST=on +OPENDAL_FS_ROOT=/tmp +``` + +Notice: The default will skip all benches if the env is not set. + ## Run tests -Currently, all tests are written in Java. It contains the Cucumber feature tests and other unit tests. +Currently, all tests are written in Java. -You can run tests with the following command: +You can run all available backends tests with the following command: ```shell ./mvnw clean verify -Dcargo-build.features=services-redis ``` +Test specific backend(such as `fs`). + +```shell +./mvnw test -Dtest=org.apache.opendal.services.FsTest +``` + +Test specific backend(such as `redis`). + +```shell +./mvnw test -Dtest=org.apache.opendal.services.RedisTest -Dcargo-build.features=services-redis +``` + > **Note:** > > The `-Dcargo-build.features=services-redis` argument is a temporary workaround. See also: diff --git a/bindings/java/src/test/java/org/apache/opendal/AbstractOperatorTest.java b/bindings/java/src/test/java/org/apache/opendal/AbstractOperatorTest.java index c7df98d53b0d..087e6896bab2 100644 --- a/bindings/java/src/test/java/org/apache/opendal/AbstractOperatorTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/AbstractOperatorTest.java @@ -37,33 +37,33 @@ @TestInstance(TestInstance.Lifecycle.PER_CLASS) public abstract class AbstractOperatorTest { - protected Operator op; + protected Optional opOptional; - protected BlockingOperator blockingOp; + protected Optional blockingOpOptional; protected abstract String schema(); @BeforeAll public void init() { String schema = this.schema(); - Optional optional = Utils.init(schema); - assertTrue(optional.isPresent()); - op = optional.get(); + opOptional = Utils.init(schema); - Optional optionalOfBlocking = Utils.initBlockingOp(schema); - assertTrue(optionalOfBlocking.isPresent()); - blockingOp = optionalOfBlocking.get(); + blockingOpOptional = Utils.initBlockingOp(schema); } @AfterAll public void clean() { - if (op != null) { - op.close(); - } + opOptional.ifPresent(op -> op.close()); + blockingOpOptional.ifPresent(op -> op.close()); } @Test public void testBlockingWrite() { + if (!blockingOpOptional.isPresent()) { + return; + } + BlockingOperator blockingOp = blockingOpOptional.get(); + String path = UUID.randomUUID().toString(); byte[] content = Utils.generateBytes(); blockingOp.write(path, content); @@ -79,6 +79,11 @@ public void testBlockingWrite() { @Test public void testBlockingRead() { + if (!blockingOpOptional.isPresent()) { + return; + } + BlockingOperator blockingOp = blockingOpOptional.get(); + Metadata metadata = blockingOp.stat(""); assertTrue(!metadata.isFile()); @@ -98,6 +103,11 @@ public void testBlockingRead() { @Test public final void testWrite() throws Exception { + if (!opOptional.isPresent()) { + return; + } + Operator op = opOptional.get(); + String path = UUID.randomUUID().toString(); byte[] content = Utils.generateBytes(); op.write(path, content).join(); @@ -113,6 +123,11 @@ public final void testWrite() throws Exception { @Test public final void testRead() throws Exception { + if (!opOptional.isPresent()) { + return; + } + Operator op = opOptional.get(); + Metadata metadata = op.stat("").get(); assertTrue(!metadata.isFile()); @@ -132,6 +147,11 @@ public final void testRead() throws Exception { @Test public void testAppend() { + if (!opOptional.isPresent()) { + return; + } + Operator op = opOptional.get(); + String path = UUID.randomUUID().toString(); String[] trunks = new String[] {Utils.generateString(), Utils.generateString(), Utils.generateString()}; diff --git a/bindings/java/src/test/java/org/apache/opendal/services/MemoryTest.java b/bindings/java/src/test/java/org/apache/opendal/services/MemoryTest.java index bc627ac9c9bf..794605301d21 100644 --- a/bindings/java/src/test/java/org/apache/opendal/services/MemoryTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/services/MemoryTest.java @@ -35,6 +35,9 @@ public String schema() { @Test @Override public void testAppend() { + if (!opOptional.isPresent()) { + return; + } assertThatThrownBy(() -> super.testAppend()) .is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.Unsupported)); } diff --git a/bindings/java/src/test/java/org/apache/opendal/services/RedisTest.java b/bindings/java/src/test/java/org/apache/opendal/services/RedisTest.java index b33e0673bf7a..209944570ac0 100644 --- a/bindings/java/src/test/java/org/apache/opendal/services/RedisTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/services/RedisTest.java @@ -35,6 +35,9 @@ public String schema() { @Test @Override public void testAppend() { + if (!opOptional.isPresent()) { + return; + } assertThatThrownBy(() -> super.testAppend()) .is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.Unsupported)); } diff --git a/bindings/java/src/test/java/org/apache/opendal/services/S3Test.java b/bindings/java/src/test/java/org/apache/opendal/services/S3Test.java index 480ebf27241d..b778819c0cd5 100644 --- a/bindings/java/src/test/java/org/apache/opendal/services/S3Test.java +++ b/bindings/java/src/test/java/org/apache/opendal/services/S3Test.java @@ -35,6 +35,9 @@ public String schema() { @Test @Override public void testAppend() { + if (!opOptional.isPresent()) { + return; + } assertThatThrownBy(() -> super.testAppend()) .is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.Unsupported)); } From 9afb744d53a513a17147a7d5f3b096fa3aeeaf9d Mon Sep 17 00:00:00 2001 From: G-XD Date: Tue, 19 Sep 2023 12:25:22 +0800 Subject: [PATCH 12/24] ci(services/redis): setup redis server --- .github/workflows/service_test_redis.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/service_test_redis.yml b/.github/workflows/service_test_redis.yml index fdfad34f77b1..de22de8f7dfb 100644 --- a/.github/workflows/service_test_redis.yml +++ b/.github/workflows/service_test_redis.yml @@ -201,6 +201,11 @@ jobs: - name: Setup Rust toolchain uses: ./.github/actions/setup + - name: Setup Redis Server + shell: bash + working-directory: fixtures/redis + run: docker-compose -f docker-compose-redis.yml up -d + - name: Test shell: bash working-directory: bindings/java From 8e621a63a53e1a0c213a0586d8916a4cc622eb0f Mon Sep 17 00:00:00 2001 From: G-XD Date: Tue, 19 Sep 2023 22:56:25 +0800 Subject: [PATCH 13/24] chore(binding/java): reuse the .env.example --- bindings/java/.env.example | 166 ------------------------------------- bindings/java/README.md | 2 +- 2 files changed, 1 insertion(+), 167 deletions(-) delete mode 100644 bindings/java/.env.example diff --git a/bindings/java/.env.example b/bindings/java/.env.example deleted file mode 100644 index 2a5b0344876b..000000000000 --- a/bindings/java/.env.example +++ /dev/null @@ -1,166 +0,0 @@ -# memory -OPENDAL_MEMORY_TEST=on -# fs -OPENDAL_FS_TEST=false -OPENDAL_FS_ROOT=/path/to/dir -OPENDAL_FS_ATOMIC_WRITE_DIR=/path/to/tempdir -# cos -OPENDAL_COS_TEST=false -OPENDAL_COS_BUCKET=opendal-testing-1318209832 -OPENDAL_COS_ENDPOINT=https://cos.ap-singapore.myqcloud.com -OPENDAL_COS_SECRET_ID= -OPENDAL_COS_SECRET_KEY= -# s3 -OPENDAL_S3_TEST=false -OPENDAL_S3_BUCKET= -OPENDAL_S3_ENDPOINT= -OPENDAL_S3_REGION= -OPENDAL_S3_ACCESS_KEY_ID= -OPENDAL_S3_SECRET_ACCESS_KEY= -# azblob -OPENDAL_AZBLOB_TEST=false -OPENDAL_AZBLOB_ROOT=/path/to/dir -OPENDAL_AZBLOB_CONTAINER= -OPENDAL_AZBLOB_ENDPOINT= -OPENDAL_AZBLOB_ACCOUNT_NAME= -OPENDAL_AZBLOB_ACCOUNT_KEY= -# hdfs -OPENDAL_HDFS_TEST=false -OPENDAL_HDFS_ROOT=/path/to/dir -OPENDAL_HDFS_NAME_NODE= -# gcs -OPENDAL_GCS_TEST=false -OPENDAL_GCS_ROOT=/path/to/dir -OPENDAL_GCS_BUCKET= -OPENDAL_GCS_CREDENTIAL= -# obs -OPENDAL_OBS_TEST=false -OPENDAL_OBS_BUCKET= -OPENDAL_OBS_ENDPOINT= -OPENDAL_OBS_ACCESS_KEY_ID= -OPENDAL_OBS_SECRET_ACCESS_KEY= -# oss -OPENDAL_OSS_TEST=false -OPENDAL_OSS_BUCKET= -OPENDAL_OSS_ENDPOINT= -OPENDAL_OSS_ACCESS_KEY_ID= -OPENDAL_OSS_ACCESS_KEY_SECRET= -# ipmfs -OPENDAL_IPMFS_TEST=false -OPENDAL_IPMFS_ROOT=/path/to/dir -OPENDAL_IPMFS_ENDPOINT=http://localhost:5001 -# ftp -OPENDAL_FTP_TEST=false -OPENDAL_FTP_ENDPOINT=ftp:// -OPENDAL_FTP_ROOT=/path/to/dir -OPENDAL_FTP_USER= -OPENDAL_FTP_PASSWORD= -# ipfs -OPENDAL_IPFS_TEST=false -OPENDAL_IPFS_ROOT=/ipfs/Qmxxxxxxxx -OPENDAL_IPFS_ENDPOINT=http://localhost:8080 -# redis -OPENDAL_REDIS_TEST=false -OPENDAL_REDIS_ENDPOINT=tcp://127.0.0.1:6379 -# OPENDAL_REDIS_CLUSTER_ENDPOINTS=rediss://127.0.0.1:6380,rediss://127.0.0.1:6381,rediss://127.0.0.1:6382,rediss://127.0.0.1:6383,rediss://127.0.0.1:6384,rediss://127.0.0.1:6385 -OPENDAL_REDIS_ROOT=/ -OPENDAL_REDIS_DB=0 -# rocksdb -OPENDAL_ROCKSDB_TEST=false -OPENDAL_ROCKSDB_DATADIR=/path/to/database -OPENDAL_ROCKSDB_ROOT=/path/to/root -# sftp -OPENDAL_SFTP_TEST=false -OPENDAL_SFTP_ENDPOINT=ssh:// -OPENDAL_SFTP_ROOT=/path/to/dir -OPENDAL_SFTP_USER= -OPENDAL_SFTP_KEY= -OPENDAL_SFTP_KNOWN_HOSTS_STRATEGY= -# sled -OPENDAL_SLED_TEST=false -OPENDAL_SLED_DATADIR=/path/to/database -OPENDAL_SLED_TREE=sled-tree -# mini-moka -OPENDAL_MINI_MOKA_TEST=false -# moka -OPENDAL_MOKA_TEST=false -# ghac -OPENDAL_GHAC_TEST=false -# memcached -OPENDAL_MEMCACHED_TEST=false -OPENDAL_MEMCACHED_ENDPOINT=tcp://127.0.0.1:11211 -OPENDAL_MEMCACHED_ROOT=/ -# webdav -OPENDAL_WEBDAV_TEST=false -OPENDAL_WEBDAV_ROOT=/tmp/opendal/ -OPENDAL_WEBDAV_ENDPOINT=http://127.0.0.1:8080 -# webhfds -OPENDAL_WEBHDFS_TEST=false -OPENDAL_WEBHDFS_ROOT=/tmp/opendal/ -OPENDAL_WEBHDFS_ENDPOINT=http://127.0.0.1:9870 -OPENDAL_WEBHDFS_DELEGATION= -OPENDAL_WEBHDFS_DISABLE_LIST_BATCH=false -# supbase -OPENDAL_SUPABASE_TEST=false -OPENDAL_SUPABASE_BUCKET= -OPENDAL_SUPABASE_ENDPOINT= -OPENDAL_SUPABASE_KEY= -# vercel artifacts -OPENDAL_VERCEL_ARTIFACTS_TEST=false -OPENDAL_VERCEL_ARTIFACTS_ACCESS_TOKEN= -# onedrive -OPENDAL_ONEDRIVE_TEST=false -OPENDAL_ONEDRIVE_ACCESS_TOKEN= -# wasabi -OPENDAL_WASABI_TEST=false -OPENDAL_WASABI_REGION= -OPENDAL_WASABI_BUCKET= -OPENDAL_WASABI_ENDPOINT= -OPENDAL_WASABI_ACCESS_KEY_ID= -OPENDAL_WASABI_SECRET_ACCESS_KEY= -# atomicserver -OPENDAL_ATOMICSERVER_TEST=false -OPENDAL_ATOMICSERVER_ROOT=/path/to/dir -OPENDAL_ATOMICSERVER_ENDPOINT=http://localhost:9883 -OPENDAL_ATOMICSERVER_PRIVATE_KEY= -OPENDAL_ATOMICSERVER_PUBLIC_KEY= -OPENDAL_ATOMICSERVER_PARENT_RESOURCE_ID=http://localhost:9883 -# foundationdb -OPENDAL_FOUNDATIONDB_TEST=false -OPENDAL_FOUNDATIONDB_ROOT=/path/to/dir -OPENDAL_FOUNDATIONDB_CONFIG_PATH=/tmp/opendal/foundationdb.conf -# redb -OPENDAL_REDB_TEST=false -OPENDAL_REDB_DATADIR=/tmp/redb -OPENDAL_REDB_TABLE=redb-table -# cacache -OPENDAL_CACACHE_TEST=false -OPENDAL_CACACHE_DATADIR=/tmp/opendal/cacache/ -# persy -OPENDAL_PERSY_TEST=false -OPENDAL_PERSY_DATAFILE=/tmp/opendal/test.persy -OPENDAL_PERSY_SEGMENT=data -OPENDAL_PERSY_INDEX=index -#dropbox -OPENDAL_DROPBOX_TEST=false -OPENDAL_DROPBOX_ROOT=/tmp/opendal/ -OPENDAL_DROPBOX_ACCESS_TOKEN= -OPENDAL_DROPBOX_REFRESH_TOKEN= -OPENDAL_DROPBOX_CLIENT_ID= -OPENDAL_DROPBOX_CLIENT_SECRET= -# etcd -OPENDAL_ETCD_TEST=false -OPENDAL_ETCD_ENDPOINTS=127.0.0.1:2379 -OPENDAL_ETCD_ROOT=/tmp/opendal/ -OPENDAL_ETCD_USERNAME= -OPENDAL_ETCD_PASSWORD= -OPENDAL_ETCD_CA_PATH= -OPENDAL_ETCD_CERT_PATH= -OPENDAL_ETCD_KEY_PATH= -# google drive -OPENDAL_GDRIVE_TEST=false -OPENDAL_GDRIVE_ROOT=/tmp/opendal/ -OPENDAL_GDRIVE_ACCESS_TOKEN= -OPENDAL_GDRIVE_REFRESH_TOKEN= -OPENDAL_GDRIVE_CLIENT_ID= -OPENDAL_GDRIVE_CLIENT_SECRET= diff --git a/bindings/java/README.md b/bindings/java/README.md index c63da4606594..fe68c44a5fa7 100644 --- a/bindings/java/README.md +++ b/bindings/java/README.md @@ -64,7 +64,7 @@ You can use Maven to build both Rust dynamic lib and JAR files with one command ## Setup Tests -Please copy `.env.example` to `src/test/resources/.env` and change the values on need. +Please copy `{project.rootdir}/.env.example` to `src/test/resources/.env` and change the values on need. Take `fs` for example, we need to enable bench on `fs` on `/tmp`. From 708d32592cb5b97e1cd263e50a1249e0b8d0513f Mon Sep 17 00:00:00 2001 From: G-XD Date: Thu, 21 Sep 2023 11:20:42 +0800 Subject: [PATCH 14/24] feat(binding/java): support info ops --- bindings/java/src/blocking_operator.rs | 25 ++ bindings/java/src/lib.rs | 97 +++++++ .../org/apache/opendal/BlockingOperator.java | 6 + .../java/org/apache/opendal/Capability.java | 270 ++++++++++++++++++ .../java/org/apache/opendal/Operator.java | 7 + .../java/org/apache/opendal/OperatorInfo.java | 40 +++ bindings/java/src/operator.rs | 30 ++ 7 files changed, 475 insertions(+) create mode 100644 bindings/java/src/main/java/org/apache/opendal/Capability.java create mode 100644 bindings/java/src/main/java/org/apache/opendal/OperatorInfo.java diff --git a/bindings/java/src/blocking_operator.rs b/bindings/java/src/blocking_operator.rs index 5b6c28aba94d..82b4ffb55a79 100644 --- a/bindings/java/src/blocking_operator.rs +++ b/bindings/java/src/blocking_operator.rs @@ -31,6 +31,7 @@ use opendal::Scheme; use crate::get_global_runtime; use crate::jmap_to_hashmap; +use crate::make_operator_info; use crate::Result; #[no_mangle] @@ -159,3 +160,27 @@ fn intern_delete(env: &mut JNIEnv, op: &mut BlockingOperator, path: JString) -> let path = env.get_string(&path)?; Ok(op.delete(path.to_str()?)?) } + +/// # Safety +/// +/// This function should not be called before the Operator are ready. +#[no_mangle] +pub unsafe extern "system" fn Java_org_apache_opendal_BlockingOperator_info<'local>( + mut env: JNIEnv<'local>, + _: JClass, + op: *mut BlockingOperator, +) -> JObject<'local> { + intern_info(&mut env, &mut *op).unwrap_or_else(|e| { + e.throw(&mut env); + JObject::null() + }) +} + +fn intern_info<'local>( + env: &mut JNIEnv<'local>, + op: &mut BlockingOperator, +) -> Result> { + let info = op.info(); + + make_operator_info(env, info) +} diff --git a/bindings/java/src/lib.rs b/bindings/java/src/lib.rs index e1703b527d75..c277f684a822 100644 --- a/bindings/java/src/lib.rs +++ b/bindings/java/src/lib.rs @@ -23,12 +23,16 @@ use crate::error::Error; use jni::objects::JObject; use jni::objects::JString; use jni::objects::{JMap, JValue}; +use jni::sys::jboolean; use jni::sys::jint; +use jni::sys::jlong; use jni::sys::JNI_VERSION_1_8; use jni::JNIEnv; use jni::JavaVM; use once_cell::sync::OnceCell; use opendal::raw::PresignedRequest; +use opendal::Capability; +use opendal::OperatorInfo; use tokio::runtime::Builder; use tokio::runtime::Runtime; @@ -142,3 +146,96 @@ fn make_presigned_request<'a>(env: &mut JNIEnv<'a>, req: PresignedRequest) -> Re )?; Ok(result) } + +fn make_operator_info<'a>(env: &mut JNIEnv<'a>, info: OperatorInfo) -> Result> { + let operator_info_class = env.find_class("org/apache/opendal/OperatorInfo")?; + + let schema = env.new_string(info.scheme().to_string())?; + let root = env.new_string(info.root().to_string())?; + let name = env.new_string(info.name().to_string())?; + let full_capability_obj = make_capability(env, info.full_capability())?; + let native_capability_obj = make_capability(env, info.native_capability())?; + + let operator_info_obj = env + .new_object( + operator_info_class, + "(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Lorg/apache/opendal/Capability;Lorg/apache/opendal/Capability;)V", + &[ + JValue::Object(&schema), + JValue::Object(&root), + JValue::Object(&name), + JValue::Object(&full_capability_obj), + JValue::Object(&native_capability_obj), + ], + )?; + + Ok(operator_info_obj) +} + +fn make_capability<'a>(env: &mut JNIEnv<'a>, cap: Capability) -> Result> { + let capability_class = env.find_class("org/apache/opendal/Capability")?; + + let write_multi_max_size = make_long(env, cap.write_multi_max_size)?; + let write_multi_min_size = make_long(env, cap.write_multi_min_size)?; + let write_multi_align_size = make_long(env, cap.write_multi_align_size)?; + let batch_max_operations = make_long(env, cap.batch_max_operations)?; + + let capability = env.new_object(capability_class, "(ZZZZZZZZZZZZZZZZZZLjava/lang/Long;Ljava/lang/Long;Ljava/lang/Long;ZZZZZZZZZZZZZZZLjava/lang/Long;Z)V", + &[ + JValue::Bool(cap.stat as jboolean), + JValue::Bool(cap.stat_with_if_match as jboolean), + JValue::Bool(cap.stat_with_if_none_match as jboolean), + JValue::Bool(cap.read as jboolean), + JValue::Bool(cap.read_can_seek as jboolean), + JValue::Bool(cap.read_can_next as jboolean), + JValue::Bool(cap.read_with_range as jboolean), + JValue::Bool(cap.read_with_if_match as jboolean), + JValue::Bool(cap.read_with_if_none_match as jboolean), + JValue::Bool(cap.read_with_override_cache_control as jboolean), + JValue::Bool(cap.read_with_override_content_disposition as jboolean), + JValue::Bool(cap.read_with_override_content_type as jboolean), + JValue::Bool(cap.write as jboolean), + JValue::Bool(cap.write_can_multi as jboolean), + JValue::Bool(cap.write_can_append as jboolean), + JValue::Bool(cap.write_with_content_type as jboolean), + JValue::Bool(cap.write_with_content_disposition as jboolean), + JValue::Bool(cap.write_with_cache_control as jboolean), + JValue::Object(&write_multi_max_size), + JValue::Object(&write_multi_min_size), + JValue::Object(&write_multi_align_size), + JValue::Bool(cap.create_dir as jboolean), + JValue::Bool(cap.delete as jboolean), + JValue::Bool(cap.copy as jboolean), + JValue::Bool(cap.rename as jboolean), + JValue::Bool(cap.list as jboolean), + JValue::Bool(cap.list_with_limit as jboolean), + JValue::Bool(cap.list_with_start_after as jboolean), + JValue::Bool(cap.list_with_delimiter_slash as jboolean), + JValue::Bool(cap.list_without_delimiter as jboolean), + JValue::Bool(cap.presign as jboolean), + JValue::Bool(cap.presign_read as jboolean), + JValue::Bool(cap.presign_stat as jboolean), + JValue::Bool(cap.presign_write as jboolean), + JValue::Bool(cap.batch as jboolean), + JValue::Bool(cap.batch_delete as jboolean), + JValue::Object(&batch_max_operations), + JValue::Bool(cap.blocking as jboolean), + ])?; + + Ok(capability) +} + +fn make_long<'a>(env: &mut JNIEnv<'a>, value: Option) -> Result> { + let long_class = env.find_class("java/lang/Long")?; + + let result; + match value { + Some(val) => { + result = env.new_object(long_class, "(J)V", &[JValue::Long(val as jlong)])?; + } + None => { + result = JObject::null(); + } + } + Ok(result) +} diff --git a/bindings/java/src/main/java/org/apache/opendal/BlockingOperator.java b/bindings/java/src/main/java/org/apache/opendal/BlockingOperator.java index 7413b2b044dd..1596f7b527a8 100644 --- a/bindings/java/src/main/java/org/apache/opendal/BlockingOperator.java +++ b/bindings/java/src/main/java/org/apache/opendal/BlockingOperator.java @@ -61,6 +61,10 @@ public Metadata stat(String path) { return new Metadata(stat(nativeHandle, path)); } + public OperatorInfo info() { + return info(nativeHandle); + } + @Override protected native void disposeInternal(long handle); @@ -73,4 +77,6 @@ public Metadata stat(String path) { private static native void delete(long nativeHandle, String path); private static native long stat(long nativeHandle, String path); + + private static native OperatorInfo info(long nativeHandle); } diff --git a/bindings/java/src/main/java/org/apache/opendal/Capability.java b/bindings/java/src/main/java/org/apache/opendal/Capability.java new file mode 100644 index 000000000000..d13857624760 --- /dev/null +++ b/bindings/java/src/main/java/org/apache/opendal/Capability.java @@ -0,0 +1,270 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.opendal; + +import lombok.ToString; + +@ToString +public class Capability { + /** + * If operator supports stat. + */ + public final boolean stat; + /** + * If operator supports stat with if match. + */ + public final boolean statWithIfMatch; + /** + * If operator supports stat with if none match. + */ + public final boolean statWithIfNoneMatch; + + /** + * If operator supports read. + */ + public final boolean read; + /** + * If operator supports seek on returning reader. + */ + public final boolean readCanSeek; + /** + * If operator supports next on returning reader. + */ + public final boolean readCanNext; + /** + * If operator supports read with range. + */ + public final boolean readWithRange; + /** + * If operator supports read with if match. + */ + public final boolean readWithIfMatch; + /** + * If operator supports read with if none match. + */ + public final boolean readWithIfNoneMatch; + /** + * If operator supports read with override cache control. + */ + public final boolean readWithOverrideCacheControl; + /** + * if operator supports read with override content disposition. + */ + public final boolean readWithOverrideContentDisposition; + /** + * if operator supports read with override content type. + */ + public final boolean readWithOverrideContentType; + + /** + * If operator supports write. + */ + public final boolean write; + /** + * If operator supports write can be called in multi times. + */ + public final boolean writeCanMulti; + /** + * If operator supports write by append. + */ + public final boolean writeCanAppend; + /** + * If operator supports write with content type. + */ + public final boolean writeWithContentType; + /** + * If operator supports write with content disposition. + */ + public final boolean writeWithContentDisposition; + /** + * If operator supports write with cache control. + */ + public final boolean writeWithCacheControl; + /** + * write_multi_max_size is the max size that services support in write_multi. + * For example, AWS S3 supports 5GiB as max in write_multi. + */ + public final Long writeMultiMaxSize; + /** + * write_multi_min_size is the min size that services support in write_multi. + * For example, AWS S3 requires at least 5MiB in write_multi expect the last one. + */ + public final Long writeMultiMinSize; + /** + * write_multi_align_size is the align size that services required in write_multi. + * For example, Google GCS requires align size to 256KiB in write_multi. + */ + public final Long writeMultiAlignSize; + + /** + * If operator supports create dir. + */ + public final boolean createDir; + + /** + * If operator supports delete. + */ + public final boolean delete; + + /** + * If operator supports copy. + */ + public final boolean copy; + + /** + * If operator supports rename. + */ + public final boolean rename; + + /** + * If operator supports list. + */ + public final boolean list; + /** + * If backend supports list with limit. + */ + public final boolean listWithLimit; + /** + * If backend supports list with start after. + */ + public final boolean listWithStartAfter; + /** + * If backend support list with using slash as delimiter. + */ + public final boolean listWithDelimiterSlash; + /** + * If backend supports list without delimiter. + */ + public final boolean listWithoutDelimiter; + + /** + * If operator supports presign. + */ + public final boolean presign; + /** + * If operator supports presign read. + */ + public final boolean presignRead; + /** + * If operator supports presign stat. + */ + public final boolean presignStat; + /** + * If operator supports presign write. + */ + public final boolean presignWrite; + + /** + * If operator supports batch. + */ + public final boolean batch; + /** + * If operator supports batch delete. + */ + public final boolean batchDelete; + /** + * The max operations that operator supports in batch. + */ + public final Long batchMaxOperations; + + /** + * If operator supports blocking. + */ + public final boolean blocking; + + public Capability( + boolean stat, + boolean statWithIfMatch, + boolean statWithIfNoneMatch, + boolean read, + boolean readCanSeek, + boolean readCanNext, + boolean readWithRange, + boolean readWithIfMatch, + boolean readWithIfNoneMatch, + boolean readWithOverrideCacheControl, + boolean readWithOverrideContentDisposition, + boolean readWithOverrideContentType, + boolean write, + boolean writeCanMulti, + boolean writeCanAppend, + boolean writeWithContentType, + boolean writeWithContentDisposition, + boolean writeWithCacheControl, + Long writeMultiMaxSize, + Long writeMultiMinSize, + Long writeMultiAlignSize, + boolean createDir, + boolean delete, + boolean copy, + boolean rename, + boolean list, + boolean listWithLimit, + boolean listWithStartAfter, + boolean listWithDelimiterSlash, + boolean listWithoutDelimiter, + boolean presign, + boolean presignRead, + boolean presignStat, + boolean presignWrite, + boolean batch, + boolean batchDelete, + Long batchMaxOperations, + boolean blocking) { + this.stat = stat; + this.statWithIfMatch = statWithIfMatch; + this.statWithIfNoneMatch = statWithIfNoneMatch; + this.read = read; + this.readCanSeek = readCanSeek; + this.readCanNext = readCanNext; + this.readWithRange = readWithRange; + this.readWithIfMatch = readWithIfMatch; + this.readWithIfNoneMatch = readWithIfNoneMatch; + this.readWithOverrideCacheControl = readWithOverrideCacheControl; + this.readWithOverrideContentDisposition = readWithOverrideContentDisposition; + this.readWithOverrideContentType = readWithOverrideContentType; + this.write = write; + this.writeCanMulti = writeCanMulti; + this.writeCanAppend = writeCanAppend; + this.writeWithContentType = writeWithContentType; + this.writeWithContentDisposition = writeWithContentDisposition; + this.writeWithCacheControl = writeWithCacheControl; + this.writeMultiMaxSize = writeMultiMaxSize; + this.writeMultiMinSize = writeMultiMinSize; + this.writeMultiAlignSize = writeMultiAlignSize; + this.createDir = createDir; + this.delete = delete; + this.copy = copy; + this.rename = rename; + this.list = list; + this.listWithLimit = listWithLimit; + this.listWithStartAfter = listWithStartAfter; + this.listWithDelimiterSlash = listWithDelimiterSlash; + this.listWithoutDelimiter = listWithoutDelimiter; + this.presign = presign; + this.presignRead = presignRead; + this.presignStat = presignStat; + this.presignWrite = presignWrite; + this.batch = batch; + this.batchDelete = batchDelete; + this.batchMaxOperations = batchMaxOperations; + this.blocking = blocking; + } +} diff --git a/bindings/java/src/main/java/org/apache/opendal/Operator.java b/bindings/java/src/main/java/org/apache/opendal/Operator.java index 4cd6c2c15972..e57e0a952a91 100644 --- a/bindings/java/src/main/java/org/apache/opendal/Operator.java +++ b/bindings/java/src/main/java/org/apache/opendal/Operator.java @@ -142,6 +142,11 @@ public CompletableFuture read(String path) { return AsyncRegistry.take(requestId); } + public CompletableFuture info() { + final long requestId = info(nativeHandle); + return AsyncRegistry.take(requestId); + } + public CompletableFuture presignRead(String path, Duration duration) { final long requestId = presignRead(nativeHandle, path, duration.toNanos()); return AsyncRegistry.take(requestId); @@ -177,6 +182,8 @@ public CompletableFuture delete(String path) { private static native long stat(long nativeHandle, String path); + private static native long info(long nativeHandle); + private static native long presignRead(long nativeHandle, String path, long duration); private static native long presignWrite(long nativeHandle, String path, long duration); diff --git a/bindings/java/src/main/java/org/apache/opendal/OperatorInfo.java b/bindings/java/src/main/java/org/apache/opendal/OperatorInfo.java new file mode 100644 index 000000000000..d9e1e600d812 --- /dev/null +++ b/bindings/java/src/main/java/org/apache/opendal/OperatorInfo.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.opendal; + +import lombok.ToString; + +@ToString +public class OperatorInfo { + public final String scheme; + public final String root; + public final String name; + public final Capability fullCapability; + public final Capability nativeCapability; + + public OperatorInfo( + String scheme, String root, String name, Capability fullCapability, Capability nativeCapability) { + this.scheme = scheme; + this.root = root; + this.name = name; + this.fullCapability = fullCapability; + this.nativeCapability = nativeCapability; + } +} diff --git a/bindings/java/src/operator.rs b/bindings/java/src/operator.rs index f18a3cb0a7a5..733bb0ea8642 100644 --- a/bindings/java/src/operator.rs +++ b/bindings/java/src/operator.rs @@ -33,6 +33,7 @@ use opendal::Scheme; use crate::get_global_runtime; use crate::jmap_to_hashmap; +use crate::make_operator_info; use crate::Result; use crate::{get_current_env, make_presigned_request}; @@ -262,6 +263,35 @@ async fn do_delete(op: &mut Operator, path: String) -> Result<()> { Ok(op.delete(&path).await?) } +// # Safety +/// +/// This function should not be called before the Operator are ready. +#[no_mangle] +pub unsafe extern "system" fn Java_org_apache_opendal_Operator_info( + mut env: JNIEnv, + _: JClass, + op: *mut Operator, +) -> jlong { + intern_info(&mut env, op).unwrap_or_else(|e| { + e.throw(&mut env); + 0 + }) +} + +fn intern_info(env: &mut JNIEnv, op: *mut Operator) -> Result { + let op = unsafe { &mut *op }; + let id = request_id(env)?; + + unsafe { get_global_runtime() }.spawn(async move { + let info = op.info(); + let mut env = unsafe { get_current_env() }; + let result = make_operator_info(&mut env, info); + complete_future(id, result.map(JValueOwned::Object)) + }); + + Ok(id) +} + /// # Safety /// /// This function should not be called before the Operator are ready. From 5b022c0f985c551ee08b74cd4505a6151d48a84b Mon Sep 17 00:00:00 2001 From: G-XD Date: Thu, 21 Sep 2023 11:48:19 +0800 Subject: [PATCH 15/24] test(binding/java): add operator info test --- .../org/apache/opendal/OperatorInfoTest.java | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 bindings/java/src/test/java/org/apache/opendal/OperatorInfoTest.java diff --git a/bindings/java/src/test/java/org/apache/opendal/OperatorInfoTest.java b/bindings/java/src/test/java/org/apache/opendal/OperatorInfoTest.java new file mode 100644 index 000000000000..67d515ded574 --- /dev/null +++ b/bindings/java/src/test/java/org/apache/opendal/OperatorInfoTest.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.opendal; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +public class OperatorInfoTest { + + @TempDir + private static Path tempDir; + + @Test + public void testBlockingOperatorInfo() { + Map conf = new HashMap<>(); + conf.put("root", tempDir.toString()); + try (BlockingOperator op = new BlockingOperator("fs", conf)) { + + OperatorInfo info = op.info(); + assertNotNull(info); + assertEquals("fs", info.scheme); + + Capability fullCapability = info.fullCapability; + assertNotNull(fullCapability); + + assertTrue(fullCapability.read); + assertTrue(fullCapability.write); + assertTrue(fullCapability.delete); + assertTrue(fullCapability.writeCanAppend); + + assertNull(fullCapability.writeMultiAlignSize); + assertNull(fullCapability.writeMultiMaxSize); + assertNull(fullCapability.writeMultiMinSize); + assertNull(fullCapability.batchMaxOperations); + + Capability nativeCapability = info.nativeCapability; + assertNotNull(nativeCapability); + } + } + + @Test + public void testOperatorInfo() { + Map conf = new HashMap<>(); + String root = "/opendal/"; + conf.put("root", root); + try (Operator op = new Operator("memory", conf)) { + + OperatorInfo info = op.info().join(); + assertNotNull(info); + assertEquals("memory", info.scheme); + assertEquals(root, info.root); + + Capability fullCapability = info.fullCapability; + assertNotNull(fullCapability); + + assertTrue(fullCapability.read); + assertTrue(fullCapability.write); + assertTrue(fullCapability.delete); + assertTrue(!fullCapability.writeCanAppend); + + assertNull(fullCapability.writeMultiAlignSize); + assertNull(fullCapability.writeMultiMaxSize); + assertNull(fullCapability.writeMultiMinSize); + assertNull(fullCapability.batchMaxOperations); + + Capability nativeCapability = info.nativeCapability; + assertNotNull(nativeCapability); + } + } +} From 1fe237c43c24af1b00c90e7f20a217815bc5e007 Mon Sep 17 00:00:00 2001 From: G-XD Date: Thu, 21 Sep 2023 12:19:14 +0800 Subject: [PATCH 16/24] test(binding/java): refactor behavior test --- .github/workflows/service_test_redis.yml | 2 +- .github/workflows/service_test_s3.yml | 2 +- bindings/java/README.md | 8 ++-- ...actOperatorTest.java => OperatorTest.java} | 38 ++++++++++++---- .../org/apache/opendal/services/FsTest.java | 30 ------------- .../apache/opendal/services/MemoryTest.java | 44 ------------------- .../apache/opendal/services/RedisTest.java | 44 ------------------- .../org/apache/opendal/services/S3Test.java | 44 ------------------- 8 files changed, 36 insertions(+), 176 deletions(-) rename bindings/java/src/test/java/org/apache/opendal/{AbstractOperatorTest.java => OperatorTest.java} (83%) delete mode 100644 bindings/java/src/test/java/org/apache/opendal/services/FsTest.java delete mode 100644 bindings/java/src/test/java/org/apache/opendal/services/MemoryTest.java delete mode 100644 bindings/java/src/test/java/org/apache/opendal/services/RedisTest.java delete mode 100644 bindings/java/src/test/java/org/apache/opendal/services/S3Test.java diff --git a/.github/workflows/service_test_redis.yml b/.github/workflows/service_test_redis.yml index de22de8f7dfb..1fa9ea6e3168 100644 --- a/.github/workflows/service_test_redis.yml +++ b/.github/workflows/service_test_redis.yml @@ -209,7 +209,7 @@ jobs: - name: Test shell: bash working-directory: bindings/java - run: ./mvnw test -Dcargo-build.features=services-redis -Dtest=org.apache.opendal.services.RedisTest + run: ./mvnw test -Dtest=org.apache.opendal.OperatorTest -DOPENDAL_TEST_SCHEMA=redis -Dcargo-build.features=services-redis env: OPENDAL_REDIS_TEST: on OPENDAL_REDIS_ENDPOINT: tcp://127.0.0.1:6379 diff --git a/.github/workflows/service_test_s3.yml b/.github/workflows/service_test_s3.yml index 10ae271398d7..207fb7ab3931 100644 --- a/.github/workflows/service_test_s3.yml +++ b/.github/workflows/service_test_s3.yml @@ -222,7 +222,7 @@ jobs: - name: Test shell: bash working-directory: bindings/java - run: ./mvnw test -Dtest=org.apache.opendal.services.S3Test + run: ./mvnw test -Dtest=org.apache.opendal.OperatorTest -DOPENDAL_TEST_SCHEMA=s3 env: OPENDAL_S3_TEST: ${{ secrets.OPENDAL_S3_TEST }} OPENDAL_S3_ROOT: ${{ secrets.OPENDAL_S3_ROOT }} diff --git a/bindings/java/README.md b/bindings/java/README.md index fe68c44a5fa7..c713ab9d0437 100644 --- a/bindings/java/README.md +++ b/bindings/java/README.md @@ -86,22 +86,24 @@ Notice: The default will skip all benches if the env is not set. Currently, all tests are written in Java. +You must set the `OPENDAL_TEST_SCHEMA` parameter to specify the backends to be tested. + You can run all available backends tests with the following command: ```shell -./mvnw clean verify -Dcargo-build.features=services-redis +./mvnw clean verify -DOPENDAL_TEST_SCHEMA=redis -Dcargo-build.features=services-redis ``` Test specific backend(such as `fs`). ```shell -./mvnw test -Dtest=org.apache.opendal.services.FsTest +./mvnw test -Dtest=org.apache.opendal.OperatorTest -DOPENDAL_TEST_SCHEMA=fs ``` Test specific backend(such as `redis`). ```shell -./mvnw test -Dtest=org.apache.opendal.services.RedisTest -Dcargo-build.features=services-redis +./mvnw test -Dtest=org.apache.opendal.OperatorTest -DOPENDAL_TEST_SCHEMA=redis -Dcargo-build.features=services-redis ``` > **Note:** diff --git a/bindings/java/src/test/java/org/apache/opendal/AbstractOperatorTest.java b/bindings/java/src/test/java/org/apache/opendal/OperatorTest.java similarity index 83% rename from bindings/java/src/test/java/org/apache/opendal/AbstractOperatorTest.java rename to bindings/java/src/test/java/org/apache/opendal/OperatorTest.java index 087e6896bab2..c832086818da 100644 --- a/bindings/java/src/test/java/org/apache/opendal/AbstractOperatorTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/OperatorTest.java @@ -32,27 +32,27 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -@TestInstance(TestInstance.Lifecycle.PER_CLASS) -public abstract class AbstractOperatorTest { +public class OperatorTest { - protected Optional opOptional; + private static final String SCHEMA_KEY = "OPENDAL_TEST_SCHEMA"; - protected Optional blockingOpOptional; + protected static Optional opOptional; - protected abstract String schema(); + protected static Optional blockingOpOptional; @BeforeAll - public void init() { - String schema = this.schema(); + public static void init() { + String schema = System.getProperty(SCHEMA_KEY); + assertTrue(schema != null && schema.length() > 0, () -> SCHEMA_KEY + " is not set"); + opOptional = Utils.init(schema); blockingOpOptional = Utils.initBlockingOp(schema); } @AfterAll - public void clean() { + public static void clean() { opOptional.ifPresent(op -> op.close()); blockingOpOptional.ifPresent(op -> op.close()); } @@ -63,6 +63,10 @@ public void testBlockingWrite() { return; } BlockingOperator blockingOp = blockingOpOptional.get(); + Capability cap = blockingOp.info().fullCapability; + if (!cap.write || !cap.read) { + return; + } String path = UUID.randomUUID().toString(); byte[] content = Utils.generateBytes(); @@ -83,6 +87,10 @@ public void testBlockingRead() { return; } BlockingOperator blockingOp = blockingOpOptional.get(); + Capability cap = blockingOp.info().fullCapability; + if (!cap.write || !cap.read) { + return; + } Metadata metadata = blockingOp.stat(""); assertTrue(!metadata.isFile()); @@ -107,6 +115,10 @@ public final void testWrite() throws Exception { return; } Operator op = opOptional.get(); + Capability cap = op.info().join().fullCapability; + if (!cap.write || !cap.read) { + return; + } String path = UUID.randomUUID().toString(); byte[] content = Utils.generateBytes(); @@ -127,6 +139,10 @@ public final void testRead() throws Exception { return; } Operator op = opOptional.get(); + Capability cap = op.info().join().fullCapability; + if (!cap.write || !cap.read) { + return; + } Metadata metadata = op.stat("").get(); assertTrue(!metadata.isFile()); @@ -151,6 +167,10 @@ public void testAppend() { return; } Operator op = opOptional.get(); + Capability cap = op.info().join().fullCapability; + if (!cap.write || !cap.writeCanAppend || !cap.read) { + return; + } String path = UUID.randomUUID().toString(); String[] trunks = new String[] {Utils.generateString(), Utils.generateString(), Utils.generateString()}; diff --git a/bindings/java/src/test/java/org/apache/opendal/services/FsTest.java b/bindings/java/src/test/java/org/apache/opendal/services/FsTest.java deleted file mode 100644 index a5f1b024cfec..000000000000 --- a/bindings/java/src/test/java/org/apache/opendal/services/FsTest.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.opendal.services; - -import org.apache.opendal.AbstractOperatorTest; - -public class FsTest extends AbstractOperatorTest { - - @Override - public String schema() { - return "fs"; - } -} diff --git a/bindings/java/src/test/java/org/apache/opendal/services/MemoryTest.java b/bindings/java/src/test/java/org/apache/opendal/services/MemoryTest.java deleted file mode 100644 index 794605301d21..000000000000 --- a/bindings/java/src/test/java/org/apache/opendal/services/MemoryTest.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.opendal.services; - -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import org.apache.opendal.AbstractOperatorTest; -import org.apache.opendal.OpenDALException; -import org.apache.opendal.condition.OpenDALExceptionCondition; -import org.junit.jupiter.api.Test; - -public class MemoryTest extends AbstractOperatorTest { - - @Override - public String schema() { - return "Memory"; - } - - @Test - @Override - public void testAppend() { - if (!opOptional.isPresent()) { - return; - } - assertThatThrownBy(() -> super.testAppend()) - .is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.Unsupported)); - } -} diff --git a/bindings/java/src/test/java/org/apache/opendal/services/RedisTest.java b/bindings/java/src/test/java/org/apache/opendal/services/RedisTest.java deleted file mode 100644 index 209944570ac0..000000000000 --- a/bindings/java/src/test/java/org/apache/opendal/services/RedisTest.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.opendal.services; - -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import org.apache.opendal.AbstractOperatorTest; -import org.apache.opendal.OpenDALException; -import org.apache.opendal.condition.OpenDALExceptionCondition; -import org.junit.jupiter.api.Test; - -public class RedisTest extends AbstractOperatorTest { - - @Override - public String schema() { - return "redis"; - } - - @Test - @Override - public void testAppend() { - if (!opOptional.isPresent()) { - return; - } - assertThatThrownBy(() -> super.testAppend()) - .is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.Unsupported)); - } -} diff --git a/bindings/java/src/test/java/org/apache/opendal/services/S3Test.java b/bindings/java/src/test/java/org/apache/opendal/services/S3Test.java deleted file mode 100644 index b778819c0cd5..000000000000 --- a/bindings/java/src/test/java/org/apache/opendal/services/S3Test.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.opendal.services; - -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import org.apache.opendal.AbstractOperatorTest; -import org.apache.opendal.OpenDALException; -import org.apache.opendal.condition.OpenDALExceptionCondition; -import org.junit.jupiter.api.Test; - -public class S3Test extends AbstractOperatorTest { - - @Override - public String schema() { - return "s3"; - } - - @Test - @Override - public void testAppend() { - if (!opOptional.isPresent()) { - return; - } - assertThatThrownBy(() -> super.testAppend()) - .is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.Unsupported)); - } -} From accfdc853e0d96073db74f58d3d9605e8d57c21b Mon Sep 17 00:00:00 2001 From: G-XD Date: Thu, 21 Sep 2023 13:08:53 +0800 Subject: [PATCH 17/24] test(binding/java): fix code style --- bindings/java/src/lib.rs | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/bindings/java/src/lib.rs b/bindings/java/src/lib.rs index c8d6d4d2b01f..9e90160d4981 100644 --- a/bindings/java/src/lib.rs +++ b/bindings/java/src/lib.rs @@ -36,8 +36,6 @@ use opendal::OperatorInfo; use tokio::runtime::Builder; use tokio::runtime::Runtime; -use crate::error::Error; - mod blocking_operator; mod error; mod metadata; @@ -230,14 +228,9 @@ fn make_capability<'a>(env: &mut JNIEnv<'a>, cap: Capability) -> Result(env: &mut JNIEnv<'a>, value: Option) -> Result> { let long_class = env.find_class("java/lang/Long")?; - let result; - match value { - Some(val) => { - result = env.new_object(long_class, "(J)V", &[JValue::Long(val as jlong)])?; - } - None => { - result = JObject::null(); - } - } + let result = match value { + Some(val) => env.new_object(long_class, "(J)V", &[JValue::Long(val as jlong)])?, + None => JObject::null(), + }; Ok(result) } From 30d4df80e089a4307d32e8f836c4a9565d01a5b7 Mon Sep 17 00:00:00 2001 From: G-XD Date: Thu, 21 Sep 2023 15:23:27 +0800 Subject: [PATCH 18/24] test(binding/java): reuse the .env file --- bindings/java/README.md | 2 +- .../java/org/apache/opendal/OperatorInfo.java | 7 ++++++- .../java/org/apache/opendal/utils/Utils.java | 18 +++++++++++++++--- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/bindings/java/README.md b/bindings/java/README.md index c713ab9d0437..70be5381923d 100644 --- a/bindings/java/README.md +++ b/bindings/java/README.md @@ -64,7 +64,7 @@ You can use Maven to build both Rust dynamic lib and JAR files with one command ## Setup Tests -Please copy `{project.rootdir}/.env.example` to `src/test/resources/.env` and change the values on need. +Please copy `{project.rootdir}/.env.example` to `{project.rootdir}/.env` and change the values on need. Take `fs` for example, we need to enable bench on `fs` on `/tmp`. diff --git a/bindings/java/src/main/java/org/apache/opendal/OperatorInfo.java b/bindings/java/src/main/java/org/apache/opendal/OperatorInfo.java index d9e1e600d812..c170f33559cb 100644 --- a/bindings/java/src/main/java/org/apache/opendal/OperatorInfo.java +++ b/bindings/java/src/main/java/org/apache/opendal/OperatorInfo.java @@ -19,6 +19,7 @@ package org.apache.opendal; +import lombok.NonNull; import lombok.ToString; @ToString @@ -30,7 +31,11 @@ public class OperatorInfo { public final Capability nativeCapability; public OperatorInfo( - String scheme, String root, String name, Capability fullCapability, Capability nativeCapability) { + @NonNull String scheme, + @NonNull String root, + @NonNull String name, + @NonNull Capability fullCapability, + @NonNull Capability nativeCapability) { this.scheme = scheme; this.root = root; this.name = name; diff --git a/bindings/java/src/test/java/org/apache/opendal/utils/Utils.java b/bindings/java/src/test/java/org/apache/opendal/utils/Utils.java index 19f01a80a66b..e36ea3fd1309 100644 --- a/bindings/java/src/test/java/org/apache/opendal/utils/Utils.java +++ b/bindings/java/src/test/java/org/apache/opendal/utils/Utils.java @@ -19,8 +19,11 @@ package org.apache.opendal.utils; +import java.io.BufferedReader; import java.io.File; -import java.io.InputStream; +import java.io.FileReader; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -85,8 +88,17 @@ public static Optional initBlockingOp(String schema) { private static Map readEnv(String schema) { final Properties properties = new Properties(); - try (InputStream is = Utils.class.getClassLoader().getResourceAsStream(ENV_NAME)) { - properties.load(is); + String projectRoot = System.getProperty("user.dir"); + + projectRoot = Optional.ofNullable(Paths.get(projectRoot)) + .map(Path::getParent) + .map(Path::getParent) + .map(Path::toString) + .orElse(projectRoot); + + try (BufferedReader reader = + new BufferedReader(new FileReader(projectRoot.toString() + File.separator + ENV_NAME))) { + properties.load(reader); } catch (Exception ignore) { } for (Map.Entry entry : System.getenv().entrySet()) { From cefcc730926e5c71a1b35f2a15a4c9d52f9b047f Mon Sep 17 00:00:00 2001 From: G-XD Date: Thu, 21 Sep 2023 15:29:04 +0800 Subject: [PATCH 19/24] ci(binding/java): fix test command --- .github/workflows/bindings_java.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/bindings_java.yml b/.github/workflows/bindings_java.yml index 21b68607bf43..fb2e0374e79a 100644 --- a/.github/workflows/bindings_java.yml +++ b/.github/workflows/bindings_java.yml @@ -84,8 +84,5 @@ jobs: # https://maven.apache.org/guides/mini/guide-reproducible-builds.html#how-to-test-my-maven-build-reproducibility shell: bash run: | - if [[ "${{ runner.os }}" != Linux ]]; then - export NO_DOCKER=true - fi ./mvnw clean install -DskipTests -Dcargo-build.features=services-redis - ./mvnw verify artifact:compare -Dcargo-build.features=services-redis + ./mvnw verify artifact:compare -DOPENDAL_TEST_SCHEMA=redis -Dcargo-build.features=services-redis From 5d9bfd966ad94f35fa65effae7b4c255dc18afe5 Mon Sep 17 00:00:00 2001 From: G-XD Date: Fri, 22 Sep 2023 00:43:06 +0800 Subject: [PATCH 20/24] test(binding/java): init all services test that OPENDAL__ON is true --- .github/workflows/service_test_redis.yml | 2 +- .github/workflows/service_test_s3.yml | 2 +- .../java/org/apache/opendal/enums/Schema.java | 111 ++++++++++++++++ .../java/org/apache/opendal/OperatorTest.java | 120 ++++++++++-------- .../java/org/apache/opendal/utils/Utils.java | 32 +---- 5 files changed, 187 insertions(+), 80 deletions(-) create mode 100644 bindings/java/src/main/java/org/apache/opendal/enums/Schema.java diff --git a/.github/workflows/service_test_redis.yml b/.github/workflows/service_test_redis.yml index 1fa9ea6e3168..4b13ee770696 100644 --- a/.github/workflows/service_test_redis.yml +++ b/.github/workflows/service_test_redis.yml @@ -209,7 +209,7 @@ jobs: - name: Test shell: bash working-directory: bindings/java - run: ./mvnw test -Dtest=org.apache.opendal.OperatorTest -DOPENDAL_TEST_SCHEMA=redis -Dcargo-build.features=services-redis + run: ./mvnw test -Dtest=org.apache.opendal.OperatorTest -Dcargo-build.features=services-redis env: OPENDAL_REDIS_TEST: on OPENDAL_REDIS_ENDPOINT: tcp://127.0.0.1:6379 diff --git a/.github/workflows/service_test_s3.yml b/.github/workflows/service_test_s3.yml index d326f48a3f02..a6ad92d0588b 100644 --- a/.github/workflows/service_test_s3.yml +++ b/.github/workflows/service_test_s3.yml @@ -254,7 +254,7 @@ jobs: - name: Test shell: bash working-directory: bindings/java - run: ./mvnw test -Dtest=org.apache.opendal.OperatorTest -DOPENDAL_TEST_SCHEMA=s3 + run: ./mvnw test -Dtest=org.apache.opendal.OperatorTest env: OPENDAL_S3_TEST: ${{ secrets.OPENDAL_S3_TEST }} OPENDAL_S3_ROOT: ${{ secrets.OPENDAL_S3_ROOT }} diff --git a/bindings/java/src/main/java/org/apache/opendal/enums/Schema.java b/bindings/java/src/main/java/org/apache/opendal/enums/Schema.java new file mode 100644 index 000000000000..5ca7519f8777 --- /dev/null +++ b/bindings/java/src/main/java/org/apache/opendal/enums/Schema.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.opendal.enums; + +public enum Schema { + Atomicserver("atomicserver"), + + Azblob("azblob"), + + Azdls("azdls"), + + Cacache("cacache"), + + Cos("cos"), + + Dashmap("dashmap"), + + Etcd("etcd"), + + Foundationdb("foundationdb"), + + Fs("fs"), + + Ftp("ftp"), + + Gcs("gcs"), + + Ghac("ghac"), + + Hdfs("hdfs"), + + Http("http"), + + Ipfs("ipfs"), + + Ipmfs("ipmfs"), + + Memcached("memcached"), + + Memory("memory"), + + MiniMoka("minimoka"), + + Moka("moka"), + + Obs("obs"), + + Onedrive("onedrive"), + + Gdrive("gdrive"), + + Dropbox("dropbox"), + + Oss("oss"), + + Persy("persy"), + + Redis("redis"), + + Postgresql("postgresql"), + + Rocksdb("rocksdb"), + + S3("s3"), + + Sftp("sftp"), + + Sled("sled"), + + Supabase("supabase"), + + VercelArtifacts("vercel-artifacts"), + + Wasabi("wasabi"), + + Webdav("webdav"), + + Webhdfs("webhdfs"), + + Redb("redb"), + + Tikv("tikv"), + ; + + private final String servicesName; + + private Schema(String name) { + this.servicesName = name; + } + + public String getServicesName() { + return servicesName; + } +} diff --git a/bindings/java/src/test/java/org/apache/opendal/OperatorTest.java b/bindings/java/src/test/java/org/apache/opendal/OperatorTest.java index c832086818da..56a746195fbf 100644 --- a/bindings/java/src/test/java/org/apache/opendal/OperatorTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/OperatorTest.java @@ -23,46 +23,56 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.Optional; import java.util.UUID; -import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.opendal.condition.OpenDALExceptionCondition; +import org.apache.opendal.enums.Schema; import org.apache.opendal.utils.Utils; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; public class OperatorTest { - private static final String SCHEMA_KEY = "OPENDAL_TEST_SCHEMA"; + protected static final List ops = new ArrayList<>(); - protected static Optional opOptional; - - protected static Optional blockingOpOptional; + protected static final List blockingOps = new ArrayList<>(); @BeforeAll public static void init() { - String schema = System.getProperty(SCHEMA_KEY); - assertTrue(schema != null && schema.length() > 0, () -> SCHEMA_KEY + " is not set"); + for (Schema schema : Schema.values()) { - opOptional = Utils.init(schema); + Optional opOptional = Utils.init(schema); + opOptional.ifPresent(op -> ops.add(op)); - blockingOpOptional = Utils.initBlockingOp(schema); + Optional blockingOpOptional = Utils.initBlockingOp(schema); + blockingOpOptional.ifPresent(op -> blockingOps.add(op)); + } } @AfterAll - public static void clean() { - opOptional.ifPresent(op -> op.close()); - blockingOpOptional.ifPresent(op -> op.close()); + public static void clean() throws InterruptedException { + ops.forEach(Operator::close); + blockingOps.forEach(BlockingOperator::close); } - @Test - public void testBlockingWrite() { - if (!blockingOpOptional.isPresent()) { - return; - } - BlockingOperator blockingOp = blockingOpOptional.get(); + private static List getOperators() { + return ops; + } + + private static List getBlockingOperators() { + return blockingOps; + } + + @ParameterizedTest(autoCloseArguments = false) + @MethodSource("getBlockingOperators") + public void testBlockingWrite(BlockingOperator blockingOp) { + Capability cap = blockingOp.info().fullCapability; if (!cap.write || !cap.read) { return; @@ -81,12 +91,9 @@ public void testBlockingWrite() { .is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.NotFound)); } - @Test - public void testBlockingRead() { - if (!blockingOpOptional.isPresent()) { - return; - } - BlockingOperator blockingOp = blockingOpOptional.get(); + @ParameterizedTest(autoCloseArguments = false) + @MethodSource("getBlockingOperators") + public void testBlockingRead(BlockingOperator blockingOp) { Capability cap = blockingOp.info().fullCapability; if (!cap.write || !cap.read) { return; @@ -99,7 +106,7 @@ public void testBlockingRead() { assertThatThrownBy(() -> blockingOp.stat(path)) .is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.NotFound)); - String content = Utils.generateString(); + byte[] content = Utils.generateBytes(); blockingOp.write(path, content); assertThat(blockingOp.read(path)).isEqualTo(content); @@ -109,13 +116,10 @@ public void testBlockingRead() { .is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.NotFound)); } - @Test - public final void testWrite() throws Exception { - if (!opOptional.isPresent()) { - return; - } - Operator op = opOptional.get(); - Capability cap = op.info().join().fullCapability; + @ParameterizedTest(autoCloseArguments = false) + @MethodSource("getOperators") + public final void testWrite(Operator op) throws Exception { + Capability cap = op.info().fullCapability; if (!cap.write || !cap.read) { return; } @@ -133,13 +137,10 @@ public final void testWrite() throws Exception { .is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.NotFound)); } - @Test - public final void testRead() throws Exception { - if (!opOptional.isPresent()) { - return; - } - Operator op = opOptional.get(); - Capability cap = op.info().join().fullCapability; + @ParameterizedTest(autoCloseArguments = false) + @MethodSource("getOperators") + public final void testRead(Operator op) throws Exception { + Capability cap = op.info().fullCapability; if (!cap.write || !cap.read) { return; } @@ -151,7 +152,7 @@ public final void testRead() throws Exception { assertThatThrownBy(() -> op.stat(path).join()) .is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.NotFound)); - String content = Utils.generateString(); + byte[] content = Utils.generateBytes(); op.write(path, content).join(); assertThat(op.read(path).join()).isEqualTo(content); @@ -161,35 +162,48 @@ public final void testRead() throws Exception { .is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.NotFound)); } - @Test - public void testAppend() { - if (!opOptional.isPresent()) { - return; - } - Operator op = opOptional.get(); - Capability cap = op.info().join().fullCapability; + @ParameterizedTest(autoCloseArguments = false) + @MethodSource("getOperators") + public void testAppend(Operator op) { + Capability cap = op.info().fullCapability; if (!cap.write || !cap.writeCanAppend || !cap.read) { return; } String path = UUID.randomUUID().toString(); - String[] trunks = new String[] {Utils.generateString(), Utils.generateString(), Utils.generateString()}; + byte[][] trunks = new byte[][] {Utils.generateBytes(), Utils.generateBytes(), Utils.generateBytes()}; for (int i = 0; i < trunks.length; i++) { op.append(path, trunks[i]).join(); - String expected = Arrays.stream(trunks).limit(i + 1).collect(Collectors.joining()); + + byte[] expected = Arrays.stream(trunks).limit(i + 1).reduce(new byte[0], (arr1, arr2) -> { + byte[] result = new byte[arr1.length + arr2.length]; + System.arraycopy(arr1, 0, result, 0, arr1.length); + System.arraycopy(arr2, 0, result, arr1.length, arr2.length); + return result; + }); + assertThat(op.read(path).join()).isEqualTo(expected); } // write overwrite existing content - String newAttempt = Utils.generateString(); + byte[] newAttempt = Utils.generateBytes(); op.write(path, newAttempt).join(); assertThat(op.read(path).join()).isEqualTo(newAttempt); for (int i = 0; i < trunks.length; i++) { op.append(path, trunks[i]).join(); - String expected = Arrays.stream(trunks).limit(i + 1).collect(Collectors.joining()); - assertThat(op.read(path).join()).isEqualTo(newAttempt + expected); + + byte[] expected = Stream.concat( + Stream.of(newAttempt), Arrays.stream(trunks).limit(i + 1)) + .reduce(new byte[0], (arr1, arr2) -> { + byte[] result = new byte[arr1.length + arr2.length]; + System.arraycopy(arr1, 0, result, 0, arr1.length); + System.arraycopy(arr2, 0, result, arr1.length, arr2.length); + return result; + }); + + assertThat(op.read(path).join()).isEqualTo(expected); } } } diff --git a/bindings/java/src/test/java/org/apache/opendal/utils/Utils.java b/bindings/java/src/test/java/org/apache/opendal/utils/Utils.java index e36ea3fd1309..54537562eb5c 100644 --- a/bindings/java/src/test/java/org/apache/opendal/utils/Utils.java +++ b/bindings/java/src/test/java/org/apache/opendal/utils/Utils.java @@ -33,6 +33,7 @@ import java.util.stream.Collectors; import org.apache.opendal.BlockingOperator; import org.apache.opendal.Operator; +import org.apache.opendal.enums.Schema; public class Utils { @@ -49,14 +50,14 @@ public class Utils { * @return If `opendal_{schema}_test` is on, construct a new Operator with given root. * Else, returns a `Empty` to represent no valid config for operator. */ - public static Optional init(String schema) { - Map conf = readEnv(schema); + public static Optional init(Schema schema) { + Map conf = readEnv(schema.getServicesName()); final String turnOnTest = conf.get(CONF_TURN_ON_TEST); if (!isTurnOn(turnOnTest)) { return Optional.empty(); } - Operator op = new Operator(schema, conf); + Operator op = new Operator(schema.getServicesName(), conf); return Optional.of(op); } @@ -67,14 +68,14 @@ public static Optional init(String schema) { * @return If `opendal_{schema}_test` is on, construct a new BlockingOperator with given root. * Else, returns a `Empty` to represent no valid config for operator. */ - public static Optional initBlockingOp(String schema) { - Map conf = readEnv(schema); + public static Optional initBlockingOp(Schema schema) { + Map conf = readEnv(schema.getServicesName()); final String turnOnTest = conf.get(CONF_TURN_ON_TEST); if (!isTurnOn(turnOnTest)) { return Optional.empty(); } - BlockingOperator op = new BlockingOperator(schema, conf); + BlockingOperator op = new BlockingOperator(schema.getServicesName(), conf); return Optional.of(op); } @@ -158,23 +159,4 @@ public static byte[] generateBytes() { return content; } - - /** - * Generates a random string of lowercase letters with a length between 1 and 256. - * - * @return the randomly generated string - */ - public static String generateString() { - Random random = new Random(); - - int length = random.nextInt(256) + 1; - StringBuilder stringBuilder = new StringBuilder(); - - for (int i = 0; i < length; i++) { - int randomChar = random.nextInt(26); - stringBuilder.append((char) ('a' + randomChar)); - } - - return stringBuilder.toString(); - } } From f3019b911024919acdf413a9318e7da6320b91b6 Mon Sep 17 00:00:00 2001 From: G-XD Date: Fri, 22 Sep 2023 00:43:48 +0800 Subject: [PATCH 21/24] ci(binding/java): del test --- .github/workflows/bindings_java.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/bindings_java.yml b/.github/workflows/bindings_java.yml index fb2e0374e79a..3aadb5a67869 100644 --- a/.github/workflows/bindings_java.yml +++ b/.github/workflows/bindings_java.yml @@ -85,4 +85,3 @@ jobs: shell: bash run: | ./mvnw clean install -DskipTests -Dcargo-build.features=services-redis - ./mvnw verify artifact:compare -DOPENDAL_TEST_SCHEMA=redis -Dcargo-build.features=services-redis From f6d8395215111c45cd280c3fd8a190c15b03886d Mon Sep 17 00:00:00 2001 From: G-XD Date: Fri, 22 Sep 2023 01:46:21 +0800 Subject: [PATCH 22/24] test(binding/java): skip test when no services turn on --- .../java/org/apache/opendal/OperatorTest.java | 31 ++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/bindings/java/src/test/java/org/apache/opendal/OperatorTest.java b/bindings/java/src/test/java/org/apache/opendal/OperatorTest.java index 56a746195fbf..96f18cde5b60 100644 --- a/bindings/java/src/test/java/org/apache/opendal/OperatorTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/OperatorTest.java @@ -23,9 +23,11 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.UUID; import java.util.stream.Stream; @@ -53,25 +55,32 @@ public static void init() { Optional blockingOpOptional = Utils.initBlockingOp(schema); blockingOpOptional.ifPresent(op -> blockingOps.add(op)); } + if (ops.isEmpty()) { + ops.add(null); + } + if (blockingOps.isEmpty()) { + blockingOps.add(null); + } } @AfterAll - public static void clean() throws InterruptedException { - ops.forEach(Operator::close); - blockingOps.forEach(BlockingOperator::close); + public static void clean() { + ops.stream().filter(Objects::nonNull).forEach(Operator::close); + blockingOps.stream().filter(Objects::nonNull).forEach(BlockingOperator::close); } - private static List getOperators() { - return ops; + private static Stream getOperators() { + return ops.stream(); } - private static List getBlockingOperators() { - return blockingOps; + private static Stream getBlockingOperators() { + return blockingOps.stream(); } @ParameterizedTest(autoCloseArguments = false) @MethodSource("getBlockingOperators") public void testBlockingWrite(BlockingOperator blockingOp) { + assumeTrue(blockingOp != null); Capability cap = blockingOp.info().fullCapability; if (!cap.write || !cap.read) { @@ -94,6 +103,8 @@ public void testBlockingWrite(BlockingOperator blockingOp) { @ParameterizedTest(autoCloseArguments = false) @MethodSource("getBlockingOperators") public void testBlockingRead(BlockingOperator blockingOp) { + assumeTrue(blockingOp != null); + Capability cap = blockingOp.info().fullCapability; if (!cap.write || !cap.read) { return; @@ -119,6 +130,8 @@ public void testBlockingRead(BlockingOperator blockingOp) { @ParameterizedTest(autoCloseArguments = false) @MethodSource("getOperators") public final void testWrite(Operator op) throws Exception { + assumeTrue(op != null); + Capability cap = op.info().fullCapability; if (!cap.write || !cap.read) { return; @@ -140,6 +153,8 @@ public final void testWrite(Operator op) throws Exception { @ParameterizedTest(autoCloseArguments = false) @MethodSource("getOperators") public final void testRead(Operator op) throws Exception { + assumeTrue(op != null); + Capability cap = op.info().fullCapability; if (!cap.write || !cap.read) { return; @@ -165,6 +180,8 @@ public final void testRead(Operator op) throws Exception { @ParameterizedTest(autoCloseArguments = false) @MethodSource("getOperators") public void testAppend(Operator op) { + assumeTrue(op != null); + Capability cap = op.info().fullCapability; if (!cap.write || !cap.writeCanAppend || !cap.read) { return; From 658a675a85bf70418b64a6df35340e3ac49e4fa4 Mon Sep 17 00:00:00 2001 From: G-XD Date: Fri, 22 Sep 2023 02:12:40 +0800 Subject: [PATCH 23/24] doc(binding/java): update doc --- bindings/java/README.md | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/bindings/java/README.md b/bindings/java/README.md index 70be5381923d..ef4f65107791 100644 --- a/bindings/java/README.md +++ b/bindings/java/README.md @@ -62,7 +62,17 @@ You can use Maven to build both Rust dynamic lib and JAR files with one command ./mvnw clean package -DskipTests=true ``` -## Setup Tests +## Run tests + +Currently, all tests are written in Java. + +You can run the base tests with the following command: + +```shell +./mvnw clean verify +``` + +## Run Service Tests Please copy `{project.rootdir}/.env.example` to `{project.rootdir}/.env` and change the values on need. @@ -77,33 +87,19 @@ into ```dotenv OPENDAL_FS_TEST=on -OPENDAL_FS_ROOT=/tmp -``` - -Notice: The default will skip all benches if the env is not set. - -## Run tests - -Currently, all tests are written in Java. - -You must set the `OPENDAL_TEST_SCHEMA` parameter to specify the backends to be tested. - -You can run all available backends tests with the following command: - -```shell -./mvnw clean verify -DOPENDAL_TEST_SCHEMA=redis -Dcargo-build.features=services-redis +OPENDAL_FS_ROOT=/opendal ``` -Test specific backend(such as `fs`). +You can run service tests of enabled with the following command: ```shell -./mvnw test -Dtest=org.apache.opendal.OperatorTest -DOPENDAL_TEST_SCHEMA=fs +./mvnw test -Dtest=org.apache.opendal.OperatorTest ``` -Test specific backend(such as `redis`). +You can run the unbound service with the following command: ```shell -./mvnw test -Dtest=org.apache.opendal.OperatorTest -DOPENDAL_TEST_SCHEMA=redis -Dcargo-build.features=services-redis +./mvnw test -Dtest=org.apache.opendal.OperatorTest -Dcargo-build.features=services-redis ``` > **Note:** From aee4b5c780f825cc20d2e9b4c3c5390d3a5ecf1d Mon Sep 17 00:00:00 2001 From: G-XD Date: Fri, 22 Sep 2023 12:24:49 +0800 Subject: [PATCH 24/24] test(binding/java): del schema enum --- .../java/org/apache/opendal/enums/Schema.java | 111 ------------------ .../java/org/apache/opendal/OperatorTest.java | 51 +++++++- .../java/org/apache/opendal/utils/Utils.java | 13 +- 3 files changed, 52 insertions(+), 123 deletions(-) delete mode 100644 bindings/java/src/main/java/org/apache/opendal/enums/Schema.java diff --git a/bindings/java/src/main/java/org/apache/opendal/enums/Schema.java b/bindings/java/src/main/java/org/apache/opendal/enums/Schema.java deleted file mode 100644 index 5ca7519f8777..000000000000 --- a/bindings/java/src/main/java/org/apache/opendal/enums/Schema.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.opendal.enums; - -public enum Schema { - Atomicserver("atomicserver"), - - Azblob("azblob"), - - Azdls("azdls"), - - Cacache("cacache"), - - Cos("cos"), - - Dashmap("dashmap"), - - Etcd("etcd"), - - Foundationdb("foundationdb"), - - Fs("fs"), - - Ftp("ftp"), - - Gcs("gcs"), - - Ghac("ghac"), - - Hdfs("hdfs"), - - Http("http"), - - Ipfs("ipfs"), - - Ipmfs("ipmfs"), - - Memcached("memcached"), - - Memory("memory"), - - MiniMoka("minimoka"), - - Moka("moka"), - - Obs("obs"), - - Onedrive("onedrive"), - - Gdrive("gdrive"), - - Dropbox("dropbox"), - - Oss("oss"), - - Persy("persy"), - - Redis("redis"), - - Postgresql("postgresql"), - - Rocksdb("rocksdb"), - - S3("s3"), - - Sftp("sftp"), - - Sled("sled"), - - Supabase("supabase"), - - VercelArtifacts("vercel-artifacts"), - - Wasabi("wasabi"), - - Webdav("webdav"), - - Webhdfs("webhdfs"), - - Redb("redb"), - - Tikv("tikv"), - ; - - private final String servicesName; - - private Schema(String name) { - this.servicesName = name; - } - - public String getServicesName() { - return servicesName; - } -} diff --git a/bindings/java/src/test/java/org/apache/opendal/OperatorTest.java b/bindings/java/src/test/java/org/apache/opendal/OperatorTest.java index 96f18cde5b60..f77529cdd793 100644 --- a/bindings/java/src/test/java/org/apache/opendal/OperatorTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/OperatorTest.java @@ -32,7 +32,6 @@ import java.util.UUID; import java.util.stream.Stream; import org.apache.opendal.condition.OpenDALExceptionCondition; -import org.apache.opendal.enums.Schema; import org.apache.opendal.utils.Utils; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -41,13 +40,55 @@ public class OperatorTest { - protected static final List ops = new ArrayList<>(); - - protected static final List blockingOps = new ArrayList<>(); + private static final List ops = new ArrayList<>(); + + private static final List blockingOps = new ArrayList<>(); + + protected static final String[] schemas = new String[] { + "atomicserver", + "azblob", + "azdls", + "cacache", + "cos", + "dashmap", + "etcd", + "foundationdb", + "fs", + "ftp", + "gcs", + "ghac", + "hdfs", + "http", + "ipfs", + "ipmfs", + "memcached", + "memory", + "minimoka", + "moka", + "obs", + "onedrive", + "gdrive", + "dropbox", + "oss", + "persy", + "redis", + "postgresql", + "rocksdb", + "s3", + "sftp", + "sled", + "supabase", + "vercel-artifacts", + "wasabi", + "webdav", + "webhdfs", + "redb", + "tikv", + }; @BeforeAll public static void init() { - for (Schema schema : Schema.values()) { + for (String schema : schemas) { Optional opOptional = Utils.init(schema); opOptional.ifPresent(op -> ops.add(op)); diff --git a/bindings/java/src/test/java/org/apache/opendal/utils/Utils.java b/bindings/java/src/test/java/org/apache/opendal/utils/Utils.java index 54537562eb5c..0121002d4973 100644 --- a/bindings/java/src/test/java/org/apache/opendal/utils/Utils.java +++ b/bindings/java/src/test/java/org/apache/opendal/utils/Utils.java @@ -33,7 +33,6 @@ import java.util.stream.Collectors; import org.apache.opendal.BlockingOperator; import org.apache.opendal.Operator; -import org.apache.opendal.enums.Schema; public class Utils { @@ -50,14 +49,14 @@ public class Utils { * @return If `opendal_{schema}_test` is on, construct a new Operator with given root. * Else, returns a `Empty` to represent no valid config for operator. */ - public static Optional init(Schema schema) { - Map conf = readEnv(schema.getServicesName()); + public static Optional init(String schema) { + Map conf = readEnv(schema); final String turnOnTest = conf.get(CONF_TURN_ON_TEST); if (!isTurnOn(turnOnTest)) { return Optional.empty(); } - Operator op = new Operator(schema.getServicesName(), conf); + Operator op = new Operator(schema, conf); return Optional.of(op); } @@ -68,14 +67,14 @@ public static Optional init(Schema schema) { * @return If `opendal_{schema}_test` is on, construct a new BlockingOperator with given root. * Else, returns a `Empty` to represent no valid config for operator. */ - public static Optional initBlockingOp(Schema schema) { - Map conf = readEnv(schema.getServicesName()); + public static Optional initBlockingOp(String schema) { + Map conf = readEnv(schema); final String turnOnTest = conf.get(CONF_TURN_ON_TEST); if (!isTurnOn(turnOnTest)) { return Optional.empty(); } - BlockingOperator op = new BlockingOperator(schema.getServicesName(), conf); + BlockingOperator op = new BlockingOperator(schema, conf); return Optional.of(op); }