Batch read

.github/workflows/pr.yaml

@@ -1,62 +1 @@
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-name: Unit Tests Check
-on:
-  pull_request:
-    branches:
-      - master
-
-jobs:
-
-  unit-tests-scala-2_11:
-    runs-on: ubuntu-latest
-    timeout-minutes: 120
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Set up JDK 1.8
-        uses: actions/setup-java@v1
-        with:
-          java-version: 1.8
-
-      - name: Set up Maven
-        uses: aahmed-se/setup-maven@v3
-        with:
-          maven-version: 3.6.1
-      - name: run precommit script
-        run: ./tools/retry.sh `pwd`/precommit.sh scala-2.11
-
-  unit-tests-scala-2_12:
-    runs-on: ubuntu-latest
-    timeout-minutes: 120
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Set up JDK 1.8
-        uses: actions/setup-java@v1
-        with:
-          java-version: 1.8
-
-      - name: Set up Maven
-        uses: aahmed-se/setup-maven@v3
-        with:
-          maven-version: 3.6.1
-
-      - name: run precommit script
-        run: ./tools/retry.sh `pwd`/precommit.sh scala-2.12
+## Licensed under the Apache License, Version 2.0 (the "License");# you may not use this file except in compliance with the License.# You may obtain a copy of the License at##     http://www.apache.org/licenses/LICENSE-2.0## Unless required by applicable law or agreed to in writing, software# distributed under the License is distributed on an "AS IS" BASIS,# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.#name: Unit Tests Checkon:  pull_request:    branches:      - masterjobs:  unit-tests-scala-2_11:    runs-on: ubuntu-latest    timeout-minutes: 120    steps:      - name: Checkout        uses: actions/checkout@v2      - name: Set up JDK 1.8        uses: actions/setup-java@v1        with:          java-version: 1.8      - name: Set up Maven        uses: aahmed-se/setup-maven@v3        with:          maven-version: 3.6.1      - name: run precommit script        run: ./tools/retry.sh `pwd`/precommit.sh scala-2.11  unit-tests-scala-2_12:    runs-on: ubuntu-latest    timeout-minutes: 120    steps:      - name: Checkout        uses: actions/checkout@v2      - name: Set up JDK 1.8        uses: actions/setup-java@v1        with:          java-version: 1.8      - name: Set up Maven        uses: aahmed-se/setup-maven@v3        with:          maven-version: 3.6.1      - name: run precommit script        run: ./tools/retry.sh `pwd`/precommit.sh scala-2.12  unit-tests-local:    runs-on: ubuntu-latest    timeout-minutes: 120    steps:      - name: Checkout        uses: actions/checkout@v2      - name: Set up JDK 1.8        uses: actions/setup-java@v1        with:          java-version: 1.8      - name: Set up Maven        uses: aahmed-se/setup-maven@v3        with:          maven-version: 3.6.1      - name: install Pulsar Service        run: |          apt update && apt install -y wget          wget -O 'apache-pulsar-2.6.1-bin.tar.gz' 'https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=pulsar/pulsar-2.6.1/apache-pulsar-2.6.1-bin.tar.gz'          tar -xvzf apache-pulsar-2.6.1-bin.tar.gz          cd apache-pulsar-2.6.1          chmod +x ./bin/*          nohup ./bin/pulsar standalone --no-stream-storage -nfw &      - name: run batch test        run: |          export PULSAR_ADMIN_URL=http://127.0.0.1:8080          export PULSAR_SERVICE_URL=pulsar://127.0.0.1:6650          export PULSAR_ZK_URL=127.0.0.1:2181          mvn -ntp install -DskipTests          mvn -ntp test -pl common -Dtest=org.apache.flink.batch.connectors.pulsar.PulsarInputFormatITest

common/pom.xml

@@ -29,6 +29,17 @@
     </properties>
 
     <dependencies>
+        <dependency>
+            <groupId>io.streamnative.connectors</groupId>
+            <artifactId>managed-ledger-shaded</artifactId>
+            <version>${project.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>*</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-table-runtime-blink_${scala.binary.version}</artifactId>

common/src/main/java/org/apache/flink/batch/connectors/pulsar/CachedClients.java

@@ -0,0 +1,179 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.batch.connectors.pulsar;
+
+import org.apache.flink.pulsar.common.ConnectorConfig;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.pulsar.PulsarVersion;
+import org.apache.pulsar.shade.com.google.common.collect.ImmutableMap;
+import org.apache.pulsar.shade.org.apache.bookkeeper.common.util.OrderedScheduler;
+import org.apache.pulsar.shade.org.apache.bookkeeper.conf.ClientConfiguration;
+import org.apache.pulsar.shade.org.apache.bookkeeper.mledger.LedgerOffloader;
+import org.apache.pulsar.shade.org.apache.bookkeeper.mledger.LedgerOffloaderFactory;
+import org.apache.pulsar.shade.org.apache.bookkeeper.mledger.ManagedLedgerConfig;
+import org.apache.pulsar.shade.org.apache.bookkeeper.mledger.ManagedLedgerFactoryConfig;
+import org.apache.pulsar.shade.org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl;
+import org.apache.pulsar.shade.org.apache.bookkeeper.mledger.impl.NullLedgerOffloader;
+import org.apache.pulsar.shade.org.apache.bookkeeper.mledger.offload.OffloaderUtils;
+import org.apache.pulsar.shade.org.apache.bookkeeper.mledger.offload.Offloaders;
+import org.apache.pulsar.shade.org.apache.bookkeeper.stats.StatsProvider;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static org.apache.flink.batch.connectors.pulsar.ConnectorUtils.createInstance;
+import static org.apache.flink.batch.connectors.pulsar.ConnectorUtils.getProperties;
+import static org.apache.pulsar.shade.com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * Cached ML / BK client to be shared among threads inside a process.
+ */
+@Slf4j
+public class CachedClients {
+
+    private static final String OFFLOADERS_DIRECTOR = "offloadersDirectory";
+    private static final String MANAGED_LEDGER_OFFLOAD_DRIVER = "managedLedgerOffloadDriver";
+    private static final String MANAGED_LEDGER_OFFLOAD_MAX_THREADS = "managedLedgerOffloadMaxThreads";
+    static CachedClients instance;
+    private final ManagedLedgerFactoryImpl managedLedgerFactory;
+    private final StatsProvider statsProvider;
+    private OrderedScheduler offloaderScheduler;
+    private Offloaders offloaderManager;
+    private LedgerOffloader offloader;
+
+    private CachedClients(ConnectorConfig config) throws Exception {
+        this.managedLedgerFactory = initManagedLedgerFactory(config);
+        this.statsProvider = createInstance(config.getStatsProvider(),
+                StatsProvider.class, getClass().getClassLoader());
+
+        // start stats provider
+        ClientConfiguration clientConfiguration = new ClientConfiguration();
+        config.getStatsProviderConfigs().forEach(clientConfiguration::setProperty);
+        this.statsProvider.start(clientConfiguration);
+
+        this.offloader = initManagedLedgerOffloader(config);
+    }
+
+    public static CachedClients getInstance(ConnectorConfig config) throws Exception {
+        synchronized (CachedClients.class) {
+            if (instance == null) {
+                instance = new CachedClients(config);
+            }
+        }
+        return instance;
+    }
+
+    public static void shutdown() throws Exception {
+        synchronized (CachedClients.class) {
+            if (instance != null) {
+                instance.statsProvider.stop();
+                instance.managedLedgerFactory.shutdown();
+                instance.offloaderScheduler.shutdown();
+                instance.offloaderManager.close();
+                instance = null;
+            }
+        }
+    }
+
+    private ManagedLedgerFactoryImpl initManagedLedgerFactory(ConnectorConfig config) throws Exception {
+        ClientConfiguration bkClientConfiguration = new ClientConfiguration()
+                .setZkServers(config.getZookeeperUri())
+                .setMetadataServiceUri("zk://" + config.getZookeeperUri() + "/ledgers")
+                .setClientTcpNoDelay(false)
+                .setUseV2WireProtocol(true)
+                .setStickyReadsEnabled(false)
+                .setAllowShadedLedgerManagerFactoryClass(true)
+                .setShadedLedgerManagerFactoryClassPrefix("org.apache.pulsar.shade.")
+                .setReadEntryTimeout(60)
+                .setThrottleValue(config.getBookkeeperThrottleValue())
+                .setNumIOThreads(config.getBookkeeperNumIOThreads())
+                .setNumWorkerThreads(config.getBookkeeperNumWorkerThreads());
+
+        ManagedLedgerFactoryConfig managedLedgerFactoryConfig = new ManagedLedgerFactoryConfig();
+        managedLedgerFactoryConfig.setMaxCacheSize(config.getManagedLedgerCacheSizeMB());
+        managedLedgerFactoryConfig.setNumManagedLedgerWorkerThreads(
+                config.getManagedLedgerNumWorkerThreads());
+        managedLedgerFactoryConfig.setNumManagedLedgerSchedulerThreads(
+                config.getManagedLedgerNumSchedulerThreads());
+
+        return new ManagedLedgerFactoryImpl(bkClientConfiguration, managedLedgerFactoryConfig);
+    }
+
+    private synchronized OrderedScheduler getOffloaderScheduler(ConnectorConfig config) {
+        if (this.offloaderScheduler == null) {
+            this.offloaderScheduler = OrderedScheduler.newSchedulerBuilder()
+                    .numThreads(config.getManagedLedgerOffloadMaxThreads())
+                    .name("pulsar-offloader").build();
+        }
+        return this.offloaderScheduler;
+    }
+
+    private LedgerOffloader initManagedLedgerOffloader(ConnectorConfig config) {
+
+        try {
+            if (StringUtils.isNotBlank(config.getManagedLedgerOffloadDriver())) {
+                checkNotNull(config.getOffloadersDirectory(),
+                        "Offloader driver is configured to be '%s' but no offloaders directory is configured.",
+                        config.getManagedLedgerOffloadDriver());
+                this.offloaderManager = OffloaderUtils.searchForOffloaders(config.getOffloadersDirectory(), "");
+                LedgerOffloaderFactory offloaderFactory = this.offloaderManager.getOffloaderFactory(
+                        config.getManagedLedgerOffloadDriver());
+
+                Map<String, String> offloaderProperties = config.getOffloaderProperties();
+                offloaderProperties.put(OFFLOADERS_DIRECTOR, config.getOffloadersDirectory());
+                offloaderProperties.put(MANAGED_LEDGER_OFFLOAD_DRIVER, config.getManagedLedgerOffloadDriver());
+                offloaderProperties
+                        .put(MANAGED_LEDGER_OFFLOAD_MAX_THREADS,
+                                String.valueOf(config.getManagedLedgerOffloadMaxThreads()));
+
+                try {
+                    return offloaderFactory.create(
+                            getProperties(offloaderProperties),
+                            ImmutableMap.of(
+                                    LedgerOffloader.METADATA_SOFTWARE_VERSION_KEY.toLowerCase(),
+                                    PulsarVersion.getVersion(),
+                                    LedgerOffloader.METADATA_SOFTWARE_GITSHA_KEY.toLowerCase(),
+                                    PulsarVersion.getGitSha()
+                            ),
+                            getOffloaderScheduler(config));
+                } catch (IOException ioe) {
+                    log.error("Failed to create offloader: ", ioe);
+                    throw new RuntimeException(ioe.getMessage(), ioe.getCause());
+                }
+            } else {
+                log.info("No ledger offloader configured, using NULL instance");
+                return NullLedgerOffloader.INSTANCE;
+            }
+        } catch (Throwable t) {
+            throw new RuntimeException(t);
+        }
+    }
+
+    public ManagedLedgerConfig getManagedLedgerConfig() {
+
+        return new ManagedLedgerConfig()
+                .setLedgerOffloader(this.offloader);
+    }
+
+    public ManagedLedgerFactoryImpl getManagedLedgerFactory() {
+        return managedLedgerFactory;
+    }
+
+    public StatsProvider getStatsProvider() {
+        return statsProvider;
+    }
+}

common/src/main/java/org/apache/flink/batch/connectors/pulsar/ConnectorUtils.java

@@ -0,0 +1,73 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.batch.connectors.pulsar;
+
+import org.apache.pulsar.common.policies.data.OffloadPolicies;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * Utilities in constructing CachedClients.
+ */
+public class ConnectorUtils {
+
+    /**
+     * Create an instance of <code>userClassName</code> using provided <code>classLoader</code>.
+     * This instance should implement the provided interface <code>xface</code>.
+     *
+     * @param userClassName user class name
+     * @param xface the interface that the reflected instance should implement
+     * @param classLoader class loader to load the class.
+     * @return the instance
+     */
+    public static <T> T createInstance(String userClassName,
+                                       Class<T> xface,
+                                       ClassLoader classLoader) {
+        Class<?> theCls;
+        try {
+            theCls = Class.forName(userClassName, true, classLoader);
+        } catch (ClassNotFoundException | NoClassDefFoundError cnfe) {
+            throw new RuntimeException("User class must be in class path", cnfe);
+        }
+        if (!xface.isAssignableFrom(theCls)) {
+            throw new RuntimeException(userClassName + " not " + xface.getName());
+        }
+        Class<T> tCls = (Class<T>) theCls.asSubclass(xface);
+        try {
+            Constructor<T> meth = tCls.getDeclaredConstructor();
+            return meth.newInstance();
+        } catch (InstantiationException ie) {
+            throw new RuntimeException("User class must be concrete", ie);
+        } catch (NoSuchMethodException e) {
+            throw new RuntimeException("User class must have a no-arg constructor", e);
+        } catch (IllegalAccessException e) {
+            throw new RuntimeException("User class must a public constructor", e);
+        } catch (InvocationTargetException e) {
+            throw new RuntimeException("User class constructor throws exception", e);
+        }
+    }
+
+    public static OffloadPolicies getProperties(Map<String, String> configMap) {
+        Properties properties = new Properties();
+        for (Map.Entry<String, String> entry : configMap.entrySet()) {
+            properties.setProperty(entry.getKey(), entry.getValue());
+        }
+        return OffloadPolicies.create(properties);
+    }
+
+}