Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: execlude zookeeper for curator #3899

Merged
merged 2 commits into from
Apr 26, 2024
Merged

feat: execlude zookeeper for curator #3899

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
aa84805
Execlude zookeeper when using curator
tobegit3hub Apr 26, 2024
f82f261
Fix local build java
tobegit3hub Apr 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion java/openmldb-batch/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,11 @@
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>3.4.14</version>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-framework</artifactId>
Expand All @@ -182,6 +186,12 @@
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<version>4.2.0</version>
<exclusions>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
</exclusions>
</dependency>

<!-- OpenMLDB -->
Expand Down
6 changes: 6 additions & 0 deletions java/openmldb-common/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,12 @@
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<version>4.2.0</version>
<exclusions>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.testng</groupId>
Expand Down
9 changes: 6 additions & 3 deletions java/openmldb-taskmanager/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,12 @@
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<version>4.2.0</version>
<exclusions>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
Expand All @@ -142,9 +148,6 @@
<scope>provided</scope>
</dependency>




<!-- Kubernetes -->
<dependency>
<groupId>io.fabric8</groupId>
Expand Down
22 changes: 11 additions & 11 deletions ...-taskmanager/src/main/java/com/_4paradigm/openmldb/taskmanager/server/JobResultSaver.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@
*/
@Slf4j
public class JobResultSaver {
private static final Log log = LogFactory.getLog(JobResultSaver.class);
private static final Log logger = LogFactory.getLog(JobResultSaver.class);

// false: unused, true: using
// 0: unused, 1: saving, 2: finished but still in use
Expand Down Expand Up @@ -92,8 +92,8 @@ public String genUniqueFileName() {
public boolean saveFile(int resultId, String jsonData) {
// No need to wait, cuz id status must have been changed by genResultId before.
// It's a check.
if (log.isDebugEnabled()) {
log.debug("save result " + resultId + ", data " + jsonData);
if (logger.isDebugEnabled()) {
logger.debug("save result " + resultId + ", data " + jsonData);
}
int status = idStatus.get(resultId);
if (status != 1) {
Expand All @@ -105,7 +105,7 @@ public boolean saveFile(int resultId, String jsonData) {
idStatus.set(resultId, 2);
idStatus.notifyAll();
}
log.info("saved all result of result " + resultId);
logger.info("saved all result of result " + resultId);
return true;
}
// save to <log path>/tmp_result/<result_id>/<unique file name>
Expand All @@ -114,7 +114,7 @@ public boolean saveFile(int resultId, String jsonData) {
File saveP = new File(savePath);
if (!saveP.exists()) {
boolean res = saveP.mkdirs();
log.info("create save path " + savePath + ", status " + res);
logger.info("create save path " + savePath + ", status " + res);
}
}
String fileFullPath = String.format("%s/%s", savePath, genUniqueFileName());
Expand All @@ -125,7 +125,7 @@ public boolean saveFile(int resultId, String jsonData) {
+ fileFullPath);
}
} catch (IOException e) {
log.error("create file failed, path " + fileFullPath, e);
logger.error("create file failed, path " + fileFullPath, e);
return false;
}

Expand All @@ -135,7 +135,7 @@ public boolean saveFile(int resultId, String jsonData) {
} catch (IOException e) {
// Write failed, we'll lost a part of result, but it's ok for show sync job
// output. So we just log it, and response the http request.
log.error("write result to file failed, path " + fileFullPath, e);
logger.error("write result to file failed, path " + fileFullPath, e);
return false;
}
return true;
Expand All @@ -151,7 +151,7 @@ public String readResult(int resultId, long timeoutMs) throws InterruptedExcepti
}
}
if (idStatus.get(resultId) != 2) {
log.warn("read result timeout, result saving may be still running, try read anyway, id " + resultId);
logger.warn("read result timeout, result saving may be still running, try read anyway, id " + resultId);
}
String output = "";
// all finished, read csv from savePath
Expand All @@ -163,7 +163,7 @@ public String readResult(int resultId, long timeoutMs) throws InterruptedExcepti
output = printFilesTostr(savePath);
FileUtils.forceDelete(saveP);
} else {
log.info("empty result for " + resultId + ", show empty string");
logger.info("empty result for " + resultId + ", show empty string");
}
// reset id
synchronized (idStatus) {
Expand All @@ -189,7 +189,7 @@ public String printFilesTostr(String fileDir) {
}
return stringWriter.toString();
} catch (Exception e) {
log.warn("read result met exception when read " + fileDir + ", " + e.getMessage());
logger.warn("read result met exception when read " + fileDir + ", " + e.getMessage());
e.printStackTrace();
return "read met exception, check the taskmanager log";
}
Expand Down Expand Up @@ -219,7 +219,7 @@ private void printFile(String file, StringWriter stringWriter, boolean printHead
csvPrinter.printRecord(iter.next());
}
} catch (Exception e) {
log.warn("error when print result file " + file + ", ignore it");
logger.warn("error when print result file " + file + ", ignore it");
e.printStackTrace();
}
}
Expand Down
2 changes: 1 addition & 1 deletion ...askmanager/src/main/java/com/_4paradigm/openmldb/taskmanager/zk/RecoverableZooKeeper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ public class RecoverableZooKeeper {
private final String quorumServers;
private final int maxMultiSize; // unused now

@edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "DE_MIGHT_IGNORE", justification = "None. Its always been this way.")
//@edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "DE_MIGHT_IGNORE", justification = "None. Its always been this way.")
public RecoverableZooKeeper(String quorumServers, int sessionTimeout, Watcher watcher) throws IOException {
// TODO: Add support for zk 'chroot'; we don't add it to the quorumServers
// String as we should.
Expand Down
Loading