download job related fix (#4754)

vesoft-inc · Oct 20, 2022 · da23f87 · da23f87
1 parent 3f69f27
commit da23f87
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 0 deletions.
diff --git a/src/common/hdfs/HdfsCommandHelper.cpp b/src/common/hdfs/HdfsCommandHelper.cpp
@@ -23,6 +23,7 @@ Status HdfsCommandHelper::ls(const std::string& hdfsHost,
     folly::Subprocess proc(std::vector<std::string>({"/bin/bash", "-c", command}));
     auto result = proc.wait();
     if (!result.exited()) {
+      LOG(INFO) << "Failed to ls: " << result.str();
       return Status::Error("Failed to ls hdfs");
     } else if (result.exitStatus() != 0) {
       LOG(INFO) << "Failed to ls: " << result.str();
@@ -49,6 +50,7 @@ Status HdfsCommandHelper::copyToLocal(const std::string& hdfsHost,
     folly::Subprocess proc(std::vector<std::string>({"/bin/bash", "-c", command}));
     auto result = proc.wait();
     if (!result.exited()) {
+      LOG(INFO) << "Failed to download: " << result.str();
       return Status::Error("Failed to download from hdfs");
     } else if (result.exitStatus() != 0) {
       LOG(INFO) << "Failed to download: " << result.str();

diff --git a/src/kvstore/RocksEngine.cpp b/src/kvstore/RocksEngine.cpp
@@ -438,6 +438,9 @@ int32_t RocksEngine::totalPartsNum() {
 
 nebula::cpp2::ErrorCode RocksEngine::ingest(const std::vector<std::string>& files,
                                             bool verifyFileChecksum) {
+  if (files.empty()) {
+    return nebula::cpp2::ErrorCode::SUCCEEDED;
+  }
   rocksdb::IngestExternalFileOptions options;
   options.move_files = FLAGS_move_files;
   options.verify_file_checksum = verifyFileChecksum;

diff --git a/src/kvstore/test/RocksEngineTest.cpp b/src/kvstore/test/RocksEngineTest.cpp
@@ -225,6 +225,8 @@ TEST_P(RocksEngineTest, IngestTest) {
   EXPECT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, engine->get("key_empty", &result));
   EXPECT_EQ("", result);
   EXPECT_EQ(nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND, engine->get("key_not_exist", &result));
+
+  EXPECT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, engine->ingest({}));
 }
 
 TEST_P(RocksEngineTest, BackupRestoreTable) {

diff --git a/src/storage/admin/DownloadTask.cpp b/src/storage/admin/DownloadTask.cpp
@@ -50,6 +50,13 @@ nebula::cpp2::ErrorCode DownloadTask::subTask(GraphSpaceID space, PartitionID pa
     }
   }
 
+  auto listResult = helper_->ls(hdfsHost_, hdfsPort_, hdfsPartPath);
+  if (!listResult.ok()) {
+    LOG(INFO) << "Can't found data of space: " << space << ", part: " << part
+              << ", just skip the part";
+    return nebula::cpp2::ErrorCode::SUCCEEDED;
+  }
+
   auto result = helper_->copyToLocal(hdfsHost_, hdfsPort_, hdfsPartPath, localPath);
   return result.ok() ? nebula::cpp2::ErrorCode::SUCCEEDED
                      : nebula::cpp2::ErrorCode::E_TASK_EXECUTION_FAILED;