Processing now works and produces H5AD, history items detect 'process…

…ing' state, resumable.
IGS · Aug 30, 2024 · 2f6642a · 2f6642a
1 parent 4bacc73
commit 2f6642a
Show file tree

Hide file tree

Showing 5 changed files with 50 additions and 18 deletions.
diff --git a/www/cgi/get_uploads_in_progress.cgi b/www/cgi/get_uploads_in_progress.cgi
@@ -71,7 +71,17 @@ def main():
             result['uploads'][-1]['status'] = 'datafile uploaded'
             result['uploads'][-1]['load_step'] = 'process-dataset'
 
-        # The user could have uploaded the data file and never clicked
+        processing_status_json_file = os.path.join(share_dir, 'status.json')
+
+        if os.path.exists(processing_status_json_file):
+            with open(processing_status_json_file, 'r') as f:
+                status_json = json.load(f)
+                processing_status = status_json.get('status', '')
+
+                if processing_status == 'processing':
+                    result['uploads'][-1]['status'] = 'processing'
+                    result['uploads'][-1]['load_step'] = 'process-dataset'
+
 
     result['success'] = 1
     print(json.dumps(result))

diff --git a/www/cgi/process_uploaded_expression_dataset.cgi b/www/cgi/process_uploaded_expression_dataset.cgi
@@ -161,6 +161,7 @@ def process_3tab(upload_dir):
         expression_matrix.append(sparse.csr_matrix(chunk.values))
         print(f"Chunks read: {rows_read}/{total_rows}", file=sys.stderr, flush=True)
         status['progress'] = percentage
+        status['message'] = f"Processed {rows_read}/{total_rows} expression matrix chunks ..."
         with open(os.path.join(upload_dir, 'status.json'), 'w') as f:
             f.write(json.dumps(status))
 

diff --git a/www/css/upload_dataset.css b/www/css/upload_dataset.css
@@ -12,6 +12,13 @@ div#logged-in-c {
 div.step-c {
 	margin-top: 20px;
 }
+div#step-process-dataset-c h3 {
+	margin-bottom: 5px;
+}
+div#step-process-dataset-c h3 span {
+	font-weight: normal;
+	font-size: 90%;
+}
 div#submissions-in-progress table {
 	border: 1px solid #ccc;
 	/*background-color: #dbdbdb;*/

diff --git a/www/js/upload_dataset.js b/www/js/upload_dataset.js
@@ -61,6 +61,12 @@ window.onload=function() {
         document.getElementById('submission-c').classList.remove('is-hidden');
     });
 
+    document.getElementById('dataset-processing-submit').addEventListener('click', (event) => {
+        event.preventDefault();
+
+        stepTo('finalize-dataset');
+    });
+
     document.getElementById('metadata-form-submit').addEventListener('click', (event) => {
         event.preventDefault();
         let errored_fields = validateMetadataForm();
@@ -163,8 +169,9 @@ const checkDatasetProcessingStatus = async () => {
     document.getElementById('dataset-processing-progress').value = data.progress;
 
     // TODO: Handle the different statuses here
-
-    console.log(data);
+    if (processing_status === 'complete') {
+        document.getElementById('dataset-processing-submit').disabled = false;
+    }
 }
 
 const populateMetadataFormFromFile = async () => {
@@ -281,6 +288,9 @@ const stepTo = (step) => {
 
     // if the step is process-dataset, we need to check on the status
     if (step === 'process-dataset') {
+        // Check the status immediately, then set an interval to keep doing it.
+        checkDatasetProcessingStatus();
+
         setInterval(() => {
             if (processing_status !== 'complete' && processing_status !== 'error') {
                 checkDatasetProcessingStatus();

diff --git a/www/upload_dataset.html b/www/upload_dataset.html
@@ -731,14 +731,6 @@ <h3>H5AD / Python</h3>
                             </span> 
                           </label>
                         </div>
-                        <div id="dataset-upload-status" class="is-hidden">
-                          <p>
-                            <span class="icon">
-                              <i class="mdi mdi-information"></i>
-                            </span>
-                            <span id="dataset-upload-status-message"></span>
-                          </p>
-                        </div>
                       </div>
                     </div>
                     <div class="column is-6">
@@ -756,10 +748,17 @@ <h3>H5AD / Python</h3>
                       </div>
                     </div>
                   </div>
+                  <div id="dataset-upload-status" class="is-hidden">
+                    <p class="mt-2 mb-0 pt-2 pb-0">
+                      <span class="icon">
+                        <i class="mdi mdi-information"></i>
+                      </span>
+                      <span id="dataset-upload-status-message"></span>
+                    </p>
+                  </div>
                   <div class="field">
                     <progress id="dataset-upload-progress" class="progress is-medium" value="0" max="100"></progress>
                   </div>
-
                   <div class="field">
                     <div class="control">
                       <button class="button is-primary" id="dataset-upload-submit" disabled>Upload dataset</button>
@@ -772,16 +771,21 @@ <h1>Step - Process dataset</h1>
                     Your dataset is being processed on the server. This may take a few minutes, depending on the size of the dataset.
                   </p>
                   <hr />
-                  <div class="columns">
-                    <div class="column">
-                      <h2>Status: <span id="step-process-dataset-status">Checking ...</span></h2>
+                  <div class="columns mb-0 pb-0">
+                    <div class="column is-one-third mb-0 pb-0">
+                      <h3>Status: <span id="step-process-dataset-status">Checking ...</span></h3>
                     </div>
-                    <div class="column">
-                      <h2>Message: <span id="step-process-dataset-status-message"></span></h2>
+                    <div class="column is-two-thirds mb-0 pb-0">
+                      <h3>Message: <span id="step-process-dataset-status-message"></span></h3>
                     </div>
                   </div>
                   <div class="field">
-                    <progress id="dataset-processing-progress" class="progress is-medium" value="0" max="100"></progress>
+                    <progress id="dataset-processing-progress" class="progress is-medium" value="" max="100"></progress>
+                  </div>
+                  <div class="field">
+                    <div class="control">
+                      <button class="button is-primary" id="dataset-processing-submit" disabled>Continue</button>
+                    </div>
                   </div>
                 </div>
               </div>