Skip to content

Commit

Permalink
Merge pull request BOINC#6092 from BOINC/dpa_submit17
Browse files Browse the repository at this point in the history
remote job submission and file management: various bug fixes
  • Loading branch information
lfield authored Feb 19, 2025
2 parents 61c5ec8 + 13ee57b commit a4627fa
Show file tree
Hide file tree
Showing 6 changed files with 196 additions and 29 deletions.
9 changes: 8 additions & 1 deletion html/inc/sandbox.inc
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ require_once("../inc/dir_hier.inc");
function sandbox_dir($user) {
$dir = parse_config(get_config(), "<sandbox_dir>");
if (!$dir) {
$dir = "../../sandbox/";
$dir = "../../sandbox";
}
if (!is_dir($dir)) {
mkdir($dir);
Expand All @@ -60,6 +60,13 @@ function sandbox_dir($user) {
return $d;
}

// return path of sandbox file
//
function sandbox_path($user, $fname) {
$dir = sandbox_dir($user);
return "$dir/$fname";
}

// parse a sandbox file's info file.
// If missing, create it.
//
Expand Down
22 changes: 11 additions & 11 deletions html/user/job_file.php
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@
require_once("../inc/submit_util.inc");

function upload_error_description($errno) {
switch($errno) {
switch ($errno) {
case UPLOAD_ERR_INI_SIZE:
return "The uploaded file exceeds upload_max_filesize of php.ini."; break;
case UPLOAD_ERR_FORM_SIZE:
Expand All @@ -100,21 +100,21 @@ function upload_error_description($errno) {
function query_files($r) {
xml_start_tag("query_files");
$user = check_remote_submit_permissions($r, null);
$absent_files = array();
$absent_files = [];
$now = time();
$delete_time = (int)$r->delete_time;
$batch_id = (int)$r->batch_id;
$fanout = parse_config(get_config(), "<uldl_dir_fanout>");
$phys_names= array();
foreach($r->phys_name as $f) {
$phys_names = [];
foreach ($r->phys_name as $f) {
$phys_names[] = (string)$f;
}
$i = 0;
foreach($phys_names as $fname) {
foreach ($phys_names as $fname) {
if (!is_valid_filename($fname)) {
xml_error(-1, 'bad filename');
}
$path = dir_hier_path($fname, project_dir() . "/download", $fanout);
$path = dir_hier_path($fname, project_dir()."/download", $fanout);

// if the job_file record is there,
// update the delete time first to avoid race condition
Expand Down Expand Up @@ -228,11 +228,11 @@ function upload_files($r) {
foreach ($_FILES as $f) {
$tmp_name = $f['tmp_name'];
$fname = $phys_names[$i];
$path = dir_hier_path($fname, project_dir() . "/download", $fanout);
$path = dir_hier_path($fname, project_dir()."/download", $fanout);

// see if file is in download hierarchy
//
switch(check_download_file($tmp_name, $path)) {
switch (check_download_file($tmp_name, $path)) {
case 0:
// file is already there
// note: check_download_file() generates .md5 in cases 1 and 2
Expand Down Expand Up @@ -287,10 +287,10 @@ function upload_files($r) {
if ($request_log) {
$request_log_dir = parse_config(get_config(), "<log_dir>");
if ($request_log_dir) {
$request_log = $request_log_dir . "/" . $request_log;
$request_log = $request_log_dir."/".$request_log;
}
if ($file = fopen($request_log, "a+")) {
fwrite($file, "\n<job_file date=\"" . date(DATE_ATOM) . "\">\n" . $_POST['request'] . "\n</job_file>\n");
fwrite($file, "\n<job_file date=\"".date(DATE_ATOM)."\">\n".$_POST['request']."\n</job_file>\n");
fclose($file);
}
}
Expand All @@ -302,7 +302,7 @@ function upload_files($r) {
xml_error(-1, "can't parse request message: ".htmlspecialchars($req), __FILE__, __LINE__);
}

switch($r->getName()) {
switch ($r->getName()) {
case 'query_files':
query_files($r);
break;
Expand Down
10 changes: 5 additions & 5 deletions html/user/submit_rpc_handler.php
Original file line number Diff line number Diff line change
Expand Up @@ -185,13 +185,13 @@ function stage_file($file, $user) {
case "local_staged":
return $file->source;
case 'sandbox':
$name = sandbox_name_to_phys_name($user, $file->source);
if (!$name) {
[$md5, $size] = sandbox_parse_info_file($user, $file->source);
if (!$md5) {
xml_error(-1, "sandbox link file $file->source not found");
}
$path = dir_hier_path($name, $download_dir, $fanout);
if (file_exists($path)) return $name;
xml_error(-1, "sandbox physical file $file->source not found");
$phys_name = job_file_name($md5);
$path = sandbox_path($user, $file->source);
stage_file_aux($path, $md5, $size, $phys_name);
case "inline":
$md5 = md5($file->source);
if (!$md5) {
Expand Down
36 changes: 24 additions & 12 deletions lib/boinc_submit.py
Original file line number Diff line number Diff line change
Expand Up @@ -148,21 +148,26 @@ def upload_files(self, local_names, phys_names, batch_id=0, delete_time=0):
if 'error' in ret:
return ret
absent = ret['absent_files']
if absent == '\n':
return ret
if type(absent) != list:
absent = [absent]
files = []
if isinstance(absent, str):
return {'success':None}
if 'file' not in absent.keys():
return {'success':None}
file = absent['file']
if type(file) != list:
file = [file]
n = 0
for f in absent:
i = int(f['file'])
files = []
upload_phys_names = []
for f in file:
i = int(f)
pn = phys_names[i]
ln = local_names[i]
upload_name = 'file_%d'%(n)
upload_phys_names.append(pn)
files.append((upload_name, (pn, open(ln, 'rb'),'application/octet-stream')))
n += 1
url = self.url + 'job_file.php'
req_xml = self.upload_files_xml(phys_names, batch_id, delete_time)
req_xml = self.upload_files_xml(upload_phys_names, batch_id, delete_time)
req = {'request': req_xml}
reply = requests.post(url, data=req, files=files)
root = ElementTree.XML(reply.text)
Expand Down Expand Up @@ -328,15 +333,22 @@ def upload_files_xml(self, phys_names, batch_id, delete_time):

# convert ElementTree to a python data structure
#
def etree_to_dict(xml, result={}):
def etree_to_dict(xml):
result = {}
for child in xml:
if len(child) == 0:
result[child.tag] = child.text
# this means the element contains data, not other elements
if child.tag in result:
if not isinstance(result[child.tag], list):
result[child.tag] = [result[child.tag]]
result[child.tag].append(child.text)
else:
result[child.tag] = child.text
else:
if child.tag in result:
if not isinstance(result[child.tag], list):
result[child.tag] = [result[child.tag]]
result[child.tag].append(etree_to_dict(child, {}))
result[child.tag].append(etree_to_dict(child))
else:
result[child.tag] = etree_to_dict(child, {})
result[child.tag] = etree_to_dict(child)
return result
146 changes: 146 additions & 0 deletions tools/boinc_submit_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
# This file is part of BOINC.
# http://boinc.berkeley.edu
# Copyright (C) 2016 University of California
#
# BOINC is free software; you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License
# as published by the Free Software Foundation,
# either version 3 of the License, or (at your option) any later version.
#
# BOINC is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with BOINC. If not, see <http://www.gnu.org/licenses/>.

# test functions for boinc_submit.py

# YOU MUST CREATE A FILE "test_auth' CONTAINING
#
# project URL
# authenticator of your account

from boinc_submit import *

# read URL and auth from a file so we don't have to include it here
#
def get_auth():
with open("test_auth", "r") as f:
url = (f.readline()).strip()
auth = (f.readline()).strip()
return [url, auth]

[url, auth] = get_auth()
s = BOINC_SERVER(url, auth)

def check_error(r):
if 'error' in r.keys():
x = r['error']
print('error: num: %s msg: %s'%(x['error_num'], x['error_msg']))
return True
return False

def test_estimate_batch():
f = FILE_DESC('main_2.sh', 'sandbox')
j = JOB_DESC([f]);
b = BATCH_DESC('uppercase', [j])
r = s.estimate_batch(b)
if check_error(r):
return
print('estimated time: ', r['seconds'])

def test_submit_batch(batch_name):
f = FILE_DESC('main_2.sh', 'sandbox')
j = JOB_DESC([f]);
b = BATCH_DESC('uppercase', [j])
r = s.submit_batch(b)
if check_error(r):
return
print('batch ID: ', r['batch_id'])

def test_query_batches():
r = s.query_batches(True)
if check_error(r):
return
print(r)

def test_query_batch(id):
r = s.query_batch(id, True, True)
if check_error(r):
return
print(r);
print('njobs: ', r['njobs']);
print('fraction_done: ', r['fraction_done']);
print('total_cpu_time: ', r['total_cpu_time']);
# ... various other fields
print('jobs:')
if 'job' in r.keys():
x = r['job']
if not isinstance(x, list):
x = [x]
for job in x:
print(' id: ', job['id'])
# ... various other fields

def test_create_batch(name):
r = s.create_batch('uppercase', name, 0)
if check_error(r):
return
print('batch ID: ', r['batch_id'])

def test_abort_batch(id):
r = s.abort_batch(id)
if check_error(r):
return
print('success')

def test_upload_files():
batch_id = 283
local_names = ('updater.cpp', 'kill_wu.cpp')
phys_names = ('dxxxb_updater.cpp', 'dxxxb_kill_wu.cpp')
r = s.upload_files(local_names, phys_names, batch_id, 0)
if check_error(r):
return
print('upload_files: success')

def test_query_files():
batch_id = 271
phys_names = ['dxxxb_updater.cpp', 'dxxx_kill_wu.cpp']
r = s.query_files(phys_names, batch_id, 0)
if check_error(r):
return
print('absent files:')
for f in r['absent_files']['file']:
print(f)

def test_get_output_file():
req = REQUEST()
[req.project, req.authenticator] = get_auth()
req.instance_name = 'uppercase_32275_1484961754.784017_0_0'
req.file_num = 1
r = get_output_file(req)
print(r)

def test_get_output_files():
req = REQUEST()
[req.project, req.authenticator] = get_auth()
req.batch_id = 271
r = get_output_files(req)
print(r)

def test_get_job_counts():
req = REQUEST()
[req.project, req.authenticator] = get_auth()
x = get_job_counts(req)
print(x.find('results_ready_to_send').text)

#test_query_batch(520)
#test_abort_batch(117)
#test_query_batches()
#test_submit_batch('batch_39')
#test_estimate_batch()
test_upload_files()
#test_query_files()
#test_create_batch('batch_140')
2 changes: 2 additions & 0 deletions tools/submit_api_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@

# test functions for submit_api.py

DEPRECATED: SEE boinc_submit_test.py

# YOU MUST CREATE A FILE "test_auth' CONTAINING
#
# project URL
Expand Down

0 comments on commit a4627fa

Please sign in to comment.