Skip to content

Commit fc85bbf

Browse files
vesellovvesellov
committed
block uploading of a file when there is insufficient storage space expected
1 parent 5539337 commit fc85bbf

9 files changed

+154
-131
lines changed

‎bitdust/access/key_ring.py

+3-2
Original file line numberDiff line numberDiff line change
@@ -462,8 +462,9 @@ def on_key_received(newpacket, info, status, error_message):
462462
# but we already have a key with that ID
463463
if my_keys.is_key_private(key_id):
464464
# we should not overwrite existing private key
465-
# TODO: check other scenarios
466-
lg.warn('received public key, but private key is already registered %r' % key_id)
465+
if my_keys.get_public_key_raw(key_id) != key_object.toPublicString():
466+
raise Exception('private key already registered : %r' % key_id)
467+
lg.warn('received public key, but matching private key is already registered %r' % key_id)
467468
p2p_service.SendAck(newpacket)
468469
return True
469470
if my_keys.get_public_key_raw(key_id) != key_object.toPublicString():

‎bitdust/interface/api.py

+11-27
Original file line numberDiff line numberDiff line change
@@ -694,6 +694,7 @@ def network_select(name):
694694

695695
def _on_network_disconnected(x):
696696
cur_base_dir = deploy.current_base_dir()
697+
# TODO: must wait shutdown and init to complete with defered
697698
shutdowner.shutdown_services()
698699
shutdowner.shutdown_local()
699700
shutdowner.shutdown_automats()
@@ -2443,33 +2444,17 @@ def file_upload_start(local_path, remote_path, wait_result=False, publish_events
24432444
if not pathID:
24442445
return ERROR('path %s was not registered yet' % remote_path)
24452446
keyID = my_keys.make_key_id(alias=key_alias, creator_glob_id=parts['customer'])
2446-
# customerID = global_id.MakeGlobalID(customer=parts['customer'], key_alias=key_alias)
24472447
pathIDfull = packetid.MakeBackupID(keyID, pathID)
24482448
if key_alias != 'master':
24492449
if not driver.is_on('service_shared_data'):
24502450
return ERROR('service_shared_data() is not started')
2451-
2452-
2453-
# def _restart_active_share(result):
2454-
# if _Debug:
2455-
# lg.args(_DebugLevel, result=result, key_id=keyID, path=path, pathID=pathID)
2456-
# if key_alias != 'master':
2457-
# from bitdust.access import shared_access_coordinator
2458-
# active_share = shared_access_coordinator.get_active_share(keyID)
2459-
# if not active_share:
2460-
# active_share = shared_access_coordinator.SharedAccessCoordinator(key_id=keyID, publish_events=publish_events)
2461-
# active_share.automat('restart')
2462-
# return result
2463-
24642451
if wait_result:
24652452
task_created_defer = Deferred()
24662453
tsk = backup_control.StartSingle(
24672454
pathID=pathIDfull,
24682455
localPath=local_path,
24692456
keyID=keyID,
24702457
)
2471-
# if key_alias != 'master':
2472-
# tsk.result_defer.addCallback(_restart_active_share)
24732458
tsk.result_defer.addCallback(
24742459
lambda result: task_created_defer.callback(
24752460
OK(
@@ -2485,14 +2470,15 @@ def file_upload_start(local_path, remote_path, wait_result=False, publish_events
24852470
)
24862471
)
24872472
)
2488-
tsk.result_defer.addErrback(lambda result: task_created_defer.callback(ERROR(
2489-
'uploading task %d for %s failed: %s' % (
2490-
tsk.number,
2491-
tsk.pathID,
2492-
result[1],
2493-
),
2494-
api_method='file_upload_start',
2495-
), ), )
2473+
tsk.result_defer.addErrback(lambda result: task_created_defer.callback(ERROR(result, api_method='file_upload_start')))
2474+
# tsk.result_defer.addErrback(lambda result: task_created_defer.callback(ERROR(
2475+
# 'uploading task %d for %s failed: %s' % (
2476+
# tsk.number,
2477+
# tsk.pathID,
2478+
# result,
2479+
# ),
2480+
# api_method='file_upload_start',
2481+
# ), ), )
24962482
backup_fs.Calculate(customer_idurl=customer_idurl, key_alias=key_alias)
24972483
backup_control.SaveFSIndex(customer_idurl, key_alias)
24982484
if _Debug:
@@ -2504,9 +2490,7 @@ def file_upload_start(local_path, remote_path, wait_result=False, publish_events
25042490
localPath=local_path,
25052491
keyID=keyID,
25062492
)
2507-
# if key_alias != 'master':
2508-
# tsk.result_defer.addCallback(_restart_active_share)
2509-
tsk.result_defer.addErrback(lambda result: lg.err('errback from api.file_upload_start.task(%s) failed with %s' % (result[0], result[1])))
2493+
tsk.result_defer.addErrback(lambda result: lg.err('errback from api.file_upload_start.task() failed with %r' % result))
25102494
backup_fs.Calculate(customer_idurl=customer_idurl, key_alias=key_alias)
25112495
backup_control.SaveFSIndex(customer_idurl, key_alias)
25122496
if _Debug:

‎bitdust/p2p/p2p_service.py

+92
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,10 @@
5151

5252
#------------------------------------------------------------------------------
5353

54+
from twisted.internet.defer import Deferred
55+
56+
#------------------------------------------------------------------------------
57+
5458
from bitdust.logs import lg
5559

5660
from bitdust.contacts import contactsdb
@@ -66,8 +70,11 @@
6670
from bitdust.crypt import signed
6771
from bitdust.crypt import my_keys
6872

73+
from bitdust.services import driver
74+
6975
from bitdust.transport import gateway
7076
from bitdust.transport import callback
77+
from bitdust.transport import packet_out
7178

7279
from bitdust.userid import my_id
7380

@@ -195,6 +202,91 @@ def outbox(outpacket, wide, callbacks, target=None, route=None, response_timeout
195202
#------------------------------------------------------------------------------
196203

197204

205+
def on_request_service_received(newpacket, info):
206+
if len(newpacket.Payload) > 1024*10:
207+
lg.warn('too long payload')
208+
SendFail(newpacket, 'too long payload')
209+
return True
210+
try:
211+
json_payload = serialization.BytesToDict(newpacket.Payload, keys_to_text=True, values_to_text=True)
212+
json_payload['name']
213+
json_payload['payload']
214+
except:
215+
lg.warn('json payload invalid')
216+
SendFail(newpacket, 'json payload invalid')
217+
return True
218+
service_name = str(json_payload['name'])
219+
if _Debug:
220+
lg.out(_Debug, 'service_p2p_hookups.RequestService {%s} from %s' % (service_name, newpacket.OwnerID))
221+
if not driver.is_exist(service_name):
222+
lg.warn('got wrong payload in %s' % service_name)
223+
SendFail(newpacket, 'service %s not exist' % service_name)
224+
return True
225+
if not driver.is_on(service_name):
226+
SendFail(newpacket, 'service %s is off' % service_name)
227+
return True
228+
try:
229+
result = driver.request(service_name, json_payload['payload'], newpacket, info)
230+
except:
231+
lg.exc()
232+
SendFail(newpacket, 'request processing failed with exception')
233+
return True
234+
if not result:
235+
if _Debug:
236+
lg.out(_Debug, 'service_p2p_hookups._send_request_service SKIP request %s' % service_name)
237+
return False
238+
if isinstance(result, Deferred):
239+
if _Debug:
240+
lg.out(_Debug, 'service_p2p_hookups._send_request_service fired delayed execution')
241+
elif isinstance(result, packet_out.PacketOut):
242+
if _Debug:
243+
lg.out(_Debug, 'service_p2p_hookups._send_request_service outbox packet sent')
244+
return True
245+
246+
247+
def on_cancel_service_received(newpacket, info):
248+
if len(newpacket.Payload) > 1024*10:
249+
SendFail(newpacket, 'too long payload')
250+
return True
251+
try:
252+
json_payload = serialization.BytesToDict(newpacket.Payload, keys_to_text=True, values_to_text=True)
253+
json_payload['name']
254+
json_payload['payload']
255+
except:
256+
SendFail(newpacket, 'json payload invalid')
257+
return True
258+
service_name = json_payload['name']
259+
if _Debug:
260+
lg.out(_Debug, 'service_p2p_hookups.CancelService {%s} from %s' % (service_name, newpacket.OwnerID))
261+
if not driver.is_exist(service_name):
262+
lg.warn('got wrong payload in %s' % newpacket)
263+
SendFail(newpacket, 'service %s not exist' % service_name)
264+
return True
265+
if not driver.is_on(service_name):
266+
SendFail(newpacket, 'service %s is off' % service_name)
267+
return True
268+
try:
269+
result = driver.cancel(service_name, json_payload['payload'], newpacket, info)
270+
except:
271+
lg.exc()
272+
SendFail(newpacket, 'request processing failed with exception')
273+
return True
274+
if not result:
275+
if _Debug:
276+
lg.out(_Debug, 'service_p2p_hookups._send_cancel_service SKIP request %s' % service_name)
277+
return False
278+
if isinstance(result, Deferred):
279+
if _Debug:
280+
lg.out(_Debug, 'service_p2p_hookups._send_cancel_service fired delayed execution')
281+
elif isinstance(result, packet_out.PacketOut):
282+
if _Debug:
283+
lg.out(_Debug, 'service_p2p_hookups._send_cancel_service outbox packet sent')
284+
return True
285+
286+
287+
#------------------------------------------------------------------------------
288+
289+
198290
def Ack(newpacket, info):
199291
if _Debug:
200292
lg.out(_DebugLevel, 'p2p_service.Ack %s from [%s] at %s://%s with %d bytes payload' % (newpacket.PacketID, nameurl.GetName(newpacket.CreatorID), info.proto, info.host, len(newpacket.Payload)))

‎bitdust/services/driver.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -474,7 +474,7 @@ def _on_started(start_result, stop_result, dependencies_results):
474474

475475
def _on_failed(err):
476476
lg.warn('failed service %s in driver.restart() : %r' % (service_name, err))
477-
restart_result.errback(str(err))
477+
restart_result.errback(Exception(str(err)))
478478
return None
479479

480480
def _do_start(stop_result=None, dependencies_results=None):

‎bitdust/services/service_p2p_hookups.py

+2-78
Original file line numberDiff line numberDiff line change
@@ -110,88 +110,12 @@ def _on_inbox_packet_received(self, newpacket, info, status, error_message):
110110
return False
111111

112112
def _on_request_service_received(self, newpacket, info):
113-
from twisted.internet.defer import Deferred
114-
from bitdust.logs import lg
115-
from bitdust.lib import serialization
116-
from bitdust.services import driver
117113
from bitdust.p2p import p2p_service
118-
from bitdust.transport import packet_out
119-
if len(newpacket.Payload) > 1024*10:
120-
lg.warn('too long payload')
121-
p2p_service.SendFail(newpacket, 'too long payload')
122-
return True
123-
try:
124-
json_payload = serialization.BytesToDict(newpacket.Payload, keys_to_text=True, values_to_text=True)
125-
json_payload['name']
126-
json_payload['payload']
127-
except:
128-
lg.warn('json payload invalid')
129-
p2p_service.SendFail(newpacket, 'json payload invalid')
130-
return True
131-
service_name = str(json_payload['name'])
132-
lg.out(self.debug_level, 'service_p2p_hookups.RequestService {%s} from %s' % (service_name, newpacket.OwnerID))
133-
if not driver.is_exist(service_name):
134-
lg.warn('got wrong payload in %s' % service_name)
135-
p2p_service.SendFail(newpacket, 'service %s not exist' % service_name)
136-
return True
137-
if not driver.is_on(service_name):
138-
p2p_service.SendFail(newpacket, 'service %s is off' % service_name)
139-
return True
140-
try:
141-
result = driver.request(service_name, json_payload['payload'], newpacket, info)
142-
except:
143-
lg.exc()
144-
p2p_service.SendFail(newpacket, 'request processing failed with exception')
145-
return True
146-
if not result:
147-
lg.out(self.debug_level, 'service_p2p_hookups._send_request_service SKIP request %s' % service_name)
148-
return False
149-
if isinstance(result, Deferred):
150-
lg.out(self.debug_level, 'service_p2p_hookups._send_request_service fired delayed execution')
151-
elif isinstance(result, packet_out.PacketOut):
152-
lg.out(self.debug_level, 'service_p2p_hookups._send_request_service outbox packet sent')
153-
return True
114+
return p2p_service.on_request_service_received(newpacket, info)
154115

155116
def _on_cancel_service_received(self, newpacket, info):
156-
from twisted.internet.defer import Deferred
157-
from bitdust.logs import lg
158-
from bitdust.lib import serialization
159-
from bitdust.services import driver
160117
from bitdust.p2p import p2p_service
161-
from bitdust.transport import packet_out
162-
if len(newpacket.Payload) > 1024*10:
163-
p2p_service.SendFail(newpacket, 'too long payload')
164-
return True
165-
try:
166-
json_payload = serialization.BytesToDict(newpacket.Payload, keys_to_text=True, values_to_text=True)
167-
json_payload['name']
168-
json_payload['payload']
169-
except:
170-
p2p_service.SendFail(newpacket, 'json payload invalid')
171-
return True
172-
service_name = json_payload['name']
173-
lg.out(self.debug_level, 'service_p2p_hookups.CancelService {%s} from %s' % (service_name, newpacket.OwnerID))
174-
if not driver.is_exist(service_name):
175-
lg.warn('got wrong payload in %s' % newpacket)
176-
p2p_service.SendFail(newpacket, 'service %s not exist' % service_name)
177-
return True
178-
if not driver.is_on(service_name):
179-
p2p_service.SendFail(newpacket, 'service %s is off' % service_name)
180-
return True
181-
try:
182-
result = driver.cancel(service_name, json_payload['payload'], newpacket, info)
183-
except:
184-
lg.exc()
185-
p2p_service.SendFail(newpacket, 'request processing failed with exception')
186-
return True
187-
if not result:
188-
lg.out(self.debug_level, 'service_p2p_hookups._send_cancel_service SKIP request %s' % service_name)
189-
return False
190-
if isinstance(result, Deferred):
191-
lg.out(self.debug_level, 'service_p2p_hookups._send_cancel_service fired delayed execution')
192-
elif isinstance(result, packet_out.PacketOut):
193-
lg.out(self.debug_level, 'service_p2p_hookups._send_cancel_service outbox packet sent')
194-
return True
118+
return p2p_service.on_cancel_service_received(newpacket, info)
195119

196120
def _on_p2p_connector_switched(self, oldstate, newstate, evt, *args, **kwargs):
197121
if newstate == 'INCOMMING?':

‎bitdust/services/service_p2p_notifications.py

-1
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,6 @@ def request(self, json_payload, newpacket, info):
141141
resp['result'] = 'denied'
142142
resp['reason'] = str(exc)
143143
service_responses_list.append(resp)
144-
lg.out(self.debug_level, 'service_p2p_notifications.request %s:%s is [%s] : %s' % (r_scope, r_action, resp['result'], resp.get('reason', 'OK')))
145144
payload = serialization.DictToBytes({'items': service_responses_list}, values_to_text=True)
146145
return p2p_service.SendAck(newpacket, payload)
147146

‎bitdust/storage/accounting.py

+19-7
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@
6767

6868
from bitdust.storage import backup_fs
6969

70+
from bitdust.userid import my_id
71+
7072
#------------------------------------------------------------------------------
7173

7274

@@ -183,28 +185,38 @@ def calculate_customers_usage_ratio(space_dict=None, used_dict=None):
183185

184186

185187
def report_consumed_storage():
188+
my_own_stats = backup_fs.total_stats()
189+
shared_stats = backup_fs.total_stats(customer_idurl=my_id.getIDURL(), exclude=True)
186190
result = {}
187191
result['suppliers_num'] = contactsdb.num_suppliers()
188192
result['needed'] = settings.getNeededBytes()
189-
# result['needed_str'] = diskspace.MakeStringFromBytes(result['needed'])
190-
result['used'] = backup_fs.total_stats()['size_backups']
191-
# result['used_str'] = diskspace.MakeStringFromBytes(result['used'])
193+
result['used'] = my_own_stats['size_backups']
192194
result['available'] = result['needed'] - result['used']
193-
# result['available_str'] = diskspace.MakeStringFromBytes(result['available'])
194195
result['needed_per_supplier'] = 0
195196
result['used_per_supplier'] = 0
196197
result['available_per_supplier'] = 0
197198
if result['suppliers_num'] > 0:
198199
result['needed_per_supplier'] = int(math.ceil(result['needed']/result['suppliers_num']))
199200
result['used_per_supplier'] = int(math.ceil(result['used']/result['suppliers_num']))
200201
result['available_per_supplier'] = result['needed_per_supplier'] - result['used_per_supplier']
201-
# result['needed_per_supplier_str'] = diskspace.MakeStringFromBytes(result['needed_per_supplier'])
202-
# result['used_per_supplier_str'] = diskspace.MakeStringFromBytes(result['used_per_supplier'])
203-
# result['available_per_supplier_str'] = diskspace.MakeStringFromBytes(result['available_per_supplier'])
204202
try:
205203
result['used_percent'] = misc.value2percent(float(result['used']), float(result['needed']))
206204
except:
207205
result['used_percent'] = '0%'
206+
result['my_catalog_items'] = my_own_stats['items']
207+
result['my_files'] = my_own_stats['files']
208+
result['my_folders'] = my_own_stats['folders']
209+
result['my_files_size'] = my_own_stats['size_files']
210+
result['my_folders_size'] = my_own_stats['size_folders']
211+
result['my_backups_size'] = my_own_stats['size_backups']
212+
result['my_keys'] = my_own_stats['keys']
213+
result['shared_catalog_items'] = shared_stats['items']
214+
result['shared_files'] = shared_stats['files']
215+
result['shared_folders'] = shared_stats['folders']
216+
result['shared_files_size'] = shared_stats['size_files']
217+
result['shared_folders_size'] = shared_stats['size_folders']
218+
result['shared_backups_size'] = shared_stats['size_backups']
219+
result['shared_keys'] = shared_stats['keys']
208220
return result
209221

210222

0 commit comments

Comments
 (0)