Use filesize cache to detect updates

synophotos/cache.py

@@ -11,12 +11,8 @@ class Cache:
 
 	filesizes: Dict[int, int] = field( factory=dict )
 
-	def filesize( self, item_id: int, filesize: int ):
-		if self.enabled:
-			self.filesizes[item_id] = filesize
-
 	def cmp_filesize( self, item_id: int, filesize: int ) -> bool:
-		return filesize == self.filesizes.get( item_id )
+		return filesize == self.filesizes.get( item_id ) if self.enabled else False
 
 # structuring/unstructuring
 

synophotos/cli.py

@@ -33,7 +33,6 @@ def cli( ctx: Context, debug: bool, force: bool, verbose: bool ):
 		synophotos = SynoPhotos( url=ctx.obj.url, account=ctx.obj.account, password=ctx.obj.password, session=ctx.obj.session )
 		if ctx.obj.config.cache:
 			synophotos.enable_cache( ctx.obj.cache )
-			log.info( f'enabled cache: {len( ctx.obj.cache.filesizes )} filesize entries' )
 
 		ctx.obj.service = synophotos
 
@@ -267,28 +266,30 @@ def show( ctx: ApplicationContext, album_id: bool, folder_id, item_id: bool, id:
 # noinspection PyShadowingNames
 @cli.command( help='sync' )
 # @option( '-a', '--album', required=False, is_flag=True, help='treat arguments as albums (the default)' ) # for now only sync albums
+@option( '-c', '--use-cache', required=False, is_flag=True, default=False, hidden=True, help='use filesize cache to detect updates (experimental)' )
 @option( '-d', '--destination', required=True, is_flag=False, help='destination folder to sync to' )
 @argument( 'albums', nargs=-1, required=False )
 @pass_obj
-def sync( ctx: ApplicationContext, albums: Tuple[str], destination: str ):
+def sync( ctx: ApplicationContext, albums: Tuple[str], destination: str, use_cache: bool ):
 	# get all existing items in all albums to be synced
 	all_albums = synophotos.albums( *albums, include_shared=True )
 	albums = { a: [] for a in all_albums }
 	for a in albums.keys():
 		albums[a] = synophotos.list_album_items( a.id )
 
-	result = prepare_sync_albums( albums, destination )
+	result = prepare_sync_albums( albums, destination, use_cache )
 
-	if len( result.additions ) == 0 and len( result.removals ) == 0:
+	if ( len( result.additions ), len( result.updates ), len( result.removals ) ) == (0, 0, 0 ):
 		pprint( f'Skipping {len( result.skips )} files, nothing to do ...' )
 		return
 
-	msg = f'Sync will [green]add {len( result.additions )} files[/green], [red]remove {len( result.removals )} files[/red] and [yellow]skip {len( result.skips )} files[/yellow], continue?'
+	msg = ( 'Sync: [green]{} additions[/green], [yellow]{} updates[/yellow], [red]{} removals[/red] and [blue]{} skips[/blue], continue?'.format( *result.lengths() ) )
 	if not confirm( msg, ctx.force ):
 		return
 
-	for i, a in result.additions:
+	for i, a in [ *result.additions, *result.updates ]:
 		item, contents = synophotos.download( item_id=i.id, passphrase=a.passphrase, thumbnail='xl', include_exif=True )
+		ctx.cache.filesizes[item.id] = item.filesize
 		write_item( item, contents, result.fs )
 	for p in result.removals:
 		remove_item( result.fs, p )

synophotos/fsio.py

@@ -1,10 +1,12 @@
 from logging import getLogger
 from os.path import dirname
-from typing import Dict, List, Tuple
+from typing import Dict, List, Optional, Tuple
 
 from attrs import define, field
+from click import get_current_context
 from fs.osfs import OSFS
 
+from synophotos import Cache
 from synophotos.photos import Album, Item
 
 log = getLogger( __name__ )
@@ -18,7 +20,16 @@ class SyncResult:
 	skips: List[Tuple[Item, Album]] = field( factory=list )
 	removals: List[str] = field( factory=list )
 
-def prepare_sync_albums( albums: Dict[Album, List[Item]], destination: str ) -> SyncResult:
+	def lengths( self ) -> Tuple[int, int, int, int]:
+		return len( self.additions ), len( self.updates ), len( self.removals ), len( self.skips )
+
+def prepare_sync_albums( albums: Dict[Album, List[Item]], destination: str, use_cache: bool = False ) -> SyncResult:
+	if use_cache:
+		cache = get_current_context().obj.cache
+		log.info( f'using cache with {len( cache.filesizes )} filesize entries to detect updates' )
+	else:
+		cache = None
+
 	fs = OSFS( root_path=destination, expand_vars=True, create=True )
 	result = SyncResult( fs = fs )
 
@@ -27,21 +38,28 @@ def prepare_sync_albums( albums: Dict[Album, List[Item]], destination: str ) ->
 			# path = f'/{album.id} - {album.name}/{item.filename}' # don't use album name as it might contain characters which cannot be used in filenames
 			if not fs.exists( _item_path( item ) ):
 				result.additions.append( ( item, album ) )
-			elif False:
-				result.updates.append( (item, album ) ) # todo: updates seem impossible as items do not have a last_modified field
+			elif cache and not cache.cmp_filesize( item.id, item.filesize ):
+				# todo: updates seem (almost) impossible as items do not have a last_modified field
+				# work around: remember file sizes and check if there are any changes
+				result.updates.append( (item, album ) )
 			else:
 				result.skips.append( (item, album ) )
 
 	# deduplicate results
 	result.additions = list( {i.id: (i, a) for i, a in result.additions}.values() )
+	result.updates = list( {i.id: (i, a) for i, a in result.updates}.values() )
 	result.skips = list( {i.id: (i, a) for i, a in result.skips}.values() )
 
 	# check for removals
-	added, skipped = [ _item_path( r[0] ) for r in result.additions ], [ _item_path( r[0] ) for r in result.skips ]
+	paths = [ _item_path( r[0] ) for r in result.additions ]
+	paths.extend( [ _item_path( r[0] ) for r in result.updates ] )
+	paths.extend( [ _item_path( r[0] ) for r in result.skips ] )
 	for f in fs.walk.files( filter=[ '*.jpg', '*.jpeg' ] ):
-		if f not in added and f not in skipped:
+		if f not in paths:
 			result.removals.append( f )
 
+	log.info( f'sync result preparation (add/update/remove/skip): {result.lengths()}' )
+
 	return result
 
 def write_item( item: Item, contents: bytes, fs: OSFS ):

synophotos/photos.py

@@ -311,6 +311,8 @@ def download( self, item_id: int, passphrase: str = None, thumbnail: Optional[Th
 		if include_exif:
 			binary = self.exif( item_id ).apply( binary, _item )
 
+		log.info( f'downloaded item {_item.filename} (id {_item.id}, {_item.filesize} bytes)' )
+
 		return _item, binary
 
 	# helpers
@@ -334,14 +336,9 @@ def folders( self, name: str ) -> List[Folder]:
 
 	def item( self, id: int, passphrase: str = None ) -> Optional[Item]:
 		if passphrase:
-			i = first( self.entry( GET_SHARED_ITEM, id=f'[{id}]', passphrase=passphrase ).as_obj( List[Item] ), None )
+			return first( self.entry( GET_SHARED_ITEM, id=f'[{id}]', passphrase=passphrase ).as_obj( List[Item] ), None )
 		else:
-			i = first( self.entry( GET_ITEM, id=f'[{id}]' ).as_obj( List[Item] ), None )
-
-		if i:
-			self.cache.filesize( i.id, i.filesize )
-
-		return i
+			return first( self.entry( GET_ITEM, id=f'[{id}]' ).as_obj( List[Item] ), None )
 
 	def root_folder( self ) -> Folder:
 		return self.folder( 0 )