diff --git a/synophotos/cache.py b/synophotos/cache.py index c754341..f5ddf8d 100644 --- a/synophotos/cache.py +++ b/synophotos/cache.py @@ -11,12 +11,8 @@ class Cache: filesizes: Dict[int, int] = field( factory=dict ) - def filesize( self, item_id: int, filesize: int ): - if self.enabled: - self.filesizes[item_id] = filesize - def cmp_filesize( self, item_id: int, filesize: int ) -> bool: - return filesize == self.filesizes.get( item_id ) + return filesize == self.filesizes.get( item_id ) if self.enabled else False # structuring/unstructuring diff --git a/synophotos/cli.py b/synophotos/cli.py index 546f786..b1ebc60 100644 --- a/synophotos/cli.py +++ b/synophotos/cli.py @@ -33,7 +33,6 @@ def cli( ctx: Context, debug: bool, force: bool, verbose: bool ): synophotos = SynoPhotos( url=ctx.obj.url, account=ctx.obj.account, password=ctx.obj.password, session=ctx.obj.session ) if ctx.obj.config.cache: synophotos.enable_cache( ctx.obj.cache ) - log.info( f'enabled cache: {len( ctx.obj.cache.filesizes )} filesize entries' ) ctx.obj.service = synophotos @@ -267,28 +266,30 @@ def show( ctx: ApplicationContext, album_id: bool, folder_id, item_id: bool, id: # noinspection PyShadowingNames @cli.command( help='sync' ) # @option( '-a', '--album', required=False, is_flag=True, help='treat arguments as albums (the default)' ) # for now only sync albums +@option( '-c', '--use-cache', required=False, is_flag=True, default=False, hidden=True, help='use filesize cache to detect updates (experimental)' ) @option( '-d', '--destination', required=True, is_flag=False, help='destination folder to sync to' ) @argument( 'albums', nargs=-1, required=False ) @pass_obj -def sync( ctx: ApplicationContext, albums: Tuple[str], destination: str ): +def sync( ctx: ApplicationContext, albums: Tuple[str], destination: str, use_cache: bool ): # get all existing items in all albums to be synced all_albums = synophotos.albums( *albums, include_shared=True ) albums = { a: [] for a in all_albums } for a in albums.keys(): albums[a] = synophotos.list_album_items( a.id ) - result = prepare_sync_albums( albums, destination ) + result = prepare_sync_albums( albums, destination, use_cache ) - if len( result.additions ) == 0 and len( result.removals ) == 0: + if ( len( result.additions ), len( result.updates ), len( result.removals ) ) == (0, 0, 0 ): pprint( f'Skipping {len( result.skips )} files, nothing to do ...' ) return - msg = f'Sync will [green]add {len( result.additions )} files[/green], [red]remove {len( result.removals )} files[/red] and [yellow]skip {len( result.skips )} files[/yellow], continue?' + msg = ( 'Sync: [green]{} additions[/green], [yellow]{} updates[/yellow], [red]{} removals[/red] and [blue]{} skips[/blue], continue?'.format( *result.lengths() ) ) if not confirm( msg, ctx.force ): return - for i, a in result.additions: + for i, a in [ *result.additions, *result.updates ]: item, contents = synophotos.download( item_id=i.id, passphrase=a.passphrase, thumbnail='xl', include_exif=True ) + ctx.cache.filesizes[item.id] = item.filesize write_item( item, contents, result.fs ) for p in result.removals: remove_item( result.fs, p ) diff --git a/synophotos/fsio.py b/synophotos/fsio.py index e4a1a6d..99348c2 100644 --- a/synophotos/fsio.py +++ b/synophotos/fsio.py @@ -1,10 +1,12 @@ from logging import getLogger from os.path import dirname -from typing import Dict, List, Tuple +from typing import Dict, List, Optional, Tuple from attrs import define, field +from click import get_current_context from fs.osfs import OSFS +from synophotos import Cache from synophotos.photos import Album, Item log = getLogger( __name__ ) @@ -18,7 +20,16 @@ class SyncResult: skips: List[Tuple[Item, Album]] = field( factory=list ) removals: List[str] = field( factory=list ) -def prepare_sync_albums( albums: Dict[Album, List[Item]], destination: str ) -> SyncResult: + def lengths( self ) -> Tuple[int, int, int, int]: + return len( self.additions ), len( self.updates ), len( self.removals ), len( self.skips ) + +def prepare_sync_albums( albums: Dict[Album, List[Item]], destination: str, use_cache: bool = False ) -> SyncResult: + if use_cache: + cache = get_current_context().obj.cache + log.info( f'using cache with {len( cache.filesizes )} filesize entries to detect updates' ) + else: + cache = None + fs = OSFS( root_path=destination, expand_vars=True, create=True ) result = SyncResult( fs = fs ) @@ -27,21 +38,28 @@ def prepare_sync_albums( albums: Dict[Album, List[Item]], destination: str ) -> # path = f'/{album.id} - {album.name}/{item.filename}' # don't use album name as it might contain characters which cannot be used in filenames if not fs.exists( _item_path( item ) ): result.additions.append( ( item, album ) ) - elif False: - result.updates.append( (item, album ) ) # todo: updates seem impossible as items do not have a last_modified field + elif cache and not cache.cmp_filesize( item.id, item.filesize ): + # todo: updates seem (almost) impossible as items do not have a last_modified field + # work around: remember file sizes and check if there are any changes + result.updates.append( (item, album ) ) else: result.skips.append( (item, album ) ) # deduplicate results result.additions = list( {i.id: (i, a) for i, a in result.additions}.values() ) + result.updates = list( {i.id: (i, a) for i, a in result.updates}.values() ) result.skips = list( {i.id: (i, a) for i, a in result.skips}.values() ) # check for removals - added, skipped = [ _item_path( r[0] ) for r in result.additions ], [ _item_path( r[0] ) for r in result.skips ] + paths = [ _item_path( r[0] ) for r in result.additions ] + paths.extend( [ _item_path( r[0] ) for r in result.updates ] ) + paths.extend( [ _item_path( r[0] ) for r in result.skips ] ) for f in fs.walk.files( filter=[ '*.jpg', '*.jpeg' ] ): - if f not in added and f not in skipped: + if f not in paths: result.removals.append( f ) + log.info( f'sync result preparation (add/update/remove/skip): {result.lengths()}' ) + return result def write_item( item: Item, contents: bytes, fs: OSFS ): diff --git a/synophotos/photos.py b/synophotos/photos.py index 1db9f3c..f805cdf 100755 --- a/synophotos/photos.py +++ b/synophotos/photos.py @@ -311,6 +311,8 @@ def download( self, item_id: int, passphrase: str = None, thumbnail: Optional[Th if include_exif: binary = self.exif( item_id ).apply( binary, _item ) + log.info( f'downloaded item {_item.filename} (id {_item.id}, {_item.filesize} bytes)' ) + return _item, binary # helpers @@ -334,14 +336,9 @@ def folders( self, name: str ) -> List[Folder]: def item( self, id: int, passphrase: str = None ) -> Optional[Item]: if passphrase: - i = first( self.entry( GET_SHARED_ITEM, id=f'[{id}]', passphrase=passphrase ).as_obj( List[Item] ), None ) + return first( self.entry( GET_SHARED_ITEM, id=f'[{id}]', passphrase=passphrase ).as_obj( List[Item] ), None ) else: - i = first( self.entry( GET_ITEM, id=f'[{id}]' ).as_obj( List[Item] ), None ) - - if i: - self.cache.filesize( i.id, i.filesize ) - - return i + return first( self.entry( GET_ITEM, id=f'[{id}]' ).as_obj( List[Item] ), None ) def root_folder( self ) -> Folder: return self.folder( 0 )