From ad486e0c7393b721a04c0ddb4813dcb78520ec2b Mon Sep 17 00:00:00 2001 From: ChemicalXandco Date: Wed, 11 Sep 2019 17:44:31 +0100 Subject: [PATCH 1/7] add LogManager class --- meshroom/core/node.py | 76 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/meshroom/core/node.py b/meshroom/core/node.py index f2e98a2de3..737238325e 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -127,6 +127,81 @@ def fromDict(self, d): self.sessionUid = d.get('sessionUid', '') +class LogManager: + supportedLevels = ['info', 'warning', 'error'] + dateTimeFormatting = '%H:%M:%S.%f' + + def __init__(self, chunk): + self.chunk = chunk + self.chunk.statusChanged.connect(self.clear) + self.progressBar = False + self.cleared = False + + def clear(self): + if self.chunk.statusName == 'RUNNING' and not self.cleared: + open(self.chunk.logFile, 'w').close() + self.cleared = True + # When the node gets ran again after error the log needs to be cleared + if self.chunk.statusName == 'ERROR': + self.cleared = False + self.progressBar = False + + def waitUntilCleared(self): + while not self.cleared: + time.sleep(0.01) + + def add(self, message, level='info'): + assert not self.progressBar + self.waitUntilCleared() + + if level not in self.supportedLevels: + self.add('Level "{}" is not supported'.format(level), 'warning') + else: + with open(self.chunk.logFile, 'a') as f: + f.write('[{}][{}] {}\n'.format(datetime.datetime.now().strftime(self.dateTimeFormatting), level, message)) + f.close() + + def makeProgressBar(self, end, message=''): + assert end > 0 + assert not self.progressBar + self.waitUntilCleared() + + self.progressEnd = end + self.currentProgressTics = 0 + + with open(self.chunk.logFile, 'a') as f: + if message: + f.write(message+'\n') + f.write('0% 10 20 30 40 50 60 70 80 90 100%\n') + f.write('|----|----|----|----|----|----|----|----|----|----|\n') + f.close() + + self.progressBar = True + + def updateProgressBar(self, value): + assert self.progressBar + assert value <= self.progressEnd + self.waitUntilCleared() + + tics = round((value/self.progressEnd)*51) + + with open(self.chunk.logFile, 'a') as f: + for i in range(tics-self.currentProgressTics): + f.write('*') + f.close() + + self.currentProgressTics = tics + + def completeProgressBar(self): + assert self.progressBar + + with open(self.chunk.logFile, 'a') as f: + f.write('\n') + f.close() + + self.progressBar = False + + runningProcesses = {} @@ -142,6 +217,7 @@ def __init__(self, node, range, parent=None): super(NodeChunk, self).__init__(parent) self.node = node self.range = range + self.log = LogManager(self) self.status = StatusData(node.name, node.nodeType, node.packageName, node.packageVersion) self.statistics = stats.Statistics() self.statusFileLastModTime = -1 From 4dfd13c0d9829b796e51116403b021871a235d80 Mon Sep 17 00:00:00 2001 From: ChemicalXandco Date: Wed, 11 Sep 2019 20:22:30 +0100 Subject: [PATCH 2/7] intergrate python logging into publish node also fix error not returning proper information (chunk.node.inputFiles.value - > [i.value for i in chunk.node.inputFiles.value]) --- meshroom/nodes/aliceVision/Publish.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/meshroom/nodes/aliceVision/Publish.py b/meshroom/nodes/aliceVision/Publish.py index 966566e69c..21f93c1221 100644 --- a/meshroom/nodes/aliceVision/Publish.py +++ b/meshroom/nodes/aliceVision/Publish.py @@ -41,9 +41,8 @@ def resolvedPaths(self, inputFiles, outDir): return paths def processChunk(self, chunk): - print("Publish") if not chunk.node.inputFiles: - print("Nothing to publish") + chunk.log.add('Nothing to publish', 'warning') return if not chunk.node.output.value: return @@ -51,13 +50,15 @@ def processChunk(self, chunk): outFiles = self.resolvedPaths(chunk.node.inputFiles.value, chunk.node.output.value) if not outFiles: - raise RuntimeError("Publish: input files listed, but nothing to publish. " - "Listed input files: {}".format(chunk.node.inputFiles.value)) + error = 'Publish: input files listed, but nothing to publish' + chunk.log.add(error, 'error') + chunk.log.add('Listed input files: {}'.format([i.value for i in chunk.node.inputFiles.value])) + raise RuntimeError(error) if not os.path.exists(chunk.node.output.value): os.mkdir(chunk.node.output.value) for iFile, oFile in outFiles.items(): - print('Publish file', iFile, 'into', oFile) + chunk.log.add('Publish file {} into {}'.format(iFile, oFile)) shutil.copyfile(iFile, oFile) - print('Publish end') + chunk.log.add('Publish end') From 87da9bfef3c67761f6cf9060e0fd98d6e2e81315 Mon Sep 17 00:00:00 2001 From: ChemicalXandco Date: Thu, 12 Sep 2019 16:52:58 +0100 Subject: [PATCH 3/7] [nodes] Publish: update version to 1.2 --- meshroom/nodes/aliceVision/Publish.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meshroom/nodes/aliceVision/Publish.py b/meshroom/nodes/aliceVision/Publish.py index 21f93c1221..9497caeba7 100644 --- a/meshroom/nodes/aliceVision/Publish.py +++ b/meshroom/nodes/aliceVision/Publish.py @@ -1,6 +1,6 @@ from __future__ import print_function -__version__ = "1.1" +__version__ = "1.2" from meshroom.core import desc import shutil From 5ef33ef64c1dd84d897051aa4bae596c46754185 Mon Sep 17 00:00:00 2001 From: ChemicalXandco Date: Sat, 14 Sep 2019 12:02:31 +0100 Subject: [PATCH 4/7] rely on standard logging object --- meshroom/core/node.py | 42 +++++++++++++++++---------- meshroom/nodes/aliceVision/Publish.py | 5 ++-- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/meshroom/core/node.py b/meshroom/core/node.py index 737238325e..be6b9db2e7 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -128,21 +128,39 @@ def fromDict(self, d): class LogManager: - supportedLevels = ['info', 'warning', 'error'] - dateTimeFormatting = '%H:%M:%S.%f' + dateTimeFormatting = '%H:%M:%S' def __init__(self, chunk): self.chunk = chunk self.chunk.statusChanged.connect(self.clear) self.progressBar = False self.cleared = False - - def clear(self): + self.logger = logging.getLogger(chunk.node.getName()) + + class Formatter(logging.Formatter): + def format(self, record): + # Make level name lower case + record.levelname = record.levelname.lower() + return logging.Formatter.format(self, record) + + def configureLogger(self): + for handler in self.logger.handlers[:]: + self.logger.removeHandler(handler) + handler = logging.FileHandler(self.chunk.logFile) + formatter = self.Formatter('[%(asctime)s.%(msecs)03d][%(levelname)s] %(message)s', self.dateTimeFormatting) + handler.setFormatter(formatter) + self.logger.addHandler(handler) + + def clear(self): if self.chunk.statusName == 'RUNNING' and not self.cleared: open(self.chunk.logFile, 'w').close() + self.configureLogger() self.cleared = True - # When the node gets ran again after error the log needs to be cleared - if self.chunk.statusName == 'ERROR': + # When the node gets ran again the log needs to be cleared + elif self.chunk.statusName in ['ERROR', 'SUCCESS']: + for handler in self.logger.handlers[:]: + # Stops the file being locked + handler.close() self.cleared = False self.progressBar = False @@ -150,16 +168,11 @@ def waitUntilCleared(self): while not self.cleared: time.sleep(0.01) - def add(self, message, level='info'): + def add(self, message, level=logging.INFO): assert not self.progressBar self.waitUntilCleared() - if level not in self.supportedLevels: - self.add('Level "{}" is not supported'.format(level), 'warning') - else: - with open(self.chunk.logFile, 'a') as f: - f.write('[{}][{}] {}\n'.format(datetime.datetime.now().strftime(self.dateTimeFormatting), level, message)) - f.close() + self.logger.log(level, message) def makeProgressBar(self, end, message=''): assert end > 0 @@ -168,6 +181,7 @@ def makeProgressBar(self, end, message=''): self.progressEnd = end self.currentProgressTics = 0 + self.progressBar = True with open(self.chunk.logFile, 'a') as f: if message: @@ -176,8 +190,6 @@ def makeProgressBar(self, end, message=''): f.write('|----|----|----|----|----|----|----|----|----|----|\n') f.close() - self.progressBar = True - def updateProgressBar(self, value): assert self.progressBar assert value <= self.progressEnd diff --git a/meshroom/nodes/aliceVision/Publish.py b/meshroom/nodes/aliceVision/Publish.py index 9497caeba7..8a2432a82e 100644 --- a/meshroom/nodes/aliceVision/Publish.py +++ b/meshroom/nodes/aliceVision/Publish.py @@ -6,6 +6,7 @@ import shutil import glob import os +import logging class Publish(desc.Node): @@ -42,7 +43,7 @@ def resolvedPaths(self, inputFiles, outDir): def processChunk(self, chunk): if not chunk.node.inputFiles: - chunk.log.add('Nothing to publish', 'warning') + chunk.log.add('Nothing to publish', logging.WARNING) return if not chunk.node.output.value: return @@ -51,7 +52,7 @@ def processChunk(self, chunk): if not outFiles: error = 'Publish: input files listed, but nothing to publish' - chunk.log.add(error, 'error') + chunk.log.add(error, logging.ERROR) chunk.log.add('Listed input files: {}'.format([i.value for i in chunk.node.inputFiles.value])) raise RuntimeError(error) From cd5beab5f430427637ac22822ea2862d213a052d Mon Sep 17 00:00:00 2001 From: ChemicalXandco Date: Sat, 14 Sep 2019 12:31:15 +0100 Subject: [PATCH 5/7] expose verbose level on Publish node --- meshroom/core/node.py | 8 ++++++++ meshroom/nodes/aliceVision/Publish.py | 11 +++++++++++ 2 files changed, 19 insertions(+) diff --git a/meshroom/core/node.py b/meshroom/core/node.py index be6b9db2e7..36a87691ae 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -213,6 +213,14 @@ def completeProgressBar(self): self.progressBar = False + def textToLevel(self, text): + if text == 'critical': return logging.CRITICAL + elif text == 'error': return logging.ERROR + elif text == 'warning': return logging.WARNING + elif text == 'info': return logging.INFO + elif text == 'debug': return logging.DEBUG + else: return logging.NOTSET + runningProcesses = {} diff --git a/meshroom/nodes/aliceVision/Publish.py b/meshroom/nodes/aliceVision/Publish.py index 8a2432a82e..917e576953 100644 --- a/meshroom/nodes/aliceVision/Publish.py +++ b/meshroom/nodes/aliceVision/Publish.py @@ -31,6 +31,15 @@ class Publish(desc.Node): description="", value="", uid=[0], + ), + desc.ChoiceParam( + name='verboseLevel', + label='Verbose Level', + description='''verbosity level (critical, error, warning, info, debug).''', + value='info', + values=['critical', 'error', 'warning', 'info', 'debug'], + exclusive=True, + uid=[], ), ] @@ -42,6 +51,8 @@ def resolvedPaths(self, inputFiles, outDir): return paths def processChunk(self, chunk): + chunk.log.logger.setLevel(chunk.log.textToLevel(chunk.node.verboseLevel.value)) + if not chunk.node.inputFiles: chunk.log.add('Nothing to publish', logging.WARNING) return From 17ecf7806426d4892329a6d5ad28ad6dcb5f6513 Mon Sep 17 00:00:00 2001 From: ChemicalXandco Date: Sun, 15 Sep 2019 15:28:59 +0100 Subject: [PATCH 6/7] use logger object directly also allow progress bar to be used even while messages are added --- meshroom/core/node.py | 29 ++++++++++++++------------- meshroom/nodes/aliceVision/Publish.py | 14 ++++++------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/meshroom/core/node.py b/meshroom/core/node.py index 36a87691ae..053c04390a 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -168,12 +168,6 @@ def waitUntilCleared(self): while not self.cleared: time.sleep(0.01) - def add(self, message, level=logging.INFO): - assert not self.progressBar - self.waitUntilCleared() - - self.logger.log(level, message) - def makeProgressBar(self, end, message=''): assert end > 0 assert not self.progressBar @@ -187,7 +181,11 @@ def makeProgressBar(self, end, message=''): if message: f.write(message+'\n') f.write('0% 10 20 30 40 50 60 70 80 90 100%\n') - f.write('|----|----|----|----|----|----|----|----|----|----|\n') + f.write('|----|----|----|----|----|----|----|----|----|----|\n\n') + if message: + self.progressBarPosition = f.tell()-14 + else: + self.progressBarPosition = f.tell()-7 f.close() def updateProgressBar(self, value): @@ -197,9 +195,12 @@ def updateProgressBar(self, value): tics = round((value/self.progressEnd)*51) - with open(self.chunk.logFile, 'a') as f: + with open(self.chunk.logFile, 'r+') as f: + text = f.read() for i in range(tics-self.currentProgressTics): - f.write('*') + text = text[:self.progressBarPosition]+'*'+text[self.progressBarPosition:] + f.seek(0) + f.write(text) f.close() self.currentProgressTics = tics @@ -207,10 +208,6 @@ def updateProgressBar(self, value): def completeProgressBar(self): assert self.progressBar - with open(self.chunk.logFile, 'a') as f: - f.write('\n') - f.close() - self.progressBar = False def textToLevel(self, text): @@ -237,7 +234,7 @@ def __init__(self, node, range, parent=None): super(NodeChunk, self).__init__(parent) self.node = node self.range = range - self.log = LogManager(self) + self.logManager = LogManager(self) self.status = StatusData(node.name, node.nodeType, node.packageName, node.packageVersion) self.statistics = stats.Statistics() self.statusFileLastModTime = -1 @@ -260,6 +257,10 @@ def name(self): def statusName(self): return self.status.status.name + @property + def logger(self): + return self.logManager.logger + @property def execModeName(self): return self.status.execMode.name diff --git a/meshroom/nodes/aliceVision/Publish.py b/meshroom/nodes/aliceVision/Publish.py index 917e576953..ebe2b9b832 100644 --- a/meshroom/nodes/aliceVision/Publish.py +++ b/meshroom/nodes/aliceVision/Publish.py @@ -6,7 +6,6 @@ import shutil import glob import os -import logging class Publish(desc.Node): @@ -51,10 +50,11 @@ def resolvedPaths(self, inputFiles, outDir): return paths def processChunk(self, chunk): - chunk.log.logger.setLevel(chunk.log.textToLevel(chunk.node.verboseLevel.value)) + chunk.logManager.waitUntilCleared() + chunk.logger.setLevel(chunk.logManager.textToLevel(chunk.node.verboseLevel.value)) if not chunk.node.inputFiles: - chunk.log.add('Nothing to publish', logging.WARNING) + chunk.logger.warning('Nothing to publish') return if not chunk.node.output.value: return @@ -63,14 +63,14 @@ def processChunk(self, chunk): if not outFiles: error = 'Publish: input files listed, but nothing to publish' - chunk.log.add(error, logging.ERROR) - chunk.log.add('Listed input files: {}'.format([i.value for i in chunk.node.inputFiles.value])) + chunk.logger.error(error) + chunk.logger.info('Listed input files: {}'.format([i.value for i in chunk.node.inputFiles.value])) raise RuntimeError(error) if not os.path.exists(chunk.node.output.value): os.mkdir(chunk.node.output.value) for iFile, oFile in outFiles.items(): - chunk.log.add('Publish file {} into {}'.format(iFile, oFile)) + chunk.logger.info('Publish file {} into {}'.format(iFile, oFile)) shutil.copyfile(iFile, oFile) - chunk.log.add('Publish end') + chunk.logger.info('Publish end') From 749fc154b1d80738a6b11debbab40deafc56958d Mon Sep 17 00:00:00 2001 From: ChemicalXandco <32775248+ChemicalXandco@users.noreply.github.com> Date: Mon, 25 Nov 2019 19:48:49 +0000 Subject: [PATCH 7/7] more reliable way to find progress bar position in log file --- meshroom/core/node.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/meshroom/core/node.py b/meshroom/core/node.py index 053c04390a..43aecf8526 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -182,10 +182,13 @@ def makeProgressBar(self, end, message=''): f.write(message+'\n') f.write('0% 10 20 30 40 50 60 70 80 90 100%\n') f.write('|----|----|----|----|----|----|----|----|----|----|\n\n') - if message: - self.progressBarPosition = f.tell()-14 - else: - self.progressBarPosition = f.tell()-7 + + f.close() + + with open(self.chunk.logFile, 'r') as f: + content = f.read() + self.progressBarPosition = content.rfind('\n') + f.close() def updateProgressBar(self, value):