diff --git a/.travis.yml b/.travis.yml index 07289baf..952b6269 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,10 @@ install: - sudo apt-get -qq update - pip install --upgrade -qq pip - sudo apt-get -qq install cdparanoia cdrdao flac gir1.2-glib-2.0 libcdio-dev libgirepository1.0-dev libiso9660-dev libsndfile1-dev sox swig libcdio-utils - - pip install musicbrainzngs mutagen pycdio==0.21 PyGObject requests setuptools setuptools_scm + # newer version of pydcio requires newer version of libcdio than travis has + - pip install pycdio==0.21 + # install rest of dependencies + - pip install -r requirements.txt # Testing dependencies - pip install twisted flake8 diff --git a/requirements.txt b/requirements.txt index c62bb6df..cdbc3733 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,5 @@ mutagen pycdio>0.20 PyGObject requests +ruamel.yaml setuptools_scm diff --git a/whipper/result/logger.py b/whipper/result/logger.py index abacf72f..df277419 100644 --- a/whipper/result/logger.py +++ b/whipper/result/logger.py @@ -1,5 +1,8 @@ import time import hashlib +import re +import ruamel.yaml as yaml +from ruamel.yaml.comments import CommentedMap as OrderedDict import whipper @@ -16,68 +19,57 @@ class WhipperLogger(result.Logger): def log(self, ripResult, epoch=time.time()): """Returns big str: logfile joined text lines""" - lines = self.logRip(ripResult, epoch=epoch) - return "\n".join(lines) + return self.logRip(ripResult, epoch) def logRip(self, ripResult, epoch): """Returns logfile lines list""" - lines = [] + riplog = OrderedDict() # Ripper version - lines.append("Log created by: whipper %s (internal logger)" % - whipper.__version__) + riplog["Log created by"] = "whipper %s (internal logger)" % ( + whipper.__version__) # Rip date date = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(epoch)).strip() - lines.append("Log creation date: %s" % date) - lines.append("") + riplog["Log creation date"] = date # Rip technical settings - lines.append("Ripping phase information:") - lines.append(" Drive: %s%s (revision %s)" % ( - ripResult.vendor, ripResult.model, ripResult.release)) - lines.append(" Extraction engine: cdparanoia %s" % - ripResult.cdparanoiaVersion) - if ripResult.cdparanoiaDefeatsCache is None: - defeat = "null" - elif ripResult.cdparanoiaDefeatsCache: - defeat = "true" - else: - defeat = "false" - lines.append(" Defeat audio cache: %s" % defeat) - lines.append(" Read offset correction: %+d" % ripResult.offset) + data = OrderedDict() + + data["Drive"] = "%s%s (revision %s)" % ( + ripResult.vendor, ripResult.model, ripResult.release) + data["Extraction engine"] = "cdparanoia %s" % ( + ripResult.cdparanoiaVersion) + data["Defeat audio cache"] = ripResult.cdparanoiaDefeatsCache + data["Read offset correction"] = ripResult.offset + # Currently unsupported by the official cdparanoia package - over = "false" # Only implemented in whipper (ripResult.overread) - if ripResult.overread: - over = "true" - lines.append(" Overread into lead-out: %s" % over) + data["Overread into lead-out"] = True if ripResult.overread else False # Next one fully works only using the patched cdparanoia package # lines.append("Fill up missing offset samples with silence: true") - lines.append(" Gap detection: cdrdao %s" % ripResult.cdrdaoVersion) + data["Gap detection"] = "cdrdao %s" % ripResult.cdrdaoVersion - isCdr = "true" if ripResult.isCdr else "false" - lines.append(" CD-R detected: %s" % isCdr) - lines.append("") + data["CD-R detected"] = ripResult.isCdr + riplog["Ripping phase information"] = data # CD metadata - lines.append("CD metadata:") - lines.append(" Release:") - lines.append(" Artist: %s" % ripResult.artist) - lines.append(" Title: %s" % ripResult.title) - lines.append(" CDDB Disc ID: %s" % ripResult. table.getCDDBDiscId()) - lines.append(" MusicBrainz Disc ID: %s" % - ripResult. table.getMusicBrainzDiscId()) - lines.append(" MusicBrainz lookup URL: %s" % - ripResult. table.getMusicBrainzSubmitURL()) + release = OrderedDict() + release["Artist"] = ripResult.artist + release["Title"] = ripResult.title + data = OrderedDict() + data["Release"] = release + data["CDDB Disc ID"] = ripResult.table.getCDDBDiscId() + data["MusicBrainz Disc ID"] = ripResult.table.getMusicBrainzDiscId() + data["MusicBrainz lookup URL"] = ( + ripResult.table.getMusicBrainzSubmitURL()) if ripResult.metadata: - lines.append(" MusicBrainz Release URL: %s" % - ripResult.metadata.url) - lines.append("") + data["MusicBrainz Release URL"] = ripResult.metadata.url + riplog["CD metadata"] = data # TOC section - lines.append("TOC:") + data = OrderedDict() table = ripResult.table # Test for HTOA presence @@ -92,149 +84,171 @@ def logRip(self, ripResult, epoch): htoastart = htoa.absolute htoaend = table.getTrackEnd(0) htoalength = table.tracks[0].getIndex(1).absolute - htoastart - lines.append(" 0:") - lines.append(" Start: %s" % common.framesToMSF(htoastart)) - lines.append(" Length: %s" % common.framesToMSF(htoalength)) - lines.append(" Start sector: %d" % htoastart) - lines.append(" End sector: %d" % htoaend) - lines.append("") + track = OrderedDict() + track["Start"] = common.framesToMSF(htoastart) + track["Length"] = common.framesToMSF(htoalength) + track["Start sector"] = htoastart + track["End sector"] = htoaend + data[0] = track # For every track include information in the TOC for t in table.tracks: start = t.getIndex(1).absolute length = table.getTrackLength(t.number) end = table.getTrackEnd(t.number) - lines.append(" %d:" % t.number) - lines.append(" Start: %s" % common.framesToMSF(start)) - lines.append(" Length: %s" % common.framesToMSF(length)) - lines.append(" Start sector: %d" % start) - lines.append(" End sector: %d" % end) - lines.append("") + track = OrderedDict() + track["Start"] = common.framesToMSF(start) + track["Length"] = common.framesToMSF(length) + track["Start sector"] = start + track["End sector"] = end + data[t.number] = track + riplog["TOC"] = data # Tracks section - lines.append("Tracks:") + data = OrderedDict() duration = 0.0 for t in ripResult.tracks: if not t.filename: continue - track_lines, ARDB_entry, ARDB_match = self.trackLog(t) + track_dict, ARDB_entry, ARDB_match = self.trackLog(t) self._inARDatabase += int(ARDB_entry) self._accuratelyRipped += int(ARDB_match) - lines.extend(track_lines) - lines.append("") + data[t.number] = track_dict duration += t.testduration + t.copyduration + riplog["Tracks"] = data # Status report - lines.append("Conclusive status report:") - arHeading = " AccurateRip summary:" + data = OrderedDict() if self._inARDatabase == 0: - lines.append("%s None of the tracks are present in the " - "AccurateRip database" % arHeading) + message = ("None of the tracks are present in the " + "AccurateRip database") else: nonHTOA = len(ripResult.tracks) if ripResult.tracks[0].number == 0: nonHTOA -= 1 if self._accuratelyRipped == 0: - lines.append("%s No tracks could be verified as accurate " - "(you may have a different pressing from the " - "one(s) in the database)" % arHeading) + message = ("No tracks could be verified as accurate " + "(you may have a different pressing from the " + "one(s) in the database)") elif self._accuratelyRipped < nonHTOA: accurateTracks = nonHTOA - self._accuratelyRipped - lines.append("%s Some tracks could not be verified as " - "accurate (%d/%d got no match)" % ( - arHeading, accurateTracks, nonHTOA)) + message = ("Some tracks could not be verified as " + "accurate (%d/%d got no match)") % ( + accurateTracks, nonHTOA) else: - lines.append("%s All tracks accurately ripped" % arHeading) + message = "All tracks accurately ripped" + data["AccurateRip summary"] = message - hsHeading = " Health status:" if self._errors: - lines.append("%s There were errors" % hsHeading) + message = "There were errors" else: - lines.append("%s No errors occurred" % hsHeading) - lines.append(" EOF: End of status report") - lines.append("") + message = "No errors occurred" + data["Health Status"] = message + data["EOF"] = "End of status report" + riplog["Conclusive status report"] = data + + riplog = yaml.dump( + riplog, + default_flow_style=False, + width=4000, + Dumper=yaml.RoundTripDumper + ) + # Add a newline after the "Log creation date" line + riplog = re.sub( + r'^(Log creation date: .*)$', + "\\1\n", + riplog, + flags=re.MULTILINE + ) + # Add a newline after a dictionary ends and returns to top-level + riplog = re.sub( + r"^(\s{2})([^\n]*)\n([A-Z][^\n]+)", + "\\1\\2\n\n\\3", + riplog, + flags=re.MULTILINE + ) + # Add a newline after a track closes + riplog = re.sub( + r"^(\s{4}[^\n]*)\n(\s{2}[0-9]+)", + "\\1\n\n\\2", + riplog, + flags=re.MULTILINE + ) + # Remove single quotes around the "Log creation date" value + riplog = re.sub( + r"^(Log creation date: )'(.*)'", + "\\1\\2", + riplog, + flags=re.MULTILINE + ) # Log hash hasher = hashlib.sha256() - hasher.update("\n".join(lines).encode("utf-8")) - lines.append("SHA-256 hash: %s" % hasher.hexdigest().upper()) - lines.append("") - return lines + hasher.update(riplog.encode("utf-8")) + riplog += "\nSHA-256 hash: %s\n" % hasher.hexdigest().upper() + return riplog def trackLog(self, trackResult): """Returns Tracks section lines: data picked from trackResult""" - lines = [] - - # Track number - lines.append(" %d:" % trackResult.number) + track = OrderedDict() # Filename (including path) of ripped track - lines.append(" Filename: %s" % trackResult.filename) + track["Filename"] = trackResult.filename # Pre-gap length pregap = trackResult.pregap if pregap: - lines.append(" Pre-gap length: %s" % common.framesToMSF(pregap)) + track["Pre-gap length"] = common.framesToMSF(pregap) # Peak level peak = trackResult.peak / 32768.0 - lines.append(" Peak level: %.6f" % peak) + track["Peak level"] = float("%.6f" % peak) # Pre-emphasis status # Only implemented in whipper (trackResult.pre_emphasis) - preEmph = "true" if trackResult.pre_emphasis else "false" - lines.append(" Pre-emphasis: %s" % preEmph) + track["Pre-emphasis"] = trackResult.pre_emphasis # Extraction speed if trackResult.copyspeed: - lines.append(" Extraction speed: %.1f X" % ( - trackResult.copyspeed)) + track["Extraction speed"] = "%.1f X" % trackResult.copyspeed # Extraction quality if trackResult.quality and trackResult.quality > 0.001: - lines.append(" Extraction quality: %.2f %%" % - (trackResult.quality * 100.0, )) + track["Extraction quality"] = "%.2f %%" % ( + trackResult.quality * 100.0, ) # Ripper Test CRC if trackResult.testcrc is not None: - lines.append(" Test CRC: %08X" % trackResult.testcrc) + track["Test CRC"] = "%08X" % trackResult.testcrc # Ripper Copy CRC if trackResult.copycrc is not None: - lines.append(" Copy CRC: %08X" % trackResult.copycrc) + track["Copy CRC"] = "%08X" % trackResult.copycrc # AccurateRip track status ARDB_entry = 0 ARDB_match = 0 for v in ("v1", "v2"): + data = OrderedDict() if trackResult.AR[v]["DBCRC"]: - lines.append(" AccurateRip %s:" % v) ARDB_entry += 1 if trackResult.AR[v]["CRC"] == trackResult.AR[v]["DBCRC"]: - lines.append(" Result: Found, exact match") + data["Result"] = "Found, exact match" ARDB_match += 1 else: - lines.append(" Result: Found, NO exact match") - lines.append( - " Confidence: %d" % trackResult.AR[v]["DBConfidence"] - ) - lines.append( - " Local CRC: %s" % trackResult.AR[v]["CRC"].upper() - ) - lines.append( - " Remote CRC: %s" % trackResult.AR[v]["DBCRC"].upper() - ) + data["Result"] = "Found, NO exact match" + data["Confidence"] = trackResult.AR[v]["DBConfidence"] + data["Local CRC"] = trackResult.AR[v]["CRC"].upper() + data["Remote CRC"] = trackResult.AR[v]["DBCRC"].upper() elif trackResult.number != 0: - lines.append(" AccurateRip %s:" % v) - lines.append( - " Result: Track not present in AccurateRip database" - ) + data["Result"] = "Track not present in AccurateRip database" + track["AccurateRip %s" % v] = data # Check if Test & Copy CRCs are equal if trackResult.testcrc == trackResult.copycrc: - lines.append(" Status: Copy OK") + track["Status"] = "Copy OK" else: self._errors = True - lines.append(" Status: Error, CRC mismatch") - return lines, bool(ARDB_entry), bool(ARDB_match) + track["Status"] = "Error, CRC mismatch" + return track, bool(ARDB_entry), bool(ARDB_match) diff --git a/whipper/test/test_result_logger.log b/whipper/test/test_result_logger.log new file mode 100644 index 00000000..bd780d70 --- /dev/null +++ b/whipper/test/test_result_logger.log @@ -0,0 +1,80 @@ +Log created by: whipper 0.7.4.dev87+gb71ec9f.d20191026 (internal logger) +Log creation date: 2019-10-26T14:25:02Z + +Ripping phase information: + Drive: HL-DT-STBD-RE WH14NS40 (revision 1.03) + Extraction engine: cdparanoia cdparanoia III 10.2 libcdio 2.0.0 x86_64-pc-linux-gnu + Defeat audio cache: true + Read offset correction: 6 + Overread into lead-out: false + Gap detection: cdrdao 1.2.4 + CD-R detected: false + +CD metadata: + Release: + Artist: Example - Symbol - Artist + Title: 'Album With: - Dashes' + CDDB Disc ID: c30bde0d + MusicBrainz Disc ID: eyjySLXGdKigAjY3_C0nbBmNUHc- + MusicBrainz lookup URL: https://musicbrainz.org/cdtoc/attach?toc=1+13+228039+150+16414+33638+51378+69369+88891+104871+121645+138672+160748+178096+194680+212628&tracks=13&id=eyjySLXGdKigAjY3_C0nbBmNUHc- + +TOC: + 1: + Start: 00:00:00 + Length: 03:36:64 + Start sector: 0 + End sector: 16263 + + 2: + Start: 03:36:64 + Length: 03:49:49 + Start sector: 16264 + End sector: 33487 + +Tracks: + 1: + Filename: ./soundtrack/Various Artists - Shark Tale - Motion Picture Soundtrack/01. Sean Paul & Ziggy Marley - Three Little Birds.flac + Peak level: 0.90036 + Pre-emphasis: + Extraction speed: 7.0 X + Extraction quality: 100.00 % + Test CRC: 0025D726 + Copy CRC: 0025D726 + AccurateRip v1: + Result: Found, exact match + Confidence: 14 + Local CRC: 95E6A189 + Remote CRC: 95E6A189 + AccurateRip v2: + Result: Found, exact match + Confidence: 11 + Local CRC: 113FA733 + Remote CRC: 113FA733 + Status: Copy OK + + 2: + Filename: ./soundtrack/Various Artists - Shark Tale - Motion Picture Soundtrack/02. Christina Aguilera feat. Missy Elliott - Car Wash (Shark Tale mix).flac + Peak level: 0.972351 + Pre-emphasis: + Extraction speed: 7.7 X + Extraction quality: 100.00 % + Test CRC: F77C14CB + Copy CRC: F77C14CB + AccurateRip v1: + Result: Found, exact match + Confidence: 14 + Local CRC: 0B3316DB + Remote CRC: 0B3316DB + AccurateRip v2: + Result: Found, exact match + Confidence: 10 + Local CRC: A0AE0E57 + Remote CRC: A0AE0E57 + Status: Copy OK + +Conclusive status report: + AccurateRip summary: All tracks accurately ripped + Health Status: No errors occurred + EOF: End of status report + +SHA-256 hash: 2B176D8C722989B25459160E335E5CC0C1A6813C9DA69F869B625FBF737C475E diff --git a/whipper/test/test_result_logger.py b/whipper/test/test_result_logger.py new file mode 100644 index 00000000..a6650586 --- /dev/null +++ b/whipper/test/test_result_logger.py @@ -0,0 +1,169 @@ +from __future__ import print_function +import hashlib +import os +import re +import unittest +import ruamel.yaml + +from whipper.result.result import TrackResult, RipResult +from whipper.result.logger import WhipperLogger + + +class MockImageTrack: + def __init__(self, number, start, end): + self.number = number + self.absolute = self.start = start + self.end = end + + def getIndex(self, num): + if num == 0: + raise KeyError + else: + return self + + +class MockImageTable: + """Mock of whipper.image.table.Table, with fake information.""" + def __init__(self): + self.tracks = [ + MockImageTrack(1, 0, 16263), + MockImageTrack(2, 16264, 33487) + ] + + def getCDDBDiscId(self): + return "c30bde0d" + + def getMusicBrainzDiscId(self): + return "eyjySLXGdKigAjY3_C0nbBmNUHc-" + + def getMusicBrainzSubmitURL(self): + return ( + "https://musicbrainz.org/cdtoc/attach?toc=1+13+228039+150+16414+" + "33638+51378+69369+88891+104871+121645+138672+160748+178096+194680" + "+212628&tracks=13&id=eyjySLXGdKigAjY3_C0nbBmNUHc-" + ) + + def getTrackLength(self, number): + return self.tracks[number-1].end - self.tracks[number-1].start + 1 + + def getTrackEnd(self, number): + return self.tracks[number-1].end + + +class LoggerTestCase(unittest.TestCase): + def setUp(self): + self.path = os.path.join(os.path.dirname(__file__)) + + def testLogger(self): + ripResult = RipResult() + ripResult.offset = 6 + ripResult.overread = False + ripResult.isCdr = False + ripResult.table = MockImageTable() + ripResult.artist = "Example - Symbol - Artist" + ripResult.title = "Album With: - Dashes" + ripResult.vendor = "HL-DT-STBD-RE " + ripResult.model = "WH14NS40" + ripResult.release = "1.03" + ripResult.cdrdaoVersion = "1.2.4" + ripResult.cdparanoiaVersion = ( + "cdparanoia III 10.2 " + "libcdio 2.0.0 x86_64-pc-linux-gnu" + ) + ripResult.cdparanoiaDefeatsCache = True + + trackResult = TrackResult() + trackResult.number = 1 + trackResult.filename = ( + "./soundtrack/Various Artists - Shark Tale - Motion Picture " + "Soundtrack/01. Sean Paul & Ziggy Marley - Three Little Birds.flac" + ) + trackResult.pregap = 0 + trackResult.peak = 29503 + trackResult.quality = 1 + trackResult.copyspeed = 7 + trackResult.testduration = 10 + trackResult.copyduration = 10 + trackResult.testcrc = 0x0025D726 + trackResult.copycrc = 0x0025D726 + trackResult.AR = { + "v1": { + "DBConfidence": 14, + "DBCRC": "95E6A189", + "CRC": "95E6A189" + }, + "v2": { + "DBConfidence": 11, + "DBCRC": "113FA733", + "CRC": "113FA733" + } + } + ripResult.tracks.append(trackResult) + + trackResult = TrackResult() + trackResult.number = 2 + trackResult.filename = ( + "./soundtrack/Various Artists - Shark Tale - Motion Picture " + "Soundtrack/02. Christina Aguilera feat. Missy Elliott - Car " + "Wash (Shark Tale mix).flac" + ) + trackResult.pregap = 0 + trackResult.peak = 31862 + trackResult.quality = 1 + trackResult.copyspeed = 7.7 + trackResult.testduration = 10 + trackResult.copyduration = 10 + trackResult.testcrc = 0xF77C14CB + trackResult.copycrc = 0xF77C14CB + trackResult.AR = { + "v1": { + "DBConfidence": 14, + "DBCRC": "0B3316DB", + "CRC": "0B3316DB" + }, + "v2": { + "DBConfidence": 10, + "DBCRC": "A0AE0E57", + "CRC": "A0AE0E57" + } + } + ripResult.tracks.append(trackResult) + logger = WhipperLogger() + actual = logger.log(ripResult) + actualLines = actual.splitlines() + expectedLines = open( + os.path.join(self.path, 'test_result_logger.log'), 'r' + ).read().splitlines() + # do not test on version line, date line, or SHA-256 hash line + self.assertListEqual(actualLines[2:-1], expectedLines[2:-1]) + + self.assertRegexpMatches( + actualLines[0], + re.compile(( + r'Log created by: whipper ' + r'[\d]+\.[\d]+.[\d]+\.dev[\w\.\+]+ \(internal logger\)' + )) + ) + self.assertRegexpMatches( + actualLines[1], + re.compile(( + r'Log creation date: ' + r'20[\d]{2}\-[\d]{2}\-[\d]{2}T[\d]{2}:[\d]{2}:[\d]{2}Z' + )) + ) + + yaml = ruamel.yaml.YAML() + parsedLog = yaml.load(actual) + self.assertEqual( + actual, + ruamel.yaml.dump( + parsedLog, + default_flow_style=False, + width=4000, + Dumper=ruamel.yaml.RoundTripDumper + ) + ) + self.assertEqual( + parsedLog['SHA-256 hash'], + hashlib.sha256("\n".join(actualLines[:-1])).hexdigest().upper() + )