Skip to content

Commit

Permalink
Simplify the conversion of very old project data
Browse files Browse the repository at this point in the history
  • Loading branch information
@vkbo
vkbo committed Jun 11, 2022
1 parent 5a5a64e commit c58868e
Show file tree
Hide file tree
Showing 2 changed files with 62 additions and 142 deletions.
98 changes: 28 additions & 70 deletions novelwriter/core/project.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -438,7 +438,7 @@ def openProject(self, fileName, overrideLock=False):
legacyList = [] # Cleanup is done later
for projItem in os.listdir(self.projPath):
logger.verbose("Project contains: %s", projItem)
if projItem.startswith("data_"):
if projItem.startswith("data_") and len(projItem) == 6:
legacyList.append(projItem)

# Project Lock
Expand Down Expand Up @@ -500,7 +500,7 @@ def openProject(self, fileName, overrideLock=False):
# Check File Type
# ===============

if not nwxRoot == "novelWriterXML":
if nwxRoot != "novelWriterXML":
self.mainGui.makeAlert(self.tr(
"Project file does not appear to be a novelWriterXML file."
), nwAlert.ERROR)
Expand Down Expand Up @@ -642,11 +642,14 @@ def openProject(self, fileName, overrideLock=False):

# Sort out old file locations
if legacyList:
errList = []
for projItem in legacyList:
errList = self._legacyDataFolder(projItem, errList)
if errList:
self.mainGui.makeAlert(errList, nwAlert.ERROR)
try:
for projItem in legacyList:
self._legacyDataFolder(projItem)
except Exception:
self.mainGui.makeAlert(self.tr(
"There was an error while converting project version 1.0. "
"Some data may not have been preserved."
), nwAlert.ERROR)

# Clean up no longer used files
self._deprecatedFiles()
Expand Down Expand Up @@ -1558,82 +1561,37 @@ def _appendSessionStats(self, idleTime):
# Legacy Data Structure Handlers
##

def _legacyDataFolder(self, theFolder, errList):
def _legacyDataFolder(self, dataDir):
"""Clean up legacy data folders.
"""
theData = os.path.join(self.projPath, theFolder)
if not os.path.isdir(theData):
errList.append(self.tr("Not a folder: {0}").format(theData))
return errList
dataPath = os.path.join(self.projPath, dataDir)
if not os.path.isdir(dataPath):
return False

logger.info("Old data folder %s found", theFolder)
logger.info("Old data folder found: %s", dataDir)

# Move Documents to Content
for dataItem in os.listdir(theData):
theFile = os.path.join(theData, dataItem)
if not os.path.isfile(theFile):
theErr = self._moveUnknownItem(theData, dataItem)
if theErr:
errList.append(theErr)
for dataItem in os.listdir(dataPath):
dataFile = os.path.join(dataPath, dataItem)
if not os.path.isfile(dataFile):
continue

if len(dataItem) == 21 and dataItem.endswith("_main.nwd"):
tHandle = theFolder[-1]+dataItem[:12]
newPath = os.path.join(self.projContent, tHandle+".nwd")
try:
os.rename(theFile, newPath)
logger.info("Moved file: %s", theFile)
logger.info("New location: %s", newPath)
except Exception:
errList.append(self.tr("Could not move: {0}").format(theFile))
logger.error("Could not move: %s", theFile)
logException()
tHandle = dataDir[-1] + dataItem[:12]
newPath = os.path.join(self.projContent, f"{tHandle}.nwd")
os.rename(dataFile, newPath)
logger.info("Moved file: %s", dataFile)

elif len(dataItem) == 21 and dataItem.endswith("_main.bak"):
try:
os.unlink(theFile)
logger.info("Deleted file: %s", theFile)
except Exception:
errList.append(self.tr("Could not delete: {0}").format(theFile))
logger.error("Could not delete: %s", theFile)
logException()

else:
theErr = self._moveUnknownItem(theData, dataItem)
if theErr:
errList.append(theErr)
os.unlink(dataFile)
logger.info("Deleted file: %s", dataFile)

# Remove Data Folder
try:
os.rmdir(theData)
logger.info("Deleted folder: %s", theFolder)
except Exception:
errList.append(self.tr("Could not delete: {0}").format(theFolder))
logger.error("Could not delete: %s", theFolder)
logException()

return errList

def _moveUnknownItem(self, theDir, theItem):
"""Move an item that doesn't belong in the project folder to
a junk folder.
"""
theJunk = os.path.join(self.projPath, "junk")
if not self._checkFolder(theJunk):
return self.tr("Could not make folder: {0}").format(theJunk)
if not os.listdir(dataPath):
os.rmdir(dataPath)
logger.info("Deleted folder: %s", dataDir)

theSrc = os.path.join(theDir, theItem)
theDst = os.path.join(theJunk, theItem)

try:
os.rename(theSrc, theDst)
logger.info("Moved to junk: %s", theSrc)
except Exception:
logger.error("Could not move item %s to junk", theSrc)
logException()
return self.tr("Could not move item {0} to {1}.").format(theSrc, theJunk)

return ""
return True

def _deprecatedFiles(self):
"""Delete files that are no longer used by novelWriter.
Expand Down
106 changes: 34 additions & 72 deletions tests/test_core/test_core_project.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -463,12 +463,15 @@ def testCoreProject_Open(monkeypatch, nwMinimal, mockGUI):
os.rename(oName, rName)

# Add some legacy stuff that cannot be removed
writeFile(os.path.join(nwMinimal, "junk"), "stuff")
os.mkdir(os.path.join(nwMinimal, "data_0"))
writeFile(os.path.join(nwMinimal, "data_0", "junk"), "stuff")
mockGUI.clear()
assert theProject.openProject(nwMinimal) is True
assert "data_0" in mockGUI.lastAlert
with monkeypatch.context() as mp:
mp.setattr(theProject, "_legacyDataFolder", causeOSError)
os.mkdir(os.path.join(nwMinimal, "data_0"))
writeFile(os.path.join(nwMinimal, "data_0", "123456789abc_main.nwd"), "stuff")
writeFile(os.path.join(nwMinimal, "data_0", "123456789abc_main.bak"), "stuff")
mockGUI.clear()
assert theProject.openProject(nwMinimal) is True
assert "version 1.0" in mockGUI.lastAlert

assert theProject.closeProject()

# END Test testCoreProject_Open
Expand Down Expand Up @@ -1141,14 +1144,6 @@ def testCoreProject_OldFormat(mockGUI, nwOldProj):
os.path.join(nwOldProj, "meta", "sessionLogOptions.json"),
]

# Add some files that shouldn't be there
deleteFiles.append(os.path.join(nwOldProj, "data_f", "whatnow.nwd"))
deleteFiles.append(os.path.join(nwOldProj, "data_f", "whatnow.txt"))

# Add some folders that shouldn't be there
os.mkdir(os.path.join(nwOldProj, "stuff"))
os.mkdir(os.path.join(nwOldProj, "data_1", "stuff"))

# Create mock files
os.mkdir(os.path.join(nwOldProj, "cache"))
for aFile in deleteFiles:
Expand All @@ -1162,20 +1157,13 @@ def testCoreProject_OldFormat(mockGUI, nwOldProj):
for aFile in deleteFiles:
assert not os.path.isfile(aFile)

assert not os.path.isdir(os.path.join(nwOldProj, "data_1", "stuff"))
assert not os.path.isdir(os.path.join(nwOldProj, "data_1"))
assert not os.path.isdir(os.path.join(nwOldProj, "data_7"))
assert not os.path.isdir(os.path.join(nwOldProj, "data_8"))
assert not os.path.isdir(os.path.join(nwOldProj, "data_9"))
assert not os.path.isdir(os.path.join(nwOldProj, "data_a"))
assert not os.path.isdir(os.path.join(nwOldProj, "data_f"))

# Check stuff that has been moved
assert os.path.isdir(os.path.join(nwOldProj, "junk"))
assert os.path.isdir(os.path.join(nwOldProj, "junk", "stuff"))
assert os.path.isfile(os.path.join(nwOldProj, "junk", "whatnow.nwd"))
assert os.path.isfile(os.path.join(nwOldProj, "junk", "whatnow.txt"))

# Check that files we want to keep are in the right place
assert os.path.isdir(os.path.join(nwOldProj, "cache"))
assert os.path.isdir(os.path.join(nwOldProj, "content"))
Expand Down Expand Up @@ -1217,7 +1205,7 @@ def testCoreProject_LegacyData(monkeypatch, mockGUI, fncDir):

with monkeypatch.context() as mp:
mp.setattr("os.unlink", causeOSError)
assert not theProject._deprecatedFiles()
assert theProject._deprecatedFiles() is False

assert theProject._deprecatedFiles()
assert not os.path.isfile(tstFile)
Expand All @@ -1226,63 +1214,36 @@ def testCoreProject_LegacyData(monkeypatch, mockGUI, fncDir):
tstFile = os.path.join(fncDir, "data_0")
writeFile(tstFile, "stuff")
assert os.path.isfile(tstFile)

errList = []
errList = theProject._legacyDataFolder(tstFile, errList)
assert len(errList) > 0

# Move folder in data folder, shouldn't be there
tstData = os.path.join(fncDir, "data_1")
errItem = os.path.join(fncDir, "data_1", "stuff")
os.mkdir(tstData)
os.mkdir(errItem)
assert os.path.isdir(tstData)
assert os.path.isdir(errItem)

# This causes a failure to create the 'junk' folder
with monkeypatch.context() as mp:
mp.setattr("os.mkdir", causeOSError)
errList = []
errList = theProject._legacyDataFolder(tstData, errList)
assert len(errList) > 0

# This causes a failure to move 'stuff' to 'junk'
with monkeypatch.context() as mp:
mp.setattr("os.rename", causeOSError)
errList = []
errList = theProject._legacyDataFolder(tstData, errList)
assert len(errList) > 0

# This should be successful
errList = []
errList = theProject._legacyDataFolder(tstData, errList)
assert len(errList) == 0
assert os.path.isdir(os.path.join(fncDir, "junk", "stuff"))
assert theProject._legacyDataFolder(tstFile) is False

# Check renaming/deleting of old document files
tstData = os.path.join(fncDir, "data_2")
tstDoc1m = os.path.join(tstData, "000000000001_main.nwd")
tstDoc1b = os.path.join(tstData, "000000000001_main.bak")
tstDoc2m = os.path.join(tstData, "000000000002_main.nwd")
tstDoc2b = os.path.join(tstData, "000000000002_main.bak")
tstDoc3m = os.path.join(tstData, "tooshort003_main.nwd")
tstDoc3b = os.path.join(tstData, "tooshort003_main.bak")

os.mkdir(tstData)
tstData2 = os.path.join(fncDir, "data_2")
tstData3 = os.path.join(fncDir, "data_3")
tstDoc1m = os.path.join(tstData2, "000000000001_main.nwd")
tstDoc1b = os.path.join(tstData2, "000000000001_main.bak")
tstDoc2m = os.path.join(tstData2, "000000000002_main.nwd")
tstDoc2b = os.path.join(tstData2, "000000000002_main.bak")
tstDoc3m = os.path.join(tstData3, "tooshort003_main.nwd")
tstDoc3b = os.path.join(tstData3, "tooshort003_main.bak")
tstDir4a = os.path.join(tstData3, "stuff")

os.mkdir(tstData2)
os.mkdir(tstData3)
writeFile(tstDoc1m, "stuff")
writeFile(tstDoc1b, "stuff")
writeFile(tstDoc2m, "stuff")
writeFile(tstDoc2b, "stuff")
writeFile(tstDoc3m, "stuff")
writeFile(tstDoc3b, "stuff")
os.mkdir(tstDir4a)

# Make the above fail
with monkeypatch.context() as mp:
mp.setattr("os.rename", causeOSError)
mp.setattr("os.unlink", causeOSError)
errList = []
errList = theProject._legacyDataFolder(tstData, errList)
assert len(errList) > 0
with pytest.raises(OSError):
theProject._legacyDataFolder(tstData2)
theProject._legacyDataFolder(tstData3)
assert os.path.isfile(tstDoc1m)
assert os.path.isfile(tstDoc1b)
assert os.path.isfile(tstDoc2m)
Expand All @@ -1291,15 +1252,16 @@ def testCoreProject_LegacyData(monkeypatch, mockGUI, fncDir):
assert os.path.isfile(tstDoc3b)

# And succeed ...
errList = []
errList = theProject._legacyDataFolder(tstData, errList)
assert len(errList) == 0
assert theProject._legacyDataFolder(tstData2) is True
assert theProject._legacyDataFolder(tstData3) is True

assert not os.path.isdir(tstData)
assert not os.path.isdir(tstData2)
assert os.path.isdir(tstData3)
assert os.path.isfile(os.path.join(fncDir, "content", "2000000000001.nwd"))
assert os.path.isfile(os.path.join(fncDir, "content", "2000000000002.nwd"))
assert os.path.isfile(os.path.join(fncDir, "junk", "tooshort003_main.nwd"))
assert os.path.isfile(os.path.join(fncDir, "junk", "tooshort003_main.bak"))
assert os.path.isfile(os.path.join(fncDir, tstData3, "tooshort003_main.nwd"))
assert os.path.isfile(os.path.join(fncDir, tstData3, "tooshort003_main.bak"))
assert os.path.isdir(tstDir4a)

# END Test testCoreProject_LegacyData

Expand Down

0 comments on commit c58868e

Please sign in to comment.