Merge pull request #107 from collective/exportimport_content_revisions

Export and import content revisions

README.rst

@@ -33,6 +33,8 @@ Features
 * Export & Import local roles
 * Export & Import order (position in parent)
 * Export & Import discussions/comments
+* Export & Import versioned content
+* Export & Import redirects
 
 Export supports:
 
@@ -188,6 +190,13 @@ You can set additional values by specifying a dict ``factory_kwargs`` that will
 Like this you can set values on the imported object that are expected to be there by subscribers to IObjectAddedEvent.
 
 
+Export versioned content
+------------------------
+
+Exporting versions of Archetypes content will not work because of a bug in plone.restapi (https://github.com/plone/plone.restapi/issues/1335).
+For export to work you need to use a version between 7.7.0 and 8.0.0 (if released) or a source-checkout of the branch 7.x.x.
+
+
 Notes on speed and large migrations
 ===================================
 

src/collective/exportimport/export_content.py

@@ -8,11 +8,13 @@
 from collective.exportimport.interfaces import IRawRichTextMarker
 from operator import itemgetter
 from plone import api
+from plone.app.layout.viewlets.content import ContentHistoryViewlet
 from plone.i18n.normalizer.interfaces import IIDNormalizer
 from plone.restapi.interfaces import ISerializeToJson
 from plone.restapi.serializer.converters import json_compatible
-from Products.CMFPlone.interfaces.constrains import ISelectableConstrainTypes
 from Products.CMFPlone.interfaces.constrains import ENABLED
+from Products.CMFPlone.interfaces.constrains import ISelectableConstrainTypes
+from Products.CMFPlone.utils import safe_unicode
 from Products.Five import BrowserView
 from Products.Five.browser.pagetemplatefile import ViewPageTemplateFile
 from zope.component import getMultiAdapter
@@ -106,6 +108,7 @@ def __call__(
         include_blobs=1,
         download_to_server=False,
         migration=True,
+        include_revisions=False,
     ):
         self.portal_type = portal_type or []
         if isinstance(self.portal_type, str):
@@ -134,6 +137,7 @@ def __call__(
             ("1", "as base-64 encoded strings"),
             ("2", "as blob paths"),
         )
+        self.include_revisions = include_revisions
 
         self.update()
 
@@ -298,6 +302,8 @@ def export_content(self):
                 item = self.fix_url(item, obj)
                 item = self.export_constraints(item, obj)
                 item = self.export_workflow_history(item, obj)
+                item = self.export_revisions(item, obj)
+
                 if self.migration:
                     item = self.update_data_for_migration(item, obj)
                 item = self.global_dict_hook(item, obj)
@@ -329,7 +335,7 @@ def portal_types(self):
                         "number": number,
                         "value": fti.id,
                         "title": translate(
-                            fti.title, domain="plone", context=self.request
+                            safe_unicode(fti.title), domain="plone", context=self.request
                         ),
                     }
                 )
@@ -464,6 +470,34 @@ def export_workflow_history(self, item, obj):
             item["workflow_history"] = results
         return item
 
+    def export_revisions(self, item, obj):
+        if not self.include_revisions:
+            return item
+        repo_tool = api.portal.get_tool("portal_repository")
+        history_metadata = repo_tool.getHistoryMetadata(obj)
+        serializer = getMultiAdapter((obj, self.request), ISerializeToJson)
+        content_history_viewlet = ContentHistoryViewlet(obj, self.request, None, None)
+        content_history_viewlet.navigation_root_url = ""
+        content_history_viewlet.site_url = ""
+        full_history = content_history_viewlet.fullHistory() or []
+        history = [i for i in full_history if i["type"] == "versioning"]
+        if not history or len(history) == 1:
+            return item
+        item["exportimport.versions"] = {}
+        # don't export the current version again
+        for history_item in history[1:]:
+            version_id = history_item["version_id"]
+            item_version = serializer(include_items=False, version=version_id)
+            item_version = self.update_data_for_migration(item_version, obj)
+            item["exportimport.versions"][version_id] = item_version
+            # inject metadata (missing for Archetypes content):
+            comment = history_metadata.retrieve(version_id)["metadata"]["sys_metadata"]["comment"]
+            if comment and comment != item["exportimport.versions"][version_id].get("changeNote"):
+                item["exportimport.versions"][version_id]["changeNote"] = comment
+        # current changenote
+        item["changeNote"] = history_metadata.retrieve(-1)["metadata"]["sys_metadata"]["comment"]
+        return item
+
 
 def fix_portal_type(portal_type):
     normalizer = getUtility(IIDNormalizer)

src/collective/exportimport/import_content.py

@@ -3,10 +3,12 @@
 from collective.exportimport import config
 from collective.exportimport.interfaces import IMigrationMarker
 from datetime import datetime
+from datetime import timedelta
 from DateTime import DateTime
 from Persistence import PersistentMapping
 from plone import api
 from plone.api.exc import InvalidParameterError
+from plone.dexterity.interfaces import IDexterityFTI
 from plone.i18n.normalizer.interfaces import IIDNormalizer
 from plone.namedfile.file import NamedBlobFile
 from plone.namedfile.file import NamedBlobImage
@@ -30,6 +32,7 @@
 import logging
 import os
 import random
+import six
 import transaction
 
 try:
@@ -109,6 +112,7 @@ def __call__(self, jsonfile=None, return_json=False, limit=None, server_file=Non
             ("2", "Update: Reuse and only overwrite imported data"),
             ("3", "Ignore: Create with a new id"),
         )
+        self.import_old_revisions = request.get("import_old_revisions", False)
 
         if not self.request.form.get("form.submitted", False):
             return self.template()
@@ -252,7 +256,7 @@ def import_new_content(self, data):  # noqa: C901
             for drop in self.DROP_PATHS:
                 if drop in item["@id"]:
                     skip = True
-                    logger.info(u"Skipping {}".format(item['@id']))
+                    logger.info(u"Skipping {}".format(item["@id"]))
             if skip:
                 continue
 
@@ -338,6 +342,15 @@ def import_new_content(self, data):  # noqa: C901
                         )
                     )
 
+            if self.import_old_revisions and item.get("exportimport.versions"):
+                # TODO: refactor into import_item to prevent duplicattion
+                new = self.import_versions(container, item)
+                if new:
+                    added.append(new.absolute_url())
+                if self.commit and not len(added) % self.commit:
+                    self.commit_hook(added, index)
+                continue
+
             if not self.update_existing:
                 # create without checking constrains and permissions
                 new = _createObjectByType(item["@type"], container, item["id"], **factory_kwargs)
@@ -399,6 +412,143 @@ def import_new_content(self, data):  # noqa: C901
 
         return added
 
+    def import_versions(self, container, item):
+        """Import one item with all its revisions..
+        We only apply hooks for the current object not for each version.
+        TODO: refactor into import_item to prevent duplicattion
+        """
+        portal_workflow = api.portal.get_tool("portal_workflow")
+
+        # Disable automatic versioning!
+        portal_types = api.portal.get_tool("portal_types")
+        fti = portal_types.get(item["@type"])
+
+        # disable versioning behavior to re-enable it after import
+        versioning_behavior = None
+        if IDexterityFTI.providedBy(fti):
+            fti_behaviors = list(fti.behaviors)
+            versioning_behaviors = [
+                "plone.versioning",
+                "plone.app.versioningbehavior.behaviors.IVersionable",
+            ]
+            for behavior in versioning_behaviors:
+                if behavior in fti_behaviors:
+                    versioning_behavior = behavior
+                    fti_behaviors.remove(behavior)
+                    fti.manage_changeProperties(behaviors=tuple(fti_behaviors))
+
+        # disable default versioning policy to re-enable it after import
+        repo_tool = api.portal.get_tool("portal_repository")
+        policy = None
+        policies = repo_tool._version_policy_mapping.get(item["@type"], [])
+        if "at_edit_autoversion" in policies:
+            policy = "at_edit_autoversion"
+            repo_tool.removePolicyFromContentType(item["@type"], policy)
+
+        for index, version in enumerate(item["exportimport.versions"].values()):
+            initial = index == 0
+            version = self.global_dict_hook(version)
+            if not version:
+                continue
+
+            # portal_type might change during a hook
+            version = self.custom_dict_hook(version)
+            if not version:
+                continue
+
+            if initial and not self.update_existing:
+                # initial version
+                new = _createObjectByType(item["@type"], container, item["id"])
+                uuid = self.set_uuid(item, new)
+                if uuid != item["UID"]:
+                    item["UID"] = uuid
+            else:
+                new = container.get(item["id"])
+
+            # import using plone.restapi deserializers
+            deserializer = getMultiAdapter((new, self.request), IDeserializeFromJson)
+            try:
+                new = deserializer(validate_all=False, data=version)
+            except Exception as error:
+                logger.warning(
+                    "cannot deserialize {}: {}".format(item["@id"], repr(error))
+                )
+                return
+
+            self.save_revision(new, version, initial)
+
+        # Finally create the current version
+        new = container.get(item["id"])
+        deserializer = getMultiAdapter((new, self.request), IDeserializeFromJson)
+        try:
+            new = deserializer(validate_all=False, data=item)
+        except Exception as error:
+            logger.warning("cannot deserialize {}: {}".format(item["@id"], repr(error)))
+            return
+
+        self.import_blob_paths(new, item)
+        self.import_constrains(new, item)
+        self.global_obj_hook(new, item)
+        self.custom_obj_hook(new, item)
+
+        if item["review_state"] and item["review_state"] != "private":
+            if portal_workflow.getChainFor(new):
+                try:
+                    api.content.transition(to_state=item["review_state"], obj=new)
+                except InvalidParameterError as e:
+                    logger.info(e)
+
+        # Import workflow_history last to drop entries created during import
+        self.import_workflow_history(new, item)
+
+        # Set modification and creation-date as a custom attribute as last step.
+        # These are reused and dropped in ResetModifiedAndCreatedDate
+        modified = item.get("modified", item.get("modification_date", None))
+        if modified:
+            modification_date = DateTime(dateutil.parser.parse(modified))
+            new.modification_date = modification_date
+            new.aq_base.modification_date_migrated = modification_date
+        created = item.get("created", item.get("creation_date", None))
+        if created:
+            creation_date = DateTime(dateutil.parser.parse(created))
+            new.creation_date = creation_date
+            new.aq_base.creation_date_migrated = creation_date
+
+        self.save_revision(new, item)
+        logger.info("Created item: {} {} with {} old versions".format(item["@type"], new.absolute_url(), len(item["exportimport.versions"])))
+
+        if policy:
+            repo_tool.addPolicyForContentType(item["@type"], policy)
+        if versioning_behavior:
+            fti_behaviors = list(fti.behaviors)
+            fti_behaviors.append(versioning_behavior)
+            fti.manage_changeProperties(behaviors=tuple(fti_behaviors))
+        return new
+
+    def save_revision(self, obj, item, initial=False):
+        """Save revision manually to set dates and changenote from exported data."""
+        rt = api.portal.get_tool("portal_repository")
+
+        modified = dateutil.parser.parse(item["modified"])
+        # add one millisecond to prevent created being before first revision
+        modified  = modified + timedelta(milliseconds=1)
+        if six.PY2:
+            import time
+            timestamp = time.mktime(modified.timetuple())
+        else:
+            timestamp = datetime.timestamp(modified)
+        from plone.app.versioningbehavior import _ as PAV
+        if initial:
+            comment = PAV(u"initial_version_changeNote", default=u"Initial version")
+        else:
+            comment = item.get("changeNote")
+        sys_metadata = {
+            "comment": comment,
+            "timestamp": timestamp,
+            "originator": None,
+        }
+        rt._recursiveSave(obj, app_metadata={}, sys_metadata=sys_metadata, autoapply=True)
+
     def handle_broken(self, item):
         """Fix some invalid values."""
         if item["id"] not in self.BUGS:
@@ -534,7 +684,7 @@ def custom_obj_hook(self, obj, item):
 
     def handle_container(self, item):
         """Specify a container per item and type using custom methods
-        Example for content_type 'Document:
+        Example for content_type Document:
 
         def handle_document_container(self, item):
             lang = item['language']['token'] if item['language'] else ''

src/collective/exportimport/templates/export_content.pt

@@ -118,6 +118,22 @@
               </label>
             </div>
 
+            <div class="field mb-3">
+              <label>
+                <input
+                    type="checkbox"
+                    class="form-check-input"
+                    name="include_revisions:boolean"
+                    id="include_revisions"
+                    tal:attributes="checked python: 'checked' if view.include_revisions else ''"
+                    />
+                Include revisions.
+                <span class="formHelp">
+                  This exports the content-history (versioning) of each exported item. Warning: This can significantly slow down the export!
+                </span>
+              </label>
+            </div>
+
             <div class="field mb-3">
               <div class="form-check">
                 <input class="form-check-input" type="radio" name="download_to_server:int" value="0" id="download_local" checked="checked">

src/collective/exportimport/templates/import_content.pt

@@ -72,6 +72,21 @@
               </label>
             </div>
 
+            <div class="field">
+              <label>
+                <input
+                    type="checkbox"
+                    name="import_old_revisions:boolean"
+                    id="import_old_revisions"
+                    tal:attributes="checked python:view.import_old_revisions"
+                    />
+                Import all old revisions
+                <span class="formHelp">
+                  This will import the content-history (versioning) for each item that has revisions. Warning: This can significantly slow down the import!
+                </span>
+              </label>
+            </div>
+
             <div class="formControls" class="form-group">
                 <input type="hidden" name="form.submitted" value="1"/>
                 <button class="btn btn-primary submit-widget button-field context"