Changes to tika, fixed reindex problem

roda-common/roda-common-data/src/main/java/org/roda/core/data/common/RodaConstants.java

@@ -340,6 +340,8 @@ public enum DateGranularity {
   public static final String API_REST_V1_DISTRIBUTED_INSTANCE = "api/v1/distributed_instances/";
   public static final String API_REST_V1_JOBS = "api/v1/jobs/";
 
+  public static final String API_REST_V1_REPRESENTATION_OTHER_METADATA = "otherMetadata";
+
   // sub-resources strings
   public static final String API_DATA = "data";
   public static final Object API_FILE = "file";
@@ -1740,6 +1742,7 @@ public String toString() {
   public static final String SEARCH_TRANSFERRED_RESOURCES = "transferred_resources";
 
   public static final String SEARCH_FIELD_PREFIX = "ui.search.fields";
+
   public static final String SEARCH_FIELD_FIELDS = "fields";
   public static final String SEARCH_FIELD_TYPE = "type";
   public static final String SEARCH_FIELD_I18N = "i18n";

roda-core/roda-core/src/main/java/org/roda/core/RodaCoreFactory.java

@@ -290,7 +290,7 @@ public Map<String, List<String>> load(Locale locale) {
           }
 
           List<String> properties = RodaCoreFactory
-            .getRodaConfigurationAsList("ui.sharedProperties.whitelist.messages.property");
+            .getRodaConfigurationAsList("ui.display.properties.tika.fixed");
           for (String propertyKey : properties) {
             if (messages.containsTranslation(propertyKey)) {
               sharedProperties.put("i18n." + propertyKey,

roda-core/roda-core/src/main/java/org/roda/core/common/MetadataFileUtils.java

@@ -26,6 +26,7 @@
 import org.jdom2.IllegalDataException;
 import org.jdom2.output.Format;
 import org.jdom2.output.XMLOutputter;
+import org.roda.core.RodaCoreFactory;
 import org.roda.core.data.v2.ip.TransferredResource;
 import org.roda.core.storage.Binary;
 import org.roda.core.storage.ContentPayload;
@@ -77,7 +78,9 @@ public static Map<String, List<String>> parseBinary(Binary binary)
     Object result = expr.evaluate(doc, XPathConstants.NODESET);
     NodeList nodes = (NodeList) result;
     for (int i = 0; i < nodes.getLength(); i++) {
-      String name = nodes.item(i).getAttributes().getNamedItem("name").getNodeValue() + "_txt";
+      String key = nodes.item(i).getAttributes().getNamedItem("name").getNodeValue();
+      String name = key + RodaCoreFactory.getRodaConfiguration()
+        .getString("core.indexedFile.tika." + key.replaceAll("[:\\s]", "") + ".solr_prefix", "_txt");
       String value = nodes.item(i).getTextContent();
       List<String> values = new ArrayList<>();
       if (otherProperties.containsKey(name)) {

roda-core/roda-core/src/main/java/org/roda/core/index/IndexModelObserver.java

@@ -259,6 +259,7 @@ private ReturnWithExceptions<Void, ModelObserver> indexRepresentation(final AIP
             aip.setHasShallowFiles(true);
             indexAIP(aip, ancestors).addTo(ret);
           }
+
           sizeInBytes += indexFile(aip, file.get(), ancestors, false).addTo(ret).getReturnedObject();
 
           if (file.get().isDirectory()) {
@@ -275,6 +276,27 @@ private ReturnWithExceptions<Void, ModelObserver> indexRepresentation(final AIP
         }
       }
 
+      // treat other metadata
+      CloseableIterable<OptionalWithCause<OtherMetadata>> allOtherMetadata = model
+        .listOtherMetadata(representation.getAipId(), representation.getId());
+
+      for (OptionalWithCause<OtherMetadata> otherMetadata : allOtherMetadata) {
+
+        String suffix = otherMetadata.get().getFileSuffix();
+        String type = otherMetadata.get().getType();
+        List<String> path = new ArrayList<>();
+
+        // suppose that all suffixes in othermetadata are like this
+        // ".metadata.<real_suffix>
+        String fileId = otherMetadata.get().getFileId();
+        suffix = fileId.substring(fileId.lastIndexOf(".")) + suffix;
+        fileId = fileId.substring(0, fileId.lastIndexOf("."));
+
+        OtherMetadata om = model.retrieveOtherMetadata(aip.getId(), representation.getId(), path, fileId, suffix, type);
+
+        otherMetadataCreated(om);
+      }
+
       // TODO support safemode
       boolean safemode = false;
 
@@ -578,7 +600,8 @@ public ReturnWithExceptions<Void, ModelObserver> representationUpdatedOn(Represe
     updatedFields.put(RodaConstants.REPRESENTATION_UPDATED_BY, representation.getUpdatedBy());
     updatedFields.put(RodaConstants.REPRESENTATION_UPDATED_ON, representation.getUpdatedOn());
     String representationUUID = IdUtils.getRepresentationId(representation.getAipId(), representation.getId());
-    SolrUtils.update(index, IndexedRepresentation.class, representationUUID, updatedFields, (ModelObserver) this).addTo(ret);
+    SolrUtils.update(index, IndexedRepresentation.class, representationUUID, updatedFields, (ModelObserver) this)
+      .addTo(ret);
     return ret;
   }
 

roda-core/roda-core/src/main/java/org/roda/core/index/schema/collections/FileCollection.java

@@ -37,7 +37,6 @@
 import org.roda.core.index.schema.AbstractSolrCollection;
 import org.roda.core.index.schema.CopyField;
 import org.roda.core.index.schema.Field;
-import org.roda.core.index.utils.IndexUtils;
 import org.roda.core.index.utils.SolrUtils;
 import org.roda.core.model.utils.ModelUtils;
 import org.roda.core.storage.Binary;
@@ -329,11 +328,9 @@ public IndexedFile fromSolrDocument(SolrDocument doc, List<String> fieldsToRetur
     // handle other properties
     Map<String, List<String>> otherProperties = new HashMap<>();
     for (String fieldName : doc.getFieldNames()) {
-      if (fieldName.endsWith("_txt")) {
-        List<String> otherProperty = SolrUtils.objectToListString(doc.get(fieldName));
-        otherProperties.put(fieldName, otherProperty);
-      }
 
+      List<String> otherProperty = SolrUtils.objectToListString(String.valueOf(doc.get(fieldName)));
+      otherProperties.put(fieldName, otherProperty);
     }
 
     FileFormat fileFormat = new FileFormat(formatDesignationName, formatDesignationVersion, mimetype, pronom, extension,

roda-core/roda-core/src/main/java/org/roda/core/model/utils/ModelUtils.java

@@ -30,7 +30,6 @@
 import org.roda.core.data.exceptions.RequestNotValidException;
 import org.roda.core.data.utils.JsonUtils;
 import org.roda.core.data.v2.IsRODAObject;
-import org.roda.core.data.v2.index.IsIndexed;
 import org.roda.core.data.v2.index.select.SelectedItems;
 import org.roda.core.data.v2.index.select.SelectedItemsFilter;
 import org.roda.core.data.v2.index.select.SelectedItemsList;
@@ -818,6 +817,22 @@ public static StoragePath getOtherMetadataStoragePath(String aipId, String repre
     return DefaultStoragePath.parse(path);
   }
 
+  public static StoragePath getOtherMetadataStoragePath(String aipId, String representationId, String fileName, String type, String fileSuffix) throws RequestNotValidException {
+
+    List<String> path;
+    if (aipId != null && representationId != null && fileName != null && fileSuffix != null && type != null) {
+      // other metadata pertaining to a file
+      path = getRepresentationOtherMetadataPath(aipId, representationId, type);
+      path.add(fileName + fileSuffix);
+    }
+    else if (aipId != null && representationId != null) {
+      path = getRepresentationOtherMetadataPath(aipId, representationId, "");
+    } else {
+      throw new RequestNotValidException("AIP id cannot be null");
+    }
+    return DefaultStoragePath.parse(path);
+  }
+
   public static <T extends Serializable> StoragePath getContainerPath(Class<T> clazz) throws RequestNotValidException {
     if (clazz.equals(RepresentationInformation.class)) {
       return getRepresentationInformationContainerPath();

roda-core/roda-core/src/main/resources/config/roda-core.properties

@@ -535,4 +535,4 @@ core.aip.lockToEdit=false
 # User registration settings
 #
 ##########################################################################
-core.user_registration.disabled = false
+core.user_registration.disabled = false

roda-core/roda-core/src/main/resources/config/roda-permissions.properties

@@ -94,6 +94,8 @@ core.permissions.org.roda.wui.api.controllers.Browser.retrieveAIPPart = READ
 core.permissions.org.roda.wui.api.controllers.Browser.retrieveAIPRepresentation = READ
 core.permissions.org.roda.wui.api.controllers.Browser.retrieveAIPRepresentationFile = READ
 core.permissions.org.roda.wui.api.controllers.Browser.retrieveAIPRepresentationPart = READ
+core.permissions.org.roda.wui.api.controllers.Browser.retrieveAIPRepresentationOthermetadata = READ
+core.permissions.org.roda.wui.api.controllers.Browser.retrieveAIPRepresentationOthermetadataFile = READ
 core.permissions.org.roda.wui.api.controllers.Browser.retrieveAIPRepresentationPreservationMetadata = READ
 core.permissions.org.roda.wui.api.controllers.Browser.retrieveAIPRepresentationPreservationMetadataFile = READ
 core.permissions.org.roda.wui.api.controllers.Browser.retrieveAncestors = READ

roda-core/roda-core/src/main/resources/config/roda-roles.properties

@@ -169,6 +169,10 @@ core.roles.org.roda.wui.api.controllers.Browser.retrieveAIPDescriptiveMetadataVe
 core.roles.org.roda.wui.api.controllers.Browser.retrieveAIPPart = aip.read
 core.roles.org.roda.wui.api.controllers.Browser.retrieveAIPRepresentation = representation.view
 core.roles.org.roda.wui.api.controllers.Browser.retrieveAIPRepresentation = representation.read
+core.roles.org.roda.wui.api.controllers.Browser.retrieveAIPRepresentationOthermetadata = representation.view
+core.roles.org.roda.wui.api.controllers.Browser.retrieveAIPRepresentationOthermetadata = representation.read
+core.roles.org.roda.wui.api.controllers.Browser.retrieveAIPRepresentationOthermetadataFile = representation.view
+core.roles.org.roda.wui.api.controllers.Browser.retrieveAIPRepresentationOthermetadataFile = representation.read
 core.roles.org.roda.wui.api.controllers.Browser.retrieveAIPRepresentationFile = representation.view
 core.roles.org.roda.wui.api.controllers.Browser.retrieveAIPRepresentationFile = representation.read
 core.roles.org.roda.wui.api.controllers.Browser.retrieveAIPRepresentationPart = representation.read

roda-ui/roda-wui/src/main/java/config/i18n/client/ClientMessages.java

@@ -122,6 +122,8 @@ public interface ClientMessages extends Messages {
 
   String changeStatusSuccessful();
 
+  String otherMetadata();
+
   String statusLabel(@Select String status);
 
   String otherStatusLabel();
@@ -142,6 +144,8 @@ public interface ClientMessages extends Messages {
 
   String chooseEntityTitle();
 
+  String tikaTitles(@Select String tika);
+
   /******* SIDEBAR ******************************************/
   String sidebarFilterDates();
 
@@ -508,6 +512,11 @@ public interface ClientMessages extends Messages {
 
   String viewRepresentationInfoPronom();
 
+  String viewTikaInformation();
+
+  String viewTechnicalMetadata();
+
+  String showTechnicalMetadata();
   String viewRepresentationInfoCreatingApplicationName();
 
   String viewRepresentationInfoCreatingApplicationVersion();

roda-ui/roda-wui/src/main/java/org/roda/wui/api/controllers/Browser.java

@@ -534,6 +534,68 @@ public static EntityResponse retrieveAIPRepresentation(User user, String aipId,
     }
   }
 
+  public static EntityResponse retrieveAIPRepresentationOthermetadata(User user, String aipId, String representationId,
+    String acceptFormat)
+    throws AuthorizationDeniedException, GenericException, NotFoundException, RequestNotValidException {
+    final ControllerAssistant controllerAssistant = new ControllerAssistant() {};
+
+    // validate input
+    BrowserHelper.validateGetAIPRepresentationParams(acceptFormat);
+
+    // check user permissions
+    controllerAssistant.checkRoles(user);
+
+    LogEntryState state = LogEntryState.SUCCESS;
+
+    try {
+      // delegate
+      IndexedRepresentation representation = BrowserHelper.retrieve(IndexedRepresentation.class,
+        IdUtils.getRepresentationId(aipId, representationId), RodaConstants.REPRESENTATION_FIELDS_TO_RETURN);
+
+      controllerAssistant.checkObjectPermissions(user, representation);
+
+      return BrowserHelper.retrieveAIPRepresentationOthermetadata(representation, acceptFormat);
+    } catch (RODAException e) {
+      state = LogEntryState.FAILURE;
+      throw e;
+    } finally {
+      // register action
+      controllerAssistant.registerAction(user, aipId, state, RodaConstants.CONTROLLER_AIP_ID_PARAM, aipId,
+        RodaConstants.CONTROLLER_REPRESENTATION_ID_PARAM, representationId);
+    }
+  }
+
+  public static EntityResponse retrieveAIPRepresentationOthermetadataFile(User user, String aipId,
+    String representationId, String filename, String type, String extension, String acceptFormat)
+    throws RequestNotValidException, AuthorizationDeniedException, NotFoundException, GenericException {
+    final ControllerAssistant controllerAssistant = new ControllerAssistant() {};
+
+    // validate input
+    BrowserHelper.validateGetFileParams(acceptFormat);
+
+    // check user permissions
+    controllerAssistant.checkRoles(user);
+
+    LogEntryState state = LogEntryState.SUCCESS;
+
+    try {
+      // delegate
+      IndexedRepresentation representation = BrowserHelper.retrieve(IndexedRepresentation.class,
+        IdUtils.getRepresentationId(aipId, representationId), RodaConstants.REPRESENTATION_FIELDS_TO_RETURN);
+
+      controllerAssistant.checkObjectPermissions(user, representation);
+
+      return BrowserHelper.retrieveAIPRepresentationOthermetadataFile(representation, acceptFormat, filename, extension, type);
+    } catch (RODAException e) {
+      state = LogEntryState.FAILURE;
+      throw e;
+    } finally {
+      // register action
+      controllerAssistant.registerAction(user, aipId, state, RodaConstants.CONTROLLER_AIP_ID_PARAM, aipId,
+        RodaConstants.CONTROLLER_REPRESENTATION_ID_PARAM, representationId);
+    }
+  }
+
   public static StreamResponse retrieveAIPPart(User user, String aipId, String part)
     throws RequestNotValidException, AuthorizationDeniedException, GenericException, NotFoundException {
     final ControllerAssistant controllerAssistant = new ControllerAssistant() {};
@@ -3234,7 +3296,8 @@ public static Notification acknowledgeNotification(User user, String notificatio
   }
 
   public static Reports listReports(User user, String id, String resourceOrSip, int start, int limit,
-    String acceptFormat) throws AuthorizationDeniedException, GenericException, RequestNotValidException, NotFoundException {
+    String acceptFormat)
+    throws AuthorizationDeniedException, GenericException, RequestNotValidException, NotFoundException {
     final ControllerAssistant controllerAssistant = new ControllerAssistant() {};
 
     // validate input