Support UTF-8 in RPSL Objects

whois-api/src/main/java/net/ripe/db/whois/api/UpdateCreator.java

@@ -2,7 +2,6 @@
 
 import net.ripe.db.whois.common.Latin1Conversion;
 import net.ripe.db.whois.common.Latin1ConversionResult;
-import net.ripe.db.whois.common.PunycodeConversion;
 import net.ripe.db.whois.update.domain.Operation;
 import net.ripe.db.whois.update.domain.Paragraph;
 import net.ripe.db.whois.update.domain.Update;
@@ -19,21 +18,14 @@ public static Update createUpdate(final Paragraph paragraph,
                                       final String rpslObject,
                                       final UpdateContext updateContext) {
 
-        final String punycodeResult = PunycodeConversion.convert(rpslObject);
+        final Latin1ConversionResult conversionResult = Latin1Conversion.convert(rpslObject);
+        final Update update = new Update(paragraph, operation, deleteReasons, conversionResult.getRpslObject());
 
-        final Latin1ConversionResult latin1ConversionResult = Latin1Conversion.convert(punycodeResult);
-
-        final Update update = new Update(paragraph, operation, deleteReasons, latin1ConversionResult.getRpslObject());
-
-        if (!punycodeResult.equals(rpslObject)) {
-            updateContext.addMessage(update, UpdateMessages.valueChangedDueToPunycodeConversion());
-        }
-
-        if (latin1ConversionResult.isGlobalSubstitution()) {
+        if (conversionResult.isGlobalSubstitution()) {
             updateContext.addMessage(update, UpdateMessages.valueChangedDueToLatin1Conversion());
         }
 
-        latin1ConversionResult.getSubstitutedAttributes().forEach(attr -> updateContext.addMessage(update, attr, UpdateMessages.valueChangedDueToLatin1Conversion(attr.getKey())));
+        conversionResult.getSubstitutedAttributes().forEach(attr -> updateContext.addMessage(update, attr, UpdateMessages.valueChangedDueToLatin1Conversion(attr.getKey())));
 
         return update;
     }

whois-api/src/main/java/net/ripe/db/whois/api/mail/dequeue/MessageParser.java

@@ -278,7 +278,7 @@ Charset getCharset(final ContentType contentType) {
             }
         }
 
-        return StandardCharsets.ISO_8859_1;
+        return StandardCharsets.UTF_8;
     }
 
     String getHeaders(final Part part) throws MessagingException {

whois-api/src/main/java/net/ripe/db/whois/api/rest/SyncUpdatesService.java

@@ -295,7 +295,6 @@ private Charset getCharset(final String contentType) {
                 }
             }
         }
-
         // application/x-www-form-urlencoded is UTF-8 by default
         return StandardCharsets.UTF_8;
     }

whois-api/src/test/java/net/ripe/db/whois/api/mail/dequeue/MessageParserTest.java

@@ -762,7 +762,7 @@ public void parse_smime_multipart_alternative() throws Exception {
 
     @Test
     public void illegal_charset() throws Exception {
-        assertThat(subject.getCharset(new ContentType("text/plain;\n\tcharset=\"_iso-2022-jp$ESC\"")), is(StandardCharsets.ISO_8859_1));
+        assertThat(subject.getCharset(new ContentType("text/plain;\n\tcharset=\"_iso-2022-jp$ESC\"")), is(StandardCharsets.UTF_8));
     }
 
     @Test

whois-api/src/test/java/net/ripe/db/whois/api/rest/SyncUpdatesServiceTestIntegration.java

@@ -722,7 +722,7 @@ public void post_url_encoded_data() {
     }
 
     @Test
-    public void post_url_encoded_data_with_latin1_charset_error() {
+    public void post_url_encoded_data_with_latin1_charset_failure() {
         rpslObjectUpdateDao.createObject(RpslObject.parse(PERSON_ANY1_TEST));
         rpslObjectUpdateDao.createObject(RpslObject.parse(MNTNER_TEST_MNTNER));
 
@@ -744,7 +744,7 @@ public void post_url_encoded_data_with_latin1_charset_error() {
     }
 
     @Test
-    public void post_url_encoded_data_with_latin1_charset() {
+    public void post_url_encoded_data_with_non_latin1_address_success() {
         rpslObjectUpdateDao.createObject(RpslObject.parse(PERSON_ANY1_TEST));
         rpslObjectUpdateDao.createObject(RpslObject.parse(MNTNER_TEST_MNTNER));
 
@@ -805,7 +805,7 @@ public void post_url_encoded_data_with_non_latin1_address() {
                   MediaType.valueOf("application/x-www-form-urlencoded; charset=UTF-8")), String.class);
 
         assertThat(databaseHelper.lookupObject(ObjectType.PERSON, "TP2-TEST").toString(),
-                containsString("address:        ???????? ?????,??????"));
+                containsString("address:        Тверская улица,москва"));
     }
 
     @Test
@@ -899,12 +899,14 @@ public void post_multipart_data_with_non_latin1_address() {
                         "source:         TEST\n" +
                         "password: emptypassword")
                 .field("NEW", "yes");
-        RestTest.target(getPort(), "whois/syncupdates/test")
+
+        final String response = RestTest.target(getPort(), "whois/syncupdates/test")
                 .request()
                 .post(Entity.entity(multipart, multipart.getMediaType()), String.class);
 
+        assertThat(response, containsString("Create SUCCEEDED: [person] TP2-TEST   Test Person"));
         assertThat(databaseHelper.lookupObject(ObjectType.PERSON, "TP2-TEST").toString(),
-                containsString("address:        ???????? ?????,??????"));
+                containsString("address:        Тверская улица,москва"));
     }
 
     @Test
@@ -922,15 +924,15 @@ public void post_multipart_data_with_control_characters_address() {
                         "source:         TEST\n" +
                         "password: emptypassword")
                 .field("NEW", "yes");
-        RestTest.target(getPort(), "whois/syncupdates/test")
+        final String response = RestTest.target(getPort(), "whois/syncupdates/test")
                 .request()
                 .post(Entity.entity(multipart, multipart.getMediaType()), String.class);
 
+        assertThat(response, containsString("Create SUCCEEDED: [person] TP2-TEST   Test Person"));
         assertThat(databaseHelper.lookupObject(ObjectType.PERSON, "TP2-TEST").toString(),
-                containsString("address:        Test???? Address"));
+                containsString("address:        Тверская улица,москва"));
     }
 
-
     @Test
     public void post_multipart_data_with_latin1_non_ascii_address() {
         databaseHelper.addObject(PERSON_ANY1_TEST);

whois-api/src/test/java/net/ripe/db/whois/api/rest/WhoisRestServiceTestIntegration.java

@@ -3085,8 +3085,9 @@ public void create_non_latin1_characters_are_substituted() {
                 .request()
                 .post(Entity.entity(map(person), MediaType.APPLICATION_XML), WhoisResources.class);
 
+        // UTF-8 characters are NOT mapped to latin1, but left as-is
         final WhoisObject responseObject = whoisResources.getWhoisObjects().get(0);
-        assertThat(responseObject.getAttributes().get(1).getValue(), is("???????? ?????,??????"));
+        assertThat(responseObject.getAttributes().get(1).getValue(), is("test \u03A3 and \u00DF characters"));
     }
 
     @Test
@@ -3293,19 +3294,18 @@ public void update_person_non_latin1_characters_are_substituted() {
                             .put(Entity.entity(whoisObjectMapper.mapRpslObjects(FormattedClientAttributeMapper.class, update), MediaType.APPLICATION_XML),
                                     WhoisResources.class);
 
-            RestTest.assertWarningCount(response, 1);
-            RestTest.assertErrorMessage(response, 0, "Warning", "Value changed due to conversion into the ISO-8859-1 (Latin-1) character set");
+            RestTest.assertErrorCount(response, 0);
 
             final RpslObject lookupObject = databaseHelper.lookupObject(ObjectType.PERSON, "TP1-TEST");
-            assertThat(lookupObject.findAttribute(AttributeType.ADDRESS).getValue(), is("        ???????? ?????,??????"));
+            assertThat(lookupObject.findAttribute(AttributeType.ADDRESS).getValue(), is("        address: Тверская улица,москва"));
         }
         {
             final WhoisResources response =
                     RestTest.target(getPort(), "whois/test/person/TP1-TEST?password=test")
                             .request()
                             .get(WhoisResources.class);
 
-            assertThat(response.getWhoisObjects().get(0).getAttributes(), hasItem(new Attribute("address", "???????? ?????,??????")));
+            assertThat(response.getWhoisObjects().get(0).getAttributes(), hasItem(new Attribute("address", "address: Тверская улица,москва")));
         }
     }
 

whois-commons/src/main/java/net/ripe/db/whois/common/jdbc/driver/ResultSetInvocationHandler.java

@@ -52,9 +52,9 @@ private void handleNext() throws SQLException {
                 row.add("<null>");
             } else if (object instanceof Blob) {
                 final Blob blob = (Blob) object;
-                row.add(new String(blob.getBytes(0, (int) blob.length()), StandardCharsets.ISO_8859_1));
+                row.add(new String(blob.getBytes(0, (int) blob.length()), StandardCharsets.UTF_8));
             } else if (object instanceof byte[]) {
-                row.add(new String((byte[]) object, StandardCharsets.ISO_8859_1));
+                row.add(new String((byte[]) object, StandardCharsets.UTF_8));
             } else {
                 row.add(object.toString());
             }

whois-commons/src/main/java/net/ripe/db/whois/common/pipeline/ChannelUtil.java

@@ -13,8 +13,10 @@
 import java.nio.charset.StandardCharsets;
 
 public final class ChannelUtil {
+
     private static final Logger LOGGER = LoggerFactory.getLogger(ChannelUtil.class);
-    public static final Charset BYTE_ENCODING = StandardCharsets.ISO_8859_1;
+
+    public static final Charset BYTE_ENCODING = StandardCharsets.UTF_8;
 
     public static final AttributeKey<InetAddress> CLIENT_IP = AttributeKey.newInstance("client-ip");
 

whois-commons/src/main/java/net/ripe/db/whois/common/support/TelnetWhoisClient.java

@@ -23,7 +23,7 @@ public class TelnetWhoisClient {
 
     public static final int DEFAULT_PORT = 43;
     public static final String DEFAULT_HOST = "localhost";
-    public static final Charset DEFAULT_CHARSET = StandardCharsets.ISO_8859_1;
+    public static final Charset DEFAULT_CHARSET = StandardCharsets.UTF_8;
     private static final int DEFAULT_TIMEOUT = (int)TimeUnit.MINUTES.toMillis(5);
     private final String host;
     private final int port;

whois-commons/src/main/java/net/ripe/db/whois/common/x509/X509CertificateWrapper.java

@@ -46,7 +46,11 @@ public static X509CertificateWrapper parse(final RpslObject rpslObject) {
             throw new IllegalArgumentException("The supplied object has no key");
         }
 
-        return parse(RpslObjectFilter.getCertificateFromKeyCert(rpslObject).getBytes(StandardCharsets.ISO_8859_1));
+        try {
+            return parse(RpslObjectFilter.getCertificateFromKeyCert(rpslObject).getBytes(StandardCharsets.UTF_8));
+        } catch (Exception e) {
+            throw new IllegalArgumentException("Error parsing X509 certificate from key-cert object", e);
+        }
     }
 
     public static X509CertificateWrapper parse(final String certificate) {

whois-commons/src/main/resources/acl_schema.sql

@@ -7,7 +7,7 @@
 /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
 /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
 /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
-/*!40101 SET NAMES utf8 */;
+/*!40101 SET NAMES utf8mb4 */;
 /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
 /*!40103 SET TIME_ZONE='+00:00' */;
 /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
@@ -21,7 +21,7 @@
 
 DROP TABLE IF EXISTS `acl_denied`;
 /*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
+/*!40101 SET character_set_client = utf8mb4 */;
 CREATE TABLE `acl_denied` (
   `prefix` varchar(50) NOT NULL,
   `comment` text,
@@ -36,7 +36,7 @@ CREATE TABLE `acl_denied` (
 
 DROP TABLE IF EXISTS `acl_event`;
 /*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
+/*!40101 SET character_set_client = utf8mb4 */;
 CREATE TABLE `acl_event` (
   `prefix` varchar(50) NOT NULL,
   `event_time` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
@@ -79,7 +79,7 @@ CREATE TABLE `acl_sso_event` (
 
 DROP TABLE IF EXISTS `acl_limit`;
 /*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
+/*!40101 SET character_set_client = utf8mb4 */;
 CREATE TABLE `acl_limit` (
   `prefix` varchar(50) NOT NULL,
   `daily_limit` int(11) NOT NULL DEFAULT '-1',
@@ -95,7 +95,7 @@ CREATE TABLE `acl_limit` (
 
 DROP TABLE IF EXISTS `acl_proxy`;
 /*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
+/*!40101 SET character_set_client = utf8mb4 */;
 CREATE TABLE `acl_proxy` (
   `prefix` varchar(50) NOT NULL,
   `comment` text,
@@ -109,7 +109,7 @@ CREATE TABLE `acl_proxy` (
 
 DROP TABLE IF EXISTS `override_users`;
 /*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
+/*!40101 SET character_set_client = utf8mb4 */;
 CREATE TABLE `override_users` (
   `username` varchar(255) NOT NULL,
   `password` varchar(64) NOT NULL,
@@ -125,7 +125,7 @@ CREATE TABLE `override_users` (
 
 DROP TABLE IF EXISTS `apikeys`;
 /*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
+/*!40101 SET character_set_client = utf8mb4 */;
 CREATE TABLE `apikeys` (
   `apikey` varchar(128) NOT NULL,
   `uri_prefix` varchar(128) NOT NULL,
@@ -140,7 +140,7 @@ CREATE TABLE `apikeys` (
 
 DROP TABLE IF EXISTS `version`;
 /*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
+/*!40101 SET character_set_client = utf8mb4 */;
 CREATE TABLE `version` (
   `version` varchar(80) DEFAULT NULL,
    PRIMARY KEY (`version`)

whois-commons/src/main/resources/mailupdates_schema.sql

@@ -7,7 +7,7 @@
 /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
 /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
 /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
-/*!40101 SET NAMES utf8 */;
+/*!40101 SET NAMES utf8mb4 */;
 /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
 /*!40103 SET TIME_ZONE='+00:00' */;
 /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
@@ -21,7 +21,7 @@
 
 DROP TABLE IF EXISTS `mailupdates`;
 /*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
+/*!40101 SET character_set_client = utf8mb4 */;
 CREATE TABLE `mailupdates` (
   `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
   `message` longblob NOT NULL,
@@ -40,7 +40,7 @@ CREATE TABLE `mailupdates` (
 
 DROP TABLE IF EXISTS `version`;
 /*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
+/*!40101 SET character_set_client = utf8mb4 */;
 CREATE TABLE `version` (
   `version` varchar(80) DEFAULT NULL,
    PRIMARY KEY (`version`)

whois-commons/src/main/resources/nrtm_schema.sql

@@ -33,6 +33,7 @@ CREATE TABLE `source`
 ) ENGINE = InnoDB
   DEFAULT CHARSET = utf8mb4;
 
+DROP TABLE IF EXISTS `version_info`;
 CREATE TABLE `version_info`
 (
     `id`             int unsigned    NOT NULL AUTO_INCREMENT,
@@ -47,6 +48,7 @@ CREATE TABLE `version_info`
 ) ENGINE = InnoDB
   DEFAULT CHARSET = utf8mb4;
 
+DROP TABLE IF EXISTS `snapshot_file`;
 CREATE TABLE `snapshot_file`
 (
     `id`         int unsigned NOT NULL AUTO_INCREMENT,
@@ -61,6 +63,7 @@ CREATE TABLE `snapshot_file`
 ) ENGINE = InnoDB
   DEFAULT CHARSET = utf8mb4;
 
+DROP TABLE IF EXISTS `delta_file`;
 CREATE TABLE `delta_file`
 (
     `id`         int unsigned NOT NULL AUTO_INCREMENT,
@@ -75,6 +78,7 @@ CREATE TABLE `delta_file`
 ) ENGINE = InnoDB
   DEFAULT CHARSET = utf8mb4;
 
+DROP TABLE IF EXISTS `notification_file`;
 CREATE TABLE `notification_file`
 (
     `id`         int unsigned    NOT NULL AUTO_INCREMENT,
@@ -86,7 +90,7 @@ CREATE TABLE `notification_file`
 ) ENGINE = InnoDB
   DEFAULT CHARSET = utf8mb4;
 
-
+DROP TABLE IF EXISTS `key_pair`;
 CREATE TABLE `key_pair`
 (
     `id`          int unsigned    NOT NULL AUTO_INCREMENT,
@@ -101,6 +105,13 @@ CREATE TABLE `key_pair`
 ) ENGINE = InnoDB
   DEFAULT CHARSET = utf8mb4;
 
+DROP TABLE IF EXISTS `version`;
+CREATE TABLE `version`
+(
+    `version` varchar(80)
+) ENGINE = InnoDB
+  DEFAULT CHARSET = utf8mb4;
+
 /*!40103 SET TIME_ZONE = @OLD_TIME_ZONE */;
 
 /*!40101 SET SQL_MODE = @OLD_SQL_MODE */;