From ed7ccfa5f629d507ccdb12f7f526018429846546 Mon Sep 17 00:00:00 2001 From: Ed Shryane Date: Fri, 23 Aug 2013 17:32:11 +0200 Subject: [PATCH 01/22] Parse UTF-8 characters in RPSL objects. --- .../whois/common/rpsl/RpslObjectBuilder.java | 7 ++- .../db/whois/common/rpsl/RpslObjectTest.java | 44 ++++++++++++++++++- 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/whois-commons/src/main/java/net/ripe/db/whois/common/rpsl/RpslObjectBuilder.java b/whois-commons/src/main/java/net/ripe/db/whois/common/rpsl/RpslObjectBuilder.java index fceeddcddc..a247b68bd0 100644 --- a/whois-commons/src/main/java/net/ripe/db/whois/common/rpsl/RpslObjectBuilder.java +++ b/whois-commons/src/main/java/net/ripe/db/whois/common/rpsl/RpslObjectBuilder.java @@ -39,9 +39,8 @@ public List getAttributes() { return attributes; } - // Note: we use ISO_8859_1 encoding everywhere as it is the only one that maps directly from byte to char (as in, it effectively is a '(char)byteValue') static List getAttributes(String input) { - return getAttributes(input.getBytes(Charsets.ISO_8859_1)); + return getAttributes(input.getBytes(Charsets.UTF_8)); } static List getAttributes(byte[] buf) { @@ -70,7 +69,7 @@ static List getAttributes(byte[] buf) { if (!readKey) throw new IllegalArgumentException("No key found"); if (start == pos) throw new IllegalArgumentException("Read zero sized key"); - final String key = new String(buf, start, pos - start, Charsets.ISO_8859_1); + final String key = new String(buf, start, pos - start, Charsets.UTF_8); // skip over ':' and continue reading the attribute value start = ++pos; @@ -100,7 +99,7 @@ static List getAttributes(byte[] buf) { stop = pos; } - final String value = new String(buf, start, stop - start, Charsets.ISO_8859_1); + final String value = new String(buf, start, stop - start, Charsets.UTF_8); newAttributes.add(new RpslAttribute(key, value)); } diff --git a/whois-commons/src/test/java/net/ripe/db/whois/common/rpsl/RpslObjectTest.java b/whois-commons/src/test/java/net/ripe/db/whois/common/rpsl/RpslObjectTest.java index e8f04ec471..1dfe87ecc4 100644 --- a/whois-commons/src/test/java/net/ripe/db/whois/common/rpsl/RpslObjectTest.java +++ b/whois-commons/src/test/java/net/ripe/db/whois/common/rpsl/RpslObjectTest.java @@ -155,7 +155,7 @@ public void parseMultipleIdenticalKeys() { } private byte[] bytesFrom(String input) { - return input.getBytes(Charsets.ISO_8859_1); + return input.getBytes(Charsets.UTF_8); } private void parseAndAssign(String input) { @@ -776,6 +776,48 @@ public void diff_separate_changes() { " source: TEST\n")); } + @Test + public void parse_greek_attribute_value() { + final RpslObject rpslObject = RpslObject.parse( + "person: Test Person\n" + + "address: Καλημέρα κόσμε\n" + + "phone: +31 20 123456\n" + + "nic-hdl: AUTO-1\n" + + "mnt-by: TEST-DBM-MNT\n" + + "changed: hostmaster@ripe.net\n" + + "source: TEST\n"); + + assertThat(rpslObject.findAttribute(AttributeType.ADDRESS).getValue(), containsString("Καλημέρα κόσμε")); + } + + @Test + public void parse_japanese_attribute_value() { + final RpslObject rpslObject = RpslObject.parse( + "person: Test Person\n" + + "address: こんにちは 世界\n" + + "phone: +31 20 123456\n" + + "nic-hdl: AUTO-1\n" + + "mnt-by: TEST-DBM-MNT\n" + + "changed: hostmaster@ripe.net\n" + + "source: TEST\n"); + + assertThat(rpslObject.findAttribute(AttributeType.ADDRESS).getValue(), containsString("こんにちは 世界")); + } + + @Test + public void parse_arabic_attribute_value() { + final RpslObject rpslObject = RpslObject.parse( + "person: Test Person\n" + + "address: حد بيتكلم انجليزي؟\n" + + "phone: +31 20 123456\n" + + "nic-hdl: AUTO-1\n" + + "mnt-by: TEST-DBM-MNT\n" + + "changed: hostmaster@ripe.net\n" + + "source: TEST\n"); + + assertThat(rpslObject.findAttribute(AttributeType.ADDRESS).getValue(), containsString("حد بيتكلم انجلي")); + } + private static Iterable convertToString(final Iterable c) { return Iterables.transform(c, new Function() { @Nullable From 46b785aff697a9624483973239f759cbc576b78d Mon Sep 17 00:00:00 2001 From: Ed Shryane Date: Fri, 23 Aug 2013 18:13:22 +0200 Subject: [PATCH 02/22] Switch to UTF-8 within the application (including the database). --- .../whois/api/mail/dequeue/MessageParser.java | 2 +- .../whois/api/whois/SyncUpdatesService.java | 2 +- .../whois/api/wsearch/LogSearchService.java | 2 +- .../api/mail/dequeue/MessageParserTest.java | 2 +- .../driver/ResultSetInvocationHandler.java | 4 +- .../db/whois/common/pipeline/ChannelUtil.java | 2 +- .../db/whois/common/rpsl/RpslAttribute.java | 4 +- .../ripe/db/whois/common/rpsl/RpslObject.java | 2 +- .../src/main/resources/acl_schema.sql | 16 +- .../src/main/resources/mailupdates_schema.sql | 4 +- .../src/main/resources/pending_schema.sql | 4 +- .../src/main/resources/scheduler_schema.sql | 4 +- .../src/main/resources/whois_schema.sql | 140 +++++++++--------- .../whois/common/dao/jdbc/DatabaseHelper.java | 23 ++- .../java/net/ripe/db/whois/WhoisFixture.java | 2 +- .../whois/query/planner/RpslAttributes.java | 4 +- .../update/keycert/PgpPublicKeyWrapper.java | 2 +- .../keycert/X509CertificateWrapper.java | 2 +- 18 files changed, 118 insertions(+), 103 deletions(-) diff --git a/whois-api/src/main/java/net/ripe/db/whois/api/mail/dequeue/MessageParser.java b/whois-api/src/main/java/net/ripe/db/whois/api/mail/dequeue/MessageParser.java index 8fcec86761..85dc7843cb 100644 --- a/whois-api/src/main/java/net/ripe/db/whois/api/mail/dequeue/MessageParser.java +++ b/whois-api/src/main/java/net/ripe/db/whois/api/mail/dequeue/MessageParser.java @@ -219,7 +219,7 @@ Charset getCharset(final ContentType contentType) { } } - return Charsets.ISO_8859_1; + return Charsets.UTF_8; } String getHeaders(final Part part) throws MessagingException { diff --git a/whois-api/src/main/java/net/ripe/db/whois/api/whois/SyncUpdatesService.java b/whois-api/src/main/java/net/ripe/db/whois/api/whois/SyncUpdatesService.java index 688a42d5f8..22e56c16c6 100644 --- a/whois-api/src/main/java/net/ripe/db/whois/api/whois/SyncUpdatesService.java +++ b/whois-api/src/main/java/net/ripe/db/whois/api/whois/SyncUpdatesService.java @@ -213,7 +213,7 @@ private Charset getCharset(final String contentType) { } } } - return Charsets.ISO_8859_1; + return Charsets.UTF_8; } @Nullable diff --git a/whois-api/src/main/java/net/ripe/db/whois/api/wsearch/LogSearchService.java b/whois-api/src/main/java/net/ripe/db/whois/api/wsearch/LogSearchService.java index 8ce6063f13..33e68307f5 100644 --- a/whois-api/src/main/java/net/ripe/db/whois/api/wsearch/LogSearchService.java +++ b/whois-api/src/main/java/net/ripe/db/whois/api/wsearch/LogSearchService.java @@ -143,7 +143,7 @@ public List getUpdateIds( final String url = String.format("http://%s:%s/api/logs/current?search=%s&date=%s&apiKey=%s", clusterMember.getHostName(), jettyConfig.getPort(Audience.INTERNAL), - URLEncoder.encode(search, "ISO-8859-1"), + URLEncoder.encode(search, "UTF-8"), date, apiKey); final Future> future = client.asyncResource(url) .accept(MediaType.APPLICATION_JSON_TYPE) diff --git a/whois-api/src/test/java/net/ripe/db/whois/api/mail/dequeue/MessageParserTest.java b/whois-api/src/test/java/net/ripe/db/whois/api/mail/dequeue/MessageParserTest.java index 3401d731a2..b8e5b755b0 100644 --- a/whois-api/src/test/java/net/ripe/db/whois/api/mail/dequeue/MessageParserTest.java +++ b/whois-api/src/test/java/net/ripe/db/whois/api/mail/dequeue/MessageParserTest.java @@ -761,7 +761,7 @@ public void parse_smime_multipart_alternative() throws Exception { @Test public void illegal_charset() throws Exception { - assertThat(subject.getCharset(new ContentType("text/plain;\n\tcharset=\"_iso-2022-jp$ESC\"")), is(Charsets.ISO_8859_1)); + assertThat(subject.getCharset(new ContentType("text/plain;\n\tcharset=\"_iso-2022-jp$ESC\"")), is(Charsets.UTF_8)); } private MimeMessage getMessage(final String message) throws MessagingException, IOException { diff --git a/whois-commons/src/main/java/net/ripe/db/whois/common/jdbc/driver/ResultSetInvocationHandler.java b/whois-commons/src/main/java/net/ripe/db/whois/common/jdbc/driver/ResultSetInvocationHandler.java index bb54066133..1a810defd2 100644 --- a/whois-commons/src/main/java/net/ripe/db/whois/common/jdbc/driver/ResultSetInvocationHandler.java +++ b/whois-commons/src/main/java/net/ripe/db/whois/common/jdbc/driver/ResultSetInvocationHandler.java @@ -52,9 +52,9 @@ private void handleNext() throws SQLException { row.add(""); } else if (object instanceof Blob) { final Blob blob = (Blob) object; - row.add(new String(blob.getBytes(0, (int) blob.length()), Charsets.ISO_8859_1)); + row.add(new String(blob.getBytes(0, (int) blob.length()), Charsets.UTF_8)); } else if (object instanceof byte[]) { - row.add(new String((byte[]) object, Charsets.ISO_8859_1)); + row.add(new String((byte[]) object, Charsets.UTF_8)); } else { row.add(object.toString()); } diff --git a/whois-commons/src/main/java/net/ripe/db/whois/common/pipeline/ChannelUtil.java b/whois-commons/src/main/java/net/ripe/db/whois/common/pipeline/ChannelUtil.java index 9384aa4e45..cdef1fb9a4 100644 --- a/whois-commons/src/main/java/net/ripe/db/whois/common/pipeline/ChannelUtil.java +++ b/whois-commons/src/main/java/net/ripe/db/whois/common/pipeline/ChannelUtil.java @@ -10,7 +10,7 @@ import java.nio.charset.Charset; public final class ChannelUtil { - public static final Charset BYTE_ENCODING = Charsets.ISO_8859_1; + public static final Charset BYTE_ENCODING = Charsets.UTF_8; private ChannelUtil() { } diff --git a/whois-commons/src/main/java/net/ripe/db/whois/common/rpsl/RpslAttribute.java b/whois-commons/src/main/java/net/ripe/db/whois/common/rpsl/RpslAttribute.java index 40e55e1e9b..c6b7a2772f 100644 --- a/whois-commons/src/main/java/net/ripe/db/whois/common/rpsl/RpslAttribute.java +++ b/whois-commons/src/main/java/net/ripe/db/whois/common/rpsl/RpslAttribute.java @@ -38,8 +38,8 @@ public final class RpslAttribute { Validate.notNull(key); Validate.notNull(value); - this.key = new String(key, Charsets.ISO_8859_1).toLowerCase(); - this.value = new String(value, Charsets.ISO_8859_1); + this.key = new String(key, Charsets.UTF_8).toLowerCase(); + this.value = new String(value, Charsets.UTF_8); this.type = AttributeType.getByNameOrNull(this.key); } diff --git a/whois-commons/src/main/java/net/ripe/db/whois/common/rpsl/RpslObject.java b/whois-commons/src/main/java/net/ripe/db/whois/common/rpsl/RpslObject.java index ea080310cf..d629138233 100644 --- a/whois-commons/src/main/java/net/ripe/db/whois/common/rpsl/RpslObject.java +++ b/whois-commons/src/main/java/net/ripe/db/whois/common/rpsl/RpslObject.java @@ -223,7 +223,7 @@ public boolean containsAttribute(final AttributeType attributeType) { @Override public void writeTo(final OutputStream out) throws IOException { - writeTo(new OutputStreamWriter(out, Charsets.ISO_8859_1)); + writeTo(new OutputStreamWriter(out, Charsets.UTF_8)); } public void writeTo(final Writer writer) throws IOException { diff --git a/whois-commons/src/main/resources/acl_schema.sql b/whois-commons/src/main/resources/acl_schema.sql index b3ebb8e016..28264f7925 100644 --- a/whois-commons/src/main/resources/acl_schema.sql +++ b/whois-commons/src/main/resources/acl_schema.sql @@ -27,7 +27,7 @@ CREATE TABLE `acl_denied` ( `comment` text, `denied_date` date NOT NULL DEFAULT '0000-00-00', PRIMARY KEY (`prefix`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -43,7 +43,7 @@ CREATE TABLE `acl_event` ( `daily_limit` int(11) NOT NULL, `event_type` varchar(20) NOT NULL, PRIMARY KEY (`prefix`,`event_time`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -59,7 +59,7 @@ CREATE TABLE `acl_limit` ( `comment` text, `unlimited_connections` bit(1) NOT NULL DEFAULT b'0', PRIMARY KEY (`prefix`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -73,7 +73,7 @@ CREATE TABLE `acl_mirror` ( `prefix` varchar(50) NOT NULL, `comment` text, PRIMARY KEY (`prefix`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -89,7 +89,7 @@ CREATE TABLE `acl_mirror_delete_log` ( `delete_comment` text, `date` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`prefix`,`date`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -103,7 +103,7 @@ CREATE TABLE `acl_proxy` ( `prefix` varchar(50) NOT NULL, `comment` text, PRIMARY KEY (`prefix`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -119,7 +119,7 @@ CREATE TABLE `override_users` ( `objecttypes` varchar(255) NOT NULL, `last_changed` date DEFAULT NULL, UNIQUE KEY `override_username_idx` (`username`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -131,7 +131,7 @@ DROP TABLE IF EXISTS `version`; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `version` ( `version` varchar(80) DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; diff --git a/whois-commons/src/main/resources/mailupdates_schema.sql b/whois-commons/src/main/resources/mailupdates_schema.sql index e888031a77..6c65835027 100644 --- a/whois-commons/src/main/resources/mailupdates_schema.sql +++ b/whois-commons/src/main/resources/mailupdates_schema.sql @@ -31,7 +31,7 @@ CREATE TABLE `mailupdates` ( `changed` int(10) unsigned DEFAULT NULL, PRIMARY KEY (`id`), UNIQUE KEY `mailupdates_uuid_idx` (`claim_uuid`) -) ENGINE=InnoDB AUTO_INCREMENT=467880 DEFAULT CHARSET=latin1; +) ENGINE=InnoDB AUTO_INCREMENT=467880 DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -43,7 +43,7 @@ DROP TABLE IF EXISTS `version`; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `version` ( `version` varchar(80) DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; diff --git a/whois-commons/src/main/resources/pending_schema.sql b/whois-commons/src/main/resources/pending_schema.sql index 6596454dfd..7d778236f6 100644 --- a/whois-commons/src/main/resources/pending_schema.sql +++ b/whois-commons/src/main/resources/pending_schema.sql @@ -7,9 +7,9 @@ create table pending_updates ( `passed_authentications` VARCHAR(100) NOT NULL, `object` longblob NOT NULL, PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; DROP TABLE IF EXISTS `version`; CREATE TABLE `version` ( `version` varchar(80) DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/whois-commons/src/main/resources/scheduler_schema.sql b/whois-commons/src/main/resources/scheduler_schema.sql index 4194bac3a4..184d2e6c3f 100644 --- a/whois-commons/src/main/resources/scheduler_schema.sql +++ b/whois-commons/src/main/resources/scheduler_schema.sql @@ -26,7 +26,7 @@ CREATE TABLE `scheduler` ( `date` date NOT NULL, `host` varchar(50) NOT NULL, PRIMARY KEY (`date`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -38,7 +38,7 @@ DROP TABLE IF EXISTS `version`; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `version` ( `version` varchar(80) DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; diff --git a/whois-commons/src/main/resources/whois_schema.sql b/whois-commons/src/main/resources/whois_schema.sql index 1db7fb646a..1e918109a1 100644 --- a/whois-commons/src/main/resources/whois_schema.sql +++ b/whois-commons/src/main/resources/whois_schema.sql @@ -29,7 +29,7 @@ CREATE TABLE `abuse_c` ( PRIMARY KEY (`pe_ro_id`,`object_id`), KEY `object_type` (`object_type`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -46,7 +46,7 @@ CREATE TABLE `abuse_mailbox` ( `object_type` tinyint(3) unsigned NOT NULL DEFAULT '0', PRIMARY KEY (`abuse_mailbox`,`object_id`), KEY `abuse_mailbox_object_id_index` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -64,7 +64,7 @@ CREATE TABLE `admin_c` ( PRIMARY KEY (`pe_ro_id`,`object_id`), KEY `object_type` (`object_type`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -83,7 +83,7 @@ CREATE TABLE `as_block` ( KEY `thread_id` (`thread_id`), KEY `begin_as` (`begin_as`), KEY `end_as` (`end_as`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -101,7 +101,7 @@ CREATE TABLE `as_set` ( PRIMARY KEY (`object_id`), KEY `as_set` (`as_set`), KEY `thread_id` (`thread_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -118,7 +118,7 @@ CREATE TABLE `aut_num` ( PRIMARY KEY (`object_id`), KEY `aut_num` (`aut_num`), KEY `thread_id` (`thread_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -131,11 +131,11 @@ DROP TABLE IF EXISTS `auth`; CREATE TABLE `auth` ( `thread_id` int(11) NOT NULL DEFAULT '0', `object_id` int(10) unsigned NOT NULL DEFAULT '0', - `auth` varchar(90) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL DEFAULT '', + `auth` varchar(90) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT '', `object_type` tinyint(3) unsigned NOT NULL DEFAULT '0', PRIMARY KEY (`auth`,`object_id`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -151,7 +151,7 @@ CREATE TABLE `auth_override` ( `date` int(10) unsigned NOT NULL DEFAULT '0', PRIMARY KEY (`date`,`object_id`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -169,7 +169,7 @@ CREATE TABLE `author` ( PRIMARY KEY (`pe_ro_id`,`object_id`), KEY `object_id` (`object_id`), KEY `object_type` (`object_type`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -186,7 +186,7 @@ CREATE TABLE `domain` ( PRIMARY KEY (`object_id`), KEY `domain` (`domain`), KEY `thread_id` (`thread_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -201,7 +201,7 @@ CREATE TABLE `ds_rdata` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `ds_rdata` varchar(128) NOT NULL DEFAULT '', PRIMARY KEY (`ds_rdata`,`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -218,7 +218,7 @@ CREATE TABLE `e_mail` ( `object_type` tinyint(3) unsigned NOT NULL DEFAULT '0', PRIMARY KEY (`e_mail`,`object_id`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -235,7 +235,7 @@ CREATE TABLE `filter_set` ( PRIMARY KEY (`object_id`), KEY `filter_set` (`filter_set`), KEY `thread_id` (`thread_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -251,7 +251,7 @@ CREATE TABLE `fingerpr` ( `fingerpr` varchar(80) NOT NULL DEFAULT '', PRIMARY KEY (`fingerpr`,`object_id`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -269,7 +269,7 @@ CREATE TABLE `form` ( PRIMARY KEY (`form_id`,`object_id`), KEY `object_id` (`object_id`), KEY `object_type` (`object_type`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -292,7 +292,7 @@ CREATE TABLE `history` ( PRIMARY KEY (`object_id`,`sequence_id`), KEY `history_pkey` (`pkey`), KEY `history_timestamp` (`timestamp`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -308,7 +308,7 @@ CREATE TABLE `ifaddr` ( `ifaddr` int(10) unsigned NOT NULL DEFAULT '0', PRIMARY KEY (`ifaddr`,`object_id`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -324,7 +324,7 @@ CREATE TABLE `inaddr_arpa` ( `begin_in` int(10) unsigned NOT NULL DEFAULT '0', `end_in` int(10) unsigned NOT NULL DEFAULT '0', PRIMARY KEY (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -346,7 +346,7 @@ CREATE TABLE `inet6num` ( KEY `i6_msb` (`i6_msb`), KEY `i6_lsb` (`i6_lsb`), KEY `thread_id` (`thread_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -365,7 +365,7 @@ CREATE TABLE `inet_rtr` ( KEY `inet_rtr` (`inet_rtr`), KEY `local_as` (`local_as`), KEY `thread_id` (`thread_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -386,7 +386,7 @@ CREATE TABLE `inetnum` ( KEY `begin_in` (`begin_in`), KEY `end_in` (`end_in`), KEY `thread_id` (`thread_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -404,7 +404,7 @@ CREATE TABLE `interface` ( `interface_v4` int(10) unsigned NOT NULL DEFAULT '0', PRIMARY KEY (`interface_v6_msp`,`interface_v6_lsp`,`interface_v4`,`object_id`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -421,7 +421,7 @@ CREATE TABLE `ip6int` ( `lsb` varchar(20) NOT NULL DEFAULT '', `prefix_length` tinyint(3) unsigned NOT NULL DEFAULT '0', PRIMARY KEY (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -438,7 +438,7 @@ CREATE TABLE `irt` ( `dummy` tinyint(4) NOT NULL DEFAULT '0', PRIMARY KEY (`object_id`), KEY `irt` (`irt`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -454,7 +454,7 @@ CREATE TABLE `irt_nfy` ( `irt_nfy` varchar(80) NOT NULL DEFAULT '', PRIMARY KEY (`irt_nfy`,`object_id`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -471,7 +471,7 @@ CREATE TABLE `key_cert` ( PRIMARY KEY (`object_id`), KEY `key_cert` (`key_cert`), KEY `thread_id` (`thread_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -494,7 +494,7 @@ CREATE TABLE `last` ( PRIMARY KEY (`object_id`,`sequence_id`), KEY `last_pkey` (`pkey`), KEY `object_type_index` (`object_type`) -) ENGINE=InnoDB AUTO_INCREMENT=12650254 DEFAULT CHARSET=latin1; +) ENGINE=InnoDB AUTO_INCREMENT=12650254 DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -511,7 +511,7 @@ CREATE TABLE `limerick` ( PRIMARY KEY (`object_id`), KEY `limerick` (`limerick`), KEY `thread_id` (`thread_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -529,7 +529,7 @@ CREATE TABLE `mbrs_by_ref` ( PRIMARY KEY (`mnt_id`,`object_id`), KEY `object_id` (`object_id`), KEY `object_type` (`object_type`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -547,7 +547,7 @@ CREATE TABLE `member_of` ( PRIMARY KEY (`set_id`,`object_id`), KEY `object_id` (`object_id`), KEY `object_type` (`object_type`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -565,7 +565,7 @@ CREATE TABLE `mnt_by` ( PRIMARY KEY (`mnt_id`,`object_id`), KEY `object_id` (`object_id`), KEY `object_type` (`object_type`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -582,7 +582,7 @@ CREATE TABLE `mnt_domains` ( `object_type` tinyint(3) unsigned NOT NULL DEFAULT '0', PRIMARY KEY (`mnt_id`,`object_id`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -600,7 +600,7 @@ CREATE TABLE `mnt_irt` ( PRIMARY KEY (`irt_id`,`object_id`), KEY `object_id` (`object_id`), KEY `object_type` (`object_type`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -618,7 +618,7 @@ CREATE TABLE `mnt_lower` ( PRIMARY KEY (`mnt_id`,`object_id`), KEY `object_id` (`object_id`), KEY `object_type` (`object_type`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -634,7 +634,7 @@ CREATE TABLE `mnt_nfy` ( `mnt_nfy` varchar(80) NOT NULL DEFAULT '', PRIMARY KEY (`mnt_nfy`,`object_id`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -651,7 +651,7 @@ CREATE TABLE `mnt_ref` ( `object_type` tinyint(3) unsigned NOT NULL DEFAULT '0', PRIMARY KEY (`mnt_id`,`object_id`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -669,7 +669,7 @@ CREATE TABLE `mnt_routes` ( PRIMARY KEY (`mnt_id`,`object_id`), KEY `object_id` (`object_id`), KEY `object_type` (`object_type`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -687,7 +687,7 @@ CREATE TABLE `mnt_routes6` ( PRIMARY KEY (`mnt_id`,`object_id`), KEY `object_id` (`object_id`), KEY `object_type` (`object_type`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -705,7 +705,7 @@ CREATE TABLE `mntner` ( PRIMARY KEY (`object_id`), KEY `mntner` (`mntner`), KEY `thread_id` (`thread_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -722,7 +722,7 @@ CREATE TABLE `names` ( `object_type` tinyint(3) unsigned NOT NULL DEFAULT '0', PRIMARY KEY (`name`,`object_id`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -744,7 +744,7 @@ CREATE TABLE `nic_hdl` ( KEY `range_end` (`range_end`), KEY `space` (`space`,`source`), KEY `thread_id` (`thread_id`) -) ENGINE=InnoDB AUTO_INCREMENT=1924502 DEFAULT CHARSET=latin1; +) ENGINE=InnoDB AUTO_INCREMENT=1924502 DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -761,7 +761,7 @@ CREATE TABLE `notify` ( `object_type` tinyint(3) unsigned NOT NULL DEFAULT '0', PRIMARY KEY (`notify`,`object_id`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -777,7 +777,7 @@ CREATE TABLE `nserver` ( `host` varchar(254) NOT NULL DEFAULT '', PRIMARY KEY (`host`,`object_id`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -794,7 +794,7 @@ CREATE TABLE `org` ( `object_type` tinyint(3) unsigned NOT NULL DEFAULT '0', PRIMARY KEY (`org_id`,`object_id`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -810,7 +810,7 @@ CREATE TABLE `org_name` ( `name` varchar(64) NOT NULL DEFAULT '', PRIMARY KEY (`name`,`object_id`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -828,7 +828,7 @@ CREATE TABLE `organisation` ( PRIMARY KEY (`organisation`,`object_id`), KEY `object_id` (`object_id`), KEY `thread_id` (`thread_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -848,7 +848,7 @@ CREATE TABLE `organisation_id` ( KEY `range_end` (`range_end`), KEY `space` (`space`,`source`), KEY `thread_id` (`thread_id`) -) ENGINE=InnoDB AUTO_INCREMENT=19153 DEFAULT CHARSET=latin1; +) ENGINE=InnoDB AUTO_INCREMENT=19153 DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -865,7 +865,7 @@ CREATE TABLE `peering_set` ( PRIMARY KEY (`object_id`), KEY `peering_set` (`peering_set`), KEY `thread_id` (`thread_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -885,7 +885,7 @@ CREATE TABLE `person_role` ( KEY `nic_hdl` (`nic_hdl`), KEY `object_type` (`object_type`), KEY `thread_id` (`thread_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -903,7 +903,7 @@ CREATE TABLE `ping_hdl` ( PRIMARY KEY (`pe_ro_id`,`object_id`), KEY `object_type` (`object_type`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -920,7 +920,7 @@ CREATE TABLE `poem` ( PRIMARY KEY (`object_id`), KEY `poem` (`poem`), KEY `thread_id` (`thread_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -939,7 +939,7 @@ CREATE TABLE `poetic_form` ( KEY `poetic_form` (`poetic_form`), KEY `thread_id` (`thread_id`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -955,7 +955,7 @@ CREATE TABLE `ref_nfy` ( `ref_nfy` varchar(80) NOT NULL DEFAULT '', PRIMARY KEY (`ref_nfy`,`object_id`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -972,7 +972,7 @@ CREATE TABLE `refer` ( `port` int(5) unsigned NOT NULL DEFAULT '43', `host` varchar(80) NOT NULL DEFAULT '', PRIMARY KEY (`object_id`,`host`,`port`,`type`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -990,7 +990,7 @@ CREATE TABLE `referral_by` ( PRIMARY KEY (`mnt_id`,`object_id`), KEY `object_id` (`object_id`), KEY `object_type` (`object_type`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -1007,7 +1007,7 @@ CREATE TABLE `rev_srv` ( `object_type` tinyint(3) unsigned NOT NULL DEFAULT '0', PRIMARY KEY (`rev_srv`,`object_id`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -1026,7 +1026,7 @@ CREATE TABLE `route` ( `dummy` tinyint(4) NOT NULL DEFAULT '0', PRIMARY KEY (`object_id`), KEY `origin` (`origin`,`prefix`,`prefix_length`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -1048,7 +1048,7 @@ CREATE TABLE `route6` ( KEY `origin` (`origin`,`r6_msb`,`r6_lsb`,`prefix_length`), KEY `r6_msb` (`r6_msb`), KEY `r6_lsb` (`r6_lsb`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -1066,7 +1066,7 @@ CREATE TABLE `route_set` ( PRIMARY KEY (`object_id`), KEY `route_set` (`route_set`), KEY `thread_id` (`thread_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -1084,7 +1084,7 @@ CREATE TABLE `rtr_set` ( PRIMARY KEY (`object_id`), KEY `rtr_set` (`rtr_set`), KEY `thread_id` (`thread_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -1104,7 +1104,7 @@ CREATE TABLE `serials` ( PRIMARY KEY (`serial_id`), KEY `object` (`object_id`,`sequence_id`), KEY `thread_id` (`thread_id`) -) ENGINE=InnoDB AUTO_INCREMENT=25050923 DEFAULT CHARSET=latin1; +) ENGINE=InnoDB AUTO_INCREMENT=25050923 DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -1120,7 +1120,7 @@ CREATE TABLE `sub_dom` ( `domain` varchar(254) NOT NULL DEFAULT '', PRIMARY KEY (`domain`,`object_id`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -1138,7 +1138,7 @@ CREATE TABLE `tech_c` ( PRIMARY KEY (`pe_ro_id`,`object_id`), KEY `object_type` (`object_type`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -1154,7 +1154,7 @@ CREATE TABLE `upd_to` ( `upd_to` varchar(80) NOT NULL DEFAULT '', PRIMARY KEY (`upd_to`,`object_id`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -1167,7 +1167,7 @@ DROP TABLE IF EXISTS `update_lock`; CREATE TABLE `update_lock` ( `global_lock` int(11) NOT NULL, PRIMARY KEY (`global_lock`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -1179,7 +1179,7 @@ DROP TABLE IF EXISTS `version`; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `version` ( `version` varchar(80) DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -1193,7 +1193,7 @@ CREATE TABLE `x509` ( `thread_id` int(11) NOT NULL DEFAULT '0', `keycert_id` int(10) unsigned NOT NULL DEFAULT '0', PRIMARY KEY (`keycert_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -1211,7 +1211,7 @@ CREATE TABLE `zone_c` ( PRIMARY KEY (`pe_ro_id`,`object_id`), KEY `object_type` (`object_type`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; @@ -1232,5 +1232,5 @@ CREATE TABLE `tags` ( `tag_id` varchar(50) NOT NULL DEFAULT '', `data` varchar(255) DEFAULT NULL, PRIMARY KEY (`object_id`, `tag_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/whois-commons/src/test/java/net/ripe/db/whois/common/dao/jdbc/DatabaseHelper.java b/whois-commons/src/test/java/net/ripe/db/whois/common/dao/jdbc/DatabaseHelper.java index 2a1bad7e79..336ad41f35 100644 --- a/whois-commons/src/test/java/net/ripe/db/whois/common/dao/jdbc/DatabaseHelper.java +++ b/whois-commons/src/test/java/net/ripe/db/whois/common/dao/jdbc/DatabaseHelper.java @@ -14,7 +14,12 @@ import net.ripe.db.whois.common.domain.BlockEvent; import net.ripe.db.whois.common.domain.User; import net.ripe.db.whois.common.jdbc.driver.LoggingDriver; -import net.ripe.db.whois.common.rpsl.*; +import net.ripe.db.whois.common.rpsl.AttributeSanitizer; +import net.ripe.db.whois.common.rpsl.ObjectMessages; +import net.ripe.db.whois.common.rpsl.ObjectType; +import net.ripe.db.whois.common.rpsl.RpslAttribute; +import net.ripe.db.whois.common.rpsl.RpslObject; +import net.ripe.db.whois.common.rpsl.RpslObjectFilter; import net.ripe.db.whois.common.source.IllegalSourceException; import net.ripe.db.whois.common.source.Source; import net.ripe.db.whois.common.source.SourceAwareDataSource; @@ -38,9 +43,19 @@ import org.springframework.util.StringValueResolver; import javax.sql.DataSource; -import java.sql.*; -import java.util.*; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Collection; +import java.util.Collections; import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -179,7 +194,7 @@ public static void addGrsDatabases(final String... sourceNames) { static void setupDatabase(final JdbcTemplate jdbcTemplate, final String propertyBase, final String name, final String... sql) { final String dbName = dbBaseName + "_" + name; - jdbcTemplate.execute("CREATE DATABASE " + dbName); + jdbcTemplate.execute("CREATE DATABASE " + dbName + " DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci"); loadScripts(new JdbcTemplate(createDataSource(dbName)), sql); diff --git a/whois-db/src/test/java/net/ripe/db/whois/WhoisFixture.java b/whois-db/src/test/java/net/ripe/db/whois/WhoisFixture.java index cb024c0766..3881d7baef 100644 --- a/whois-db/src/test/java/net/ripe/db/whois/WhoisFixture.java +++ b/whois-db/src/test/java/net/ripe/db/whois/WhoisFixture.java @@ -83,7 +83,7 @@ public class WhoisFixture { private static final String SYNCUPDATES_INSTANCE = "TEST"; - private static final String CHARSET = "ISO-8859-1"; + private static final String CHARSET = "UTF-8"; static { Slf4JLogConfiguration.init(); diff --git a/whois-query/src/main/java/net/ripe/db/whois/query/planner/RpslAttributes.java b/whois-query/src/main/java/net/ripe/db/whois/query/planner/RpslAttributes.java index 8a2328271c..470b7a49d1 100644 --- a/whois-query/src/main/java/net/ripe/db/whois/query/planner/RpslAttributes.java +++ b/whois-query/src/main/java/net/ripe/db/whois/query/planner/RpslAttributes.java @@ -18,7 +18,7 @@ public RpslAttributes(final Iterable attributes) { @Override public void writeTo(final OutputStream out) throws IOException { - final OutputStreamWriter writer = new OutputStreamWriter(out, Charsets.ISO_8859_1); + final OutputStreamWriter writer = new OutputStreamWriter(out, Charsets.UTF_8); for (final RpslAttribute attribute : attributes) { attribute.writeTo(writer); @@ -40,6 +40,6 @@ public byte[] toByteArray() { @Override public String toString() { - return new String(toByteArray(), Charsets.ISO_8859_1); + return new String(toByteArray(), Charsets.UTF_8); } } diff --git a/whois-update/src/main/java/net/ripe/db/whois/update/keycert/PgpPublicKeyWrapper.java b/whois-update/src/main/java/net/ripe/db/whois/update/keycert/PgpPublicKeyWrapper.java index 6ee6ad0961..47ff7c7af3 100644 --- a/whois-update/src/main/java/net/ripe/db/whois/update/keycert/PgpPublicKeyWrapper.java +++ b/whois-update/src/main/java/net/ripe/db/whois/update/keycert/PgpPublicKeyWrapper.java @@ -42,7 +42,7 @@ public static PgpPublicKeyWrapper parse(final RpslObject object) { } try { - final byte[] bytes = new RpslObjectFilter(object).getCertificateFromKeyCert().getBytes(Charsets.ISO_8859_1); + final byte[] bytes = new RpslObjectFilter(object).getCertificateFromKeyCert().getBytes(Charsets.UTF_8); final ArmoredInputStream armoredInputStream = (ArmoredInputStream) PGPUtil.getDecoderStream(new ByteArrayInputStream(bytes)); PGPPublicKey masterKey = null; List subKeys = Lists.newArrayList(); diff --git a/whois-update/src/main/java/net/ripe/db/whois/update/keycert/X509CertificateWrapper.java b/whois-update/src/main/java/net/ripe/db/whois/update/keycert/X509CertificateWrapper.java index 1187148924..1600b0602e 100644 --- a/whois-update/src/main/java/net/ripe/db/whois/update/keycert/X509CertificateWrapper.java +++ b/whois-update/src/main/java/net/ripe/db/whois/update/keycert/X509CertificateWrapper.java @@ -33,7 +33,7 @@ public static X509CertificateWrapper parse(final RpslObject rpslObject) { } try { - final byte[] bytes = new RpslObjectFilter(rpslObject).getCertificateFromKeyCert().getBytes(Charsets.ISO_8859_1); + final byte[] bytes = new RpslObjectFilter(rpslObject).getCertificateFromKeyCert().getBytes(Charsets.UTF_8); X509CertParser parser = new X509CertParser(); parser.engineInit(new ByteArrayInputStream(bytes)); From bbe1b42492bfff04d289bab4e3f73589b6c210be Mon Sep 17 00:00:00 2001 From: Ed Shryane Date: Mon, 26 Aug 2013 12:42:49 +0200 Subject: [PATCH 03/22] Encode test client query response as UTF-8. --- .../common/support/DummyWhoisClient.java | 4 +- .../integration/PersonIntegrationSpec.groovy | 55 +++++++++++++++++++ 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/whois-commons/src/test/java/net/ripe/db/whois/common/support/DummyWhoisClient.java b/whois-commons/src/test/java/net/ripe/db/whois/common/support/DummyWhoisClient.java index bc73f84f27..7fc4f4d22f 100644 --- a/whois-commons/src/test/java/net/ripe/db/whois/common/support/DummyWhoisClient.java +++ b/whois-commons/src/test/java/net/ripe/db/whois/common/support/DummyWhoisClient.java @@ -34,7 +34,7 @@ public static String query(final int port, final String query) { public static String query(final int port, final String query, final int timeout) { try { - return new DummyWhoisClient("127.0.0.1", port).sendQuery(query, Charset.forName("ISO-8859-1"), timeout); + return new DummyWhoisClient("127.0.0.1", port).sendQuery(query, Charset.forName("UTF-8"), timeout); } catch (IOException e) { throw new IllegalStateException("Unable to execute query"); } @@ -52,7 +52,7 @@ public DummyWhoisClient(String host, int port) { } public String sendQuery(String query) throws IOException { - return sendQuery(query, Charset.forName("ISO-8859-1")); + return sendQuery(query, Charset.forName("UTF-8")); } public String sendQuery(String query, Charset charset) throws IOException { diff --git a/whois-db/src/test/groovy/spec/integration/PersonIntegrationSpec.groovy b/whois-db/src/test/groovy/spec/integration/PersonIntegrationSpec.groovy index 7faa4f074d..75c503faf3 100644 --- a/whois-db/src/test/groovy/spec/integration/PersonIntegrationSpec.groovy +++ b/whois-db/src/test/groovy/spec/integration/PersonIntegrationSpec.groovy @@ -365,6 +365,61 @@ class PersonIntegrationSpec extends BaseWhoisSourceSpec { response =~ /Mandatory attribute \"mnt-by\" is missing/ } + def "create person mail update with non-ASCII UTF-8 character"() { + given: + def person = new Message(body: """\ + person: Test Person3 + address: Flughafenstra\u00DFe 109/a + address: München, Germany + phone: +49 282 411141 + fax-no: +49 282 411140 + nic-hdl: TP3-TEST + changed: dbtest@ripe.net 20120101 + mnt-by: UPD-MNT + source: TEST + password: update + """.stripIndent()) + + when: + def message = send person + def response = ackFor message + + then: + println response + response =~ /Create SUCCEEDED: \[person\] TP3-TEST/ + + then: + def query = query "-r -T person TP3-TEST" + query =~ /address:\s+Flughafenstraße 109\/a/ + query =~ /address:\s+München, Germany/ + } + + def "create person syncupdate with non-ASCII UTF-8 character"() { + given: + def create = syncUpdate new SyncUpdate(data: """\ + person: Test Person3 + address: Flughafenstra\u00DFe 109/a + address: München, Germany + phone: +49 282 411141 + fax-no: +49 282 411140 + nic-hdl: TP3-TEST + changed: dbtest@ripe.net 20120101 + mnt-by: UPD-MNT + source: TEST + password: update + """.stripIndent()) + + when: + create =~ /Create SUCCEEDED: \[person\] TP3-TEST/ + + then: + def query = query "-r -T person TP3-TEST" + query =~ /address:\s+Flughafenstraße 109\/a/ + query =~ /address:\s+München, Germany/ + } + + + def "create person with generated nic-hdl"() { given: def person = new SyncUpdate(data: """\ From 3947248e05383c95d317d45efa87d12ba0196ff7 Mon Sep 17 00:00:00 2001 From: Ed Shryane Date: Mon, 26 Aug 2013 12:43:31 +0200 Subject: [PATCH 04/22] Export database with UTF-8 encoding. --- .../ripe/db/whois/scheduler/task/export/ExportFileWriter.java | 2 +- .../whois/scheduler/task/export/ExportFileWriterFactory.java | 4 ++-- .../scheduler/task/export/ExportDatabaseTestIntegration.java | 4 ++-- .../scheduler/task/export/ExportFileWriterFactoryTest.java | 4 ++-- .../db/whois/scheduler/task/export/ExportFileWriterTest.java | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/whois-scheduler/src/main/java/net/ripe/db/whois/scheduler/task/export/ExportFileWriter.java b/whois-scheduler/src/main/java/net/ripe/db/whois/scheduler/task/export/ExportFileWriter.java index 882e61ad8e..6909e21f89 100644 --- a/whois-scheduler/src/main/java/net/ripe/db/whois/scheduler/task/export/ExportFileWriter.java +++ b/whois-scheduler/src/main/java/net/ripe/db/whois/scheduler/task/export/ExportFileWriter.java @@ -70,7 +70,7 @@ private Writer getWriter(final String filename) throws IOException { final File file = new File(baseDir, filename + ".gz"); final FileOutputStream fileOutputStream = new FileOutputStream(file); try { - writer = new BufferedWriter(new OutputStreamWriter(new GZIPOutputStream(fileOutputStream), Charsets.ISO_8859_1)); + writer = new BufferedWriter(new OutputStreamWriter(new GZIPOutputStream(fileOutputStream), Charsets.UTF_8)); writer.write(QueryMessages.termsAndConditionsDump().toString()); writerMap.put(filename, writer); } catch (IOException e) { diff --git a/whois-scheduler/src/main/java/net/ripe/db/whois/scheduler/task/export/ExportFileWriterFactory.java b/whois-scheduler/src/main/java/net/ripe/db/whois/scheduler/task/export/ExportFileWriterFactory.java index 1adf2e1c78..cce4892595 100644 --- a/whois-scheduler/src/main/java/net/ripe/db/whois/scheduler/task/export/ExportFileWriterFactory.java +++ b/whois-scheduler/src/main/java/net/ripe/db/whois/scheduler/task/export/ExportFileWriterFactory.java @@ -47,8 +47,8 @@ public List createExportFileWriters(final File baseDir, final initDirs(fullDirNew, fullDir, splitDirNew, splitDir, internalDir); try { - FileCopyUtils.copy(String.valueOf(lastSerial).getBytes(Charsets.ISO_8859_1), new File(fullDirNew, CURRENTSERIAL_FILENAME)); - FileCopyUtils.copy(String.valueOf(lastSerial).getBytes(Charsets.ISO_8859_1), new File(fullDir, CURRENTSERIAL_FILENAME)); + FileCopyUtils.copy(String.valueOf(lastSerial).getBytes(Charsets.UTF_8), new File(fullDirNew, CURRENTSERIAL_FILENAME)); + FileCopyUtils.copy(String.valueOf(lastSerial).getBytes(Charsets.UTF_8), new File(fullDir, CURRENTSERIAL_FILENAME)); } catch (IOException e) { throw new RuntimeException("Writing current serial", e); } diff --git a/whois-scheduler/src/test/java/net/ripe/db/whois/scheduler/task/export/ExportDatabaseTestIntegration.java b/whois-scheduler/src/test/java/net/ripe/db/whois/scheduler/task/export/ExportDatabaseTestIntegration.java index eb867e6fa4..6fe9ba4156 100644 --- a/whois-scheduler/src/test/java/net/ripe/db/whois/scheduler/task/export/ExportDatabaseTestIntegration.java +++ b/whois-scheduler/src/test/java/net/ripe/db/whois/scheduler/task/export/ExportDatabaseTestIntegration.java @@ -421,9 +421,9 @@ private void checkFile(final String name, final String... expectedContents) thro final boolean isDumpFile = name.endsWith(".gz"); if (isDumpFile) { - reader = new InputStreamReader(new GZIPInputStream(new BufferedInputStream(new FileInputStream(file))), Charsets.ISO_8859_1); + reader = new InputStreamReader(new GZIPInputStream(new BufferedInputStream(new FileInputStream(file))), Charsets.UTF_8); } else { - reader = new InputStreamReader(new BufferedInputStream(new FileInputStream(file)), Charsets.ISO_8859_1); + reader = new InputStreamReader(new BufferedInputStream(new FileInputStream(file)), Charsets.UTF_8); } final String contents = FileCopyUtils.copyToString(reader); diff --git a/whois-scheduler/src/test/java/net/ripe/db/whois/scheduler/task/export/ExportFileWriterFactoryTest.java b/whois-scheduler/src/test/java/net/ripe/db/whois/scheduler/task/export/ExportFileWriterFactoryTest.java index 7d2537dad4..48d90cc3cc 100644 --- a/whois-scheduler/src/test/java/net/ripe/db/whois/scheduler/task/export/ExportFileWriterFactoryTest.java +++ b/whois-scheduler/src/test/java/net/ripe/db/whois/scheduler/task/export/ExportFileWriterFactoryTest.java @@ -88,13 +88,13 @@ public void isLastSerialFile_created() throws IOException { final File currentSerialFile = new File(folder.getRoot(), "dbase/RIPE.CURRENTSERIAL"); assertThat(currentSerialFile.exists(), Matchers.is(true)); - final String savedSerial = new String(FileCopyUtils.copyToByteArray(currentSerialFile), Charsets.ISO_8859_1); + final String savedSerial = new String(FileCopyUtils.copyToByteArray(currentSerialFile), Charsets.UTF_8); assertThat(savedSerial, Matchers.is(String.valueOf(LAST_SERIAL))); final File newSerialFile = new File(folder.getRoot(), "dbase_new/RIPE.CURRENTSERIAL"); assertThat(newSerialFile.exists(), Matchers.is(true)); - final String newSavedSerial = new String(FileCopyUtils.copyToByteArray(currentSerialFile), Charsets.ISO_8859_1); + final String newSavedSerial = new String(FileCopyUtils.copyToByteArray(currentSerialFile), Charsets.UTF_8); assertThat(newSavedSerial, Matchers.is(String.valueOf(LAST_SERIAL))); } } diff --git a/whois-scheduler/src/test/java/net/ripe/db/whois/scheduler/task/export/ExportFileWriterTest.java b/whois-scheduler/src/test/java/net/ripe/db/whois/scheduler/task/export/ExportFileWriterTest.java index 7c99180472..de09134e29 100644 --- a/whois-scheduler/src/test/java/net/ripe/db/whois/scheduler/task/export/ExportFileWriterTest.java +++ b/whois-scheduler/src/test/java/net/ripe/db/whois/scheduler/task/export/ExportFileWriterTest.java @@ -98,7 +98,7 @@ public void write() throws IOException { } private void checkFile(final File file, final String expectedContents) throws IOException { - final String content = FileCopyUtils.copyToString(new InputStreamReader(new GZIPInputStream(new FileInputStream(file)), Charsets.ISO_8859_1)); + final String content = FileCopyUtils.copyToString(new InputStreamReader(new GZIPInputStream(new FileInputStream(file)), Charsets.UTF_8)); Assert.assertThat(content, Matchers.is(QueryMessages.termsAndConditionsDump() + "\n" + expectedContents)); } From 780f69552a531ebdae752064ee9733164c32da73 Mon Sep 17 00:00:00 2001 From: Ed Shryane Date: Mon, 9 Sep 2013 16:30:45 +0200 Subject: [PATCH 05/22] Tested that single-byte latin1 characters > ASCII in the object BLOB column will need to be converted to utf8 separately (e.g. using MySQL CONVERT). --- .../integration/SimpleTestIntegration.java | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/whois-query/src/test/java/net/ripe/db/whois/query/integration/SimpleTestIntegration.java b/whois-query/src/test/java/net/ripe/db/whois/query/integration/SimpleTestIntegration.java index 8910cda68b..c2374566c2 100644 --- a/whois-query/src/test/java/net/ripe/db/whois/query/integration/SimpleTestIntegration.java +++ b/whois-query/src/test/java/net/ripe/db/whois/query/integration/SimpleTestIntegration.java @@ -4,6 +4,7 @@ import net.ripe.db.whois.common.DateTimeProvider; import net.ripe.db.whois.common.IntegrationTest; import net.ripe.db.whois.common.dao.RpslObjectUpdateInfo; +import net.ripe.db.whois.common.dao.jdbc.DatabaseHelper; import net.ripe.db.whois.common.domain.CIString; import net.ripe.db.whois.common.iptree.IpTreeUpdater; import net.ripe.db.whois.common.rpsl.RpslObject; @@ -15,6 +16,7 @@ import net.ripe.db.whois.query.support.AbstractWhoisIntegrationTest; import org.junit.After; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; import org.springframework.beans.factory.annotation.Autowired; @@ -708,4 +710,33 @@ public void invalid_combination() { assertThat(response, containsString("ERROR:110: multiple use of flag")); assertThat(response, containsString("The flag \"-v\" cannot be used multiple times.")); } + + @Test + public void query_utf8_encoded_object() throws Exception { + databaseHelper.getWhoisTemplate().execute( + "UPDATE last SET object = CONVERT(_latin1'" + + "domain: 117.80.81.in-addr.arpa\n" + + "descr: 66121 Saarbrücken\n" + + "' USING utf8) WHERE pkey = '117.80.81.in-addr.arpa'"); + + final String response = DummyWhoisClient.query(QueryServer.port, "117.80.81.in-addr.arpa"); + + // 0x00FC is UTF-8 representation of u-umlaut (2 bytes) + assertThat(response, containsString("Saarbr\u00FCcken")); + } + + @Ignore("TODO: single byte latin1 characters > ASCII in object BLOB column will need to be converted to UTF8") + @Test + public void query_latin1_encoded_object_in_utf8_table() throws Exception { + // 0xFC is latin1 representation of u-umlaut (1 byte) + databaseHelper.getWhoisTemplate().execute( + "UPDATE last SET object = _latin1'" + + "domain: 117.80.81.in-addr.arpa\n" + + "descr: 66121 Saarbrücken" + + "' WHERE pkey = '117.80.81.in-addr.arpa'"); + + final String response = DummyWhoisClient.query(QueryServer.port, "117.80.81.in-addr.arpa"); + + assertThat(response, containsString("Saarbr\u00FCcken")); + } } From 6eb4812bfc9d8ac247eb5885740428fbc312f6a4 Mon Sep 17 00:00:00 2001 From: Edward Shryane Date: Wed, 17 Dec 2014 17:58:08 +0100 Subject: [PATCH 06/22] Fixed UTF8 tests (no conversion to latin1 on this branch). --- .../SyncUpdatesServiceTestIntegration.java | 55 ++++++++++--------- .../rest/WhoisRestServiceTestIntegration.java | 11 ++-- .../MailMessageIntegrationSpec.groovy | 9 ++- .../whois/update/domain/UpdateMessages.java | 4 -- .../update/handler/SingleUpdateHandler.java | 15 ----- 5 files changed, 37 insertions(+), 57 deletions(-) diff --git a/whois-api/src/test/java/net/ripe/db/whois/api/rest/SyncUpdatesServiceTestIntegration.java b/whois-api/src/test/java/net/ripe/db/whois/api/rest/SyncUpdatesServiceTestIntegration.java index e7b6d70c68..e393db392d 100644 --- a/whois-api/src/test/java/net/ripe/db/whois/api/rest/SyncUpdatesServiceTestIntegration.java +++ b/whois-api/src/test/java/net/ripe/db/whois/api/rest/SyncUpdatesServiceTestIntegration.java @@ -11,7 +11,6 @@ import net.ripe.db.whois.common.rpsl.RpslObject; import net.ripe.db.whois.update.mail.MailSenderStub; import org.glassfish.jersey.media.multipart.FormDataMultiPart; -import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; import org.springframework.beans.factory.annotation.Autowired; @@ -520,7 +519,7 @@ public void post_url_encoded_data() throws Exception { } @Test - public void post_url_encoded_data_with_latin1_charset() throws Exception { + public void post_url_encoded_data_with_latin1_charset_failure() throws Exception { rpslObjectUpdateDao.createObject(RpslObject.parse(PERSON_ANY1_TEST)); rpslObjectUpdateDao.createObject(RpslObject.parse(MNTNER_TEST_MNTNER)); @@ -542,10 +541,8 @@ public void post_url_encoded_data_with_latin1_charset() throws Exception { assertThat(response, containsString("Flughafenstraße 109/a")); } - // TODO we should return a informational warning indicating less of information due to conversion into latin1 - @Ignore @Test - public void post_url_encoded_data_with_non_latin1_address() throws Exception { + public void post_url_encoded_data_with_non_latin1_address_success() throws Exception { rpslObjectUpdateDao.createObject(RpslObject.parse(PERSON_ANY1_TEST)); rpslObjectUpdateDao.createObject(RpslObject.parse(MNTNER_TEST_MNTNER)); @@ -562,13 +559,12 @@ public void post_url_encoded_data_with_non_latin1_address() throws Exception { "password: emptypassword"), MediaType.valueOf("application/x-www-form-urlencoded; charset=UTF-8")), String.class); - assertThat(response, containsString("Attribute \"address\" value changed due to conversion into the ISO-8859-1 (Latin-1) character set")); + assertThat(response, containsString("Create SUCCEEDED: [person] TP2-TEST Test Person again")); + assertThat(databaseHelper.lookupObject(ObjectType.PERSON, "TP2-TEST").toString(), containsString("address: Тверская улица,москва")); } - // TODO: [ES] no warning on conversion of cyrillic characters to latin-1 charset - @Ignore @Test - public void post_multipart_data_with_non_latin1_address() throws Exception { + public void post_multipart_data_with_non_latin1_address_success() throws Exception { databaseHelper.addObject(PERSON_ANY1_TEST); databaseHelper.addObject(MNTNER_TEST_MNTNER); @@ -583,11 +579,13 @@ public void post_multipart_data_with_non_latin1_address() throws Exception { "source: TEST\n" + "password: emptypassword") .field("NEW", "yes"); + final String response = RestTest.target(getPort(), "whois/syncupdates/test") .request() .post(Entity.entity(multipart, multipart.getMediaType()), String.class); - assertThat(response, containsString("Attribute \"address\" value changed due to conversion into the ISO-8859-1 (Latin-1) character set")); + assertThat(response, containsString("Create SUCCEEDED: [person] TP2-TEST Test Person")); + assertThat(databaseHelper.lookupObject(ObjectType.PERSON, "TP2-TEST").toString(), containsString("address: Тверская улица,москва")); } @Test @@ -598,17 +596,19 @@ public void create_person_with_filtered_source() throws Exception { final FormDataMultiPart multipart = new FormDataMultiPart() .field("DATA", "person: Test Person\n" + - "address: Home\n" + - "phone: +31 6 12345678\n" + - "nic-hdl: TP2-TEST\n" + - "mnt-by: mntner\n" + - "changed: dbtest@ripe.net 20120101\n" + - "source: TEST #Filtered\n" + - "password: emptypassword") + "address: Home\n" + + "phone: +31 6 12345678\n" + + "nic-hdl: TP2-TEST\n" + + "mnt-by: mntner\n" + + "changed: dbtest@ripe.net 20120101\n" + + "source: TEST #Filtered\n" + + "password: emptypassword") .field("NEW", "yes"); + final String response = RestTest.target(getPort(), "whois/syncupdates/test") .request() .post(Entity.entity(multipart, multipart.getMediaType()), String.class); + assertThat(response, containsString("Create FAILED")); assertThat(response, containsString("***Error: End of line comments not allowed on \"source:\" attribute")); } @@ -621,23 +621,24 @@ public void update_person_with_filtered_source() throws Exception { final FormDataMultiPart multipart = new FormDataMultiPart() .field("DATA", "person: Test Person\n" + - "address: Home\n" + - "phone: +31 6 12345678\n" + - "nic-hdl: TP1-TEST\n" + - "mnt-by: mntner\n" + - "remarks: test remark\n" + - "remarks: another test remark\n" + - "changed: dbtest@ripe.net 20120101\n" + - "source: TEST #Filtered\n" + - "password: emptypassword"); + "address: Home\n" + + "phone: +31 6 12345678\n" + + "nic-hdl: TP1-TEST\n" + + "mnt-by: mntner\n" + + "remarks: test remark\n" + + "remarks: another test remark\n" + + "changed: dbtest@ripe.net 20120101\n" + + "source: TEST #Filtered\n" + + "password: emptypassword"); + final String response = RestTest.target(getPort(), "whois/syncupdates/test") .request() .post(Entity.entity(multipart, multipart.getMediaType()), String.class); + assertThat(response, containsString("Modify FAILED")); assertThat(response, containsString("End of line comments not allowed on \"source:\" attribute")); } - // helper methods private MimeMessage getMessage(final String to) throws MessagingException { diff --git a/whois-api/src/test/java/net/ripe/db/whois/api/rest/WhoisRestServiceTestIntegration.java b/whois-api/src/test/java/net/ripe/db/whois/api/rest/WhoisRestServiceTestIntegration.java index 68ab2037f4..b8811e04ef 100644 --- a/whois-api/src/test/java/net/ripe/db/whois/api/rest/WhoisRestServiceTestIntegration.java +++ b/whois-api/src/test/java/net/ripe/db/whois/api/rest/WhoisRestServiceTestIntegration.java @@ -1811,9 +1811,9 @@ public void create_utf8_character_encoding() { .request() .post(Entity.entity(whoisObjectMapper.mapRpslObjects(FormattedClientAttributeMapper.class, person), MediaType.APPLICATION_XML), WhoisResources.class); - // UTF-8 characters are mapped to latin1. Characters outside the latin1 charset are substituted by '?' + // UTF-8 characters are NOT mapped to latin1, but left as-is final WhoisObject responseObject = whoisResources.getWhoisObjects().get(0); - assertThat(responseObject.getAttributes().get(1).getValue(), is("test ? and \u00DF characters")); + assertThat(responseObject.getAttributes().get(1).getValue(), is("test \u03A3 and \u00DF characters")); } @Test @@ -1956,11 +1956,10 @@ public void update_person_with_non_latin_chars() throws Exception { .put(Entity.entity(whoisObjectMapper.mapRpslObjects(FormattedClientAttributeMapper.class, update), MediaType.APPLICATION_XML), WhoisResources.class); - RestTest.assertErrorCount(response, 1); - RestTest.assertErrorMessage(response, 0, "Warning", "Attribute \"%s\" value changed due to conversion into the ISO-8859-1 (Latin-1) character set", "address"); + RestTest.assertErrorCount(response, 0); final RpslObject lookupObject = databaseHelper.lookupObject(ObjectType.PERSON, "TP1-TEST"); - assertThat(lookupObject.findAttribute(AttributeType.ADDRESS).getValue(), is(" address: ???????? ?????,??????")); + assertThat(lookupObject.findAttribute(AttributeType.ADDRESS).getValue(), is(" address: Тверская улица,москва")); } { final WhoisResources response = @@ -1968,7 +1967,7 @@ public void update_person_with_non_latin_chars() throws Exception { .request() .get(WhoisResources.class); - assertThat(response.getWhoisObjects().get(0).getAttributes(), hasItem(new Attribute("address", "address: ???????? ?????,??????"))); + assertThat(response.getWhoisObjects().get(0).getAttributes(), hasItem(new Attribute("address", "address: Тверская улица,москва"))); } } diff --git a/whois-endtoend/src/test/groovy/net/ripe/db/whois/spec/integration/MailMessageIntegrationSpec.groovy b/whois-endtoend/src/test/groovy/net/ripe/db/whois/spec/integration/MailMessageIntegrationSpec.groovy index 914f40e6f2..cfe0dcc396 100644 --- a/whois-endtoend/src/test/groovy/net/ripe/db/whois/spec/integration/MailMessageIntegrationSpec.groovy +++ b/whois-endtoend/src/test/groovy/net/ripe/db/whois/spec/integration/MailMessageIntegrationSpec.groovy @@ -1,8 +1,6 @@ package net.ripe.db.whois.spec.integration - import net.ripe.db.whois.common.IntegrationTest import net.ripe.db.whois.spec.domain.Message -import spock.lang.Ignore; @org.junit.experimental.categories.Category(IntegrationTest.class) class MailMessageIntegrationSpec extends BaseWhoisSourceSpec { @@ -594,8 +592,7 @@ class MailMessageIntegrationSpec extends BaseWhoisSourceSpec { ack.contents =~ /\*\*\*Warning: All keywords were ignored/ } - @Ignore - def "warning on conversion of non latin-1 address"() { + def "no conversion of non latin-1 address"() { when: def message = send "Date: Fri, 4 Jan 2013 15:29:59 +0100\n" + "From: noreply@ripe.net\n" + @@ -620,7 +617,9 @@ class MailMessageIntegrationSpec extends BaseWhoisSourceSpec { ack.success ack.summary.nrFound == 1 - ack.contents =~ /\*\*\*Warning: Attribute "address" value changed due to conversion into the ISO-8859-1 (Latin-1) character set/ + then: + def query = queryObject("-r FP1-TEST", "person", "First Person") + query =~ /address: Тверская улица,москва/ } } diff --git a/whois-update/src/main/java/net/ripe/db/whois/update/domain/UpdateMessages.java b/whois-update/src/main/java/net/ripe/db/whois/update/domain/UpdateMessages.java index 1a6d196f12..2f21df84c0 100644 --- a/whois-update/src/main/java/net/ripe/db/whois/update/domain/UpdateMessages.java +++ b/whois-update/src/main/java/net/ripe/db/whois/update/domain/UpdateMessages.java @@ -594,8 +594,4 @@ public static Message sponsoringOrgNotAllowedWithStatus(final CharSequence statu public static Message sponsoringOrgMustBePresent() { return new Message(Type.ERROR, "This resource object must be created with a sponsoring-org attribute"); } - - public static Message valueChangedDueToLatin1Conversion(String attributeName) { - return new Message(Type.WARNING, "Attribute \"%s\" value changed due to conversion into the ISO-8859-1 (Latin-1) character set", attributeName); - } } diff --git a/whois-update/src/main/java/net/ripe/db/whois/update/handler/SingleUpdateHandler.java b/whois-update/src/main/java/net/ripe/db/whois/update/handler/SingleUpdateHandler.java index ef86df7da0..5fd123b10f 100644 --- a/whois-update/src/main/java/net/ripe/db/whois/update/handler/SingleUpdateHandler.java +++ b/whois-update/src/main/java/net/ripe/db/whois/update/handler/SingleUpdateHandler.java @@ -9,7 +9,6 @@ import net.ripe.db.whois.common.rpsl.ObjectMessages; import net.ripe.db.whois.common.rpsl.ObjectTemplate; import net.ripe.db.whois.common.rpsl.ObjectType; -import net.ripe.db.whois.common.rpsl.RpslAttribute; import net.ripe.db.whois.common.rpsl.RpslObject; import net.ripe.db.whois.common.rpsl.RpslObjectFilter; import net.ripe.db.whois.update.authentication.Authenticator; @@ -133,10 +132,6 @@ public void handle(final Origin origin, final Keyword keyword, final Update upda // re-generate preparedUpdate preparedUpdate = new PreparedUpdate(update, originalObject, updatedObjectWithAutoKeys, action, overrideOptions); - // Currently the database uses the latin-1 characterset. Non convertable characters are stored as ?. - // We warn about information loss. - warnForNotLatinAttributeValues(update, updateContext); - // run business validation & pending updates hack final boolean businessRulesOk = updateObjectHandler.validateBusinessRules(preparedUpdate, updateContext); // TODO: [AH] pending updates is scattered across the code @@ -159,16 +154,6 @@ public void handle(final Origin origin, final Keyword keyword, final Update upda } } - @CheckForNull - private void warnForNotLatinAttributeValues(final Update update, final UpdateContext updateContext ) { - RpslObject submittedObject = update.getSubmittedObject(); - for( RpslAttribute attribute: submittedObject.getAttributes() ) { - if( ! CharacterSetConversion.isConvertableIntoLatin1(attribute.getValue() )) { - updateContext.addMessage(update, UpdateMessages.valueChangedDueToLatin1Conversion(attribute.getKey()) ); - } - } - } - @CheckForNull private RpslObject getOriginalObject(final Update update, final UpdateContext updateContext, final OverrideOptions overrideOptions) { RpslObject originalObject; From 9bae706d31c1d413b1acf3c341bbef77f82a2271 Mon Sep 17 00:00:00 2001 From: Edward Shryane Date: Thu, 18 Dec 2014 00:38:13 +0100 Subject: [PATCH 07/22] Fixed UTF8 in test --- .../test/groovy/net/ripe/db/whois/spec/update/OrgSpec.groovy | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/whois-endtoend/src/test/groovy/net/ripe/db/whois/spec/update/OrgSpec.groovy b/whois-endtoend/src/test/groovy/net/ripe/db/whois/spec/update/OrgSpec.groovy index 16e7fbd898..f9b24b0354 100644 --- a/whois-endtoend/src/test/groovy/net/ripe/db/whois/spec/update/OrgSpec.groovy +++ b/whois-endtoend/src/test/groovy/net/ripe/db/whois/spec/update/OrgSpec.groovy @@ -394,7 +394,6 @@ class OrgSpec extends BaseQueryUpdateSpec { queryObject("-r -T organisation ORG-XA1-TEST", "organisation", "ORG-XA1-TEST") } - // FIXME: [AH] the character 'Ő' was converted to a ? in the acknowledgement message def "create organisation with auto-1 and weird (invalid) chars in name, missing auth"() { expect: queryObjectNotFound("-r -T organisation ORG-AA1-TEST", "organisation", "ORG-AA1-TEST") @@ -429,7 +428,7 @@ class OrgSpec extends BaseQueryUpdateSpec { ack.countErrorWarnInfo(1, 0, 0) - ack.errorMessagesFor("Create", "[organisation] auto-1") == ["Syntax error in Hö öns mö åäö ?Ú"] + ack.errorMessagesFor("Create", "[organisation] auto-1") == ["Syntax error in Hö öns mö åäö ŐÚ"] queryObjectNotFound("-r -T organisation ORG-AA1-TEST", "organisation", "ORG-AA1-TEST") } From a4146e6439aca5a4882ebf2f25f0860f7b29ec8c Mon Sep 17 00:00:00 2001 From: Edward Shryane Date: Tue, 16 May 2017 11:44:49 +0200 Subject: [PATCH 08/22] Merge branch 'master' into utf8 --- .../net/ripe/db/whois/update/handler/SingleUpdateHandler.java | 1 + 1 file changed, 1 insertion(+) diff --git a/whois-update/src/main/java/net/ripe/db/whois/update/handler/SingleUpdateHandler.java b/whois-update/src/main/java/net/ripe/db/whois/update/handler/SingleUpdateHandler.java index 7a253075f5..52a523b652 100644 --- a/whois-update/src/main/java/net/ripe/db/whois/update/handler/SingleUpdateHandler.java +++ b/whois-update/src/main/java/net/ripe/db/whois/update/handler/SingleUpdateHandler.java @@ -9,6 +9,7 @@ import net.ripe.db.whois.common.rpsl.ObjectMessages; import net.ripe.db.whois.common.rpsl.ObjectTemplate; import net.ripe.db.whois.common.rpsl.ObjectType; +import net.ripe.db.whois.common.rpsl.RpslAttribute; import net.ripe.db.whois.common.rpsl.RpslObject; import net.ripe.db.whois.common.rpsl.RpslObjectFilter; import net.ripe.db.whois.common.rpsl.ValidationMessages; From 801865e64999379d22604241934bdc8648d857c9 Mon Sep 17 00:00:00 2001 From: Ed Shryane Date: Sun, 12 Jul 2020 21:44:51 +0200 Subject: [PATCH 09/22] Deleted obsolete files --- .../src/main/resources/pending_schema.sql | 15 ------ .../src/main/resources/scheduler_schema.sql | 53 ------------------- 2 files changed, 68 deletions(-) delete mode 100644 whois-commons/src/main/resources/pending_schema.sql delete mode 100644 whois-commons/src/main/resources/scheduler_schema.sql diff --git a/whois-commons/src/main/resources/pending_schema.sql b/whois-commons/src/main/resources/pending_schema.sql deleted file mode 100644 index 7d778236f6..0000000000 --- a/whois-commons/src/main/resources/pending_schema.sql +++ /dev/null @@ -1,15 +0,0 @@ -DROP TABLE IF EXISTS `pending_updates`; -create table pending_updates ( - `id` int(10) unsigned NOT NULL AUTO_INCREMENT, - `object_type` tinyint(3) unsigned NOT NULL, - `pkey` varchar(254) NOT NULL, - `stored_date` date NOT NULL DEFAULT '0000-00-00', - `passed_authentications` VARCHAR(100) NOT NULL, - `object` longblob NOT NULL, - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - -DROP TABLE IF EXISTS `version`; -CREATE TABLE `version` ( - `version` varchar(80) DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/whois-commons/src/main/resources/scheduler_schema.sql b/whois-commons/src/main/resources/scheduler_schema.sql deleted file mode 100644 index 184d2e6c3f..0000000000 --- a/whois-commons/src/main/resources/scheduler_schema.sql +++ /dev/null @@ -1,53 +0,0 @@ --- MySQL dump 10.13 Distrib 5.1.61, for redhat-linux-gnu (x86_64) --- --- Host: localhost Database: SCHEDULER_RIPE --- ------------------------------------------------------ --- Server version 5.1.61-log - -/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; -/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; -/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; -/*!40101 SET NAMES utf8 */; -/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; -/*!40103 SET TIME_ZONE='+00:00' */; -/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; -/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; -/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; -/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; - --- --- Table structure for table `scheduler` --- - -DROP TABLE IF EXISTS `scheduler`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `scheduler` ( - `date` date NOT NULL, - `host` varchar(50) NOT NULL, - PRIMARY KEY (`date`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Table structure for table `version` --- - -DROP TABLE IF EXISTS `version`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `version` ( - `version` varchar(80) DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=utf8; -/*!40101 SET character_set_client = @saved_cs_client */; -/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; - -/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; -/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; -/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; -/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; -/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; -/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; -/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; - --- Dump completed on 2013-02-19 17:03:30 From 31ec4e32995610d0c53096c8910504a986c60098 Mon Sep 17 00:00:00 2001 From: Ed Shryane Date: Sun, 12 Jul 2020 21:56:08 +0200 Subject: [PATCH 10/22] Use utf8mb4 character set instead of utf8. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ref. "How to support full Unicode in MySQL databases" https://mathiasbynens.be/notes/mysql-utf8mb4 Ref. "In MySQL, never use “utf8”. Use “utf8mb4”." https://medium.com/@adamhooper/in-mysql-never-use-utf8-use-utf8mb4-11761243e434 --- .../src/main/resources/acl_schema.sql | 28 +- .../src/main/resources/mailupdates_schema.sql | 10 +- .../src/main/resources/whois_schema.sql | 254 +++++++++--------- .../whois/common/dao/jdbc/DatabaseHelper.java | 2 +- 4 files changed, 147 insertions(+), 147 deletions(-) diff --git a/whois-commons/src/main/resources/acl_schema.sql b/whois-commons/src/main/resources/acl_schema.sql index 586eaa292f..fae28d2adf 100644 --- a/whois-commons/src/main/resources/acl_schema.sql +++ b/whois-commons/src/main/resources/acl_schema.sql @@ -7,7 +7,7 @@ /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; -/*!40101 SET NAMES utf8 */; +/*!40101 SET NAMES utf8mb4 */; /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; /*!40103 SET TIME_ZONE='+00:00' */; /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; @@ -21,13 +21,13 @@ DROP TABLE IF EXISTS `acl_denied`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `acl_denied` ( `prefix` varchar(50) NOT NULL, `comment` text, `denied_date` date NOT NULL DEFAULT '0000-00-00', PRIMARY KEY (`prefix`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -36,14 +36,14 @@ CREATE TABLE `acl_denied` ( DROP TABLE IF EXISTS `acl_event`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `acl_event` ( `prefix` varchar(50) NOT NULL, `event_time` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00', `daily_limit` int(11) NOT NULL, `event_type` varchar(20) NOT NULL, PRIMARY KEY (`prefix`,`event_time`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -52,14 +52,14 @@ CREATE TABLE `acl_event` ( DROP TABLE IF EXISTS `acl_limit`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `acl_limit` ( `prefix` varchar(50) NOT NULL, `daily_limit` int(11) NOT NULL DEFAULT '-1', `comment` text, `unlimited_connections` bit(1) NOT NULL DEFAULT b'0', PRIMARY KEY (`prefix`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -68,12 +68,12 @@ CREATE TABLE `acl_limit` ( DROP TABLE IF EXISTS `acl_proxy`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `acl_proxy` ( `prefix` varchar(50) NOT NULL, `comment` text, PRIMARY KEY (`prefix`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -82,14 +82,14 @@ CREATE TABLE `acl_proxy` ( DROP TABLE IF EXISTS `override_users`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `override_users` ( `username` varchar(255) NOT NULL, `password` varchar(64) NOT NULL, `objecttypes` varchar(255) NOT NULL, `last_changed` date DEFAULT NULL, UNIQUE KEY `override_username_idx` (`username`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -98,7 +98,7 @@ CREATE TABLE `override_users` ( DROP TABLE IF EXISTS `apikeys`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `apikeys` ( `apikey` varchar(128) NOT NULL, `uri_prefix` varchar(128) NOT NULL, @@ -112,10 +112,10 @@ CREATE TABLE `apikeys` ( DROP TABLE IF EXISTS `version`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `version` ( `version` varchar(80) DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; diff --git a/whois-commons/src/main/resources/mailupdates_schema.sql b/whois-commons/src/main/resources/mailupdates_schema.sql index 6c65835027..fb2613a4a8 100644 --- a/whois-commons/src/main/resources/mailupdates_schema.sql +++ b/whois-commons/src/main/resources/mailupdates_schema.sql @@ -7,7 +7,7 @@ /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; -/*!40101 SET NAMES utf8 */; +/*!40101 SET NAMES utf8mb4 */; /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; /*!40103 SET TIME_ZONE='+00:00' */; /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; @@ -21,7 +21,7 @@ DROP TABLE IF EXISTS `mailupdates`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `mailupdates` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, `message` longblob NOT NULL, @@ -31,7 +31,7 @@ CREATE TABLE `mailupdates` ( `changed` int(10) unsigned DEFAULT NULL, PRIMARY KEY (`id`), UNIQUE KEY `mailupdates_uuid_idx` (`claim_uuid`) -) ENGINE=InnoDB AUTO_INCREMENT=467880 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=467880 DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -40,10 +40,10 @@ CREATE TABLE `mailupdates` ( DROP TABLE IF EXISTS `version`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `version` ( `version` varchar(80) DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; diff --git a/whois-commons/src/main/resources/whois_schema.sql b/whois-commons/src/main/resources/whois_schema.sql index 8b4f4b3121..d8ea29518b 100644 --- a/whois-commons/src/main/resources/whois_schema.sql +++ b/whois-commons/src/main/resources/whois_schema.sql @@ -7,7 +7,7 @@ /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; -/*!40101 SET NAMES utf8 */; +/*!40101 SET NAMES utf8mb4 */; /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; /*!40103 SET TIME_ZONE='+00:00' */; /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; @@ -21,7 +21,7 @@ DROP TABLE IF EXISTS `abuse_c`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `abuse_c` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `pe_ro_id` int(10) unsigned NOT NULL DEFAULT '0', @@ -29,7 +29,7 @@ CREATE TABLE `abuse_c` ( PRIMARY KEY (`pe_ro_id`,`object_id`), KEY `object_type` (`object_type`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -38,14 +38,14 @@ CREATE TABLE `abuse_c` ( DROP TABLE IF EXISTS `abuse_mailbox`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `abuse_mailbox` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `abuse_mailbox` varchar(80) NOT NULL DEFAULT '', `object_type` tinyint(3) unsigned NOT NULL DEFAULT '0', PRIMARY KEY (`abuse_mailbox`,`object_id`), KEY `abuse_mailbox_object_id_index` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -54,7 +54,7 @@ CREATE TABLE `abuse_mailbox` ( DROP TABLE IF EXISTS `admin_c`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `admin_c` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `pe_ro_id` int(10) unsigned NOT NULL DEFAULT '0', @@ -62,7 +62,7 @@ CREATE TABLE `admin_c` ( PRIMARY KEY (`pe_ro_id`,`object_id`), KEY `object_type` (`object_type`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -71,7 +71,7 @@ CREATE TABLE `admin_c` ( DROP TABLE IF EXISTS `as_block`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `as_block` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `begin_as` int(10) unsigned NOT NULL DEFAULT '0', @@ -79,7 +79,7 @@ CREATE TABLE `as_block` ( PRIMARY KEY (`object_id`), KEY `begin_as` (`begin_as`), KEY `end_as` (`end_as`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -88,13 +88,13 @@ CREATE TABLE `as_block` ( DROP TABLE IF EXISTS `as_set`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `as_set` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `as_set` varchar(80) NOT NULL DEFAULT '', PRIMARY KEY (`object_id`), KEY `as_set` (`as_set`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -103,13 +103,13 @@ CREATE TABLE `as_set` ( DROP TABLE IF EXISTS `aut_num`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `aut_num` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `aut_num` char(13) NOT NULL DEFAULT '', PRIMARY KEY (`object_id`), KEY `aut_num` (`aut_num`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -118,14 +118,14 @@ CREATE TABLE `aut_num` ( DROP TABLE IF EXISTS `auth`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `auth` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', - `auth` varchar(90) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT '', + `auth` varchar(90) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', `object_type` tinyint(3) unsigned NOT NULL DEFAULT '0', PRIMARY KEY (`auth`,`object_id`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -134,7 +134,7 @@ CREATE TABLE `auth` ( DROP TABLE IF EXISTS `author`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `author` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `pe_ro_id` int(10) unsigned NOT NULL DEFAULT '0', @@ -142,7 +142,7 @@ CREATE TABLE `author` ( PRIMARY KEY (`pe_ro_id`,`object_id`), KEY `object_id` (`object_id`), KEY `object_type` (`object_type`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -151,13 +151,13 @@ CREATE TABLE `author` ( DROP TABLE IF EXISTS `domain`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `domain` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `domain` varchar(254) NOT NULL DEFAULT '', PRIMARY KEY (`object_id`), KEY `domain` (`domain`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -166,12 +166,12 @@ CREATE TABLE `domain` ( DROP TABLE IF EXISTS `ds_rdata`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `ds_rdata` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `ds_rdata` varchar(128) NOT NULL DEFAULT '', PRIMARY KEY (`ds_rdata`,`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -180,14 +180,14 @@ CREATE TABLE `ds_rdata` ( DROP TABLE IF EXISTS `e_mail`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `e_mail` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `e_mail` varchar(80) NOT NULL DEFAULT '', `object_type` tinyint(3) unsigned NOT NULL DEFAULT '0', PRIMARY KEY (`e_mail`,`object_id`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -196,7 +196,7 @@ CREATE TABLE `e_mail` ( DROP TABLE IF EXISTS `status`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `status` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `status` varchar(80) NOT NULL DEFAULT '', @@ -211,13 +211,13 @@ CREATE TABLE `status` ( DROP TABLE IF EXISTS `filter_set`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `filter_set` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `filter_set` varchar(80) NOT NULL DEFAULT '', PRIMARY KEY (`object_id`), KEY `filter_set` (`filter_set`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -226,13 +226,13 @@ CREATE TABLE `filter_set` ( DROP TABLE IF EXISTS `fingerpr`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `fingerpr` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `fingerpr` varchar(80) NOT NULL DEFAULT '', PRIMARY KEY (`fingerpr`,`object_id`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -241,7 +241,7 @@ CREATE TABLE `fingerpr` ( DROP TABLE IF EXISTS `form`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `form` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `form_id` int(10) unsigned NOT NULL DEFAULT '0', @@ -249,7 +249,7 @@ CREATE TABLE `form` ( PRIMARY KEY (`form_id`,`object_id`), KEY `object_id` (`object_id`), KEY `object_type` (`object_type`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -258,7 +258,7 @@ CREATE TABLE `form` ( DROP TABLE IF EXISTS `history`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `history` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `sequence_id` int(10) unsigned NOT NULL DEFAULT '0', @@ -269,7 +269,7 @@ CREATE TABLE `history` ( PRIMARY KEY (`object_id`,`sequence_id`), KEY `history_pkey` (`pkey`), KEY `history_timestamp` (`timestamp`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -278,13 +278,13 @@ CREATE TABLE `history` ( DROP TABLE IF EXISTS `ifaddr`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `ifaddr` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `ifaddr` int(10) unsigned NOT NULL DEFAULT '0', PRIMARY KEY (`ifaddr`,`object_id`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -293,13 +293,13 @@ CREATE TABLE `ifaddr` ( DROP TABLE IF EXISTS `inaddr_arpa`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `inaddr_arpa` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `begin_in` int(10) unsigned NOT NULL DEFAULT '0', `end_in` int(10) unsigned NOT NULL DEFAULT '0', PRIMARY KEY (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -308,7 +308,7 @@ CREATE TABLE `inaddr_arpa` ( DROP TABLE IF EXISTS `inet6num`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `inet6num` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `i6_msb` varchar(20) NOT NULL DEFAULT '', @@ -319,7 +319,7 @@ CREATE TABLE `inet6num` ( KEY `netname` (`netname`), KEY `i6_msb` (`i6_msb`), KEY `i6_lsb` (`i6_lsb`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -328,7 +328,7 @@ CREATE TABLE `inet6num` ( DROP TABLE IF EXISTS `inet_rtr`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `inet_rtr` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `inet_rtr` varchar(254) NOT NULL DEFAULT '', @@ -336,7 +336,7 @@ CREATE TABLE `inet_rtr` ( PRIMARY KEY (`object_id`), KEY `inet_rtr` (`inet_rtr`), KEY `local_as` (`local_as`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -345,7 +345,7 @@ CREATE TABLE `inet_rtr` ( DROP TABLE IF EXISTS `inetnum`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `inetnum` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `begin_in` int(10) unsigned NOT NULL DEFAULT '0', @@ -355,7 +355,7 @@ CREATE TABLE `inetnum` ( KEY `netname` (`netname`), KEY `begin_in` (`begin_in`), KEY `end_in` (`end_in`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -364,7 +364,7 @@ CREATE TABLE `inetnum` ( DROP TABLE IF EXISTS `interface`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `interface` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `interface_v6_msp` varchar(20) NOT NULL DEFAULT '', @@ -372,7 +372,7 @@ CREATE TABLE `interface` ( `interface_v4` int(10) unsigned NOT NULL DEFAULT '0', PRIMARY KEY (`interface_v6_msp`,`interface_v6_lsp`,`interface_v4`,`object_id`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -381,13 +381,13 @@ CREATE TABLE `interface` ( DROP TABLE IF EXISTS `irt`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `irt` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `irt` varchar(80) NOT NULL DEFAULT '0', PRIMARY KEY (`object_id`), KEY `irt` (`irt`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -396,13 +396,13 @@ CREATE TABLE `irt` ( DROP TABLE IF EXISTS `irt_nfy`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `irt_nfy` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `irt_nfy` varchar(80) NOT NULL DEFAULT '', PRIMARY KEY (`irt_nfy`,`object_id`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -411,13 +411,13 @@ CREATE TABLE `irt_nfy` ( DROP TABLE IF EXISTS `key_cert`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `key_cert` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `key_cert` varchar(32) NOT NULL DEFAULT '', PRIMARY KEY (`object_id`), KEY `key_cert` (`key_cert`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -426,7 +426,7 @@ CREATE TABLE `key_cert` ( DROP TABLE IF EXISTS `last`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `last` ( `object_id` int(10) unsigned NOT NULL AUTO_INCREMENT, `sequence_id` int(10) unsigned NOT NULL DEFAULT '1', @@ -437,7 +437,7 @@ CREATE TABLE `last` ( PRIMARY KEY (`object_id`,`sequence_id`), KEY `last_pkey` (`pkey`), KEY `object_type_index` (`object_type`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -446,7 +446,7 @@ CREATE TABLE `last` ( DROP TABLE IF EXISTS `mbrs_by_ref`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `mbrs_by_ref` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `mnt_id` int(10) unsigned NOT NULL DEFAULT '0', @@ -454,7 +454,7 @@ CREATE TABLE `mbrs_by_ref` ( PRIMARY KEY (`mnt_id`,`object_id`), KEY `object_id` (`object_id`), KEY `object_type` (`object_type`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -463,7 +463,7 @@ CREATE TABLE `mbrs_by_ref` ( DROP TABLE IF EXISTS `member_of`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `member_of` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `set_id` int(10) unsigned NOT NULL DEFAULT '0', @@ -471,7 +471,7 @@ CREATE TABLE `member_of` ( PRIMARY KEY (`set_id`,`object_id`), KEY `object_id` (`object_id`), KEY `object_type` (`object_type`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -480,7 +480,7 @@ CREATE TABLE `member_of` ( DROP TABLE IF EXISTS `mnt_by`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `mnt_by` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `mnt_id` int(10) unsigned NOT NULL DEFAULT '0', @@ -488,7 +488,7 @@ CREATE TABLE `mnt_by` ( PRIMARY KEY (`mnt_id`,`object_id`), KEY `object_id` (`object_id`), KEY `object_type` (`object_type`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -497,14 +497,14 @@ CREATE TABLE `mnt_by` ( DROP TABLE IF EXISTS `mnt_domains`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `mnt_domains` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `mnt_id` int(10) unsigned NOT NULL DEFAULT '0', `object_type` tinyint(3) unsigned NOT NULL DEFAULT '0', PRIMARY KEY (`mnt_id`,`object_id`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -513,7 +513,7 @@ CREATE TABLE `mnt_domains` ( DROP TABLE IF EXISTS `mnt_irt`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `mnt_irt` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `irt_id` int(10) unsigned NOT NULL DEFAULT '0', @@ -521,7 +521,7 @@ CREATE TABLE `mnt_irt` ( PRIMARY KEY (`irt_id`,`object_id`), KEY `object_id` (`object_id`), KEY `object_type` (`object_type`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -530,7 +530,7 @@ CREATE TABLE `mnt_irt` ( DROP TABLE IF EXISTS `mnt_lower`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `mnt_lower` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `mnt_id` int(10) unsigned NOT NULL DEFAULT '0', @@ -538,7 +538,7 @@ CREATE TABLE `mnt_lower` ( PRIMARY KEY (`mnt_id`,`object_id`), KEY `object_id` (`object_id`), KEY `object_type` (`object_type`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -547,13 +547,13 @@ CREATE TABLE `mnt_lower` ( DROP TABLE IF EXISTS `mnt_nfy`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `mnt_nfy` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `mnt_nfy` varchar(80) NOT NULL DEFAULT '', PRIMARY KEY (`mnt_nfy`,`object_id`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -562,14 +562,14 @@ CREATE TABLE `mnt_nfy` ( DROP TABLE IF EXISTS `mnt_ref`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `mnt_ref` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `mnt_id` int(10) unsigned NOT NULL DEFAULT '0', `object_type` tinyint(3) unsigned NOT NULL DEFAULT '0', PRIMARY KEY (`mnt_id`,`object_id`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -578,7 +578,7 @@ CREATE TABLE `mnt_ref` ( DROP TABLE IF EXISTS `mnt_routes`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `mnt_routes` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `mnt_id` int(10) unsigned NOT NULL DEFAULT '0', @@ -586,7 +586,7 @@ CREATE TABLE `mnt_routes` ( PRIMARY KEY (`mnt_id`,`object_id`), KEY `object_id` (`object_id`), KEY `object_type` (`object_type`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -595,13 +595,13 @@ CREATE TABLE `mnt_routes` ( DROP TABLE IF EXISTS `mntner`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `mntner` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `mntner` varchar(80) NOT NULL DEFAULT '', PRIMARY KEY (`object_id`), KEY `mntner` (`mntner`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -610,14 +610,14 @@ CREATE TABLE `mntner` ( DROP TABLE IF EXISTS `names`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `names` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `name` varchar(64) NOT NULL DEFAULT '', `object_type` tinyint(3) unsigned NOT NULL DEFAULT '0', PRIMARY KEY (`name`,`object_id`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -626,7 +626,7 @@ CREATE TABLE `names` ( DROP TABLE IF EXISTS `nic_hdl`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `nic_hdl` ( `range_id` int(10) unsigned NOT NULL AUTO_INCREMENT, `range_start` int(10) unsigned NOT NULL DEFAULT '0', @@ -637,7 +637,7 @@ CREATE TABLE `nic_hdl` ( KEY `range_start` (`range_start`), KEY `range_end` (`range_end`), KEY `space` (`space`,`source`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -646,14 +646,14 @@ CREATE TABLE `nic_hdl` ( DROP TABLE IF EXISTS `notify`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `notify` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `notify` varchar(80) NOT NULL DEFAULT '', `object_type` tinyint(3) unsigned NOT NULL DEFAULT '0', PRIMARY KEY (`notify`,`object_id`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -662,13 +662,13 @@ CREATE TABLE `notify` ( DROP TABLE IF EXISTS `nserver`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `nserver` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `host` varchar(254) NOT NULL DEFAULT '', PRIMARY KEY (`host`,`object_id`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -677,14 +677,14 @@ CREATE TABLE `nserver` ( DROP TABLE IF EXISTS `org`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `org` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `org_id` int(10) unsigned NOT NULL DEFAULT '0', `object_type` tinyint(3) unsigned NOT NULL DEFAULT '0', PRIMARY KEY (`org_id`,`object_id`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -693,13 +693,13 @@ CREATE TABLE `org` ( DROP TABLE IF EXISTS `org_name`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `org_name` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `name` varchar(64) NOT NULL DEFAULT '', PRIMARY KEY (`name`,`object_id`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -708,13 +708,13 @@ CREATE TABLE `org_name` ( DROP TABLE IF EXISTS `organisation`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `organisation` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `organisation` varchar(80) NOT NULL DEFAULT '', PRIMARY KEY (`organisation`,`object_id`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -723,7 +723,7 @@ CREATE TABLE `organisation` ( DROP TABLE IF EXISTS `organisation_id`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `organisation_id` ( `range_id` int(10) unsigned NOT NULL AUTO_INCREMENT, `range_end` int(10) unsigned NOT NULL DEFAULT '0', @@ -731,7 +731,7 @@ CREATE TABLE `organisation_id` ( `source` char(10) NOT NULL DEFAULT '', PRIMARY KEY (`range_id`), UNIQUE KEY `space` (`space`,`source`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -740,13 +740,13 @@ CREATE TABLE `organisation_id` ( DROP TABLE IF EXISTS `peering_set`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `peering_set` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `peering_set` varchar(80) NOT NULL DEFAULT '', PRIMARY KEY (`object_id`), KEY `peering_set` (`peering_set`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -755,7 +755,7 @@ CREATE TABLE `peering_set` ( DROP TABLE IF EXISTS `person_role`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `person_role` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `nic_hdl` varchar(30) NOT NULL DEFAULT '', @@ -763,7 +763,7 @@ CREATE TABLE `person_role` ( PRIMARY KEY (`object_id`), KEY `nic_hdl` (`nic_hdl`), KEY `object_type` (`object_type`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -772,7 +772,7 @@ CREATE TABLE `person_role` ( DROP TABLE IF EXISTS `ping_hdl`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `ping_hdl` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `pe_ro_id` int(10) unsigned NOT NULL DEFAULT '0', @@ -780,7 +780,7 @@ CREATE TABLE `ping_hdl` ( PRIMARY KEY (`pe_ro_id`,`object_id`), KEY `object_type` (`object_type`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -789,13 +789,13 @@ CREATE TABLE `ping_hdl` ( DROP TABLE IF EXISTS `poem`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `poem` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `poem` varchar(80) NOT NULL DEFAULT '', PRIMARY KEY (`object_id`), KEY `poem` (`poem`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -804,14 +804,14 @@ CREATE TABLE `poem` ( DROP TABLE IF EXISTS `poetic_form`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `poetic_form` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `poetic_form` varchar(80) NOT NULL DEFAULT '', PRIMARY KEY (`poetic_form`), KEY `poetic_form` (`poetic_form`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -820,13 +820,13 @@ CREATE TABLE `poetic_form` ( DROP TABLE IF EXISTS `ref_nfy`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `ref_nfy` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `ref_nfy` varchar(80) NOT NULL DEFAULT '', PRIMARY KEY (`ref_nfy`,`object_id`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -835,7 +835,7 @@ CREATE TABLE `ref_nfy` ( DROP TABLE IF EXISTS `route`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `route` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `prefix` int(10) unsigned NOT NULL DEFAULT '0', @@ -843,7 +843,7 @@ CREATE TABLE `route` ( `origin` varchar(13) NOT NULL DEFAULT '', PRIMARY KEY (`object_id`), KEY `origin` (`origin`,`prefix`,`prefix_length`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -852,7 +852,7 @@ CREATE TABLE `route` ( DROP TABLE IF EXISTS `route6`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `route6` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `r6_msb` varchar(20) NOT NULL DEFAULT '', @@ -863,7 +863,7 @@ CREATE TABLE `route6` ( KEY `origin` (`origin`,`r6_msb`,`r6_lsb`,`prefix_length`), KEY `r6_msb` (`r6_msb`), KEY `r6_lsb` (`r6_lsb`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -872,13 +872,13 @@ CREATE TABLE `route6` ( DROP TABLE IF EXISTS `route_set`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `route_set` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `route_set` varchar(80) NOT NULL DEFAULT '', PRIMARY KEY (`object_id`), KEY `route_set` (`route_set`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -887,13 +887,13 @@ CREATE TABLE `route_set` ( DROP TABLE IF EXISTS `rtr_set`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `rtr_set` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `rtr_set` varchar(80) NOT NULL DEFAULT '', PRIMARY KEY (`object_id`), KEY `rtr_set` (`rtr_set`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -902,7 +902,7 @@ CREATE TABLE `rtr_set` ( DROP TABLE IF EXISTS `serials`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `serials` ( `serial_id` int(11) NOT NULL AUTO_INCREMENT, `object_id` int(10) unsigned NOT NULL DEFAULT '0', @@ -911,7 +911,7 @@ CREATE TABLE `serials` ( `operation` tinyint(4) unsigned NOT NULL DEFAULT '0', PRIMARY KEY (`serial_id`), KEY `object` (`object_id`,`sequence_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -920,14 +920,14 @@ CREATE TABLE `serials` ( DROP TABLE IF EXISTS `sponsoring_org`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `sponsoring_org` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `org_id` int(10) unsigned NOT NULL DEFAULT '0', `object_type` tinyint(3) unsigned NOT NULL DEFAULT '0', PRIMARY KEY (`org_id`,`object_id`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -936,7 +936,7 @@ CREATE TABLE `sponsoring_org` ( DROP TABLE IF EXISTS `tags`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `tags` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `tag_id` varchar(50) NOT NULL DEFAULT '', @@ -951,7 +951,7 @@ CREATE TABLE `tags` ( DROP TABLE IF EXISTS `tech_c`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `tech_c` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `pe_ro_id` int(10) unsigned NOT NULL DEFAULT '0', @@ -959,7 +959,7 @@ CREATE TABLE `tech_c` ( PRIMARY KEY (`pe_ro_id`,`object_id`), KEY `object_type` (`object_type`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -968,13 +968,13 @@ CREATE TABLE `tech_c` ( DROP TABLE IF EXISTS `upd_to`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `upd_to` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `upd_to` varchar(80) NOT NULL DEFAULT '', PRIMARY KEY (`upd_to`,`object_id`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -983,16 +983,16 @@ CREATE TABLE `upd_to` ( DROP TABLE IF EXISTS `update_lock`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `update_lock` ( `global_lock` int(11) NOT NULL, PRIMARY KEY (`global_lock`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; DROP TABLE IF EXISTS `transfer_update_lock`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `transfer_update_lock` ( `global_lock` int(11) NOT NULL, PRIMARY KEY (`global_lock`) @@ -1005,10 +1005,10 @@ CREATE TABLE `transfer_update_lock` ( DROP TABLE IF EXISTS `version`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `version` ( `version` varchar(80) DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -1017,11 +1017,11 @@ CREATE TABLE `version` ( DROP TABLE IF EXISTS `x509`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `x509` ( `keycert_id` int(10) unsigned NOT NULL DEFAULT '0', PRIMARY KEY (`keycert_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -1030,7 +1030,7 @@ CREATE TABLE `x509` ( DROP TABLE IF EXISTS `zone_c`; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; +/*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `zone_c` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', `pe_ro_id` int(10) unsigned NOT NULL DEFAULT '0', @@ -1038,7 +1038,7 @@ CREATE TABLE `zone_c` ( PRIMARY KEY (`pe_ro_id`,`object_id`), KEY `object_type` (`object_type`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; diff --git a/whois-commons/src/test/java/net/ripe/db/whois/common/dao/jdbc/DatabaseHelper.java b/whois-commons/src/test/java/net/ripe/db/whois/common/dao/jdbc/DatabaseHelper.java index 5946cdd478..fe2e0a52fb 100644 --- a/whois-commons/src/test/java/net/ripe/db/whois/common/dao/jdbc/DatabaseHelper.java +++ b/whois-commons/src/test/java/net/ripe/db/whois/common/dao/jdbc/DatabaseHelper.java @@ -217,7 +217,7 @@ public static void addGrsDatabases(final String... sourceNames) { static void setupDatabase(final JdbcTemplate jdbcTemplate, final String propertyBase, final String name, final String... sql) { final String dbName = dbBaseName + "_" + name; - jdbcTemplate.execute("CREATE DATABASE " + dbName + " DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci"); + jdbcTemplate.execute("CREATE DATABASE " + dbName + " DEFAULT CHARACTER SET utf8mb4 DEFAULT COLLATE utf8mb4_unicode_ci"); loadScripts(new JdbcTemplate(createDataSource(dbName)), sql); From 681a9b30ff079d2c447f8f22fbdce3b1828d9889 Mon Sep 17 00:00:00 2001 From: Ed Shryane Date: Tue, 11 Aug 2020 13:48:41 +0200 Subject: [PATCH 11/22] Created script to convert Whois schema to UTF8. --- .../src/main/resources/patch/whois-utf8.sql | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 whois-commons/src/main/resources/patch/whois-utf8.sql diff --git a/whois-commons/src/main/resources/patch/whois-utf8.sql b/whois-commons/src/main/resources/patch/whois-utf8.sql new file mode 100644 index 0000000000..10eac0adc0 --- /dev/null +++ b/whois-commons/src/main/resources/patch/whois-utf8.sql @@ -0,0 +1,99 @@ +-- Convert Whois schema to UTF8 +-- +-- TODO: [ES] how to track character set in object BLOB column in history and last tables? +-- Do we add a new 'utf8' tinyint / boolean column, or store full character set name? +-- Do we store *both* a UTF8 version *and* a latin-1 transliteration? + +-- TODO: [ES] update database and client defaults +-- [client] +-- default-character-set=utf8mb4 +-- [mysql] +-- default-character-set=utf8mb4 +-- [mysqld] +-- collation-server = utf8mb4_unicode_ci +-- init-connect='SET NAMES utf8mb4' +-- character-set-server = utf8mb4 +-- + +-- TODO: [ES] do we run this script separately on the master and slave(s) ? + +-- TODO: [ES] how much temporary disk space is needed for the migration? + +-- TODO: [ES] how much time is needed for the migration? +-- eg. history table = 52 mins, last table = 17 mins (tested on metal server) + +-- TODO: Also convert mirror databases, TEST, mailupdates etc. + +SET autocommit = 0; +START TRANSACTION; + +-- DATABASE +alter database WHOIS_UPDATE_RIPE character set = 'utf8mb4' collate = 'utf8mb4_general_ci'; + +-- TABLES +ALTER TABLE abuse_c CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE abuse_mailbox CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE admin_c CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE as_block CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE as_set CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE aut_num CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE auth CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE author CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE domain CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE ds_rdata CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE e_mail CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE filter_set CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE fingerpr CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE form CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE history CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE ifaddr CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE inaddr_arpa CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE inet6num CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE inet_rtr CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE inetnum CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE interface CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE irt CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE irt_nfy CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE key_cert CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE last CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE mbrs_by_ref CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE member_of CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE mnt_by CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE mnt_domains CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE mnt_irt CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE mnt_lower CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE mnt_nfy CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE mnt_ref CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE mnt_routes CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE mntner CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE names CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE nic_hdl CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE notify CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE nserver CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE org CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE org_name CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE organisation CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE organisation_id CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE peering_set CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE person_role CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE ping_hdl CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE poem CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE poetic_form CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE ref_nfy CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE route CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE route6 CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE route_set CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE rtr_set CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE serials CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE sponsoring_org CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE status CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE tags CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE tech_c CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE transfer_update_lock CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE upd_to CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE update_lock CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE version CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE x509 CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE zone_c CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; + +COMMIT; From 511c6ea745f24e13f4b97314b6f6c36d4346d2a9 Mon Sep 17 00:00:00 2001 From: Ed Shryane Date: Wed, 12 Aug 2020 13:37:47 +0200 Subject: [PATCH 12/22] Added conversion script for versions database --- .../main/resources/patch/versions-utf8.sql | 20 +++++++++++++++++++ .../src/main/resources/patch/whois-utf8.sql | 1 + 2 files changed, 21 insertions(+) create mode 100644 whois-commons/src/main/resources/patch/versions-utf8.sql diff --git a/whois-commons/src/main/resources/patch/versions-utf8.sql b/whois-commons/src/main/resources/patch/versions-utf8.sql new file mode 100644 index 0000000000..43361c8058 --- /dev/null +++ b/whois-commons/src/main/resources/patch/versions-utf8.sql @@ -0,0 +1,20 @@ +-- Convert Versions schema to UTF8 +-- + +-- TODO: [ES] How long to run in production? +-- Took 66 mins on a standalone metal server + +-- TODO: how much extra disk space is needed during the conversion? + +SET autocommit = 0; +START TRANSACTION; + +-- DATABASE +alter database VERSIONS_RIPE character set = 'utf8mb4' collate = 'utf8mb4_general_ci'; + +-- TABLES +ALTER TABLE object_reference CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE object_version CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE serials CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; + +COMMIT; diff --git a/whois-commons/src/main/resources/patch/whois-utf8.sql b/whois-commons/src/main/resources/patch/whois-utf8.sql index 10eac0adc0..6eaf127af2 100644 --- a/whois-commons/src/main/resources/patch/whois-utf8.sql +++ b/whois-commons/src/main/resources/patch/whois-utf8.sql @@ -21,6 +21,7 @@ -- TODO: [ES] how much time is needed for the migration? -- eg. history table = 52 mins, last table = 17 mins (tested on metal server) +-- Entire script ran in 82m12.701s on a standalone metal server -- TODO: Also convert mirror databases, TEST, mailupdates etc. From 213cc9556b8b06e6cfff5d59252f0136d01e3c94 Mon Sep 17 00:00:00 2001 From: Ed Shryane Date: Mon, 15 Mar 2021 17:32:16 +0100 Subject: [PATCH 13/22] Fixed tests following merge --- .../SyncUpdatesServiceTestIntegration.java | 77 +++++++++++++++++-- 1 file changed, 71 insertions(+), 6 deletions(-) diff --git a/whois-api/src/test/java/net/ripe/db/whois/api/rest/SyncUpdatesServiceTestIntegration.java b/whois-api/src/test/java/net/ripe/db/whois/api/rest/SyncUpdatesServiceTestIntegration.java index eeec4466fb..6fec408418 100644 --- a/whois-api/src/test/java/net/ripe/db/whois/api/rest/SyncUpdatesServiceTestIntegration.java +++ b/whois-api/src/test/java/net/ripe/db/whois/api/rest/SyncUpdatesServiceTestIntegration.java @@ -216,7 +216,6 @@ public void update_object_dryrun_parameter() { assertThat(response, containsString("***Info: Dry-run performed, no changes to the database have been made")); } - @Ignore("TODO: [ES] implement character substitution") @Test public void non_break_spaces_are_substituted_with_regular_space() { databaseHelper.addObject(PERSON_ANY1_TEST); @@ -728,7 +727,7 @@ public void post_url_encoded_data_with_non_latin1_address_error() { MediaType.valueOf("application/x-www-form-urlencoded; charset=UTF-8")), String.class); assertThat(response, containsString("***Error: Unrecognized source: INVALID")); - assertThat(response, containsString("address: Тверская улица,москва")); + assertThat(response, containsString("address: ???????? ?????,??????")); } @Test @@ -752,7 +751,6 @@ public void post_url_encoded_data_with_non_latin1_address() { containsString("address: Тверская улица,москва")); } - @Ignore("TODO: [ES] implement character substitution") @Test public void post_url_encoded_data_with_control_characters_address() { rpslObjectUpdateDao.createObject(RpslObject.parse(PERSON_ANY1_TEST)); @@ -774,6 +772,52 @@ public void post_url_encoded_data_with_control_characters_address() { containsString("address: Test???? Address")); } + @Test + public void post_url_encoded_data_with_latin1_email_address_converted_to_punycode() { + rpslObjectUpdateDao.createObject(RpslObject.parse(PERSON_ANY1_TEST)); + rpslObjectUpdateDao.createObject(RpslObject.parse(MNTNER_TEST_MNTNER)); + + final String response = RestTest.target(getPort(), "whois/syncupdates/test") + .request() + .post( Entity.entity("DATA=" + SyncUpdateUtils.encode( + "person: Test Person\n" + + "address: Zürich\n" + + "e-mail: no-reply@zürich.example\n" + + "phone: +31-6-123456\n" + + "nic-hdl: TP2-TEST\n" + + "mnt-by: mntner-mnt\n" + + "source: TEST\n" + + "password: emptypassword"), + MediaType.valueOf("application/x-www-form-urlencoded; charset=UTF-8")), String.class); + + assertThat(response, containsString("***Warning: Value changed due to conversion of IDN email address(es) into\n Punycode\n")); + assertThat(databaseHelper.lookupObject(ObjectType.PERSON, "TP2-TEST").toString(), + containsString("e-mail: no-reply@xn--zrich-kva.example")); + } + + @Test + public void post_url_encoded_data_with_cyrillic_email_address_converted_to_punycode() { + rpslObjectUpdateDao.createObject(RpslObject.parse(PERSON_ANY1_TEST)); + rpslObjectUpdateDao.createObject(RpslObject.parse(MNTNER_TEST_MNTNER)); + + final String response = RestTest.target(getPort(), "whois/syncupdates/test") + .request() + .post( Entity.entity("DATA=" + SyncUpdateUtils.encode( + "person: Test Person\n" + + "address: Moscow\n" + + "e-mail: no-reply@москва.ru\n" + + "phone: +31-6-123456\n" + + "nic-hdl: TP2-TEST\n" + + "mnt-by: mntner-mnt\n" + + "source: TEST\n" + + "password: emptypassword"), + MediaType.valueOf("application/x-www-form-urlencoded; charset=UTF-8")), String.class); + + assertThat(response, containsString("***Warning: Value changed due to conversion of IDN email address(es) into\n Punycode\n")); + assertThat(databaseHelper.lookupObject(ObjectType.PERSON, "TP2-TEST").toString(), + containsString("e-mail: no-reply@xn--80adxhks.ru")); + } + @Test public void post_multipart_data_with_non_latin1_address() { databaseHelper.addObject(PERSON_ANY1_TEST); @@ -819,10 +863,8 @@ public void post_multipart_data_with_control_characters_address() { .post(Entity.entity(multipart, multipart.getMediaType()), String.class); assertThat(response, containsString("Create SUCCEEDED: [person] TP2-TEST Test Person")); - - // TODO: [ES] control characters are *not* stripped out assertThat(databaseHelper.lookupObject(ObjectType.PERSON, "TP2-TEST").toString(), - containsString("address: Test\u000B\f\u007F\u008F Address")); + containsString("address: Тверская улица,москва")); } @Test @@ -870,6 +912,29 @@ public void post_multipart_data_with_latin1_non_ascii_address_latin1_encoded() { assertThat(databaseHelper.lookupObject(ObjectType.PERSON, "TP2-TEST").toString(), containsString("address: ÅçÅç")); } + @Test + public void post_multipart_data_with_latin1_email_address_converted_to_punycode() { + databaseHelper.addObject(PERSON_ANY1_TEST); + databaseHelper.addObject(MNTNER_TEST_MNTNER); + + final FormDataMultiPart multipart = new FormDataMultiPart() + .field("DATA", + "person: Test Person\n" + + "address: Zürich\n" + + "e-mail: no-reply@zürich.example\n" + + "phone: +31 6 12345678\n" + + "nic-hdl: TP2-TEST\n" + + "mnt-by: mntner-mnt\n" + + "source: TEST\n" + + "password: emptypassword") + .field("NEW", "yes"); + RestTest.target(getPort(), "whois/syncupdates/test") + .request() + .post(Entity.entity(multipart, new MediaType("multipart", "form-data", StandardCharsets.ISO_8859_1.displayName())), String.class); + + assertThat(databaseHelper.lookupObject(ObjectType.PERSON, "TP2-TEST").toString(), containsString("e-mail: no-reply@xn--zrich-kva.example")); + } + @Test public void create_person_with_filtered_source() { databaseHelper.addObject(PERSON_ANY1_TEST); From 5397a575eac58169834223ba36052dae002d3445 Mon Sep 17 00:00:00 2001 From: Ed Shryane Date: Mon, 22 Jan 2024 18:00:03 +0100 Subject: [PATCH 14/22] Merge branch 'master' into utf8 --- .../src/main/resources/patch/whois-utf8.sql | 1 - .../common/support/DummyWhoisClient.java | 87 ------------------- .../keycert/X509CertificateWrapper.java | 1 - 3 files changed, 89 deletions(-) delete mode 100644 whois-commons/src/test/java/net/ripe/db/whois/common/support/DummyWhoisClient.java diff --git a/whois-commons/src/main/resources/patch/whois-utf8.sql b/whois-commons/src/main/resources/patch/whois-utf8.sql index 6eaf127af2..0ad8448874 100644 --- a/whois-commons/src/main/resources/patch/whois-utf8.sql +++ b/whois-commons/src/main/resources/patch/whois-utf8.sql @@ -88,7 +88,6 @@ ALTER TABLE rtr_set CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ ALTER TABLE serials CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; ALTER TABLE sponsoring_org CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; ALTER TABLE status CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE tags CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; ALTER TABLE tech_c CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; ALTER TABLE transfer_update_lock CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; ALTER TABLE upd_to CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; diff --git a/whois-commons/src/test/java/net/ripe/db/whois/common/support/DummyWhoisClient.java b/whois-commons/src/test/java/net/ripe/db/whois/common/support/DummyWhoisClient.java deleted file mode 100644 index 7fc4f4d22f..0000000000 --- a/whois-commons/src/test/java/net/ripe/db/whois/common/support/DummyWhoisClient.java +++ /dev/null @@ -1,87 +0,0 @@ -package net.ripe.db.whois.common.support; - -import net.ripe.db.whois.common.aspects.RetryFor; -import org.apache.commons.io.IOUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.util.FileCopyUtils; - -import java.io.*; -import java.net.Socket; -import java.net.SocketException; -import java.net.SocketTimeoutException; -import java.nio.charset.Charset; - -public class DummyWhoisClient { - private static final Logger LOGGER = LoggerFactory.getLogger(DummyWhoisClient.class); - - private String host; - private int port; - - public static String query(final int port, final String query) { - for (int attempt = 1; attempt <= 3; attempt++) { - LOGGER.info("Query {} attempt {}", query, attempt); - DummyWhoisClient client = new DummyWhoisClient("127.0.0.1", port); - try { - return client.sendQuery(query); - } catch (IOException e) { - LOGGER.warn("Query {} attempt {} failed", query, attempt); - } - } - - throw new IllegalStateException("Unable to execute query"); - } - - public static String query(final int port, final String query, final int timeout) { - try { - return new DummyWhoisClient("127.0.0.1", port).sendQuery(query, Charset.forName("UTF-8"), timeout); - } catch (IOException e) { - throw new IllegalStateException("Unable to execute query"); - } - } - - - public DummyWhoisClient(int port) { - this.port = port; - this.host = "localhost"; - } - - public DummyWhoisClient(String host, int port) { - this.port = port; - this.host = host; - } - - public String sendQuery(String query) throws IOException { - return sendQuery(query, Charset.forName("UTF-8")); - } - - public String sendQuery(String query, Charset charset) throws IOException { - return sendQuery(query, charset, -1); - } - - @RetryFor(IOException.class) - public String sendQuery(final String query, final Charset charset, final int timeout) throws IOException { - final Socket socket = new Socket(host, port); - if (timeout > 0) { - socket.setSoTimeout(timeout); - } - - PrintWriter serverWriter = new PrintWriter(socket.getOutputStream(), true); - BufferedReader serverReader = new BufferedReader(new InputStreamReader(socket.getInputStream(), charset)); - StringWriter responseWriter = new StringWriter(); - - serverWriter.println(query); - - try { - FileCopyUtils.copy(serverReader, responseWriter); - } catch (SocketTimeoutException | SocketException ignored) { - LOGGER.warn("IO error", ignored); - } finally { - IOUtils.closeQuietly(serverWriter); - IOUtils.closeQuietly(serverReader); - IOUtils.closeQuietly(socket); - } - - return responseWriter.toString(); - } -} diff --git a/whois-update/src/main/java/net/ripe/db/whois/update/keycert/X509CertificateWrapper.java b/whois-update/src/main/java/net/ripe/db/whois/update/keycert/X509CertificateWrapper.java index 058eddc212..c15c2c5253 100644 --- a/whois-update/src/main/java/net/ripe/db/whois/update/keycert/X509CertificateWrapper.java +++ b/whois-update/src/main/java/net/ripe/db/whois/update/keycert/X509CertificateWrapper.java @@ -7,7 +7,6 @@ import net.ripe.db.whois.common.rpsl.RpslObject; import net.ripe.db.whois.common.rpsl.RpslObjectFilter; import org.bouncycastle.jce.provider.BouncyCastleProvider; -import org.bouncycastle.util.StreamParsingException; import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; From 44be90ba776d76c55da44256879a4242f7465e3d Mon Sep 17 00:00:00 2001 From: Ed Shryane Date: Wed, 27 Mar 2024 13:43:32 +0100 Subject: [PATCH 15/22] Add patch files for remaining DB schemas --- .../src/main/resources/patch/acl-utf8.sql | 21 +++++++++++++ .../main/resources/patch/internals-utf8.sql | 31 +++++++++++++++++++ .../main/resources/patch/mailupdates-utf8.sql | 13 ++++++++ .../src/main/resources/patch/nrtm-utf8.sql | 18 +++++++++++ 4 files changed, 83 insertions(+) create mode 100644 whois-commons/src/main/resources/patch/acl-utf8.sql create mode 100644 whois-commons/src/main/resources/patch/internals-utf8.sql create mode 100644 whois-commons/src/main/resources/patch/mailupdates-utf8.sql create mode 100644 whois-commons/src/main/resources/patch/nrtm-utf8.sql diff --git a/whois-commons/src/main/resources/patch/acl-utf8.sql b/whois-commons/src/main/resources/patch/acl-utf8.sql new file mode 100644 index 0000000000..cb6ba27c0b --- /dev/null +++ b/whois-commons/src/main/resources/patch/acl-utf8.sql @@ -0,0 +1,21 @@ +-- convert ACL schema to UTF8 + + +SET autocommit = 0; +START TRANSACTION; + +-- DATABASE +alter database ACL_RIPE character set = 'utf8mb4' collate = 'utf8mb4_general_ci'; + +-- TABLES +ALTER TABLE acl_denied CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE acl_event CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE acl_sso_denied CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE acl_sso_event CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE acl_limit CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE acl_proxy CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE override_users CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE apikeys CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE version CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; + +COMMIT; diff --git a/whois-commons/src/main/resources/patch/internals-utf8.sql b/whois-commons/src/main/resources/patch/internals-utf8.sql new file mode 100644 index 0000000000..fe0e3e223e --- /dev/null +++ b/whois-commons/src/main/resources/patch/internals-utf8.sql @@ -0,0 +1,31 @@ +-- convert INTERNALS schema to UTF8 + + + +SET autocommit = 0; +START TRANSACTION; + +-- DATABASE +alter database INTERNALS_RIPE character set = 'utf8mb4' collate = 'utf8mb4_general_ci'; + +-- TABLES +ALTER TABLE authoritative_resource CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE legacy_autnums CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE email_links CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE forgot_password_audit_log CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE default_maintainer_history CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE default_maintainer_sync_history CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE default_maintainer_in_progress CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE default_maintainer_sync CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE abuse_email CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE abuse_org_email CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE abuse_ticket CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE shedlock CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE non_auth_route CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE non_auth_route_unregistered_space CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE environment CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE outgoing_message CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE email_status CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE version CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; + +COMMIT; diff --git a/whois-commons/src/main/resources/patch/mailupdates-utf8.sql b/whois-commons/src/main/resources/patch/mailupdates-utf8.sql new file mode 100644 index 0000000000..125421362f --- /dev/null +++ b/whois-commons/src/main/resources/patch/mailupdates-utf8.sql @@ -0,0 +1,13 @@ +-- convert MAILUPDATES schema to UTF8 + +SET autocommit = 0; +START TRANSACTION; + +-- DATABASE +alter database MAILUPDATES_RIPE character set = 'utf8mb4' collate = 'utf8mb4_general_ci'; + +-- TABLES +ALTER TABLE mailupdates CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE version CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; + +COMMIT; diff --git a/whois-commons/src/main/resources/patch/nrtm-utf8.sql b/whois-commons/src/main/resources/patch/nrtm-utf8.sql new file mode 100644 index 0000000000..271cafb5de --- /dev/null +++ b/whois-commons/src/main/resources/patch/nrtm-utf8.sql @@ -0,0 +1,18 @@ +-- convert NRTM schema to UTF8 + +SET autocommit = 0; +START TRANSACTION; + +-- DATABASE +alter database NRTM_RIPE character set = 'utf8mb4' collate = 'utf8mb4_general_ci'; + +-- TABLES +ALTER TABLE version CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE source CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE version_info CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE snapshot_file CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE delta_file CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE notification_file CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; +ALTER TABLE key_pair CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; + +COMMIT; From 2e4e2cef6f755bb67c8497069e66d5650940a8f1 Mon Sep 17 00:00:00 2001 From: Ed Shryane Date: Wed, 27 Mar 2024 13:44:11 +0100 Subject: [PATCH 16/22] Specify UTF8 in all schemas (not latin1) --- .../src/main/resources/internals_schema.sql | 45 ++++++++++--------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/whois-commons/src/main/resources/internals_schema.sql b/whois-commons/src/main/resources/internals_schema.sql index 740d1fe600..75426fef12 100644 --- a/whois-commons/src/main/resources/internals_schema.sql +++ b/whois-commons/src/main/resources/internals_schema.sql @@ -7,19 +7,14 @@ CREATE TABLE `authoritative_resource` ( `resource` varchar(128) NOT NULL, PRIMARY KEY (`id`), KEY(`source`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; - -DROP TABLE IF EXISTS `version`; -CREATE TABLE `version` ( - `version` varchar(80) DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; DROP TABLE IF EXISTS `legacy_autnums`; CREATE TABLE `legacy_autnums` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, `autnum` varchar(16) NOT NULL, PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; DROP TABLE IF EXISTS `email_links`; CREATE TABLE `email_links` ( @@ -33,7 +28,7 @@ CREATE TABLE `email_links` ( `expired_by` varchar(256) DEFAULT NULL, PRIMARY KEY (`id`), UNIQUE KEY `unique_key` (`hash`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; DROP TABLE IF EXISTS `forgot_password_audit_log`; CREATE TABLE `forgot_password_audit_log` ( @@ -47,7 +42,7 @@ CREATE TABLE `forgot_password_audit_log` ( `user_sso_email` varchar(256) DEFAULT NULL, PRIMARY KEY (`id`), CONSTRAINT FOREIGN KEY (`hash`) REFERENCES `email_links` (`hash`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; DROP TABLE IF EXISTS `default_maintainer_history`; CREATE TABLE `default_maintainer_history` ( @@ -59,7 +54,7 @@ CREATE TABLE `default_maintainer_history` ( `email` varchar(256), `in_progress` tinyint(1) DEFAULT NULL, PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; DROP TABLE IF EXISTS `default_maintainer_sync_history`; CREATE TABLE `default_maintainer_sync_history` ( @@ -69,7 +64,7 @@ CREATE TABLE `default_maintainer_sync_history` ( `timestamp` timestamp NOT NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(), `email` varchar(256) NOT NULL, `is_synchronised` tinyint(1) DEFAULT 0,PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; DROP TABLE IF EXISTS `default_maintainer_in_progress`; @@ -83,7 +78,7 @@ CREATE TABLE `default_maintainer_in_progress` ( PRIMARY KEY (`id`), UNIQUE KEY `org_idx` (`org`), UNIQUE KEY `mntner_idx` (`mntner`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; DROP TABLE IF EXISTS `default_maintainer_sync`; CREATE TABLE `default_maintainer_sync` ( @@ -93,7 +88,7 @@ CREATE TABLE `default_maintainer_sync` ( `timestamp` timestamp NOT NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(), `email` varchar(256) NOT NULL, PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; DROP TABLE IF EXISTS `abuse_email`; CREATE TABLE `abuse_email` ( @@ -105,7 +100,7 @@ CREATE TABLE `abuse_email` ( `status` varchar(7) NOT NULL, `link_sent_at` datetime, PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; CREATE UNIQUE INDEX abuse_email_address_i on `abuse_email`(`address`); @@ -122,7 +117,7 @@ CREATE TABLE `abuse_org_email` ( `abuse_nic_hdl` varchar(30) NOT NULL, PRIMARY KEY (`id`), CONSTRAINT FOREIGN KEY (`email_id`) REFERENCES `abuse_email` (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; DROP TABLE IF EXISTS `abuse_ticket`; CREATE TABLE `abuse_ticket` ( @@ -135,7 +130,7 @@ CREATE TABLE `abuse_ticket` ( `ticket_type` char(1) NOT NULL, `state` char(1) NOT NULL, PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; CREATE INDEX abuse_ticket_org_id_i ON abuse_ticket(org_id); CREATE UNIQUE INDEX abuse_ticket_ticket_id_i on abuse_ticket(ticket_id); @@ -147,27 +142,27 @@ CREATE TABLE `shedlock` ( `locked_at` TIMESTAMP(3) NULL, `locked_by` VARCHAR(255), PRIMARY KEY (`name`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; DROP TABLE IF EXISTS `non_auth_route`; CREATE TABLE `non_auth_route` ( `object_pkey` VARCHAR(254) NOT NULL, `created_at` DATE NOT NULL, PRIMARY KEY (`object_pkey`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; DROP TABLE IF EXISTS `non_auth_route_unregistered_space`; CREATE TABLE `non_auth_route_unregistered_space` ( `object_pkey` VARCHAR(254) NOT NULL, `created_at` DATE NOT NULL, PRIMARY KEY (`object_pkey`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; DROP TABLE IF EXISTS `environment`; CREATE TABLE `environment` ( `name` varchar(8) NOT NULL, PRIMARY KEY (`name`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; DROP TABLE IF EXISTS `outgoing_message`; CREATE TABLE `outgoing_message` ( @@ -175,7 +170,7 @@ CREATE TABLE `outgoing_message` ( `email` varchar(80) NOT NULL, `last_update` datetime DEFAULT now(), PRIMARY KEY (`message_id`, `email`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; CREATE INDEX outgoing_message_email_i ON outgoing_message(email); @@ -185,4 +180,10 @@ CREATE TABLE `email_status` ( `status` varchar(120) NOT NULL, `last_update` datetime DEFAULT now(), PRIMARY KEY (`email`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +DROP TABLE IF EXISTS `version`; +CREATE TABLE `version` ( + `version` varchar(80) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + From 201eba5c8ad5209a3ac36dcfc0de4ca7daee1cc1 Mon Sep 17 00:00:00 2001 From: Ed Shryane Date: Wed, 27 Mar 2024 13:45:18 +0100 Subject: [PATCH 17/22] Corrected NRTM schema (put the drop table just before the create table; removed non-existant snapshot_object table; use 'utf8mb4' and not 'utf8'. --- .../src/main/resources/nrtm_schema.sql | 41 +++++++++---------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/whois-commons/src/main/resources/nrtm_schema.sql b/whois-commons/src/main/resources/nrtm_schema.sql index 5bbf64e37e..c59d6f1546 100644 --- a/whois-commons/src/main/resources/nrtm_schema.sql +++ b/whois-commons/src/main/resources/nrtm_schema.sql @@ -1,7 +1,7 @@ /*!40101 SET @OLD_CHARACTER_SET_CLIENT = @@CHARACTER_SET_CLIENT */; /*!40101 SET @OLD_CHARACTER_SET_RESULTS = @@CHARACTER_SET_RESULTS */; /*!40101 SET @OLD_COLLATION_CONNECTION = @@COLLATION_CONNECTION */; -/*!40101 SET NAMES utf8 */; +/*!40101 SET NAMES utf8mb4 */; /*!40103 SET @OLD_TIME_ZONE = @@TIME_ZONE */; /*!40103 SET TIME_ZONE = '+00:00' */; /*!40014 SET @OLD_UNIQUE_CHECKS = @@UNIQUE_CHECKS, UNIQUE_CHECKS = 0 */; @@ -9,21 +9,7 @@ /*!40101 SET @OLD_SQL_MODE = @@SQL_MODE, SQL_MODE = 'NO_AUTO_VALUE_ON_ZERO' */; /*!40111 SET @OLD_SQL_NOTES = @@SQL_NOTES, SQL_NOTES = 0 */; -DROP TABLE IF EXISTS `version`; -CREATE TABLE `version` -( - `version` varchar(80) -) ENGINE = InnoDB - DEFAULT CHARSET = utf8; - -DROP TABLE IF EXISTS `notification_file`; -DROP TABLE IF EXISTS `delta_file`; -DROP TABLE IF EXISTS `snapshot_file`; -DROP TABLE IF EXISTS `snapshot_object`; -DROP TABLE IF EXISTS `version_info`; DROP TABLE IF EXISTS `source`; -DROP TABLE IF EXISTS `key_pair`; - CREATE TABLE `source` ( `id` int unsigned NOT NULL AUTO_INCREMENT, @@ -31,8 +17,9 @@ CREATE TABLE `source` PRIMARY KEY (`id`), UNIQUE KEY `source__name_uk` (`name`) ) ENGINE = InnoDB - DEFAULT CHARSET = utf8; + DEFAULT CHARSET = utf8mb4; +DROP TABLE IF EXISTS `version_info`; CREATE TABLE `version_info` ( `id` int unsigned NOT NULL AUTO_INCREMENT, @@ -45,8 +32,9 @@ CREATE TABLE `version_info` PRIMARY KEY (`id`), UNIQUE KEY `version_info__session__source__version__type__uk` (`session_id`, `source_id`, `version`, `type`) ) ENGINE = InnoDB - DEFAULT CHARSET = utf8; + DEFAULT CHARSET = utf8mb4; +DROP TABLE IF EXISTS `snapshot_file`; CREATE TABLE `snapshot_file` ( `id` int unsigned NOT NULL AUTO_INCREMENT, @@ -59,8 +47,9 @@ CREATE TABLE `snapshot_file` UNIQUE KEY `snapshot_file__name__uk` (`name`), CONSTRAINT `snapshot_file__version_id__fk` FOREIGN KEY (`version_id`) REFERENCES `version_info` (`id`) ) ENGINE = InnoDB - DEFAULT CHARSET = utf8; + DEFAULT CHARSET = utf8mb4; +DROP TABLE IF EXISTS `delta_file`; CREATE TABLE `delta_file` ( `id` int unsigned NOT NULL AUTO_INCREMENT, @@ -73,8 +62,9 @@ CREATE TABLE `delta_file` UNIQUE KEY `delta_file__name__uk` (`name`), CONSTRAINT `delta_file__version_id__fk` FOREIGN KEY (`version_id`) REFERENCES `version_info` (`id`) ) ENGINE = InnoDB - DEFAULT CHARSET = utf8; + DEFAULT CHARSET = utf8mb4; +DROP TABLE IF EXISTS `notification_file`; CREATE TABLE `notification_file` ( `id` int unsigned NOT NULL AUTO_INCREMENT, @@ -84,9 +74,9 @@ CREATE TABLE `notification_file` PRIMARY KEY (`id`), CONSTRAINT `notification_file__version_id__fk` FOREIGN KEY (`version_id`) REFERENCES `version_info` (`id`) ) ENGINE = InnoDB - DEFAULT CHARSET = utf8; - + DEFAULT CHARSET = utf8mb4; +DROP TABLE IF EXISTS `key_pair`; CREATE TABLE `key_pair` ( `id` int unsigned NOT NULL AUTO_INCREMENT, @@ -98,7 +88,14 @@ CREATE TABLE `key_pair` UNIQUE KEY `public_key_name_uk` (`public_key`), PRIMARY KEY (`id`) ) ENGINE = InnoDB - DEFAULT CHARSET = utf8; + DEFAULT CHARSET = utf8mb4; + +DROP TABLE IF EXISTS `version`; +CREATE TABLE `version` +( + `version` varchar(80) +) ENGINE = InnoDB + DEFAULT CHARSET = utf8mb4; /*!40103 SET TIME_ZONE = @OLD_TIME_ZONE */; From bc221044efdbd40e59a921a126219dfdbe749aea Mon Sep 17 00:00:00 2001 From: Ed Shryane Date: Wed, 27 Mar 2024 13:46:01 +0100 Subject: [PATCH 18/22] Removed redundant character set and collation from Whois auth table VARCHAR column --- whois-commons/src/main/resources/whois_schema.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/whois-commons/src/main/resources/whois_schema.sql b/whois-commons/src/main/resources/whois_schema.sql index dbb4768387..54b902dd65 100644 --- a/whois-commons/src/main/resources/whois_schema.sql +++ b/whois-commons/src/main/resources/whois_schema.sql @@ -121,7 +121,7 @@ DROP TABLE IF EXISTS `auth`; /*!40101 SET character_set_client = utf8mb4 */; CREATE TABLE `auth` ( `object_id` int(10) unsigned NOT NULL DEFAULT '0', - `auth` varchar(90) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', + `auth` varchar(90) NOT NULL DEFAULT '', `object_type` tinyint(3) unsigned NOT NULL DEFAULT '0', PRIMARY KEY (`auth`,`object_id`), KEY `object_id` (`object_id`) From 90c516d8bb178b1100c1ba170ce7d6b26d22caad Mon Sep 17 00:00:00 2001 From: Ed Shryane Date: Wed, 27 Mar 2024 15:42:18 +0100 Subject: [PATCH 19/22] schema charset is utf8mb4 not latin1 --- whois-commons/src/main/resources/acl_schema.sql | 6 +++--- whois-commons/src/main/resources/versions_schema.sql | 6 +++--- whois-commons/src/main/resources/whois_schema.sql | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/whois-commons/src/main/resources/acl_schema.sql b/whois-commons/src/main/resources/acl_schema.sql index 0da2304150..d941d0c659 100644 --- a/whois-commons/src/main/resources/acl_schema.sql +++ b/whois-commons/src/main/resources/acl_schema.sql @@ -54,7 +54,7 @@ CREATE TABLE `acl_sso_denied` ( `comment` text, `denied_date` date NOT NULL DEFAULT '0000-00-00', PRIMARY KEY (`sso_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -70,7 +70,7 @@ CREATE TABLE `acl_sso_event` ( `daily_limit` int(11) NOT NULL, `event_type` varchar(20) NOT NULL, PRIMARY KEY (`sso_id`,`event_time`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -130,7 +130,7 @@ CREATE TABLE `apikeys` ( `apikey` varchar(128) NOT NULL, `uri_prefix` varchar(128) NOT NULL, `comment` varchar(255) NOT NULL -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- diff --git a/whois-commons/src/main/resources/versions_schema.sql b/whois-commons/src/main/resources/versions_schema.sql index c29dcbda18..b051b6ec19 100644 --- a/whois-commons/src/main/resources/versions_schema.sql +++ b/whois-commons/src/main/resources/versions_schema.sql @@ -13,7 +13,7 @@ CREATE TABLE `object_version` ( KEY `from_timestamp` (`from_timestamp`), KEY `to_timestamp` (`to_timestamp`), KEY `revision` (`revision`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; DROP TABLE IF EXISTS `object_reference`; CREATE TABLE `object_reference` ( @@ -26,10 +26,10 @@ CREATE TABLE `object_reference` ( KEY `to_version` (`to_version`), CONSTRAINT FOREIGN KEY (`from_version`) REFERENCES `object_version` (`id`), CONSTRAINT FOREIGN KEY (`to_version`) REFERENCES `object_version` (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; DROP TABLE IF EXISTS `serials`; CREATE TABLE `serials` ( `serial_id` int(11) NOT NULL, PRIMARY KEY (`serial_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; diff --git a/whois-commons/src/main/resources/whois_schema.sql b/whois-commons/src/main/resources/whois_schema.sql index 54b902dd65..fc6f69bc6f 100644 --- a/whois-commons/src/main/resources/whois_schema.sql +++ b/whois-commons/src/main/resources/whois_schema.sql @@ -203,7 +203,7 @@ CREATE TABLE `status` ( `object_type` tinyint(3) unsigned NOT NULL DEFAULT '0', PRIMARY KEY (`status`,`object_id`), KEY `object_id` (`object_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; -- -- Table structure for table `filter_set` @@ -981,7 +981,7 @@ DROP TABLE IF EXISTS `transfer_update_lock`; CREATE TABLE `transfer_update_lock` ( `global_lock` int(11) NOT NULL, PRIMARY KEY (`global_lock`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -- From 1ed092a3d474b32d8ff2e1e960af4a01e2ebeb6b Mon Sep 17 00:00:00 2001 From: Ed Shryane Date: Tue, 29 Oct 2024 19:36:13 +0100 Subject: [PATCH 20/22] remove obsolete patch files (database schema is already converted to UTF8). --- .../src/main/resources/patch/acl-utf8.sql | 21 ------------------- .../main/resources/patch/mailupdates-utf8.sql | 13 ------------ .../src/main/resources/patch/nrtm-utf8.sql | 18 ---------------- 3 files changed, 52 deletions(-) delete mode 100644 whois-commons/src/main/resources/patch/acl-utf8.sql delete mode 100644 whois-commons/src/main/resources/patch/mailupdates-utf8.sql delete mode 100644 whois-commons/src/main/resources/patch/nrtm-utf8.sql diff --git a/whois-commons/src/main/resources/patch/acl-utf8.sql b/whois-commons/src/main/resources/patch/acl-utf8.sql deleted file mode 100644 index cb6ba27c0b..0000000000 --- a/whois-commons/src/main/resources/patch/acl-utf8.sql +++ /dev/null @@ -1,21 +0,0 @@ --- convert ACL schema to UTF8 - - -SET autocommit = 0; -START TRANSACTION; - --- DATABASE -alter database ACL_RIPE character set = 'utf8mb4' collate = 'utf8mb4_general_ci'; - --- TABLES -ALTER TABLE acl_denied CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE acl_event CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE acl_sso_denied CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE acl_sso_event CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE acl_limit CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE acl_proxy CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE override_users CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE apikeys CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE version CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; - -COMMIT; diff --git a/whois-commons/src/main/resources/patch/mailupdates-utf8.sql b/whois-commons/src/main/resources/patch/mailupdates-utf8.sql deleted file mode 100644 index 125421362f..0000000000 --- a/whois-commons/src/main/resources/patch/mailupdates-utf8.sql +++ /dev/null @@ -1,13 +0,0 @@ --- convert MAILUPDATES schema to UTF8 - -SET autocommit = 0; -START TRANSACTION; - --- DATABASE -alter database MAILUPDATES_RIPE character set = 'utf8mb4' collate = 'utf8mb4_general_ci'; - --- TABLES -ALTER TABLE mailupdates CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE version CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; - -COMMIT; diff --git a/whois-commons/src/main/resources/patch/nrtm-utf8.sql b/whois-commons/src/main/resources/patch/nrtm-utf8.sql deleted file mode 100644 index 271cafb5de..0000000000 --- a/whois-commons/src/main/resources/patch/nrtm-utf8.sql +++ /dev/null @@ -1,18 +0,0 @@ --- convert NRTM schema to UTF8 - -SET autocommit = 0; -START TRANSACTION; - --- DATABASE -alter database NRTM_RIPE character set = 'utf8mb4' collate = 'utf8mb4_general_ci'; - --- TABLES -ALTER TABLE version CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE source CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE version_info CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE snapshot_file CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE delta_file CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE notification_file CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE key_pair CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; - -COMMIT; From 84dc093fd92bee85d7dce443cbe3d397458d2a25 Mon Sep 17 00:00:00 2001 From: Ed Shryane Date: Tue, 29 Oct 2024 19:38:45 +0100 Subject: [PATCH 21/22] remove obsolete patch files --- .../main/resources/patch/internals-utf8.sql | 31 ------ .../main/resources/patch/versions-utf8.sql | 20 ---- .../src/main/resources/patch/whois-utf8.sql | 99 ------------------- 3 files changed, 150 deletions(-) delete mode 100644 whois-commons/src/main/resources/patch/internals-utf8.sql delete mode 100644 whois-commons/src/main/resources/patch/versions-utf8.sql delete mode 100644 whois-commons/src/main/resources/patch/whois-utf8.sql diff --git a/whois-commons/src/main/resources/patch/internals-utf8.sql b/whois-commons/src/main/resources/patch/internals-utf8.sql deleted file mode 100644 index fe0e3e223e..0000000000 --- a/whois-commons/src/main/resources/patch/internals-utf8.sql +++ /dev/null @@ -1,31 +0,0 @@ --- convert INTERNALS schema to UTF8 - - - -SET autocommit = 0; -START TRANSACTION; - --- DATABASE -alter database INTERNALS_RIPE character set = 'utf8mb4' collate = 'utf8mb4_general_ci'; - --- TABLES -ALTER TABLE authoritative_resource CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE legacy_autnums CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE email_links CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE forgot_password_audit_log CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE default_maintainer_history CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE default_maintainer_sync_history CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE default_maintainer_in_progress CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE default_maintainer_sync CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE abuse_email CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE abuse_org_email CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE abuse_ticket CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE shedlock CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE non_auth_route CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE non_auth_route_unregistered_space CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE environment CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE outgoing_message CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE email_status CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE version CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; - -COMMIT; diff --git a/whois-commons/src/main/resources/patch/versions-utf8.sql b/whois-commons/src/main/resources/patch/versions-utf8.sql deleted file mode 100644 index 43361c8058..0000000000 --- a/whois-commons/src/main/resources/patch/versions-utf8.sql +++ /dev/null @@ -1,20 +0,0 @@ --- Convert Versions schema to UTF8 --- - --- TODO: [ES] How long to run in production? --- Took 66 mins on a standalone metal server - --- TODO: how much extra disk space is needed during the conversion? - -SET autocommit = 0; -START TRANSACTION; - --- DATABASE -alter database VERSIONS_RIPE character set = 'utf8mb4' collate = 'utf8mb4_general_ci'; - --- TABLES -ALTER TABLE object_reference CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE object_version CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE serials CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; - -COMMIT; diff --git a/whois-commons/src/main/resources/patch/whois-utf8.sql b/whois-commons/src/main/resources/patch/whois-utf8.sql deleted file mode 100644 index 0ad8448874..0000000000 --- a/whois-commons/src/main/resources/patch/whois-utf8.sql +++ /dev/null @@ -1,99 +0,0 @@ --- Convert Whois schema to UTF8 --- --- TODO: [ES] how to track character set in object BLOB column in history and last tables? --- Do we add a new 'utf8' tinyint / boolean column, or store full character set name? --- Do we store *both* a UTF8 version *and* a latin-1 transliteration? - --- TODO: [ES] update database and client defaults --- [client] --- default-character-set=utf8mb4 --- [mysql] --- default-character-set=utf8mb4 --- [mysqld] --- collation-server = utf8mb4_unicode_ci --- init-connect='SET NAMES utf8mb4' --- character-set-server = utf8mb4 --- - --- TODO: [ES] do we run this script separately on the master and slave(s) ? - --- TODO: [ES] how much temporary disk space is needed for the migration? - --- TODO: [ES] how much time is needed for the migration? --- eg. history table = 52 mins, last table = 17 mins (tested on metal server) --- Entire script ran in 82m12.701s on a standalone metal server - --- TODO: Also convert mirror databases, TEST, mailupdates etc. - -SET autocommit = 0; -START TRANSACTION; - --- DATABASE -alter database WHOIS_UPDATE_RIPE character set = 'utf8mb4' collate = 'utf8mb4_general_ci'; - --- TABLES -ALTER TABLE abuse_c CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE abuse_mailbox CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE admin_c CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE as_block CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE as_set CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE aut_num CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE auth CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE author CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE domain CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE ds_rdata CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE e_mail CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE filter_set CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE fingerpr CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE form CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE history CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE ifaddr CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE inaddr_arpa CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE inet6num CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE inet_rtr CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE inetnum CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE interface CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE irt CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE irt_nfy CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE key_cert CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE last CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE mbrs_by_ref CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE member_of CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE mnt_by CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE mnt_domains CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE mnt_irt CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE mnt_lower CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE mnt_nfy CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE mnt_ref CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE mnt_routes CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE mntner CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE names CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE nic_hdl CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE notify CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE nserver CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE org CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE org_name CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE organisation CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE organisation_id CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE peering_set CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE person_role CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE ping_hdl CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE poem CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE poetic_form CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE ref_nfy CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE route CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE route6 CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE route_set CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE rtr_set CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE serials CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE sponsoring_org CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE status CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE tech_c CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE transfer_update_lock CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE upd_to CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE update_lock CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE version CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE x509 CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; -ALTER TABLE zone_c CONVERT TO CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_ci'; - -COMMIT; From 1b194f3882cd45912cc021ea9ce7363857f49751 Mon Sep 17 00:00:00 2001 From: Ed Shryane Date: Tue, 29 Oct 2024 21:46:05 +0100 Subject: [PATCH 22/22] fixed test --- .../db/whois/query/integration/SimpleTestIntegration.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/whois-query/src/test/java/net/ripe/db/whois/query/integration/SimpleTestIntegration.java b/whois-query/src/test/java/net/ripe/db/whois/query/integration/SimpleTestIntegration.java index 22869bc7a7..7b7f02b6b5 100644 --- a/whois-query/src/test/java/net/ripe/db/whois/query/integration/SimpleTestIntegration.java +++ b/whois-query/src/test/java/net/ripe/db/whois/query/integration/SimpleTestIntegration.java @@ -8,7 +8,6 @@ import net.ripe.db.whois.common.rpsl.RpslObject; import net.ripe.db.whois.common.support.TelnetWhoisClient; import net.ripe.db.whois.query.QueryMessages; -import net.ripe.db.whois.query.QueryServer; import net.ripe.db.whois.query.support.AbstractQueryIntegrationTest; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; @@ -510,7 +509,7 @@ public void query_utf8_encoded_object() throws Exception { "descr: 66121 Saarbrücken\n" + "' USING utf8) WHERE pkey = '117.80.81.in-addr.arpa'"); - final String response = TelnetWhoisClient.queryLocalhost(QueryServer.port, "117.80.81.in-addr.arpa"); + final String response = TelnetWhoisClient.queryLocalhost(queryServer.getPort(), "117.80.81.in-addr.arpa"); // 0x00FC is UTF-8 representation of u-umlaut (2 bytes) assertThat(response, containsString("Saarbr\u00FCcken")); @@ -526,7 +525,7 @@ public void query_latin1_encoded_object_in_utf8_table() throws Exception { "descr: 66121 Saarbrücken" + "' WHERE pkey = '117.80.81.in-addr.arpa'"); - final String response = TelnetWhoisClient.queryLocalhost(QueryServer.port, "117.80.81.in-addr.arpa"); + final String response = TelnetWhoisClient.queryLocalhost(queryServer.getPort(), "117.80.81.in-addr.arpa"); assertThat(response, containsString("Saarbr\u00FCcken")); }