Skip to content

Commit

Permalink
Fix serialization of long (> 64k) strings, fixes apache#155
Browse files Browse the repository at this point in the history
  • Loading branch information
@gnodet
gnodet committed Oct 26, 2020
1 parent 4cec95e commit e33f3fc
Show file tree
Hide file tree
Showing 2 changed files with 173 additions and 24 deletions.
146 changes: 122 additions & 24 deletions common/src/main/java/org/jboss/fuse/mvnd/common/Message.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,11 @@

import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.EOFException;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.io.UTFDataFormatException;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
Expand Down Expand Up @@ -66,23 +68,23 @@ public long timestamp() {
static void writeStringList(DataOutputStream output, List<String> value) throws IOException {
output.writeInt(value.size());
for (String v : value) {
output.writeUTF(v);
writeUTF(output, v);
}
}

static void writeStringMap(DataOutputStream output, Map<String, String> value) throws IOException {
output.writeInt(value.size());
for (Map.Entry<String, String> e : value.entrySet()) {
output.writeUTF(e.getKey());
output.writeUTF(e.getValue());
writeUTF(output, e.getKey());
writeUTF(output, e.getValue());
}
}

static List<String> readStringList(DataInputStream input) throws IOException {
ArrayList<String> l = new ArrayList<>();
int nb = input.readInt();
for (int i = 0; i < nb; i++) {
l.add(input.readUTF());
l.add(readUTF(input));
}
return l;
}
Expand All @@ -91,13 +93,109 @@ static Map<String, String> readStringMap(DataInputStream input) throws IOExcepti
LinkedHashMap<String, String> m = new LinkedHashMap<>();
int nb = input.readInt();
for (int i = 0; i < nb; i++) {
String k = input.readUTF();
String v = input.readUTF();
String k = readUTF(input);
String v = readUTF(input);
m.put(k, v);
}
return m;
}

private static final String INVALID_BYTE = "Invalid byte";
private static final int UTF_BUFS_CHAR_CNT = 256;
private static final int UTF_BUFS_BYTE_CNT = UTF_BUFS_CHAR_CNT * 3;
private static final ThreadLocal<byte[]> BUF_TLS = ThreadLocal.withInitial(() -> new byte[UTF_BUFS_BYTE_CNT]);

static String readUTF(DataInputStream input) throws IOException {
byte[] byteBuf = BUF_TLS.get();
int len = input.readInt();
final char[] chars = new char[len];
int i = 0, cnt = 0, charIdx = 0;
while (charIdx < len) {
if (i == cnt) {
cnt = input.read(byteBuf, 0, Math.min(UTF_BUFS_BYTE_CNT, len - charIdx));
if (cnt < 0) {
throw new EOFException();
}
i = 0;
}
final int a = byteBuf[i++] & 0xff;
if (a < 0x80) {
// low bit clear
chars[charIdx++] = (char) a;
} else if (a < 0xc0) {
throw new UTFDataFormatException(INVALID_BYTE);
} else if (a < 0xe0) {
if (i == cnt) {
cnt = input.read(byteBuf, 0, Math.min(UTF_BUFS_BYTE_CNT, len - charIdx));
if (cnt < 0) {
throw new EOFException();
}
i = 0;
}
final int b = byteBuf[i++] & 0xff;
if ((b & 0xc0) != 0x80) {
throw new UTFDataFormatException(INVALID_BYTE);
}
chars[charIdx++] = (char) ((a & 0x1f) << 6 | b & 0x3f);
} else if (a < 0xf0) {
if (i == cnt) {
cnt = input.read(byteBuf, 0, Math.min(UTF_BUFS_BYTE_CNT, len - charIdx));
if (cnt < 0) {
throw new EOFException();
}
i = 0;
}
final int b = byteBuf[i++] & 0xff;
if ((b & 0xc0) != 0x80) {
throw new UTFDataFormatException(INVALID_BYTE);
}
if (i == cnt) {
cnt = input.read(byteBuf, 0, Math.min(UTF_BUFS_BYTE_CNT, len - charIdx));
if (cnt < 0) {
throw new EOFException();
}
i = 0;
}
final int c = byteBuf[i++] & 0xff;
if ((c & 0xc0) != 0x80) {
throw new UTFDataFormatException(INVALID_BYTE);
}
chars[charIdx++] = (char) ((a & 0x0f) << 12 | (b & 0x3f) << 6 | c & 0x3f);
} else {
throw new UTFDataFormatException(INVALID_BYTE);
}
}
return String.valueOf(chars);
}

static void writeUTF(DataOutputStream output, String s) throws IOException {
byte[] byteBuf = BUF_TLS.get();
final int length = s.length();
output.writeInt(length);
int strIdx = 0;
int byteIdx = 0;
while (strIdx < length) {
final char c = s.charAt(strIdx++);
if (c > 0 && c <= 0x7f) {
byteBuf[byteIdx++] = (byte) c;
} else if (c <= 0x07ff) {
byteBuf[byteIdx++] = (byte) (0xc0 | 0x1f & c >> 6);
byteBuf[byteIdx++] = (byte) (0x80 | 0x3f & c);
} else {
byteBuf[byteIdx++] = (byte) (0xe0 | 0x0f & c >> 12);
byteBuf[byteIdx++] = (byte) (0x80 | 0x3f & c >> 6);
byteBuf[byteIdx++] = (byte) (0x80 | 0x3f & c);
}
if (byteIdx > UTF_BUFS_BYTE_CNT - 4) {
output.write(byteBuf, 0, byteIdx);
byteIdx = 0;
}
}
if (byteIdx > 0) {
output.write(byteBuf, 0, byteIdx);
}
}

public static class BuildRequest extends Message {
final List<String> args;
final String workingDir;
Expand All @@ -106,8 +204,8 @@ public static class BuildRequest extends Message {

public static Message read(DataInputStream input) throws IOException {
List<String> args = readStringList(input);
String workingDir = input.readUTF();
String projectDir = input.readUTF();
String workingDir = readUTF(input);
String projectDir = readUTF(input);
Map<String, String> env = readStringMap(input);
return new BuildRequest(args, workingDir, projectDir, env);
}
Expand Down Expand Up @@ -148,8 +246,8 @@ public String toString() {
public void write(DataOutputStream output) throws IOException {
output.write(BUILD_REQUEST);
writeStringList(output, args);
output.writeUTF(workingDir);
output.writeUTF(projectDir);
writeUTF(output, workingDir);
writeUTF(output, projectDir);
writeStringMap(output, env);
}
}
Expand All @@ -160,9 +258,9 @@ public static class BuildException extends Message {
final String stackTrace;

public static Message read(DataInputStream input) throws IOException {
String message = input.readUTF();
String className = input.readUTF();
String stackTrace = input.readUTF();
String message = readUTF(input);
String className = readUTF(input);
String stackTrace = readUTF(input);
return new BuildException(message, className, stackTrace);
}

Expand Down Expand Up @@ -206,9 +304,9 @@ public String toString() {
@Override
public void write(DataOutputStream output) throws IOException {
output.write(BUILD_EXCEPTION);
output.writeUTF(message);
output.writeUTF(className);
output.writeUTF(stackTrace);
writeUTF(output, message);
writeUTF(output, className);
writeUTF(output, stackTrace);
}
}

Expand All @@ -223,8 +321,8 @@ public enum Type {

public static Message read(DataInputStream input) throws IOException {
BuildEvent.Type type = BuildEvent.Type.values()[input.read()];
String projectId = input.readUTF();
String display = input.readUTF();
String projectId = readUTF(input);
String display = readUTF(input);
return new BuildEvent(type, projectId, display);
}

Expand Down Expand Up @@ -259,8 +357,8 @@ public String toString() {
public void write(DataOutputStream output) throws IOException {
output.write(BUILD_EVENT);
output.write(type.ordinal());
output.writeUTF(projectId);
output.writeUTF(display);
writeUTF(output, projectId);
writeUTF(output, display);
}
}

Expand All @@ -269,8 +367,8 @@ public static class BuildMessage extends Message {
final String message;

public static Message read(DataInputStream input) throws IOException {
String projectId = input.readUTF();
String message = input.readUTF();
String projectId = readUTF(input);
String message = readUTF(input);
return new BuildMessage(projectId.isEmpty() ? null : projectId, message);
}

Expand Down Expand Up @@ -298,8 +396,8 @@ public String toString() {
@Override
public void write(DataOutputStream output) throws IOException {
output.write(BUILD_MESSAGE);
output.writeUTF(projectId != null ? projectId : "");
output.writeUTF(message);
writeUTF(output, projectId != null ? projectId : "");
writeUTF(output, message);
}
}

Expand Down
51 changes: 51 additions & 0 deletions common/src/test/java/org/jboss/fuse/mvnd/common/MessageTest.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
/*
* Copyright 2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jboss.fuse.mvnd.common;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import org.junit.jupiter.api.Test;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;

public class MessageTest {

@Test
public void testBigMessage() throws IOException {
StringBuilder stringToWrite = new StringBuilder();
for (int i = 0; i < 66000; ++i) {
stringToWrite.append("a");
}
Message msg = new Message.BuildMessage("project", stringToWrite.toString());
ByteArrayOutputStream baos = new ByteArrayOutputStream();
try (DataOutputStream daos = new DataOutputStream(baos)) {
msg.write(daos);
}

Message msg2;
ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
try (DataInputStream dis = new DataInputStream(bais)) {
msg2 = Message.read(dis);
}

assertTrue(msg2 instanceof Message.BuildMessage);
assertEquals(stringToWrite, ((Message.BuildMessage) msg2).getMessage());
}
}

0 comments on commit e33f3fc

Please sign in to comment.