Skip to content

Commit

Permalink
Completed all the Primitive OI for all numeric types, with unit tests.
Browse files Browse the repository at this point in the history
  • Loading branch information
Roberto Congiu committed Jan 22, 2014
1 parent 002c53a commit 3788ede
Show file tree
Hide file tree
Showing 8 changed files with 328 additions and 15 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,12 @@
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.openx.data.jsonserde.objectinspector.primitive.JavaStringByteObjectInspector;
import org.openx.data.jsonserde.objectinspector.primitive.JavaStringDoubleObjectInspector;
import org.openx.data.jsonserde.objectinspector.primitive.JavaStringFloatObjectInspector;
import org.openx.data.jsonserde.objectinspector.primitive.JavaStringIntObjectInspector;
import org.openx.data.jsonserde.objectinspector.primitive.JavaStringLongObjectInspector;
import org.openx.data.jsonserde.objectinspector.primitive.JavaStringShortObjectInspector;

/**
*
Expand Down Expand Up @@ -167,8 +171,12 @@ public static JsonMapObjectInspector getJsonMapObjectInspector(
= new EnumMap<PrimitiveCategory, AbstractPrimitiveJavaObjectInspector>(PrimitiveCategory.class);

static {
primitiveOICache.put(PrimitiveCategory.BYTE, new JavaStringByteObjectInspector());
primitiveOICache.put(PrimitiveCategory.SHORT, new JavaStringShortObjectInspector());
primitiveOICache.put(PrimitiveCategory.INT, new JavaStringIntObjectInspector());
primitiveOICache.put(PrimitiveCategory.LONG, new JavaStringLongObjectInspector());
primitiveOICache.put(PrimitiveCategory.FLOAT, new JavaStringFloatObjectInspector());
primitiveOICache.put(PrimitiveCategory.DOUBLE, new JavaStringDoubleObjectInspector());
}

/**
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
/*======================================================================*
* Copyright (c) 2011, OpenX Technologies, Inc. All rights reserved. *
* *
* Licensed under the New BSD License (the "License"); you may not use *
* this file except in compliance with the License. Unless required *
* by applicable law or agreed to in writing, software distributed *
* under the License is distributed on an "AS IS" BASIS, WITHOUT *
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. *
* See the License for the specific language governing permissions and *
* limitations under the License. See accompanying LICENSE file. *
*======================================================================*/

package org.openx.data.jsonserde.objectinspector.primitive;

import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableByteObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableIntObjectInspector;
import org.apache.hadoop.io.IntWritable;

/**
*
* @author rcongiu
*/
public class JavaStringByteObjectInspector extends AbstractPrimitiveJavaObjectInspector
implements SettableByteObjectInspector {

public JavaStringByteObjectInspector() {
super(PrimitiveObjectInspectorUtils.longTypeEntry);
}

@Override
public Object getPrimitiveWritableObject(Object o) {
if(o == null) return null;

if(o instanceof String) {
return new IntWritable(Integer.parseInt((String)o));
} else {
return new IntWritable(((Integer) o).intValue());
}
}

@Override
public byte get(Object o) {
if(o instanceof String) {
return Byte.parseByte((String)o);
} else {
return ((Byte) o).byteValue();
}
}

@Override
public Object create(byte value) {
return Byte.valueOf(value);
}

@Override
public Object set(Object o, byte value) {
return Byte.valueOf(value);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
/*======================================================================*
* Copyright (c) 2011, OpenX Technologies, Inc. All rights reserved. *
* *
* Licensed under the New BSD License (the "License"); you may not use *
* this file except in compliance with the License. Unless required *
* by applicable law or agreed to in writing, software distributed *
* under the License is distributed on an "AS IS" BASIS, WITHOUT *
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. *
* See the License for the specific language governing permissions and *
* limitations under the License. See accompanying LICENSE file. *
*======================================================================*/

package org.openx.data.jsonserde.objectinspector.primitive;

import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableDoubleObjectInspector;
import org.apache.hadoop.io.DoubleWritable;

/**
*
* @author rcongiu
*/
public class JavaStringDoubleObjectInspector extends AbstractPrimitiveJavaObjectInspector
implements SettableDoubleObjectInspector {

public JavaStringDoubleObjectInspector() {
super(PrimitiveObjectInspectorUtils.doubleTypeEntry);
}

@Override
public Object getPrimitiveWritableObject(Object o) {
if(o == null) return null;

if(o instanceof String) {
return new DoubleWritable(Double.parseDouble((String)o));
} else {
return new DoubleWritable(((Double) o).doubleValue());
}
}

@Override
public double get(Object o) {

if(o instanceof String) {
return Double.parseDouble((String)o);
} else {
return (((Double) o).doubleValue());
}
}

@Override
public Object create(double value) {
return Double.valueOf(value);
}

@Override
public Object set(Object o, double value) {
return Double.valueOf(value);
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
/*======================================================================*
* Copyright (c) 2011, OpenX Technologies, Inc. All rights reserved. *
* *
* Licensed under the New BSD License (the "License"); you may not use *
* this file except in compliance with the License. Unless required *
* by applicable law or agreed to in writing, software distributed *
* under the License is distributed on an "AS IS" BASIS, WITHOUT *
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. *
* See the License for the specific language governing permissions and *
* limitations under the License. See accompanying LICENSE file. *
*======================================================================*/

package org.openx.data.jsonserde.objectinspector.primitive;

import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableFloatObjectInspector;
import org.apache.hadoop.io.FloatWritable;

/**
*
* @author rcongiu
*/
public class JavaStringFloatObjectInspector extends AbstractPrimitiveJavaObjectInspector
implements SettableFloatObjectInspector {

public JavaStringFloatObjectInspector() {
super(PrimitiveObjectInspectorUtils.floatTypeEntry);
}

@Override
public Object getPrimitiveWritableObject(Object o) {
if(o == null) return null;

if(o instanceof String) {
return new FloatWritable(Float.parseFloat((String)o));
} else {
return new FloatWritable(((Float) o).floatValue());
}
}

@Override
public float get(Object o) {

if(o instanceof String) {
return Float.parseFloat((String)o);
} else {
return (((Float) o).floatValue());
}
}

@Override
public Object create(float value) {
return Float.valueOf(value);
}

@Override
public Object set(Object o, float value) {
return Float.valueOf(value);
}

}
Original file line number Diff line number Diff line change
@@ -1,8 +1,15 @@
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
/*======================================================================*
* Copyright (c) 2011, OpenX Technologies, Inc. All rights reserved. *
* *
* Licensed under the New BSD License (the "License"); you may not use *
* this file except in compliance with the License. Unless required *
* by applicable law or agreed to in writing, software distributed *
* under the License is distributed on an "AS IS" BASIS, WITHOUT *
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. *
* See the License for the specific language governing permissions and *
* limitations under the License. See accompanying LICENSE file. *
*======================================================================*/

package org.openx.data.jsonserde.objectinspector.primitive;

import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,15 @@
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
/*======================================================================*
* Copyright (c) 2011, OpenX Technologies, Inc. All rights reserved. *
* *
* Licensed under the New BSD License (the "License"); you may not use *
* this file except in compliance with the License. Unless required *
* by applicable law or agreed to in writing, software distributed *
* under the License is distributed on an "AS IS" BASIS, WITHOUT *
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. *
* See the License for the specific language governing permissions and *
* limitations under the License. See accompanying LICENSE file. *
*======================================================================*/

package org.openx.data.jsonserde.objectinspector.primitive;

import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
/*======================================================================*
* Copyright (c) 2011, OpenX Technologies, Inc. All rights reserved. *
* *
* Licensed under the New BSD License (the "License"); you may not use *
* this file except in compliance with the License. Unless required *
* by applicable law or agreed to in writing, software distributed *
* under the License is distributed on an "AS IS" BASIS, WITHOUT *
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. *
* See the License for the specific language governing permissions and *
* limitations under the License. See accompanying LICENSE file. *
*======================================================================*/

package org.openx.data.jsonserde.objectinspector.primitive;

import org.apache.hadoop.hive.serde2.io.ShortWritable;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableShortObjectInspector;

/**
*
* @author rcongiu
*/
public class JavaStringShortObjectInspector
extends AbstractPrimitiveJavaObjectInspector
implements SettableShortObjectInspector {

public JavaStringShortObjectInspector() {
super(PrimitiveObjectInspectorUtils.shortTypeEntry);
}

@Override
public Object getPrimitiveWritableObject(Object o) {
if(o == null) return null;

if(o instanceof String) {
return new ShortWritable(Short.parseShort((String)o));
} else {
return new ShortWritable(((Short) o).shortValue());
}
}

@Override
public short get(Object o) {

if(o instanceof String) {
return Short.parseShort((String)o);
} else {
return (((Short) o).shortValue());
}
}

@Override
public Object create(short value) {
return Short.valueOf(value);
}

@Override
public Object set(Object o, short value) {
return Short.valueOf(value);
}
}
54 changes: 49 additions & 5 deletions src/test/java/org/openx/data/jsonserde/JsonSerDeTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -32,15 +32,20 @@
import org.apache.hadoop.hive.serde.Constants;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaBooleanObjectInspector;
import org.apache.hadoop.io.Writable;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import static org.junit.Assert.*;
import org.openx.data.jsonserde.objectinspector.primitive.JavaStringByteObjectInspector;
import org.openx.data.jsonserde.objectinspector.primitive.JavaStringDoubleObjectInspector;
import org.openx.data.jsonserde.objectinspector.primitive.JavaStringFloatObjectInspector;
import org.openx.data.jsonserde.objectinspector.primitive.JavaStringIntObjectInspector;
import org.openx.data.jsonserde.objectinspector.primitive.JavaStringLongObjectInspector;
import org.openx.data.jsonserde.objectinspector.primitive.JavaStringShortObjectInspector;

/**
*
Expand Down Expand Up @@ -288,15 +293,54 @@ public JsonSerDe getNumericSerde() throws SerDeException {
}

@Test
public void testNumbers() throws SerDeException {
public void testNumbers() throws SerDeException, JSONException {
System.out.println("testNumbers");

JsonSerDe serde = getNumericSerde();
Text line = new Text("{ cboolean:true, ctinyint:1, csmallint:200}");

Text line = new Text("{ cboolean:true, ctinyint:1, csmallint:200, cint:12345,cbigint:123446767687867, cfloat:3.1415, cdouble:43424234234.4243423}");

StructObjectInspector soi = (StructObjectInspector) serde.getObjectInspector();

JSONObject result = (JSONObject) serde.deserialize(line);

StructField sf = soi.getStructFieldRef("cboolean");

assertTrue(sf.getFieldObjectInspector() instanceof JavaBooleanObjectInspector);
JavaBooleanObjectInspector jboi = (JavaBooleanObjectInspector) sf.getFieldObjectInspector();
assertEquals(true, jboi.get(result.get("cboolean")));

sf = soi.getStructFieldRef("ctinyint");
assertTrue(sf.getFieldObjectInspector() instanceof JavaStringByteObjectInspector);
JavaStringByteObjectInspector boi = (JavaStringByteObjectInspector) sf.getFieldObjectInspector();
assertEquals(1, boi.get(result.get("ctinyint")));

sf = soi.getStructFieldRef("csmallint");
assertTrue(sf.getFieldObjectInspector() instanceof JavaStringShortObjectInspector);
JavaStringShortObjectInspector shoi = (JavaStringShortObjectInspector) sf.getFieldObjectInspector();
assertEquals(200, shoi.get(result.get("csmallint")));


sf = soi.getStructFieldRef("cint");
assertTrue(sf.getFieldObjectInspector() instanceof JavaStringIntObjectInspector);
JavaStringIntObjectInspector oi = (JavaStringIntObjectInspector) sf.getFieldObjectInspector();
assertEquals(12345, oi.get(result.get("cint")));

sf = soi.getStructFieldRef("cbigint");
assertTrue(sf.getFieldObjectInspector() instanceof JavaStringLongObjectInspector);
JavaStringLongObjectInspector bioi = (JavaStringLongObjectInspector) sf.getFieldObjectInspector();
assertEquals(123446767687867L , bioi.get(result.get("cbigint")));

sf = soi.getStructFieldRef("cfloat");
assertTrue(sf.getFieldObjectInspector() instanceof JavaStringFloatObjectInspector);
JavaStringFloatObjectInspector foi = (JavaStringFloatObjectInspector) sf.getFieldObjectInspector();
assertEquals(3.1415 , foi.get(result.get("cfloat")),0.001);

sf = soi.getStructFieldRef("cdouble");
assertTrue(sf.getFieldObjectInspector() instanceof JavaStringDoubleObjectInspector);
JavaStringDoubleObjectInspector doi = (JavaStringDoubleObjectInspector) sf.getFieldObjectInspector();
assertEquals(43424234234.4243423 , doi.get(result.get("cdouble")),0.001);
}


@Test
public void testSerializeWithMapping() throws SerDeException, JSONException {
Expand Down

0 comments on commit 3788ede

Please sign in to comment.