Skip to content

Commit

Permalink
More cleanup for PR
Browse files Browse the repository at this point in the history
  • Loading branch information
Sunjeet committed Feb 6, 2025
1 parent 1edbaa4 commit 045ab77
Show file tree
Hide file tree
Showing 4 changed files with 2 additions and 52 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -244,61 +244,12 @@ protected void applyShardToChecksum(HollowChecksum checksum, HollowSchema withSc
checksum.applyInt(findVarLengthFieldHashCode(startByte, endByte, numBitsForField, fieldIdx));
}
}
// if (withSchema.getName().equals("TestObject")) {
// System.out.println("SNAP: DEBUG: checksum=" + checksum + " ordinal= " + ordinal);
// }
}

ordinal = populatedOrdinals.nextSetBit(ordinal + 1);
}
}

protected void applyOrdinalToChecksum(HollowChecksum checksum, HollowSchema withSchema, int ordinal, int shardOrdinal) {
int numBitsForField;
long bitOffset;
long endByte;
long startByte;

if(!(withSchema instanceof HollowObjectSchema))
throw new IllegalArgumentException("HollowObjectTypeReadState can only calculate checksum with a HollowObjectSchema: " + schema.getName());

HollowObjectSchema commonSchema = schema.findCommonSchema((HollowObjectSchema)withSchema);

List<String> commonFieldNames = new ArrayList<String>();
for(int i=0;i<commonSchema.numFields();i++)
commonFieldNames.add(commonSchema.getFieldName(i));
Collections.sort(commonFieldNames);

int fieldIndexes[] = new int[commonFieldNames.size()];
for(int i=0;i<commonFieldNames.size();i++) {
fieldIndexes[i] = schema.getPosition(commonFieldNames.get(i));
}

checksum.applyInt(ordinal);
for(int i=0;i<fieldIndexes.length;i++) {
int fieldIdx = fieldIndexes[i];
bitOffset = fieldOffset(shardOrdinal, fieldIdx);
numBitsForField = dataElements.bitsPerField[fieldIdx];
if(!schema.getFieldType(fieldIdx).isVariableLength()) {
long fixedLengthValue = numBitsForField <= 56 ?
dataElements.fixedLengthData.getElementValue(bitOffset, numBitsForField)
: dataElements.fixedLengthData.getLargeElementValue(bitOffset, numBitsForField);

if(fixedLengthValue == dataElements.nullValueForField[fieldIdx])
checksum.applyInt(Integer.MAX_VALUE);
else
checksum.applyLong(fixedLengthValue);
} else {
endByte = dataElements.fixedLengthData.getElementValue(bitOffset, numBitsForField);
startByte = shardOrdinal != 0 ? dataElements.fixedLengthData.getElementValue(bitOffset - dataElements.bitsPerRecord, numBitsForField) : 0;
checksum.applyInt(findVarLengthFieldHashCode(startByte, endByte, numBitsForField, fieldIdx));
}
}
if (withSchema.getName().equals("TestObject")) {
System.out.println("SNAP: DEBUG: checksum=" + checksum + " ordinal= " + ordinal);
}
}

public long getApproximateHeapFootprintInBytes() {
long bitsPerFixedLengthData = (long)dataElements.bitsPerRecord * (dataElements.maxOrdinal + 1);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,7 @@ public void writeDelta(OutputStream os, ProducerOptionalBlobPartConfig.OptionalB
partStreamsByType = partStreams.getStreamsByType();

stateEngine.prepareForWrite();

if(stateEngine.isRestored())
stateEngine.ensureAllNecessaryStatesRestored();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,8 @@
import com.netflix.hollow.core.schema.HollowListSchema;
import java.io.DataOutputStream;
import java.io.IOException;
import java.util.logging.Logger;

public class HollowListTypeWriteState extends HollowTypeWriteState {
private static final Logger LOG = Logger.getLogger(HollowListTypeWriteState.class.getName());

/// statistics required for writing fixed length list data
private int bitsPerListPointer;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -194,7 +194,7 @@ public void calculateSnapshot() {
}

public void writeSnapshot(DataOutputStream os) throws IOException {
LOG.log(Level.INFO, String.format("Writing snapshot with num shards = %s, revNumShards = %s, max shard ordinals = %s", numShards, revNumShards, Arrays.toString(maxShardOrdinal)));
LOG.log(Level.FINE, String.format("Writing snapshot with num shards = %s, revNumShards = %s, max shard ordinals = %s", numShards, revNumShards, Arrays.toString(maxShardOrdinal)));
/// for unsharded blobs, support pre v2.1.0 clients
if(numShards == 1) {
writeSnapshotShard(os, 0);
Expand Down

0 comments on commit 045ab77

Please sign in to comment.