Skip to content

Commit

Permalink
[NOID] Fix flaky geocode server errors (#3527)
Browse files Browse the repository at this point in the history 
* [NOID] Fix flaky geocode server errors

* [NOID] added waitForServerResponseOK in other test-cases, changed time measuring handling
  • Loading branch information
@vga91
vga91 authored Apr 14, 2023
1 parent d79eed0 commit 7a9fec4
Showing 1 changed file with 65 additions and 23 deletions.
88 changes: 65 additions & 23 deletions core/src/test/java/apoc/spatial/GeocodeTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,19 +6,24 @@
import org.junit.*;
import org.neo4j.graphdb.QueryExecutionException;
import org.neo4j.configuration.GraphDatabaseInternalSettings;
import org.neo4j.graphdb.Result;
import org.neo4j.internal.helpers.collection.Iterators;
import org.neo4j.test.rule.DbmsRule;
import org.neo4j.test.rule.ImpermanentDbmsRule;

import java.io.InputStream;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Consumer;
import java.util.stream.Stream;

import static apoc.ApocConfig.apocConfig;
import static apoc.util.MapUtil.map;
import static apoc.util.TestUtil.*;
import static org.junit.Assert.*;
import static org.neo4j.test.assertion.Assert.assertEventually;

public class GeocodeTest {

Expand Down Expand Up @@ -175,35 +180,37 @@ private long testGeocode(String provider, long throttle, boolean reverseGeocode,
setupSupplier(provider, throttle);
InputStream is = getClass().getResourceAsStream("/spatial.json");
Map tests = JsonUtil.OBJECT_MAPPER.readValue(is, Map.class);
long start = System.currentTimeMillis();
AtomicLong time = new AtomicLong();

if(reverseGeocode) {
for(Object address : (List) tests.get("events")) {
testReverseGeocodeAddress(((Map)address).get("lat"), ((Map)address).get("lon"), config);
testReverseGeocodeAddress(((Map)address).get("lat"), ((Map)address).get("lon"), time, config);
}
} else {
for (Object address : (List) tests.get("addresses")) {
testGeocodeAddress((Map) address, (String) config.getOrDefault("provider", provider), config);
testGeocodeAddress((Map) address, (String) config.getOrDefault("provider", provider), time, config);
}
}

return System.currentTimeMillis() - start;
return time.get();
}

private void testReverseGeocodeAddress(Object latitude, Object longitude, Map<String, Object> config) {
private void testReverseGeocodeAddress(Object latitude, Object longitude, AtomicLong time, Map<String, Object> config) {
ignoreQuotaError(() -> {
testResult(db, "CALL apoc.spatial.reverseGeocode($latitude, $longitude, false, $config)",
map("latitude", latitude, "longitude", longitude, "config", config), (row) -> {
assertTrue(row.hasNext());
row.forEachRemaining((r)->{
String query = "CALL apoc.spatial.reverseGeocode($latitude, $longitude, false, $config)";
Map<String, Object> params = Map.of("latitude", latitude, "longitude", longitude, "config", config);
waitForServerResponseOK(query, params, time,
(res) -> {
assertTrue(res.hasNext());
res.forEachRemaining((r) -> {
assertNotNull(r.get("description"));
assertNotNull(r.get("location"));
assertNotNull(r.get("data"));
});
});
});
}

private void ignoreQuotaError(Runnable runnable) {
try {
runnable.run();
Expand All @@ -224,39 +231,51 @@ private void setupSupplier(String providerName, long throttle) {
apocConfig().setProperty(Geocode.PREFIX + "." + providerName + ".throttle", Long.toString(throttle));
}

private void testGeocodeAddress(Map map, String provider, Map<String, Object> config) {
private void testGeocodeAddress(Map map, String provider, AtomicLong time, Map<String, Object> config) {
ignoreQuotaError(() -> {
testResult(db,"CALL apoc.spatial.geocode('FRANCE',1,true,$config)",
map("config", config), (row)->{
row.forEachRemaining((r)->{
String query = "CALL apoc.spatial.geocode('FRANCE',1,true,$config)";
Map<String, Object> params = Map.of("config", config);
waitForServerResponseOK(query, params, time,
(res) -> {
res.forEachRemaining((r) -> {
assertNotNull(r.get("description"));
assertNotNull(r.get("location"));
assertNotNull(r.get("data"));
});
});
});
String geocodeQuery = "CALL apoc.spatial.geocode($url,0)";
if (map.containsKey("noresults")) {
for (String field : new String[]{"address", "noresults"}) {
checkJsonFields(map, field);
}
System.out.println("map = " + map);
testCallEmpty(db, "CALL apoc.spatial.geocode($url,0)", map("url", map.get("address").toString()));
waitForServerResponseOK(geocodeQuery,
map("url", map.get("address").toString()),
time,
(res) -> assertFalse(res.hasNext())
);
} else if (map.containsKey("count")) {
if (((Map) map.get("count")).containsKey(provider)) {
for (String field : new String[]{"address", "count"}) {
checkJsonFields(map, field);
}
testCallCount(db, "CALL apoc.spatial.geocode($url,0)",
waitForServerResponseOK(geocodeQuery,
map("url", map.get("address").toString()),
((Number) ((Map) map.get("count")).get(provider)).intValue());
time,
(res) -> {
long actual = Iterators.count(res);
int expected = ((Number) ((Map) map.get("count")).get(provider)).intValue();
assertEquals(expected, actual);
});
}
} else {
for (String field : new String[]{"address", "osm"}) {
checkJsonFields(map, field);
}
testGeocodeAddress(map.get("address").toString(),
getCoord(map, provider, "latitude"),
getCoord(map, provider, "longitude"),
getCoord(map, provider, "longitude"),
time,
config);
}
}
Expand All @@ -271,9 +290,10 @@ private void checkJsonFields(Map map, String field) {
assertTrue("Expected " + field + " field", map.containsKey(field));
}

private void testGeocodeAddress(String address, double lat, double lon, Map<String, Object> config) {
testResult(db, "CALL apoc.spatial.geocodeOnce($url, $config)",
map("url", address, "config", config),
private void testGeocodeAddress(String address, double lat, double lon, AtomicLong time, Map<String, Object> config) {
String query = "CALL apoc.spatial.geocodeOnce($url, $config)";
Map<String, Object> params = Map.of("url", address, "config", config);
waitForServerResponseOK(query, params, time,
(result) -> {
if (result.hasNext()) {
Map<String, Object> row = result.next();
Expand All @@ -289,4 +309,26 @@ private void testGeocodeAddress(String address, double lat, double lon, Map<Stri
}
});
}

private void waitForServerResponseOK(String query, Map<String, Object> params, AtomicLong time, Consumer<Result> resultObjectFunction) {
assertEventually(() -> {
try {
long start = System.currentTimeMillis();
db.executeTransactionally(query,
params,
res -> {
resultObjectFunction.accept(res);
return null;
});

time.addAndGet( System.currentTimeMillis() - start );
return true;
} catch (Exception e) {
if (e.getMessage().contains("Server returned HTTP response code")) {
return false;
}
throw e;
}
}, (value) -> value, 20L, TimeUnit.SECONDS);
}
}

0 comments on commit 7a9fec4

Please sign in to comment.