diff --git a/Jenkinsfile b/Jenkinsfile index 57fcf3e..5c99587 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -504,29 +504,30 @@ def resolveBranchNo(String featureBranchPRMinusNo) { } def resolveBranchName(String featureBranchPRMinusNo, String orgName, String repoName) { - - // get pull request number - def branchNoMatcher = featureBranchPRMinusNo =~ /PR-(.*)/ - assert branchNoMatcher.find() - - def prNo = branchNoMatcher[0][1] + def prNo = extractPrNumber(featureBranchPRMinusNo) // curl the repo based on the feature branch no to get the branch information /// Note: only works for public repos! Otherwise credentials needs to be passed - def curlUrl = "curl https://api.github.com/repos/" + orgName + "/" + repoName + "/pulls/" + prNo - def response = curlUrl.execute().text - def matcher = response =~ /\"label\":\s\"(.+)\"/ - - assert matcher.find() + String response = curlByPR(prNo, orgName, repoName) + log("i", "API response:" + response) + def jsonResponse = readJSON text: response + def branchName = jsonResponse.head.ref - // get split the label to account for PRs from forks - def split = matcher[0][1] =~ /(.*):(.*)/ - - assert matcher.find() + return branchName +} - def username = split[0][1] - def branch = split[0][2] +def extractPrNumber(String featureBranchPRMinusNo) { + // get pull request number + def branchNoMatcher = featureBranchPRMinusNo =~ /PR-(.*)/ + assert branchNoMatcher.find() - return branch + String prNo = branchNoMatcher[0][1] + log("i", "PR number: " + prNo + " of class " + prNo.getClass()) + return prNo +} +def curlByPR(String prId, String orgName, String repoName) { + def curlUrl = "set +x && curl -s https://api.github.com/repos/" + orgName + "/" + repoName + "/pulls/" + prId + String jsonResponseString = sh(script: curlUrl, returnStdout: true) + return jsonResponseString } \ No newline at end of file diff --git a/src/main/java/edu/ie3/tools/Converter.java b/src/main/java/edu/ie3/tools/Converter.java index 9317209..d5a8403 100644 --- a/src/main/java/edu/ie3/tools/Converter.java +++ b/src/main/java/edu/ie3/tools/Converter.java @@ -412,7 +412,7 @@ private void validation() { private List checkForPreviousEntries(List entities) { LinkedList checkedEntities = new LinkedList<>(); Map> entitiesByDate = - entities.stream().collect(Collectors.groupingBy(ICONWeatherModel::getDate)); + entities.stream().collect(Collectors.groupingBy(ICONWeatherModel::getTime)); entitiesByDate.forEach( (date, entitiesAtDate) -> checkedEntities.addAll(checkForPreviousEntries(entitiesAtDate, date))); diff --git a/src/main/java/edu/ie3/tools/models/persistence/CoordinateModel.java b/src/main/java/edu/ie3/tools/models/persistence/CoordinateModel.java index 9fc3665..81d3df9 100644 --- a/src/main/java/edu/ie3/tools/models/persistence/CoordinateModel.java +++ b/src/main/java/edu/ie3/tools/models/persistence/CoordinateModel.java @@ -29,7 +29,7 @@ + "AND longitude BETWEEN :minLongitude AND :maxLongitude"), }) @Entity(name = "icon_coordinates") -public class CoordinateModel extends edu.ie3.tools.models.persistence.AbstractCoordinateModel { +public class CoordinateModel extends AbstractCoordinateModel { public static final String COSMO_Coordinates = "CoordinateModel.COSMO_Coordinates"; public static final String ICON_Coordinates = "CoordinateModel.ICON_Coordinates"; diff --git a/src/main/java/edu/ie3/tools/models/persistence/ICONWeatherModel.java b/src/main/java/edu/ie3/tools/models/persistence/ICONWeatherModel.java index 71fb02f..54dfffa 100644 --- a/src/main/java/edu/ie3/tools/models/persistence/ICONWeatherModel.java +++ b/src/main/java/edu/ie3/tools/models/persistence/ICONWeatherModel.java @@ -21,8 +21,8 @@ public class ICONWeatherModel implements Serializable { private static final long serialVersionUID = -3506091597737302833L; @Id - @Column(name = "datum", nullable = false) - private ZonedDateTime date; + @Column(name = "time", nullable = false) + private ZonedDateTime time; @Id @ManyToOne(cascade = CascadeType.DETACH) @@ -138,8 +138,8 @@ public class ICONWeatherModel implements Serializable { public ICONWeatherModel() {} - public ICONWeatherModel(ZonedDateTime date, CoordinateModel coordinate) { - this.date = date; + public ICONWeatherModel(ZonedDateTime time, CoordinateModel coordinate) { + this.time = time; this.coordinate = coordinate; } @@ -162,7 +162,7 @@ public static ICONWeatherModel getInterpolatedEntity(ICONWeatherModel... entitie } public ICONWeatherKey getKey() { - return new ICONWeatherKey(coordinate, date); + return new ICONWeatherKey(coordinate, time); } public static String getSQLUpsertStatement(Collection entities) { @@ -172,56 +172,58 @@ public static String getSQLUpsertStatement(Collection entities public static String getSQLUpsertStatement( Collection entities, String database_schema) { StringBuilder upsertStatementBuilder = new StringBuilder(); - upsertStatementBuilder.append( - "INSERT INTO " - + database_schema - + ".weather(\n" - + "\tdatum, alb_rad, asob_s, aswdifd_s, aswdifu_s, aswdir_s, sobs_rad, p_20m, p_65m, p_131m, t_131m, t_2m, t_g, u_10m, u_131m, u_20m, u_216m, u_65m, v_10m, v_131m, v_20m, v_216m, v_65m, w_131m, w_20m, w_216m, w_65m, z0, coordinate_id)\n" - + "\t VALUES "); + upsertStatementBuilder + .append("INSERT INTO ") + .append(database_schema) + .append(".weather(\n") + .append( + "\ttime, alb_rad, asob_s, aswdifd_s, aswdifu_s, aswdir_s, sobs_rad, p_20m, p_65m, p_131m, t_131m, t_2m, t_g, u_10m, u_131m, u_20m, u_216m, u_65m, v_10m, v_131m, v_20m, v_216m, v_65m, w_131m, w_20m, w_216m, w_65m, z0, coordinate_id)\n") + .append("\t VALUES "); entities.forEach( - entity -> upsertStatementBuilder.append(entity.getSQLInsertValuesString() + ", ")); + entity -> upsertStatementBuilder.append(entity.getSQLInsertValuesString()).append(", ")); int lastComma = upsertStatementBuilder.lastIndexOf(","); upsertStatementBuilder.deleteCharAt(lastComma); - upsertStatementBuilder.append("ON CONFLICT (coordinate_id, datum) DO UPDATE \n" + " SET "); - upsertStatementBuilder.append( - "datum=excluded.datum,\n" - + " alb_rad=excluded.alb_rad,\n" - + " asob_s=excluded.asob_s,\n" - + " aswdifd_s=excluded.aswdifd_s,\n" - + " aswdifu_s=excluded.aswdifu_s,\n" - + " aswdir_s=excluded.aswdir_s,\n" - + " sobs_rad=excluded.sobs_rad,\n" - + " p_20m=excluded.p_20m,\n" - + " p_65m=excluded.p_65m,\n" - + " p_131m=excluded.p_131m,\n" - + " t_131m=excluded.t_131m,\n" - + " t_2m=excluded.t_2m,\n" - + " t_g=excluded.t_g,\n" - + " u_10m=excluded.u_10m,\n" - + " u_131m=excluded.u_131m,\n" - + " u_20m=excluded.u_20m,\n" - + " u_216m=excluded.u_216m,\n" - + " u_65m=excluded.u_65m,\n" - + " v_10m=excluded.v_10m,\n" - + " v_131m=excluded.v_131m,\n" - + " v_20m=excluded.v_20m,\n" - + " v_216m=excluded.v_216m,\n" - + " v_65m=excluded.v_65m,\n" - + " w_131m=excluded.w_131m,\n" - + " w_20m=excluded.w_20m,\n" - + " w_216m=excluded.w_216m,\n" - + " w_65m=excluded.w_65m,\n" - + " z0=excluded.z0,\n" - + " coordinate_id=excluded.coordinate_id;"); + upsertStatementBuilder + .append("ON CONFLICT (coordinate_id, time) DO UPDATE \n" + " SET ") + .append( + "time=excluded.time,\n" + + " alb_rad=excluded.alb_rad,\n" + + " asob_s=excluded.asob_s,\n" + + " aswdifd_s=excluded.aswdifd_s,\n" + + " aswdifu_s=excluded.aswdifu_s,\n" + + " aswdir_s=excluded.aswdir_s,\n" + + " sobs_rad=excluded.sobs_rad,\n" + + " p_20m=excluded.p_20m,\n" + + " p_65m=excluded.p_65m,\n" + + " p_131m=excluded.p_131m,\n" + + " t_131m=excluded.t_131m,\n" + + " t_2m=excluded.t_2m,\n" + + " t_g=excluded.t_g,\n" + + " u_10m=excluded.u_10m,\n" + + " u_131m=excluded.u_131m,\n" + + " u_20m=excluded.u_20m,\n" + + " u_216m=excluded.u_216m,\n" + + " u_65m=excluded.u_65m,\n" + + " v_10m=excluded.v_10m,\n" + + " v_131m=excluded.v_131m,\n" + + " v_20m=excluded.v_20m,\n" + + " v_216m=excluded.v_216m,\n" + + " v_65m=excluded.v_65m,\n" + + " w_131m=excluded.w_131m,\n" + + " w_20m=excluded.w_20m,\n" + + " w_216m=excluded.w_216m,\n" + + " w_65m=excluded.w_65m,\n" + + " z0=excluded.z0,\n" + + " coordinate_id=excluded.coordinate_id;"); return upsertStatementBuilder.toString(); } - public ZonedDateTime getDate() { - return date; + public ZonedDateTime getTime() { + return time; } - public void setDate(ZonedDateTime date) { - this.date = date; + public void setTime(ZonedDateTime date) { + this.time = date; } public Double getAlb_rad() { @@ -714,13 +716,13 @@ public void setCoordinate(CoordinateModel coordinate) { this.coordinate = coordinate; } - // (datum, alb_rad, asob_s, aswdifd_s, aswdifu_s, aswdir_s,sobs_rad,p_20m,p_65m,p_131m, t_131m, + // (time, alb_rad, asob_s, aswdifd_s, aswdifu_s, aswdir_s,sobs_rad,p_20m,p_65m,p_131m, t_131m, // t_2m, t_g, u_10m, u_131m, u_20m, // u_216m, u_65m, v_10m, v_131m, v_20m, v_216m, v_65m, w_131m, w_20m, w_216m, w_65m, z0, // coordinate_id) public String getSQLInsertValuesString() { String insertValues = "("; - insertValues += "'" + ConfigurationParameters.SQL_FORMATTER(date) + "', "; + insertValues += "'" + ConfigurationParameters.SQL_FORMATTER(time) + "', "; insertValues += alb_rad + ", "; insertValues += asob_s + ", "; insertValues += aswdifd_s + ", "; @@ -765,7 +767,7 @@ public static String getPSQLFindString(String database_schema) { + ".weather w JOIN " + database_schema + ".icon_coordinates c ON w.coordinate_id = c.id " - + "WHERE datum=? AND coordinate_id = ANY(?);"; + + "WHERE time=? AND coordinate_id = ANY(?);"; return query; } @@ -774,7 +776,7 @@ public boolean equals(Object o) { if (this == o) return true; if (!(o instanceof ICONWeatherModel)) return false; ICONWeatherModel that = (ICONWeatherModel) o; - return Objects.equals(date, that.date) + return Objects.equals(time, that.time) && Objects.equals(coordinate, that.coordinate) && Objects.equals(alb_rad, that.alb_rad) && Objects.equals(asob_s, that.asob_s) @@ -807,14 +809,14 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(coordinate, date); + return Objects.hash(coordinate, time); } @Override public String toString() { return "ICONWeatherModel{" - + "date=" - + date + + "time=" + + time + "\n, coordinate=" + coordinate + ", alb_rad=" diff --git a/src/main/java/edu/ie3/tools/models/persistence/keys/ICONWeatherKey.java b/src/main/java/edu/ie3/tools/models/persistence/keys/ICONWeatherKey.java index a7ece96..a435292 100644 --- a/src/main/java/edu/ie3/tools/models/persistence/keys/ICONWeatherKey.java +++ b/src/main/java/edu/ie3/tools/models/persistence/keys/ICONWeatherKey.java @@ -13,11 +13,11 @@ public class ICONWeatherKey implements Serializable { private CoordinateModel coordinate; - private ZonedDateTime date; + private ZonedDateTime time; - public ICONWeatherKey(CoordinateModel coordinate, ZonedDateTime date) { + public ICONWeatherKey(CoordinateModel coordinate, ZonedDateTime time) { this.coordinate = coordinate; - this.date = date; + this.time = time; } public ICONWeatherKey() {} @@ -30,16 +30,16 @@ public void setCoordinate(CoordinateModel coordinate) { this.coordinate = coordinate; } - public ZonedDateTime getDate() { - return date; + public ZonedDateTime getTime() { + return time; } - public void setDate(ZonedDateTime date) { - this.date = date; + public void setTime(ZonedDateTime date) { + this.time = date; } @Override public String toString() { - return "ICONWeatherKey{" + "coordinate=" + coordinate + ", date=" + date + '}'; + return "ICONWeatherKey{" + "coordinate=" + coordinate + ", time=" + time + '}'; } } diff --git a/src/main/java/edu/ie3/tools/utils/DatabaseController.java b/src/main/java/edu/ie3/tools/utils/DatabaseController.java index da0cf53..af08cfc 100644 --- a/src/main/java/edu/ie3/tools/utils/DatabaseController.java +++ b/src/main/java/edu/ie3/tools/utils/DatabaseController.java @@ -200,7 +200,7 @@ private Collection jdbcFindWeather( coordinateModel.setLongitude(rs.getDouble("longitude")); ICONWeatherModel weather = new ICONWeatherModel( - ZonedDateTime.of(rs.getTimestamp("datum").toLocalDateTime(), ZoneId.of("UTC")), + ZonedDateTime.of(rs.getTimestamp("time").toLocalDateTime(), ZoneId.of("UTC")), coordinateModel); for (Parameter parameter : Parameter.values()) { diff --git a/src/test/java/edu/ie3/tools/ConverterIT.java b/src/test/java/edu/ie3/tools/ConverterIT.java index 0aa4939..d169019 100644 --- a/src/test/java/edu/ie3/tools/ConverterIT.java +++ b/src/test/java/edu/ie3/tools/ConverterIT.java @@ -200,11 +200,11 @@ public void testAmountOfData() { assertEquals(numberOfCoordinates * 3, numberOfRows); statement = connect.createStatement(); - rs = statement.executeQuery("SELECT DISTINCT(datum) AS datum FROM icon.weather;"); + rs = statement.executeQuery("SELECT DISTINCT(time) AS time FROM icon.weather;"); statement.closeOnCompletion(); Set dates = new HashSet<>(); while (rs.next()) { - dates.add(rs.getTimestamp("datum").toString()); + dates.add(rs.getTimestamp("time").toString()); } // contains three dates, as two date points should overlap // (Modelrun 3 + Timestep 0 == Modelrun 0 + Timestep 3 == 3 o' Clock) @@ -347,7 +347,7 @@ private String getValueQuery(int coord_id, String dateString, String param) { + param + " as value from icon.weather where coordinate_id = " + coord_id - + " and datum = '" + + " and time = '" + dateString + "';"; return query; diff --git a/src/test/java/edu/ie3/tools/models/persistence/ICONWeatherModelTest.java b/src/test/java/edu/ie3/tools/models/persistence/ICONWeatherModelTest.java index 497007d..5716d1a 100644 --- a/src/test/java/edu/ie3/tools/models/persistence/ICONWeatherModelTest.java +++ b/src/test/java/edu/ie3/tools/models/persistence/ICONWeatherModelTest.java @@ -58,17 +58,17 @@ public void setUp() throws Exception { @Test public void getInterpolatedEntity() { ICONWeatherModel interpolatedEntity = ICONWeatherModel.getInterpolatedEntity(weather); - interpolatedEntity.setDate(date); + interpolatedEntity.setTime(date); interpolatedEntity.setCoordinate(coordinate); assertEquals(weather, interpolatedEntity); // should be the same ICONWeatherModel newWeather = new ICONWeatherModel(date, coordinate); - interpolatedEntity.setDate(date); + interpolatedEntity.setTime(date); interpolatedEntity.setCoordinate(coordinate); newWeather.setAlb_rad(alb_rad); interpolatedEntity = ICONWeatherModel.getInterpolatedEntity(weather, newWeather); - interpolatedEntity.setDate(date); + interpolatedEntity.setTime(date); interpolatedEntity.setCoordinate(coordinate); assertEquals( weather, @@ -82,7 +82,7 @@ public void getInterpolatedEntity() { } HashMap collectedParameterValues = collectParameterValues(testEntities); interpolatedEntity = ICONWeatherModel.getInterpolatedEntity(testEntities); - interpolatedEntity.setDate(date); + interpolatedEntity.setTime(date); interpolatedEntity.setCoordinate(coordinate); assertEquals(collectedParameterValues.get("alb_rad") / 10, interpolatedEntity.getAlb_rad()); @@ -117,20 +117,20 @@ public void getInterpolatedEntity() { @Test public void getSQLUpsertStatement() { ICONWeatherModel randWeather1 = generateRandomWeatherEntity(); - randWeather1.setDate(date); + randWeather1.setTime(date); randWeather1.setCoordinate(coordinate); ICONWeatherModel randWeather2 = generateRandomWeatherEntity(); - randWeather2.setDate(date); + randWeather2.setTime(date); randWeather2.setCoordinate(coordinate); Collection entities = Arrays.asList(weather, randWeather1, randWeather2); String generatedUpsertStatement = ICONWeatherModel.getSQLUpsertStatement(entities, "test"); String sqlInsertInto = - "INSERT INTO test.weather(\n\tdatum, alb_rad, asob_s, aswdifd_s, aswdifu_s, aswdir_s, sobs_rad, p_20m, p_65m, p_131m, t_131m, t_2m, t_g, " + "INSERT INTO test.weather(\n\ttime, alb_rad, asob_s, aswdifd_s, aswdifu_s, aswdir_s, sobs_rad, p_20m, p_65m, p_131m, t_131m, t_2m, t_g, " + "u_10m, u_131m, u_20m, u_216m, u_65m, v_10m, v_131m, v_20m, v_216m, v_65m, w_131m, w_20m, w_216m, w_65m, " + "z0, coordinate_id)\n\t VALUES "; assertTrue(generatedUpsertStatement.startsWith(sqlInsertInto)); String sqlOnConflict = - " ON CONFLICT (coordinate_id, datum) DO UPDATE \n SET datum=excluded.datum,\n " + " ON CONFLICT (coordinate_id, time) DO UPDATE \n SET time=excluded.time,\n " + "alb_rad=excluded.alb_rad,\n asob_s=excluded.asob_s,\n aswdifd_s=excluded.aswdifd_s,\n " + "aswdifu_s=excluded.aswdifu_s,\n aswdir_s=excluded.aswdir_s,\n sobs_rad=excluded.sobs_rad,\n " + "p_20m=excluded.p_20m,\n p_65m=excluded.p_65m,\n p_131m=excluded.p_131m,\n t_131m=excluded.t_131m,\n " diff --git a/src/test/resources/sql/initDatabase.sql b/src/test/resources/sql/initDatabase.sql index fb225a1..19f6e04 100644 --- a/src/test/resources/sql/initDatabase.sql +++ b/src/test/resources/sql/initDatabase.sql @@ -5,7 +5,7 @@ CREATE TABLE icon.icon_coordinates ( coordinate_type character varying(255), CONSTRAINT pk_icon_coordinates PRIMARY KEY (id)); CREATE TABLE icon.weather ( - datum timestamp without time zone NOT NULL, alb_rad double precision, asob_s double precision, + time timestamp without time zone NOT NULL, alb_rad double precision, asob_s double precision, aswdifd_s double precision, aswdifu_s double precision, aswdir_s double precision, sobs_rad double precision, p_20m double precision, p_65m double precision, p_131m double precision, t_131m double precision, t_2m double precision, t_g double precision, u_10m double precision, @@ -14,7 +14,7 @@ CREATE TABLE icon.weather ( v_65m double precision, w_131m double precision, w_20m double precision, w_216m double precision, w_65m double precision, z0 double precision, coordinate_id integer NOT NULL, - CONSTRAINT pk_weather PRIMARY KEY (coordinate_id, datum), + CONSTRAINT pk_weather PRIMARY KEY (coordinate_id, time), CONSTRAINT fk_weather_coordinates FOREIGN KEY (coordinate_id) REFERENCES icon.icon_coordinates (id) MATCH SIMPLE ON UPDATE NO ACTION ON DELETE NO ACTION);