Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adapt the column name from "date" to "time" #3

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 19 additions & 18 deletions Jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -616,29 +616,30 @@ def resolveBranchNo(String featureBranchPRMinusNo) {
}

def resolveBranchName(String featureBranchPRMinusNo, String orgName, String repoName) {

// get pull request number
def branchNoMatcher = featureBranchPRMinusNo =~ /PR-(.*)/
assert branchNoMatcher.find()

def prNo = branchNoMatcher[0][1]
def prNo = extractPrNumber(featureBranchPRMinusNo)

// curl the repo based on the feature branch no to get the branch information
/// Note: only works for public repos! Otherwise credentials needs to be passed
def curlUrl = "curl https://api.github.com/repos/" + orgName + "/" + repoName + "/pulls/" + prNo
def response = curlUrl.execute().text
def matcher = response =~ /\"label\":\s\"(.+)\"/

assert matcher.find()
String response = curlByPR(prNo, orgName, repoName)
log("i", "API response:" + response)
def jsonResponse = readJSON text: response
def branchName = jsonResponse.head.ref
Comment on lines +511 to +514
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this actually do the same thing as the old implementation?


// get split the label to account for PRs from forks
def split = matcher[0][1] =~ /(.*):(.*)/

assert matcher.find()
return branchName
}

def username = split[0][1]
def branch = split[0][2]
def extractPrNumber(String featureBranchPRMinusNo) {
// get pull request number
def branchNoMatcher = featureBranchPRMinusNo =~ /PR-(.*)/
assert branchNoMatcher.find()

return branch
String prNo = branchNoMatcher[0][1]
log("i", "PR number: " + prNo + " of class " + prNo.getClass())
return prNo
}

def curlByPR(String prId, String orgName, String repoName) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would rename prId to prNo as it is called in line 625 and the method extractPrNumber also indicate that it is the number we are actually after.

def curlUrl = "set +x && curl -s https://api.github.com/repos/" + orgName + "/" + repoName + "/pulls/" + prId
String jsonResponseString = sh(script: curlUrl, returnStdout: true)
return jsonResponseString
}
2 changes: 1 addition & 1 deletion src/main/java/edu/ie3/tools/Converter.java
Original file line number Diff line number Diff line change
Expand Up @@ -407,7 +407,7 @@ private void validation() {
private List<ICONWeatherModel> checkForPreviousEntries(List<ICONWeatherModel> entities) {
LinkedList<ICONWeatherModel> checkedEntities = new LinkedList<>();
Map<ZonedDateTime, List<ICONWeatherModel>> entitiesByDate =
entities.stream().collect(Collectors.groupingBy(ICONWeatherModel::getDate));
entities.stream().collect(Collectors.groupingBy(ICONWeatherModel::getTime));
entitiesByDate.forEach(
(date, entitiesAtDate) ->
checkedEntities.addAll(checkForPreviousEntries(entitiesAtDate, date)));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
+ "AND longitude BETWEEN :minLongitude AND :maxLongitude"),
})
@Entity(name = "icon_coordinates")
public class CoordinateModel extends edu.ie3.tools.models.persistence.AbstractCoordinateModel {
public class CoordinateModel extends AbstractCoordinateModel {

public static final String COSMO_Coordinates = "CoordinateModel.COSMO_Coordinates";
public static final String ICON_Coordinates = "CoordinateModel.ICON_Coordinates";
Expand Down
110 changes: 56 additions & 54 deletions src/main/java/edu/ie3/tools/models/persistence/ICONWeatherModel.java
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ public class ICONWeatherModel implements Serializable {
private static final long serialVersionUID = -3506091597737302833L;

@Id
@Column(name = "datum", nullable = false)
private ZonedDateTime date;
@Column(name = "time", nullable = false)
private ZonedDateTime time;

@Id
@ManyToOne(cascade = CascadeType.DETACH)
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Don't know how you feel about this. As the many-to-one relation has no @JoinColumns annotation, it defaults back to coordinate_id, as this field ist named coordinate and the database id of CoordinateModel is id. However, this is quite implicit and I would like it to be more explicit.

Expand Down Expand Up @@ -138,8 +138,8 @@ public class ICONWeatherModel implements Serializable {

public ICONWeatherModel() {}

public ICONWeatherModel(ZonedDateTime date, CoordinateModel coordinate) {
this.date = date;
public ICONWeatherModel(ZonedDateTime time, CoordinateModel coordinate) {
this.time = time;
this.coordinate = coordinate;
}

Expand All @@ -162,7 +162,7 @@ public static ICONWeatherModel getInterpolatedEntity(ICONWeatherModel... entitie
}

public ICONWeatherKey getKey() {
return new ICONWeatherKey(coordinate, date);
return new ICONWeatherKey(coordinate, time);
}

public static String getSQLUpsertStatement(Collection<ICONWeatherModel> entities) {
Expand All @@ -172,56 +172,58 @@ public static String getSQLUpsertStatement(Collection<ICONWeatherModel> entities
public static String getSQLUpsertStatement(
Collection<ICONWeatherModel> entities, String database_schema) {
StringBuilder upsertStatementBuilder = new StringBuilder();
upsertStatementBuilder.append(
"INSERT INTO "
+ database_schema
+ ".weather(\n"
+ "\tdatum, alb_rad, asob_s, aswdifd_s, aswdifu_s, aswdir_s, sobs_rad, p_20m, p_65m, p_131m, t_131m, t_2m, t_g, u_10m, u_131m, u_20m, u_216m, u_65m, v_10m, v_131m, v_20m, v_216m, v_65m, w_131m, w_20m, w_216m, w_65m, z0, coordinate_id)\n"
+ "\t VALUES ");
upsertStatementBuilder
.append("INSERT INTO ")
.append(database_schema)
.append(".weather(\n")
.append(
"\ttime, alb_rad, asob_s, aswdifd_s, aswdifu_s, aswdir_s, sobs_rad, p_20m, p_65m, p_131m, t_131m, t_2m, t_g, u_10m, u_131m, u_20m, u_216m, u_65m, v_10m, v_131m, v_20m, v_216m, v_65m, w_131m, w_20m, w_216m, w_65m, z0, coordinate_id)\n")
.append("\t VALUES ");
entities.forEach(
entity -> upsertStatementBuilder.append(entity.getSQLInsertValuesString() + ", "));
entity -> upsertStatementBuilder.append(entity.getSQLInsertValuesString()).append(", "));
int lastComma = upsertStatementBuilder.lastIndexOf(",");
upsertStatementBuilder.deleteCharAt(lastComma);
upsertStatementBuilder.append("ON CONFLICT (coordinate_id, datum) DO UPDATE \n" + " SET ");
upsertStatementBuilder.append(
"datum=excluded.datum,\n"
+ " alb_rad=excluded.alb_rad,\n"
+ " asob_s=excluded.asob_s,\n"
+ " aswdifd_s=excluded.aswdifd_s,\n"
+ " aswdifu_s=excluded.aswdifu_s,\n"
+ " aswdir_s=excluded.aswdir_s,\n"
+ " sobs_rad=excluded.sobs_rad,\n"
+ " p_20m=excluded.p_20m,\n"
+ " p_65m=excluded.p_65m,\n"
+ " p_131m=excluded.p_131m,\n"
+ " t_131m=excluded.t_131m,\n"
+ " t_2m=excluded.t_2m,\n"
+ " t_g=excluded.t_g,\n"
+ " u_10m=excluded.u_10m,\n"
+ " u_131m=excluded.u_131m,\n"
+ " u_20m=excluded.u_20m,\n"
+ " u_216m=excluded.u_216m,\n"
+ " u_65m=excluded.u_65m,\n"
+ " v_10m=excluded.v_10m,\n"
+ " v_131m=excluded.v_131m,\n"
+ " v_20m=excluded.v_20m,\n"
+ " v_216m=excluded.v_216m,\n"
+ " v_65m=excluded.v_65m,\n"
+ " w_131m=excluded.w_131m,\n"
+ " w_20m=excluded.w_20m,\n"
+ " w_216m=excluded.w_216m,\n"
+ " w_65m=excluded.w_65m,\n"
+ " z0=excluded.z0,\n"
+ " coordinate_id=excluded.coordinate_id;");
upsertStatementBuilder
.append("ON CONFLICT (coordinate_id, time) DO UPDATE \n" + " SET ")
.append(
"time=excluded.time,\n"
+ " alb_rad=excluded.alb_rad,\n"
+ " asob_s=excluded.asob_s,\n"
+ " aswdifd_s=excluded.aswdifd_s,\n"
+ " aswdifu_s=excluded.aswdifu_s,\n"
+ " aswdir_s=excluded.aswdir_s,\n"
+ " sobs_rad=excluded.sobs_rad,\n"
+ " p_20m=excluded.p_20m,\n"
+ " p_65m=excluded.p_65m,\n"
+ " p_131m=excluded.p_131m,\n"
+ " t_131m=excluded.t_131m,\n"
+ " t_2m=excluded.t_2m,\n"
+ " t_g=excluded.t_g,\n"
+ " u_10m=excluded.u_10m,\n"
+ " u_131m=excluded.u_131m,\n"
+ " u_20m=excluded.u_20m,\n"
+ " u_216m=excluded.u_216m,\n"
+ " u_65m=excluded.u_65m,\n"
+ " v_10m=excluded.v_10m,\n"
+ " v_131m=excluded.v_131m,\n"
+ " v_20m=excluded.v_20m,\n"
+ " v_216m=excluded.v_216m,\n"
+ " v_65m=excluded.v_65m,\n"
+ " w_131m=excluded.w_131m,\n"
+ " w_20m=excluded.w_20m,\n"
+ " w_216m=excluded.w_216m,\n"
+ " w_65m=excluded.w_65m,\n"
+ " z0=excluded.z0,\n"
+ " coordinate_id=excluded.coordinate_id;");
return upsertStatementBuilder.toString();
}

public ZonedDateTime getDate() {
return date;
public ZonedDateTime getTime() {
return time;
}

public void setDate(ZonedDateTime date) {
this.date = date;
public void setTime(ZonedDateTime date) {
this.time = date;
}

public Double getAlb_rad() {
Expand Down Expand Up @@ -714,13 +716,13 @@ public void setCoordinate(CoordinateModel coordinate) {
this.coordinate = coordinate;
}

// (datum, alb_rad, asob_s, aswdifd_s, aswdifu_s, aswdir_s,sobs_rad,p_20m,p_65m,p_131m, t_131m,
// (time, alb_rad, asob_s, aswdifd_s, aswdifu_s, aswdir_s,sobs_rad,p_20m,p_65m,p_131m, t_131m,
// t_2m, t_g, u_10m, u_131m, u_20m,
// u_216m, u_65m, v_10m, v_131m, v_20m, v_216m, v_65m, w_131m, w_20m, w_216m, w_65m, z0,
// coordinate_id)
public String getSQLInsertValuesString() {
String insertValues = "(";
insertValues += "'" + ConfigurationParameters.SQL_FORMATTER(date) + "', ";
insertValues += "'" + ConfigurationParameters.SQL_FORMATTER(time) + "', ";
insertValues += alb_rad + ", ";
insertValues += asob_s + ", ";
insertValues += aswdifd_s + ", ";
Expand Down Expand Up @@ -765,7 +767,7 @@ public static String getPSQLFindString(String database_schema) {
+ ".weather w JOIN "
+ database_schema
+ ".icon_coordinates c ON w.coordinate_id = c.id "
+ "WHERE datum=? AND coordinate_id = ANY(?);";
+ "WHERE time=? AND coordinate_id = ANY(?);";
return query;
}

Expand All @@ -774,7 +776,7 @@ public boolean equals(Object o) {
if (this == o) return true;
if (!(o instanceof ICONWeatherModel)) return false;
ICONWeatherModel that = (ICONWeatherModel) o;
return Objects.equals(date, that.date)
return Objects.equals(time, that.time)
&& Objects.equals(coordinate, that.coordinate)
&& Objects.equals(alb_rad, that.alb_rad)
&& Objects.equals(asob_s, that.asob_s)
Expand Down Expand Up @@ -807,14 +809,14 @@ public boolean equals(Object o) {

@Override
public int hashCode() {
return Objects.hash(coordinate, date);
return Objects.hash(coordinate, time);
}

@Override
public String toString() {
return "ICONWeatherModel{"
+ "date="
+ date
+ "time="
+ time
+ "\n, coordinate="
+ coordinate
+ ", alb_rad="
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,11 @@
public class ICONWeatherKey implements Serializable {

private CoordinateModel coordinate;
private ZonedDateTime date;
private ZonedDateTime time;

public ICONWeatherKey(CoordinateModel coordinate, ZonedDateTime date) {
public ICONWeatherKey(CoordinateModel coordinate, ZonedDateTime time) {
this.coordinate = coordinate;
this.date = date;
this.time = time;
}

public ICONWeatherKey() {}
Expand All @@ -30,16 +30,16 @@ public void setCoordinate(CoordinateModel coordinate) {
this.coordinate = coordinate;
}

public ZonedDateTime getDate() {
return date;
public ZonedDateTime getTime() {
return time;
}

public void setDate(ZonedDateTime date) {
this.date = date;
public void setTime(ZonedDateTime date) {
this.time = date;
}

@Override
public String toString() {
return "ICONWeatherKey{" + "coordinate=" + coordinate + ", date=" + date + '}';
return "ICONWeatherKey{" + "coordinate=" + coordinate + ", time=" + time + '}';
}
}
2 changes: 1 addition & 1 deletion src/main/java/edu/ie3/tools/utils/DatabaseController.java
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ private Collection<ICONWeatherModel> jdbcFindWeather(
coordinateModel.setLongitude(rs.getDouble("longitude"));
ICONWeatherModel weather =
new ICONWeatherModel(
ZonedDateTime.of(rs.getTimestamp("datum").toLocalDateTime(), ZoneId.of("UTC")),
ZonedDateTime.of(rs.getTimestamp("time").toLocalDateTime(), ZoneId.of("UTC")),
coordinateModel);

for (Parameter parameter : Parameter.values()) {
Expand Down
6 changes: 3 additions & 3 deletions src/test/java/edu/ie3/tools/ConverterIT.java
Original file line number Diff line number Diff line change
Expand Up @@ -200,11 +200,11 @@ public void testAmountOfData() {
assertEquals(numberOfCoordinates * 3, numberOfRows);

statement = connect.createStatement();
rs = statement.executeQuery("SELECT DISTINCT(datum) AS datum FROM icon.weather;");
rs = statement.executeQuery("SELECT DISTINCT(time) AS time FROM icon.weather;");
statement.closeOnCompletion();
Set<String> dates = new HashSet<>();
while (rs.next()) {
dates.add(rs.getTimestamp("datum").toString());
dates.add(rs.getTimestamp("time").toString());
}
// contains three dates, as two date points should overlap
// (Modelrun 3 + Timestep 0 == Modelrun 0 + Timestep 3 == 3 o' Clock)
Expand Down Expand Up @@ -347,7 +347,7 @@ private String getValueQuery(int coord_id, String dateString, String param) {
+ param
+ " as value from icon.weather where coordinate_id = "
+ coord_id
+ " and datum = '"
+ " and time = '"
+ dateString
+ "';";
return query;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,17 +58,17 @@ public void setUp() throws Exception {
@Test
public void getInterpolatedEntity() {
ICONWeatherModel interpolatedEntity = ICONWeatherModel.getInterpolatedEntity(weather);
interpolatedEntity.setDate(date);
interpolatedEntity.setTime(date);
interpolatedEntity.setCoordinate(coordinate);
assertEquals(weather, interpolatedEntity); // should be the same

ICONWeatherModel newWeather = new ICONWeatherModel(date, coordinate);
interpolatedEntity.setDate(date);
interpolatedEntity.setTime(date);
interpolatedEntity.setCoordinate(coordinate);
newWeather.setAlb_rad(alb_rad);

interpolatedEntity = ICONWeatherModel.getInterpolatedEntity(weather, newWeather);
interpolatedEntity.setDate(date);
interpolatedEntity.setTime(date);
interpolatedEntity.setCoordinate(coordinate);
assertEquals(
weather,
Expand All @@ -82,7 +82,7 @@ public void getInterpolatedEntity() {
}
HashMap<String, Double> collectedParameterValues = collectParameterValues(testEntities);
interpolatedEntity = ICONWeatherModel.getInterpolatedEntity(testEntities);
interpolatedEntity.setDate(date);
interpolatedEntity.setTime(date);
interpolatedEntity.setCoordinate(coordinate);

assertEquals(collectedParameterValues.get("alb_rad") / 10, interpolatedEntity.getAlb_rad());
Expand Down Expand Up @@ -117,20 +117,20 @@ public void getInterpolatedEntity() {
@Test
public void getSQLUpsertStatement() {
ICONWeatherModel randWeather1 = generateRandomWeatherEntity();
randWeather1.setDate(date);
randWeather1.setTime(date);
randWeather1.setCoordinate(coordinate);
ICONWeatherModel randWeather2 = generateRandomWeatherEntity();
randWeather2.setDate(date);
randWeather2.setTime(date);
randWeather2.setCoordinate(coordinate);
Collection<ICONWeatherModel> entities = Arrays.asList(weather, randWeather1, randWeather2);
String generatedUpsertStatement = ICONWeatherModel.getSQLUpsertStatement(entities, "test");
String sqlInsertInto =
"INSERT INTO test.weather(\n\tdatum, alb_rad, asob_s, aswdifd_s, aswdifu_s, aswdir_s, sobs_rad, p_20m, p_65m, p_131m, t_131m, t_2m, t_g, "
"INSERT INTO test.weather(\n\ttime, alb_rad, asob_s, aswdifd_s, aswdifu_s, aswdir_s, sobs_rad, p_20m, p_65m, p_131m, t_131m, t_2m, t_g, "
+ "u_10m, u_131m, u_20m, u_216m, u_65m, v_10m, v_131m, v_20m, v_216m, v_65m, w_131m, w_20m, w_216m, w_65m, "
+ "z0, coordinate_id)\n\t VALUES ";
assertTrue(generatedUpsertStatement.startsWith(sqlInsertInto));
String sqlOnConflict =
" ON CONFLICT (coordinate_id, datum) DO UPDATE \n SET datum=excluded.datum,\n "
" ON CONFLICT (coordinate_id, time) DO UPDATE \n SET time=excluded.time,\n "
+ "alb_rad=excluded.alb_rad,\n asob_s=excluded.asob_s,\n aswdifd_s=excluded.aswdifd_s,\n "
+ "aswdifu_s=excluded.aswdifu_s,\n aswdir_s=excluded.aswdir_s,\n sobs_rad=excluded.sobs_rad,\n "
+ "p_20m=excluded.p_20m,\n p_65m=excluded.p_65m,\n p_131m=excluded.p_131m,\n t_131m=excluded.t_131m,\n "
Expand Down
4 changes: 2 additions & 2 deletions src/test/resources/sql/initDatabase.sql
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ CREATE TABLE icon.icon_coordinates (
coordinate_type character varying(255), CONSTRAINT pk_icon_coordinates PRIMARY KEY (id));

CREATE TABLE icon.weather (
datum timestamp without time zone NOT NULL, alb_rad double precision, asob_s double precision,
time timestamp without time zone NOT NULL, alb_rad double precision, asob_s double precision,
aswdifd_s double precision, aswdifu_s double precision, aswdir_s double precision, sobs_rad double precision,
p_20m double precision, p_65m double precision, p_131m double precision,
t_131m double precision, t_2m double precision, t_g double precision, u_10m double precision,
Expand All @@ -14,7 +14,7 @@ CREATE TABLE icon.weather (
v_65m double precision, w_131m double precision, w_20m double precision, w_216m double precision,
w_65m double precision, z0 double precision,
coordinate_id integer NOT NULL,
CONSTRAINT pk_weather PRIMARY KEY (coordinate_id, datum),
CONSTRAINT pk_weather PRIMARY KEY (coordinate_id, time),
CONSTRAINT fk_weather_coordinates FOREIGN KEY (coordinate_id) REFERENCES icon.icon_coordinates (id)
MATCH SIMPLE ON UPDATE NO ACTION ON DELETE NO ACTION);

Expand Down