Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

NPI-3446 Fix SP3 parse and write of data section flags, plus address CLK and POS nodata causing misalignment #45

Merged
merged 6 commits into from
Aug 15, 2024
Merged

NPI-3446 Fix SP3 parse and write of data section flags, plus address CLK and POS nodata causing misalignment #45

Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
f9ab891
NPI-3446 various bugfixes in sp3 parse and write:
treefern Aug 13, 2024
f4170e1
NPI-3446 Fix SP3 writing of FLAGS columns using pre-concatenation to …
treefern Aug 14, 2024
2f51ab6
NPI-3446 Remove deprecated argument to dataframe groupby(). Value use…
treefern Aug 14, 2024
9f33165
NPI-3446 Fix SP3 header parse bug with orbit accuracy codes, introduc…
treefern Aug 14, 2024
8773a23
NPI-3446 Add raw flag to escaped string definitions to make linter ha…
treefern Aug 15, 2024
d4d5dd1
NPI-3446 Fix error reporting in trop.py (exception handling would hav…
treefern Aug 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
124 changes: 86 additions & 38 deletions gnssanalysis/gn_io/sp3.py
View file
Open in desktop
treefern marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@
_RE_SP3_HEAD_FDESCR = _re.compile(rb"\%c[ ]+(\w{1})[ ]+cc[ ](\w{3})")


_SP3_DEF_PV_WIDTH = [1, 3, 14, 14, 14, 14, 1, 2, 1, 2, 1, 2, 1, 3, 1, 1, 1, 1, 1, 1]
_SP3_DEF_PV_WIDTH = [1, 3, 14, 14, 14, 14, 1, 2, 1, 2, 1, 2, 1, 3, 1, 1, 1, 2, 1, 1]
treefern marked this conversation as resolved.
Show resolved Hide resolved
_SP3_DEF_PV_NAME = [
"PV_FLAG",
"PRN",
Expand All @@ -77,10 +77,79 @@
"Orbit_Pred_Flag",
]

SP3_POSITION_COLUMNS = [
[
"EST",
"EST",
"EST",
"EST",
"STD",
"STD",
"STD",
"STD",
"FLAGS",
"FLAGS",
"FLAGS",
"FLAGS",
],
[
"X",
"Y",
"Z",
"CLK",
"X",
"Y",
"Z",
"CLK",
"Clock_Event",
"Clock_Pred",
"Maneuver",
"Orbit_Pred",
],
]

SP3_VELOCITY_COLUMNS = [
[
"EST",
"EST",
"EST",
"EST",
"STD",
"STD",
"STD",
"STD",
"FLAGS",
"FLAGS",
"FLAGS",
"FLAGS",
],
[
"VX",
"VY",
"VZ",
"VCLOCK",
"VX",
"VY",
"VZ",
"VCLOCK",
"Clock_Event",
"Clock_Pred",
"Maneuver",
"Orbit_Pred",
],
]

# Nodata ie NaN constants for SP3 format
SP3_CLOCK_NODATA_STRING = " 999999.999999"

# NOTE: the CLOCK and POS NODATA strings below are technicaly incorrect.
# The specification requires a leading space on the CLOCK value, but Pandas DataFrame.to_string() (and others?) insist
# on adding a space between columns (so this cancels out the missing space here).
# For the POS value, no leading spaces are required by the SP3 spec, but we need the total width to be 13 chars,
# not 14 (the official width of the column i.e. F14.6), again because Pandas insists on adding a further space.
# See comment in gen_sp3_content() line ~622 for further discussion.
SP3_CLOCK_NODATA_STRING = "999999.999999"
SP3_CLOCK_NODATA_NUMERIC = 999999
SP3_POS_NODATA_STRING = " 0.000000"
SP3_POS_NODATA_STRING = " 0.000000"
SP3_POS_NODATA_NUMERIC = 0
SP3_CLOCK_STD_NODATA = -1000
SP3_POS_STD_NODATA = -100
Expand Down Expand Up @@ -155,6 +224,12 @@ def _process_sp3_block(
return _pd.DataFrame()
epochs_dt = _pd.to_datetime(_pd.Series(date).str.slice(2, 21).values.astype(str), format=r"%Y %m %d %H %M %S")
temp_sp3 = _pd.read_fwf(_io.StringIO(data), widths=widths, names=names)
# TODO set datatypes per column in advance
temp_sp3["Clock_Event_Flag"] = temp_sp3["Clock_Event_Flag"].fillna(" ")
temp_sp3["Clock_Pred_Flag"] = temp_sp3["Clock_Pred_Flag"].fillna(" ")
temp_sp3["Maneuver_Flag"] = temp_sp3["Maneuver_Flag"].fillna(" ")
temp_sp3["Orbit_Pred_Flag"] = temp_sp3["Orbit_Pred_Flag"].fillna(" ")

dt_index = _np.repeat(a=_gn_datetime.datetime2j2000(epochs_dt.values), repeats=len(temp_sp3))
temp_sp3.set_index(dt_index, inplace=True)
temp_sp3.index.name = "J2000"
Expand Down Expand Up @@ -216,26 +291,12 @@ def read_sp3(
if pOnly or parsed_header.HEAD.loc["PV_FLAG"] == "P":
sp3_df = sp3_df.loc[sp3_df.index.get_level_values("PV_FLAG") == "P"]
sp3_df.index = sp3_df.index.droplevel("PV_FLAG")
# TODO consider exception handling if EP rows encountered
else:
position_df = sp3_df.xs("P", level="PV_FLAG")
velocity_df = sp3_df.xs("V", level="PV_FLAG")
velocity_df.columns = [
[
"EST",
"EST",
"EST",
"EST",
"STD",
"STD",
"STD",
"STD",
"a1",
"a2",
"a3",
"a4",
],
["VX", "VY", "VZ", "VCLOCK", "VX", "VY", "VZ", "VCLOCK", "", "", "", ""],
]
# TODO consider exception handling if EV rows encountered
velocity_df.columns = SP3_VELOCITY_COLUMNS
sp3_df = _pd.concat([position_df, velocity_df], axis=1)

# sp3_df.drop(columns="PV_FLAG", inplace=True)
Expand Down Expand Up @@ -279,23 +340,7 @@ def _reformat_df(sp3_df: _pd.DataFrame) -> _pd.DataFrame:
# remove PRN and PV_FLAG columns
sp3_df = sp3_df.drop(columns=["PRN", "PV_FLAG"])
# rename columns x_coordinate -> [EST, X], y_coordinate -> [EST, Y]
sp3_df.columns = [
[
"EST",
"EST",
"EST",
"EST",
"STD",
"STD",
"STD",
"STD",
"a1",
"a2",
"a3",
"a4",
],
["X", "Y", "Z", "CLK", "X", "Y", "Z", "CLK", "", "", "", ""],
]
sp3_df.columns = SP3_POSITION_COLUMNS
return sp3_df


Expand Down Expand Up @@ -551,6 +596,7 @@ def gen_sp3_content(
# options that .sort_index() provides
sp3_df = sp3_df.sort_index(ascending=True)
out_df = sp3_df["EST"]
flags_df = sp3_df["FLAGS"] # Prediction, maneuver, etc.
# If we have STD information transform it to the output format (integer exponents) and add to dataframe
if "STD" in sp3_df:
# In future we should pull this information from the header on read and store it in the dataframe attributes
Expand Down Expand Up @@ -580,7 +626,7 @@ def clk_log(x):
std_df.attrs = {}
std_df = std_df.transform({"X": pos_log, "Y": pos_log, "Z": pos_log, "CLK": clk_log})
std_df = std_df.rename(columns=lambda x: "STD_" + x)
out_df = _pd.concat([out_df, std_df], axis="columns")
out_df = _pd.concat([out_df, std_df, flags_df], axis="columns")

def prn_formatter(x):
return f"P{x}"
Expand Down Expand Up @@ -673,6 +719,8 @@ def clk_std_formatter(x):
# relevant columns.
# NOTE: NaN and infinity values do NOT invoke the formatter, though you can put a string in a primarily numeric
# column, so we format the nodata values ahead of time, above.
# NOTE: you CAN'T mix datatypes as described above, in Pandas 3 and above, so this approach will need to be
# updated to use chained calls to format().
epoch_vals.to_string(
buf=out_buf,
index=False,
Expand Down
Loading