Skip to content

Commit

Permalink
Added code to read (initial draft) of HAWQS disaggregation API result…
Browse files Browse the repository at this point in the history 
…s, unzip the data, and link to the AQUATOX simulation.

Fixed an issue with default output graphs.
  • Loading branch information
@jsclough
jsclough committed Mar 8, 2024
1 parent e031e91 commit 0d1fc1f
Show file tree
Hide file tree
Showing 4 changed files with 113 additions and 25 deletions.
4 changes: 3 additions & 1 deletion Data.Simulate.AQUATOX/AQTSegment.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5081,7 +5081,9 @@ TGraphSetup AddVarsToGraph(string name, AllVariables firstvar, AllVariables last

if (var==AllVariables.Volume)
{
result.YItems.Add(new SeriesID() { nm = TSV.OutputText(2), lyr = TSV.Layer, ns = TSV.NState, typ = TSV.SVType, indx = 2 }); //add inflow
int inflowcolumn = 2;
if (PSetup.SaveBRates.Val) inflowcolumn = 5; // after inflow, outflow, and evap in units of "percent"
result.YItems.Add(new SeriesID() { nm = TSV.OutputText(inflowcolumn), lyr = TSV.Layer, ns = TSV.NState, typ = TSV.SVType, indx = inflowcolumn }); //add inflow
}

}
Expand Down
4 changes: 3 additions & 1 deletion Data.Simulate.AQUATOX/AQ_Sim_2D.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
using System.Threading.Tasks;
using Microsoft.VisualBasic.FileIO;
using AQUATOX.OrgMatter;
using Microsoft.AspNetCore.DataProtection.KeyManagement;

namespace AQUATOX.AQSim_2D

Expand Down Expand Up @@ -139,13 +138,16 @@ public class HAWQSInput
public int warmupYears { get; set; } = 2;
public string outputPrintSetting { get; set; } = "daily";
public ReportData reportData { get; set; } = new ReportData();
public bool disaggregateComids { get; set; } = false;
}

public class SetHrus
{
public string method { get; set; } = "none";
public double target { get; set; } = 0;
public string units { get; set; } = "none";
public string[] exemptLanduse { get; set; } = new string[] { "AGWF", "AGWR", "AGWT", "RIWF", "RIWN", "UPWF", "UPWN", "WATR", "WETF", "WETL", "WETN" };
public string[] noAreaRedistribution { get; set; } = new string[] { "AGWF", "AGWR", "AGWT", "RIWF", "RIWN", "UPWF", "UPWN", "WATR", "WETF", "WETL", "WETN" };
}

public class ReportData
Expand Down
2 changes: 2 additions & 0 deletions GUI/GUI.AQUATOX/GUI.AQUATOX.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -186,12 +186,14 @@
</ItemGroup>

<ItemGroup>
<PackageReference Include="7z.Libs" Version="23.1.0" />
<PackageReference Include="Microsoft.Web.WebView2" Version="1.0.2277.86" />
<PackageReference Include="SDSLite" Version="2.0.3" />
<PackageReference Include="Microsoft.DotNet.UpgradeAssistant.Extensions.Default.Analyzers" Version="0.4.355802">
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="Microsoft.Windows.Compatibility" Version="8.0.1" />
<PackageReference Include="SevenZipSharp.Interop" Version="19.1.0" />
<PackageReference Include="System.Windows.Forms.DataVisualization" Version="1.0.0-prerelease.20110.1" />
</ItemGroup>

Expand Down
128 changes: 105 additions & 23 deletions GUI/GUI.AQUATOX/MultiSegForm.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,11 @@
using Data;
using Globals;
using Hawqs;
using Microsoft.VisualBasic;
using Microsoft.Web.WebView2.Core;
using MongoDB.Driver;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using SevenZip;
using System;
using System.Collections.Generic;
using System.Collections.Specialized;
Expand All @@ -24,10 +24,11 @@
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Reflection;
using System.Windows.Forms;
using System.Windows.Forms.DataVisualization.Charting;
using Utilities;
using static AQUATOX.AQSim_2D.AQSim_2D;
using System.Globalization;

namespace GUI.AQUATOX

Expand Down Expand Up @@ -404,8 +405,16 @@ private void UpdateScreen()
else if (validJSON)
{
if (isLake0D) StatusLabel.Text = "Site Selected";
else if (isHUC0D) StatusLabel.Text = "HUC Selected";
else StatusLabel.Text = "Stream Network Created";
else if (isHUC0D)
{
if (File.Exists(BaseDir + "output_rch_daily.csv")) StatusLabel.Text = "HAWQS Run Completed";
else StatusLabel.Text = "HUC Selected";
}
else
{
if (File.Exists(BaseDir + "output_rch_disaggregated.csv")) StatusLabel.Text = "HAWQS Run Completed";
StatusLabel.Text = "Stream Network Created";
}
}
else if (validDirectory) StatusLabel.Text = "Model Not Initiated";
else StatusLabel.Text = "Invalid Directory Specified";
Expand Down Expand Up @@ -2402,7 +2411,61 @@ static List<string> MergeLists(List<string> list1, List<string> list2)
Hawqs.Hawqs HAWQS_Sim = null;
runstatus HAWQS_RunStatus = null;

async private void HAWQS_Click(object sender, EventArgs e)
public class CSVAggregator
{
public static async Task<string> DownloadAndAggregateCSV(string fileUrl)
{
var tempFilePath = Path.GetTempFileName();
var outputStringBuilder = new StringBuilder();

// Download file
using (var httpClient = new HttpClient())
{
var response = await httpClient.GetAsync(fileUrl, HttpCompletionOption.ResponseHeadersRead);
response.EnsureSuccessStatusCode();

using (var fs = new FileStream(tempFilePath, FileMode.Create))
using (var stream = await response.Content.ReadAsStreamAsync())
{
await stream.CopyToAsync(fs);
}
}

SevenZipCompressor.SetLibraryPath(Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), @"x64\7z.dll")); //fixme deployment
// Extract and aggregate CSVs
using (var extractor = new SevenZipExtractor(tempFilePath))
{
foreach (var entry in extractor.ArchiveFileData.Where(file => file.FileName.EndsWith(".csv") && file.Size > 170)) // 170 byte files simply contain an error message
{
using (var entryStream = new MemoryStream())
{
extractor.ExtractFile(entry.FileName, entryStream);
entryStream.Position = 0;

using (var reader = new StreamReader(entryStream))
{
bool isFirstFile = outputStringBuilder.Length == 0;
while (!reader.EndOfStream)
{
var line = reader.ReadLine();
if (isFirstFile || !reader.EndOfStream)
{
outputStringBuilder.AppendLine(line);
}
isFirstFile = false;
}
}
}
}
}

File.Delete(tempFilePath);
return outputStringBuilder.ToString();
}
}


async private void HAWQS_Click(object sender, EventArgs e)
{
if (!VerifyStreamNetwork()) return;
if (Lake0D > 0)
Expand All @@ -2422,12 +2485,13 @@ async private void HAWQS_Click(object sender, EventArgs e)
string[] outputHUCs;

string hucstr;
if (!isHUC0D)
if (!isHUC0D) //streamnetwork code first
{
List<string> H14s = Render_RelevantH14s(out string pourpoint);
HAWQSInp.downstreamSubbasin = pourpoint;

HAWQSInp.dataset = "HUC14";
HAWQSInp.disaggregateComids = true;
hucstr = "14";
List<string> outhucs = new();
AQT2D.HAWQSInf.modelDomain = H14s;
Expand All @@ -2445,7 +2509,7 @@ async private void HAWQS_Click(object sender, EventArgs e)
}
outputHUCs = outhucs.ToArray();
}
else
else //HUC0D code
{
hucstr = HUCStr(HUC0D); // "8" to "14"
HAWQSInp.dataset = "HUC" + hucstr;
Expand Down Expand Up @@ -2557,10 +2621,14 @@ async private void HAWQS_Click(object sender, EventArgs e)
TSafeUpdateProgress(progress);
}

Task<List<HawqsOutput>> GPDT = HAWQS_Sim.GetProjectData(HAWQS_apikey, HAWQS_RunStatus.id, false);
Task<List<HawqsOutput>> GPDT = HAWQS_Sim.GetProjectData(HAWQS_apikey, "702" , false); //HAWQS_RunStatus.id FIXME
await (GPDT);
List<HawqsOutput> LHO = GPDT.Result;

string urlName = "output_rch_daily.csv";
if (!isHUC0D) urlName = "output_rch_daily_comids.7z";


if (LHO.Count == 0) AddToProcessLog("WARNING: No HAWQS Results Returned.");
else
{
Expand All @@ -2572,11 +2640,11 @@ async private void HAWQS_Click(object sender, EventArgs e)
while ((!foundrch) && (indx < LHO.Count))
{
url = LHO[indx].url;
foundrch = url.Contains("output_rch_daily.csv");
foundrch = url.Contains(urlName);
indx++;
}

if (!foundrch) AddToProcessLog("ERROR: daily reach output CSV not found in HAWQS data URLs.");
if (!foundrch) AddToProcessLog("ERROR: daily reach output "+urlName+" not found in HAWQS data URLs.");
else
{
AddToProcessLog("INFO: Downloading daily reach data from " + url);
Expand All @@ -2585,11 +2653,20 @@ async private void HAWQS_Click(object sender, EventArgs e)

try
{
using (var fs = await hc.GetStreamAsync(url))
using (var fileStream = new FileStream(BaseDir + "output_rch_daily.csv", FileMode.Create))
await fs.CopyToAsync(fileStream);
AddToProcessLog("INFO: Saved daily reach data to " + BaseDir + "output_rch_daily.csv");
hc.Dispose();
if (!isHUC0D) // aggregate all the CSVs in the 7z file
{
string csvstring = await CSVAggregator.DownloadAndAggregateCSV(url);
File.WriteAllText(BaseDir + "output_rch_disaggregated.csv",csvstring);
AddToProcessLog("INFO: Saved daily reach data to " + BaseDir + "output_rch_disaggregated.csv");
}
else //download data for HUC0D
{
using (var fs = await hc.GetStreamAsync(url))
using (var fileStream = new FileStream(BaseDir + "output_rch_daily.csv", FileMode.Create))
await fs.CopyToAsync(fileStream);
AddToProcessLog("INFO: Saved daily reach data to " + BaseDir + "output_rch_daily.csv");
hc.Dispose();
}
}
catch (Exception ex)
{
Expand Down Expand Up @@ -2626,7 +2703,10 @@ private void ReadHAWQS_Click(object sender, EventArgs e)
if (!VerifyStreamNetwork()) return;

BaseDir = basedirBox.Text;

string RCHfileN = BaseDir + "output_rch_daily.csv";
if (HUC0D == "") RCHfileN = BaseDir + "output_rch_disaggregated.csv";

if (!File.Exists(RCHfileN))
{
AddToProcessLog("ERROR: can't find reach output file: " + RCHfileN);
Expand Down Expand Up @@ -2663,14 +2743,16 @@ void AddHAWQSRchData(long longKey, DateTime dateTimeKey, HAWQSRCHRow data) //ad
for (int i = 1; i < csvlines.Length; i++)
{
string[] rowdata = csvlines[i].Split(','); //split
DateTime date = DateTime.ParseExact(rowdata[0], "yyyy-MM-dd", System.Globalization.CultureInfo.InvariantCulture);
HAWQSRCHRow row = new();
row.lat = Convert.ToDouble(rowdata[1]);
row.lon = Convert.ToDouble(rowdata[2]);
long SubBasin = Convert.ToInt64(rowdata[3]);
row.vals = new double[rowdata.Length - 4];
for (int j = 4; j < rowdata.Length; j++) row.vals[j - 4] = Convert.ToDouble(rowdata[j]);
AddHAWQSRchData(SubBasin, date, row);
if (DateTime.TryParseExact(rowdata[0], "yyyy-MM-dd", System.Globalization.CultureInfo.InvariantCulture, DateTimeStyles.None, out DateTime date))
{
HAWQSRCHRow row = new();
row.lat = Convert.ToDouble(rowdata[1]);
row.lon = Convert.ToDouble(rowdata[2]);
long SubBasin = Convert.ToInt64(rowdata[3]);
row.vals = new double[rowdata.Length - 4];
for (int j = 4; j < rowdata.Length; j++) row.vals[j - 4] = Convert.ToDouble(rowdata[j]);
AddHAWQSRchData(SubBasin, date, row);
}
}

AddToProcessLog("INPUTS: read HAWQS file " + RCHfileN);
Expand Down

0 comments on commit 0d1fc1f

Please sign in to comment.