v0.2.48..v0.2.49 changeset ParallelBoundedApiReader.h

diff --git a/hoot-core/src/main/cpp/hoot/core/io/ParallelBoundedApiReader.h b/hoot-core/src/main/cpp/hoot/core/io/ParallelBoundedApiReader.h
new file mode 100644
index 0000000..c75fe9a
--- /dev/null
+++ b/hoot-core/src/main/cpp/hoot/core/io/ParallelBoundedApiReader.h
@@ -0,0 +1,170 @@
+/*
+ * This file is part of Hootenanny.
+ *
+ * Hootenanny is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --------------------------------------------------------------------
+ *
+ * The following copyright notices are generated automatically. If you
+ * have a new notice to add, please use the format:
+ * " * @copyright Copyright ..."
+ * This will properly maintain the copyright information. DigitalGlobe
+ * copyrights will be updated automatically.
+ *
+ * @copyright Copyright (C) 2019 DigitalGlobe (http://www.digitalglobe.com/)
+ */
+
+#ifndef PARALLEL_BOUNDED_API_READER_H
+#define PARALLEL_BOUNDED_API_READER_H
+
+//  Standard
+#include <mutex>
+#include <thread>
+#include <queue>
+
+//  Geos
+#include <geos/geom/Envelope.h>
+
+//  Qt
+#include <QString>
+#include <QUrl>
+
+namespace hoot
+{
+
+/**
+ * @brief The ParallelBoundedApiReader class is a base class for HTTP reader classes that are bounded
+ *  by an envelope bounding box.  If the bounding box exceeds the `reader.http.bbox.max.size` value
+ *  (default of 0.25 degrees) then it is split up and processed.  Some APIs will respond with an error
+ *  if the amount of data for an area surpasses a certain threashold (50k elements in the OSM API for
+ *  example) that area is divided into quarters and reprocessed until all areas are successfully read.
+ */
+class ParallelBoundedApiReader
+{
+public:
+  /**
+   * @brief ParallelBoundedApiReader - Constructor
+   * @param useOsmApiBboxFormat True for using the x1,y1,x2,y2 format, false for the x1,x2,y1,y2 format
+   * @param addProjection True adds the projection ",EPSG:4326" to the query string
+   */
+  ParallelBoundedApiReader(bool useOsmApiBboxFormat = true, bool addProjection = false);
+  /** Destructor that stops all threads if necessary */
+  virtual ~ParallelBoundedApiReader();
+  /**
+   * @brief beginRead - Start the reading process by dividing up the envelope if necessary
+   *   and starting the receiving threads
+   * @param endpoint URL to the HTTP endpoint to query
+   * @param envelope Bounding box of the area to query
+   */
+  void beginRead(const QUrl& endpoint, const geos::geom::Envelope& envelope);
+  /**
+   * @brief isComplete
+   * @return True if all bounding boxes have been successfully queried
+   */
+  bool isComplete();
+  /**
+   * @brief getSingleResult
+   * @param result Single, full query result (output)
+   * @return True if there is a result to return, false otherwise
+   */
+  bool getSingleResult(QString& result);
+  /**
+   * @brief hasMoreResults
+   * @return True if there are more results to process
+   */
+  bool hasMoreResults();
+  /**
+   * @brief wait Wait on the threads to join
+   */
+  void wait();
+  /**
+   * @brief stop Stop the threads and then wait() for them to join
+   */
+  void stop();
+  /**
+   * @brief setMaxThreads
+   * @param max_threads Maximum number of reciever threads to spin up
+   */
+  void setMaxThreads(int max_threads) { _threadCount = max_threads; }
+
+  bool isError() { return _fatalError; }
+
+protected:
+  /**
+   * @brief _process Thread function that does the actual work of sending
+   *  the HTTP request for a give area and responds to the result
+   */
+  void _process();
+  /**
+   * @brief _sleep Sleep the current thread
+   */
+  void _sleep();
+  /**
+   * @brief writeDebugMap Write out the reponse from the API to a file for
+   *  debugging purposes
+   * @param data Response from API to write to file
+   * @param name Name of file to write in $HOOT_HOME/tmp/
+   */
+  void writeDebugMap(const QString& data, const QString& name);
+  /** Type of data that is being downloaded, for internal use in derived classes */
+  enum DataType
+  {
+    Text,
+    OsmXml,
+    GeoJson,
+    Json
+  };
+  DataType _dataType;
+  /** URL of the API endpoint to query */
+  QUrl _url;
+  /** List of result strings, one for each HTTP response */
+  QStringList _resultsList;
+  /** Total number of results received, should match _totalEnvelopes at the end to ensure all data has arrived */
+  int _totalResults;
+  /** Mutex guarding the results list */
+  std::mutex _resultsMutex;
+  /** Essentially the work queue of bounding boxes that the threads query from */
+  std::queue<geos::geom::Envelope> _bboxes;
+  /** Total number of envelopes created to query */
+  int _totalEnvelopes;
+  /** Mutex guarding the bounding box list */
+  std::mutex _bboxMutex;
+  /** Flag indicating that the _bboxes list is still being loaded, set to false when completely loaded */
+  bool _bboxContinue;
+  /** Grid division size (0.25 degrees lat/lon default) */
+  double _coordGridSize;
+  /** Maximum size of an area that can be downloaded */
+  double _maxGridSize;
+  /** Number of threads to process the HTTP requests */
+  int _threadCount;
+  /** Processing thread pool */
+  std::vector<std::thread> _threads;
+  /** Set to true if there was a fatal error in the query */
+  bool _fatalError;
+  /** Flag set to true if the bounding box is to be output in the x1,y1,x2,y2 format and
+   *  false will output x1,x2,y1,y2 format */
+  bool _useOsmApiBboxFormat;
+  /** Add the projection (",EPSG:4326") to the end of the bounding box, required for some APIs */
+  bool _addProjection;
+  /** Flag to stop all running threads from continuing */
+  bool _continueRunning;
+  /** File number used to create unique filenames for debug output */
+  int _filenumber;
+  /** Mutex guarding the file number */
+  std::mutex _filenumberMutex;
+};
+
+}
+
+#endif  //  PARALLEL_BOUNDED_API_READER_H