feat: Allo to setup the UPPER_LIMIT parameter in Weacast downloaders (c…

…lose #179)
kalisio · Oct 8, 2020 · 315d569 · 315d569
1 parent 3048a25
commit 315d569
Show file tree

Hide file tree

Showing 7 changed files with 26 additions and 10 deletions.
diff --git a/.env.default b/.env.default
@@ -179,11 +179,16 @@ WEACAST_DEBUG=
 # Weacast loaders
 ARPEGE_WORLD_LOADER_IMAGE=weacast/weacast-arpege
 ARPEGE_WORLD_LOADER_TAG=world-1.4.1
+ARPEGE_WORLD_LOADER_UPPER_LIMIT=367200
 ARPEGE_EUROPE_LOADER_IMAGE=weacast/weacast-arpege
 ARPEGE_EUROPE_LOADER_TAG=europe-1.4.1
+ARPEGE_EUROPE_LOADER_UPPER_LIMIT=367200
 AROME_FRANCE_LOADER_IMAGE=weacast/weacast-arome
 AROME_FRANCE_LOADER_TAG=france-1.4.1
+AROME_FRANCE_LOADER_UPPER_LIMIT=151200
 AROME_FRANCE_HIGH_LOADER_IMAGE=weacast/weacast-arome
 AROME_FRANCE_HIGH_LOADER_TAG=france-high-1.4.1
+AROME_FRANCE_HIGH_LOADER_UPPER_LIMIT=151200
 GFS_WORLD_LOADER_IMAGE=weacast/weacast-gfs
 GFS_WORLD_LOADER_TAG=world-1.4.1
+GFS_WORLD_LOADER_UPPER_LIMIT=864000
diff --git a/deploy/arome-france-high-loader.yml b/deploy/arome-france-high-loader.yml
@@ -6,6 +6,7 @@ services:
     environment:
       - DB_URL=${WEACAST_LOADERS_DB_URL}
       - SYNC_DB_URL=${WEACAST_LOADERS_SYNC_DB_URL}
+      - UPPER_LIMIT=${AROME_FRANCE_HIGH_LOADER_UPPER_LIMIT}
       - DEBUG=${AROME_RANCE_HIGH_LOADER_DEBUG}
       - HTTP_PROXY
       - HTTPS_PROXY

diff --git a/deploy/arome-france-loader.yml b/deploy/arome-france-loader.yml
@@ -6,6 +6,7 @@ services:
     environment:
       - DB_URL=${WEACAST_LOADERS_DB_URL}
       - SYNC_DB_URL=${WEACAST_LOADERS_SYNC_DB_URL}
+      - UPPER_LIMIT=${AROME_FRANCE_LOADER_UPPER_LIMIT}
       - DEBUG=${AROME_FRANCE_LOADER_DEBUG}
       - HTTP_PROXY
       - HTTPS_PROXY

diff --git a/deploy/arpege-europe-loader.yml b/deploy/arpege-europe-loader.yml
@@ -6,6 +6,7 @@ services:
     environment:
       - DB_URL=${WEACAST_LOADERS_DB_URL}
       - SYNC_DB_URL=${WEACAST_LOADERS_SYNC_DB_URL}
+      - UPPER_LIMIT=${ARPEGE_EUROPE_LOADER_UPPER_LIMIT}
       - DEBUG=${ARPEGE_EUROPE_LOADER_DEBUG}
       - HTTP_PROXY
       - HTTPS_PROXY

diff --git a/deploy/arpege-world-loader.yml b/deploy/arpege-world-loader.yml
@@ -6,6 +6,7 @@ services:
     environment:
       - DB_URL=${WEACAST_LOADERS_DB_URL}
       - SYNC_DB_URL=${WEACAST_LOADERS_SYNC_DB_URL}
+      - UPPER_LIMIT=${ARPEGE_WORLD_LOADER_UPPER_LIMIT}
       - DEBUG=${ARPEGE_WORLD_LOADER_DEBUG}
       - HTTP_PROXY
       - HTTPS_PROXY

diff --git a/deploy/gfs-world-loader.yml b/deploy/gfs-world-loader.yml
@@ -6,6 +6,7 @@ services:
     environment:
       - DB_URL=${WEACAST_LOADERS_DB_URL}
       - SYNC_DB_URL=${WEACAST_LOADERS_SYNC_DB_URL}
+      - UPPER_LIMIT=${GFS_WORLD_LOADER_UPPER_LIMIT}
       - DEBUG=${GFS_WORLD_LOADER_DEBUG}
       - HTTP_PROXY
       - HTTPS_PROXY

diff --git a/docs/reference/environment.md b/docs/reference/environment.md
@@ -299,20 +299,26 @@ The **MapProxy** service is preconfigured to run [Gunicorn](https://gunicorn.org
 | Variable | Description | Default value |
 | --- | --- | --- |
 | `WEACAST_LOADERS_DB_URL` | The database URL where to write the data. | - |
-| `ARPEGE_WOLRD_LOADER_IMAGE` | The image to be used to scrape Arpege World data. | `localhost:5000/arpege-world-loader` |
-| `ARPEGE_WORLD_LOADER_TAG` | The tag of the image to be used to scrape Arpege World data. | `1.4.1` |
+| `ARPEGE_WOLRD_LOADER_IMAGE` | The image to be used to scrape Arpege World data. | `weacast/weacast-arpege` |
+| `ARPEGE_WORLD_LOADER_TAG` | The tag of the image to be used to scrape Arpege World data. | `world-1.4.1` |
+| `ARPEGE_WORLD_LOADER_UPPER_LIMIT` | The time limit, in seconds, of the forecast data to be downloaded. | `367200` (102 hours)|
 | `ARPEGE_WORLD_LOADER_DEBUG` | The namespaces to enable debug output. Set it to `krawler*` to enable full debug output. | `` |
-| `ARPEGE_EUROPE_LOADER_IMAGE` | The image to be used to scrape Arpege Europe data. | `localhost:5000/arpege-europe-loader` |
-| `ARPEGE_EUROPE_LOADER_TAG` | The tag of the image to be used to scrape Arpege Europe data. | `1.4.1` |
+| `ARPEGE_EUROPE_LOADER_IMAGE` | The image to be used to scrape Arpege Europe data. | `weacast/weacast-arpege` |
+| `ARPEGE_EUROPE_LOADER_TAG` | The tag of the image to be used to scrape Arpege Europe data. | `europe-1.4.1` |
+| `ARPEGE_EUROPE_LOADER_UPPER_LIMIT` | The time limit, in seconds, of the forecast data to be downloaded. | `367200` (102 hours)|
 | `ARPEGE_EUROPE_LOADER_DEBUG` | The namespaces to enable debug output. Set it to `krawler*` to enable full debug output. | `` |
-| `AROME_FRANCE_LOADER_IMAGE` | The image to be used to scrape Arome France data. | `localhost:5000/arome-france-loader` |
-| `AROME_FRANCE_LOADER_TAG` | The tag of the image to be used to scrape Arome France World data. | `1.4.1` |
+| `AROME_FRANCE_LOADER_IMAGE` | The image to be used to scrape Arome France data. | `weacast/weacast-arome` |
+| `AROME_FRANCE_LOADER_TAG` | The tag of the image to be used to scrape Arome France World data. | `france-1.4.1` |
+| `AROME_FRANCE_LOADER_UPPER_LIMIT` | The time limit, in seconds, of the forecast data to be downloaded. | `151200` (42 hours)|
 | `AROME_FRANCE_LOADER_DEBUG` | The namespaces to enable debug output. Set it to `krawler*` to enable full debug output. | `` |
-| `AROME_FRANCE_HIGH_LOADER_IMAGE` | The image to be used to scrape Arome France data. | `localhost:5000/arome-france-loader` |
-| `AROME_FRANCE_HIGH_LOADER_TAG` | The tag of the image to be used to scrape Arome France World data. | `1.4.1` |
+| `AROME_FRANCE_HIGH_LOADER_IMAGE` | The image to be used to scrape Arome France data. | `weacast/weacast-arome` |
+
+| `AROME_FRANCE_HIGH_LOADER_TAG` | The tag of the image to be used to scrape Arome France World data. | `france-high-1.4.1` |
+| `AROME_FRANCE_HIGH_LOADER_UPPER_LIMIT` | The time limit, in seconds, of the forecast data to be downloaded. | `151200`(42 hours)|
 | `AROME_FRANCE_HIGH_LOADER_DEBUG` |The namespaces to enable debug output. Set it to `krawler*` to enable full debug output. | `` |
-| `GFS_WORLD_LOADER_IMAGE` | The image to be used to scrape GFS World data. | `localhost:5000/gfs-world-loader` |
-| `GFS_WORLD_LOADER_TAG` | The tag of the image to be used to scrape Arpege World data. | `1.4.1` |
+| `GFS_WORLD_LOADER_IMAGE` | The image to be used to scrape GFS World data. | `weacast/weacast-gfs` |
+| `GFS_WORLD_LOADER_TAG` | The tag of the image to be used to scrape Arpege World data. | `world-1.4.1` |
+| `GFS_WORLD_LOADER_UPPER_LIMIT` | The time limit, in seconds, of the forecast data to be downloaded. | `864000` (10 days) |
 | `GFS_WORLD_DEBUG` | The namespaces to enable debug output. Set it to `krawler*` to enable full debug output. | `` |