Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[automate-gatway] Add data-lifecycle endpoint (#2157)
As a requirement for building web UI to control Data Lifecycle jobs, we needed
to expose the data lifecycle functionality in the gateway. While contemplating
implementation options we noticed that the existing Node Lifecycle page required
two gateway API calls to get the configuration and another two to set it for the
existing two Node Lifecycle jobs. Rather than build several additional ad hoc
gateway APIs to for the the purge jobs, we determined that it would be better to
build a new top-level `data-lifecycle` endpoint where we could expose a unified
single interface for data lifecycle jobs and move all complexity to the backend.
The new data lifecycle API is:
```
GET /data-lifecycle/status
PUT /data-lifecycle/config
POST /data-lifecycle/run
GET /data-lifecycle/infra/status
PUT /data-lifecycle/infra/config
POST /data-lifecycle/infra/run
GET /data-lifecycle/compliance/status
PUT /data-lifecycle/compliance/config
POST /data-lifecycle/compliance/run
GET /data-lifecycle/event-feed/status
PUT /data-lifecycle/event-feed/config
POST /data-lifecycle/event-feed/run
```
Example aggregate status response:
```json
{
"infra": {
"jobs": [
{
"name": "delete_nodes",
"disabled": false,
"recurrence": "FREQ=SECONDLY;DTSTART=20191106T180240Z;INTERVAL=600",
"threshold": "2d",
"purge_policies": null,
"last_elapsed": "0.041154s",
"next_due_at": "2019-11-07T22:32:40Z",
"last_enqueued_at": "2019-11-07T22:22:40.017947Z",
"last_started_at": null,
"last_ended_at": null
},
{
"name": "missing_nodes",
"disabled": false,
"recurrence": "FREQ=SECONDLY;DTSTART=20191106T180240Z;INTERVAL=600",
"threshold": "2d",
"purge_policies": null,
"last_elapsed": "0.043205s",
"next_due_at": "2019-11-07T22:32:40Z",
"last_enqueued_at": "2019-11-07T22:22:40.026315Z",
"last_started_at": null,
"last_ended_at": null
},
{
"name": "missing_nodes_for_deletion",
"disabled": false,
"recurrence": "FREQ=SECONDLY;DTSTART=20191106T180240Z;INTERVAL=600",
"threshold": "31d",
"purge_policies": null,
"last_elapsed": "0.048030s",
"next_due_at": "2019-11-07T22:32:40Z",
"last_enqueued_at": "2019-11-07T22:22:40.000590Z",
"last_started_at": null,
"last_ended_at": null
},
{
"name": "periodic_purge_timeseries",
"disabled": false,
"recurrence": "FREQ=DAILY;DTSTART=20191106T180240Z;INTERVAL=2",
"threshold": "",
"purge_policies": {
"elasticsearch": [
{
"name": "actions",
"index": "actions",
"older_than_days": 29,
"custom_purge_field": "",
"disabled": false
},
{
"name": "converge-history",
"index": "converge-history",
"older_than_days": 0,
"custom_purge_field": "",
"disabled": false
}
],
"postgres": []
},
"last_elapsed": "0.019777s",
"next_due_at": "2019-11-08T18:02:40Z",
"last_enqueued_at": "0001-01-01T00:00:00Z",
"last_started_at": "2019-11-07T19:37:16.463036Z",
"last_ended_at": "2019-11-07T19:37:16.482813Z"
}
]
},
"compliance": {
"jobs": [
{
"name": "periodic_purge",
"disabled": true,
"recurrence": "FREQ=DAILY;DTSTART=20191106T180323Z;INTERVAL=2",
"threshold": "",
"purge_policies": {
"elasticsearch": [
{
"name": "compliance-reports",
"index": "comp-5-r",
"older_than_days": 100,
"custom_purge_field": "",
"disabled": false
},
{
"name": "compliance-scans",
"index": "comp-5-s",
"older_than_days": 100,
"custom_purge_field": "",
"disabled": false
}
],
"postgres": []
},
"last_elapsed": "0.015443s",
"next_due_at": "2019-11-08T18:03:23Z",
"last_enqueued_at": "0001-01-01T00:00:00Z",
"last_started_at": "2019-11-07T18:03:23.000254Z",
"last_ended_at": "2019-11-07T18:03:23.015697Z"
}
]
},
"event_feed": {
"jobs": [
{
"name": "periodic_purge",
"disabled": true,
"recurrence": "FREQ=DAILY;DTSTART=20191106T180243Z;INTERVAL=2",
"threshold": "",
"purge_policies": {
"elasticsearch": [
{
"name": "feed",
"index": "eventfeed-2-feeds",
"older_than_days": 60,
"custom_purge_field": "pub_timestamp",
"disabled": true
}
],
"postgres": []
},
"last_elapsed": "0.817205s",
"next_due_at": "2019-11-08T18:02:43Z",
"last_enqueued_at": "0001-01-01T00:00:00Z",
"last_started_at": "2019-11-07T18:02:43.000265Z",
"last_ended_at": "2019-11-07T18:02:43.817470Z"
}
]
},
"services": null
}
```
Example aggregate configuration payload
```json
{ "infra": {
"job_settings": [
{ "name":"delete_nodes",
"recurrence": "FREQ=SECONDLY;DTSTART=20191106T180240Z;INTERVAL=600",
"threshold": "2d"
},
{ "name":"missing_nodes",
"recurrence": "FREQ=SECONDLY;DTSTART=20191106T180240Z;INTERVAL=600",
"threshold": "2d"
},
{ "name":"missing_nodes_for_deletion",
"recurrence": "FREQ=SECONDLY;DTSTART=20191106T180240Z;INTERVAL=600",
"threshold": "31d"
},
{ "name":"periodic_purge_timeseries",
"recurrence": "FREQ=DAILY;DTSTART=20191106T180240Z;INTERVAL=2",
"purge_policies": {
"elasticsearch": [
{
"policy_name": "actions",
"older_than_days": 29,
"disabled": false
},
{
"policy_name": "converge-history",
"older_than_days": 29,
"disabled": false
}
]
}
}
]
},
"compliance": {
"job_settings": [
{
"name": "periodic_purge",
"disabled": true,
"recurrence": "FREQ=DAILY;DTSTART=20191106T180323Z;INTERVAL=2",
"purge_policies": {
"elasticsearch": [
{
"policy_name": "compliance-reports",
"older_than_days": 100,
"disabled": false
},
{
"policy_name": "compliance-scans",
"older_than_days": 100,
"disabled": false
}
]
}
}
]
},
"event_feed": {
"job_settings": [
{
"name": "periodic_purge",
"disabled": true,
"recurrence": "FREQ=DAILY;DTSTART=20191106T180243Z;INTERVAL=2",
"purge_policies": {
"elasticsearch": [
{
"policy_name": "feed",
"older_than_days": 60,
"disabled": true
}
]
}
}
]
}
}
```
Relates to:
* #1208
* #2108- Loading branch information
1 parent
f60964f
commit 16592de