Skip to content

Commit

Permalink
RHINENG-4700_and_5552: Use deletion for culling and rename attributes…
Browse files Browse the repository at this point in the history 
… in custom staleness (#1573)

* Use deletion for culling and rename attributes in custom staleness

* feat(RHINENG-2631): Support new reporter values for yupana (#1557)

* add db migration script for columns renaming and update config.py

* removed a commented out line

* Add migration scripts for renaming stalenss attributes

---------

Co-authored-by: Richard Oliveri <[email protected]>
  • Loading branch information
@thearifismail @roliveri
thearifismail and roliveri authored Dec 6, 2023
1 parent 586e0d7 commit e82471f
Show file tree
Hide file tree
Showing 19 changed files with 294 additions and 251 deletions.
12 changes: 6 additions & 6 deletions api/staleness.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,12 @@ def _get_return_data():
"id": "1ba078bb-8461-474c-8498-1e50a1975cfb",
"account_id": "0123456789",
"org_id": "123",
"conventional_staleness_delta": "1",
"conventional_stale_warning_delta": "7",
"conventional_culling_delta": "14",
"immutable_staleness_delta": "2",
"immutable_stale_warning_delta": "120",
"immutable_culling_delta": "180",
"conventional_time_to_stale": "1",
"conventional_time_to_stale_warning": "7",
"conventional_time_to_delete": "14",
"immutable_time_to_stale": "2",
"immutable_time_to_stale_warning": "120",
"immutable_time_to_delete": "180",
"created_at": "2023-07-28T14:32:16.353082",
"updated_at": "2023-07-28T14:32:16.353082",
}
Expand Down
24 changes: 12 additions & 12 deletions api/staleness_query.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,12 +38,12 @@ def _build_staleness_sys_default(org_id, config=None):
{
"id": "system_default",
"org_id": org_id,
"conventional_staleness_delta": config.conventional_staleness_seconds,
"conventional_stale_warning_delta": config.conventional_stale_warning_seconds,
"conventional_culling_delta": config.conventional_culling_seconds,
"immutable_staleness_delta": config.immutable_staleness_seconds,
"immutable_stale_warning_delta": config.immutable_stale_warning_seconds,
"immutable_culling_delta": config.immutable_culling_seconds,
"conventional_time_to_stale": config.conventional_time_to_stale_seconds,
"conventional_time_to_stale_warning": config.conventional_time_to_stale_warning_seconds,
"conventional_time_to_delete": config.conventional_time_to_delete_seconds,
"immutable_time_to_stale": config.immutable_time_to_stale_seconds,
"immutable_time_to_stale_warning": config.immutable_time_to_stale_warning_seconds,
"immutable_time_to_delete": config.immutable_time_to_delete_seconds,
"created_on": "N/A",
"modified_on": "N/A",
}
Expand All @@ -58,12 +58,12 @@ def _build_serialized_acc_staleness_obj(staleness):
{
"id": str(staleness.id),
"org_id": staleness.org_id,
"conventional_staleness_delta": staleness.conventional_staleness_delta,
"conventional_stale_warning_delta": staleness.conventional_stale_warning_delta,
"conventional_culling_delta": staleness.conventional_culling_delta,
"immutable_staleness_delta": staleness.immutable_staleness_delta,
"immutable_stale_warning_delta": staleness.immutable_stale_warning_delta,
"immutable_culling_delta": staleness.immutable_culling_delta,
"conventional_time_to_stale": staleness.conventional_time_to_stale,
"conventional_time_to_stale_warning": staleness.conventional_time_to_stale_warning,
"conventional_time_to_delete": staleness.conventional_time_to_delete,
"immutable_time_to_stale": staleness.immutable_time_to_stale,
"immutable_time_to_stale_warning": staleness.immutable_time_to_stale_warning,
"immutable_time_to_delete": staleness.immutable_time_to_delete,
"created_on": staleness.created_on,
"modified_on": staleness.modified_on,
}
Expand Down
24 changes: 16 additions & 8 deletions app/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -230,21 +230,29 @@ def __init__(self, runtime_environment):
minutes=int(os.environ.get("CULLING_CULLED_OFFSET_MINUTES", "0")),
)

self.conventional_staleness_seconds = int(os.environ.get("CONVENTIONAL_STALENESS_SECONDS", 104400)) # 29 hours
self.conventional_time_to_stale_seconds = int(
os.environ.get("CONVENTIONAL_TIME_TO_STALE_SECONDS", 104400)
) # 29 hours

self.conventional_stale_warning_seconds = os.environ.get(
"CONVENTIONAL_STALENESS_WARNING_SECONDS", self.days_to_seconds(7)
self.conventional_time_to_stale_warning_seconds = os.environ.get(
"CONVENTIONAL_TIME_TO_STALE_WARNING_SECONDS", self.days_to_seconds(7)
)

self.conventional_culling_seconds = os.environ.get("CONVENTIONAL_CULLING_SECONDS", self.days_to_seconds(14))
self.conventional_time_to_delete_seconds = os.environ.get(
"CONVENTIONAL_TIME_TO_DELETE_SECONDS", self.days_to_seconds(14)
)

self.immutable_staleness_seconds = os.environ.get("IMMUTABLE_STALENESS_SECONDS", self.days_to_seconds(2))
self.immutable_time_to_stale_seconds = os.environ.get(
"IMMUTABLE_TIME_TO_STALE_SECONDS", self.days_to_seconds(2)
)

self.immutable_stale_warning_seconds = os.environ.get(
"IMMUTABLE_STALENESS_WARNING_SECONDS", self.days_to_seconds(180)
self.immutable_time_to_stale_warning_seconds = os.environ.get(
"IMMUTABLE_TIME_TO_STALE_WARNING_SECONDS", self.days_to_seconds(180)
)

self.immutable_culling_seconds = os.environ.get("IMMUTABLE_CULLING_SECONDS", self.days_to_seconds(730))
self.immutable_time_to_delete_seconds = os.environ.get(
"IMMUTABLE_TIME_TO_DELETE_SECONDS", self.days_to_seconds(730)
)

self.xjoin_graphql_url = os.environ.get("XJOIN_GRAPHQL_URL", "http://localhost:4000/graphql")

Expand Down
12 changes: 6 additions & 6 deletions app/culling.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,14 +44,14 @@ def __init__(self, staleness, host_type):

self.staleness_host_type = {
None: {
"stale": staleness["conventional_staleness_delta"],
"warning": staleness["conventional_stale_warning_delta"],
"culled": staleness["conventional_culling_delta"],
"stale": staleness["conventional_time_to_stale"],
"warning": staleness["conventional_time_to_stale_warning"],
"culled": staleness["conventional_time_to_delete"],
},
"edge": {
"stale": staleness["immutable_staleness_delta"],
"warning": staleness["immutable_stale_warning_delta"],
"culled": staleness["immutable_culling_delta"],
"stale": staleness["immutable_time_to_stale"],
"warning": staleness["immutable_time_to_stale_warning"],
"culled": staleness["immutable_time_to_delete"],
},
}

Expand Down
72 changes: 36 additions & 36 deletions app/models.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -549,50 +549,50 @@ class Staleness(db.Model):
def __init__(
self,
org_id,
conventional_staleness_delta=None,
conventional_stale_warning_delta=None,
conventional_culling_delta=None,
immutable_staleness_delta=None,
immutable_stale_warning_delta=None,
immutable_culling_delta=None,
conventional_time_to_stale=None,
conventional_time_to_stale_warning=None,
conventional_time_to_delete=None,
immutable_time_to_stale=None,
immutable_time_to_stale_warning=None,
immutable_time_to_delete=None,
):
if not org_id:
raise ValidationException("Staleness org_id cannot be null.")

self.org_id = org_id
self.conventional_staleness_delta = conventional_staleness_delta
self.conventional_stale_warning_delta = conventional_stale_warning_delta
self.conventional_culling_delta = conventional_culling_delta
self.immutable_staleness_delta = immutable_staleness_delta
self.immutable_stale_warning_delta = immutable_stale_warning_delta
self.immutable_culling_delta = immutable_culling_delta
self.conventional_time_to_stale = conventional_time_to_stale
self.conventional_time_to_stale_warning = conventional_time_to_stale_warning
self.conventional_time_to_delete = conventional_time_to_delete
self.immutable_time_to_stale = immutable_time_to_stale
self.immutable_time_to_stale_warning = immutable_time_to_stale_warning
self.immutable_time_to_delete = immutable_time_to_delete

def days_to_seconds(n_days):
factor = 86400
return n_days * factor

def update(self, input_acc):
if input_acc.conventional_staleness_delta:
self.conventional_staleness_delta = input_acc.conventional_staleness_delta
if input_acc.conventional_stale_warning_delta:
self.conventional_stale_warning_delta = input_acc.conventional_stale_warning_delta
if input_acc.conventional_culling_delta:
self.conventional_culling_delta = input_acc.conventional_culling_delta
if input_acc.immutable_staleness_delta:
self.immutable_staleness_delta = input_acc.immutable_staleness_delta
if input_acc.immutable_stale_warning_delta:
self.immutable_stale_warning_delta = input_acc.immutable_stale_warning_delta
if input_acc.immutable_culling_delta:
self.immutable_culling_delta = input_acc.immutable_culling_delta
if input_acc.conventional_time_to_stale:
self.conventional_time_to_stale = input_acc.conventional_time_to_stale
if input_acc.conventional_time_to_stale_warning:
self.conventional_time_to_stale_warning = input_acc.conventional_time_to_stale_warning
if input_acc.conventional_time_to_delete:
self.conventional_time_to_delete = input_acc.conventional_time_to_delete
if input_acc.immutable_time_to_stale:
self.immutable_time_to_stale = input_acc.immutable_time_to_stale
if input_acc.immutable_time_to_stale_warning:
self.immutable_time_to_stale_warning = input_acc.immutable_time_to_stale_warning
if input_acc.immutable_time_to_delete:
self.immutable_time_to_delete = input_acc.immutable_time_to_delete

id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
org_id = db.Column(db.String(36), nullable=False)
conventional_staleness_delta = db.Column(db.Integer, default=104400, nullable=False)
conventional_stale_warning_delta = db.Column(db.Integer, default=days_to_seconds(7), nullable=False)
conventional_culling_delta = db.Column(db.Integer, default=days_to_seconds(14), nullable=False)
immutable_staleness_delta = db.Column(db.Integer, default=days_to_seconds(2), nullable=False)
immutable_stale_warning_delta = db.Column(db.Integer, default=days_to_seconds(180), nullable=False)
immutable_culling_delta = db.Column(db.Integer, default=days_to_seconds(730), nullable=False)
conventional_time_to_stale = db.Column(db.Integer, default=104400, nullable=False)
conventional_time_to_stale_warning = db.Column(db.Integer, default=days_to_seconds(7), nullable=False)
conventional_time_to_delete = db.Column(db.Integer, default=days_to_seconds(14), nullable=False)
immutable_time_to_stale = db.Column(db.Integer, default=days_to_seconds(2), nullable=False)
immutable_time_to_stale_warning = db.Column(db.Integer, default=days_to_seconds(180), nullable=False)
immutable_time_to_delete = db.Column(db.Integer, default=days_to_seconds(730), nullable=False)
created_on = db.Column(db.DateTime(timezone=True), default=_time_now)
modified_on = db.Column(db.DateTime(timezone=True), default=_time_now, onupdate=_time_now)

Expand Down Expand Up @@ -862,12 +862,12 @@ def __init__(self, *args, **kwargs):


class StalenessSchema(MarshmallowSchema):
conventional_staleness_delta = fields.Integer(validate=marshmallow_validate.Range(min=1, max=2147483647))
conventional_stale_warning_delta = fields.Integer(validate=marshmallow_validate.Range(min=1, max=2147483647))
conventional_culling_delta = fields.Integer(validate=marshmallow_validate.Range(min=1, max=2147483647))
immutable_staleness_delta = fields.Integer(validate=marshmallow_validate.Range(min=1, max=2147483647))
immutable_stale_warning_delta = fields.Integer(validate=marshmallow_validate.Range(min=1, max=2147483647))
immutable_culling_delta = fields.Integer(validate=marshmallow_validate.Range(min=1, max=2147483647))
conventional_time_to_stale = fields.Integer(validate=marshmallow_validate.Range(min=1, max=2147483647))
conventional_time_to_stale_warning = fields.Integer(validate=marshmallow_validate.Range(min=1, max=2147483647))
conventional_time_to_delete = fields.Integer(validate=marshmallow_validate.Range(min=1, max=2147483647))
immutable_time_to_stale = fields.Integer(validate=marshmallow_validate.Range(min=1, max=2147483647))
immutable_time_to_stale_warning = fields.Integer(validate=marshmallow_validate.Range(min=1, max=2147483647))
immutable_time_to_delete = fields.Integer(validate=marshmallow_validate.Range(min=1, max=2147483647))

def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
38 changes: 18 additions & 20 deletions app/serialization.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,24 +122,22 @@ def serialize_host(
# TODO: In future, this must handle groups staleness

if host.system_profile_facts.get("host_type") == "edge":
stale_timestamp = staleness_timestamps.stale_timestamp(
host.modified_on, staleness["immutable_staleness_delta"]
)
stale_timestamp = staleness_timestamps.stale_timestamp(host.modified_on, staleness["immutable_time_to_stale"])
stale_warning_timestamp = staleness_timestamps.stale_warning_timestamp(
host.modified_on, staleness["immutable_stale_warning_delta"]
host.modified_on, staleness["immutable_time_to_stale_warning"]
)
culled_timestamp = staleness_timestamps.culled_timestamp(
host.modified_on, staleness["immutable_culling_delta"]
host.modified_on, staleness["immutable_time_to_delete"]
)
else:
stale_timestamp = staleness_timestamps.stale_timestamp(
host.modified_on, staleness["conventional_staleness_delta"]
host.modified_on, staleness["conventional_time_to_stale"]
)
stale_warning_timestamp = staleness_timestamps.stale_warning_timestamp(
host.modified_on, staleness["conventional_stale_warning_delta"]
host.modified_on, staleness["conventional_time_to_stale_warning"]
)
culled_timestamp = staleness_timestamps.culled_timestamp(
host.modified_on, staleness["conventional_culling_delta"]
host.modified_on, staleness["conventional_time_to_delete"]
)

serialized_host = {**serialize_canonical_facts(host.canonical_facts)}
Expand Down Expand Up @@ -381,12 +379,12 @@ def serialize_staleness_response(staleness):
return {
"id": _serialize_uuid(staleness.id),
"org_id": staleness.org_id,
"conventional_staleness_delta": staleness.conventional_staleness_delta,
"conventional_stale_warning_delta": staleness.conventional_stale_warning_delta,
"conventional_culling_delta": staleness.conventional_culling_delta,
"immutable_staleness_delta": staleness.immutable_staleness_delta,
"immutable_stale_warning_delta": staleness.immutable_stale_warning_delta,
"immutable_culling_delta": staleness.immutable_culling_delta,
"conventional_time_to_stale": staleness.conventional_time_to_stale,
"conventional_time_to_stale_warning": staleness.conventional_time_to_stale_warning,
"conventional_time_to_delete": staleness.conventional_time_to_delete,
"immutable_time_to_stale": staleness.immutable_time_to_stale,
"immutable_time_to_stale_warning": staleness.immutable_time_to_stale_warning,
"immutable_time_to_delete": staleness.immutable_time_to_delete,
"created": "N/A" if staleness.created_on == "N/A" else _serialize_datetime(staleness.created_on),
"updated": "N/A" if staleness.modified_on == "N/A" else _serialize_datetime(staleness.modified_on),
}
Expand All @@ -398,10 +396,10 @@ def serialize_staleness_to_dict(staleness_obj) -> dict:
to a simple dictionary. This contains less information
"""
return {
"conventional_staleness_delta": staleness_obj.conventional_staleness_delta,
"conventional_stale_warning_delta": staleness_obj.conventional_stale_warning_delta,
"conventional_culling_delta": staleness_obj.conventional_culling_delta,
"immutable_staleness_delta": staleness_obj.immutable_staleness_delta,
"immutable_stale_warning_delta": staleness_obj.immutable_stale_warning_delta,
"immutable_culling_delta": staleness_obj.immutable_culling_delta,
"conventional_time_to_stale": staleness_obj.conventional_time_to_stale,
"conventional_time_to_stale_warning": staleness_obj.conventional_time_to_stale_warning,
"conventional_time_to_delete": staleness_obj.conventional_time_to_delete,
"immutable_time_to_stale": staleness_obj.immutable_time_to_stale,
"immutable_time_to_stale_warning": staleness_obj.immutable_time_to_stale_warning,
"immutable_time_to_delete": staleness_obj.immutable_time_to_delete,
}
24 changes: 12 additions & 12 deletions lib/staleness.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,23 +9,23 @@

def add_staleness(staleness_data) -> Staleness:
logger.debug("Creating a new AccountStaleness: %s", staleness_data)
conventional_staleness_delta = staleness_data.get("conventional_staleness_delta")
conventional_stale_warning_delta = staleness_data.get("conventional_stale_warning_delta")
conventional_culling_delta = staleness_data.get("conventional_culling_delta")
immutable_staleness_delta = staleness_data.get("immutable_staleness_delta")
immutable_stale_warning_delta = staleness_data.get("immutable_stale_warning_delta")
immutable_culling_delta = staleness_data.get("immutable_culling_delta")
conventional_time_to_stale = staleness_data.get("conventional_time_to_stale")
conventional_time_to_stale_warning = staleness_data.get("conventional_time_to_stale_warning")
conventional_time_to_delete = staleness_data.get("conventional_time_to_delete")
immutable_time_to_stale = staleness_data.get("immutable_time_to_stale")
immutable_time_to_stale_warning = staleness_data.get("immutable_time_to_stale_warning")
immutable_time_to_delete = staleness_data.get("immutable_time_to_delete")
org_id = get_current_identity().org_id

with session_guard(db.session):
new_staleness = Staleness(
org_id=org_id,
conventional_staleness_delta=conventional_staleness_delta,
conventional_stale_warning_delta=conventional_stale_warning_delta,
conventional_culling_delta=conventional_culling_delta,
immutable_staleness_delta=immutable_staleness_delta,
immutable_stale_warning_delta=immutable_stale_warning_delta,
immutable_culling_delta=immutable_culling_delta,
conventional_time_to_stale=conventional_time_to_stale,
conventional_time_to_stale_warning=conventional_time_to_stale_warning,
conventional_time_to_delete=conventional_time_to_delete,
immutable_time_to_stale=immutable_time_to_stale,
immutable_time_to_stale_warning=immutable_time_to_stale_warning,
immutable_time_to_delete=immutable_time_to_delete,
)
db.session.add(new_staleness)
db.session.flush()
Expand Down
Loading

0 comments on commit e82471f

Please sign in to comment.