Merge branch 'master' into physical_backup_restore

dashboard/src2/pages/ReleaseGroupBenchSites.vue

@@ -428,7 +428,7 @@ export default {
 				},
 			];
 		},
-		runDocMethod(name, methodName) {
+		runBenchMethod(name, methodName) {
 			const method = createResource({
 				url: 'press.api.client.run_doc_method',
 			});

press-semgrep-rules.yml

@@ -22,6 +22,7 @@ rules:
         - python
       references:
         - https://docs.python-guide.org/writing/gotchas/#mutable-default-arguments
+
   - id: except-with-db-code
     languages:
       - python
@@ -32,33 +33,33 @@ rules:
           except ...:
             $ERR_HANDL_BLK
       - pattern-either:
-        - pattern: |
-            try:
-              ...
-            except ...:
-              ...
-              $DOC.save(...)
-              ...
-              raise
-              ...
-        - pattern: |
-            try:
-              ...
-            except ...:
-              ...
-              frappe. ... .set_value(...)
-              ...
-              raise
-              ...
-        - pattern: |
-            try:
-              ...
-            except ...:
-              ...
-              $DOC.db_set(...)
-              ...
-              raise
-              ...
+          - pattern: |
+              try:
+                ...
+              except ...:
+                ...
+                $DOC.save(...)
+                ...
+                raise
+                ...
+          - pattern: |
+              try:
+                ...
+              except ...:
+                ...
+                frappe. ... .set_value(...)
+                ...
+                raise
+                ...
+          - pattern: |
+              try:
+                ...
+              except ...:
+                ...
+                $DOC.db_set(...)
+                ...
+                raise
+                ...
       - pattern-not: |
           try:
             ...
@@ -95,3 +96,30 @@ rules:
     message: except block has no db commit before raise. The db changes made won't persist assuming innodb tables.
     severity: ERROR
 
+  - id: retries-without-until
+    languages:
+      - yaml
+    patterns:
+      - pattern: |
+          ...
+          retries: $RETRIES
+          delay: $DELAY
+          ...
+
+      - pattern-not: |
+          ...
+          retries: $RETRIES
+          delay: $DELAY
+          until: $UNTIL
+          ...
+
+    paths:
+      include:
+        - 'press/playbooks/**/*.yml'
+    message: retry block doesn't have until condition. Only works with ansible 2.16 and above.
+    severity: ERROR
+    metadata:
+      category: correctness
+      references:
+        - https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_loops.html#retrying-a-task-until-a-condition-is-met
+        - https://docs.ansible.com/ansible/latest/reference_appendices/release_and_maintenance.html#ansible-community-changelogs

press/agent.py

@@ -1024,6 +1024,9 @@ def update_monitor_rules(self, rules, routes):
 	def get_job_status(self, id):
 		return self.get(f"jobs/{id}")
 
+	def cancel_job(self, id):
+		return self.post(f"jobs/{id}/cancel")
+
 	def get_site_sid(self, site, user=None):
 		if user:
 			data = {"user": user}

press/exceptions.py

@@ -51,3 +51,7 @@ class SiteUnderMaintenance(ValidationError):
 
 class SiteAlreadyArchived(ValidationError):
 	pass
+
+
+class InactiveDomains(ValidationError):
+	pass

press/infrastructure/doctype/virtual_machine_migration/virtual_machine_migration.py

@@ -282,6 +282,7 @@ def migration_steps(self):
 			(self.update_mounts, NoWait),
 			(self.update_bind_mount_permissions, NoWait),
 			(self.update_plan, NoWait),
+			(self.update_tls_certificate, NoWait),
 		]
 
 		steps = []
@@ -359,6 +360,7 @@ def terminate_previous_machine(self) -> StepStatus:
 		if copied_machine.status == "Terminated":
 			return StepStatus.Success
 		if copied_machine.status == "Pending":
+			copied_machine.sync()
 			return StepStatus.Pending
 
 		copied_machine.disable_termination_protection()
@@ -496,6 +498,24 @@ def update_plan(self) -> StepStatus:
 			server._change_plan(plan)
 		return StepStatus.Success
 
+	def update_tls_certificate(self) -> StepStatus:
+		"Update TLS certificate"
+		server = self.machine.get_server()
+		server.update_tls_certificate()
+
+		plays = frappe.get_all(
+			"Ansible Play",
+			{"server": server.name, "play": "Setup TLS Certificates", "creation": (">", self.creation)},
+			["status"],
+			order_by="creation desc",
+			limit=1,
+		)
+		if not plays:
+			return StepStatus.Failure
+		if plays[0].status == "Success":
+			return StepStatus.Success
+		return StepStatus.Failure
+
 	@frappe.whitelist()
 	def execute(self):
 		self.status = "Running"

press/playbooks/filebeat_update.yml

@@ -26,6 +26,8 @@
       apt:
         name: filebeat
         state: latest
+      result: result
+      until: result.rc == 0
       retries: 5
       delay: 120
 

press/playbooks/roles/earlyoom_memory_limits/tasks/main.yml

@@ -5,6 +5,8 @@
       - earlyoom
     state:
       present
+  register: result
+  until: result.rc == 0
   retries: 5
   delay: 120
 

press/playbooks/roles/extend_ec2_volume/tasks/main.yml

@@ -15,6 +15,8 @@
 
 - name: Extend Partition
   command: 'growpart {{ device }} 1'
+  register: result
+  until: result.rc == 0
   retries: 10
   delay: 10
   when: partitioned_disk

press/playbooks/roles/mariadb_10_4_to_10_6/tasks/main.yml

@@ -41,6 +41,8 @@
       - mariadb-client
       - libmariadbclient18
     state: latest
+  register: result
+  until: result.rc == 0
   retries: 5
   delay: 120
 

press/press/doctype/agent_job/agent_job.js

@@ -52,6 +52,7 @@ frappe.ui.form.on('Agent Job', {
 				__('Succeed and Process Job Updates'),
 				'succeed_and_process_job_updates',
 			],
+			[__('Cancel Job'), 'cancel_job', ['Pending', "Running"].includes(frm.doc.status)],
 		].forEach(([label, method, condition]) => {
 			frm.add_custom_button(
 				label,

press/press/doctype/agent_job/agent_job.py

@@ -307,6 +307,11 @@ def fail_and_process_job_updates(self):
 	def process_job_updates(self):
 		process_job_updates(self.name)
 
+	@frappe.whitelist()
+	def cancel_job(self):
+		agent = Agent(self.server, server_type=self.server_type)
+		agent.cancel_job(self.job_id)
+
 	def on_trash(self):
 		steps = frappe.get_all("Agent Job Step", filters={"agent_job": self.name})
 		for step in steps:

press/press/doctype/release_group_variable/release_group_variable.json

@@ -20,7 +20,7 @@
   },
   {
    "fieldname": "value",
-   "fieldtype": "Data",
+   "fieldtype": "Text",
    "in_list_view": 1,
    "label": "Value",
    "reqd": 1
@@ -36,7 +36,7 @@
  "index_web_pages_for_search": 1,
  "istable": 1,
  "links": [],
- "modified": "2024-02-11 14:56:12.851224",
+ "modified": "2025-01-20 15:12:45.664299",
  "modified_by": "Administrator",
  "module": "Press",
  "name": "Release Group Variable",

press/press/doctype/release_group_variable/release_group_variable.py

@@ -9,7 +9,7 @@ class ReleaseGroupVariable(Document):
 	# begin: auto-generated types
 	# This code is auto-generated. Do not modify anything in this block.
 
-	from typing import TYPE_CHECKING
+	from typing import TYPE_CHECKING, ClassVar
 
 	if TYPE_CHECKING:
 		from frappe.types import DF
@@ -19,16 +19,15 @@ class ReleaseGroupVariable(Document):
 		parent: DF.Data
 		parentfield: DF.Data
 		parenttype: DF.Data
-		value: DF.Data
+		value: DF.Text
 	# end: auto-generated types
 
-	dashboard_fields = ["key", "value"]
+	dashboard_fields: ClassVar = ["key", "value"]
 
 	@staticmethod
 	def get_list_query(query, filters=None, **list_args):
 		environmentVariable = frappe.qb.DocType("Release Group Variable")
 		query = query.where(environmentVariable.internal == 0).orderby(
 			environmentVariable.key, order=frappe.qb.asc
 		)
-		configs = query.run(as_dict=True)
-		return configs
+		return query.run(as_dict=True)

press/press/doctype/site/site.py

@@ -1449,6 +1449,15 @@ def get_login_sid(self, user="Administrator"):
 			try:
 				agent = Agent(self.server)
 				sid = agent.get_site_sid(self, user)
+			except requests.HTTPError as e:
+				if "validate_ip_address" in str(e):
+					frappe.throw(
+						f"Login with {user}'s credentials is IP restricted. Please remove the same and try again.",
+						frappe.ValidationError,
+					)
+				elif f"User {user} does not exist" in str(e):
+					frappe.throw(f"User {user} does not exist in the site", frappe.ValidationError)
+				raise e
 			except AgentRequestSkippedException:
 				frappe.throw(
 					"Server is unresponsive. Please try again in some time.",

press/press/doctype/site_migration/site_migration.py

@@ -12,6 +12,7 @@
 from press.agent import Agent
 from press.exceptions import (
 	CannotChangePlan,
+	InactiveDomains,
 	InsufficientSpaceOnServer,
 	MissingAppsInBench,
 	OngoingAgentJob,
@@ -76,6 +77,7 @@ class SiteMigration(Document):
 	def before_insert(self):
 		self.validate_apps()
 		self.validate_bench()
+		self.check_for_inactive_domains()
 		self.check_enough_space_on_destination_server()
 		if get_ongoing_migration(self.site, scheduled=True):
 			frappe.throw(f"Ongoing/Scheduled Site Migration for the site {frappe.bold(self.site)} exists.")
@@ -123,11 +125,21 @@ def validate_apps(self):
 				MissingAppsInBench,
 			)
 
+	def check_for_inactive_domains(self):
+		if domains := frappe.db.get_all(
+			"Site Domain", {"site": self.site, "status": ("!=", "Active")}, pluck="name"
+		):
+			frappe.throw(
+				f"Inactive custom domains exist: {','.join(domains)}. Please remove or fix the same.",
+				InactiveDomains,
+			)
+
 	@frappe.whitelist()
 	def start(self):
 		self.status = "Pending"
 		self.save()
 		self.check_for_ongoing_agent_jobs()
+		self.check_for_inactive_domains()
 		self.validate_apps()
 		self.check_enough_space_on_destination_server()
 		site: Site = frappe.get_doc("Site", self.site)
@@ -683,11 +695,13 @@ def run_scheduled_migrations():
 		try:
 			site_migration.start()
 		except OngoingAgentJob:
-			pass
+			pass  # ongoing jobs will finish in some time
 		except MissingAppsInBench as e:
 			site_migration.cleanup_and_fail(reason=str(e), force_activate=True)
 		except InsufficientSpaceOnServer as e:
 			site_migration.cleanup_and_fail(reason=str(e), force_activate=True)
+		except InactiveDomains as e:
+			site_migration.cleanup_and_fail(reason=str(e), force_activate=True)
 		except Exception as e:
 			log_error("Site Migration Start Error", exception=e)