Error monitoring

CHANGELOG.md

@@ -3,9 +3,16 @@
 ## Pending
 
 ### Added
+- Support Error Monitoring. See
+  [integration docs](https://scoutapm.com/docs/python/error-monitoring).
+  ([PR #651](https://github.com/scoutapp/scout_apm_python/pull/651))
+- Deprecate `backtrace.capture` in favor of `backtrace.capture_backtrace`
 
 ### Fixed
 - Setup metadata keywords now contains an array of strings.
+- Remove non-project paths from traces.
+  ([Issue #416](https://github.com/scoutapp/scout_apm_python/issues/416))
+
 
 ## [2.20.0] 2021-07-21
 - Removed parsing queue time from Amazon ALB header, X-Amzn-Trace-Id.

setup.py

@@ -67,6 +67,7 @@
         'urllib3[secure] < 1.25 ; python_version < "3.5"',
         'urllib3[secure] < 2 ; python_version >= "3.5"',
         "wrapt>=1.10,<2.0",
+        "requests",
     ],
     keywords=["apm", "performance monitoring", "development"],
     classifiers=[

src/scout_apm/api/__init__.py

@@ -1,9 +1,12 @@
 # coding=utf-8
 from __future__ import absolute_import, division, print_function, unicode_literals
 
+import sys
+
 import scout_apm.core
 from scout_apm.compat import ContextDecorator, text
 from scout_apm.core.config import ScoutConfig
+from scout_apm.core.error import ErrorMonitor
 from scout_apm.core.tracked_request import TrackedRequest
 
 # The async_ module can only be shipped on Python 3.6+
@@ -19,6 +22,7 @@ class AsyncDecoratorMixin(object):
     "BackgroundTransaction",
     "Config",
     "Context",
+    "Error",
     "WebTransaction",
     "install",
     "instrument",
@@ -143,3 +147,51 @@ def rename_transaction(name):
     if name is not None:
         tracked_request = TrackedRequest.instance()
         tracked_request.tag("transaction.name", name)
+
+
+class Error(object):
+    @classmethod
+    def capture(
+        cls,
+        exception,
+        request_path=None,
+        request_params=None,
+        session=None,
+        custom_controller=None,
+        custom_params=None,
+    ):
+        """
+        Capture the exception manually.
+
+        Utilizes sys.exc_info to gather the traceback. This has the side
+        effect that if another exception is raised before calling
+        ``Error.capture``, the traceback will match the most recently
+        raised exception.
+
+        Includes any context added for the TrackedRequest.
+
+        :exception: Any exception.
+        :request_path: Any String identifying the relative path of the request.
+              Example: "/hello-world/"
+        :request_params: Any json-serializable dict representing the
+              querystring parameters.
+              Example: {"page": 1}
+        :session: Any json-serializable dict representing the
+              request session.
+              Example: {"step": 0}
+        :custom_controller: Any String identifying the controller or job.
+              Example: "send_email"
+        :custom_params: Any json-serializable dict.
+              Example: {"to": "[email protected]", "from": "[email protected]"}
+        :returns: nothing.
+        """
+        if isinstance(exception, Exception):
+            exc_info = (exception.__class__, exception, sys.exc_info()[2])
+            ErrorMonitor.send(
+                exc_info,
+                request_path=request_path,
+                request_params=request_params,
+                session=session,
+                custom_controller=custom_controller,
+                custom_params=custom_params,
+            )

src/scout_apm/celery.py

@@ -2,14 +2,34 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 import datetime as dt
+import logging
 
 from celery.signals import before_task_publish, task_failure, task_postrun, task_prerun
 
+try:
+    import django
+
+    if django.VERSION < (3, 1):
+        from django.views.debug import get_safe_settings
+    else:
+        from django.views.debug import SafeExceptionReporterFilter
+
+        def get_safe_settings():
+            return SafeExceptionReporterFilter().get_safe_settings()
+
+
+except ImportError:
+    # Django not installed
+    get_safe_settings = None
+
 import scout_apm.core
 from scout_apm.compat import datetime_to_timestamp
 from scout_apm.core.config import scout_config
+from scout_apm.core.error import ErrorMonitor
 from scout_apm.core.tracked_request import TrackedRequest
 
+logger = logging.getLogger(__name__)
+
 
 def before_task_publish_callback(headers=None, properties=None, **kwargs):
     if "scout_task_start" not in headers:
@@ -52,10 +72,52 @@ def task_postrun_callback(task=None, **kwargs):
     tracked_request.stop_span()
 
 
-def task_failure_callback(task_id=None, **kwargs):
+def task_failure_callback(
+    sender,
+    task_id=None,
+    exception=None,
+    args=None,
+    kwargs=None,
+    traceback=None,
+    **remaining
+):
     tracked_request = TrackedRequest.instance()
     tracked_request.tag("error", "true")
 
+    custom_controller = sender.name
+    custom_params = {
+        "celery": {
+            "task_id": task_id,
+            "args": args,
+            "kwargs": kwargs,
+        }
+    }
+
+    # Look up the django settings if populated.
+    environment = None
+    if get_safe_settings:
+        try:
+            environment = get_safe_settings()
+        except django.core.exceptions.ImproperlyConfigured as exc:
+            # Django not setup correctly
+            logger.debug(
+                "Celery integration does not have django configured properly: %r", exc
+            )
+            pass
+        except Exception as exc:
+            logger.debug(
+                "Celery task_failure callback exception: %r", exc, exc_info=exc
+            )
+            pass
+
+    exc_info = (exception.__class__, exception, traceback)
+    ErrorMonitor.send(
+        exc_info,
+        environment=environment,
+        custom_params=custom_params,
+        custom_controller=custom_controller,
+    )
+
 
 def install(app=None):
     if app is not None:

src/scout_apm/compat.py

@@ -2,6 +2,7 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 import datetime as dt
+import gzip
 import inspect
 import sys
 from functools import wraps
@@ -151,9 +152,27 @@ def urllib3_cert_pool_manager(**kwargs):
     return urllib3.PoolManager(cert_reqs=CERT_REQUIRED, ca_certs=certifi.where())
 
 
+if sys.version_info >= (3, 2):
+
+    def gzip_compress(data):
+        return gzip.compress(data)
+
+
+else:
+    import io
+
+    def gzip_compress(data):
+        """Reimplementation gzip.compress for python 2.7"""
+        buf = io.BytesIO()
+        with gzip.GzipFile(fileobj=buf, mode="wb") as f:
+            f.write(data)
+        return buf.getvalue()
+
+
 __all__ = [
     "ContextDecorator",
     "datetime_to_timestamp",
+    "gzip_compress",
     "kwargs_only",
     "parse_qsl",
     "queue",

src/scout_apm/core/__init__.py

@@ -12,6 +12,7 @@
 from scout_apm.core.agent.manager import CoreAgentManager
 from scout_apm.core.agent.socket import CoreAgentSocketThread
 from scout_apm.core.config import scout_config
+from scout_apm.core.error_service import ErrorServiceThread
 from scout_apm.core.metadata import report_app_metadata
 
 logger = logging.getLogger(__name__)
@@ -61,7 +62,7 @@ def install(config=None):
 def shutdown():
     timeout_seconds = scout_config.value("shutdown_timeout_seconds")
 
-    def callback(queue_size):
+    def apm_callback(queue_size):
         if scout_config.value("shutdown_message_enabled"):
             print(  # noqa: T001
                 (
@@ -75,6 +76,25 @@ def callback(queue_size):
                 file=sys.stderr,
             )
 
+    def error_callback(queue_size):
+        if scout_config.value("shutdown_message_enabled"):
+            print(  # noqa: T001
+                (
+                    "Scout draining {queue_size} error{s} for up to"
+                    + " {timeout_seconds} seconds"
+                ).format(
+                    queue_size=queue_size,
+                    s=("" if queue_size == 1 else "s"),
+                    timeout_seconds=timeout_seconds,
+                ),
+                file=sys.stderr,
+            )
+
     CoreAgentSocketThread.wait_until_drained(
-        timeout_seconds=timeout_seconds, callback=callback
+        timeout_seconds=timeout_seconds, callback=apm_callback
     )
+
+    if scout_config.value("errors_enabled"):
+        ErrorServiceThread.wait_until_drained(
+            timeout_seconds=timeout_seconds, callback=error_callback
+        )

src/scout_apm/core/backtrace.py

@@ -2,9 +2,11 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 import itertools
+import os
 import sys
 import sysconfig
 import traceback
+import warnings
 
 # Maximum non-Scout frames to target retrieving
 LIMIT = 50
@@ -21,29 +23,66 @@ def filter_frames(frames):
             yield frame
 
 
+def module_filepath(module, filepath):
+    """Get the filepath relative to the base module."""
+    root_module = module.split(".", 1)[0]
+    if root_module == module:
+        return os.path.basename(filepath)
+
+    module_dir = sys.modules[root_module].__file__.rsplit(os.sep, 2)[0]
+    return filepath.split(module_dir, 1)[-1].lstrip(os.sep)
+
+
+def filepath(frame):
+    """Get the filepath for frame."""
+    module = frame.f_globals.get("__name__", None)
+    filepath = frame.f_code.co_filename
+
+    if filepath.endswith(".pyc"):
+        filepath = filepath[:-1]
+
+    if not module:
+        return filepath
+    return module_filepath(module, filepath)
+
+
 if sys.version_info >= (3, 5):
 
-    def frame_walker():
-        """Iterate over each frame of the stack.
+    def stacktrace_walker(tb):
+        """Iterate over each frame of the stack downards for exceptions."""
+        for frame, lineno in traceback.walk_tb(tb):
+            name = frame.f_code.co_name
+            yield {"file": filepath(frame), "line": lineno, "function": name}
+
+    def backtrace_walker():
+        """Iterate over each frame of the stack upwards.
 
         Taken from python3/traceback.ExtractSummary.extract to support
         iterating over the entire stack, but without creating a large
         data structure.
         """
-        for frame, lineno in traceback.walk_stack(sys._getframe().f_back):
-            co = frame.f_code
-            filename = co.co_filename
-            name = co.co_name
-            yield {"file": filename, "line": lineno, "function": name}
-
-    def capture():
-        return list(itertools.islice(filter_frames(frame_walker()), LIMIT))
+        start_frame = sys._getframe().f_back
+        for frame, lineno in traceback.walk_stack(start_frame):
+            name = frame.f_code.co_name
+            yield {"file": filepath(frame), "line": lineno, "function": name}
 
 
 else:
 
-    def frame_walker():
-        """Iterate over each frame of the stack.
+    def stacktrace_walker(tb):
+        """Iterate over each frame of the stack downards for exceptions."""
+        while tb is not None:
+            lineno = tb.tb_lineno
+            name = tb.tb_frame.f_code.co_name
+            yield {
+                "file": filepath(tb.tb_frame),
+                "line": lineno,
+                "function": name,
+            }
+            tb = tb.tb_next
+
+    def backtrace_walker():
+        """Iterate over each frame of the stack upwards.
 
         Taken from python2.7/traceback.extract_stack to support iterating
         over the entire stack, but without creating a large data structure.
@@ -52,15 +91,29 @@ def frame_walker():
             raise ZeroDivisionError
         except ZeroDivisionError:
             # Get the current frame
-            f = sys.exc_info()[2].tb_frame.f_back
-
-        while f is not None:
-            lineno = f.f_lineno
-            co = f.f_code
-            filename = co.co_filename
-            name = co.co_name
-            yield {"file": filename, "line": lineno, "function": name}
-            f = f.f_back
-
-    def capture():
-        return list(itertools.islice(filter_frames(frame_walker()), LIMIT))
+            frame = sys.exc_info()[2].tb_frame.f_back
+
+        while frame is not None:
+            lineno = frame.f_lineno
+            name = frame.f_code.co_name
+            yield {"file": filepath(frame), "line": lineno, "function": name}
+            frame = frame.f_back
+
+
+def capture_backtrace():
+    walker = filter_frames(backtrace_walker())
+    return list(itertools.islice(walker, LIMIT))
+
+
+def capture_stacktrace(tb):
+    walker = stacktrace_walker(tb)
+    return list(reversed(list(itertools.islice(walker, LIMIT))))
+
+
+def capture():
+    warnings.warn(
+        "capture is deprecated, instead use capture_backtrace instead.",
+        DeprecationWarning,
+        2,
+    )
+    return capture_backtrace()