Skip to content

Commit

Permalink
Address major performance issue with recreating requests sessions ove…
Browse files Browse the repository at this point in the history
…r and over
  • Loading branch information
matteius committed Oct 24, 2024
1 parent 16c1346 commit ae2b85c
Show file tree
Hide file tree
Showing 3 changed files with 91 additions and 3 deletions.
1 change: 0 additions & 1 deletion pipenv/routines/install.py
Original file line number Diff line number Diff line change
Expand Up @@ -701,7 +701,6 @@ def do_init(
pypi_mirror=pypi_mirror,
categories=categories,
)
err.print(packages_updated)

if not allow_global and not deploy and "PIPENV_ACTIVE" not in os.environ:
console.print(
Expand Down
89 changes: 89 additions & 0 deletions pipenv/utils/developers.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
import cProfile
import functools
import os
import pstats
import threading
from datetime import datetime
from pstats import SortKey

# Keep track of lock stats
lock_stats = {"acquire_count": 0, "wait_times": [], "acquire_locations": []}

# Patch threading.Lock to track usage
original_lock = threading.Lock


def instrumented_lock():
lock = original_lock()
original_acquire = lock.acquire

@functools.wraps(original_acquire)
def traced_acquire(*args, **kwargs):
import traceback

lock_stats["acquire_count"] += 1
# Get call location
stack = traceback.extract_stack()
# Get the caller's frame (excluding this function)
caller = stack[-2]
lock_stats["acquire_locations"].append(f"{caller.filename}:{caller.lineno}")
return original_acquire(*args, **kwargs)

lock.acquire = traced_acquire
return lock


# Apply the patch
threading.Lock = instrumented_lock


def profile_method(output_dir="profiles"):
"""
Decorator to profile pipenv method execution.
"""

def decorator(func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
# Create output directory if it doesn't exist
os.makedirs(output_dir, exist_ok=True)

# Create profile filename with timestamp
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
profile_name = f"{func.__name__}_{timestamp}"
profile_path = os.path.join(output_dir, f"{profile_name}.prof")

# Setup profiler
profiler = cProfile.Profile()

# Start profiling
profiler.enable()

try:
# Run the actual pipenv command
result = func(*args, **kwargs)

return result
finally:
# Stop profiling
profiler.disable()

# Save stats
stats = pstats.Stats(profiler)
stats.sort_stats(SortKey.CUMULATIVE)
stats.dump_stats(profile_path)
print(f"\nProfile saved to: {profile_path}")
stats.print_stats(20)
# Print lock statistics
print("\nLock Statistics:")
print(f"Total lock acquisitions: {lock_stats['acquire_count']}")
print("\nTop 10 lock acquisition locations:")
from collections import Counter

locations = Counter(lock_stats["acquire_locations"])
for loc, count in locations.most_common(10):
print(f"{count:5d} times: {loc}")

return wrapper

return decorator
4 changes: 2 additions & 2 deletions pipenv/utils/resolver.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import sys
import tempfile
import warnings
from functools import lru_cache
from functools import cached_property, lru_cache
from pathlib import Path
from typing import Dict, List, Optional

Expand Down Expand Up @@ -308,7 +308,7 @@ def pip_options(self):
)
return pip_options

@property
@cached_property
def session(self):
return self.pip_command._build_session(self.pip_options)

Expand Down

0 comments on commit ae2b85c

Please sign in to comment.