Add tracemalloc utility view to API and improve k6 scripts for memory leak debugging

api/conf/urls/__init__.py

@@ -9,6 +9,7 @@
 from django.contrib import admin
 from django.urls import include, path
 from django.views.generic import RedirectView
+from rest_framework.decorators import api_view
 from rest_framework.routers import SimpleRouter
 
 from api.views.audio_views import AudioViewSet
@@ -17,6 +18,34 @@
 from conf.urls.auth_tokens import urlpatterns as auth_tokens_urlpatterns
 from conf.urls.deprecations import urlpatterns as deprecations_urlpatterns
 from conf.urls.openapi import urlpatterns as openapi_urlpatterns
+from conf.wsgi import TRACING
+
+
+if TRACING:
+    prev_snapshot = None
+
+    @api_view(["GET"])
+    def _trace_view(request):
+        global prev_snapshot
+        import pprint
+        import time
+        import tracemalloc
+
+        from django.http.response import HttpResponse
+
+        snapshot = tracemalloc.take_snapshot()
+        if prev_snapshot:
+            top_stats = snapshot.compare_to(prev_snapshot, "lineno", cumulative=True)[
+                :10
+            ]
+        else:
+            top_stats = snapshot.statistics("lineno", cumulative=True)[:10]
+
+        prev_snapshot = snapshot
+
+        content = f"{time.time()}\n" f"{pprint.pformat(top_stats, indent=4)}"
+
+        return HttpResponse(content=content, status=200, content_type="text/plain")
 
 
 versioned_paths = [
@@ -37,5 +66,8 @@
     path("v1/", include(versioned_paths)),
 ]
 
+if TRACING:
+    urlpatterns.append(path("_trace/", _trace_view))
+
 if settings.ENVIRONMENT == "local":
     urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)

api/conf/wsgi.py

@@ -9,6 +9,15 @@
 
 import os
 
+
+if os.getenv("ENABLE_TRACE_VIEW", "0") == "1":
+    TRACING = True
+    import tracemalloc
+
+    tracemalloc.start()
+else:
+    TRACING = False
+
 from django.core.wsgi import get_wsgi_application
 
 

api/env.template

@@ -66,3 +66,4 @@ IS_PROXIED=False
 
 FILTER_DEAD_LINKS_BY_DEFAULT=False
 ENABLE_FILTERED_INDEX_QUERIES=True
+ENABLE_TRACE_VIEW=0

utilities/load_testing/justfile

@@ -13,3 +13,6 @@ api host=("http://" + DOCKER_HOST + ":8000"):
 
 k6 url="https://api-dev.openverse.engineering" +extra_args="": build
     docker run --rm -e API_TOKEN=$API_TOKEN -e API_URL={{ url }} -v {{ invocation_directory() }}/k6:/app {{ extra_args }} openverse-load-testing:latest k6 run main.js
+
+k6-local:
+    @API_TOKEN="" just k6 http://localhost:50280/v1/ --net=host

utilities/load_testing/k6/main.js

@@ -1,6 +1,6 @@
 import { group } from "k6"
 import { searchBy } from "./search.js"
-import { getProvider, getRandomWord } from "./utils.js"
+import { getProvider } from "./utils.js"
 
 const createScenario = (mediaType, pageSize, funcName) => {
   return {
@@ -64,7 +64,6 @@ const searchByField = (paramFunc, followLinks = false) => {
   )
 }
 
-export const searchByRandomWord = () =>
-  searchByField(() => `q=${getRandomWord()}`, true)
+export const searchByRandomWord = () => searchByField(() => `randomWord`, true)
 export const searchByProvider = () =>
   searchByField((media_type) => `source=${getProvider(media_type)}`, false)

utilities/load_testing/k6/search.js

@@ -6,10 +6,16 @@ import {
   makeResponseFailedCheck,
   REQUEST_HEADERS,
   SLEEP_DURATION,
+  discoveredUnconsumedTags,
+  getRandomWord,
 } from "./utils.js"
 
 export const searchBy = (param, page, media_type, page_size, followLinks) => {
-  let url = `${API_URL}${media_type}/?${param}&page=${page}&page_size=${page_size}&filter_dead=false`
+  if (param === "randomWord") {
+    param = `q=${getRandomWord()}`
+  }
+
+  let url = `${API_URL}${media_type}/?${param}&page=${page}&page_size=${page_size}&filter_dead=true`
   const response = http.get(url, { headers: REQUEST_HEADERS })
 
   const checkResponseFailed = makeResponseFailedCheck(param, page)
@@ -23,6 +29,9 @@ export const searchBy = (param, page, media_type, page_size, followLinks) => {
   const pageCount = parsedResp["page_count"]
   const detailUrls = parsedResp["results"].map((i) => i.detail_url)
   const relatedUrls = parsedResp["results"].map((i) => i.related_url)
+  parsedResp["results"].forEach((i) =>
+    i["tags"].forEach((t) => discoveredUnconsumedTags.add(t["name"]))
+  )
 
   // Don't view details/related if not requested
   if (!followLinks) {

utilities/load_testing/k6/utils.js

@@ -5,12 +5,17 @@ import { randomItem } from "https://jslib.k6.io/k6-utils/1.2.0/index.js"
 export const API_URL =
   __ENV.API_URL || "https://api-dev.openverse.engineering/v1/"
 export const SLEEP_DURATION = 0.1
-// Use the random words list available locally, but filter any words that end with apostrophe-s
-const WORDS = open("/usr/share/dict/words")
-  .split("\n")
-  .filter((w) => !w.endsWith("'s"))
 
-export const getRandomWord = () => randomItem(WORDS)
+export const discoveredUnconsumedTags = new Set()
+
+export const getRandomWord = () => {
+  if (discoveredUnconsumedTags.size) {
+    const tag = randomItem(Array.from(discoveredUnconsumedTags))
+    discoveredUnconsumedTags.delete(tag)
+    return tag
+  }
+  return randomItem(["dog", "run", "cat", "honey", "happy"])
+}
 
 export const getProvider = (media_type) => {
   let url = `${API_URL}${media_type}/stats`