Skip to content

Commit 3ee5f86

Browse files
authored
Python gc in python thread (#54)
* some simplification/speed improvements * This aint workin' yet * all tests pass with cooperative reference cleanup * editing comments * simplification after reading more of concurrent-deque api * This aint workin' yet * all tests pass with cooperative reference cleanup * editing comments * simplification after reading more of concurrent-deque api * updated tech.resource
1 parent c17c7a0 commit 3ee5f86

File tree

6 files changed

+150
-86
lines changed

6 files changed

+150
-86
lines changed

project.clj

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
:url "https://www.eclipse.org/legal/epl-2.0/"}
66
:dependencies [[org.clojure/clojure "1.10.1"]
77
[camel-snake-kebab "0.4.0"]
8-
[techascent/tech.datatype "4.69"]
8+
[techascent/tech.datatype "4.70"]
99
[org.clojure/data.json "0.2.7"]]
1010
:repl-options {:init-ns user}
1111
:java-source-paths ["java"])

src/libpython_clj/python/bridge.clj

+6-6
Original file line numberDiff line numberDiff line change
@@ -40,24 +40,25 @@
4040
[expose-bridge-to-python!
4141
pybridge->bridge
4242
create-bridge-from-att-map]]
43+
[libpython-clj.python.gc :as pygc]
4344
[clojure.stacktrace :as st]
4445
[tech.jna :as jna]
4546
[tech.v2.tensor :as dtt]
4647
[tech.v2.datatype.casting :as casting]
4748
[tech.v2.datatype.functional :as dtype-fn]
4849
[tech.v2.datatype :as dtype]
49-
[tech.resource :as resource]
5050
[clojure.set :as c-set]
5151
[clojure.tools.logging :as log])
5252
(:import [java.util Map RandomAccess List Map$Entry Iterator UUID]
53-
[java.util.concurrent ConcurrentHashMap]
53+
[java.util.concurrent ConcurrentHashMap ConcurrentLinkedQueue]
5454
[clojure.lang IFn Symbol Keyword Seqable
5555
Fn MapEntry Range LongRange]
5656
[tech.v2.datatype ObjectReader ObjectWriter ObjectMutable
5757
ObjectIter MutableRemove]
5858
[tech.v2.datatype.typed_buffer TypedBuffer]
5959
[tech.v2.tensor.protocols PTensor]
6060
[com.sun.jna Pointer]
61+
[tech.resource GCReference]
6162
[java.io Writer]
6263
[libpython_clj.jna JVMBridge
6364
CFunction$KeyWordFunction
@@ -539,8 +540,7 @@
539540
long-addr (get-attr ctypes "data")
540541
hash-ary {:ctypes-map ctypes}
541542
ptr-val (-> (Pointer. long-addr)
542-
(resource/track #(get hash-ary :ctypes-map)
543-
pyobj/*pyobject-tracking-flags*))]
543+
(pygc/track #(get hash-ary :ctypes-map)))]
544544
{:ptr ptr-val
545545
:datatype np-dtype
546546
:shape shape
@@ -782,7 +782,7 @@
782782
[& body]
783783
`(delay
784784
(with-gil
785-
(with-bindings {#'pyobj/*pyobject-tracking-flags* [:gc]}
785+
(with-bindings {#'pygc/*stack-gc-context* nil}
786786
~@body))))
787787

788788

@@ -1094,7 +1094,7 @@
10941094
initial-buffer
10951095
(->py-tuple shape)
10961096
(->py-tuple strides))]
1097-
(resource/track retval #(get buffer-desc :ptr) pyobj/*pyobject-tracking-flags*))))
1097+
(pygc/track retval #(get buffer-desc :ptr)))))
10981098

10991099

11001100
(extend-type Object

src/libpython_clj/python/gc.clj

+67
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
(ns libpython-clj.python.gc
2+
"Binding of various sort of gc semantics optimized specifically for
3+
libpython-clj. For general bindings, see tech.resource"
4+
(:import [java.util.concurrent ConcurrentHashMap ConcurrentLinkedDeque]
5+
[java.lang.ref ReferenceQueue]
6+
[tech.resource GCReference]))
7+
8+
9+
(set! *warn-on-reflection* true)
10+
11+
12+
(defonce ^:dynamic *stack-gc-context* nil)
13+
(defn stack-context
14+
^ConcurrentLinkedDeque []
15+
*stack-gc-context*)
16+
17+
18+
(defonce reference-queue-var (ReferenceQueue.))
19+
(defn reference-queue
20+
^ReferenceQueue []
21+
reference-queue-var)
22+
23+
24+
(defonce ptr-set-var (ConcurrentHashMap/newKeySet))
25+
(defn ptr-set
26+
^java.util.Set []
27+
ptr-set-var)
28+
29+
30+
(defn track
31+
[item dispose-fn]
32+
(let [ptr-val (GCReference. item (reference-queue) (fn [ptr-val]
33+
(.remove (ptr-set) ptr-val)
34+
(dispose-fn)))
35+
^ConcurrentLinkedDeque stack-context (stack-context)]
36+
;;We have to keep track of the pointer. If we do not the pointer gets gc'd then
37+
;;it will not be put on the reference queue when the object itself is gc'd.
38+
;;Nice little gotcha there.
39+
(if stack-context
40+
(.add stack-context ptr-val)
41+
;;Ensure we don't lose track of the weak reference. If it gets cleaned up
42+
;;the gc system will fail.
43+
(.add (ptr-set) ptr-val))
44+
item))
45+
46+
47+
(defn clear-reference-queue
48+
[]
49+
(when-let [next-ref (.poll (reference-queue))]
50+
(.run ^Runnable next-ref)
51+
(recur)))
52+
53+
54+
(defn clear-stack-context
55+
[]
56+
(when-let [next-ref (.pollLast (stack-context))]
57+
(.run ^Runnable next-ref)
58+
(recur)))
59+
60+
61+
(defmacro with-stack-context
62+
[& body]
63+
`(with-bindings {#'*stack-gc-context* (ConcurrentLinkedDeque.)}
64+
(try
65+
~@body
66+
(finally
67+
(clear-stack-context)))))

src/libpython_clj/python/interpreter.clj

+3-2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
(ns libpython-clj.python.interpreter
22
(:require [libpython-clj.jna :as libpy ]
33
[libpython-clj.jna.base :as libpy-base]
4-
[tech.resource :as resource]
4+
[libpython-clj.python.gc :as pygc]
55
[libpython-clj.python.logging
66
:refer [log-error log-warn log-info]]
77
[tech.jna :as jna]
@@ -351,6 +351,7 @@ print(json.dumps(
351351
(try
352352
~@body
353353
(finally
354+
(pygc/clear-reference-queue)
354355
(when new-binding?#
355356
(release-gil! interp#)))))))))
356357

@@ -437,7 +438,7 @@ print(json.dumps(
437438
(recur library-names))))
438439
;;Set program name
439440
(when-let [program-name (or program-name *program-name* "")]
440-
(resource/stack-resource-context
441+
(pygc/with-stack-context
441442
(libpy/PySys_SetArgv 0 (-> program-name
442443
(jna/string->wide-ptr)))))
443444
(let [type-symbols (libpy/lookup-type-symbols)

src/libpython_clj/python/object.clj

+73-73
Original file line numberDiff line numberDiff line change
@@ -33,12 +33,13 @@
3333
:as py-proto]
3434
[libpython-clj.jna.base :as libpy-base]
3535
[libpython-clj.jna :as libpy]
36+
[libpython-clj.python.gc :as pygc]
3637
[clojure.stacktrace :as st]
3738
[tech.jna :as jna]
38-
[tech.resource :as resource]
3939
[tech.v2.datatype :as dtype]
4040
[tech.v2.datatype.protocols :as dtype-proto]
4141
[tech.v2.datatype.casting :as casting]
42+
[tech.resource :as resource]
4243
[tech.v2.tensor]
4344
[clojure.tools.logging :as log])
4445
(:import [com.sun.jna Pointer CallbackReference]
@@ -96,13 +97,10 @@
9697
(py-proto/->jvm item options)))
9798

9899

99-
(def ^:dynamic *object-reference-logging* false)
100-
100+
(def object-reference-logging nil)
101101

102-
(def ^:dynamic *object-reference-tracker* nil)
103102

104-
105-
(def ^:dynamic *pyobject-tracking-flags* [:gc])
103+
(def object-reference-tracker nil)
106104

107105

108106
(defn incref
@@ -130,41 +128,44 @@
130128
(if (and pyobj
131129
(not= (Pointer/nativeValue (libpy/as-pyobj pyobj))
132130
(Pointer/nativeValue (libpy/as-pyobj (libpy/Py_None)))))
133-
(let [interpreter (ensure-bound-interpreter)
134-
pyobj-value (Pointer/nativeValue (libpy/as-pyobj pyobj))
135-
py-type-name (name (python-type pyobj))]
136-
(when *object-reference-logging*
137-
(let [obj-data (PyObject. (Pointer. pyobj-value))]
138-
(println (format "tracking object - 0x%x:%4d:%s"
139-
pyobj-value
140-
(.ob_refcnt obj-data)
141-
py-type-name))))
142-
(when *object-reference-tracker*
143-
(swap! *object-reference-tracker*
144-
update pyobj-value #(inc (or % 0))))
145-
;;We ask the garbage collector to track the python object and notify
146-
;;us when it is released. We then decref on that event.
147-
(resource/track pyobj
148-
#(with-gil
149-
(try
150-
(let [refcount (refcount pyobj)
151-
obj-data (PyObject. (Pointer. pyobj-value))]
152-
(if (< refcount 1)
153-
(log/errorf "Fatal error -- releasing object - 0x%x:%4d:%s
131+
(do
132+
(ensure-bound-interpreter)
133+
(let [pyobj-value (Pointer/nativeValue (libpy/as-pyobj pyobj))
134+
py-type-name (name (python-type pyobj))]
135+
(when object-reference-logging
136+
(let [obj-data (PyObject. (Pointer. pyobj-value))]
137+
(println (format "tracking object - 0x%x:%4d:%s"
138+
pyobj-value
139+
(.ob_refcnt obj-data)
140+
py-type-name))))
141+
(when object-reference-tracker
142+
(swap! object-reference-tracker
143+
update pyobj-value #(inc (or % 0))))
144+
;;We ask the garbage collector to track the python object and notify
145+
;;us when it is released. We then decref on that event.
146+
(pygc/track pyobj
147+
;;No longer with-gil. Because cleanup is cooperative, the gil is
148+
;;guaranteed to be captured here already.
149+
#(try
150+
;;Intentionally overshadow pyobj. We cannot access it here.
151+
(let [pyobj (Pointer. pyobj-value)
152+
refcount (refcount pyobj)
153+
obj-data (PyObject. pyobj)]
154+
(if (< refcount 1)
155+
(log/errorf "Fatal error -- releasing object - 0x%x:%4d:%s
154156
Object's refcount is bad. Crash is imminent" pyobj-value refcount py-type-name)
155-
(when *object-reference-logging*
156-
(println (format "releasing object - 0x%x:%4d:%s"
157-
pyobj-value
158-
(.ob_refcnt obj-data)
159-
py-type-name))))
160-
(when *object-reference-tracker*
161-
(swap! *object-reference-tracker*
162-
update pyobj-value (fn [arg]
163-
(dec (or arg 0))))))
164-
(libpy/Py_DecRef (Pointer. pyobj-value))
165-
(catch Throwable e
166-
(log/error e "Exception while releasing object"))))
167-
*pyobject-tracking-flags*))
157+
(when object-reference-logging
158+
(println (format "releasing object - 0x%x:%4d:%s"
159+
pyobj-value
160+
(.ob_refcnt obj-data)
161+
py-type-name))))
162+
(when object-reference-tracker
163+
(swap! object-reference-tracker
164+
update pyobj-value (fn [arg]
165+
(dec (or arg 0))))))
166+
(libpy/Py_DecRef (Pointer. pyobj-value))
167+
(catch Throwable e
168+
(log/error e "Exception while releasing object"))))))
168169
(do
169170
;;Special handling for PyNone types
170171
(libpy/Py_DecRef pyobj)
@@ -173,9 +174,8 @@ Object's refcount is bad. Crash is imminent" pyobj-value refcount py-type-name)
173174

174175
(defmacro stack-resource-context
175176
[& body]
176-
`(with-bindings {#'*pyobject-tracking-flags* [:stack :gc]}
177-
(resource/stack-resource-context
178-
~@body)))
177+
`(pygc/with-stack-context
178+
~@body))
179179

180180

181181
(defn incref-wrap-pyobject
@@ -391,37 +391,37 @@ Object's refcount is bad. Crash is imminent" pyobj-value refcount py-type-name)
391391
doc
392392
function]
393393
:as method-data}]
394-
(resource/stack-resource-context
395-
(when-not (cfunc-instance? function)
396-
(throw (Exception.
397-
(format "Callbacks must implement one of the CFunction interfaces:
394+
;;Here we really do need a resource stack context
395+
(when-not (cfunc-instance? function)
396+
(throw (Exception.
397+
(format "Callbacks must implement one of the CFunction interfaces:
398398
%s" (type function)))))
399-
(let [meth-flags (long (cond
400-
(instance? CFunction$NoArgFunction function)
401-
@libpy/METH_NOARGS
402-
403-
(instance? CFunction$TupleFunction function)
404-
@libpy/METH_VARARGS
405-
406-
(instance? CFunction$KeyWordFunction function)
407-
(bit-or @libpy/METH_KEYWORDS @libpy/METH_VARARGS)
408-
:else
409-
(throw (ex-info (format "Failed due to type: %s"
410-
(type function))
411-
{}))))
412-
name-ptr (jna/string->ptr name)
413-
doc-ptr (jna/string->ptr doc)]
414-
(set! (.ml_name method-def) name-ptr)
415-
(set! (.ml_meth method-def) (CallbackReference/getFunctionPointer function))
416-
(set! (.ml_flags method-def) (int meth-flags))
417-
(set! (.ml_doc method-def) doc-ptr)
418-
(.write method-def)
419-
(pyinterp/conj-forever! (assoc method-data
420-
:name-ptr name-ptr
421-
:doc-ptr doc-ptr
422-
:callback-object function
423-
:method-definition method-def))
424-
method-def)))
399+
(let [meth-flags (long (cond
400+
(instance? CFunction$NoArgFunction function)
401+
@libpy/METH_NOARGS
402+
403+
(instance? CFunction$TupleFunction function)
404+
@libpy/METH_VARARGS
405+
406+
(instance? CFunction$KeyWordFunction function)
407+
(bit-or @libpy/METH_KEYWORDS @libpy/METH_VARARGS)
408+
:else
409+
(throw (ex-info (format "Failed due to type: %s"
410+
(type function))
411+
{}))))
412+
name-ptr (jna/string->ptr-untracked name)
413+
doc-ptr (jna/string->ptr-untracked doc)]
414+
(set! (.ml_name method-def) name-ptr)
415+
(set! (.ml_meth method-def) (CallbackReference/getFunctionPointer function))
416+
(set! (.ml_flags method-def) (int meth-flags))
417+
(set! (.ml_doc method-def) doc-ptr)
418+
(.write method-def)
419+
(pyinterp/conj-forever! (assoc method-data
420+
:name-ptr name-ptr
421+
:doc-ptr doc-ptr
422+
:callback-object function
423+
:method-definition method-def))
424+
method-def))
425425

426426

427427
(defn method-def-data->method-def

test/libpython_clj/stress_test.clj

-4
Original file line numberDiff line numberDiff line change
@@ -56,10 +56,6 @@ def getmultidata():
5656
(gd-fn)))
5757

5858

59-
;;If you want to see how the sausage is made...
60-
(alter-var-root #'libpython-clj.python.object/*object-reference-logging*
61-
(constantly false))
62-
6359
;;Ensure that failure to open resource context before tracking for stack
6460
;;related things causes immediate failure. Unless you feel like being pedantic,
6561
;;this isn't necessary. The python library automatically switches to normal gc-only

0 commit comments

Comments
 (0)