From bbcd7eac8e247c557be603020a5c7706af995a4e Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Fri, 22 Nov 2019 12:56:33 -0700 Subject: [PATCH 001/456] editing --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 82bac21..729fedc 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,12 @@ JNA libpython bindings to the tech ecosystem. REPL. +#### Clojure Conj 2019 + +* [slides](https://docs.google.com/presentation/d/1uegYhpS6P2AtEfhpg6PlgBmTSIPqCXvFTWcGYG_Qk2o/edit?usp=sharing) +* video -- coming soon!! + + We have a [video](https://www.youtube.com/watch?v=ajDiGS73i2o) up of a scicloj community discussion with demos. From 211b81cd274e1ebf469796f59b3410fcc6289939 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Fri, 22 Nov 2019 13:54:05 -0700 Subject: [PATCH 002/456] Adding nil test for ->jvm and as-jvm. maps use getordefault for ifn overload now, and example of bridge. --- project.clj | 2 +- src/libpython_clj/python.clj | 3 +- src/libpython_clj/python/bridge.clj | 4 +-- src/libpython_clj/python/object.clj | 3 +- test/libpython_clj/python_test.clj | 51 +++++++++++++++++++++++++++++ testcode/__init__.py | 4 +++ 6 files changed, 62 insertions(+), 5 deletions(-) diff --git a/project.clj b/project.clj index 16cb360..c067819 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.9" +(defproject cnuernber/libpython-clj "1.10-SNAPSHOT" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" diff --git a/src/libpython_clj/python.clj b/src/libpython_clj/python.clj index dbfa5cd..634e53f 100644 --- a/src/libpython_clj/python.clj +++ b/src/libpython_clj/python.clj @@ -50,7 +50,8 @@ (export-symbols libpython-clj.python.interop - libpython-clj-module-name) + libpython-clj-module-name + create-bridge-from-att-map) (export-symbols libpython-clj.python.bridge diff --git a/src/libpython_clj/python/bridge.clj b/src/libpython_clj/python/bridge.clj index 6117cfe..9be1ab4 100644 --- a/src/libpython_clj/python/bridge.clj +++ b/src/libpython_clj/python/bridge.clj @@ -292,7 +292,7 @@ (defn- py-impl-call-raw [att-name att-map arglist] (if-let [py-fn (get att-map att-name)] - (-> (py-proto/do-call-fn py-fn (map as-python arglist) nil)) + (py-proto/do-call-fn py-fn (map as-python arglist) nil) (throw (UnsupportedOperationException. (format "Python object has no attribute: %s" att-name))))) @@ -371,7 +371,7 @@ (MapEntry. k v))))))] (.iterator ^Iterable mapentry-seq))) IFn - (invoke [this arg] (.get this arg)) + (invoke [this arg] (.getOrDefault this arg nil)) (invoke [this k v] (.put this k v)) (applyTo [this arglist] (let [arglist (vec arglist)] diff --git a/src/libpython_clj/python/object.clj b/src/libpython_clj/python/object.clj index 80a6c91..9d2303f 100644 --- a/src/libpython_clj/python/object.clj +++ b/src/libpython_clj/python/object.clj @@ -91,7 +91,8 @@ (defn ->jvm "Copy an object into the jvm (if it wasn't there already.)" [item & [options]] - (py-proto/->jvm item options)) + (when item + (py-proto/->jvm item options))) (def ^:dynamic *object-reference-logging* false) diff --git a/test/libpython_clj/python_test.clj b/test/libpython_clj/python_test.clj index 20bfcfc..348dcd3 100644 --- a/test/libpython_clj/python_test.clj +++ b/test/libpython_clj/python_test.clj @@ -197,3 +197,54 @@ (py/call-attr item "doit_noerr")) (is (= ["enter" "exit: None"] (py/->jvm fn-list)))))) + + +(deftest arrow-as-fns-with-nil + (py/initialize!) + (is (= nil (py/->jvm nil))) + (is (= nil (py/as-jvm nil)))) + + +(deftest pydict-nil-get + (py/initialize!) + (let [dict (py/->python {:a 1 :b {:a 1 :b 2}}) + bridged (py/as-jvm dict)] + (is (= nil (bridged nil))))) + + +(deftest custom-clojure-item + (let [att-map {"clojure_fn" (py/->python #(vector 1 2 3))} + my-python-item (py/create-bridge-from-att-map + ;;First is the jvm object that this bridge stands for. If this + ;;gets garbage collected then the python side will be removed + ;;also. + att-map + ;;second is the list of attributes. In this case, since this + ;;object isn't iterable or anything, this function will do. + att-map + ) + py-mod (py/import-module "testcode")] + (is (= [1 2 3] + (py/call-attr py-mod "calling_custom_clojure_fn" my-python-item))) + + ;;Now this case is harder. Let's say we have something that is iterable and we + ;;want this to be reflected in python. In that case we have to call 'as-python' + ;;on the iterator and that 'should' work. + + (let [my-obj (reify + Iterable + (iterator [this] (.iterator [4 5 6]))) + ;;Note that attributes themselves have to be python objects and wrapping + ;;this with as-python makes that function wrap whatever it returns in a + ;;bridging python object also. + att-map {"__iter__" (py/as-python #(.iterator my-obj))} + my-python-item (py/create-bridge-from-att-map + my-obj + ;;second is the list of attributes. In this case, since this + ;;object isn't iterable or anything, this function will do. + att-map + )] + (is (= [4 5 6] + (vec my-obj))) + (is (= [4 5 6] + (vec (py/call-attr py-mod "for_iter" my-python-item))))))) diff --git a/testcode/__init__.py b/testcode/__init__.py index c17ccd9..8e0922e 100644 --- a/testcode/__init__.py +++ b/testcode/__init__.py @@ -18,3 +18,7 @@ def for_iter(arg): for item in arg: retval.append(item) return retval + + +def calling_custom_clojure_fn(arg): + return arg.clojure_fn() From 1aacd27322f0a2ffe3047825d4084a8dfc44e0d9 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Fri, 22 Nov 2019 13:54:45 -0700 Subject: [PATCH 003/456] 1.10 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index c067819..1ba9374 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.10-SNAPSHOT" +(defproject cnuernber/libpython-clj "1.10" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" From 41ce31bf7db990be64789065630725f29d71581e Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Fri, 22 Nov 2019 13:54:53 -0700 Subject: [PATCH 004/456] snap --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 1ba9374..fdb0679 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.10" +(defproject cnuernber/libpython-clj "1.11-SNAPSHOT" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" From 01840b2063b35996df25aebff9b8a4a80f2538fc Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Fri, 22 Nov 2019 16:50:20 -0700 Subject: [PATCH 005/456] edit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 729fedc..ccf5ea6 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,8 @@ JNA libpython bindings to the tech ecosystem. #### Clojure Conj 2019 +* [video](https://www.youtube.com/watch?v=vQPW16_jixs) * [slides](https://docs.google.com/presentation/d/1uegYhpS6P2AtEfhpg6PlgBmTSIPqCXvFTWcGYG_Qk2o/edit?usp=sharing) -* video -- coming soon!! We have a [video](https://www.youtube.com/watch?v=ajDiGS73i2o) up of a scicloj community From c9449c3c105cac8240afcb3ac8cd8b1515c9a534 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 24 Nov 2019 11:47:14 -0700 Subject: [PATCH 006/456] Fixing crash bug with iteration. Adding scoped execution. --- project.clj | 2 +- src/libpython_clj/jna/base.clj | 27 +++- src/libpython_clj/jna/interpreter.clj | 11 +- src/libpython_clj/python.clj | 21 +++- src/libpython_clj/python/interop.clj | 117 ++++++++--------- src/libpython_clj/python/interpreter.clj | 20 +-- src/libpython_clj/python/object.clj | 152 +++++++++++++++-------- test/libpython_clj/stress_test.clj | 72 +++++++++++ 8 files changed, 295 insertions(+), 127 deletions(-) create mode 100644 test/libpython_clj/stress_test.clj diff --git a/project.clj b/project.clj index fdb0679..91645e1 100644 --- a/project.clj +++ b/project.clj @@ -4,7 +4,7 @@ :license {:name "EPL-2.0" :url "https://www.eclipse.org/legal/epl-2.0/"} :dependencies [[org.clojure/clojure "1.10.1"] - [techascent/tech.datatype "4.53"] + [techascent/tech.datatype "4.54"] [camel-snake-kebab "0.4.0"]] :repl-options {:init-ns user} :java-source-paths ["java"]) diff --git a/src/libpython_clj/jna/base.clj b/src/libpython_clj/jna/base.clj index d4fad57..ed90fa7 100644 --- a/src/libpython_clj/jna/base.clj +++ b/src/libpython_clj/jna/base.clj @@ -46,11 +46,36 @@ (ensure-pyobj item)) -(defmacro def-pylib-fn +(def ^:dynamic *gil-captured* false) + + +(defmacro def-no-gil-pylib-fn + "Define a pylib function where the gil doesn't need to be captured to call." [fn-name docstring & args] `(jna/def-jna-fn *python-library* ~fn-name ~docstring ~@args)) +(defmacro def-pylib-fn + [fn-name docstring rettype & argpairs] + `(defn ~fn-name + ~docstring + ~(mapv first argpairs) + (when-not *gil-captured* + (throw (Exception. "Failure to capture gil when calling into libpython"))) + (let [~'tvm-fn (jna/find-function ~(str fn-name) *python-library*) + ~'fn-args (object-array + ~(mapv (fn [[arg-symbol arg-coersion]] + (when (= arg-symbol arg-coersion) + (throw (ex-info (format "Argument symbol (%s) cannot match coersion (%s)" + arg-symbol arg-coersion) + {}))) + `(~arg-coersion ~arg-symbol)) + argpairs))] + ~(if rettype + `(.invoke (jna-base/to-typed-fn ~'tvm-fn) ~rettype ~'fn-args) + `(.invoke (jna-base/to-typed-fn ~'tvm-fn) ~'fn-args))))) + + (def size-t-type (type (jna/size-t 0))) diff --git a/src/libpython_clj/jna/interpreter.clj b/src/libpython_clj/jna/interpreter.clj index 8db711c..07cdddb 100644 --- a/src/libpython_clj/jna/interpreter.clj +++ b/src/libpython_clj/jna/interpreter.clj @@ -1,6 +1,7 @@ (ns libpython-clj.jna.interpreter (:require [libpython-clj.jna.base :refer [def-pylib-fn + def-no-gil-pylib-fn ensure-pyobj ensure-pytuple ensure-pydict @@ -28,7 +29,7 @@ ;; calls Py_Initialize() and Py_Finalize() more than once. -(def-pylib-fn Py_InitializeEx +(def-no-gil-pylib-fn Py_InitializeEx "This function works like Py_Initialize() if initsigs is 1. If initsigs is 0, it skips initialization registration of signal handlers, which might be useful when Python is embedded." @@ -37,14 +38,14 @@ -(def-pylib-fn Py_IsInitialized +(def-no-gil-pylib-fn Py_IsInitialized "Return true (nonzero) when the Python interpreter has been initialized, false (zero) if not. After Py_Finalize() is called, this returns false until Py_Initialize() is called again." Integer) -(def-pylib-fn Py_FinalizeEx +(def-no-gil-pylib-fn Py_FinalizeEx "Undo all initializations made by Py_Initialize() and subsequent use of Python/C API functions, and destroy all sub-interpreters (see Py_NewInterpreter() below) that were created and not yet destroyed since the last call to Py_Initialize(). Ideally, this @@ -62,7 +63,7 @@ ;; System Functionality -(def-pylib-fn PySys_SetArgv +(def-no-gil-pylib-fn PySys_SetArgv "This function works like PySys_SetArgvEx() with updatepath set to 1 unless the python interpreter was started with the -I. @@ -226,7 +227,7 @@ ;;Acquire the GIL of the given thread state -(def-pylib-fn PyEval_RestoreThread +(def-no-gil-pylib-fn PyEval_RestoreThread "Acquire the global interpreter lock (if it has been created and thread support is enabled) and set the thread state to tstate, which must not be NULL. If the lock has been created, the current thread must not have acquired it, otherwise deadlock ensues. diff --git a/src/libpython_clj/python.clj b/src/libpython_clj/python.clj index 634e53f..65a077a 100644 --- a/src/libpython_clj/python.clj +++ b/src/libpython_clj/python.clj @@ -2,7 +2,7 @@ (:require [tech.parallel.utils :refer [export-symbols]] [libpython-clj.python.interop :as pyinterop] [libpython-clj.python.interpreter :as pyinterp - :refer [with-gil with-interpreter]] + :refer [with-interpreter]] [libpython-clj.python.object :as pyobj] [libpython-clj.python.bridge] [libpython-clj.jna :as pyjna] @@ -49,6 +49,25 @@ ->jvm) +(defmacro stack-resource-context + "Create a stack-based resource context. All python objects allocated within this + context will be released at the termination of this context. + !!This means that no python objects can escape from this context!! + You must use copy semantics (->jvm) for anything escaping this context." + [& body] + `(pyobj/stack-resource-context + ~@body)) + + +(defmacro with-gil + "Capture the gil for an extended amount of time. This can greatly speed up + operations as the mutex is captured and held once as opposed to find grained + grabbing/releasing of the mutex." + [& body] + `(pyinterp/with-gil + ~@body)) + + (export-symbols libpython-clj.python.interop libpython-clj-module-name create-bridge-from-att-map) diff --git a/src/libpython_clj/python/interop.clj b/src/libpython_clj/python/interop.clj index dd7c7b3..cbd0809 100644 --- a/src/libpython_clj/python/interop.clj +++ b/src/libpython_clj/python/interop.clj @@ -23,6 +23,7 @@ [tech.jna :as jna] ;;need memset [tech.v2.datatype.nio-buffer :as nio-buf] + [tech.resource :as resource] [libpython-clj.python.object :refer [wrap-pyobject incref-wrap-pyobject incref @@ -205,58 +206,62 @@ tp_iternext ;;may be nil ] :as type-definition}] - (when (or (not type-name) - (= 0 (count type-name))) - (throw (ex-info "Cannot create unnamed type." {}))) - (let [tp_new (or tp_new - (reify CFunction$tp_new - (pyinvoke [this self varargs kw_args] - (libpy/PyType_GenericNew self varargs kw_args)))) - module-name (get-attr module "__name__") - docstring-ptr (jna/string->ptr docstring) - type-name-ptr (jna/string->ptr (str module-name "." type-name)) - tp_flags (long (or tp_flags - (bit-or @libpy/Py_TPFLAGS_DEFAULT - @libpy/Py_TPFLAGS_BASETYPE))) - ;;We allocate our memory manually here else the system will gc the - ;;type object memory when the type goes out of scope. - new-mem (jna/malloc-untracked type-obj-size) - _ (nio-buf/memset new-mem 0 type-obj-size) - new-type (PyTypeObject. new-mem)] - (set! (.tp_name new-type) type-name-ptr) - (set! (.tp_doc new-type) docstring-ptr) - (set! (.tp_basicsize new-type) tp_basicsize) - (set! (.tp_flags new-type) tp_flags) - (set! (.tp_new new-type) tp_new) - (set! (.tp_dealloc new-type) tp_dealloc) - (set! (.tp_getattr new-type) tp_getattr) - (set! (.tp_setattr new-type) tp_setattr) - (set! (.tp_methods new-type) (method-def-data-seq->method-def-ref - method-definitions)) - (when tp_iter - (set! (.tp_iter new-type) tp_iter)) - (when tp_iternext - (set! (.tp_iternext new-type) tp_iternext)) - (let [type-ready (libpy/PyType_Ready new-type)] - (if (>= 0 type-ready) - (do - (libpy/Py_IncRef new-type) - (.read new-type) - (libpy/PyModule_AddObject module (str type-name "_type") new-type) - ;;We are careful to keep track of the static data we give to python. - ;;the GC cannot now remove any of this stuff pretty much - ;;forever now. - (conj-forever! (assoc type-definition - :tp_name type-name-ptr - :tp_doc docstring-ptr - :tp_new tp_new - :tp_dealloc tp_dealloc - :tp_getattr tp_getattr - :tp_setattr tp_setattr)) - (libpy/Py_IncRef new-type) - new-type) - (throw (ex-info (format "Type failed to register: %d" type-ready) - {})))))) + (resource/stack-resource-context + (when (or (not type-name) + (= 0 (count type-name))) + (throw (ex-info "Cannot create unnamed type." {}))) + (let [tp_new (or tp_new + (reify CFunction$tp_new + (pyinvoke [this self varargs kw_args] + (libpy/PyType_GenericNew self varargs kw_args)))) + module-name (get-attr module "__name__") + ;;These get leaked. Really, types are global objects that cannot be released. + ;;Until we can destroy interpreters, it isn't worth the effort to track the + ;;type and memory related to the type. + docstring-ptr (jna/string->ptr-untracked docstring) + type-name-ptr (jna/string->ptr-untracked (str module-name "." type-name)) + tp_flags (long (or tp_flags + (bit-or @libpy/Py_TPFLAGS_DEFAULT + @libpy/Py_TPFLAGS_BASETYPE))) + ;;We allocate our memory manually here else the system will gc the + ;;type object memory when the type goes out of scope. + new-mem (jna/malloc-untracked type-obj-size) + _ (nio-buf/memset new-mem 0 type-obj-size) + new-type (PyTypeObject. new-mem)] + (set! (.tp_name new-type) type-name-ptr) + (set! (.tp_doc new-type) docstring-ptr) + (set! (.tp_basicsize new-type) tp_basicsize) + (set! (.tp_flags new-type) tp_flags) + (set! (.tp_new new-type) tp_new) + (set! (.tp_dealloc new-type) tp_dealloc) + (set! (.tp_getattr new-type) tp_getattr) + (set! (.tp_setattr new-type) tp_setattr) + (set! (.tp_methods new-type) (method-def-data-seq->method-def-ref + method-definitions)) + (when tp_iter + (set! (.tp_iter new-type) tp_iter)) + (when tp_iternext + (set! (.tp_iternext new-type) tp_iternext)) + (let [type-ready (libpy/PyType_Ready new-type)] + (if (>= 0 type-ready) + (do + (libpy/Py_IncRef new-type) + (.read new-type) + (libpy/PyModule_AddObject module (str type-name "_type") new-type) + ;;We are careful to keep track of the static data we give to python. + ;;the GC cannot now remove any of this stuff pretty much + ;;forever now. + (conj-forever! (assoc type-definition + :tp_name type-name-ptr + :tp_doc docstring-ptr + :tp_new tp_new + :tp_dealloc tp_dealloc + :tp_getattr tp_getattr + :tp_setattr tp_setattr)) + (libpy/Py_IncRef new-type) + new-type) + (throw (ex-info (format "Type failed to register: %d" type-ready) + {}))))))) (defn pybridge->bridge ^JVMBridge [^Pointer pybridge] @@ -415,7 +420,7 @@ (defn create-var-writer "Returns an unregistered bridge" - ^Pointer [writer-var] + ^Pointer [writer-var varname] (with-gil (create-bridge-from-att-map writer-var @@ -428,17 +433,17 @@ (defn get-or-create-var-writer - [writer-var] + [writer-var varname] (if-let [existing-writer (find-jvm-bridge-entry (get-object-handle writer-var) (ensure-interpreter))] (:pyobject existing-writer) - (create-var-writer writer-var))) + (create-var-writer writer-var varname))) (defn setup-std-writer [writer-var sys-mod-attname] (with-gil (let [sys-module (import-module "sys") - std-out-writer (get-or-create-var-writer writer-var)] + std-out-writer (get-or-create-var-writer writer-var sys-mod-attname)] (py-proto/set-attr! sys-module sys-mod-attname std-out-writer) :ok))) diff --git a/src/libpython_clj/python/interpreter.clj b/src/libpython_clj/python/interpreter.clj index df9c4c5..df326d7 100644 --- a/src/libpython_clj/python/interpreter.clj +++ b/src/libpython_clj/python/interpreter.clj @@ -1,5 +1,6 @@ (ns libpython-clj.python.interpreter (:require [libpython-clj.jna :as libpy] + [libpython-clj.jna.base :as libpy-base] [tech.resource :as resource] [libpython-clj.python.logging :refer [log-error log-warn log-info]] @@ -196,12 +197,13 @@ ;;No interpreters bound (not *current-thread-interpreter*) (locking interpreter - (try - (with-bindings {#'*current-thread-interpreter* interpreter} - (acquire-gil! interpreter) - (body-fn)) - (finally - (release-gil! interpreter)))) + (with-bindings {#'*current-thread-interpreter* interpreter} + (acquire-gil! interpreter) + (with-bindings {#'libpy-base/*gil-captured* true} + (try + (body-fn) + (finally + (release-gil! interpreter)))))) ;;Switch interpreters in the current thread...deadlock ;;is possible here. (not (identical? interpreter *current-thread-interpreter*)) @@ -244,8 +246,10 @@ (resource/stack-resource-context (libpy/PySys_SetArgv 0 (-> program-name (jna/string->wide-ptr))))) - (let [type-symbols (libpy/lookup-type-symbols)] - (construct-main-interpreter! (libpy/PyEval_SaveThread) type-symbols)))) + (let [type-symbols (libpy/lookup-type-symbols) + context (with-bindings {#'libpy-base/*gil-captured* true} + (libpy/PyEval_SaveThread))] + (construct-main-interpreter! context type-symbols)))) (def ^:dynamic *python-error-handler* nil) diff --git a/src/libpython_clj/python/object.clj b/src/libpython_clj/python/object.clj index 9d2303f..cf165fe 100644 --- a/src/libpython_clj/python/object.clj +++ b/src/libpython_clj/python/object.clj @@ -44,6 +44,7 @@ [tech.v2.datatype.casting :as casting] [tech.v2.tensor]) (:import [com.sun.jna Pointer CallbackReference] + [com.sun.jna.ptr PointerByReference] [libpython_clj.jna PyObject CFunction$KeyWordFunction @@ -98,6 +99,12 @@ (def ^:dynamic *object-reference-logging* false) +(def ^:dynamic *object-reference-tracker* nil) + + +(def ^:dynamic *pyobject-tracking-flags* [:gc]) + + (defn wrap-pyobject "Wrap object such that when it is no longer accessible via the program decref is called. Used for new references. This is some of the meat of the issue, however, @@ -106,7 +113,10 @@ [pyobj & [skip-check-error?]] (when-not skip-check-error? (check-error-throw)) - (when pyobj + ;;We don't wrap pynone + (when (and pyobj + (not= (Pointer/nativeValue (jna/as-ptr pyobj)) + (Pointer/nativeValue (jna/as-ptr (libpy/Py_None))))) (let [interpreter (ensure-bound-interpreter) pyobj-value (Pointer/nativeValue (jna/as-ptr pyobj)) py-type-name (name (python-type pyobj))] @@ -116,6 +126,9 @@ pyobj-value (.ob_refcnt obj-data) py-type-name)))) + (when *object-reference-tracker* + (swap! *object-reference-tracker* + update pyobj-value #(inc (or % 0)))) ;;We ask the garbage collector to track the python object and notify ;;us when it is released. We then decref on that event. (resource/track pyobj @@ -127,10 +140,21 @@ pyobj-value (.ob_refcnt obj-data) py-type-name)))) + (when *object-reference-tracker* + (swap! *object-reference-tracker* + update pyobj-value (fn [arg] + (dec (or arg 0))))) (libpy/Py_DecRef (Pointer. pyobj-value)) (catch Throwable e (log-error "Exception while releasing object: %s" e)))) - [:gc])))) + *pyobject-tracking-flags*)))) + + +(defmacro stack-resource-context + [& body] + `(with-bindings {#'*pyobject-tracking-flags* [:stack :gc]} + (resource/stack-resource-context + ~@body))) (defn incref-wrap-pyobject @@ -363,36 +387,37 @@ doc function] :as method-data}] - (let [callback (if (or (instance? CFunction$KeyWordFunction function) - (instance? CFunction$TupleFunction function) - (instance? CFunction$NoArgFunction function)) - function - (wrap-clojure-fn function)) - meth-flags (long (cond - (instance? CFunction$NoArgFunction callback) - @libpy/METH_NOARGS - - (instance? CFunction$TupleFunction callback) - @libpy/METH_VARARGS - - (instance? CFunction$KeyWordFunction callback) - (bit-or @libpy/METH_KEYWORDS @libpy/METH_VARARGS) - :else - (throw (ex-info (format "Failed due to type: %s" - (type callback)))))) - name-ptr (jna/string->ptr name) - doc-ptr (jna/string->ptr doc)] - (set! (.ml_name method-def) name-ptr) - (set! (.ml_meth method-def) (CallbackReference/getFunctionPointer callback)) - (set! (.ml_flags method-def) (int meth-flags)) - (set! (.ml_doc method-def) doc-ptr) - (.write method-def) - (pyinterp/conj-forever! (assoc method-data - :name-ptr name-ptr - :doc-ptr doc-ptr - :callback-object callback - :method-definition method-def)) - method-def)) + (resource/stack-resource-context + (let [callback (if (or (instance? CFunction$KeyWordFunction function) + (instance? CFunction$TupleFunction function) + (instance? CFunction$NoArgFunction function)) + function + (wrap-clojure-fn function)) + meth-flags (long (cond + (instance? CFunction$NoArgFunction callback) + @libpy/METH_NOARGS + + (instance? CFunction$TupleFunction callback) + @libpy/METH_VARARGS + + (instance? CFunction$KeyWordFunction callback) + (bit-or @libpy/METH_KEYWORDS @libpy/METH_VARARGS) + :else + (throw (ex-info (format "Failed due to type: %s" + (type callback)))))) + name-ptr (jna/string->ptr name) + doc-ptr (jna/string->ptr doc)] + (set! (.ml_name method-def) name-ptr) + (set! (.ml_meth method-def) (CallbackReference/getFunctionPointer callback)) + (set! (.ml_flags method-def) (int meth-flags)) + (set! (.ml_doc method-def) doc-ptr) + (.write method-def) + (pyinterp/conj-forever! (assoc method-data + :name-ptr name-ptr + :doc-ptr doc-ptr + :callback-object callback + :method-definition method-def)) + method-def))) (defn method-def-data->method-def @@ -691,29 +716,46 @@ "This is a tough function to get right. The iterator could return nil as in you could have a list of python none types or something so you have to iterate till you get a StopIteration error." - [iter-fn item-conversion-fn] - (with-gil nil - (let [interpreter (ensure-bound-interpreter)] - (let [py-iter (py-proto/call iter-fn) - next-fn (fn [last-item] - (with-interpreter interpreter - (try - [(-> (py-proto/call-attr py-iter "__next__") - item-conversion-fn)] - (catch Throwable e - nil)))) - cur-item-store (atom (next-fn nil))] - (reify ObjectIter - jna/PToPtr - (is-jna-ptr-convertible? [item] true) - (->ptr-backing-store [item] py-iter) - (hasNext [obj-iter] - (not (nil? @cur-item-store))) - (next [obj-iter] - (-> (swap-vals! cur-item-store next-fn) - ffirst)) - (current [obj-iter] - (first @cur-item-store))))))) + [iter-fn & [item-conversion-fn]] + (with-gil nil + (let [interpreter (ensure-bound-interpreter)] + (let [py-iter (py-proto/call iter-fn) + py-next-fn (when py-iter (py-proto/get-attr py-iter "__next__")) + next-fn (fn [last-item] + (with-interpreter interpreter + (let [retval (libpy/PyObject_CallObject py-next-fn nil)] + (if (libpy/PyErr_Occurred) + (let [ptype (PointerByReference.) + pvalue (PointerByReference.) + ptraceback (PointerByReference.) + _ (libpy/PyErr_Fetch ptype pvalue ptraceback) + ptype (jna/->ptr-backing-store ptype) + pvalue (jna/->ptr-backing-store pvalue) + ptraceback (jna/->ptr-backing-store ptraceback)] + (if (= ptype + (libpy/PyExc_StopIteration)) + (do + (libpy/Py_DecRef ptype) + (when pvalue (libpy/Py_DecRef pvalue)) + (when ptraceback (libpy/Py_DecRef ptraceback)) + nil) + (do (libpy/PyErr_Restore ptype pvalue ptraceback) + (check-error-throw)))) + [(cond-> (wrap-pyobject retval) + item-conversion-fn + item-conversion-fn)])))) + cur-item-store (atom (next-fn nil))] + (reify ObjectIter + jna/PToPtr + (is-jna-ptr-convertible? [item] true) + (->ptr-backing-store [item] py-iter) + (hasNext [obj-iter] + (not (nil? @cur-item-store))) + (next [obj-iter] + (-> (swap-vals! cur-item-store next-fn) + ffirst)) + (current [obj-iter] + (first @cur-item-store))))))) (defn python->jvm-iterable diff --git a/test/libpython_clj/stress_test.clj b/test/libpython_clj/stress_test.clj new file mode 100644 index 0000000..94a2ab7 --- /dev/null +++ b/test/libpython_clj/stress_test.clj @@ -0,0 +1,72 @@ +(ns libpython-clj.stress-test + "A set of tests meant to crash the system or just run the system out of + memory if it isn't setup correctly." + (:require [libpython-clj.python :as py])) + + +(defn get-data + [] + (let [gd-fn + (-> (py/run-simple-string " +def getdata(): + while True: + yield {'a': 1, 'b': 2} +") + :globals + (get "getdata"))] + (gd-fn))) + +;;If you want to see how the sausage is made... +(alter-var-root #'libpython-clj.python.object/*object-reference-logging* + (constantly false)) + +;;Ensure that failure to open resource context before tracking for stack +;;related things causes immediate failure. +(alter-var-root #'tech.resource.stack/*resource-context* + (constantly nil)) + +(defn forever-test + [] + (py/initialize! :no-io-redirect? false) + (doseq [items (partition 999 (get-data))] + ;;One way is to use the GC + (time + (do + (last + (eduction + (map py/->jvm) + (map (partial into {})) + items)) + (System/gc))) + ;;A faster way is to grab the gil and use the resource system + ;;This also ensures that resources within that block do not escape + (time + (py/with-gil + (py/stack-resource-context + (last + (eduction + (map py/->jvm) + (map (partial into {})) + items))))))) + + +(defn str-marshal-test + [] + (py/initialize!) + (let [test-str (py/->python "a nice string to work with")] + (time + (py/with-gil + (py/stack-resource-context + (dotimes [iter 100000] + (py/->jvm test-str))))))) + + +(defn dict-marshal-test + [] + (py/initialize!) + (let [test-item (py/->python {:a 1 :b 2})] + (time + (py/with-gil + (py/stack-resource-context + (dotimes [iter 10000] + (py/->jvm test-item))))))) From 8a3967d9a9b9ed2f0945b281048ea9acf1a2dee7 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 24 Nov 2019 11:47:35 -0700 Subject: [PATCH 007/456] 1.11 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 91645e1..723cc84 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.11-SNAPSHOT" +(defproject cnuernber/libpython-clj "1.11" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" From 2ff91403a27a71ecf54afacaa76b7138a98507d0 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 24 Nov 2019 11:47:48 -0700 Subject: [PATCH 008/456] snap --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 723cc84..f05c0c2 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.11" +(defproject cnuernber/libpython-clj "1.12-SNAPSHOT" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" From 05a12ed634a85db1064ed14863806baf087a5982 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Mon, 25 Nov 2019 11:03:52 -0700 Subject: [PATCH 009/456] Lots of hardening and perf testing. Found another crash bug in the way the None type was handled --- docs/scopes-and-gc.md | 73 ++++++++++++++++++++ src/libpython_clj/python.clj | 19 +++++- src/libpython_clj/python/bridge.clj | 6 +- src/libpython_clj/python/interop.clj | 5 +- src/libpython_clj/python/object.clj | 35 ++++++---- test/libpython_clj/python_test.clj | 8 +++ test/libpython_clj/stress_test.clj | 99 +++++++++++++++++++++------- 7 files changed, 203 insertions(+), 42 deletions(-) create mode 100644 docs/scopes-and-gc.md diff --git a/docs/scopes-and-gc.md b/docs/scopes-and-gc.md new file mode 100644 index 0000000..c76434f --- /dev/null +++ b/docs/scopes-and-gc.md @@ -0,0 +1,73 @@ +# Scopes And Garbage Collection + + +libpython-clj now supports stack-based scoping rules so you can guarantee all python +objects created during a section of code will be released by a certain point. + + +Using the stack-based scoping looks like: + + + +```clojure +user> (require '[libpython-clj.python :as py]) +nil +user> (py/initialize!) +... (logging elided) +:ok +user> (py/stack-resource-context + (-> (py/->py-dict {:a 1 :b 2}) + ;;Note - Without this call you guarantee a crash. + (py/->jvm))) +{"a" 1, "b" 2} +``` + + +You must either call ->jvm or return a keyword at the end of your scope. + +In the case where you are processing a batch of items (which we recommend for perf +reasons), you can also grab the GIL at the top of your thing: + +```clojure +user> (def dict-seq (py/as-jvm (py/->py-list (repeat 1000 (py/->py-dict {:a 1 :b 2}))))) +#'user/dict-seq + + +user> (def ignored (time (mapv py/->jvm dict-seq))) +"Elapsed time: 2200.556506 msecs" +#'user/ignored +user> (def ignored (time (py/with-gil (mapv py/->jvm dict-seq)))) +"Elapsed time: 2095.815518 msecs" +#'user/ignored +``` + + +The hidden thing above, regardless of if you grab the gil or not is that you are +actually holding onto a lot of python objects that could be released. Hence if you +aren't disciplined about calling System/gc or if the jvm gc just decides not to run +you could be allocating a lot of native-heap objects. Plus what you don't see is +that if you call System/gc the resource thread dedicated to releasing things will +have a lot of work to do. + +For production use cases where you need a bit more assurance that things get released, +please consider both grabbing the gil *and* opening a resource context: + + +```clojure +user> (def ignored (time (py/with-gil-stack-rc-context + (->> (repeatedly 1000 #(py/->py-dict {:a 1 :b 2})) + (py/->py-list) + (py/as-jvm) + (mapv py/->jvm))))) + +"Elapsed time: 3246.847595 msecs" +#'user/ignored +``` + +This took a second longer! But, you *know* that all python objects allocated within +that scope are released. Before, you would be in essence hoping that things would be +released soon enough. + +Again, for production contexts we recommend batch processing objects *and* using +the `with-gil-stack-rc-context` function call that correctly grabs the gil, opens +a resource context and then releases anything allocated within that context. diff --git a/src/libpython_clj/python.clj b/src/libpython_clj/python.clj index 65a077a..3c667ca 100644 --- a/src/libpython_clj/python.clj +++ b/src/libpython_clj/python.clj @@ -53,7 +53,10 @@ "Create a stack-based resource context. All python objects allocated within this context will be released at the termination of this context. !!This means that no python objects can escape from this context!! - You must use copy semantics (->jvm) for anything escaping this context." + You must use copy semantics (->jvm) for anything escaping this context. + Furthermore, if you are returning generic python objects you may need + to call (into {}) or something like that just to ensure that absolutely + everything is copied into the jvm." [& body] `(pyobj/stack-resource-context ~@body)) @@ -68,6 +71,20 @@ ~@body)) +(defmacro with-gil-stack-rc-context + "Capture the gil, open a resource context. The resource context is released + before the gil is leading to much faster resource collection. See documentation + on `stack-resource-context` for multiple warnings; the most important one being + that if a python object escapes this context your program will eventually, at + some undefined point in the future crash. That being said, this is the recommended + pathway to use in production contexts where you want defined behavior and timings + related to use of python." + [& body] + `(with-gil + (stack-resource-context + ~@body))) + + (export-symbols libpython-clj.python.interop libpython-clj-module-name create-bridge-from-att-map) diff --git a/src/libpython_clj/python/bridge.clj b/src/libpython_clj/python/bridge.clj index 9be1ab4..e791835 100644 --- a/src/libpython_clj/python/bridge.clj +++ b/src/libpython_clj/python/bridge.clj @@ -220,10 +220,12 @@ (->python [item# options#] pyobj#) py-proto/PBridgeToPython (as-python [item# options#] pyobj#) - py-proto/PCopyToJVM - (->jvm [item# options#] item#) py-proto/PBridgeToJVM (as-jvm [item# options#] item#) + py-proto/PCopyToJVM + (->jvm [item# options#] + (with-interpreter interpreter# + (->jvm pyobj# options#))) py-proto/PPyObject (dir [item#] (with-interpreter interpreter# diff --git a/src/libpython_clj/python/interop.clj b/src/libpython_clj/python/interop.clj index cbd0809..ea80a1b 100644 --- a/src/libpython_clj/python/interop.clj +++ b/src/libpython_clj/python/interop.clj @@ -383,7 +383,7 @@ (getAttr [bridge att-name] (if-let [retval (get att-map att-name)] (incref retval) - (libpy/Py_None))) + (incref (libpy/Py_None)))) (setAttr [bridge att-name att-value] (throw (ex-info "Cannot set attributes" {}))) (dir [bridge] dir-data) @@ -425,7 +425,8 @@ (create-bridge-from-att-map writer-var {"write" (->python (fn [& args] - (.write ^Writer @writer-var (str (first args))))) + (.write ^Writer @writer-var (str (first args))) + )) "flush" (->python (fn [& args])) "isatty" (->python (fn [& args] (libpy/Py_False))) diff --git a/src/libpython_clj/python/object.clj b/src/libpython_clj/python/object.clj index cf165fe..3b454e3 100644 --- a/src/libpython_clj/python/object.clj +++ b/src/libpython_clj/python/object.clj @@ -114,7 +114,7 @@ (when-not skip-check-error? (check-error-throw)) ;;We don't wrap pynone - (when (and pyobj + (if (and pyobj (not= (Pointer/nativeValue (jna/as-ptr pyobj)) (Pointer/nativeValue (jna/as-ptr (libpy/Py_None))))) (let [interpreter (ensure-bound-interpreter) @@ -147,7 +147,11 @@ (libpy/Py_DecRef (Pointer. pyobj-value)) (catch Throwable e (log-error "Exception while releasing object: %s" e)))) - *pyobject-tracking-flags*)))) + *pyobject-tracking-flags*)) + (do + ;;Special handling for PyNone types + (libpy/Py_DecRef pyobj) + nil))) (defmacro stack-resource-context @@ -175,6 +179,11 @@ pyobj)) +(defn refcount + [pyobj] + (.ob_refcnt (PyObject. (jna/as-ptr pyobj)))) + + (defn py-true [] (libpy/Py_True)) @@ -371,7 +380,7 @@ (let [retval (apply fn-obj (->jvm args))] (if (nil? retval) - (libpy/Py_None) + (incref (libpy/Py_None)) (->python retval))) (catch Throwable e (log-error (format "%s:%s" e (with-out-str @@ -661,7 +670,7 @@ (extend-protocol py-proto/PyCall Pointer (do-call-fn [callable arglist kw-arg-map] - (with-gil nil + (with-gil (-> (cond (seq kw-arg-map) (libpy/PyObject_Call callable (->py-tuple arglist) @@ -687,7 +696,7 @@ (defn python->jvm-copy-hashmap [pyobj & [map-items]] - (with-gil nil + (with-gil (when-not (= 1 (libpy/PyMapping_Check pyobj)) (throw (ex-info (format "Object does not implement the mapping protocol: %s" (python-type pyobj))))) @@ -700,7 +709,7 @@ (defn python->jvm-copy-persistent-vector [pyobj] - (with-gil nil + (with-gil (when-not (= 1 (libpy/PySequence_Check pyobj)) (throw (ex-info (format "Object does not implement sequence protocol: %s" (python-type pyobj))))) @@ -717,7 +726,7 @@ you could have a list of python none types or something so you have to iterate till you get a StopIteration error." [iter-fn & [item-conversion-fn]] - (with-gil nil + (with-gil (let [interpreter (ensure-bound-interpreter)] (let [py-iter (py-proto/call iter-fn) py-next-fn (when py-iter (py-proto/get-attr py-iter "__next__")) @@ -763,7 +772,7 @@ maintains a reference to the python object, however, so this method isn't necessarily safe." [pyobj & [item-conversion-fn]] - (with-gil nil + (with-gil (when-not (has-attr? pyobj "__iter__") (throw (ex-info (format "object is not iterable: %s" (python-type pyobj)) @@ -782,13 +791,13 @@ (defmethod pyobject->jvm :int [pyobj] - (with-gil nil + (with-gil (libpy/PyLong_AsLongLong pyobj))) (defmethod pyobject->jvm :float [pyobj] - (with-gil nil + (with-gil (libpy/PyFloat_AsDouble pyobj))) @@ -799,13 +808,13 @@ (defmethod pyobject->jvm :str [pyobj] - (with-gil nil + (with-gil (py-string->string pyobj))) (defn pyobj-true? [pyobj] - (with-gil nil + (with-gil (= 1 (libpy/PyObject_IsTrue pyobj)))) @@ -839,7 +848,7 @@ ;;numpy types (defn numpy-scalar->jvm [pyobj] - (with-gil nil + (with-gil (-> (py-proto/get-attr pyobj "data") (py-proto/get-item (->py-tuple [])) ->jvm))) diff --git a/test/libpython_clj/python_test.clj b/test/libpython_clj/python_test.clj index 348dcd3..493d817 100644 --- a/test/libpython_clj/python_test.clj +++ b/test/libpython_clj/python_test.clj @@ -248,3 +248,11 @@ (vec my-obj))) (is (= [4 5 6] (vec (py/call-attr py-mod "for_iter" my-python-item))))))) + + +(deftest bridged-dict-to-jvm + (py/initialize!) + (let [py-dict (py/->py-dict {:a 1 :b 2}) + bridged (py/as-jvm py-dict) + copied-back (py/->jvm bridged)] + (is (instance? clojure.lang.PersistentArrayMap copied-back)))) diff --git a/test/libpython_clj/stress_test.clj b/test/libpython_clj/stress_test.clj index 94a2ab7..88a5e32 100644 --- a/test/libpython_clj/stress_test.clj +++ b/test/libpython_clj/stress_test.clj @@ -1,33 +1,70 @@ (ns libpython-clj.stress-test "A set of tests meant to crash the system or just run the system out of memory if it isn't setup correctly." - (:require [libpython-clj.python :as py])) + (:require [libpython-clj.python :as py] + [libpython-clj.jna :as libpy] + [libpython-clj.python.object :as pyobject])) -(defn get-data - [] - (let [gd-fn - (-> (py/run-simple-string " +(def test-script + (memoize + (fn [] + (-> (py/run-simple-string " +import itertools + def getdata(): while True: yield {'a': 1, 'b': 2} + +def print_data(): + print(\"hey\") + +def getmultidata(): + a = {'disableReason': 'None', 'generationNumber': 1186, 'leDomain': 0, 'protocol': 'FC', 'index': 197, 'authentication': 'None', 'physicalID': 4, 'cFlags': ['0x1'], 'wwn': '20:c5:00:05:1e:ba:a7:00', 'scn': 'Offline', 'health': 'OFFLINE', 'connectionSpeed': 'N8Gbps', 'shareIFID': None, 'id': '586a6086ce8d6e5d86a0a04b', 'flags': ['0x4001', 'PRESENT', 'LED'], 'portID': '01c500', 'storagecenterSnapshotType': 'live', 'transitionCount': 1, 'portType': 17.0, 'podPort': None, 'name': 'slot9 port21', 'deviceNumber': 267654, 'isPartOtherAD': False, 'creditRecovery': 'Inactive', 'distance': 'normal', 'storagecenterSnapshotTime': '2019-11-04T14:12:20.740000+00:00', 'scnID': 2, 'portIFID': '4392001d', 'fcFastwrite': False, 'state': 'Offline', 'wwnConnected': [], 'faa': 'Inactive', 'localSwcFlags': ['0x0'], 'stateID': 2, 'aoq': 'Inactive', 'physical': 'No_Light', 'peerBeacon': False} + b = {'disableReason': 'None', 'generationNumber': 0, 'leDomain': 0, 'protocol': 'FC', 'index': 777, 'authentication': 'None', 'physicalID': 6, 'cFlags': ['0x1'], 'wwn': '50:00:53:32:3d:6f:73:09', 'scn': 'Online', 'health': 'HEALTHY', 'connectionSpeed': 'N8Gbps', 'shareIFID': None, 'id': '586a603560a298494d6a5e97', 'flags': ['0x24b03', 'PRESENT', 'ACTIVE', 'F_PORT', 'G_PORT', 'LOGICAL_ONLINE', 'LOGIN', 'NOELP', 'LED', 'ACCEPT'], 'portID': '3d8d80', 'storagecenterSnapshotType': 'deleted', 'transitionCount': 0, 'portType': 17.0, 'podPort': None, 'name': 'slot1 port57', 'deviceNumber': 365521, 'isPartOtherAD': None, 'creditRecovery': 'Inactive', 'distance': 'normal', 'storagecenterSnapshotTime': '2018-06-07T02:12:16.083000+00:00', 'scnID': 1, 'portIFID': '43120039', 'fcFastwrite': False, 'state': 'Online', 'wwnConnected': ['21:00:00:24:ff:f7:37:01'], 'faa': 'Inactive', 'localSwcFlags': ['0x0'], 'stateID': 1, 'aoq': 'Inactive', 'physical': 'In_Sync', 'peerBeacon': False} + c = {'disableReason': 'None', 'generationNumber': 0, 'leDomain': 0, 'protocol': 'FC', 'index': 127, 'authentication': 'None', 'physicalID': 4, 'cFlags': ['0x1'], 'wwn': '20:7f:00:05:33:61:48:01', 'scn': 'Offline', 'health': 'OFFLINE', 'connectionSpeed': 'N8Gbps', 'shareIFID': None, 'id': '5c7d9d73c146f2bb9dd1a48c', 'flags': ['0x1', 'PRESENT'], 'portID': '42e000', 'storagecenterSnapshotType': 'live', 'transitionCount': 1, 'portType': 17.0, 'podPort': None, 'name': 'slot12 port15', 'deviceNumber': 382664, 'isPartOtherAD': None, 'creditRecovery': 'Inactive', 'distance': 'normal', 'storagecenterSnapshotTime': '2019-03-04T21:49:38.234000+00:00', 'scnID': 2, 'portIFID': '43c20037', 'fcFastwrite': False, 'state': 'Offline', 'wwnConnected': [], 'faa': None, 'localSwcFlags': ['0x0'], 'stateID': 2, 'aoq': None, 'physical': 'No_Light', 'peerBeacon': False} + d = {'disableReason': 'None', 'generationNumber': 506, 'leDomain': 0, 'protocol': 'FC', 'index': 187, 'authentication': 'None', 'physicalID': 4, 'cFlags': ['0x1'], 'wwn': '20:bb:00:05:33:24:6a:01', 'scn': 'Offline', 'health': 'OFFLINE', 'connectionSpeed': 'N8Gbps', 'shareIFID': None, 'id': '586a604a1d2c89767dc345f1', 'flags': ['0x4001', 'PRESENT', 'LED'], 'portID': '3ed100', 'storagecenterSnapshotType': 'live', 'transitionCount': 1, 'portType': 17.0, 'podPort': None, 'name': 'slot4 port27 DISABLED server 640461 seg config', 'deviceNumber': 365522, 'isPartOtherAD': None, 'creditRecovery': 'Inactive', 'distance': 'normal', 'storagecenterSnapshotTime': '2019-10-18T13:10:24.968000+00:00', 'scnID': 2, 'portIFID': '4342100b', 'fcFastwrite': False, 'state': 'Offline', 'wwnConnected': [], 'faa': 'Inactive', 'localSwcFlags': ['0x0'], 'stateID': 2, 'aoq': 'Inactive', 'physical': 'No_Light', 'peerBeacon': False} + for x in itertools.cycle([a, b, c, d]): + yield x ") - :globals - (get "getdata"))] + :globals)))) + + +(defn get-data + [] + (let [gd-fn (-> (test-script) + (get "getdata"))] + (gd-fn))) + + +(defn print-data + [] + (let [pr-fn (-> (test-script) + (get "print_data"))] + (pr-fn))) + + +(defn get-multi-data + [] + (let [gd-fn (-> (test-script) + (get "getmultidata"))] (gd-fn))) + ;;If you want to see how the sausage is made... (alter-var-root #'libpython-clj.python.object/*object-reference-logging* (constantly false)) ;;Ensure that failure to open resource context before tracking for stack -;;related things causes immediate failure. +;;related things causes immediate failure. Unless you feel like being pedantic, +;;this isn't necessary. The python library automatically switches to normal gc-only +;;mode if a resource context is open. (alter-var-root #'tech.resource.stack/*resource-context* (constantly nil)) (defn forever-test [] - (py/initialize! :no-io-redirect? false) + (py/initialize!) (doseq [items (partition 999 (get-data))] ;;One way is to use the GC (time @@ -41,13 +78,21 @@ def getdata(): ;;A faster way is to grab the gil and use the resource system ;;This also ensures that resources within that block do not escape (time - (py/with-gil - (py/stack-resource-context - (last - (eduction - (map py/->jvm) - (map (partial into {})) - items))))))) + (py/with-gil-stack-rc-context + (last + (eduction + (map py/->jvm) + (map (partial into {})) + items)))))) + + +(defn multidata-test + [] + (py/initialize!) + (doseq [items (partition 1000 (get-multi-data))] + (time (do (py/with-gil-stack-rc-context + (mapv py/->jvm items)) + :ok)))) (defn str-marshal-test @@ -55,10 +100,9 @@ def getdata(): (py/initialize!) (let [test-str (py/->python "a nice string to work with")] (time - (py/with-gil - (py/stack-resource-context - (dotimes [iter 100000] - (py/->jvm test-str))))))) + (py/with-gil-stack-rc-context + (dotimes [iter 100000] + (py/->jvm test-str)))))) (defn dict-marshal-test @@ -66,7 +110,14 @@ def getdata(): (py/initialize!) (let [test-item (py/->python {:a 1 :b 2})] (time - (py/with-gil - (py/stack-resource-context - (dotimes [iter 10000] - (py/->jvm test-item))))))) + (py/with-gil-stack-rc-context + (dotimes [iter 10000] + (py/->jvm test-item)))))) + + +(defn print-stress-test + [] + (py/initialize!) + (dotimes [iter 10000] + (with-out-str + (print-data)))) From 32dab03caefb64103a777c9ac709743a830b258d Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Mon, 25 Nov 2019 11:04:21 -0700 Subject: [PATCH 010/456] 1.12 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index f05c0c2..3be2873 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.12-SNAPSHOT" +(defproject cnuernber/libpython-clj "1.12" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" From 6ada8d863b8a341e02ef235e299dd8cb21118a93 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Mon, 25 Nov 2019 11:04:36 -0700 Subject: [PATCH 011/456] snap --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 3be2873..6fcbaf4 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.12" +(defproject cnuernber/libpython-clj "1.13-SNAPSHOT" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" From 592c99b00aaa05842c19cb82c78950f48a2da374 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Mon, 25 Nov 2019 11:11:35 -0700 Subject: [PATCH 012/456] editing --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ccf5ea6..04fb127 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,8 @@ JNA libpython bindings to the tech ecosystem. * Bridge between JVM objects and Python objects easily; use Python in your Java and use some Java in your Python. * Python objects are linked to the JVM GC such that when they are no longer reachable - from the JVM their references are released. + from the JVM their references are released. Scope based resource contexts are + [also available](https://github.com/cnuernber/libpython-clj/blob/master/docs/scopes-and-gc.md). * The exact same binary can run top of on multiple version of python reducing version dependency chain management issues. * Development of new functionality is faster because it can be done from purely from the @@ -302,6 +303,7 @@ distance. ## Further Information * [design documentation](docs/design.md) +* [scope and garbage collection docs](https://github.com/cnuernber/libpython-clj/blob/master/docs/scopes-and-gc.md) * [examples](example/README.md) * [docker setup](https://github.com/scicloj/docker-hub) * [pandas bindings (!!)](https://github.com/alanmarazzi/panthera) From 529840fd543e2b49245728d3b64044f756648cbb Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 26 Nov 2019 06:57:02 -0700 Subject: [PATCH 013/456] editing --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 04fb127..c325cb4 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ JNA libpython bindings to the tech ecosystem. * Bridge between JVM objects and Python objects easily; use Python in your Java and use some Java in your Python. * Python objects are linked to the JVM GC such that when they are no longer reachable - from the JVM their references are released. Scope based resource contexts are + from the JVM their references are released. Scope based resource contexts are [also available](https://github.com/cnuernber/libpython-clj/blob/master/docs/scopes-and-gc.md). * The exact same binary can run top of on multiple version of python reducing version dependency chain management issues. @@ -25,7 +25,7 @@ We have a [video](https://www.youtube.com/watch?v=ajDiGS73i2o) up of a scicloj c discussion with demos. -TechAscent has a [blog post](http://www.techascent.com/blog/functions-across-languages.html) +TechAscent has a [blog post](http://www.techascent.com/blog/functions-across-languages.html) up about how the inter-language bindings actually work at a very low level. @@ -115,7 +115,7 @@ there are java wrappers over the top of them. For instance, `Object.toString` f its implementation to the python function `__str__`. ```clojure - +(def bridged (run-simple-string "print('hey')")) (instance? java.util.Map (:globals bridged)) true user> (:globals bridged) From c6be956dd2e4e51fa097732bcc97f308c8a291b3 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 26 Nov 2019 10:02:49 -0700 Subject: [PATCH 014/456] Syntax sugar and major perf upgrade copying dicts into jvm space. --- README.md | 23 +++++++ src/libpython_clj/jna/concrete/dict.clj | 6 +- src/libpython_clj/python.clj | 49 +++++++++++++++ src/libpython_clj/python/bridge.clj | 79 ++++++++++++++++++------- src/libpython_clj/python/object.clj | 18 +++++- test/libpython_clj/python_test.clj | 14 +++++ 6 files changed, 160 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index c325cb4..8d0cbcc 100644 --- a/README.md +++ b/README.md @@ -257,6 +257,29 @@ Execution error (ExceptionInfo) at libpython-clj.python.interpreter/check-error- SyntaxError: EOL while scanning string literal ``` +### Some Syntax Sugar +```clojure +user> (py/from-import numpy linspace) +#'user/linspace +user> (linspace 2 3 :num 10) +[2. 2.11111111 2.22222222 2.33333333 2.44444444 2.55555556 + 2.66666667 2.77777778 2.88888889 3. ] +user> (doc linspace) +------------------------- +user/linspace + + Return evenly spaced numbers over a specified interval. + + Returns `num` evenly spaced samples, calculated over the + interval [`start`, `stop`]. + +``` + +* `from-import` - sugar around python `from a import b`. Takes multiple b's. +* `import-as` - surgar around python `import a as b`. +* `a$` - call an attribute using symbol att name. Keywords map to kwargs +* `c$` - call an object mapping keywords to fn-args + ### Numpy Speaking of numpy, you can move data between numpy and java easily. diff --git a/src/libpython_clj/jna/concrete/dict.clj b/src/libpython_clj/jna/concrete/dict.clj index 2902568..93548a0 100644 --- a/src/libpython_clj/jna/concrete/dict.clj +++ b/src/libpython_clj/jna/concrete/dict.clj @@ -192,9 +192,9 @@ structure, and since the structure is sparse, the offsets are not consecutive." Integer [p ensure-pyobj] - [ppos jna/as-ptr] - [pkey jna/as-ptr] - [pvalue jna/as-ptr]) + [ppos (partial jna/ensure-type jna/size-t-ref-type)] + [pkey jna/ensure-ptr-ptr] + [pvalue jna/ensure-ptr-ptr]) (def-pylib-fn PyDict_Merge diff --git a/src/libpython_clj/python.clj b/src/libpython_clj/python.clj index 3c667ca..27f7667 100644 --- a/src/libpython_clj/python.clj +++ b/src/libpython_clj/python.clj @@ -91,6 +91,9 @@ (export-symbols libpython-clj.python.bridge + args->pos-kw-args + cfn + afn as-jvm as-python ->numpy @@ -276,3 +279,49 @@ retval#) (catch Throwable e# (with-exit-error-handler ~varname e#)))))))) + + +(defmacro a$ + "Call an attribute of an object. Similar calling conventions to afn except: + Keywords must be compile time constants. So this won't work with 'apply'. On the + other hand, building the positional and kw argmaps happens at compile time as + opposed to at runtime. The attr name can be a symbol." + [item attr & args] + (let [attr-name (if (symbol? attr) + (name attr) + attr) + [pos-args kw-args] (args->pos-kw-args args)] + `(call-attr-kw ~item ~attr-name ~pos-args ~kw-args))) + + +(defmacro c$ + "Call an object. Similar calling conventions to cfn except: + Keywords must be compile time constants. So this won't work with 'apply'. On the + other hand, building the positional and kw argmaps happens at compile time as + opposed to at runtime." + [item & args] + (let [[pos-args kw-args] (args->pos-kw-args args)] + `(call-kw ~item ~pos-args ~kw-args))) + + +(defmacro import-as + "Import a module and assign it to a var. Documentation is included." + [module-path varname] + `(let [~'mod-data (import-module ~(name module-path))] + (def ~varname (import-module ~(name module-path))) + (alter-meta! #'~varname assoc :doc (get-attr ~'mod-data "__doc__")) + #'~varname)) + + +(defmacro from-import + "Support for the from a import b,c style of importing modules and symbols in python. + Documentation is included." + [module-path item & args] + `(do + (let [~'mod-data (import-module ~(name module-path))] + ~@(map (fn [varname] + `(let [~'var-data (get-attr ~'mod-data ~(name varname))] + (def ~varname ~'var-data) + (alter-meta! #'~varname assoc :doc (get-attr ~'var-data "__doc__")) + #'~varname)) + (concat [item] args))))) diff --git a/src/libpython_clj/python/bridge.clj b/src/libpython_clj/python/bridge.clj index e791835..07a43fc 100644 --- a/src/libpython_clj/python/bridge.clj +++ b/src/libpython_clj/python/bridge.clj @@ -517,6 +517,55 @@ (python->jvm-iterator iter-fn as-jvm)))) +(defn args->pos-kw-args + "Utility function that, given a list of arguments, separates them + into positional and keyword arguments. Throws an exception if the + keyword argument is not followed by any more arguments." + [arglist] + (loop [args arglist + pos-args [] + kw-args nil] + (if-not (seq args) + [pos-args kw-args] + (let [arg (first args) + [pos-args kw-args args] + (if (keyword? arg) + (if-not (seq (rest args)) + (throw (Exception. + (format "Keyword arguments must be followed by another arg: %s" + (str arglist)))) + [pos-args (assoc kw-args arg (first (rest args))) + (drop 2 args)]) + [(conj pos-args (first args)) + kw-args + (rest args)])] + (recur args pos-args kw-args))))) + + +(defn cfn + "Call an object. + Arguments are passed in positionally. Any keyword + arguments are paired with the next arg, gathered, and passed into the + system as *kwargs. + + Not having an argument after a keyword argument is an error." + [item & args] + (let [[pos-args kw-args] (args->pos-kw-args args)] + (py-proto/call-kw item pos-args kw-args))) + + +(defn afn + "Call an attribute of an object. + Arguments are passed in positionally. Any keyword + arguments are paired with the next arg, gathered, and passed into the + system as *kwargs. + + Not having an argument after a keyword argument is an error." + [item attr & args] + (let [[pos-args kw-args] (args->pos-kw-args args)] + (py-proto/call-attr-kw item attr pos-args kw-args))) + + (defn generic-python-as-jvm "Given a generic pyobject, wrap it in a read-only map interface where the keys are the attributes." @@ -542,48 +591,32 @@ ;;uggh (invoke [this] (with-interpreter interpreter - (-> (libpy/PyObject_CallObject pyobj nil) - wrap-pyobject - as-jvm))) + (cfn this))) (invoke [this arg0] (with-interpreter interpreter - (-> (libpy/PyObject_CallObject pyobj (as-tuple [arg0])) - wrap-pyobject - as-jvm))) + (cfn this arg0))) (invoke [this arg0 arg1] (with-interpreter interpreter - (-> (libpy/PyObject_CallObject pyobj (as-tuple [arg0 arg1])) - wrap-pyobject - as-jvm))) + (cfn this arg0 arg1))) (invoke [this arg0 arg1 arg2] (with-interpreter interpreter - (-> (libpy/PyObject_CallObject pyobj (as-tuple [arg0 arg1 arg2])) - wrap-pyobject - as-jvm))) + (cfn this arg0 arg1 arg2))) (invoke [this arg0 arg1 arg2 arg3] (with-interpreter interpreter - (-> (libpy/PyObject_CallObject - pyobj (as-tuple [arg0 arg1 arg2 arg3])) - wrap-pyobject - as-jvm))) + (cfn this arg0 arg1 arg2 arg3))) (invoke [this arg0 arg1 arg2 arg3 arg4] (with-interpreter interpreter - (-> (libpy/PyObject_CallObject - pyobj (as-tuple [arg0 arg1 arg2 arg3 arg4])) - wrap-pyobject - as-jvm))) + (cfn this arg0 arg1 arg2 arg3 arg4))) (applyTo [this arglist] (with-interpreter interpreter - (-> (libpy/PyObject_CallObject pyobj (as-tuple arglist)) - wrap-pyobject - as-jvm))) + (apply cfn this arglist))) ;;Mark this as executable Fn PyFunction diff --git a/src/libpython_clj/python/object.clj b/src/libpython_clj/python/object.clj index 3b454e3..f503e4c 100644 --- a/src/libpython_clj/python/object.clj +++ b/src/libpython_clj/python/object.clj @@ -92,7 +92,7 @@ (defn ->jvm "Copy an object into the jvm (if it wasn't there already.)" [item & [options]] - (when item + (when-not (nil? item) (py-proto/->jvm item options))) @@ -835,7 +835,19 @@ (defmethod pyobject->jvm :dict [pyobj] - (python->jvm-copy-hashmap pyobj)) + (with-gil + (let [ppos (jna/size-t-ref 0) + pkey (PointerByReference.) + pvalue (PointerByReference.) + retval (transient {})] + (loop [next-retval (libpy/PyDict_Next pyobj ppos pkey pvalue)] + (if (not= 0 next-retval) + (do + (assoc! retval + (->jvm (jna/as-ptr pkey)) + (->jvm (jna/as-ptr pvalue))) + (recur (libpy/PyDict_Next pyobj ppos pkey pvalue))) + (persistent! retval)))))) (defmethod pyobject->jvm :set @@ -919,7 +931,7 @@ (python->jvm-copy-persistent-vector pyobj))) ;;Sequences become persistent vectors (= 1 (libpy/PySequence_Check pyobj)) - (python->jvm-copy-persistent-vector) + (python->jvm-copy-persistent-vector pyobj) :else {:type (python-type pyobj) :value (Pointer/nativeValue (jna/as-ptr pyobj))})) diff --git a/test/libpython_clj/python_test.clj b/test/libpython_clj/python_test.clj index 493d817..793220f 100644 --- a/test/libpython_clj/python_test.clj +++ b/test/libpython_clj/python_test.clj @@ -1,6 +1,7 @@ (ns libpython-clj.python-test (:require [libpython-clj.python :as py] [tech.v2.datatype :as dtype] + [tech.v2.datatype.functional :as dfn] [tech.v2.tensor :as dtt] [clojure.test :refer :all]) (:import [java.io StringWriter] @@ -256,3 +257,16 @@ bridged (py/as-jvm py-dict) copied-back (py/->jvm bridged)] (is (instance? clojure.lang.PersistentArrayMap copied-back)))) + + +(deftest calling-conventions + (py/initialize!) + (let [np (py/import-module "numpy") + linspace (py/get-attr np "linspace")] + + (is (dfn/equals [2.000 2.250 2.500 2.750 3.000] + (py/as-tensor (linspace 2 3 :num 5)))) + (is (dfn/equals [2.000 2.250 2.500 2.750 3.000] + (py/as-tensor (py/c$ linspace 2 3 :num 5)))) + (is (dfn/equals [2.000 2.250 2.500 2.750 3.000] + (py/as-tensor (py/a$ np linspace 2 3 :num 5)))))) From 91c820eccdfae15810e421c37309be6f560d640a Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 26 Nov 2019 10:03:16 -0700 Subject: [PATCH 015/456] 1.13 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 6fcbaf4..836b721 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.13-SNAPSHOT" +(defproject cnuernber/libpython-clj "1.13" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" From 6cd5394ba4445f39d48809d8dbddc0c4559ea7a1 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 26 Nov 2019 10:03:24 -0700 Subject: [PATCH 016/456] snap --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 836b721..66a539c 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.13" +(defproject cnuernber/libpython-clj "1.14-SNAPSHOT" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" From 4cf7c965c39530a29a76971ff8e71cc8fd44a698 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 26 Nov 2019 10:42:41 -0700 Subject: [PATCH 017/456] Trying out auto-bridging some types for the top level API. --- src/libpython_clj/python.clj | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/src/libpython_clj/python.clj b/src/libpython_clj/python.clj index 27f7667..71229cc 100644 --- a/src/libpython_clj/python.clj +++ b/src/libpython_clj/python.clj @@ -38,13 +38,9 @@ (export-symbols libpython-clj.python.object - ->py-dict ->py-float - ->py-list ->py-long ->py-string - ->py-tuple - ->py-fn ->python ->jvm) @@ -100,6 +96,32 @@ as-numpy) +(defn ->py-dict + "Create a python dictionary" + [item] + (-> (pyobj/->py-dict item) + (as-jvm))) + + +(defn ->py-list + "Create a python list" + [item] + (-> (pyobj/->py-list item) + (as-jvm))) + + +(defn ->py-tuple + [item] + (-> (pyobj/->py-tuple item) + (as-jvm))) + + +(defn ->py-fn + [item] + (-> (pyobj/->py-fn item) + (as-jvm))) + + (defn run-simple-string "Run a string expression returning a map of {:globals :locals :result}. From 17cf3428111c69ffc1b23940111dc84a496bd323 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 26 Nov 2019 10:43:59 -0700 Subject: [PATCH 018/456] editing --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8d0cbcc..1f45a5e 100644 --- a/README.md +++ b/README.md @@ -278,7 +278,7 @@ user/linspace * `from-import` - sugar around python `from a import b`. Takes multiple b's. * `import-as` - surgar around python `import a as b`. * `a$` - call an attribute using symbol att name. Keywords map to kwargs -* `c$` - call an object mapping keywords to fn-args +* `c$` - call an object mapping keywords to kwargs ### Numpy From e0e5619f1e511daeff68fab9a0e7f9f2e9773638 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 26 Nov 2019 13:20:54 -0700 Subject: [PATCH 019/456] editign --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 1f45a5e..4d0c12c 100644 --- a/README.md +++ b/README.md @@ -325,6 +325,7 @@ distance. ## Further Information +* [development discussion forum](https://clojurians.zulipchat.com/#narrow/stream/215609-libpython-clj-dev) * [design documentation](docs/design.md) * [scope and garbage collection docs](https://github.com/cnuernber/libpython-clj/blob/master/docs/scopes-and-gc.md) * [examples](example/README.md) From eb6ac5f4c671ce294c5483cd95f52335e6e6c59c Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Wed, 27 Nov 2019 15:12:53 -0700 Subject: [PATCH 020/456] small datatype upgrade --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 66a539c..4f41a76 100644 --- a/project.clj +++ b/project.clj @@ -4,7 +4,7 @@ :license {:name "EPL-2.0" :url "https://www.eclipse.org/legal/epl-2.0/"} :dependencies [[org.clojure/clojure "1.10.1"] - [techascent/tech.datatype "4.54"] + [techascent/tech.datatype "4.56"] [camel-snake-kebab "0.4.0"]] :repl-options {:init-ns user} :java-source-paths ["java"]) From 45de248b0e93fb9c3a3a21ef5ea36781e209245b Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Fri, 29 Nov 2019 10:56:52 -0700 Subject: [PATCH 021/456] Attempted to make keras example as clojury as possible. --- example/project.clj | 2 +- example/src/keras_simple.clj | 74 +++++++++++++++--------------------- 2 files changed, 32 insertions(+), 44 deletions(-) diff --git a/example/project.clj b/example/project.clj index 32a53ad..4063c45 100644 --- a/example/project.clj +++ b/example/project.clj @@ -4,4 +4,4 @@ :license {:name "EPL-2.0" :url "https://www.eclipse.org/legal/epl-2.0/"} :dependencies [[org.clojure/clojure "1.10.1"] - [cnuernber/libpython-clj "1.9"]]) + [cnuernber/libpython-clj "1.13"]]) diff --git a/example/src/keras_simple.clj b/example/src/keras_simple.clj index 7986129..09c91e0 100644 --- a/example/src/keras_simple.clj +++ b/example/src/keras_simple.clj @@ -2,6 +2,10 @@ "https://machinelearningmastery.com/tutorial-first-neural-network-python-keras/" (:require [libpython-clj.python :refer [import-module + import-as + from-import + a$ ;;compile time call-attr + afn ;;runtime call-attr get-item get-attr python-type @@ -22,83 +26,67 @@ (py/initialize!) -(defonce np (import-module "numpy")) -(defonce builtins (import-module "builtins")) -(defonce keras (import-module "keras")) -(defonce keras-models (import-module "keras.models")) -(defonce keras-layers (import-module "keras.layers")) -(defonce c-types (import-module "ctypes")) +(import-as numpy np) +(import-as builtins builtins) +(from-import builtins slice) +(import-as keras keras) +(import-as keras.models keras-models) +(import-as keras.layers keras-layers) +(import-as ctypes c-types) -(defn slice - ([] - (call-attr builtins "slice" nil)) - ([start] - (call-attr builtins "slice" start)) - ([start stop] - (call-attr builtins "slice" start stop)) - ([start stop incr] - (call-attr builtins "slice" start stop incr))) +(defonce initial-data (a$ np loadtxt "pima-indians-diabetes.data.csv" :delimiter ",")) -(defonce initial-data (call-attr-kw np "loadtxt" - ["pima-indians-diabetes.data.csv"] - {"delimiter" ","})) +(def features (get-item initial-data [(slice nil) (slice 0 8)])) -(def features (get-item initial-data [(slice) (slice 0 8)])) - -(def labels (get-item initial-data [(slice) (slice 8 9)])) +(def labels (get-item initial-data [(slice nil) (slice 8 9)])) (defn dense-layer - [output-size & {:as kwords}] - (call-attr-kw keras-layers "Dense" [output-size] kwords)) + [output-size & args] + (apply py/afn keras-layers "Dense" output-size args)) (defn sequential-model [] - (call-attr keras-models "Sequential")) + (a$ keras-models Sequential)) (defn add-layer! [model layer] - (call-attr model "add" layer) + (a$ model add layer) model) (defn compile-model! - [model & {:as kw-args}] - (call-attr-kw model "compile" [] - kw-args) + [model & args] + (apply py/afn model "compile" args) model) (def model (-> (sequential-model) - (add-layer! (dense-layer 12 "input_dim" 8 "activation" "relu")) - (add-layer! (dense-layer 8 "activation" "relu")) - (add-layer! (dense-layer 1 "activation" "sigmoid")) - (compile-model! "loss" "binary_crossentropy" - "optimizer" "adam" - "metrics" (py/->py-list ["accuracy"])))) - -;;model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) + (add-layer! (dense-layer 12 :input_dim 8 :activation :relu)) + (add-layer! (dense-layer 8 :activation :relu)) + (add-layer! (dense-layer 1 :activation :sigmoid)) + (compile-model! :loss :binary_crossentropy + :optimizer :adam + :metrics (py/->py-list [:accuracy])))) (defn fit-model - [model features labels & {:as kw-args}] - (call-attr-kw model "fit" - [features labels] - kw-args) + [model features labels & args] + (apply py/afn model "fit" features labels args) model) (def fitted-model (fit-model model features labels - "epochs" 150 - "batch_size" 10)) + :epochs 150 + :batch_size 10)) (defn eval-model [model features lables] (let [model-names (->> (get-attr model "metrics_names") (mapv keyword))] - (->> (call-attr model "evaluate" features labels) + (->> (a$ model evaluate features labels) (map vector model-names) (into {})))) From 78a7c6becf6ca51bcb71e481e396efe79935e7ee Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sat, 30 Nov 2019 08:11:57 -0700 Subject: [PATCH 022/456] Found link of persistent datastructures in python. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 4d0c12c..12fd15b 100644 --- a/README.md +++ b/README.md @@ -334,6 +334,7 @@ distance. * [nextjournal notebook](https://nextjournal.com/chrisn/fun-with-matplotlib) * [scicloj video](https://www.youtube.com/watch?v=ajDiGS73i2o) * [Clojure/Python interop technical blog post](www.techascent.com/blogs/functions-across-languages.html) +* [persistent datastructures in python](https://github.com/tobgu/pyrsistent) ## Resources From 0a5eefa4228c864c001f5595d9d5dda643f3b128 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sat, 30 Nov 2019 08:47:51 -0700 Subject: [PATCH 023/456] tiny update to keras-simple --- example/src/keras_simple.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/example/src/keras_simple.clj b/example/src/keras_simple.clj index 09c91e0..52604b7 100644 --- a/example/src/keras_simple.clj +++ b/example/src/keras_simple.clj @@ -30,9 +30,9 @@ (import-as builtins builtins) (from-import builtins slice) (import-as keras keras) +(from-import keras.layers Dense) (import-as keras.models keras-models) (import-as keras.layers keras-layers) -(import-as ctypes c-types) (defonce initial-data (a$ np loadtxt "pima-indians-diabetes.data.csv" :delimiter ",")) @@ -44,7 +44,7 @@ (defn dense-layer [output-size & args] - (apply py/afn keras-layers "Dense" output-size args)) + (apply py/cfn Dense output-size args)) (defn sequential-model From 50d848fd50fd890eefc655176cb4f0a8d1f3a7fd Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Mon, 2 Dec 2019 17:26:28 -0700 Subject: [PATCH 024/456] adding more libraries to be tried dynamically. --- CHANGELOG.md | 15 +++++++++++++++ src/libpython_clj/jna/base.clj | 7 +++++++ src/libpython_clj/python.clj | 5 +---- src/libpython_clj/python/interpreter.clj | 21 +++++++++++++++++++-- 4 files changed, 42 insertions(+), 6 deletions(-) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..7ccfda6 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,15 @@ +# Time for a ChangeLog! + + +## 1.14 + +libpython-clj now searches for several shared libraries instead of being hardcoded +to just one of them. Because of this, there is now: +```clojure +libpython-clj.jna.base/*python-library-names* +``` + +This is a sequence of library names that will be tried in order. + +You can also pass in the desired library name as part of the initialize! call and +only this name will be tried. diff --git a/src/libpython_clj/jna/base.clj b/src/libpython_clj/jna/base.clj index ed90fa7..b179fd6 100644 --- a/src/libpython_clj/jna/base.clj +++ b/src/libpython_clj/jna/base.clj @@ -9,6 +9,13 @@ (def ^:dynamic *python-library* "python3.6m") +(def ^:dynamic *python-library-names* ["python3.7m" "python3.6m"]) + + +(defn library-names + [] + *python-library-names*) + (defprotocol PToPyObjectPtr (convertible-to-pyobject-ptr? [item]) diff --git a/src/libpython_clj/python.clj b/src/libpython_clj/python.clj index 71229cc..d59a1b7 100644 --- a/src/libpython_clj/python.clj +++ b/src/libpython_clj/python.clj @@ -194,11 +194,8 @@ :no-io-redirect - there if you don't want python stdout and stderr redirection to *out* and *err*." [& {:keys [program-name no-io-redirect? library-path]}] - (when library-path - (alter-var-root #'libpython-clj.jna.base/*python-library* - (constantly library-path))) (when-not @pyinterp/*main-interpreter* - (pyinterp/initialize! program-name) + (pyinterp/initialize! program-name library-path) ;;setup bridge mechansim and io redirection (pyinterop/register-bridge-type!) (when-not no-io-redirect? diff --git a/src/libpython_clj/python/interpreter.clj b/src/libpython_clj/python/interpreter.clj index df326d7..5c08b30 100644 --- a/src/libpython_clj/python/interpreter.clj +++ b/src/libpython_clj/python/interpreter.clj @@ -236,11 +236,28 @@ (defonce ^:dynamic *program-name* "") +(defn- try-load-python-library! + [libname] + (try + (jna/load-library libname) + (alter-var-root #'libpy-base/*python-library* (constantly libname)) + (libpy/Py_InitializeEx 0) + libname + (catch Exception e))) + + (defn initialize! - [& [program-name]] + [& [program-name python-library-name]] (when-not @*main-interpreter* (log-info "executing python initialize!") - (libpy/Py_InitializeEx 0) + (let [user-names (when python-library-name + [python-library-name]) + library-names (or user-names (libpy-base/library-names))] + (when-not (->> library-names + (filter try-load-python-library!) + first) + (throw (Exception. (format "Failed to initialize python library. +Attempted libraries %s" library-names))))) ;;Set program name (when-let [program-name (or program-name *program-name* "")] (resource/stack-resource-context From ec8157d5fadfaee225b7b8e312df27ee4d454a3c Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Mon, 2 Dec 2019 17:26:57 -0700 Subject: [PATCH 025/456] 1.14 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 4f41a76..14b91be 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.14-SNAPSHOT" +(defproject cnuernber/libpython-clj "1.14" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" From 689bc330f0eacb2c80aeead61c6bc1a6cbb1dc03 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Mon, 2 Dec 2019 17:27:07 -0700 Subject: [PATCH 026/456] snap --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 14b91be..8e2e91d 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.14" +(defproject cnuernber/libpython-clj "1.15-SNAPSHOT" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" From 9c2f4821e3e335c9d27ab79219b907da5961e7cd Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Mon, 2 Dec 2019 17:30:06 -0700 Subject: [PATCH 027/456] Changed location of python-library-name --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 12fd15b..b02628c 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ INFO: Reference thread starting This dynamically finds the python shared library and loads it. If you desire a different shared library you can override -[here](https://github.com/cnuernber/libpython-clj/blob/142c0dbc7056d0f5cd1969548a127a119f641c86/src/libpython_clj/jna/base.clj#L12). +[here](https://github.com/cnuernber/libpython-clj/blob/master/src/libpython_clj/jna/base.clj#L12). ### Execute Some Python From 04d8d1e8b1a48bb1eec31457c355f81ca4e46884 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 3 Dec 2019 16:20:30 -0700 Subject: [PATCH 028/456] Sytax sugar experimentation. --- CHANGELOG.md | 10 +++++++ README.md | 16 +++++++++-- src/libpython_clj/python.clj | 45 ++++++++++++++++++++++++++++-- test/libpython_clj/python_test.clj | 9 ++++++ 4 files changed, 76 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ccfda6..c57b767 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Time for a ChangeLog! +## 1.15-SNAPSHOT + +Moar syntax sugar -- +```clojure +user> (py/$. numpy linspace) + +user> (py/$.. numpy random shuffle) + +``` + ## 1.14 diff --git a/README.md b/README.md index b02628c..633fde7 100644 --- a/README.md +++ b/README.md @@ -258,6 +258,7 @@ SyntaxError: EOL while scanning string literal ``` ### Some Syntax Sugar + ```clojure user> (py/from-import numpy linspace) #'user/linspace @@ -277,8 +278,19 @@ user/linspace * `from-import` - sugar around python `from a import b`. Takes multiple b's. * `import-as` - surgar around python `import a as b`. -* `a$` - call an attribute using symbol att name. Keywords map to kwargs -* `c$` - call an object mapping keywords to kwargs +* `$a` - call an attribute using symbol att name. Keywords map to kwargs +* `$c` - call an object mapping keywords to kwargs +* `$.` - get an attribute. Can pass in symbol, string, or keyword +* `$..` - get an attribute. If more args are present, get the attribute on that +result. + +```clojure +user> (py/$. numpy linspace) + +user> (py/$.. numpy random shuffle) + +``` + ### Numpy diff --git a/src/libpython_clj/python.clj b/src/libpython_clj/python.clj index d59a1b7..9521431 100644 --- a/src/libpython_clj/python.clj +++ b/src/libpython_clj/python.clj @@ -299,12 +299,16 @@ (catch Throwable e# (with-exit-error-handler ~varname e#)))))))) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + (defmacro a$ "Call an attribute of an object. Similar calling conventions to afn except: Keywords must be compile time constants. So this won't work with 'apply'. On the other hand, building the positional and kw argmaps happens at compile time as - opposed to at runtime. The attr name can be a symbol." + opposed to at runtime. The attr name can be a symbol. + + DEPRECATION POSSIBLE - use $a." [item attr & args] (let [attr-name (if (symbol? attr) (name attr) @@ -317,12 +321,49 @@ "Call an object. Similar calling conventions to cfn except: Keywords must be compile time constants. So this won't work with 'apply'. On the other hand, building the positional and kw argmaps happens at compile time as - opposed to at runtime." + opposed to at runtime. + + DEPRECATION POSSIBLE - use $c." [item & args] (let [[pos-args kw-args] (args->pos-kw-args args)] `(call-kw ~item ~pos-args ~kw-args))) + +(defmacro $a + "Call an attribute of an object. Similar calling conventions to afn except: + Keywords must be compile time constants. So this won't work with 'apply'. On the + other hand, building the positional and kw argmaps happens at compile time as + opposed to at runtime. The attr name can be a symbol." + [item attr & args] + `(a$ ~item ~attr ~@args)) + + +(defmacro $c + "Call an object. Similar calling conventions to cfn except: + Keywords must be compile time constants. So this won't work with 'apply'. On the + other hand, building the positional and kw argmaps happens at compile time as + opposed to at runtime." + [item & args] + `(c$ ~item ~@args)) + + +(defmacro $. + "Get the attribute of an object." + [item attname] + `(get-attr ~item ~(name attname))) + + +(defmacro $.. + "Get the attribute of an object. If there are extra args, apply successive + get-attribute calls to the arguments." + [item attname & args] + `(-> (get-attr ~item ~(name attname)) + ~@(->> args + (map (fn [arg] + `(get-attr ~(name arg))))))) + + (defmacro import-as "Import a module and assign it to a var. Documentation is included." [module-path varname] diff --git a/test/libpython_clj/python_test.clj b/test/libpython_clj/python_test.clj index 793220f..66c8b95 100644 --- a/test/libpython_clj/python_test.clj +++ b/test/libpython_clj/python_test.clj @@ -270,3 +270,12 @@ (py/as-tensor (py/c$ linspace 2 3 :num 5)))) (is (dfn/equals [2.000 2.250 2.500 2.750 3.000] (py/as-tensor (py/a$ np linspace 2 3 :num 5)))))) + +(deftest syntax-sugar + (py/initialize!) + (let [np (py/import-module "numpy")] + (is (= (str (py/$. np linspace)) + (str (py/get-attr np "linspace")))) + (is (= (str (py/$.. np random shuffle)) + (str (-> (py/get-attr np "random") + (py/get-attr "shuffle"))))))) From 0fa71d13d2cb0c3869bf8d2bee9602a04c798968 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 3 Dec 2019 16:23:43 -0700 Subject: [PATCH 029/456] Making experimental nature of sugar apparent. --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 633fde7..ac9b8f5 100644 --- a/README.md +++ b/README.md @@ -280,6 +280,10 @@ user/linspace * `import-as` - surgar around python `import a as b`. * `$a` - call an attribute using symbol att name. Keywords map to kwargs * `$c` - call an object mapping keywords to kwargs + + +#### Experimental Sugar (SNAPSHOT) + * `$.` - get an attribute. Can pass in symbol, string, or keyword * `$..` - get an attribute. If more args are present, get the attribute on that result. From 3df8b6a793fdc1e21caa624e27bf825b03838341 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 3 Dec 2019 16:24:57 -0700 Subject: [PATCH 030/456] 1.15 --- CHANGELOG.md | 2 +- project.clj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c57b767..2e735ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Time for a ChangeLog! -## 1.15-SNAPSHOT +## 1.15 Moar syntax sugar -- ```clojure diff --git a/project.clj b/project.clj index 8e2e91d..50939fc 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.15-SNAPSHOT" +(defproject cnuernber/libpython-clj "1.15" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" From 3286631463271a16cde80809c5d1ad9cfa4c5b1d Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 3 Dec 2019 16:27:12 -0700 Subject: [PATCH 031/456] editing --- CHANGELOG.md | 4 ++++ README.md | 5 ++++- project.clj | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e735ec..d76ada8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Time for a ChangeLog! + +## 1.16-SNAPSHOT + + ## 1.15 Moar syntax sugar -- diff --git a/README.md b/README.md index ac9b8f5..e69b4de 100644 --- a/README.md +++ b/README.md @@ -282,7 +282,10 @@ user/linspace * `$c` - call an object mapping keywords to kwargs -#### Experimental Sugar (SNAPSHOT) +#### Experimental Sugar + +We are trying to find the best way to handle attributes in order to shorten +generic python notebook-type usage. The currently implemented direction is: * `$.` - get an attribute. Can pass in symbol, string, or keyword * `$..` - get an attribute. If more args are present, get the attribute on that diff --git a/project.clj b/project.clj index 50939fc..0e84e52 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.15" +(defproject cnuernber/libpython-clj "1.16-SNAPSHOT" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" From 66ef14ab5970c9eb14632dd60e509e8668101254 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Thu, 5 Dec 2019 07:26:09 -0700 Subject: [PATCH 032/456] small updates. --- example/project.clj | 2 +- example/src/keras_simple.clj | 2 +- src/libpython_clj/python.clj | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/example/project.clj b/example/project.clj index 4063c45..0464bad 100644 --- a/example/project.clj +++ b/example/project.clj @@ -4,4 +4,4 @@ :license {:name "EPL-2.0" :url "https://www.eclipse.org/legal/epl-2.0/"} :dependencies [[org.clojure/clojure "1.10.1"] - [cnuernber/libpython-clj "1.13"]]) + [cnuernber/libpython-clj "1.15"]]) diff --git a/example/src/keras_simple.clj b/example/src/keras_simple.clj index 52604b7..44a2236 100644 --- a/example/src/keras_simple.clj +++ b/example/src/keras_simple.clj @@ -84,7 +84,7 @@ (defn eval-model [model features lables] - (let [model-names (->> (get-attr model "metrics_names") + (let [model-names (->> ($. model metrics_names) (mapv keyword))] (->> (a$ model evaluate features labels) (map vector model-names) diff --git a/src/libpython_clj/python.clj b/src/libpython_clj/python.clj index 9521431..241605d 100644 --- a/src/libpython_clj/python.clj +++ b/src/libpython_clj/python.clj @@ -264,7 +264,7 @@ (when (and ptype pvalue ptraceback (not suppress-error?)) (do - ;;MAnuall incref here because we cannot detach the object + ;;Manual incref here because we cannot detach the object ;;from our gc decref hook added during earlier pyerr-fetch handler. (pyjna/Py_IncRef ptype) (pyjna/Py_IncRef pvalue) From 77687de777b640bf0d83b0bfb6d11328e47ac59c Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Thu, 5 Dec 2019 13:56:13 -0700 Subject: [PATCH 033/456] Added test for nontrivial function definition. --- testcode/__init__.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/testcode/__init__.py b/testcode/__init__.py index 8e0922e..d2fa00b 100644 --- a/testcode/__init__.py +++ b/testcode/__init__.py @@ -22,3 +22,17 @@ def for_iter(arg): def calling_custom_clojure_fn(arg): return arg.clojure_fn() + + + +def complex_fn(a, b, c: str=5, *args, d=10, **kwargs): + return {"a" : a, + "b" : b, + "c" : c, + "args" : args, + "d": d, + "kwargs": kwargs} + + +complex_fn_testcases = {"complex_fn(1, 2, c=10, d=10, e=10)":complex_fn(1, 2, c=10, d=10, e=10), + "complex_fn(1, 2, 10, 11, 12, d=10, e=10)":complex_fn(1, 2, 10, 11, 12, d=10, e=10)} From c3070c6f7096915d5f56c38a0d4b9c9072eb2d81 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Thu, 5 Dec 2019 14:13:57 -0700 Subject: [PATCH 034/456] generic testing of more complex fncall syntax. --- src/libpython_clj/python.clj | 16 ++++++------- src/libpython_clj/python/bridge.clj | 35 ++++++++++++++++++++++------- test/libpython_clj/fncall_test.clj | 29 ++++++++++++++++++++++++ test/libpython_clj/python_test.clj | 20 +++-------------- 4 files changed, 66 insertions(+), 34 deletions(-) create mode 100644 test/libpython_clj/fncall_test.clj diff --git a/src/libpython_clj/python.clj b/src/libpython_clj/python.clj index 241605d..4e97098 100644 --- a/src/libpython_clj/python.clj +++ b/src/libpython_clj/python.clj @@ -4,7 +4,7 @@ [libpython-clj.python.interpreter :as pyinterp :refer [with-interpreter]] [libpython-clj.python.object :as pyobj] - [libpython-clj.python.bridge] + [libpython-clj.python.bridge :as pybridge] [libpython-clj.jna :as pyjna] [tech.jna :as jna]) (:import [com.sun.jna Pointer] @@ -310,11 +310,9 @@ DEPRECATION POSSIBLE - use $a." [item attr & args] - (let [attr-name (if (symbol? attr) - (name attr) - attr) - [pos-args kw-args] (args->pos-kw-args args)] - `(call-attr-kw ~item ~attr-name ~pos-args ~kw-args))) + (let [[pos-args kw-args] (args->pos-kw-args args)] + `(call-attr-kw ~item ~(pybridge/key-sym-str->str attr) + ~pos-args ~kw-args))) (defmacro c$ @@ -351,17 +349,17 @@ (defmacro $. "Get the attribute of an object." [item attname] - `(get-attr ~item ~(name attname))) + `(get-attr ~item ~(pybridge/key-sym-str->str attname))) (defmacro $.. "Get the attribute of an object. If there are extra args, apply successive get-attribute calls to the arguments." [item attname & args] - `(-> (get-attr ~item ~(name attname)) + `(-> (get-attr ~item ~(pybridge/key-sym-str->str attname)) ~@(->> args (map (fn [arg] - `(get-attr ~(name arg))))))) + `(get-attr ~(pybridge/key-sym-str->str arg))))))) (defmacro import-as diff --git a/src/libpython_clj/python/bridge.clj b/src/libpython_clj/python/bridge.clj index 07a43fc..f7a531e 100644 --- a/src/libpython_clj/python/bridge.clj +++ b/src/libpython_clj/python/bridge.clj @@ -524,22 +524,27 @@ [arglist] (loop [args arglist pos-args [] - kw-args nil] + kw-args nil + found-kw? false] (if-not (seq args) [pos-args kw-args] (let [arg (first args) - [pos-args kw-args args] + [pos-args kw-args args found-kw?] (if (keyword? arg) (if-not (seq (rest args)) (throw (Exception. (format "Keyword arguments must be followed by another arg: %s" (str arglist)))) [pos-args (assoc kw-args arg (first (rest args))) - (drop 2 args)]) - [(conj pos-args (first args)) - kw-args - (rest args)])] - (recur args pos-args kw-args))))) + (drop 2 args) true]) + (if found-kw? + (throw (Exception. + (format "Positional arguments are not allowed after keyword arguments: %s" + arglist))) + [(conj pos-args (first args)) + kw-args + (rest args) found-kw?]))] + (recur args pos-args kw-args found-kw?))))) (defn cfn @@ -554,6 +559,19 @@ (py-proto/call-kw item pos-args kw-args))) +(defn key-sym-str->str + [attr-name] + (cond + (or (keyword? attr-name) + (symbol? attr-name)) + (name attr-name) + (string? attr-name) + attr-name + :else + (throw (Exception. + "Only keywords, symbols, or strings can be used to access attributes.")))) + + (defn afn "Call an attribute of an object. Arguments are passed in positionally. Any keyword @@ -563,7 +581,8 @@ Not having an argument after a keyword argument is an error." [item attr & args] (let [[pos-args kw-args] (args->pos-kw-args args)] - (py-proto/call-attr-kw item attr pos-args kw-args))) + (py-proto/call-attr-kw item (key-sym-str->str attr) + pos-args kw-args))) (defn generic-python-as-jvm diff --git a/test/libpython_clj/fncall_test.clj b/test/libpython_clj/fncall_test.clj new file mode 100644 index 0000000..c2dbacc --- /dev/null +++ b/test/libpython_clj/fncall_test.clj @@ -0,0 +1,29 @@ +(ns libpython-clj.fncall-test + (:require [libpython-clj.python :as py] + [clojure.test :refer :all])) + + +(py/initialize!) + + + +(deftest complex-fn-test + (let [testmod (py/import-module "testcode") + testcases (py/$. testmod complex_fn_testcases)] + (is (= (-> (get testcases "complex_fn(1, 2, c=10, d=10, e=10)") + (py/->jvm)) + (-> (py/$a testmod complex_fn 1 2 :c 10 :d 10 :e 10) + (py/->jvm)))) + (is (= (-> (get testcases "complex_fn(1, 2, 10, 11, 12, d=10, e=10)") + (py/->jvm)) + (-> (py/$a testmod complex_fn 1 2 10 11 12 :d 10 :e 10) + (py/->jvm)))) + + (is (= (-> (get testcases "complex_fn(1, 2, c=10, d=10, e=10)") + (py/->jvm)) + (-> (apply py/afn testmod "complex_fn" [1 2 :c 10 :d 10 :e 10]) + (py/->jvm)))) + (is (= (-> (get testcases "complex_fn(1, 2, 10, 11, 12, d=10, e=10)") + (py/->jvm)) + (-> (apply py/afn testmod "complex_fn" [1 2 10 11 12 :d 10 :e 10]) + (py/->jvm)))))) diff --git a/test/libpython_clj/python_test.clj b/test/libpython_clj/python_test.clj index 66c8b95..e150733 100644 --- a/test/libpython_clj/python_test.clj +++ b/test/libpython_clj/python_test.clj @@ -7,9 +7,10 @@ (:import [java.io StringWriter] [java.util Map List])) + (py/initialize!) + (deftest stdout-and-stderr - (py/initialize!) (is (= "hey\n" (with-out-str (py/run-simple-string "print('hey')")))) (is (= "hey\n" (let [custom-writer (StringWriter.)] @@ -21,7 +22,6 @@ (deftest dicts - (py/initialize!) (let [py-dict (py/->python {:a 1 :b 2})] (is (= :dict (py/python-type py-dict))) (is (= 2 (-> (py/call-attr py-dict "__len__") @@ -40,7 +40,6 @@ (deftest lists - (py/initialize!) (let [py-list (py/->py-list [4 3 2 1])] (is (= :list (py/python-type py-list))) (is (= 4 (-> (py/call-attr py-list "__len__") @@ -58,7 +57,6 @@ (deftest global-dict - (py/initialize!) (let [main-module (py/add-module "__main__") ^Map globals (-> (py/module-dict main-module) (py/as-jvm))] @@ -72,7 +70,6 @@ (deftest numpy-and-back - (py/initialize!) (let [jvm-tens (dtt/->tensor (->> (range 9) (partition 3)))] ;;zero-copy can't work on jvm datastructures with current JNA tech. @@ -109,7 +106,6 @@ dtt/->jvm))))))) (deftest numpy-scalars - (py/initialize!) (let [np (py/import-module "numpy") scalar-constructors (concat ["float64" "float32"] @@ -123,7 +119,6 @@ (deftest dict-with-complex-key - (py/initialize!) (let [py-dict (py/->python {["a" "b"] 1 ["c" "d"] 2}) bridged (py/as-jvm py-dict)] @@ -136,20 +131,17 @@ (deftest simple-print-crashed - (py/initialize!) (let [numpy (py/import-module "numpy")] (println (py/as-tensor (py/call-attr numpy "ones" [3 3]))))) (deftest true-false-list - (py/initialize!) (is (= [false true] (-> '(false true) py/->py-list py/->jvm)))) (deftest true-false-true-numpy - (py/initialize!) (let [numpy (py/import-module "numpy")] (is (= [true false true] (->> (for [a (py/call-attr numpy "array" [true false true])] @@ -158,7 +150,6 @@ (deftest aspy-iter - (py/initialize!) (let [testcode-module (py/import-module "testcode")] (is (= [1 2 3 4 5] (-> (py/call-attr testcode-module @@ -171,7 +162,6 @@ (deftest basic-with-test - (py/initialize!) (let [testcode-module (py/import-module "testcode")] (let [fn-list (py/->py-list [])] (is (nil? @@ -201,13 +191,11 @@ (deftest arrow-as-fns-with-nil - (py/initialize!) (is (= nil (py/->jvm nil))) (is (= nil (py/as-jvm nil)))) (deftest pydict-nil-get - (py/initialize!) (let [dict (py/->python {:a 1 :b {:a 1 :b 2}}) bridged (py/as-jvm dict)] (is (= nil (bridged nil))))) @@ -252,7 +240,6 @@ (deftest bridged-dict-to-jvm - (py/initialize!) (let [py-dict (py/->py-dict {:a 1 :b 2}) bridged (py/as-jvm py-dict) copied-back (py/->jvm bridged)] @@ -260,9 +247,8 @@ (deftest calling-conventions - (py/initialize!) (let [np (py/import-module "numpy") - linspace (py/get-attr np "linspace")] + linspace (py/$. np linspace)] (is (dfn/equals [2.000 2.250 2.500 2.750 3.000] (py/as-tensor (linspace 2 3 :num 5)))) From 9407189bc3c31ff41db19a62f9dd736af37a9500 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Fri, 6 Dec 2019 09:38:23 -0700 Subject: [PATCH 035/456] 1.16 --- project.clj | 2 +- src/libpython_clj/python/interpreter.clj | 9 ++++----- test/libpython_clj/fncall_test.clj | 2 ++ 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/project.clj b/project.clj index 0e84e52..785453e 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.16-SNAPSHOT" +(defproject cnuernber/libpython-clj "1.16" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" diff --git a/src/libpython_clj/python/interpreter.clj b/src/libpython_clj/python/interpreter.clj index 5c08b30..e446b29 100644 --- a/src/libpython_clj/python/interpreter.clj +++ b/src/libpython_clj/python/interpreter.clj @@ -253,11 +253,10 @@ (let [user-names (when python-library-name [python-library-name]) library-names (or user-names (libpy-base/library-names))] - (when-not (->> library-names - (filter try-load-python-library!) - first) - (throw (Exception. (format "Failed to initialize python library. -Attempted libraries %s" library-names))))) + (loop [[library-name & library-names] library-names] + (if (and library-name + (not (try-load-python-library! library-name))) + (recur library-names)))) ;;Set program name (when-let [program-name (or program-name *program-name* "")] (resource/stack-resource-context diff --git a/test/libpython_clj/fncall_test.clj b/test/libpython_clj/fncall_test.clj index c2dbacc..aa43dfd 100644 --- a/test/libpython_clj/fncall_test.clj +++ b/test/libpython_clj/fncall_test.clj @@ -6,6 +6,8 @@ (py/initialize!) +(py/import-as testcode testmod) +(py/from-import inspect getfullargspec) (deftest complex-fn-test (let [testmod (py/import-module "testcode") From dd4fa650e7998c005d64ed1f51ff503fbcedccb4 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Fri, 6 Dec 2019 09:40:37 -0700 Subject: [PATCH 036/456] Updating changelog. --- CHANGELOG.md | 10 +++++++++- project.clj | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d76ada8..25e9561 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,15 @@ # Time for a ChangeLog! -## 1.16-SNAPSHOT +## 1.16 + +Fixed a bug where the system would load multiple python libraries, not stopping +after the first valid library loaded. There are two ways to control the system's +python library loading mechanism: + +1. Pass in a library name in initialize! +2. alter-var-root the list of libraries in libpython-clj.jna.base before + calling initialize!. ## 1.15 diff --git a/project.clj b/project.clj index 785453e..b106221 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.16" +(defproject cnuernber/libpython-clj "1.17-SNAPSHOT" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" From 4f264eb9055a03778b2e88918a32952f245ac613 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sat, 7 Dec 2019 17:07:29 -0700 Subject: [PATCH 037/456] First crack at creating custom classes. --- src/libpython_clj/jna.clj | 3 +- src/libpython_clj/jna/concrete/cfunction.clj | 17 +++++++ src/libpython_clj/jna/concrete/type.clj | 4 +- src/libpython_clj/python.clj | 52 +++++++++++++++++++- src/libpython_clj/python/interop.clj | 14 ++++-- src/libpython_clj/python/protocols.clj | 3 ++ test/libpython_clj/classes_test.clj | 41 +++++++++++++++ test/libpython_clj/stress_test.clj | 46 ++++++++++++++--- 8 files changed, 167 insertions(+), 13 deletions(-) create mode 100644 test/libpython_clj/classes_test.clj diff --git a/src/libpython_clj/jna.clj b/src/libpython_clj/jna.clj index 906de77..a1f4bde 100644 --- a/src/libpython_clj/jna.clj +++ b/src/libpython_clj/jna.clj @@ -272,7 +272,8 @@ METH_O METH_STATIC METH_VARARGS - PyCFunction_New) + PyCFunction_New + PyInstanceMethod_New) (export-symbols libpython-clj.jna.concrete.import diff --git a/src/libpython_clj/jna/concrete/cfunction.clj b/src/libpython_clj/jna/concrete/cfunction.clj index 38e534f..d078a9e 100644 --- a/src/libpython_clj/jna/concrete/cfunction.clj +++ b/src/libpython_clj/jna/concrete/cfunction.clj @@ -55,3 +55,20 @@ Pointer [method-def (partial jna/ensure-type PyMethodDef)] [self jna/as-ptr]) + + +(def-pylib-fn PyInstanceMethod_New + "Return value: New reference. + + Return a new instance method object, with func being any callable object func is the + function that will be called when the instance method is called." + Pointer + [func ensure-pyobj]) + + +(def-pylib-fn PyInstanceMethod_Function + "Return value: Borrowed reference. + + Return the function object associated with the instance method im." + Pointer + [im ensure-pyobj]) diff --git a/src/libpython_clj/jna/concrete/type.clj b/src/libpython_clj/jna/concrete/type.clj index d043859..323a906 100644 --- a/src/libpython_clj/jna/concrete/type.clj +++ b/src/libpython_clj/jna/concrete/type.clj @@ -110,8 +110,8 @@ type’s tp_alloc slot." Pointer [type (partial jna/ensure-type PyTypeObject)] - [args ensure-pyobj] - [kwds ensure-pyobj]) + [args #(when % (ensure-pyobj %))] + [kwds #(when % (ensure-pyobj %))]) (def-pylib-fn PyType_Ready diff --git a/src/libpython_clj/python.clj b/src/libpython_clj/python.clj index 4e97098..3ff0970 100644 --- a/src/libpython_clj/python.clj +++ b/src/libpython_clj/python.clj @@ -10,7 +10,10 @@ (:import [com.sun.jna Pointer] [com.sun.jna.ptr PointerByReference] [java.io Writer] - [libpython_clj.jna PyObject])) + [libpython_clj.jna PyObject + CFunction$KeyWordFunction + CFunction$TupleFunction + CFunction$NoArgFunction])) (set! *warn-on-reflection* true) @@ -111,17 +114,64 @@ (defn ->py-tuple + "Create a python tuple" [item] (-> (pyobj/->py-tuple item) (as-jvm))) (defn ->py-fn + "Make a python function. If clojure function is passed in the arguments are + marshalled from python to clojure, the function called, and the return value will be + marshalled back." [item] (-> (pyobj/->py-fn item) (as-jvm))) +(defn make-tuple-fn + "Reify the appropriate interface for tuple functions and return a new python cfunction + object. Args are exposed as a python tuple to the embedded code. This is a low level + function and you may never need it." + [clj-fn & [options]] + (-> (reify CFunction$TupleFunction + (pyinvoke [_ self args] + (let [retval + (apply clj-fn (as-jvm args))] + (pyobj/incref (->python retval))))) + (pyobj/->py-fn options))) + +(declare import-module) + + +(def ^:private builtins (delay (import-module "builtins"))) + + +(defn create-class + "Create a new class object. Any callable values in the cls-hashmap + will be presented as instance methods. + Things in the cls hashmap had better be either atoms or already converted + python objects. You may get surprised otherwise; you have been warned. + See the classes-test file in test/libpython-clj" + [name bases cls-hashmap] + (with-gil + (let [cls-hashmap (->> cls-hashmap + (map (fn [[k v]] + [k (if (callable? v) + (-> (pyjna/PyInstanceMethod_New v) + (pyobj/wrap-pyobject)) + v)]))) + cls-dict (reduce (fn [cls-dict [k v]] + (set-item! cls-dict k (->python v)) + cls-dict) + (->py-dict {}) + cls-hashmap) + bases (->py-tuple bases) + builtins @builtins + new-cls (call-attr builtins "type" name bases cls-dict)] + new-cls))) + + (defn run-simple-string "Run a string expression returning a map of {:globals :locals :result}. diff --git a/src/libpython_clj/python/interop.clj b/src/libpython_clj/python/interop.clj index ea80a1b..221d5ce 100644 --- a/src/libpython_clj/python/interop.clj +++ b/src/libpython_clj/python/interop.clj @@ -198,6 +198,7 @@ method-definitions tp_flags ;;may be nil tp_basicsize ;;size of binary type + tp_init ;;init fn, may be nil tp_new ;;may be nil, will use generic tp_dealloc ;;may be nil, will use generic tp_getattr ;;may *not* be nil @@ -213,12 +214,15 @@ (let [tp_new (or tp_new (reify CFunction$tp_new (pyinvoke [this self varargs kw_args] - (libpy/PyType_GenericNew self varargs kw_args)))) + (let [retval (libpy/PyType_GenericNew self varargs kw_args)] + (println retval) + retval) + ))) module-name (get-attr module "__name__") ;;These get leaked. Really, types are global objects that cannot be released. ;;Until we can destroy interpreters, it isn't worth the effort to track the ;;type and memory related to the type. - docstring-ptr (jna/string->ptr-untracked docstring) + docstring-ptr (when docstring (jna/string->ptr-untracked docstring)) type-name-ptr (jna/string->ptr-untracked (str module-name "." type-name)) tp_flags (long (or tp_flags (bit-or @libpy/Py_TPFLAGS_DEFAULT @@ -229,6 +233,7 @@ _ (nio-buf/memset new-mem 0 type-obj-size) new-type (PyTypeObject. new-mem)] (set! (.tp_name new-type) type-name-ptr) + (set! (.tp_init new-type) tp_init) (set! (.tp_doc new-type) docstring-ptr) (set! (.tp_basicsize new-type) tp_basicsize) (set! (.tp_flags new-type) tp_flags) @@ -255,9 +260,12 @@ :tp_name type-name-ptr :tp_doc docstring-ptr :tp_new tp_new + :tp_init tp_init :tp_dealloc tp_dealloc :tp_getattr tp_getattr - :tp_setattr tp_setattr)) + :tp_setattr tp_setattr + :tp_iter tp_iter + :tp_iternext tp_iternext)) (libpy/Py_IncRef new-type) new-type) (throw (ex-info (format "Type failed to register: %d" type-ready) diff --git a/src/libpython_clj/python/protocols.clj b/src/libpython_clj/python/protocols.clj index 415f515..b4b72ab 100644 --- a/src/libpython_clj/python/protocols.clj +++ b/src/libpython_clj/python/protocols.clj @@ -88,6 +88,9 @@ are converted into a {:type :pyobject-address} pairs.")) (extend-type Object + PPyObject + (callable? [_] false) + (has-attr? [_ _] false) PPyAttMap (att-type-map [item] (->> (dir item) diff --git a/test/libpython_clj/classes_test.clj b/test/libpython_clj/classes_test.clj new file mode 100644 index 0000000..d80fb5f --- /dev/null +++ b/test/libpython_clj/classes_test.clj @@ -0,0 +1,41 @@ +(ns libpython-clj.classes-test + (:require [libpython-clj.python :as py] + [libpython-clj.jna :as pylib] + [clojure.test :refer :all] + [clojure.pprint :as pp] + [clojure.edn :as edn])) + + + +(py/initialize!) + +(deftest new-cls-test + (let [cls-obj (py/create-class + "Stock" nil + ;;Using as-py-fn instead of ->py-fn to avoid any marshalling. + ;;What happens is the self object gets marshalled to a + ;;persistent map. This isn't what we want. + {"__init__" (py/make-tuple-fn + (fn [self name shares price] + (py/set-attr! self "name" name) + (py/set-attr! self "shares" shares) + (py/set-attr! self "price" price) + nil)) + "cost" (py/make-tuple-fn + (fn [self] + (* (py/$. self shares) + (py/$. self price)))) + "__str__" (py/make-tuple-fn + (fn [self] + ;;Self is just a dict so it converts to a hashmap + (pr-str {"name" (py/$. self name) + "shares" (py/$. self shares) + "price" (py/$. self price)}))) + "clsattr" 55}) + new-instance (cls-obj "ACME" 50 90)] + (is (= 4500 + (py/$a new-instance cost))) + (is (= 55 (py/$. new-instance clsattr))) + + (is (= {"name" "ACME", "shares" 50, "price" 90} + (edn/read-string (.toString new-instance)))))) diff --git a/test/libpython_clj/stress_test.clj b/test/libpython_clj/stress_test.clj index 88a5e32..939d2d0 100644 --- a/test/libpython_clj/stress_test.clj +++ b/test/libpython_clj/stress_test.clj @@ -3,7 +3,12 @@ memory if it isn't setup correctly." (:require [libpython-clj.python :as py] [libpython-clj.jna :as libpy] - [libpython-clj.python.object :as pyobject])) + [libpython-clj.python.object :as pyobject] + [clojure.test :refer :all] + [clojure.edn :as edn])) + + +(py/initialize!) (def test-script @@ -64,7 +69,6 @@ def getmultidata(): (defn forever-test [] - (py/initialize!) (doseq [items (partition 999 (get-data))] ;;One way is to use the GC (time @@ -88,7 +92,6 @@ def getmultidata(): (defn multidata-test [] - (py/initialize!) (doseq [items (partition 1000 (get-multi-data))] (time (do (py/with-gil-stack-rc-context (mapv py/->jvm items)) @@ -97,7 +100,6 @@ def getmultidata(): (defn str-marshal-test [] - (py/initialize!) (let [test-str (py/->python "a nice string to work with")] (time (py/with-gil-stack-rc-context @@ -107,7 +109,6 @@ def getmultidata(): (defn dict-marshal-test [] - (py/initialize!) (let [test-item (py/->python {:a 1 :b 2})] (time (py/with-gil-stack-rc-context @@ -117,7 +118,40 @@ def getmultidata(): (defn print-stress-test [] - (py/initialize!) (dotimes [iter 10000] (with-out-str (print-data)))) + + +(defn new-cls-stress-test + [] + (dotimes [iter 1000] + (py/with-gil-stack-rc-context + (let [test-cls (py/create-class "testcls" nil + {"__init__" (py/make-tuple-fn + (fn [self name shares price] + (py/set-attr! self "name" name) + (py/set-attr! self "shares" shares) + (py/set-attr! self "price" price) + nil)) + "cost" (py/make-tuple-fn + (fn [self] + (* (py/$. self shares) + (py/$. self price)))) + "__str__" (py/make-tuple-fn + (fn [self] + ;;Self is just a dict so it converts to a hashmap + (pr-str {"name" (py/$. self name) + "shares" (py/$. self shares) + "price" (py/$. self price)}))) + "testvar" 55} + ) + new-inst (test-cls "ACME" 50 90)] + (is (= 4500 + (py/$a new-inst cost))) + (is (= 55 (py/$. new-inst testvar))) + + (is (= {"name" "ACME", "shares" 50, "price" 90} + (edn/read-string (.toString new-inst)))) + + )))) From da33cc4123af3cbc75a6beee12585970c32e9928 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 8 Dec 2019 09:14:10 -0700 Subject: [PATCH 038/456] Refactored pathway for generating classes dynamically. --- src/libpython_clj/jna.clj | 19 ++- src/libpython_clj/python.clj | 68 ++++----- src/libpython_clj/python/object.clj | 215 +++++++++++++++++++++------- test/libpython_clj/classes_test.clj | 45 +++--- 4 files changed, 236 insertions(+), 111 deletions(-) diff --git a/src/libpython_clj/jna.clj b/src/libpython_clj/jna.clj index a1f4bde..04b42ec 100644 --- a/src/libpython_clj/jna.clj +++ b/src/libpython_clj/jna.clj @@ -7,9 +7,10 @@ [tech.parallel.utils :refer [export-symbols]] [libpython-clj.jna.base :refer [def-pylib-fn - ensure-pyobj]] + ensure-pyobj] + :as libpy-base] [libpython-clj.jna.base] - [libpython-clj.jna.interpreter] + [libpython-clj.jna.interpreter :as jnainterp] [libpython-clj.jna.protocols.object] [libpython-clj.jna.protocols.iterator] [libpython-clj.jna.protocols.sequence] @@ -55,6 +56,20 @@ PyRun_StringFlags) +(defmacro ^:private export-type-symbols + [] + `(do + ~@(->> (vals jnainterp/type-symbol-table) + (map (fn [sym-name] + `(defn ~(symbol sym-name) + ~(format "%s type object" sym-name) + [] + (libpy-base/find-pylib-symbol ~sym-name))))))) + + +(export-type-symbols) + + (export-symbols libpython-clj.jna.protocols.object Py_DecRef Py_IncRef diff --git a/src/libpython_clj/python.clj b/src/libpython_clj/python.clj index 3ff0970..a3a6864 100644 --- a/src/libpython_clj/python.clj +++ b/src/libpython_clj/python.clj @@ -45,7 +45,10 @@ ->py-long ->py-string ->python - ->jvm) + ->jvm + make-tuple-fn + make-tuple-instance-fn + create-class) (defmacro stack-resource-context @@ -84,6 +87,26 @@ ~@body))) +(defn set-attrs! + "Set a sequence of [name value] attributes. + Returns item" + [item att-seq] + (with-gil + (doseq [[k v] att-seq] + (set-attr! item k v))) + item) + + +(defn set-items! + "Set a sequence of [name value]. + Returns item" + [item item-seq] + (with-gil + (doseq [[k v] item-seq] + (set-item! item k v))) + item) + + (export-symbols libpython-clj.python.interop libpython-clj-module-name create-bridge-from-att-map) @@ -129,49 +152,6 @@ (as-jvm))) -(defn make-tuple-fn - "Reify the appropriate interface for tuple functions and return a new python cfunction - object. Args are exposed as a python tuple to the embedded code. This is a low level - function and you may never need it." - [clj-fn & [options]] - (-> (reify CFunction$TupleFunction - (pyinvoke [_ self args] - (let [retval - (apply clj-fn (as-jvm args))] - (pyobj/incref (->python retval))))) - (pyobj/->py-fn options))) - -(declare import-module) - - -(def ^:private builtins (delay (import-module "builtins"))) - - -(defn create-class - "Create a new class object. Any callable values in the cls-hashmap - will be presented as instance methods. - Things in the cls hashmap had better be either atoms or already converted - python objects. You may get surprised otherwise; you have been warned. - See the classes-test file in test/libpython-clj" - [name bases cls-hashmap] - (with-gil - (let [cls-hashmap (->> cls-hashmap - (map (fn [[k v]] - [k (if (callable? v) - (-> (pyjna/PyInstanceMethod_New v) - (pyobj/wrap-pyobject)) - v)]))) - cls-dict (reduce (fn [cls-dict [k v]] - (set-item! cls-dict k (->python v)) - cls-dict) - (->py-dict {}) - cls-hashmap) - bases (->py-tuple bases) - builtins @builtins - new-cls (call-attr builtins "type" name bases cls-dict)] - new-cls))) - - (defn run-simple-string "Run a string expression returning a map of {:globals :locals :result}. diff --git a/src/libpython_clj/python/object.clj b/src/libpython_clj/python/object.clj index f503e4c..c15f465 100644 --- a/src/libpython_clj/python/object.clj +++ b/src/libpython_clj/python/object.clj @@ -32,8 +32,6 @@ :refer [pyobject->jvm python-type] :as py-proto] - [libpython-clj.python.logging - :refer [log-error log-warn log-info]] [libpython-clj.jna.base :as libpy-base] [libpython-clj.jna :as libpy] [clojure.stacktrace :as st] @@ -42,7 +40,8 @@ [tech.v2.datatype :as dtype] [tech.v2.datatype.protocols :as dtype-proto] [tech.v2.datatype.casting :as casting] - [tech.v2.tensor]) + [tech.v2.tensor] + [clojure.tools.logging :as log]) (:import [com.sun.jna Pointer CallbackReference] [com.sun.jna.ptr PointerByReference] [libpython_clj.jna @@ -52,7 +51,7 @@ CFunction$NoArgFunction PyMethodDef] [java.nio.charset StandardCharsets] - [tech.v2.datatype ObjectIter] + [tech.v2.datatype ObjectIter ObjectReader] [tech.v2.datatype.typed_buffer TypedBuffer] [tech.v2.tensor.protocols PTensor] [java.util RandomAccess Map Set Map$Entry] @@ -146,7 +145,7 @@ (dec (or arg 0))))) (libpy/Py_DecRef (Pointer. pyobj-value)) (catch Throwable e - (log-error "Exception while releasing object: %s" e)))) + (log/error e "Exception while releasing object")))) *pyobject-tracking-flags*)) (do ;;Special handling for PyNone types @@ -370,25 +369,11 @@ (def ^:dynamic *item-tuple-cutoff* 8) -(defn wrap-clojure-fn - [fn-obj] - (when-not (fn? fn-obj) - (throw (ex-info "This is not a function." {}))) - (reify CFunction$TupleFunction - (pyinvoke [this self args] - (try - (let [retval - (apply fn-obj (->jvm args))] - (if (nil? retval) - (incref (libpy/Py_None)) - (->python retval))) - (catch Throwable e - (log-error (format "%s:%s" e (with-out-str - (st/print-stack-trace e)))) - (libpy/PyErr_SetString (libpy/PyExc_Exception) - (format "%s:%s" e (with-out-str - (st/print-stack-trace e)))) - nil))))) +(defn- cfunc-instance? + [function] + (or (instance? CFunction$KeyWordFunction function) + (instance? CFunction$TupleFunction function) + (instance? CFunction$NoArgFunction function))) (defn apply-method-def-data! @@ -397,34 +382,34 @@ function] :as method-data}] (resource/stack-resource-context - (let [callback (if (or (instance? CFunction$KeyWordFunction function) - (instance? CFunction$TupleFunction function) - (instance? CFunction$NoArgFunction function)) - function - (wrap-clojure-fn function)) - meth-flags (long (cond - (instance? CFunction$NoArgFunction callback) + (when-not (cfunc-instance? function) + (throw (Exception. + (format "Callbacks must implement one of the CFunction interfaces: +%s" (type function))))) + (let [meth-flags (long (cond + (instance? CFunction$NoArgFunction function) @libpy/METH_NOARGS - (instance? CFunction$TupleFunction callback) + (instance? CFunction$TupleFunction function) @libpy/METH_VARARGS - (instance? CFunction$KeyWordFunction callback) + (instance? CFunction$KeyWordFunction function) (bit-or @libpy/METH_KEYWORDS @libpy/METH_VARARGS) :else (throw (ex-info (format "Failed due to type: %s" - (type callback)))))) + (type function)) + {})))) name-ptr (jna/string->ptr name) doc-ptr (jna/string->ptr doc)] (set! (.ml_name method-def) name-ptr) - (set! (.ml_meth method-def) (CallbackReference/getFunctionPointer callback)) + (set! (.ml_meth method-def) (CallbackReference/getFunctionPointer function)) (set! (.ml_flags method-def) (int meth-flags)) (set! (.ml_doc method-def) doc-ptr) (.write method-def) (pyinterp/conj-forever! (assoc method-data :name-ptr name-ptr :doc-ptr doc-ptr - :callback-object callback + :callback-object function :method-definition method-def)) method-def))) @@ -434,29 +419,161 @@ (apply-method-def-data! (PyMethodDef.) method-data)) + +(defn- cfunc-impl->pyobject + [cfunc {:keys [method-name + documentation + py-self] + :or {method-name "unnamed_function" + documentation "not documented"}}] + (with-gil + (let [py-self (or py-self (->python {}))] + (-> (libpy/PyCFunction_New (method-def-data->method-def + {:name method-name + :doc documentation + :function cfunc}) + ;;This is a nice little tidbit, cfunction_new + ;;steals the reference. + (libpy/Py_IncRef py-self)) + (wrap-pyobject))))) + + +(defn py-tuple->borrowed-reference-reader + "Given a python tuple, return an object reader that iterates + through the items. If you hold onto one of these items you need + to add to it's reference count. If unsure of what you are doing + don't use this method, use ->jvm." + [tuple] + (with-gil + (let [interpreter (ensure-bound-interpreter) + n-items (long (libpy/PyObject_Length tuple))] + (reify ObjectReader + (lsize [_] n-items) + (read [_ idx] + (with-interpreter interpreter + (libpy/PyTuple_GetItem tuple idx))))))) + + +(defn ->python-incref + "Convert to python and add a reference. This is necessary for return values from + functions as the ->python pathway adds a reference but it also tracks it and + releases it when it is not in use any more. Thus python ends up holding onto + something with fewer refcounts than it should have. If you are just messing + around in the repl you only need ->python. There is an expectation that the + return value of a function call is a new reference and not a borrowed reference + hence this pathway." + [item] + (-> (->python item) + (incref))) + + + +(defn make-tuple-fn + "Given a clojure function, create a python tuple function. + arg-convert is applied to arguments before the clojure function + gets them and result-converter is applied to the outbound result. + Exceptions are caught, logged, and propagated to python. + + arg-converter: A function to be called on arguments before they get to + clojure. Defaults to ->jvm. + result-converter: A function to be called on the return value before it + makes it back to python. Defaults to ->python-incref. + method-name: Name of function exposed to python. + documentation: Documentation of function exposed to python." + [fn-obj & {:keys [arg-converter + result-converter] + :or {arg-converter ->jvm + result-converter ->python-incref} + :as options}] + (with-gil + (-> (reify CFunction$TupleFunction + (pyinvoke [this self args] + (try + (let [argseq (cond->> (py-tuple->borrowed-reference-reader args) + arg-converter + (map arg-converter))] + (cond-> (apply fn-obj argseq) + result-converter + (result-converter))) + (catch Throwable e + (log/error e "Error executing clojure function.") + (libpy/PyErr_SetString (libpy/PyExc_Exception) + (format "%s:%s" e (with-out-str + (st/print-stack-trace e)))) + nil)))) + (cfunc-impl->pyobject options)))) + + (defn ->py-fn "Create a python callback from a clojure fn. If clojure fn, then tuple arguments are used. If keyword arguments are desired, the pass in something derived from: libpython-clj.jna.CFunction$KeyWordFunction. If a pure fn is passed in, arguments are marshalled from python if possible and then to-python in the case of successful execution. An exception will set the error - indicator." - ([fn-obj {:keys [method-name documentation py-self] - :or {method-name "unnamed_function" - documentation "not documented"}}] - (with-gil - (let [py-self (or py-self (->python {}))] - (wrap-pyobject (libpy/PyCFunction_New (method-def-data->method-def - {:name method-name - :doc documentation - :function fn-obj}) - ;;This is a nice little tidbit, cfunction_new - ;;steals the reference. - (libpy/Py_IncRef py-self)))))) + indicator. + Options are + method-name: Name of function exposed to python. + documentation: Documentation of function exposed to python. + py-self: The 'self' object to be used for the function." + ([fn-obj {:keys [] + :as options}] + (cond + (instance? clojure.lang.IFn fn-obj) + (apply make-tuple-fn fn-obj (apply concat options)) + (cfunc-instance? fn-obj) + (cfunc-impl->pyobject fn-obj options) + :else + (throw (Exception. "fn-obj is neither a CFunction nor clojure callable.")))) ([fn-obj] (->py-fn fn-obj {}))) +(defn py-fn->instance-fn + "Given a python callable, return an instance function meant to be used + in class definitions." + [py-fn] + (with-gil + (-> (libpy/PyInstanceMethod_New py-fn) + (wrap-pyobject)))) + + +(defn make-tuple-instance-fn + "Make an instance function. In this case the default behavior is to + pass raw python object ptr args to the clojure function without marshalling + as that can add confusion and unnecessary overhead. Self will be the first argument. + Callers can change this behavior by setting the 'arg-converter' option as in + 'make-tuple-fn'. + Options are the same as make-tuple-fn." + [clj-fn & {:keys [arg-converter] + :as options}] + (with-gil + (-> (apply make-tuple-fn + clj-fn + ;;Explicity set arg-convert to override make-tuple-fn's default + ;;->jvm arg-converter. + (->> (assoc options :arg-converter arg-converter) + (apply concat))) + ;;Mark this as an instance function. + (py-fn->instance-fn)))) + + +(defn create-class + "Create a new class object. Any callable values in the cls-hashmap + will be presented as instance methods. + Things in the cls hashmap had better be either atoms or already converted + python objects. You may get surprised otherwise; you have been warned. + See the classes-test file in test/libpython-clj" + [name bases cls-hashmap] + (with-gil + (let [cls-dict (reduce (fn [cls-dict [k v]] + (py-proto/set-item! cls-dict k (->python v)) + cls-dict) + (->py-dict {}) + cls-hashmap) + bases (->py-tuple bases) + new-cls (py-proto/call (libpy/PyType_Type) name bases cls-dict)] + (py-proto/as-jvm new-cls nil)))) + (extend-protocol py-proto/PCopyToPython Number diff --git a/test/libpython_clj/classes_test.clj b/test/libpython_clj/classes_test.clj index d80fb5f..e68d7cc 100644 --- a/test/libpython_clj/classes_test.clj +++ b/test/libpython_clj/classes_test.clj @@ -6,36 +6,49 @@ [clojure.edn :as edn])) - (py/initialize!) + (deftest new-cls-test + ;;The crux of this is making instance functions. The make-tuple-instance-fn pathway + ;;is pretty close to the metal and as such it does no marshalling of parameters by + ;;default. You can turn on marshalling of all parameters adding `arg-converter` + ;;optional argument (this may make your life easier) or you can marshal just the + ;;parameter you have to (self in the examples below). (let [cls-obj (py/create-class "Stock" nil - ;;Using as-py-fn instead of ->py-fn to avoid any marshalling. - ;;What happens is the self object gets marshalled to a - ;;persistent map. This isn't what we want. - {"__init__" (py/make-tuple-fn + {"__init__" (py/make-tuple-instance-fn (fn [self name shares price] - (py/set-attr! self "name" name) - (py/set-attr! self "shares" shares) - (py/set-attr! self "price" price) + ;;Because we did not use an arg-converter, all the + ;;arguments above are raw jna Pointers - borrowed + ;;references. + (py/set-attrs! self {"name" name + "shares" shares + "price" price}) + ;;If you don't return nil from __init__ that is an + ;;error. nil)) - "cost" (py/make-tuple-fn + "cost" (py/make-tuple-instance-fn (fn [self] (* (py/$. self shares) - (py/$. self price)))) - "__str__" (py/make-tuple-fn + (py/$. self price))) + + ;;Convert self to something that auto-marshals things. + ;;This pathway will autoconvert all arguments to the function. + :arg-converter py/as-jvm + :method-name "cost") + "__str__" (py/make-tuple-instance-fn (fn [self] - ;;Self is just a dict so it converts to a hashmap - (pr-str {"name" (py/$. self name) - "shares" (py/$. self shares) - "price" (py/$. self price)}))) + ;;Alternative to using arg-converter. This way you can + ;;explicitly control which arguments are converted. + (let [self (py/as-jvm self)] + (pr-str {"name" (py/$. self name) + "shares" (py/$. self shares) + "price" (py/$. self price)})))) "clsattr" 55}) new-instance (cls-obj "ACME" 50 90)] (is (= 4500 (py/$a new-instance cost))) (is (= 55 (py/$. new-instance clsattr))) - (is (= {"name" "ACME", "shares" 50, "price" 90} (edn/read-string (.toString new-instance)))))) From 72043c8c9c1fa90a570b24095f61dae52e0aaf92 Mon Sep 17 00:00:00 2001 From: "J.J. Tolton" Date: Sun, 8 Dec 2019 11:21:56 -0500 Subject: [PATCH 039/456] Add alpha framework for require-python feature (#12) * Add alpha framework for require-python feature * more docstring * less docstring * less docstring * remove trace statements * remove unused code; formatting * spacing * relevant examples * add developer note * typos * note formatting * fix documentation typo * fix load-py-fn invocation in load-python-lib --- src/libpython_clj/require.clj | 253 ++++++++++++++++++++++++++++++++++ 1 file changed, 253 insertions(+) create mode 100644 src/libpython_clj/require.clj diff --git a/src/libpython_clj/require.clj b/src/libpython_clj/require.clj new file mode 100644 index 0000000..6d46246 --- /dev/null +++ b/src/libpython_clj/require.clj @@ -0,0 +1,253 @@ +(ns libpython-clj.require + (:refer-clojure :exclude [fn? doc]) + (:require [libpython-clj.python :as py])) + +(def ^:private builtins (py/import-module "builtins")) + +(def ^:private inspect (py/import-module "inspect")) + +(def ^:private argspec (py/get-attr inspect "getfullargspec")) + +(def ^:private py-source (py/get-attr inspect "getsource")) + +(def ^:private types (py/import-module "types")) + +(def ^:private fn-type (py/get-attr types "FunctionType")) + +(def ^:private method-type (py/get-attr types "MethodType")) + +(def ^:private isinstance? (py/get-attr builtins "isinstance")) + +(def ^:private fn? #(isinstance? % fn-type)) + +(def ^:private method? #(isinstance? % method-type)) + +(def ^:private doc #(try + (py/get-attr % "__doc__") + (catch Exception e + nil))) + +(def ^:private importlib (py/import-module "importlib")) + +(def ^:private reload-module (py/get-attr importlib "reload")) + +(def ^:priviate import-module (py/get-attr importlib "import_module")) + +(def ^{:private false :public true} + get-pydoc doc) + +(def ^:private vars (py/get-attr builtins "vars")) + +(defn ^:private py-fn-argspec [f] + (let [spec (argspec f)] + {:args (py/->jvm (py/get-attr spec "args")) + :varargs (py/->jvm (py/get-attr spec "varargs")) + :varkw (py/->jvm (py/get-attr spec "varkw")) + :defaults (py/->jvm (py/get-attr spec "defaults")) + :kwonlyargs (py/->jvm (py/get-attr spec "kwonlyargs")) + :kwonlydefaults (py/->jvm (py/get-attr spec "kwonlydefaults")) + :annotations (py/->jvm (py/get-attr spec "annotations"))})) + +(defn ^:private py-class-argspec [class] + (let [constructor (py/get-attr class "__init__")] + (py-fn-argspec constructor))) + +(defn pyargspec [x] + (cond + (fn? x) (py-fn-argspec x) + (method? x) (py-fn-argspec x) + :else (py-class-argspec x))) + +(defn ^:private load-py-fn [f fn-name fn-module-name-or-ns] + (let [fn-argspec (pyargspec f) + fn-docstr (get-pydoc f) + fn-ns (symbol (str fn-module-name-or-ns)) + fn-sym (symbol fn-name)] + (intern fn-ns fn-sym + (with-meta f + (merge + fn-argspec + {:doc fn-docstr + :name fn-name}))))) + +(defn ^:private load-python-lib [req] + (let [supported-flags #{:reload} + [module-name & etc] req + flags (into #{} + (filter supported-flags) + etc) + etc (into {} + (comp + (remove supported-flags) + (partition-all 2) + (map vec)) + etc) + reload? (:reload flags) + this-module (import-module (str module-name)) + module-name-or-ns (:as etc module-name) + exclude (into #{} (:exclude etc)) + refer (cond + (= :all (:refer etc)) #{:all} + (= :* (:refer etc)) #{:*} + :else (into + #{} + (:refer etc))) + current-ns *ns* + current-ns-sym (symbol (str current-ns))] + + + ;; if the current namespace is already loaded, unless + ;; the :reload flag is specified, this will be a no-op + (when (not (and (find-ns module-name-or-ns) + (not reload?)));; :reload behavior + + ;; TODO: should we track things referred into the existing + ;; ..: *ns* with an atom and clear them on :reload? + + (when reload? + (remove-ns module-name) + (reload-module this-module)) + + ;; bind the python module to its symbolic name + ;; in the current namespace + (intern current-ns-sym module-name-or-ns this-module) + + ;; create namespace for module and bind python + ;; values to namespace symbols + (doseq [[k pyfn?] (seq (py/as-jvm (vars this-module)))] + (try + (load-py-fn pyfn? (symbol k) module-name-or-ns) + (catch Exception e + (try + (let [ns module-name-or-ns + symbol (symbol k)] + (in-ns ns) + (intern ns symbol pyfn?)) + (finally + (in-ns current-ns-sym)))))) + + ;; behavior for [.. :refer :all], [.. :refer [...]], and + ;; [.. :refer :*] + ;; TODO: code is a bit repetitive maybe + (cond + ;; include everything into the current namespace, + ;; ignore __all__ directive + (refer :all) + (doseq + [[k pyfn?] + (seq (py/as-jvm (vars this-module))) + :when (not (exclude (symbol k)))] + (try + (load-py-fn pyfn? (symbol k) current-ns-sym) + (catch Exception e + (let [symbol (symbol k)] + (intern *ns* symbol pyfn?))))) + ;; only include that specfied by __all__ attribute + ;; of python module if specified, else same as :all + (refer :*) + (let [hasattr (py/get-attr builtins "hasattr") + getattr (py/get-attr builtins "getattr")] + (if (hasattr this-module "__all__") + (doseq + [[k pyfn?] + (for [item-name (getattr this-module "__all__")] + [(symbol item-name) + (getattr this-module item-name)]) + :when (not (exclude (symbol k)))] + (try + (load-py-fn pyfn? (symbol k) current-ns-sym) + (catch Exception e + (let [symbol (symbol k)] + (intern *ns* symbol pyfn?))))) + (doseq + [[k pyfn?] + (seq (py/as-jvm (vars this-module))) + :when (not (exclude (symbol k)))] + (try + (load-py-fn pyfn? (symbol k) current-ns-sym) + (catch Exception e + (let [symbol (symbol k)] + (intern *ns* symbol pyfn?))))))) + + ;; [.. :refer [..]] behavior + :else + (doseq [r refer + :let [pyfn? (py/get-attr this-module (str r))]] + (if (or (fn? pyfn?) (method? pyfn?)) + (load-py-fn pyfn?) + (intern current-ns-sym r pyfn?))))))) + +(defn require-python [reqs] + "## Basic usage ## + + (require-python 'math) + (math/sin 1.0) ;;=> 0.8414709848078965 + + (require-python '[math :as maaaath]) + + (maaaath/sin 1.0) ;;=> 0.8414709848078965 + + (require-python '(math csv)) + (require-python '([math :as pymath] csv)) + (require-python '([math :as pymath] [csv :as py-csv]) + (require-python 'concurrent.futures) + (require-python '[concurrent.futures :as fs]) + + (require-python '[requests :refer [get post]]) + + (requests/get \"https//www.google.com\") ;;=> + (get \"https//www.google.com\") ;;=> + + + ## Use with custom modules ## + + For use with a custom namespace foo.py while developing, you can + use: + + (require-python '[foo :reload]) + + NOTE: unless you specify the :reload flag, + ..: the module will NOT reload. If the :reload flag is set, + ..: the behavior mimics importlib.reload + + ## Setting up classpath for custom modules ## + + Note: you may need to setup your PYTHONPATH correctly. + One technique to do this is, if your foo.py lives at + /path/to/foodir/foo.py: + + (require-python 'sys) + (py/call-attr (py/get-attr sys \"path\") + \"append\" + \"/path/to/foodir\") + + Another option is + + (require-python 'os) + (os/chdir \"/path/to/foodir\") + + + ## For library developers ## + + If you want to intern all symbols to your current namespace, + you can do the following -- + + (require-python '[math :refer :all]) + + However, if you only want to use + those things designated by the module under the __all__ attribute, + you can do + + (require-python '[operators :refer :*])" + + (cond + (list? reqs) + (doseq [req (vec reqs)] (require-python req)) + (symbol? reqs) + (load-python-lib (vector reqs)) + (vector? reqs) + (load-python-lib reqs))) + +(comment + (require-python '([clojure :refer [parenthesis]] __future__)) + (py/set-attr! __future__ "braces" parenthesis)) From 5c09e99df953e5a060b953db90a694678fad9e75 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 8 Dec 2019 09:54:49 -0700 Subject: [PATCH 040/456] Metadata is now applied to symbol. --- src/libpython_clj/py_modules/numpy.clj | 5 ++- src/libpython_clj/require.clj | 44 ++++++++++++++------------ 2 files changed, 26 insertions(+), 23 deletions(-) diff --git a/src/libpython_clj/py_modules/numpy.clj b/src/libpython_clj/py_modules/numpy.clj index afda395..81bbf46 100644 --- a/src/libpython_clj/py_modules/numpy.clj +++ b/src/libpython_clj/py_modules/numpy.clj @@ -1,13 +1,12 @@ (ns libpython-clj.py-modules.numpy (:require [libpython-clj.python :as py] - [libpython-clj.export-module-symbols - :refer [export-module-symbols]]) + [libpython-clj.require :refer [require-python]]) (:refer-clojure :exclude [test take str sort short require repeat partition mod min max long load int identity float empty double conj char cast byte])) -(export-module-symbols "numpy") +(require-python '[numpy :refer :* :reload]) (comment diff --git a/src/libpython_clj/require.clj b/src/libpython_clj/require.clj index 6d46246..c7f5cc9 100644 --- a/src/libpython_clj/require.clj +++ b/src/libpython_clj/require.clj @@ -2,6 +2,8 @@ (:refer-clojure :exclude [fn? doc]) (:require [libpython-clj.python :as py])) +(py/initialize!) + (def ^:private builtins (py/import-module "builtins")) (def ^:private inspect (py/import-module "inspect")) @@ -58,17 +60,18 @@ (method? x) (py-fn-argspec x) :else (py-class-argspec x))) +(defn pymetadata [fn-name x] + (let [fn-argspec (pyargspec x) + fn-docstr (get-pydoc x)] + (merge + fn-argspec + {:doc fn-docstr + :name fn-name}))) + (defn ^:private load-py-fn [f fn-name fn-module-name-or-ns] - (let [fn-argspec (pyargspec f) - fn-docstr (get-pydoc f) - fn-ns (symbol (str fn-module-name-or-ns)) + (let [fn-ns (symbol (str fn-module-name-or-ns)) fn-sym (symbol fn-name)] - (intern fn-ns fn-sym - (with-meta f - (merge - fn-argspec - {:doc fn-docstr - :name fn-name}))))) + (intern fn-ns (with-meta fn-sym (metadata fn-name f)) f))) (defn ^:private load-python-lib [req] (let [supported-flags #{:reload} @@ -104,7 +107,7 @@ ;; TODO: should we track things referred into the existing ;; ..: *ns* with an atom and clear them on :reload? - (when reload? + (when reload? (remove-ns module-name) (reload-module this-module)) @@ -168,7 +171,7 @@ (catch Exception e (let [symbol (symbol k)] (intern *ns* symbol pyfn?))))))) - + ;; [.. :refer [..]] behavior :else (doseq [r refer @@ -177,9 +180,9 @@ (load-py-fn pyfn?) (intern current-ns-sym r pyfn?))))))) -(defn require-python [reqs] +(defn require-python "## Basic usage ## - + (require-python 'math) (math/sin 1.0) ;;=> 0.8414709848078965 @@ -212,16 +215,16 @@ ## Setting up classpath for custom modules ## - Note: you may need to setup your PYTHONPATH correctly. - One technique to do this is, if your foo.py lives at + Note: you may need to setup your PYTHONPATH correctly. + One technique to do this is, if your foo.py lives at /path/to/foodir/foo.py: (require-python 'sys) - (py/call-attr (py/get-attr sys \"path\") - \"append\" + (py/call-attr (py/get-attr sys \"path\") + \"append\" \"/path/to/foodir\") - Another option is + Another option is (require-python 'os) (os/chdir \"/path/to/foodir\") @@ -230,15 +233,16 @@ ## For library developers ## If you want to intern all symbols to your current namespace, - you can do the following -- + you can do the following -- (require-python '[math :refer :all]) - However, if you only want to use + However, if you only want to use those things designated by the module under the __all__ attribute, you can do (require-python '[operators :refer :*])" + [reqs] (cond (list? reqs) From 4a0747011a4a1bbaf58a952b598218c6ed37be03 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 8 Dec 2019 09:58:15 -0700 Subject: [PATCH 041/456] Deprecating export macro. --- src/libpython_clj/export_module_symbols.clj | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libpython_clj/export_module_symbols.clj b/src/libpython_clj/export_module_symbols.clj index 30aac0c..b6e48c4 100644 --- a/src/libpython_clj/export_module_symbols.clj +++ b/src/libpython_clj/export_module_symbols.clj @@ -1,6 +1,8 @@ (ns libpython-clj.export-module-symbols "A macro that will export all the symbols from a python module and make - them functions in the current clojure namespace." + them functions in the current clojure namespace. + + DEPRECATED - please use 'libpython-clj.require/require-python" (:require [libpython-clj.python :as py])) @@ -8,6 +10,7 @@ (defmacro export-module-symbols + "DEPRECATED - please use 'libpython-clj.require/require-python" [py-mod-name] (py/initialize!) (let [mod-data (py/import-module py-mod-name)] From 245e76b81294499ec5132d333eaa1dc9c62c4ab9 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 8 Dec 2019 10:41:59 -0700 Subject: [PATCH 042/456] Preparing for next release. --- CHANGELOG.md | 36 ++++++++++++++++++++++++++++++++++++ project.clj | 2 +- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25e9561..abd6c10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,41 @@ # Time for a ChangeLog! +## 1.20-SNAPSHOT + +With two many huge things we had to skip a few versions! + +#### require-python + +`require-python` works like require but it works on python modules. +`require-python` dynamically loads the module and exports it's symbols into +a clojure namespace. There are many options available for this pathway. + + +This implements a big step towards embedding python in Clojure in a simple, +clear, and easy to use way. One important thing to consider is the require +has a `:reload:` option to allow you to actively develop a python module and +test it via clojure. + + +This excellent work was done by [James Tolton](https://github.com/jjtolton). + + +#### Clojure-defined Python classes + +You can now extend a tuple of python classes (or implement a new one). This system +allows, among many things, us to use frameworks that use derivation as part of their +public API. Please see [classes-test](test/libpython-clj.classes-test) for a documented +example of a simple pathway through the new API. Note that if you use vanilla +`->py-fn` functions as part of the class definition you won't get access to the `self` +object. + + +#### Bugfixes + +A general stability bugfix was made that was involved in the interoperation of +Clojure functions within Python. Clojure functions weren't currently adding +a refcount to their return values. + ## 1.16 diff --git a/project.clj b/project.clj index b106221..ec008c2 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.17-SNAPSHOT" +(defproject cnuernber/libpython-clj "1.20-SNAPSHOT" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" From 87bdc05b86cd20885f0a0071da1ebe249db0d016 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 8 Dec 2019 10:54:23 -0700 Subject: [PATCH 043/456] editing --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index abd6c10..f09152e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,7 @@ This excellent work was done by [James Tolton](https://github.com/jjtolton). You can now extend a tuple of python classes (or implement a new one). This system allows, among many things, us to use frameworks that use derivation as part of their -public API. Please see [classes-test](test/libpython-clj.classes-test) for a documented +public API. Please see [classes-test](test/libpython_clj.classes-test) for a documented example of a simple pathway through the new API. Note that if you use vanilla `->py-fn` functions as part of the class definition you won't get access to the `self` object. From d9f5e3fe8f029deaaf815ecfd4343cb38165626d Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 8 Dec 2019 10:54:58 -0700 Subject: [PATCH 044/456] editing --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f09152e..c684f37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,7 @@ This excellent work was done by [James Tolton](https://github.com/jjtolton). You can now extend a tuple of python classes (or implement a new one). This system allows, among many things, us to use frameworks that use derivation as part of their -public API. Please see [classes-test](test/libpython_clj.classes-test) for a documented +public API. Please see [classes-test](test/libpython_clj/classes_test.clj) for a documented example of a simple pathway through the new API. Note that if you use vanilla `->py-fn` functions as part of the class definition you won't get access to the `self` object. From bdfcb5da637d1ad6b710f5580a58284474bc8a2d Mon Sep 17 00:00:00 2001 From: "J.J. Tolton" Date: Sun, 8 Dec 2019 13:20:22 -0500 Subject: [PATCH 045/456] convert python argspec to clojure arglists; properly assign metadata for discoverability (#14) --- src/libpython_clj/require.clj | 105 ++++++++++++++++++++++++++++++---- 1 file changed, 94 insertions(+), 11 deletions(-) diff --git a/src/libpython_clj/require.clj b/src/libpython_clj/require.clj index c7f5cc9..8a8ff39 100644 --- a/src/libpython_clj/require.clj +++ b/src/libpython_clj/require.clj @@ -2,8 +2,6 @@ (:refer-clojure :exclude [fn? doc]) (:require [libpython-clj.python :as py])) -(py/initialize!) - (def ^:private builtins (py/import-module "builtins")) (def ^:private inspect (py/import-module "inspect")) @@ -60,18 +58,106 @@ (method? x) (py-fn-argspec x) :else (py-class-argspec x))) +(defn pyarglists + ([argspec] (pyarglists argspec + (if-let [defaults + (not-empty (:defaults argspec))] + defaults + []))) + ([argspec defaults] (pyarglists argspec defaults [])) + ([{:as argspec + args :args + varkw :varkw + varargs :varargs + kwonlydefaults :kwonlydefaults + kwonlyargs :kwonlyargs} + defaults res] + (println argspec) + (println defaults) + (let [n-args (count args) + n-defaults (count defaults) + n-pos-args (- n-args n-defaults) + pos-args (transduce + (comp + (take n-pos-args) + (map symbol)) + (completing conj) + [] + args) + kw-default-args (transduce + (comp + (drop n-pos-args) + (map symbol)) + (completing conj) + [] + args) + or-map (transduce + (comp + (partition-all 2) + (map vec) + (map (fn [[k v]] [(symbol k) v]))) + (completing (partial apply assoc)) + {} + (concat + (interleave kw-default-args defaults) + (flatten (seq kwonlydefaults)))) + as-varkw (when (not (nil? varkw)) + {:as (symbol varkw)}) + default-map (transduce + (comp + (partition-all 2) + (map vec) + (map (fn [[k v]] [(symbol k) (keyword k)]))) + (completing (partial apply assoc)) + {} + (concat + (interleave kw-default-args defaults) + (flatten (seq kwonlydefaults)))) + kwargs-map (merge default-map + (when (not-empty or-map) + {:or or-map}) + (when (not-empty as-varkw) + as-varkw)) + opt-args + (cond + (and (empty? kwargs-map) + (nil? varargs)) '() + (empty? kwargs-map) (list '& [(symbol varargs)]) + (nil? varargs) (list '& [or-map]) + :else (list '& [(symbol varargs) + kwargs-map])) + + arglist (concat (list* pos-args) opt-args)] + (let [arglists (conj res arglist) + defaults' (if (not-empty defaults) (pop defaults) []) + argspec' (update argspec :args + (fn [args] + (if (not-empty args) + (pop args) + args)))] + + (if (and (empty? defaults) (empty? defaults')) + arglists + (recur argspec defaults' arglists)))))) + + (defn pymetadata [fn-name x] - (let [fn-argspec (pyargspec x) - fn-docstr (get-pydoc x)] + (let [fn-argspec (pyargspec x) + fn-docstr (get-pydoc x) + fn-arglists (pyarglists fn-argspec)] (merge fn-argspec - {:doc fn-docstr - :name fn-name}))) + {:doc fn-docstr + :arglists fn-arglists + :name fn-name}))) + + (defn ^:private load-py-fn [f fn-name fn-module-name-or-ns] (let [fn-ns (symbol (str fn-module-name-or-ns)) fn-sym (symbol fn-name)] - (intern fn-ns (with-meta fn-sym (metadata fn-name f)) f))) + (intern fn-ns (with-meta fn-sym (pymetadata fn-name f)) f))) + (defn ^:private load-python-lib [req] (let [supported-flags #{:reload} @@ -96,10 +182,7 @@ #{} (:refer etc))) current-ns *ns* - current-ns-sym (symbol (str current-ns))] - - - ;; if the current namespace is already loaded, unless + current-ns-sym (symbol (str current-ns))];; if the current namespace is already loaded, unless ;; the :reload flag is specified, this will be a no-op (when (not (and (find-ns module-name-or-ns) (not reload?)));; :reload behavior From 70db576c401fcf185eaf31b4d0f20630033bb738 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 8 Dec 2019 12:52:42 -0700 Subject: [PATCH 046/456] testing require-python --- src/libpython_clj/require.clj | 252 ++++++++++++++------- test/libpython_clj/require_python_test.clj | 21 ++ 2 files changed, 192 insertions(+), 81 deletions(-) create mode 100644 test/libpython_clj/require_python_test.clj diff --git a/src/libpython_clj/require.clj b/src/libpython_clj/require.clj index c7f5cc9..83ca6ed 100644 --- a/src/libpython_clj/require.clj +++ b/src/libpython_clj/require.clj @@ -1,6 +1,7 @@ (ns libpython-clj.require (:refer-clojure :exclude [fn? doc]) - (:require [libpython-clj.python :as py])) + (:require [libpython-clj.python :as py] + [clojure.tools.logging :as log])) (py/initialize!) @@ -27,7 +28,7 @@ (def ^:private doc #(try (py/get-attr % "__doc__") (catch Exception e - nil))) + ""))) (def ^:private importlib (py/import-module "importlib")) @@ -58,20 +59,123 @@ (cond (fn? x) (py-fn-argspec x) (method? x) (py-fn-argspec x) + (string? x) "" + (number? x) "" :else (py-class-argspec x))) -(defn pymetadata [fn-name x] + +(defn pyarglists + ([argspec] (pyarglists argspec + (if-let [defaults + (not-empty (:defaults argspec))] + defaults + []))) + ([argspec defaults] (pyarglists argspec defaults [])) + ([{:as argspec + args :args + varkw :varkw + varargs :varargs + kwonlydefaults :kwonlydefaults + kwonlyargs :kwonlyargs} + defaults res] + (let [n-args (count args) + n-defaults (count defaults) + n-pos-args (- n-args n-defaults) + pos-args (transduce + (comp + (take n-pos-args) + (map symbol)) + (completing conj) + [] + args) + kw-default-args (transduce + (comp + (drop n-pos-args) + (map symbol)) + (completing conj) + [] + args) + or-map (transduce + (comp + (partition-all 2) + (map vec) + (map (fn [[k v]] [(symbol k) v]))) + (completing (partial apply assoc)) + {} + (concat + (interleave kw-default-args defaults) + (flatten (seq kwonlydefaults)))) + + as-varkw (when (not (nil? varkw)) + {:as (symbol varkw)}) + default-map (transduce + (comp + (partition-all 2) + (map vec) + (map (fn [[k v]] [(symbol k) (keyword k)]))) + (completing (partial apply assoc)) + {} + (concat + (interleave kw-default-args defaults) + (flatten (seq kwonlydefaults)))) + + kwargs-map (merge default-map + (when (not-empty or-map) + {:or or-map}) + (when (not-empty as-varkw) + as-varkw)) + + + + + opt-args + (cond + (and (empty? kwargs-map) + (nil? varargs)) '() + (empty? kwargs-map) (list '& [(symbol varargs)]) + (nil? varargs) (list '& [or-map]) + :else (list '& [(symbol varargs) + kwargs-map])) + + arglist (concat (list* pos-args) opt-args)] + (let [arglists (conj res arglist) + defaults' (if (not-empty defaults) (pop defaults) []) + argspec' (update argspec :args + (fn [args] + (if (not-empty args) + (pop args) + args)))] + + (if (and (empty? defaults) (empty? defaults')) + arglists + (recur argspec' defaults' arglists)))))) + + +(defn py-fn-metadata [fn-name x] (let [fn-argspec (pyargspec x) fn-docstr (get-pydoc x)] (merge fn-argspec {:doc fn-docstr - :name fn-name}))) + :name fn-name} + ;;TODO -- + ;;We need to figure out a syntax that allows the actual functions + ;;to be called. The arglists, while they look good the compiler parses + ;;them and then errors out. + + ;; (when (py/callable? x) + ;; (try + ;; {:arglists (pyarglists fn-argspec)} + ;; (catch Throwable e + ;; nil))) + ))) + (defn ^:private load-py-fn [f fn-name fn-module-name-or-ns] (let [fn-ns (symbol (str fn-module-name-or-ns)) fn-sym (symbol fn-name)] - (intern fn-ns (with-meta fn-sym (metadata fn-name f)) f))) + (intern fn-ns (with-meta fn-sym (py-fn-metadata fn-name f)) f))) + (defn ^:private load-python-lib [req] (let [supported-flags #{:reload} @@ -86,7 +190,6 @@ (map vec)) etc) reload? (:reload flags) - this-module (import-module (str module-name)) module-name-or-ns (:as etc module-name) exclude (into #{} (:exclude etc)) refer (cond @@ -96,89 +199,76 @@ #{} (:refer etc))) current-ns *ns* - current-ns-sym (symbol (str current-ns))] + current-ns-sym (symbol (str current-ns)) + python-namespace (find-ns module-name-or-ns) + this-module (import-module (str module-name))] - - ;; if the current namespace is already loaded, unless - ;; the :reload flag is specified, this will be a no-op - (when (not (and (find-ns module-name-or-ns) - (not reload?)));; :reload behavior - - ;; TODO: should we track things referred into the existing - ;; ..: *ns* with an atom and clear them on :reload? - - (when reload? - (remove-ns module-name) - (reload-module this-module)) + (when reload? + (remove-ns module-name) + (reload-module this-module)) + (create-ns module-name-or-ns) ;; bind the python module to its symbolic name ;; in the current namespace - (intern current-ns-sym module-name-or-ns this-module) + ;; create namespace for module and bind python ;; values to namespace symbols - (doseq [[k pyfn?] (seq (py/as-jvm (vars this-module)))] - (try - (load-py-fn pyfn? (symbol k) module-name-or-ns) - (catch Exception e - (try - (let [ns module-name-or-ns - symbol (symbol k)] - (in-ns ns) - (intern ns symbol pyfn?)) - (finally - (in-ns current-ns-sym)))))) - - ;; behavior for [.. :refer :all], [.. :refer [...]], and - ;; [.. :refer :*] - ;; TODO: code is a bit repetitive maybe - (cond - ;; include everything into the current namespace, - ;; ignore __all__ directive - (refer :all) - (doseq - [[k pyfn?] - (seq (py/as-jvm (vars this-module))) - :when (not (exclude (symbol k)))] + (when (or reload? + (not python-namespace)) + ;;Mutably define the root namespace. + (doseq [att-name (py/dir this-module)] + (let [v (py/get-attr this-module att-name)] (try - (load-py-fn pyfn? (symbol k) current-ns-sym) - (catch Exception e - (let [symbol (symbol k)] - (intern *ns* symbol pyfn?))))) - ;; only include that specfied by __all__ attribute - ;; of python module if specified, else same as :all - (refer :*) - (let [hasattr (py/get-attr builtins "hasattr") - getattr (py/get-attr builtins "getattr")] - (if (hasattr this-module "__all__") - (doseq - [[k pyfn?] - (for [item-name (getattr this-module "__all__")] - [(symbol item-name) - (getattr this-module item-name)]) - :when (not (exclude (symbol k)))] - (try - (load-py-fn pyfn? (symbol k) current-ns-sym) - (catch Exception e - (let [symbol (symbol k)] - (intern *ns* symbol pyfn?))))) - (doseq - [[k pyfn?] - (seq (py/as-jvm (vars this-module))) - :when (not (exclude (symbol k)))] - (try - (load-py-fn pyfn? (symbol k) current-ns-sym) - (catch Exception e - (let [symbol (symbol k)] - (intern *ns* symbol pyfn?))))))) - - ;; [.. :refer [..]] behavior - :else - (doseq [r refer - :let [pyfn? (py/get-attr this-module (str r))]] - (if (or (fn? pyfn?) (method? pyfn?)) - (load-py-fn pyfn?) - (intern current-ns-sym r pyfn?))))))) + (when v + (if (py/callable? v) + (load-py-fn v (symbol att-name) module-name-or-ns) + (intern module-name-or-ns (symbol att-name) v))) + (catch Throwable e + (log/warnf e "Failed to require symbol %s" att-name)))))) + + + (let [python-namespace (find-ns module-name-or-ns) + ;;ns-publics is a map of symbol to var. Var's have metadata on them. + public-data (->> (ns-publics python-namespace) + (remove #(exclude (first %))) + (into {}))] + + ;;Always make the loaded namespace available to the current namespace. + (intern current-ns-sym (with-meta module-name-or-ns + {:doc (doc this-module)})) + (let [refer-symbols + (cond + ;; include everything into the current namespace, + ;; ignore __all__ directive + (or (refer :all) + (and (not (py/has-attr? this-module "__all__")) + (refer :*))) + (keys public-data) + ;; only include that specfied by __all__ attribute + ;; of python module if specified, else same as :all + (refer :*) + (->> (py/get-attr this-module "__all__") + (map (fn [item-name] + (let [item-sym (symbol item-name)] + (when (contains? public-data item-sym) + item-sym)))) + (remove nil?)) + + ;; [.. :refer [..]] behavior + :else + (do + (when-let [missing (->> refer + (remove (partial contains? public-data)) + seq)] + (throw (Exception. + (format "'refer' symbols not found: %s" + (vec missing))))) + refer))] + (doseq [[s v] (select-keys public-data refer-symbols)] + (intern current-ns-sym + (with-meta s (meta v)) + (deref v))))))) (defn require-python "## Basic usage ## diff --git a/test/libpython_clj/require_python_test.clj b/test/libpython_clj/require_python_test.clj new file mode 100644 index 0000000..76b127d --- /dev/null +++ b/test/libpython_clj/require_python_test.clj @@ -0,0 +1,21 @@ +(ns libpython-clj.require-python-test + (:require [libpython-clj.require :refer [require-python]] + [clojure.test :refer :all])) + + +;;Since this test mutates the global environment we have to accept that +;;it may not always work. + +(require-python '[math + :refer :* + :exclude [sin cos] + ]) + + +(deftest base-require-test + (let [publics (ns-publics (find-ns 'libpython-clj.require-python-test))] + (is (contains? publics 'acos)) + (is (contains? publics 'floor)) + (is (not (contains? publics 'sin))) + (is (= 10.0 (double (floor 10.1)))) + (is (thrown? Throwable (require-python '[math :refer [blah]]))))) From 894162cdc849e5cce726dc6f18e7d5813638bf46 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 8 Dec 2019 13:03:01 -0700 Subject: [PATCH 047/456] oof, bad merge! --- src/libpython_clj/require.clj | 53 ++--------------------------------- 1 file changed, 2 insertions(+), 51 deletions(-) diff --git a/src/libpython_clj/require.clj b/src/libpython_clj/require.clj index a6e0fed..83ca6ed 100644 --- a/src/libpython_clj/require.clj +++ b/src/libpython_clj/require.clj @@ -3,6 +3,8 @@ (:require [libpython-clj.python :as py] [clojure.tools.logging :as log])) +(py/initialize!) + (def ^:private builtins (py/import-module "builtins")) (def ^:private inspect (py/import-module "inspect")) @@ -61,10 +63,7 @@ (number? x) "" :else (py-class-argspec x))) -<<<<<<< HEAD -======= ->>>>>>> bdfcb5da637d1ad6b710f5580a58284474bc8a2d (defn pyarglists ([argspec] (pyarglists argspec (if-let [defaults @@ -79,11 +78,6 @@ kwonlydefaults :kwonlydefaults kwonlyargs :kwonlyargs} defaults res] -<<<<<<< HEAD -======= - (println argspec) - (println defaults) ->>>>>>> bdfcb5da637d1ad6b710f5580a58284474bc8a2d (let [n-args (count args) n-defaults (count defaults) n-pos-args (- n-args n-defaults) @@ -111,10 +105,7 @@ (concat (interleave kw-default-args defaults) (flatten (seq kwonlydefaults)))) -<<<<<<< HEAD -======= ->>>>>>> bdfcb5da637d1ad6b710f5580a58284474bc8a2d as-varkw (when (not (nil? varkw)) {:as (symbol varkw)}) default-map (transduce @@ -127,22 +118,16 @@ (concat (interleave kw-default-args defaults) (flatten (seq kwonlydefaults)))) -<<<<<<< HEAD -======= ->>>>>>> bdfcb5da637d1ad6b710f5580a58284474bc8a2d kwargs-map (merge default-map (when (not-empty or-map) {:or or-map}) (when (not-empty as-varkw) as-varkw)) -<<<<<<< HEAD -======= ->>>>>>> bdfcb5da637d1ad6b710f5580a58284474bc8a2d opt-args (cond (and (empty? kwargs-map) @@ -163,7 +148,6 @@ (if (and (empty? defaults) (empty? defaults')) arglists -<<<<<<< HEAD (recur argspec' defaults' arglists)))))) @@ -185,31 +169,12 @@ ;; (catch Throwable e ;; nil))) ))) -======= - (recur argspec defaults' arglists)))))) - - -(defn pymetadata [fn-name x] - (let [fn-argspec (pyargspec x) - fn-docstr (get-pydoc x) - fn-arglists (pyarglists fn-argspec)] - (merge - fn-argspec - {:doc fn-docstr - :arglists fn-arglists - :name fn-name}))) - ->>>>>>> bdfcb5da637d1ad6b710f5580a58284474bc8a2d (defn ^:private load-py-fn [f fn-name fn-module-name-or-ns] (let [fn-ns (symbol (str fn-module-name-or-ns)) fn-sym (symbol fn-name)] -<<<<<<< HEAD (intern fn-ns (with-meta fn-sym (py-fn-metadata fn-name f)) f))) -======= - (intern fn-ns (with-meta fn-sym (pymetadata fn-name f)) f))) ->>>>>>> bdfcb5da637d1ad6b710f5580a58284474bc8a2d (defn ^:private load-python-lib [req] @@ -234,7 +199,6 @@ #{} (:refer etc))) current-ns *ns* -<<<<<<< HEAD current-ns-sym (symbol (str current-ns)) python-namespace (find-ns module-name-or-ns) this-module (import-module (str module-name))] @@ -243,19 +207,6 @@ (remove-ns module-name) (reload-module this-module)) (create-ns module-name-or-ns) -======= - current-ns-sym (symbol (str current-ns))];; if the current namespace is already loaded, unless - ;; the :reload flag is specified, this will be a no-op - (when (not (and (find-ns module-name-or-ns) - (not reload?)));; :reload behavior - - ;; TODO: should we track things referred into the existing - ;; ..: *ns* with an atom and clear them on :reload? - - (when reload? - (remove-ns module-name) - (reload-module this-module)) ->>>>>>> bdfcb5da637d1ad6b710f5580a58284474bc8a2d ;; bind the python module to its symbolic name ;; in the current namespace From 0576b6d965a078583218a48418b12ff52322e36d Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 8 Dec 2019 13:27:45 -0700 Subject: [PATCH 048/456] editing --- CHANGELOG.md | 5 ++++- src/libpython_clj/require.clj | 17 ++++++++--------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c684f37..376bece 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,10 @@ has a `:reload:` option to allow you to actively develop a python module and test it via clojure. -This excellent work was done by [James Tolton](https://github.com/jjtolton). +This excellent work was in large part done by [James Tolton](https://github.com/jjtolton). + + +* [require-python-test](test/libpython_clj/require_python_test.clj) #### Clojure-defined Python classes diff --git a/src/libpython_clj/require.clj b/src/libpython_clj/require.clj index 83ca6ed..7c71e64 100644 --- a/src/libpython_clj/require.clj +++ b/src/libpython_clj/require.clj @@ -217,15 +217,14 @@ (when (or reload? (not python-namespace)) ;;Mutably define the root namespace. - (doseq [att-name (py/dir this-module)] - (let [v (py/get-attr this-module att-name)] - (try - (when v - (if (py/callable? v) - (load-py-fn v (symbol att-name) module-name-or-ns) - (intern module-name-or-ns (symbol att-name) v))) - (catch Throwable e - (log/warnf e "Failed to require symbol %s" att-name)))))) + (doseq [[att-name v] (vars this-module)] + (try + (when v + (if (py/callable? v) + (load-py-fn v (symbol att-name) module-name-or-ns) + (intern module-name-or-ns (symbol att-name) v))) + (catch Throwable e + (log/warnf e "Failed to require symbol %s" att-name))))) (let [python-namespace (find-ns module-name-or-ns) From 310b3b021295db2f7aa600fd10c99441a656f676 Mon Sep 17 00:00:00 2001 From: "J.J. Tolton" Date: Mon, 9 Dec 2019 12:07:50 -0500 Subject: [PATCH 049/456] fix for arglists not compiling bug (#16) * fix for arglists not compiling bug * or-map -> kwarg-map * remove TODO --- src/libpython_clj/require.clj | 35 +++++++++++++---------------------- 1 file changed, 13 insertions(+), 22 deletions(-) diff --git a/src/libpython_clj/require.clj b/src/libpython_clj/require.clj index 7c71e64..8729454 100644 --- a/src/libpython_clj/require.clj +++ b/src/libpython_clj/require.clj @@ -133,11 +133,11 @@ (and (empty? kwargs-map) (nil? varargs)) '() (empty? kwargs-map) (list '& [(symbol varargs)]) - (nil? varargs) (list '& [or-map]) + (nil? varargs) (list '& [kwargs-map]) :else (list '& [(symbol varargs) kwargs-map])) - - arglist (concat (list* pos-args) opt-args)] + + arglist ((comp vec concat) (list* pos-args) opt-args)] (let [arglists (conj res arglist) defaults' (if (not-empty defaults) (pop defaults) []) argspec' (update argspec :args @@ -157,18 +157,12 @@ (merge fn-argspec {:doc fn-docstr - :name fn-name} - ;;TODO -- - ;;We need to figure out a syntax that allows the actual functions - ;;to be called. The arglists, while they look good the compiler parses - ;;them and then errors out. - - ;; (when (py/callable? x) - ;; (try - ;; {:arglists (pyarglists fn-argspec)} - ;; (catch Throwable e - ;; nil))) - ))) + :name fn-name} + (when (py/callable? x) + (try + {:arglists (pyarglists fn-argspec)} + (catch Throwable e + nil)))))) (defn ^:private load-py-fn [f fn-name fn-module-name-or-ns] @@ -208,12 +202,12 @@ (reload-module this-module)) (create-ns module-name-or-ns) - ;; bind the python module to its symbolic name - ;; in the current namespace + ;; bind the python module to its symbolic name + ;; in the current namespace - ;; create namespace for module and bind python - ;; values to namespace symbols + ;; create namespace for module and bind python + ;; values to namespace symbols (when (or reload? (not python-namespace)) ;;Mutably define the root namespace. @@ -341,6 +335,3 @@ (vector? reqs) (load-python-lib reqs))) -(comment - (require-python '([clojure :refer [parenthesis]] __future__)) - (py/set-attr! __future__ "braces" parenthesis)) From 64fcb4bed8eb3af070dbb2209abbd32d18bfba50 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Mon, 9 Dec 2019 10:21:18 -0700 Subject: [PATCH 050/456] Adding one more flag and testing it. --- src/libpython_clj/require.clj | 29 +++++++++++++++------- test/libpython_clj/require_python_test.clj | 3 +-- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/src/libpython_clj/require.clj b/src/libpython_clj/require.clj index 8729454..fa4cfae 100644 --- a/src/libpython_clj/require.clj +++ b/src/libpython_clj/require.clj @@ -136,7 +136,7 @@ (nil? varargs) (list '& [kwargs-map]) :else (list '& [(symbol varargs) kwargs-map])) - + arglist ((comp vec concat) (list* pos-args) opt-args)] (let [arglists (conj res arglist) defaults' (if (not-empty defaults) (pop defaults) []) @@ -151,28 +151,31 @@ (recur argspec' defaults' arglists)))))) -(defn py-fn-metadata [fn-name x] +(defn py-fn-metadata [fn-name x {:keys [no-arglists?]}] (let [fn-argspec (pyargspec x) fn-docstr (get-pydoc x)] (merge fn-argspec {:doc fn-docstr - :name fn-name} - (when (py/callable? x) + :name fn-name} + (when (and (py/callable? x) + (not no-arglists?)) (try {:arglists (pyarglists fn-argspec)} (catch Throwable e nil)))))) -(defn ^:private load-py-fn [f fn-name fn-module-name-or-ns] +(defn ^:private load-py-fn [f fn-name fn-module-name-or-ns + options] (let [fn-ns (symbol (str fn-module-name-or-ns)) fn-sym (symbol fn-name)] - (intern fn-ns (with-meta fn-sym (py-fn-metadata fn-name f)) f))) + (intern fn-ns (with-meta fn-sym (py-fn-metadata fn-name f + options)) f))) (defn ^:private load-python-lib [req] - (let [supported-flags #{:reload} + (let [supported-flags #{:reload :no-arglists} [module-name & etc] req flags (into #{} (filter supported-flags) @@ -184,6 +187,7 @@ (map vec)) etc) reload? (:reload flags) + no-arglists? (:no-arglists flags) module-name-or-ns (:as etc module-name) exclude (into #{} (:exclude etc)) refer (cond @@ -215,7 +219,9 @@ (try (when v (if (py/callable? v) - (load-py-fn v (symbol att-name) module-name-or-ns) + (load-py-fn v (symbol att-name) module-name-or-ns + {:no-arglists? + no-arglists?}) (intern module-name-or-ns (symbol att-name) v))) (catch Throwable e (log/warnf e "Failed to require symbol %s" att-name))))) @@ -284,6 +290,12 @@ (requests/get \"https//www.google.com\") ;;=> (get \"https//www.google.com\") ;;=> + In some cases we may generate invalid arglists metadata for the clojure compiler. + In those cases we have a flag, :no-arglists that will disable adding arglists to + the generated metadata for the vars. Use the reload flag below if you need to + force reload a namespace where invalid arglists have been generated. + + (require-python '[numpy :refer [linspace] :no-arglists :as np]) ## Use with custom modules ## @@ -334,4 +346,3 @@ (load-python-lib (vector reqs)) (vector? reqs) (load-python-lib reqs))) - diff --git a/test/libpython_clj/require_python_test.clj b/test/libpython_clj/require_python_test.clj index 76b127d..4eb3508 100644 --- a/test/libpython_clj/require_python_test.clj +++ b/test/libpython_clj/require_python_test.clj @@ -8,8 +8,7 @@ (require-python '[math :refer :* - :exclude [sin cos] - ]) + :exclude [sin cos]]) (deftest base-require-test From d9dbba254a2667a3e50e18a86fc500495e40886f Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Mon, 9 Dec 2019 10:22:17 -0700 Subject: [PATCH 051/456] 1.20 --- CHANGELOG.md | 2 +- project.clj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 376bece..c8f0075 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Time for a ChangeLog! -## 1.20-SNAPSHOT +## 1.20 With two many huge things we had to skip a few versions! diff --git a/project.clj b/project.clj index ec008c2..2d1d578 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.20-SNAPSHOT" +(defproject cnuernber/libpython-clj "1.20" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" From 0c73ad8f90cb9759662e9f6ebe9c2475814b0b4f Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Mon, 9 Dec 2019 10:22:24 -0700 Subject: [PATCH 052/456] snap --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 2d1d578..850ae23 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.20" +(defproject cnuernber/libpython-clj "1.21-SNAPSHOT" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" From dbc5ad1da11e7f8f6668499fd3a75b01a61028fd Mon Sep 17 00:00:00 2001 From: "J.J. Tolton" Date: Tue, 10 Dec 2019 11:05:51 -0500 Subject: [PATCH 053/456] fix for arglists on some functions (#17) --- src/libpython_clj/require.clj | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/src/libpython_clj/require.clj b/src/libpython_clj/require.clj index fa4cfae..1a05ae7 100644 --- a/src/libpython_clj/require.clj +++ b/src/libpython_clj/require.clj @@ -15,9 +15,15 @@ (def ^:private types (py/import-module "types")) -(def ^:private fn-type (py/get-attr types "FunctionType")) +(def ^:private fn-type + (py/call-attr builtins "tuple" + [(py/get-attr types "FunctionType") + (py/get-attr types "BuiltinFunctionType")])) -(def ^:private method-type (py/get-attr types "MethodType")) +(def ^:private method-type + (py/call-attr builtins "tuple" + [(py/get-attr types "MethodType") + (py/get-attr types "BuiltinMethodType")])) (def ^:private isinstance? (py/get-attr builtins "isinstance")) @@ -42,23 +48,35 @@ (def ^:private vars (py/get-attr builtins "vars")) (defn ^:private py-fn-argspec [f] - (let [spec (argspec f)] + (if-let [spec (try (argspec f) (catch Throwable e nil))] {:args (py/->jvm (py/get-attr spec "args")) :varargs (py/->jvm (py/get-attr spec "varargs")) :varkw (py/->jvm (py/get-attr spec "varkw")) :defaults (py/->jvm (py/get-attr spec "defaults")) :kwonlyargs (py/->jvm (py/get-attr spec "kwonlyargs")) :kwonlydefaults (py/->jvm (py/get-attr spec "kwonlydefaults")) - :annotations (py/->jvm (py/get-attr spec "annotations"))})) + :annotations (py/->jvm (py/get-attr spec "annotations"))} + (py-fn-argspec (py/get-attr f "__init__")))) (defn ^:private py-class-argspec [class] (let [constructor (py/get-attr class "__init__")] (py-fn-argspec constructor))) (defn pyargspec [x] + ;; TODO: certain builtin functions have + ;; ..: signatures that are found in the first line + ;; ..: of their docstring, aka, print. + ;; ..: These seem to be uniform enough that + ;; ..: most IDEs have a way of creating stubs + ;; ..: for the signature. If there is a uniform way + ;; ..: to do this that doesn't simply involve an + ;; ..: army of devs doing transcription I'd like to + ;; ..: pull that in here (cond (fn? x) (py-fn-argspec x) (method? x) (py-fn-argspec x) + ;; (builtin-function? x) (py-builtin-fn-argspec x) + ;; (builtin-method? x) (py-builtin-method-argspec x) (string? x) "" (number? x) "" :else (py-class-argspec x))) From 5f3035215f5acecf45ab75e237a57475e782ebf7 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 10 Dec 2019 09:22:54 -0700 Subject: [PATCH 054/456] added test for inifinite sequence. --- test/libpython_clj/python_test.clj | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/test/libpython_clj/python_test.clj b/test/libpython_clj/python_test.clj index e150733..4404c9f 100644 --- a/test/libpython_clj/python_test.clj +++ b/test/libpython_clj/python_test.clj @@ -1,5 +1,6 @@ (ns libpython-clj.python-test (:require [libpython-clj.python :as py] + [libpython-clj.require :refer [require-python]] [tech.v2.datatype :as dtype] [tech.v2.datatype.functional :as dfn] [tech.v2.tensor :as dtt] @@ -7,7 +8,7 @@ (:import [java.io StringWriter] [java.util Map List])) - (py/initialize!) +(py/initialize!) (deftest stdout-and-stderr @@ -265,3 +266,15 @@ (is (= (str (py/$.. np random shuffle)) (str (-> (py/get-attr np "random") (py/get-attr "shuffle"))))))) + + +(deftest infinite-seq + (let [islice (-> (py/import-module "itertools") + (py/get-attr "islice"))] + + (is (= (vec (range 10)) + ;;Range is an infinite sequence + (-> (range) + (py/as-python) + (islice 0 10) + (vec)))))) From e5613eb7d99abb7f81e36d22e15a30a30ca9ca5d Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 10 Dec 2019 11:13:29 -0700 Subject: [PATCH 055/456] Checking in what I have to move to bigger monitor. --- src/libpython_clj/python/bridge.clj | 6 +++++- src/libpython_clj/python/interop.clj | 7 ++++++- src/libpython_clj/python/interpreter.clj | 13 +++++++++++-- src/libpython_clj/python/object.clj | 2 +- test/libpython_clj/python_test.clj | 10 ++++++++-- 5 files changed, 31 insertions(+), 7 deletions(-) diff --git a/src/libpython_clj/python/bridge.clj b/src/libpython_clj/python/bridge.clj index f7a531e..114ba6c 100644 --- a/src/libpython_clj/python/bridge.clj +++ b/src/libpython_clj/python/bridge.clj @@ -260,7 +260,11 @@ (py-proto/att-type-map pyobj#))) py-proto/PyCall (do-call-fn [callable# arglist# kw-arg-map#] - (-> (py-proto/do-call-fn pyobj# arglist# kw-arg-map#) + (-> (py-proto/do-call-fn pyobj# (mapv as-python arglist#) + (->> kw-arg-map# + (map (fn [[k# v#]] + [k# (as-python v#)])) + (into {}))) (as-jvm))) Object (toString [this#] diff --git a/src/libpython_clj/python/interop.clj b/src/libpython_clj/python/interop.clj index 221d5ce..999326c 100644 --- a/src/libpython_clj/python/interop.clj +++ b/src/libpython_clj/python/interop.clj @@ -368,7 +368,12 @@ (throw (ex-info "Failed to find bridge type" {}))) bridge-type (PyTypeObject. bridge-type-ptr) ^Pointer new-py-obj (libpy/_PyObject_New bridge-type) - pybridge (JVMBridgeType. new-py-obj)] + pybridge (JVMBridgeType. new-py-obj) + ] + (println (format "Creating bridge: 0x%x -> %s:%s" + (Pointer/nativeValue new-py-obj) + (get-object-handle (.interpreter bridge)) + (get-object-handle (.wrappedObject bridge)))) (set! (.jvm_interpreter_handle pybridge) (get-object-handle (.interpreter bridge))) (set! (.jvm_handle pybridge) (get-object-handle (.wrappedObject bridge))) diff --git a/src/libpython_clj/python/interpreter.clj b/src/libpython_clj/python/interpreter.clj index e446b29..96e4d74 100644 --- a/src/libpython_clj/python/interpreter.clj +++ b/src/libpython_clj/python/interpreter.clj @@ -80,18 +80,26 @@ ^JVMBridge [handle interpreter] (if-let [bridge-obj (find-jvm-bridge-entry handle interpreter)] (:jvm-bridge bridge-obj) - (throw (ex-info "Unable to find bridge for interpreter %s and handle %s" - interpreter handle)))) + (throw (Exception. + (format "Unable to find bridge for interpreter %s and handle %s" + interpreter handle))))) (defn register-bridge! [^JVMBridge bridge ^PyObject bridge-pyobject] (let [interpreter (.interpreter bridge) bridge-handle (get-object-handle (.wrappedObject bridge))] + (when (contains? (get-in @(:interpreter-state* interpreter) + [:bridge-objects]) + bridge-handle) + (throw (Exception. (format "Bridge already exists!! - %s" bridge-handle)))) (swap! (:interpreter-state* interpreter) assoc-in [:bridge-objects bridge-handle] {:jvm-bridge bridge :pyobject bridge-pyobject}) + (println "ADDING-BRIDGE" bridge-handle + (keys (get-in @(:interpreter-state* interpreter) + [:bridge-objects]))) :ok)) @@ -99,6 +107,7 @@ [^JVMBridge bridge] (let [interpreter (.interpreter bridge) bridge-handle (get-object-handle (.wrappedObject bridge))] + (println "REMOVING-BRIDGE:" bridge-handle) (swap! (:interpreter-state* interpreter) update :bridge-objects dissoc bridge-handle) :ok)) diff --git a/src/libpython_clj/python/object.clj b/src/libpython_clj/python/object.clj index c15f465..4c230d4 100644 --- a/src/libpython_clj/python/object.clj +++ b/src/libpython_clj/python/object.clj @@ -95,7 +95,7 @@ (py-proto/->jvm item options))) -(def ^:dynamic *object-reference-logging* false) +(def ^:dynamic *object-reference-logging* true) (def ^:dynamic *object-reference-tracker* nil) diff --git a/test/libpython_clj/python_test.clj b/test/libpython_clj/python_test.clj index 4404c9f..08d0307 100644 --- a/test/libpython_clj/python_test.clj +++ b/test/libpython_clj/python_test.clj @@ -1,6 +1,5 @@ (ns libpython-clj.python-test (:require [libpython-clj.python :as py] - [libpython-clj.require :refer [require-python]] [tech.v2.datatype :as dtype] [tech.v2.datatype.functional :as dfn] [tech.v2.tensor :as dtt] @@ -11,6 +10,14 @@ (py/initialize!) +(defn crashit + [] + (let [test-data (py/->python (py/as-python [1 2])) + test-py-data (py/->py-tuple [test-data test-data]) + _ :ignored] + :ok)) + + (deftest stdout-and-stderr (is (= "hey\n" (with-out-str (py/run-simple-string "print('hey')")))) @@ -275,6 +282,5 @@ (is (= (vec (range 10)) ;;Range is an infinite sequence (-> (range) - (py/as-python) (islice 0 10) (vec)))))) From 164ce13ddc63ca5c5bc0c2d8d956658118f1a6b0 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 10 Dec 2019 15:04:58 -0700 Subject: [PATCH 056/456] Fixing really tough bridging errors. --- java/libpython_clj/jna/JVMBridge.java | 2 - src/libpython_clj/python.clj | 3 + src/libpython_clj/python/bridge.clj | 108 +++++++++++++---------- src/libpython_clj/python/interop.clj | 52 ++++------- src/libpython_clj/python/interpreter.clj | 16 ++-- src/libpython_clj/python/object.clj | 10 +-- 6 files changed, 96 insertions(+), 95 deletions(-) diff --git a/java/libpython_clj/jna/JVMBridge.java b/java/libpython_clj/jna/JVMBridge.java index f63ee0a..5906057 100644 --- a/java/libpython_clj/jna/JVMBridge.java +++ b/java/libpython_clj/jna/JVMBridge.java @@ -9,8 +9,6 @@ public interface JVMBridge extends AutoCloseable public Pointer getAttr(String name); public void setAttr(String name, Pointer val); public String[] dir(); - // Next has to have special treatment; in this case it is more manually wrapped. - public Object nextFn (); public Object interpreter(); public Object wrappedObject(); // Called from python when the python mirror is deleted. diff --git a/src/libpython_clj/python.clj b/src/libpython_clj/python.clj index a3a6864..d902be1 100644 --- a/src/libpython_clj/python.clj +++ b/src/libpython_clj/python.clj @@ -45,6 +45,8 @@ ->py-long ->py-string ->python + ;;Used when you are returning a value from a function. + ->python-incref ->jvm make-tuple-fn make-tuple-instance-fn @@ -118,6 +120,7 @@ afn as-jvm as-python + as-python-incref ;; Used when returning a value from a function to python. ->numpy as-numpy) diff --git a/src/libpython_clj/python/bridge.clj b/src/libpython_clj/python/bridge.clj index 114ba6c..8df49d1 100644 --- a/src/libpython_clj/python/bridge.clj +++ b/src/libpython_clj/python/bridge.clj @@ -34,7 +34,8 @@ ->py-tuple ->py-dict ->py-string - ->py-fn]] + ->py-fn] + :as pyobj] [libpython-clj.python.interop :as pyinterop :refer [expose-bridge-to-python! @@ -200,6 +201,24 @@ (->jvm pyobj)) +(defn- mostly-copy-arg + "This is the dirty boundary between the languages. Copying as often faster + for simple things but we have to be careful not to attempt a copy of things that + are only iterable (and not random access)." + [arg] + (cond + (jna/as-ptr arg) + (jna/as-ptr arg) + (instance? RandomAccess arg) + (->python arg) + (instance? Map arg) + (->python arg) + (instance? Iterable arg) + (as-python arg) + :else + (->python arg))) + + (defmacro bridge-pyobject [pyobj interpreter & body] `(let [pyobj# ~pyobj @@ -260,12 +279,14 @@ (py-proto/att-type-map pyobj#))) py-proto/PyCall (do-call-fn [callable# arglist# kw-arg-map#] - (-> (py-proto/do-call-fn pyobj# (mapv as-python arglist#) - (->> kw-arg-map# - (map (fn [[k# v#]] - [k# (as-python v#)])) - (into {}))) - (as-jvm))) + (with-interpreter interpreter# + (let [arglist# (mapv mostly-copy-arg arglist#) + kw-arg-map# (->> kw-arg-map# + (map (fn [[k# v#]] + [k# (mostly-copy-arg v#)])) + (into {}))] + (-> (py-proto/do-call-fn pyobj# arglist# kw-arg-map#) + (as-jvm))))) Object (toString [this#] (->jvm (py-proto/call-attr pyobj# "__str__"))) @@ -507,7 +528,8 @@ long-addr (get-attr ctypes "data") hash-ary {:ctypes-map ctypes} ptr-val (-> (Pointer. long-addr) - (resource/track #(get hash-ary :ctypes-map) [:gc]))] + (resource/track #(get hash-ary :ctypes-map) + pyobj/*pyobject-tracking-flags*))] {:ptr ptr-val :datatype np-dtype :shape shape @@ -682,47 +704,43 @@ (with-out-str (st/print-stack-trace e#))))))) -(defmacro impl-tuple-function - [& body] - `(reify CFunction$TupleFunction - (pyinvoke [this# ~'self ~'args] - (wrap-jvm-context - (-> (let [~'args (as-jvm ~'args)] - ~@body) - ;;as-python can create a bridge object or just a ptr - as-python - ;;convert any bridge objects to actual jna ptrs to match - ;;interface definitions - jna/->ptr-backing-store))))) - - -(defn jvm-fn->iface - [jvm-fn] - (impl-tuple-function - (apply jvm-fn args))) - - -(defn as-py-fn - [jvm-fn] - (-> (jvm-fn->iface jvm-fn) - ->py-fn)) - (defn jvm-iterator-as-python ^Pointer [^Iterator item] (with-gil nil (let [next-fn #(if (.hasNext item) (-> (.next item) + ;;As python tracks the object in a jvm context (as-python) - (jna/->ptr-backing-store)) + (jna/->ptr-backing-store) + ;;But we are handing the object back to python which is expecting + ;;a new reference. + (pyobj/incref)) (do (libpy/PyErr_SetNone (err/PyExc_StopIteration)) nil)) att-map - {"__next__" (->py-fn next-fn)}] - (create-bridge-from-att-map item att-map - :next-fn next-fn)))) + {"__next__" (pyobj/make-tuple-fn next-fn + :arg-converter nil + :result-converter nil)}] + (create-bridge-from-att-map item att-map)))) + + +(defn as-python-incref + "Convert to python and add a reference. Necessary for return values from + functions as python expects a new reference and the as-python pathway + ensures the jvm garbage collector also sees the reference." + [item] + (-> (as-python item) + (pyobj/incref))) + + +(defn- as-py-fn + ^Pointer [obj-fn] + (pyobj/make-tuple-fn obj-fn + :arg-converter as-jvm + :result-converter as-python-incref)) (defn jvm-map-as-python @@ -763,13 +781,13 @@ "append" (as-py-fn #(.add jvm-data %)) "insert" (as-py-fn #(.add jvm-data (int %1) %2)) "pop" (as-py-fn (fn [& args] - (let [index (int (if (first args) - (first args) - -1)) - index (if (< index 0) - (- (.size jvm-data) index) - index)] - #(.remove jvm-data index))))}] + (let [index (int (if (first args) + (first args) + -1)) + index (if (< index 0) + (- (.size jvm-data) index) + index)] + #(.remove jvm-data index))))}] (create-bridge-from-att-map jvm-data att-map)))) @@ -869,7 +887,7 @@ initial-buffer (->py-tuple shape) (->py-tuple strides))] - (resource/track retval #(get buffer-desc :ptr) [:gc])))) + (resource/track retval #(get buffer-desc :ptr) pyobj/*pyobject-tracking-flags*)))) (extend-type Object diff --git a/src/libpython_clj/python/interop.clj b/src/libpython_clj/python/interop.clj index 999326c..7aceb6d 100644 --- a/src/libpython_clj/python/interop.clj +++ b/src/libpython_clj/python/interop.clj @@ -215,7 +215,6 @@ (reify CFunction$tp_new (pyinvoke [this self varargs kw_args] (let [retval (libpy/PyType_GenericNew self varargs kw_args)] - (println retval) retval) ))) module-name (get-attr module "__name__") @@ -274,7 +273,7 @@ (defn pybridge->bridge ^JVMBridge [^Pointer pybridge] (let [bridge-type (JVMBridgeType. pybridge)] - (get-jvm-bridge (.jvm_handle bridge-type) + (get-jvm-bridge (Pointer/nativeValue pybridge) (.jvm_interpreter_handle bridge-type)))) @@ -311,7 +310,7 @@ e (with-out-str (st/print-stack-trace e)))))) - (unregister-bridge! bridge)) + (unregister-bridge! bridge self)) (catch Throwable e (log-error e))) (try @@ -343,17 +342,20 @@ 0)))) :tp_iter (reify CFunction$NoArgFunction (pyinvoke [this self] - (let [attr - (-> (pybridge->bridge self) - (.getAttr "__iter__"))] - (py-proto/call attr)))) - + (if-let [attr (-> (pybridge->bridge self) + (.getAttr "__iter__"))] + (libpy/PyObject_CallObject (jna/as-ptr attr) nil) + (do + (libpy/PyErr_SetNone (libpy/PyExc_Exception)) + nil)))) :tp_iternext (reify CFunction$NoArgFunction (pyinvoke [this self] - (when-let [next - (-> (pybridge->bridge self) - (.nextFn))] - (next))))})))) + (if-let [next (-> (pybridge->bridge self) + (.getAttr "__next__"))] + (libpy/PyObject_CallObject (jna/as-ptr next) nil) + (do + (libpy/PyErr_SetNone (libpy/PyExc_Exception)) + nil))))})))) (defn expose-bridge-to-python! @@ -368,23 +370,17 @@ (throw (ex-info "Failed to find bridge type" {}))) bridge-type (PyTypeObject. bridge-type-ptr) ^Pointer new-py-obj (libpy/_PyObject_New bridge-type) - pybridge (JVMBridgeType. new-py-obj) - ] - (println (format "Creating bridge: 0x%x -> %s:%s" - (Pointer/nativeValue new-py-obj) - (get-object-handle (.interpreter bridge)) - (get-object-handle (.wrappedObject bridge)))) + pybridge (JVMBridgeType. new-py-obj)] (set! (.jvm_interpreter_handle pybridge) (get-object-handle (.interpreter bridge))) (set! (.jvm_handle pybridge) (get-object-handle (.wrappedObject bridge))) (.write pybridge) - (register-bridge! bridge pybridge) - (-> (.getPointer pybridge) - wrap-pyobject)))) + (register-bridge! bridge new-py-obj) + (wrap-pyobject new-py-obj)))) (defn create-bridge-from-att-map - [src-item att-map & {:keys [next-fn]}] + [src-item att-map] (with-gil (let [interpreter (ensure-bound-interpreter) dir-data (->> (keys att-map) @@ -402,21 +398,11 @@ (dir [bridge] dir-data) (interpreter [bridge] interpreter) (wrappedObject [bridge] src-item) - (nextFn [bridge] next-fn) - libpy-base/PToPyObjectPtr - (->py-object-ptr [item] - (with-gil - (if-let [existing-bridge (find-jvm-bridge-entry - (get-object-handle src-item) - (ensure-bound-interpreter))] - (:pyobject existing-bridge) - (expose-bridge-to-python! item)))) py-proto/PCopyToJVM (->jvm [item options] src-item) py-proto/PBridgeToJVM (as-jvm [item options] src-item))] - (expose-bridge-to-python! bridge) - (libpy-base/->py-object-ptr bridge)))) + (expose-bridge-to-python! bridge)))) (defmethod py-proto/pyobject->jvm :jvm-bridge diff --git a/src/libpython_clj/python/interpreter.clj b/src/libpython_clj/python/interpreter.clj index 96e4d74..45bf054 100644 --- a/src/libpython_clj/python/interpreter.clj +++ b/src/libpython_clj/python/interpreter.clj @@ -86,28 +86,24 @@ (defn register-bridge! - [^JVMBridge bridge ^PyObject bridge-pyobject] + [^JVMBridge bridge ^Pointer bridge-pyobject] (let [interpreter (.interpreter bridge) - bridge-handle (get-object-handle (.wrappedObject bridge))] + bridge-handle (Pointer/nativeValue bridge-pyobject)] (when (contains? (get-in @(:interpreter-state* interpreter) [:bridge-objects]) bridge-handle) - (throw (Exception. (format "Bridge already exists!! - %s" bridge-handle)))) + (throw (Exception. (format "Bridge already exists!! - 0x%x" bridge-handle)))) (swap! (:interpreter-state* interpreter) assoc-in [:bridge-objects bridge-handle] {:jvm-bridge bridge - :pyobject bridge-pyobject}) - (println "ADDING-BRIDGE" bridge-handle - (keys (get-in @(:interpreter-state* interpreter) - [:bridge-objects]))) + :pyobject bridge-handle}) :ok)) (defn unregister-bridge! - [^JVMBridge bridge] + [^JVMBridge bridge ^Pointer bridge-pyobject] (let [interpreter (.interpreter bridge) - bridge-handle (get-object-handle (.wrappedObject bridge))] - (println "REMOVING-BRIDGE:" bridge-handle) + bridge-handle (Pointer/nativeValue bridge-pyobject)] (swap! (:interpreter-state* interpreter) update :bridge-objects dissoc bridge-handle) :ok)) diff --git a/src/libpython_clj/python/object.clj b/src/libpython_clj/python/object.clj index 4c230d4..b2c3456 100644 --- a/src/libpython_clj/python/object.clj +++ b/src/libpython_clj/python/object.clj @@ -95,7 +95,7 @@ (py-proto/->jvm item options))) -(def ^:dynamic *object-reference-logging* true) +(def ^:dynamic *object-reference-logging* false) (def ^:dynamic *object-reference-tracker* nil) @@ -427,14 +427,14 @@ :or {method-name "unnamed_function" documentation "not documented"}}] (with-gil - (let [py-self (or py-self (->python {}))] + ;;This is a nice little tidbit, cfunction_new + ;;steals the reference. + (let [py-self (when py-self (incref (->python py-self)))] (-> (libpy/PyCFunction_New (method-def-data->method-def {:name method-name :doc documentation :function cfunc}) - ;;This is a nice little tidbit, cfunction_new - ;;steals the reference. - (libpy/Py_IncRef py-self)) + py-self) (wrap-pyobject))))) From 620966bca9b8ed662cd4589b730c5cb6c4e21b9d Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Wed, 11 Dec 2019 09:40:18 -0700 Subject: [PATCH 057/456] 1.21 --- CHANGELOG.md | 11 +++++++++++ project.clj | 2 +- src/libpython_clj/python/bridge.clj | 2 +- test/libpython_clj/python_test.clj | 10 ++++++++++ 4 files changed, 23 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c8f0075..9f2a799 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Time for a ChangeLog! +## 1.21 + +Bugfix release. Passing infinite sequences to python functions was +causing a hang as libpython-clj attempted to copy the sequence. The +current calling convention does a shallow copy of things that are list-like +or map-like, while bridging things that are iterable or don't fall into +the above categories. + +This exposed a bug that caused reference counting to be subtly wrong when +python iterated through a bridged object. And that was my life for a day. + ## 1.20 With two many huge things we had to skip a few versions! diff --git a/project.clj b/project.clj index 850ae23..cc2287a 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.21-SNAPSHOT" +(defproject cnuernber/libpython-clj "1.21" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" diff --git a/src/libpython_clj/python/bridge.clj b/src/libpython_clj/python/bridge.clj index 8df49d1..e8b1993 100644 --- a/src/libpython_clj/python/bridge.clj +++ b/src/libpython_clj/python/bridge.clj @@ -216,7 +216,7 @@ (instance? Iterable arg) (as-python arg) :else - (->python arg))) + (as-python arg))) (defmacro bridge-pyobject diff --git a/test/libpython_clj/python_test.clj b/test/libpython_clj/python_test.clj index 08d0307..a3f8401 100644 --- a/test/libpython_clj/python_test.clj +++ b/test/libpython_clj/python_test.clj @@ -284,3 +284,13 @@ (-> (range) (islice 0 10) (vec)))))) + + +(deftest persistent-vector-nparray + (testing "Create numpy array from nested persistent vectors" + (let [ary-data (-> (py/import-module "numpy") + (py/$a array [[1 2 3] + [4 5 6]]))] + (is (dfn/equals (dtt/->tensor [[1 2 3] + [4 5 6]]) + (py/as-tensor ary-data)))))) From da45409906ada252521968193bb0c9a2b705fd09 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Wed, 11 Dec 2019 09:40:34 -0700 Subject: [PATCH 058/456] snap --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index cc2287a..609e296 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.21" +(defproject cnuernber/libpython-clj "1.22-SNAPSHOT" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" From 711a15bd1df54ff01150c059768dc20842ef262a Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Wed, 11 Dec 2019 11:07:40 -0700 Subject: [PATCH 059/456] editing --- README.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e69b4de..3cde429 100644 --- a/README.md +++ b/README.md @@ -54,13 +54,22 @@ This is important to realize in that the code below doesn't look like python bec referencing the item and attribute systems by name and not via '.' or '[]'. +### Installation + +#### Ubuntu + ```console -sudo apt install libpython3.6-dev +sudo apt install libpython3.6 # numpy and pandas are required for unit tests. Numpy is required for # zero copy support. python3.6 -m pip install numpy pandas --user ``` +#### MacOSX + +Python installation instructions [here](https://docs.python-guide.org/starting/install3/osx/). + + ### Initialize python From 12e07fe94047260710561912a54a2a610122dc4b Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Wed, 11 Dec 2019 16:20:49 -0700 Subject: [PATCH 060/456] Some conda exploration. --- dockerfiles/Dockerfile | 34 ++++++++++++++++++++++++++++++++++ scripts/build-docker | 7 +++++++ scripts/docker-repl | 10 ++++++++++ 3 files changed, 51 insertions(+) create mode 100644 dockerfiles/Dockerfile create mode 100755 scripts/build-docker create mode 100755 scripts/docker-repl diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile new file mode 100644 index 0000000..e7a3411 --- /dev/null +++ b/dockerfiles/Dockerfile @@ -0,0 +1,34 @@ +# We will use Ubuntu for our image +FROM ubuntu:latest + +# Updating Ubuntu packages + + +RUN apt-get -qq update && apt-get -qq -y install curl wget bzip2 openjdk-8-jdk-headless \ + && curl -sSL https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o /tmp/miniconda.sh \ + && bash /tmp/miniconda.sh -bfp /usr/local \ + && rm -rf /tmp/miniconda.sh \ + && conda install -y python=3 \ + && conda update conda \ + && curl -O https://download.clojure.org/install/linux-install-1.10.1.492.sh \ + && chmod +x linux-install-1.10.1.492.sh \ + && ./linux-install-1.10.1.492.sh && rm linux-install-1.10.1.492.sh \ + && wget https://raw.githubusercontent.com/technomancy/leiningen/stable/bin/lein \ + && chmod a+x lein \ + && mv lein /usr/bin \ + && apt-get -qq -y autoremove \ + && apt-get autoclean \ + && rm -rf /var/lib/apt/lists/* /var/log/dpkg.log \ + && conda clean --all --yes + + +ENV PATH /opt/conda/bin:$PATH + + +ARG USERID +ARG GROUPID +ARG USERNAME + +RUN groupadd -g $GROUPID $USERNAME +RUN useradd -u $USERID -g $GROUPID $USERNAME +RUN mkdir /home/$USERNAME && chown $USERNAME:$USERNAME /home/$USERNAME \ No newline at end of file diff --git a/scripts/build-docker b/scripts/build-docker new file mode 100755 index 0000000..5a1da08 --- /dev/null +++ b/scripts/build-docker @@ -0,0 +1,7 @@ +#!/bin/bash + +set -e + +pushd dockerfiles +docker build -t docker-conda -f Dockerfile --build-arg USERID=$(id -u) --build-arg GROUPID=$(id -u) --build-arg USERNAME=$USER . +popd diff --git a/scripts/docker-repl b/scripts/docker-repl new file mode 100755 index 0000000..508fde7 --- /dev/null +++ b/scripts/docker-repl @@ -0,0 +1,10 @@ +#!/bin/bash + + +docker run --rm -it -u $(id -u):$(id -g) \ + -e LEIN_REPL_HOST="0.0.0.0" \ + -v /$HOME/.m2:/home/$USER/.m2 \ + -v /$HOME/.lein:/home/$USER/.lein \ + -v $(pwd)/:/libpython-clj \ + --net=host -w /libpython-clj \ + docker-conda $@ From 12b23f24b96f1d5ebf98cc316e57e6164b90877c Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Wed, 11 Dec 2019 16:48:41 -0700 Subject: [PATCH 061/456] Small updates. --- scripts/docker-repl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/docker-repl b/scripts/docker-repl index 508fde7..c41dddb 100755 --- a/scripts/docker-repl +++ b/scripts/docker-repl @@ -1,5 +1,8 @@ #!/bin/bash +## This is incomplete, you still have to mess around with pythonpath and such. The +## goal at this stage is to get this docker container to launch a repl that allows +## initialize! to work out of the box. docker run --rm -it -u $(id -u):$(id -g) \ -e LEIN_REPL_HOST="0.0.0.0" \ @@ -8,3 +11,7 @@ docker run --rm -it -u $(id -u):$(id -g) \ -v $(pwd)/:/libpython-clj \ --net=host -w /libpython-clj \ docker-conda $@ + + # docker-conda lein update-in :dependencies conj \[nrepl\ \"0.6.0\"\]\ + # -- update-in :plugins conj \[cider/cider-nrepl\ \"0.22.4\"\]\ + # -- repl :headless :host localhost From 6df574d4c925e2ec9708ee253b9fafcc534f8a96 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Thu, 12 Dec 2019 08:17:36 -0700 Subject: [PATCH 062/456] Checking on a dockerfile that builds conda and libpython-clj. --- dockerfiles/{Dockerfile => CondaDockerfile} | 0 scripts/build-conda-docker | 7 +++++++ scripts/build-docker | 7 ------- scripts/{docker-repl => run-conda-docker} | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) rename dockerfiles/{Dockerfile => CondaDockerfile} (100%) create mode 100755 scripts/build-conda-docker delete mode 100755 scripts/build-docker rename scripts/{docker-repl => run-conda-docker} (96%) diff --git a/dockerfiles/Dockerfile b/dockerfiles/CondaDockerfile similarity index 100% rename from dockerfiles/Dockerfile rename to dockerfiles/CondaDockerfile diff --git a/scripts/build-conda-docker b/scripts/build-conda-docker new file mode 100755 index 0000000..d323d22 --- /dev/null +++ b/scripts/build-conda-docker @@ -0,0 +1,7 @@ +#!/bin/bash + +set -e + +pushd dockerfiles +docker build -t docker-conda -f CondaDockerfile --build-arg USERID=$(id -u) --build-arg GROUPID=$(id -u) --build-arg USERNAME=$USER . +popd diff --git a/scripts/build-docker b/scripts/build-docker deleted file mode 100755 index 5a1da08..0000000 --- a/scripts/build-docker +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -set -e - -pushd dockerfiles -docker build -t docker-conda -f Dockerfile --build-arg USERID=$(id -u) --build-arg GROUPID=$(id -u) --build-arg USERNAME=$USER . -popd diff --git a/scripts/docker-repl b/scripts/run-conda-docker similarity index 96% rename from scripts/docker-repl rename to scripts/run-conda-docker index c41dddb..848f691 100755 --- a/scripts/docker-repl +++ b/scripts/run-conda-docker @@ -10,7 +10,7 @@ docker run --rm -it -u $(id -u):$(id -g) \ -v /$HOME/.lein:/home/$USER/.lein \ -v $(pwd)/:/libpython-clj \ --net=host -w /libpython-clj \ - docker-conda $@ + docker-conda /bin/bash # docker-conda lein update-in :dependencies conj \[nrepl\ \"0.6.0\"\]\ # -- update-in :plugins conj \[cider/cider-nrepl\ \"0.22.4\"\]\ From cf14225817d62303bdf6d4e43d1bf0bf94e49db6 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Thu, 12 Dec 2019 10:08:32 -0700 Subject: [PATCH 063/456] Working through new knowledge and new apis --- dockerfiles/CondaDockerfile | 4 +- src/libpython_clj/jna.clj | 4 +- src/libpython_clj/jna/interpreter.clj | 32 ++++++++++++ src/libpython_clj/python/interpreter.clj | 62 ++++++++++++++++++++++-- 4 files changed, 95 insertions(+), 7 deletions(-) diff --git a/dockerfiles/CondaDockerfile b/dockerfiles/CondaDockerfile index e7a3411..8f976e9 100644 --- a/dockerfiles/CondaDockerfile +++ b/dockerfiles/CondaDockerfile @@ -31,4 +31,6 @@ ARG USERNAME RUN groupadd -g $GROUPID $USERNAME RUN useradd -u $USERID -g $GROUPID $USERNAME -RUN mkdir /home/$USERNAME && chown $USERNAME:$USERNAME /home/$USERNAME \ No newline at end of file +RUN mkdir /home/$USERNAME && chown $USERNAME:$USERNAME /home/$USERNAME +USER $USERNAME +RUN conda create -n pyclj python=3.6 && echo "source activate pyclj" > /home/$USERNAME/.bashrc \ No newline at end of file diff --git a/src/libpython_clj/jna.clj b/src/libpython_clj/jna.clj index 04b42ec..0a5df41 100644 --- a/src/libpython_clj/jna.clj +++ b/src/libpython_clj/jna.clj @@ -38,11 +38,13 @@ (export-symbols libpython-clj.jna.interpreter + PySys_SetArgv + Py_SetProgramName + Py_SetPythonHome Py_InitializeEx Py_IsInitialized Py_FinalizeEx PyRun_SimpleString - PySys_SetArgv get-type-name lookup-type-symbols Py_None diff --git a/src/libpython_clj/jna/interpreter.clj b/src/libpython_clj/jna/interpreter.clj index 07cdddb..9a6bacc 100644 --- a/src/libpython_clj/jna/interpreter.clj +++ b/src/libpython_clj/jna/interpreter.clj @@ -18,6 +18,38 @@ [libpython_clj.jna PyObject])) + +(def-no-gil-pylib-fn Py_SetProgramName + "This function should be called before Py_Initialize() is called for the first time, + if it is called at all. It tells the interpreter the value of the argv[0] argument to + the main() function of the program (converted to wide characters). This is used by + Py_GetPath() and some other functions below to find the Python run-time libraries + relative to the interpreter executable. The default value is 'python'. The argument + should point to a zero-terminated wide character string in static storage whose + contents will not change for the duration of the program’s execution. No code in the + Python interpreter will change the contents of this storage. + + Use Py_DecodeLocale() to decode a bytes string to get a wchar_* string." + nil + [name jna/ensure-ptr]) + + + + +(def-no-gil-pylib-fn Py_SetPythonHome + "Set the default “home” directory, that is, the location of the standard Python + libraries. See PYTHONHOME for the meaning of the argument string. + + The argument should point to a zero-terminated character string in static storage + whose contents will not change for the duration of the program’s execution. No code + in the Python interpreter will change the contents of this storage. + + Use Py_DecodeLocale() to decode a bytes string to get a wchar_* string." + nil + [home jna/ensure-ptr]) + + + ;; Bugs and caveats: The destruction of modules and objects in modules is done in random ;; order; this may cause destructors (__del__() methods) to fail when they depend on ;; other objects (even functions) or modules. Dynamically loaded extension modules diff --git a/src/libpython_clj/python/interpreter.clj b/src/libpython_clj/python/interpreter.clj index 45bf054..cee8926 100644 --- a/src/libpython_clj/python/interpreter.clj +++ b/src/libpython_clj/python/interpreter.clj @@ -4,7 +4,9 @@ [tech.resource :as resource] [libpython-clj.python.logging :refer [log-error log-warn log-info]] - [tech.jna :as jna]) + [tech.jna :as jna] + [clojure.java.shell :as sh] + [clojure.string :as s]) (:import [libpython_clj.jna JVMBridge PyObject] @@ -241,6 +243,9 @@ (defonce ^:dynamic *program-name* "") + + + (defn- try-load-python-library! [libname] (try @@ -251,13 +256,60 @@ (catch Exception e))) +(defn- ignore-shell-errors + [& args] + (try + (apply sh/sh args) + (catch Throwable e nil))) + + +(defn- find-python-home + [& [python-home]] + (cond + python-home + python-home + (seq (System/getenv "PYTHONHOME")) + (System/getenv "PYTHONHOME") + :else + (let [{:keys [out err exit]} (ignore-shell-errors "python3-config" "--prefix")] + (when (= 0 exit) + (s/trimr out))))) + + +(defn- find-python-lib-version + [] + (let [{:keys [out err exit]} (ignore-shell-errors "python3" "--version")] + (when (= 0 exit) + (when-let [parts (->> (s/split out #"\.") + (take 2) + seq)] + (let [^String first-part (first parts) + first-part (if (.startsWith first-part "Python ") + (.substring first-part (count "Python ")) + first-part)] + (s/join "." (concat [first-part] (rest parts)))))))) + + +(defn set-java-library-path! + [python-home] + ) + + (defn initialize! - [& [program-name python-library-name]] + [& {:keys [program-name + python-library-name + python-home]}] (when-not @*main-interpreter* (log-info "executing python initialize!") - (let [user-names (when python-library-name - [python-library-name]) - library-names (or user-names (libpy-base/library-names))] + (let [python-home (find-python-home python-home) + lib-version (find-python-lib-version) + library-names (if python-library-name + [python-library-name] + (concat + [(str "python" lib-version "m")] + (libpy-base/library-names)))] + (when python-home + (set-java-library-path! python-home)) (loop [[library-name & library-names] library-names] (if (and library-name (not (try-load-python-library! library-name))) From 2b90f9f0fbf6efc41a2c90f2049ff1531f021273 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Thu, 12 Dec 2019 11:48:32 -0700 Subject: [PATCH 064/456] Upgrades to enable python to run more reliably. --- dockerfiles/CondaDockerfile | 3 +- scripts/run-conda-docker | 7 +- src/libpython_clj/python.clj | 9 +- src/libpython_clj/python/interpreter.clj | 109 +++++++++++++++-------- 4 files changed, 86 insertions(+), 42 deletions(-) diff --git a/dockerfiles/CondaDockerfile b/dockerfiles/CondaDockerfile index 8f976e9..038c986 100644 --- a/dockerfiles/CondaDockerfile +++ b/dockerfiles/CondaDockerfile @@ -33,4 +33,5 @@ RUN groupadd -g $GROUPID $USERNAME RUN useradd -u $USERID -g $GROUPID $USERNAME RUN mkdir /home/$USERNAME && chown $USERNAME:$USERNAME /home/$USERNAME USER $USERNAME -RUN conda create -n pyclj python=3.6 && echo "source activate pyclj" > /home/$USERNAME/.bashrc \ No newline at end of file +RUN conda create -n pyclj python=3.6 && conda install -n pyclj numpy mxnet\ + && echo "source activate pyclj" > /home/$USERNAME/.bashrc \ No newline at end of file diff --git a/scripts/run-conda-docker b/scripts/run-conda-docker index 848f691..d193f96 100755 --- a/scripts/run-conda-docker +++ b/scripts/run-conda-docker @@ -11,7 +11,6 @@ docker run --rm -it -u $(id -u):$(id -g) \ -v $(pwd)/:/libpython-clj \ --net=host -w /libpython-clj \ docker-conda /bin/bash - - # docker-conda lein update-in :dependencies conj \[nrepl\ \"0.6.0\"\]\ - # -- update-in :plugins conj \[cider/cider-nrepl\ \"0.22.4\"\]\ - # -- repl :headless :host localhost + # lein update-in :dependencies conj \[nrepl\ \"0.6.0\"\]\ + # -- update-in :plugins conj \[cider/cider-nrepl\ \"0.22.4\"\]\ + # -- repl :headless :host localhost diff --git a/src/libpython_clj/python.clj b/src/libpython_clj/python.clj index d902be1..35a6d0d 100644 --- a/src/libpython_clj/python.clj +++ b/src/libpython_clj/python.clj @@ -226,9 +226,14 @@ :program-name - optional but will show up in error messages from python. :no-io-redirect - there if you don't want python stdout and stderr redirection to *out* and *err*." - [& {:keys [program-name no-io-redirect? library-path]}] + [& {:keys [program-name + library-path + python-home + no-io-redirect?]}] (when-not @pyinterp/*main-interpreter* - (pyinterp/initialize! program-name library-path) + (pyinterp/initialize! :program-name program-name + :library-path library-path + :python-home python-home) ;;setup bridge mechansim and io redirection (pyinterop/register-bridge-type!) (when-not no-io-redirect? diff --git a/src/libpython_clj/python/interpreter.clj b/src/libpython_clj/python/interpreter.clj index cee8926..ba4d386 100644 --- a/src/libpython_clj/python/interpreter.clj +++ b/src/libpython_clj/python/interpreter.clj @@ -6,13 +6,15 @@ :refer [log-error log-warn log-info]] [tech.jna :as jna] [clojure.java.shell :as sh] - [clojure.string :as s]) + [clojure.string :as s] + [clojure.tools.logging :as log]) (:import [libpython_clj.jna JVMBridge PyObject] [com.sun.jna Pointer] [com.sun.jna.ptr PointerByReference] - [java.io StringWriter])) + [java.io StringWriter] + [java.nio.file Paths Path])) (set! *warn-on-reflection* true) @@ -245,17 +247,6 @@ - -(defn- try-load-python-library! - [libname] - (try - (jna/load-library libname) - (alter-var-root #'libpy-base/*python-library* (constantly libname)) - (libpy/Py_InitializeEx 0) - libname - (catch Exception e))) - - (defn- ignore-shell-errors [& args] (try @@ -280,39 +271,87 @@ [] (let [{:keys [out err exit]} (ignore-shell-errors "python3" "--version")] (when (= 0 exit) - (when-let [parts (->> (s/split out #"\.") - (take 2) - seq)] - (let [^String first-part (first parts) - first-part (if (.startsWith first-part "Python ") - (.substring first-part (count "Python ")) - first-part)] - (s/join "." (concat [first-part] (rest parts)))))))) + ;;Anaconda prints version info only to the error stream. + (when-let [version-info (first (filter seq [out err]))] + (log/infof "Python detected: %s" version-info) + (let [version-data (re-find #"\d+\.\d+\.\d+" version-info) + parts (->> (s/split version-data #"\.") + (take 2) + seq)] + (s/join "." parts)))))) + + +(defn append-java-library-path! + [new-search-path] + (let [existing-paths (-> (System/getProperty "java.library.path") + (s/split #":"))] + (when-not (contains? (set existing-paths) new-search-path) + (let [new-path-str (s/join ":" (concat [new-search-path] + existing-paths))] + (log/infof "Setting java library path: %s" new-path-str) + (System/setProperty "java.library.path" new-path-str))))) + + +(defonce ^:private python-home-wide-ptr* (atom nil)) + + +(defn- try-load-python-library! + [libname python-home-wide-ptr] + (try + (jna/load-library libname) + (alter-var-root #'libpy-base/*python-library* (constantly libname)) + (when python-home-wide-ptr + (libpy/Py_SetPythonHome python-home-wide-ptr)) + (libpy/Py_InitializeEx 0) + libname + (catch Exception e))) -(defn set-java-library-path! - [python-home] - ) +(defn- detect-startup-info + [{:keys [library-path python-home]}] + (let [python-home (find-python-home python-home) + java-library-path-addendum + (when python-home + (-> (Paths/get python-home (into-array String ["lib"])) + (.toString))) + lib-version (find-python-lib-version) + libname (when (seq lib-version) + (str "python" lib-version "m")) + retval + {:python-home python-home + :lib-version lib-version + :libname libname + :java-library-path-addendum java-library-path-addendum}] + (log/infof "Startup info detected: %s" retval) + retval)) (defn initialize! [& {:keys [program-name - python-library-name - python-home]}] + library-path + python-home] + :as options}] (when-not @*main-interpreter* - (log-info "executing python initialize!") - (let [python-home (find-python-home python-home) - lib-version (find-python-lib-version) - library-names (if python-library-name - [python-library-name] + (log-info "Executing python initialize!") + (let [{:keys [python-home libname java-library-path-addendum] :as startup-info} + (detect-startup-info options) + library-names (cond + library-path + [library-path] + libname (concat - [(str "python" lib-version "m")] - (libpy-base/library-names)))] + [libname] + (libpy-base/library-names)) + :else + (libpy-base/library-names))] + (reset! python-home-wide-ptr* nil) (when python-home - (set-java-library-path! python-home)) + (append-java-library-path! java-library-path-addendum) + ;;This can never be released if load-library succeeeds + (reset! python-home-wide-ptr* (jna/string->wide-ptr python-home))) (loop [[library-name & library-names] library-names] (if (and library-name - (not (try-load-python-library! library-name))) + (not (try-load-python-library! library-name @python-home-wide-ptr*))) (recur library-names)))) ;;Set program name (when-let [program-name (or program-name *program-name* "")] From 03787e789c8fb9b8d4a2e3f1dc3936a4be41dedf Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Thu, 12 Dec 2019 11:53:13 -0700 Subject: [PATCH 065/456] 1.22 --- CHANGELOG.md | 9 +++++++++ project.clj | 2 +- src/libpython_clj/python/interpreter.clj | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f2a799..528b96a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Time for a ChangeLog! +## 1.22 + +Working to make more python environments work out of the box. Currently have a +testcase for conda working in a clean install of a docker container. There is now a +new method: `libpython-clj.python.interpreter/detect-startup-info` that attempts +call python3-config and python3 --version in order to automagically configure the +python library. + + ## 1.21 Bugfix release. Passing infinite sequences to python functions was diff --git a/project.clj b/project.clj index 609e296..dba79b2 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.22-SNAPSHOT" +(defproject cnuernber/libpython-clj "1.22" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" diff --git a/src/libpython_clj/python/interpreter.clj b/src/libpython_clj/python/interpreter.clj index ba4d386..484da8b 100644 --- a/src/libpython_clj/python/interpreter.clj +++ b/src/libpython_clj/python/interpreter.clj @@ -307,7 +307,7 @@ (catch Exception e))) -(defn- detect-startup-info +(defn detect-startup-info [{:keys [library-path python-home]}] (let [python-home (find-python-home python-home) java-library-path-addendum From 94dbef6bd61c4bb48ea2d9c83603a9c1f6c8e88d Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Thu, 12 Dec 2019 11:53:23 -0700 Subject: [PATCH 066/456] snap --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index dba79b2..ca4dfbf 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.22" +(defproject cnuernber/libpython-clj "1.23-SNAPSHOT" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" From 015af9c24f1a7ed0b4de6fdda809828ef47a5e72 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Thu, 12 Dec 2019 14:37:40 -0700 Subject: [PATCH 067/456] Equality/hash/better tostring for types. --- CHANGELOG.md | 46 +++++++++++++++++++++- src/libpython_clj/jna.clj | 1 + src/libpython_clj/jna/protocols/object.clj | 28 ++++++++++++- src/libpython_clj/python.clj | 5 ++- src/libpython_clj/python/bridge.clj | 11 +++++- src/libpython_clj/python/interop.clj | 4 -- src/libpython_clj/python/object.clj | 25 ++++++++++++ test/libpython_clj/python_test.clj | 20 ++++++---- 8 files changed, 123 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 528b96a..0700fb4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,54 @@ # Time for a ChangeLog! + +## 1.23-SNAPSHOT + + +Equals, hashcode, nice default .toString of python types: + +```clojure +user> (require '[libpython-clj.python :as py]) +nil +user> (def test-tuple (py/->py-tuple [1 2])) +#'user/test-tuple +user> (require '[libpython-clj.require :refer [require-python]]) +nil +user> (require-python '[builtins :as bt]) +nil +user> (bt/type test-tuple) +builtins.tuple +user> test-tuple +(1, 2) +user> (def new-tuple (py/->py-tuple [3 4])) +#'user/new-tuple +user> (= test-tuple new-tuple) +false +user> (= test-tuple (py/->py-tuple [1 2])) +true +user> (.hashCode test-tuple) +2130570162 +user> (.hashCode (py/->py-tuple [1 2])) +2130570162 +user> (require-python '[numpy :as np]) +nil +user> (def np-ary (np/array [1 2 3])) +#'user/np-ary +user> np-ary +[1 2 3] +user> (bt/type np-ary) +numpy.ndarray +user> (py/python-type *1) +:type +``` + + ## 1.22 Working to make more python environments work out of the box. Currently have a testcase for conda working in a clean install of a docker container. There is now a new method: `libpython-clj.python.interpreter/detect-startup-info` that attempts -call python3-config and python3 --version in order to automagically configure the -python library. +call `python3-config --prefix` and `python3 --version` in order to automagically +configure the python library. ## 1.21 diff --git a/src/libpython_clj/jna.clj b/src/libpython_clj/jna.clj index 0a5df41..2121f5d 100644 --- a/src/libpython_clj/jna.clj +++ b/src/libpython_clj/jna.clj @@ -96,6 +96,7 @@ PyObject_Call PyObject_CallObject PyObject_Hash + PyObject_IsInstance PyObject_IsTrue PyObject_Not PyObject_Length diff --git a/src/libpython_clj/jna/protocols/object.clj b/src/libpython_clj/jna/protocols/object.clj index d83d8b2..e2f9691 100644 --- a/src/libpython_clj/jna/protocols/object.clj +++ b/src/libpython_clj/jna/protocols/object.clj @@ -42,6 +42,7 @@ keyword) v])) (into {}))) +(def bool-fn-value-set (set (vals bool-fn-table))) (defn bool-fn-constant @@ -51,7 +52,7 @@ (long item) (keyword? item) (get bool-fn-table item)) - value-set (set (vals bool-fn-table))] + value-set bool-fn-value-set] (when-not (contains? value-set value) (throw (ex-info (format "Unrecognized bool fn %s" item) {}))) (int value))) @@ -309,10 +310,33 @@ Changed in version 3.2: The return type is now Py_hash_t. This is a signed integer the same size as Py_ssize_t." - (size-t-type) + size-t-type [o ensure-pyobj]) +(def-pylib-fn PyObject_IsInstance + "Return 1 if inst is an instance of the class cls or a subclass of cls, or 0 if + not. On error, returns -1 and sets an exception. + + If cls is a tuple, the check will be done against every entry in cls. The result + will be 1 when at least one of the checks returns 1, otherwise it will be 0. + + If cls has a __instancecheck__() method, it will be called to determine the + subclass status as described in PEP 3119. Otherwise, inst is an instance of cls + if its class is a subclass of cls. + + An instance inst can override what is considered its class by having a __class__ + attribute. + + An object cls can override if it is considered a class, and what its base + classes are, by having a __bases__ attribute (which must be a tuple of base + classes)." + Integer + [inst ensure-pyobj] + [cls ensure-pyobj]) + + + (def-pylib-fn PyObject_IsTrue "Returns 1 if the object o is considered to be true, and 0 otherwise. This is equivalent to the Python expression not not o. On failure, return -1." diff --git a/src/libpython_clj/python.clj b/src/libpython_clj/python.clj index 35a6d0d..e33c5d7 100644 --- a/src/libpython_clj/python.clj +++ b/src/libpython_clj/python.clj @@ -50,7 +50,10 @@ ->jvm make-tuple-fn make-tuple-instance-fn - create-class) + create-class + is-instance? + hash-code + equals?) (defmacro stack-resource-context diff --git a/src/libpython_clj/python/bridge.clj b/src/libpython_clj/python/bridge.clj index e8b1993..afa532c 100644 --- a/src/libpython_clj/python/bridge.clj +++ b/src/libpython_clj/python/bridge.clj @@ -289,7 +289,16 @@ (as-jvm))))) Object (toString [this#] - (->jvm (py-proto/call-attr pyobj# "__str__"))) + (with-interpreter interpreter# + (if (= 1 (libpy/PyObject_IsInstance pyobj# (libpy/PyType_Type))) + (format "%s.%s" + (->jvm (py-proto/get-attr pyobj# "__module__")) + (->jvm (py-proto/get-attr pyobj# "__name__"))) + (->jvm (py-proto/call-attr pyobj# "__str__"))))) + (equals [this# other#] + (pyobj/equals? this# other#)) + (hashCode [this#] + (.hashCode ^Object (pyobj/hash-code this#))) ~@body) {:type :pyobject}))) diff --git a/src/libpython_clj/python/interop.clj b/src/libpython_clj/python/interop.clj index 7aceb6d..935eed6 100644 --- a/src/libpython_clj/python/interop.clj +++ b/src/libpython_clj/python/interop.clj @@ -166,10 +166,6 @@ (into-array Object [(str fieldname)]))) -(defn create-python-type - [type-name module-name method-def-data]) - - (defn method-def-data-seq->method-def-ref ^PyMethodDef$ByReference [method-def-data-seq] (when (seq method-def-data-seq) diff --git a/src/libpython_clj/python/object.clj b/src/libpython_clj/python/object.clj index b2c3456..ecf7e42 100644 --- a/src/libpython_clj/python/object.clj +++ b/src/libpython_clj/python/object.clj @@ -1052,3 +1052,28 @@ :else {:type (python-type pyobj) :value (Pointer/nativeValue (jna/as-ptr pyobj))})) + + +(defn is-instance? + "Returns true if inst is an instance of type. + False otherwise." + [py-inst py-type] + (with-gil + (= 1 (libpy/PyObject_IsInstance (->python py-inst) + ;;The type has to be a python type already. + py-type)))) + + +(defn hash-code + ^long [py-inst] + (with-gil + (long (libpy/PyObject_Hash (->python py-inst))))) + + +(defn equals? + "Returns true of the python equals operator returns 1." + [lhs rhs] + (with-gil + (= 1 (libpy/PyObject_RichCompareBool (->python lhs) + (->python rhs) + :py-eq)))) diff --git a/test/libpython_clj/python_test.clj b/test/libpython_clj/python_test.clj index a3f8401..28259ee 100644 --- a/test/libpython_clj/python_test.clj +++ b/test/libpython_clj/python_test.clj @@ -10,13 +10,6 @@ (py/initialize!) -(defn crashit - [] - (let [test-data (py/->python (py/as-python [1 2])) - test-py-data (py/->py-tuple [test-data test-data]) - _ :ignored] - :ok)) - (deftest stdout-and-stderr (is (= "hey\n" (with-out-str @@ -294,3 +287,16 @@ (is (dfn/equals (dtt/->tensor [[1 2 3] [4 5 6]]) (py/as-tensor ary-data)))))) + + +(deftest python-tuple-equals + (testing "Python tuples have nice equal semantics." + (let [lhs (py/->py-tuple [1 2]) + same (py/->py-tuple [1 2]) + not-same (py/->py-tuple [3 4])] + (is (= lhs same)) + (is (not= lhs not-same)) + (is (= (.hashCode lhs) + (.hashCode same))) + (is (not= (.hashCode lhs) + (.hashCode not-same)))))) From b9941fb571d975fa20e153429834dd5236d025e1 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Thu, 12 Dec 2019 14:38:38 -0700 Subject: [PATCH 068/456] 1.23 --- CHANGELOG.md | 2 +- project.clj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0700fb4..f897c68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Time for a ChangeLog! -## 1.23-SNAPSHOT +## 1.23 Equals, hashcode, nice default .toString of python types: diff --git a/project.clj b/project.clj index ca4dfbf..24f3e3e 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.23-SNAPSHOT" +(defproject cnuernber/libpython-clj "1.23" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" From 669ecca619e8b38a3d0a3a36a739ed0474bdbf8b Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Thu, 12 Dec 2019 14:38:45 -0700 Subject: [PATCH 069/456] snap --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 24f3e3e..2df8bf0 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.23" +(defproject cnuernber/libpython-clj "1.24-SNAPSHOT" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" From a2ec5110b1c798b891815a77c98c5bb5e10cdbf9 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Fri, 13 Dec 2019 08:57:20 -0700 Subject: [PATCH 070/456] 1.24 --- project.clj | 4 ++-- src/libpython_clj/python/bridge.clj | 2 +- test/libpython_clj/python_test.clj | 7 +++++++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/project.clj b/project.clj index 2df8bf0..f5b7371 100644 --- a/project.clj +++ b/project.clj @@ -1,10 +1,10 @@ -(defproject cnuernber/libpython-clj "1.24-SNAPSHOT" +(defproject cnuernber/libpython-clj "1.24" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" :url "https://www.eclipse.org/legal/epl-2.0/"} :dependencies [[org.clojure/clojure "1.10.1"] - [techascent/tech.datatype "4.56"] + [techascent/tech.datatype "4.61"] [camel-snake-kebab "0.4.0"]] :repl-options {:init-ns user} :java-source-paths ["java"]) diff --git a/src/libpython_clj/python/bridge.clj b/src/libpython_clj/python/bridge.clj index afa532c..9dd4e82 100644 --- a/src/libpython_clj/python/bridge.clj +++ b/src/libpython_clj/python/bridge.clj @@ -209,7 +209,7 @@ (cond (jna/as-ptr arg) (jna/as-ptr arg) - (instance? RandomAccess arg) + (dtype/reader? arg) (->python arg) (instance? Map arg) (->python arg) diff --git a/test/libpython_clj/python_test.clj b/test/libpython_clj/python_test.clj index 28259ee..306e1cb 100644 --- a/test/libpython_clj/python_test.clj +++ b/test/libpython_clj/python_test.clj @@ -300,3 +300,10 @@ (.hashCode same))) (is (not= (.hashCode lhs) (.hashCode not-same)))))) + + +(deftest range-nparray + (let [ary-data (-> (py/import-module "numpy") + (py/$a array (range 10)))] + (is (dfn/equals (dtt/->tensor (range 10)) + (py/as-tensor ary-data))))) From 5a936943a6be54488a66b7607db04dcb5b2948b8 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Fri, 13 Dec 2019 08:57:27 -0700 Subject: [PATCH 071/456] snap --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index f5b7371..6b3f82a 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.24" +(defproject cnuernber/libpython-clj "1.25-SNAPSHOT" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" From 3bf1ec9d819dc7bf59799c67a78a11e35e09d79e Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Fri, 13 Dec 2019 08:59:33 -0700 Subject: [PATCH 072/456] changelog update. --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f897c68..716d0f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,14 @@ # Time for a ChangeLog! +## 1.24 + + +Clojure's range is now respected in two different ways: +* `(range)` - bridges to a python iterable +* `(range 5)` - copies to a python list + + ## 1.23 From a16fb2dad8f71f7d9eee8aa6c407c3abc6e598ab Mon Sep 17 00:00:00 2001 From: "J.J. Tolton" Date: Fri, 13 Dec 2019 16:34:38 -0500 Subject: [PATCH 073/456] Tools for finding system library paths (#22) * * tools for finding python library paths * Update project.clj --- project.clj | 3 +- src/libpython_clj/python/interpreter.clj | 108 ++++++++++++++++++++++- 2 files changed, 109 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index 6b3f82a..66c867b 100644 --- a/project.clj +++ b/project.clj @@ -4,7 +4,8 @@ :license {:name "EPL-2.0" :url "https://www.eclipse.org/legal/epl-2.0/"} :dependencies [[org.clojure/clojure "1.10.1"] + [camel-snake-kebab "0.4.0"] [techascent/tech.datatype "4.61"] - [camel-snake-kebab "0.4.0"]] + [cheshire "5.9.0"]] :repl-options {:init-ns user} :java-source-paths ["java"]) diff --git a/src/libpython_clj/python/interpreter.clj b/src/libpython_clj/python/interpreter.clj index 484da8b..d3c44ca 100644 --- a/src/libpython_clj/python/interpreter.clj +++ b/src/libpython_clj/python/interpreter.clj @@ -6,8 +6,10 @@ :refer [log-error log-warn log-info]] [tech.jna :as jna] [clojure.java.shell :as sh] + [clojure.java.io :as io] [clojure.string :as s] - [clojure.tools.logging :as log]) + [clojure.tools.logging :as log] + [cheshire.core :as json]) (:import [libpython_clj.jna JVMBridge PyObject] @@ -25,6 +27,110 @@ (System/identityHashCode obj)) +(defn py-system-attribute [executable attr] + (let [{out :out err :err} + (sh executable + "-c" + (format "import sys,json; print(getattr(sys, '%s'))" attr))] + (clojure.string/trim out))) + +(defn py-system-version [executable] + (let [{out :out err :err} + (sh executable "-c" + (format "import sys, json; print(list(getattr(sys, 'version_info')[:3]))"))] + (json/parse-string out))) + + +(defn python-system-info + "An information map about the Python system information provided + by a Python executable (string). + + :platform (operating system information) + :prefix + A string giving the site-specific directory prefix where the platform independent Python files are installed; by default, this is the string '/usr/local'. This can be set at build time with the --prefix argument to the configure script. The main collection of Python library modules is installed in the directory prefix/lib/pythonX.Y while the platform independent header files (all except pyconfig.h) are stored in prefix/include/pythonX.Y, where X.Y is the version number of Python, for example 3.2. + Note If a virtual environment is in effect, this value will be changed in site.py to point to the virtual environment. The value for the Python installation will still be available, via base_prefix. + + :base-prefix + Set during Python startup, before site.py is run, to the same value as prefix. If not running in a virtual environment, the values will stay the same; if site.py finds that a virtual environment is in use, the values of prefix and exec_prefix will be changed to point to the virtual environment, whereas base_prefix and base_exec_prefix will remain pointing to the base Python installation (the one which the virtual environment was created from). + + :executable + A string giving the absolute path of the executable binary for the Python interpreter, on systems where this makes sense. If Python is unable to retrieve the real path to its executable, sys.executable will be an empty string or None. + + :exec-prefix + A string giving the site-specific directory prefix where the platform-dependent Python files are installed; by default, this is also '/usr/local'. This can be set at build time with the --exec-prefix argument to the configure script. Specifically, all configuration files (e.g. the pyconfig.h header file) are installed in the directory exec_prefix/lib/pythonX.Y/config, and shared library modules are installed in exec_prefix/lib/pythonX.Y/lib-dynload, where X.Y is the version number of Python, for example 3.2. + Note If a virtual environment is in effect, this value will be changed in site.py to point to the virtual environment. The value for the Python installation will still be available, via base_exec_prefix. + + :base-exec-prefix + Set during Python startup, before site.py is run, to the same value as exec_prefix. If not running in a virtual environment, the values will stay the same; if site.py finds that a virtual environment is in use, the values of prefix and exec_prefix will be changed to point to the virtual environment, whereas base_prefix and base_exec_prefix will remain pointing to the base Python installation (the one which the virtual environment was created from). + + :version + (list python-major python-minor python-micro)" + [executable] + (letfn [(system-attribte [x] + (py-system-attribute executable x))] + + {:platform (system-attribte "platform") + :prefix (system-attribte "prefix") + :base-prefix (system-attribte "base_prefix") + :executable (system-attribte "executable") + :exec-prefix (system-attribte "exec_prefix") + :base-exec-prefix (system-attribte "base_exec_prefix") + :version (py-system-version executable)})) + +(defn python-library-regex [system-info] + (let [{version :version + platform :platform} system-info + [major minor micro] (vec version)] + (re-pattern + (format + (condp (partial =) (keyword platform) + ;; TODO: not sure what the strings returned by + ;; ..: mac and windows are for sys.platform + :linux "libpython%s.%sm.so$" + :mac "libpython%s.%sm.dylib$" + :windows "python%s.%sm.dll$") + major minor)))) + +(defn python-library-paths + "Returns vector of matching python libraries in order of: + - virtual-env (library) + - installation prefix (library) + - default installation location + - virtual-env (executable) + - installation prefix (executable) + - default executable location" + [system-info python-regex] + (transduce + (comp + (map io/file) + (map file-seq) + cat + (map str) + (filter #(re-find python-regex %))) + (fn + ([[seen results]] results) + ([[seen? results] input] + (if (not (seen? input)) + [(conj seen? input) (conj results input)] + [seen? results]))) + ;; [seen? results] + [#{} []] + ((comp + vec + (juxt :base-prefix :prefix :base-exec-prefix :exec-prefix)) + system-info))) + +(comment + ;; library paths workflow + + (let [executable "python3.7" + system-info (python-system-info executable) + pyregex (python-library-regex system-info)] + (python-library-paths system-info pyregex)) + ;;=> ["/usr/lib/x86_64-linux-gnu/libpython3.7m.so" "/usr/lib/python3.7/config-3.7m-x86_64-linux-gnu/libpython3.7m.so"] + ) + + ;;All interpreters share the same type symbol table as types are uniform ;;across initializations. So given an unknown item, we can in constant time ;;get the type of that item if we have seen it before. From 8ec9452ff40179efbf02b9e0204311fb6b2087f1 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Fri, 13 Dec 2019 15:20:56 -0700 Subject: [PATCH 074/456] Now have mxnet importing correctly into the docker container. --- dockerfiles/CondaDockerfile | 1 + scripts/conda-repl | 9 +++++++++ scripts/run-conda-docker | 2 +- src/libpython_clj/python/interpreter.clj | 19 ++++++++++++++----- 4 files changed, 25 insertions(+), 6 deletions(-) create mode 100755 scripts/conda-repl diff --git a/dockerfiles/CondaDockerfile b/dockerfiles/CondaDockerfile index 038c986..1320a77 100644 --- a/dockerfiles/CondaDockerfile +++ b/dockerfiles/CondaDockerfile @@ -10,6 +10,7 @@ RUN apt-get -qq update && apt-get -qq -y install curl wget bzip2 openjdk-8-jdk-h && rm -rf /tmp/miniconda.sh \ && conda install -y python=3 \ && conda update conda \ + && conda install -y mxnet numpy \ && curl -O https://download.clojure.org/install/linux-install-1.10.1.492.sh \ && chmod +x linux-install-1.10.1.492.sh \ && ./linux-install-1.10.1.492.sh && rm linux-install-1.10.1.492.sh \ diff --git a/scripts/conda-repl b/scripts/conda-repl new file mode 100755 index 0000000..45f6921 --- /dev/null +++ b/scripts/conda-repl @@ -0,0 +1,9 @@ +#!/bin/bash + +source activate pyclj + +export LD_LIBRARY_PATH="/home/chrisn/.conda/envs/pyclj/lib" + +lein update-in :dependencies conj \[nrepl\ \"0.6.0\"\]\ + -- update-in :plugins conj \[cider/cider-nrepl\ \"0.22.4\"\]\ + -- repl :headless :host localhost diff --git a/scripts/run-conda-docker b/scripts/run-conda-docker index d193f96..0e1688c 100755 --- a/scripts/run-conda-docker +++ b/scripts/run-conda-docker @@ -10,7 +10,7 @@ docker run --rm -it -u $(id -u):$(id -g) \ -v /$HOME/.lein:/home/$USER/.lein \ -v $(pwd)/:/libpython-clj \ --net=host -w /libpython-clj \ - docker-conda /bin/bash + docker-conda scripts/conda-repl # lein update-in :dependencies conj \[nrepl\ \"0.6.0\"\]\ # -- update-in :plugins conj \[cider/cider-nrepl\ \"0.22.4\"\]\ # -- repl :headless :host localhost diff --git a/src/libpython_clj/python/interpreter.clj b/src/libpython_clj/python/interpreter.clj index d3c44ca..66b7906 100644 --- a/src/libpython_clj/python/interpreter.clj +++ b/src/libpython_clj/python/interpreter.clj @@ -5,7 +5,7 @@ [libpython-clj.python.logging :refer [log-error log-warn log-info]] [tech.jna :as jna] - [clojure.java.shell :as sh] + [clojure.java.shell :refer [sh]] [clojure.java.io :as io] [clojure.string :as s] [clojure.tools.logging :as log] @@ -356,7 +356,7 @@ (defn- ignore-shell-errors [& args] (try - (apply sh/sh args) + (apply sh args) (catch Throwable e nil))) @@ -399,15 +399,18 @@ (defonce ^:private python-home-wide-ptr* (atom nil)) +(defonce ^:private python-path-wide-ptr* (atom nil)) (defn- try-load-python-library! - [libname python-home-wide-ptr] + [libname python-home-wide-ptr python-path-wide-ptr] (try (jna/load-library libname) (alter-var-root #'libpy-base/*python-library* (constantly libname)) (when python-home-wide-ptr (libpy/Py_SetPythonHome python-home-wide-ptr)) + (when python-path-wide-ptr + (libpy/Py_SetProgramName python-path-wide-ptr)) (libpy/Py_InitializeEx 0) libname (catch Exception e))) @@ -451,13 +454,19 @@ :else (libpy-base/library-names))] (reset! python-home-wide-ptr* nil) + (reset! python-path-wide-ptr* nil) (when python-home (append-java-library-path! java-library-path-addendum) ;;This can never be released if load-library succeeeds - (reset! python-home-wide-ptr* (jna/string->wide-ptr python-home))) + (reset! python-home-wide-ptr* (jna/string->wide-ptr python-home)) + (reset! python-path-wide-ptr* (jna/string->wide-ptr + (format "%s/bin/python3" + python-home)))) (loop [[library-name & library-names] library-names] (if (and library-name - (not (try-load-python-library! library-name @python-home-wide-ptr*))) + (not (try-load-python-library! library-name + @python-home-wide-ptr* + @python-path-wide-ptr*))) (recur library-names)))) ;;Set program name (when-let [program-name (or program-name *program-name* "")] From 2cafb93cb69ea61a66b374196d1244c4567a33ab Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Fri, 13 Dec 2019 21:14:46 -0700 Subject: [PATCH 075/456] correct ld config export --- scripts/conda-repl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/conda-repl b/scripts/conda-repl index 45f6921..c39c4b3 100755 --- a/scripts/conda-repl +++ b/scripts/conda-repl @@ -2,7 +2,7 @@ source activate pyclj -export LD_LIBRARY_PATH="/home/chrisn/.conda/envs/pyclj/lib" +export LD_LIBRARY_PATH="$(python3-config --prefix)/lib" lein update-in :dependencies conj \[nrepl\ \"0.6.0\"\]\ -- update-in :plugins conj \[cider/cider-nrepl\ \"0.22.4\"\]\ From 4a9ae56c2de5f8143def337d61371471b44d8927 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sat, 14 Dec 2019 13:41:25 -0700 Subject: [PATCH 076/456] Python-require issue. --- CHANGELOG.md | 6 ++++++ src/libpython_clj/require.clj | 17 +++++++++++------ test/libpython_clj/require_python_test.clj | 5 ++++- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 716d0f9..ed2f42b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,12 @@ # Time for a ChangeLog! +## 1.25 + + +Fixed (with tests) major issue with require-python. + + ## 1.24 diff --git a/src/libpython_clj/require.clj b/src/libpython_clj/require.clj index 1a05ae7..2c19c48 100644 --- a/src/libpython_clj/require.clj +++ b/src/libpython_clj/require.clj @@ -219,10 +219,13 @@ python-namespace (find-ns module-name-or-ns) this-module (import-module (str module-name))] - (when reload? - (remove-ns module-name) - (reload-module this-module)) - (create-ns module-name-or-ns) + (cond + reload? + (do + (remove-ns module-name) + (reload-module this-module)) + (not python-namespace) + (create-ns module-name-or-ns)) ;; bind the python module to its symbolic name ;; in the current namespace @@ -252,8 +255,10 @@ (into {}))] ;;Always make the loaded namespace available to the current namespace. - (intern current-ns-sym (with-meta module-name-or-ns - {:doc (doc this-module)})) + (intern current-ns-sym + (with-meta module-name-or-ns + {:doc (doc this-module)}) + this-module) (let [refer-symbols (cond ;; include everything into the current namespace, diff --git a/test/libpython_clj/require_python_test.clj b/test/libpython_clj/require_python_test.clj index 4eb3508..a0b16c5 100644 --- a/test/libpython_clj/require_python_test.clj +++ b/test/libpython_clj/require_python_test.clj @@ -1,5 +1,6 @@ (ns libpython-clj.require-python-test (:require [libpython-clj.require :refer [require-python]] + [libpython-clj.python :as py] [clojure.test :refer :all])) @@ -8,7 +9,8 @@ (require-python '[math :refer :* - :exclude [sin cos]]) + :exclude [sin cos] + :as pymath]) (deftest base-require-test @@ -17,4 +19,5 @@ (is (contains? publics 'floor)) (is (not (contains? publics 'sin))) (is (= 10.0 (double (floor 10.1)))) + (is (= pymath (py/import-module "math"))) (is (thrown? Throwable (require-python '[math :refer [blah]]))))) From edb80cd9c610c3b55b2b769cbbb317452e5d7d00 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sat, 14 Dec 2019 13:41:57 -0700 Subject: [PATCH 077/456] 1.25 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 66c867b..985cefa 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.25-SNAPSHOT" +(defproject cnuernber/libpython-clj "1.25" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" From e7ef6ed3e40d4538672f3ff2beababf5db550ebc Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sat, 14 Dec 2019 13:42:38 -0700 Subject: [PATCH 078/456] snap --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 985cefa..d9103d4 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.25" +(defproject cnuernber/libpython-clj "1.26-SNAPSHOT" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" From 16da3d885f29bde59ea219c9438b9d3654387971 Mon Sep 17 00:00:00 2001 From: "J.J. Tolton" Date: Sat, 14 Dec 2019 16:19:36 -0500 Subject: [PATCH 079/456] Multiple system calls -> single system call in python library path finding (#23) * * tools for finding python library paths * * change to single system call * refactor symbol names * refactor symbol names * Update interpreter.clj spacing --- src/libpython_clj/python/interpreter.clj | 50 +++++++++++++----------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/src/libpython_clj/python/interpreter.clj b/src/libpython_clj/python/interpreter.clj index 66b7906..bee7435 100644 --- a/src/libpython_clj/python/interpreter.clj +++ b/src/libpython_clj/python/interpreter.clj @@ -27,19 +27,19 @@ (System/identityHashCode obj)) -(defn py-system-attribute [executable attr] +(defn python-system-data [executable] (let [{out :out err :err} - (sh executable - "-c" - (format "import sys,json; print(getattr(sys, '%s'))" attr))] - (clojure.string/trim out))) - -(defn py-system-version [executable] - (let [{out :out err :err} - (sh executable "-c" - (format "import sys, json; print(list(getattr(sys, 'version_info')[:3]))"))] - (json/parse-string out))) - + (sh executable "-c" "import sys, json; +print(json.dumps( +{\"platform\": sys.platform, + \"prefix\": sys.prefix, + \"base_prefix\": sys.base_prefix, + \"executable\": sys.executable, + \"base_exec_prefix\": sys.base_exec_prefix, + \"exec_prefix\": sys.exec_prefix, + \"version\": list(sys.version_info)[:3]}))")] + (when (clojure.string/blank? err) + (json/parse-string out true)))) (defn python-system-info "An information map about the Python system information provided @@ -66,16 +66,22 @@ :version (list python-major python-minor python-micro)" [executable] - (letfn [(system-attribte [x] - (py-system-attribute executable x))] - - {:platform (system-attribte "platform") - :prefix (system-attribte "prefix") - :base-prefix (system-attribte "base_prefix") - :executable (system-attribte "executable") - :exec-prefix (system-attribte "exec_prefix") - :base-exec-prefix (system-attribte "base_exec_prefix") - :version (py-system-version executable)})) + (let [{platform :platform + prefix :prefix + base-prefix :base_prefix + executable :executable + exec-prefix :exec_prefix + base-exec-prefix :base_exec_prefix + version :version} + (python-system-data executable)] + {:platform platform + :prefix prefix + :base-prefix base-prefix + :executable executable + :exec-prefix exec-prefix + :base-exec-prefix base-exec-prefix + :version version})) + (defn python-library-regex [system-info] (let [{version :version From e6718c1c08ac02c83a09616c703bb73de7b12abd Mon Sep 17 00:00:00 2001 From: "J.J. Tolton" Date: Sat, 14 Dec 2019 16:20:56 -0500 Subject: [PATCH 080/456] Issue 26 (Python generators in Clojure transducer context) (#27) * add transducing converter * add tests * add sugar * add sugar * revert require.clj --- src/libpython_clj/sugar.clj | 48 +++++++++++++++++++ test/libpython_clj/sugar_test.clj | 79 +++++++++++++++++++++++++++++++ 2 files changed, 127 insertions(+) create mode 100644 src/libpython_clj/sugar.clj create mode 100644 test/libpython_clj/sugar_test.clj diff --git a/src/libpython_clj/sugar.clj b/src/libpython_clj/sugar.clj new file mode 100644 index 0000000..5972ca5 --- /dev/null +++ b/src/libpython_clj/sugar.clj @@ -0,0 +1,48 @@ +(ns libpython-clj.sugar + (:require [libpython-clj.python :as py])) + + +(let [{{:strs [pyxfn]} :globals} + (py/run-simple-string " +import collections + + +class Deque: + + def __init__(self): + self.q = collections.deque() + + def __iter__(self): + while True: + yield next(self) + + def __next__(self): + return self.q.popleft() + + + def append(self, x): + self.q.appendleft(x) + + def pop(self): + self.q.popleft() + + +def pyxfn(g, *args, **kwargs): + q = Deque() + gx = g(q, *args, **kwargs) + return q, gx +")] + (def ^:private -pyxfn pyxfn)) + +(let [builtins (py/import-module "builtins") + pynext (py/get-attr builtins "next")] + (defn pyxfn + [g & args] + (fn [rf] + (let [[q gx] (apply -pyxfn g args)] + (fn + ([] (rf)) + ([result] (rf result)) + ([result input] + (py/$a q append input) + (rf result (pynext gx)))))))) diff --git a/test/libpython_clj/sugar_test.clj b/test/libpython_clj/sugar_test.clj new file mode 100644 index 0000000..00adf24 --- /dev/null +++ b/test/libpython_clj/sugar_test.clj @@ -0,0 +1,79 @@ +(ns libpython-clj.sugar-test + (:require [libpython-clj.sugar :as pysug :reload true] + [clojure.test :refer :all] + [libpython-clj.python :as py])) + +(deftest test-pyxfn + (let [{{:strs [addem addx addxkwargs stringify addbang]} :globals} + (py/run-simple-string + " +def addem(nums): + for num in nums: + yield num + num + + +def addx(nums, x): + for num in nums: + yield num + x + + +def addxkwargs(nums, *, x=None): + if x is None: + raise Exception('x should not be None') + for num in nums: + yield num + x + + +def stringify(xs): + for x in xs: + yield str(x) + + +def addbang(xs): + for x in xs: + yield '{}!'.format(x) +")] + (is (= [0 2 4 6 8 10 12 14 16 18] + (into [] (pysug/pyxfn addem) (range 10)))) + + (is (= [1 2 3 4 5] + (into [] (pysug/pyxfn addx 1) (range 5)))) + + (is (= [1 2 3 4 5] + (into [] (pysug/pyxfn addxkwargs :x 1) (range 5)))) + + (is (= ["00!" "11!" "22!" "33!"] + (transduce + (comp + (pysug/pyxfn stringify) + (pysug/pyxfn addem) + (pysug/pyxfn addbang)) + (completing conj) + [] + (range 4)))) (is (= ["33!" "22!" "11!" "00!"] + (transduce + (comp + (pysug/pyxfn stringify) + (pysug/pyxfn addem) + (pysug/pyxfn addbang)) + (fn + ([result] ((comp vec reverse) result)) + ([result input] + (conj result input))) + [] + (range 4)))) + + (is (= [["22!" "33!"] ["00!" "11!"]] + (transduce + (comp + (pysug/pyxfn stringify) + (pysug/pyxfn addem) + (pysug/pyxfn addbang) + (partition-all 2)) + (fn + ([result] ((comp vec reverse) result)) + ([result input] + (conj result input))) + [] + (range 4)))))) + From 985ac7e6cba0840b03e83f0b8683de74f420673c Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sat, 14 Dec 2019 14:24:37 -0700 Subject: [PATCH 081/456] updating changelog --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed2f42b..f05e9a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,15 @@ # Time for a ChangeLog! +## 1.26-SNAPSHOT + + + +* [python startup work](https://github.com/cnuernber/libpython-clj/commit/16da3d885f29bde59ea219c9438b9d3654387971) +* [python generates & clojure transducers](https://github.com/cnuernber/libpython-clj/pull/27) + + + ## 1.25 From 170dc9a783fd8a9956baf7654b72c0528ee1938e Mon Sep 17 00:00:00 2001 From: "J.J. Tolton" Date: Sat, 14 Dec 2019 17:17:37 -0500 Subject: [PATCH 082/456] Issue 20 (require reload syntax) (#24) * conform require-python to unary or binary flags * refactor to parse-flags route * remove spacing deltas * remove spacing * remove spacing * readability * fix tests * fix tests * merge master --- src/libpython_clj/require.clj | 100 +++++++++++++++++---- test/libpython_clj/require_python_test.clj | 31 ++++++- 2 files changed, 114 insertions(+), 17 deletions(-) diff --git a/src/libpython_clj/require.clj b/src/libpython_clj/require.clj index 2c19c48..b2f88d2 100644 --- a/src/libpython_clj/require.clj +++ b/src/libpython_clj/require.clj @@ -191,16 +191,84 @@ (intern fn-ns (with-meta fn-sym (py-fn-metadata fn-name f options)) f))) +(defn- parse-flags + "FSM style parser for flags. Designed to support both + unary style flags aka '[foo :reload] and + boolean flags '[foo :reload true] to support Clojure + style 'require syntax. Possibly overengineered." + [supported-flags reqs] + + (letfn [(supported-flag-item + ;; scanned a supported tag token + [supported-flags flag results item items] + (cond + ;; add flag, continue scanning + (true? item) (next-flag-item + supported-flags + (conj results flag) + (first items) + (rest items)) + + ;; don't add flag, continue scanning + (false? item) (next-flag-item + supported-flags + results + (first items) + (rest items)) + + + :else + ;; unary flag -- add flag but scan current item/s + (next-flag-item + supported-flags + (conj results flag) + item + items))) + + + ;; scan flags + (next-flag-item [supported-flags results item items] + (cond + ;; supported flag scanned, begin FSM parse + (get supported-flags item) + (let [flag (get supported-flags item) + remaining-flags (clojure.set/difference + supported-flags #{flag})] + (supported-flag-item + remaining-flags + flag + results + (first items) + (rest items))) + + ;; FSM complete + (nil? item) (into #{} results) + + ;; no flag scanned, continue scanning + :else (recur + supported-flags + results + (first items) + (rest items)))) + + ;; entrypoint + (get-flags [supported-flags reqs] + (next-flag-item supported-flags + [] + (first reqs) + (rest reqs)))] + (trampoline get-flags supported-flags reqs))) + + (defn ^:private load-python-lib [req] (let [supported-flags #{:reload :no-arglists} [module-name & etc] req - flags (into #{} - (filter supported-flags) - etc) + flags (flags* supported-flags etc) etc (into {} (comp (remove supported-flags) + (remove boolean?) (partition-all 2) (map vec)) etc) @@ -208,16 +276,16 @@ no-arglists? (:no-arglists flags) module-name-or-ns (:as etc module-name) exclude (into #{} (:exclude etc)) - refer (cond - (= :all (:refer etc)) #{:all} - (= :* (:refer etc)) #{:*} - :else (into - #{} - (:refer etc))) - current-ns *ns* - current-ns-sym (symbol (str current-ns)) - python-namespace (find-ns module-name-or-ns) - this-module (import-module (str module-name))] + refer (cond + (= :all (:refer etc)) #{:all} + (= :* (:refer etc)) #{:*} + :else (into + #{} + (:refer etc))) + current-ns *ns* + current-ns-sym (symbol (str current-ns)) + python-namespace (find-ns module-name-or-ns) + this-module (import-module (str module-name))] (cond reload? @@ -250,9 +318,9 @@ (let [python-namespace (find-ns module-name-or-ns) ;;ns-publics is a map of symbol to var. Var's have metadata on them. - public-data (->> (ns-publics python-namespace) - (remove #(exclude (first %))) - (into {}))] + public-data (->> (ns-publics python-namespace) + (remove #(exclude (first %))) + (into {}))] ;;Always make the loaded namespace available to the current namespace. (intern current-ns-sym diff --git a/test/libpython_clj/require_python_test.clj b/test/libpython_clj/require_python_test.clj index a0b16c5..6ff616e 100644 --- a/test/libpython_clj/require_python_test.clj +++ b/test/libpython_clj/require_python_test.clj @@ -1,5 +1,5 @@ (ns libpython-clj.require-python-test - (:require [libpython-clj.require :refer [require-python]] + (:require [libpython-clj.require :as req :refer [require-python]] [libpython-clj.python :as py] [clojure.test :refer :all])) @@ -21,3 +21,32 @@ (is (= 10.0 (double (floor 10.1)))) (is (= pymath (py/import-module "math"))) (is (thrown? Throwable (require-python '[math :refer [blah]]))))) + + +(deftest parse-flags-test + ;; sanity check + (is (= #{:reload :foo} + #{:foo :reload})) + (let [parse-flags #'req/parse-flags] + (is (= #{:reload} (parse-flags + #{:reload} + '[:reload foo]))) + (is (= #{} (parse-flags #{} '[:reload foo]))) + (is (= #{:reload} (parse-flags #{:reload} '[:reload true]))) + (is (= #{:reload} (parse-flags #{:reload} '[:reload :as foo]))) + (is (= #{:reload} (parse-flags #{:reload} '[:reload foo :as]))) + (is (= #{:reload} (parse-flags #{:reload} '[foo :reload :as bar]))) + (is (= #{} (parse-flags #{:reload} '[:reload false]))) + (is (= #{} (parse-flags #{:reload} '[:reload false :reload]))) + (is (= #{} (parse-flags #{:reload} '[:reload false :reload true]))) + (is (= #{:reload} (parse-flags #{:reload} '[:reload :reload false]))) + (is (= #{:reload} (parse-flags #{:reload} '[:reload true :reload false]))) + (is (= #{:a :b}) (parse-flags #{:a :b} '[:a true :b])) + (is (= #{:a :b}) (parse-flags #{:a :b} '[:a :b])) + (is (= #{:a :b} (parse-flags #{:a :b} '[:a true :b]))) + (is (= #{:a :b} (parse-flags #{:a :b} '[:a :b true]))) + (is (= #{:a :b} (parse-flags #{:a :b} '[:a true :b true]))) + (is (= #{:b} (parse-flags #{:a :b} '[:a false :b true]))) + (is (= #{:b} (parse-flags #{:a :b} '[:b true :a false]))) + (is (= #{:b} (parse-flags #{:a :b} '[:b :a false]))) + (is (= #{:b} (parse-flags #{:a :b} '[:b :a false :a true]))))) From 3bc16d343ee4bf7e9a70d8d14a595fbfeaf8a889 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sat, 14 Dec 2019 15:31:11 -0700 Subject: [PATCH 083/456] small merge issue. --- src/libpython_clj/require.clj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libpython_clj/require.clj b/src/libpython_clj/require.clj index b2f88d2..684464e 100644 --- a/src/libpython_clj/require.clj +++ b/src/libpython_clj/require.clj @@ -197,7 +197,7 @@ boolean flags '[foo :reload true] to support Clojure style 'require syntax. Possibly overengineered." [supported-flags reqs] - + (letfn [(supported-flag-item ;; scanned a supported tag token [supported-flags flag results item items] @@ -216,7 +216,7 @@ (first items) (rest items)) - + :else ;; unary flag -- add flag but scan current item/s (next-flag-item @@ -264,7 +264,7 @@ (defn ^:private load-python-lib [req] (let [supported-flags #{:reload :no-arglists} [module-name & etc] req - flags (flags* supported-flags etc) + flags (parse-flags supported-flags etc) etc (into {} (comp (remove supported-flags) From 4084ddfcd271a9c336bf67aa163c17cc3e945a44 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sat, 14 Dec 2019 16:17:01 -0700 Subject: [PATCH 084/456] 1.26 --- project.clj | 2 +- src/libpython_clj/require.clj | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index d9103d4..a78f432 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.26-SNAPSHOT" +(defproject cnuernber/libpython-clj "1.26" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" diff --git a/src/libpython_clj/require.clj b/src/libpython_clj/require.clj index 684464e..80d5d3d 100644 --- a/src/libpython_clj/require.clj +++ b/src/libpython_clj/require.clj @@ -291,7 +291,8 @@ reload? (do (remove-ns module-name) - (reload-module this-module)) + (reload-module this-module) + (create-ns module-name-or-ns)) (not python-namespace) (create-ns module-name-or-ns)) From 09ca9e3c547d9d549b456d61b49289fed943b4cd Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sat, 14 Dec 2019 16:17:20 -0700 Subject: [PATCH 085/456] snap --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index a78f432..5831f26 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.26" +(defproject cnuernber/libpython-clj "1.27-SNAPSHOT" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" From 70c2ae842eb34c85380f488c01e03a064cb2341e Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sat, 14 Dec 2019 16:19:01 -0700 Subject: [PATCH 086/456] Changelog update --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f05e9a7..c48d224 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,13 @@ # Time for a ChangeLog! -## 1.26-SNAPSHOT - +## 1.26 * [python startup work](https://github.com/cnuernber/libpython-clj/commit/16da3d885f29bde59ea219c9438b9d3654387971) * [python generates & clojure transducers](https://github.com/cnuernber/libpython-clj/pull/27) +* [requre-python reload fix](https://github.com/cnuernber/libpython-clj/pull/24) +* Bugfix with require-python :reload semantics. From b2c4ba7974a732d4c1f086bfaf6e81d0d23fe304 Mon Sep 17 00:00:00 2001 From: "J.J. Tolton" Date: Sun, 15 Dec 2019 11:11:39 -0500 Subject: [PATCH 087/456] Issue 25 - data driven require.clj pathway (#28) * light overhaul; tests added * whitespace * formatting --- src/libpython_clj/require.clj | 233 +++++++++++++-------- test/libpython_clj/require_python_test.clj | 78 ++++++- 2 files changed, 219 insertions(+), 92 deletions(-) diff --git a/src/libpython_clj/require.clj b/src/libpython_clj/require.clj index 80d5d3d..f738dac 100644 --- a/src/libpython_clj/require.clj +++ b/src/libpython_clj/require.clj @@ -81,7 +81,6 @@ (number? x) "" :else (py-class-argspec x))) - (defn pyarglists ([argspec] (pyarglists argspec (if-let [defaults @@ -142,10 +141,6 @@ {:or or-map}) (when (not-empty as-varkw) as-varkw)) - - - - opt-args (cond (and (empty? kwargs-map) @@ -168,7 +163,6 @@ arglists (recur argspec' defaults' arglists)))))) - (defn py-fn-metadata [fn-name x {:keys [no-arglists?]}] (let [fn-argspec (pyargspec x) fn-docstr (get-pydoc x)] @@ -183,7 +177,6 @@ (catch Throwable e nil)))))) - (defn ^:private load-py-fn [f fn-name fn-module-name-or-ns options] (let [fn-ns (symbol (str fn-module-name-or-ns)) @@ -215,8 +208,6 @@ results (first items) (rest items)) - - :else ;; unary flag -- add flag but scan current item/s (next-flag-item @@ -225,7 +216,6 @@ item items))) - ;; scan flags (next-flag-item [supported-flags results item items] (cond @@ -259,9 +249,37 @@ (rest reqs)))] (trampoline get-flags supported-flags reqs))) - - -(defn ^:private load-python-lib [req] +(defn- extract-refer-symbols + [{:keys [refer this-module]} public-data] + (cond + ;; include everything into the current namespace, + ;; ignore __all__ directive + (or (refer :all) + (and (not (py/has-attr? this-module "__all__")) + (refer :*))) + (keys public-data) + ;; only include that specfied by __all__ attribute + ;; of python module if specified, else same as :all + (refer :*) + (->> (py/get-attr this-module "__all__") + (map (fn [item-name] + (let [item-sym (symbol item-name)] + (when (contains? public-data item-sym) + item-sym)))) + (remove nil?)) + + ;; [.. :refer [..]] behavior + :else + (do + (when-let [missing (->> refer + (remove (partial contains? public-data)) + seq)] + (throw (Exception. + (format "'refer' symbols not found: %s" + (vec missing))))) + refer))) + +(defn ^:private python-lib-configuration [req] (let [supported-flags #{:reload :no-arglists} [module-name & etc] req flags (parse-flags supported-flags etc) @@ -286,80 +304,122 @@ current-ns-sym (symbol (str current-ns)) python-namespace (find-ns module-name-or-ns) this-module (import-module (str module-name))] + {:supported-flags supported-flags + :etc etc + :reload? reload? + :no-arglists? no-arglists? + :module-name module-name + :module-name-or-ns module-name-or-ns + :exclude exclude + :refer refer + :current-ns current-ns + :current-ns-sym current-ns-sym + :python-namespace python-namespace + :this-module this-module})) + +(defn- extract-public-data + [{:keys [exclude python-namespace module-name-or-ns]}] + (let [python-namespace + (or python-namespace + (find-ns module-name-or-ns))] + (->> (ns-publics python-namespace) + (remove #(exclude (first %))) + (into {})))) + +(defn- reload-python-ns! + [module-name this-module module-name-or-ns] + (do + (remove-ns module-name) + (reload-module this-module) + (create-ns module-name-or-ns))) + +(defn- create-python-ns! + [module-name-or-ns] + (create-ns module-name-or-ns)) + +(defn ^:private maybe-reload-or-create-ns! + [{:keys [reload? + this-module + module-name + module-name-or-ns] + python-namespace :python-namespace}] + (cond + reload? (reload-python-ns! module-name + this-module + module-name-or-ns) + (not python-namespace) (create-python-ns! module-name-or-ns))) + +(defn enhanced-python-lib-configuration + [{:keys [python-namespace exclude this-module] + :as lib-config}] + (let [public-data (extract-public-data lib-config)] + (merge + lib-config + {:public-data public-data + :refer-symbols (extract-refer-symbols lib-config + public-data)}))) + +(defn- bind-py-symbols-to-ns! + [{:keys [reload? + python-namespace + this-module + module-name-or-ns + no-arglists?]}] + (when (or reload? (not python-namespace)) + ;;Mutably define the root namespace. + (doseq [[att-name v] (vars this-module)] + (try + (when v + (if (py/callable? v) + (load-py-fn v (symbol att-name) module-name-or-ns + {:no-arglists? + no-arglists?}) + (intern module-name-or-ns (symbol att-name) v))) + (catch Throwable e + (log/warnf e "Failed to require symbol %s" att-name)))))) + +(defn- bind-module-ns! + [{:keys [current-ns-sym module-name-or-ns this-module]}] + (intern current-ns-sym + (with-meta module-name-or-ns + {:doc (doc this-module)}) + this-module)) + +(defn- intern-public-and-refer-symbols! + [{:keys [public-data refer-symbols current-ns-sym]}] + (doseq [[s v] (select-keys public-data refer-symbols)] + (intern current-ns-sym + (with-meta s (meta v)) + (deref v)))) + +(defn preload-python-lib! [req] + (let [lib-config (python-lib-configuration req)] + (maybe-reload-or-create-ns! lib-config) + (bind-py-symbols-to-ns! lib-config) + (bind-module-ns! lib-config) + (enhanced-python-lib-configuration lib-config))) - (cond - reload? - (do - (remove-ns module-name) - (reload-module this-module) - (create-ns module-name-or-ns)) - (not python-namespace) - (create-ns module-name-or-ns)) - - ;; bind the python module to its symbolic name - ;; in the current namespace - - - ;; create namespace for module and bind python - ;; values to namespace symbols - (when (or reload? - (not python-namespace)) - ;;Mutably define the root namespace. - (doseq [[att-name v] (vars this-module)] - (try - (when v - (if (py/callable? v) - (load-py-fn v (symbol att-name) module-name-or-ns - {:no-arglists? - no-arglists?}) - (intern module-name-or-ns (symbol att-name) v))) - (catch Throwable e - (log/warnf e "Failed to require symbol %s" att-name))))) - - - (let [python-namespace (find-ns module-name-or-ns) - ;;ns-publics is a map of symbol to var. Var's have metadata on them. - public-data (->> (ns-publics python-namespace) - (remove #(exclude (first %))) - (into {}))] - - ;;Always make the loaded namespace available to the current namespace. - (intern current-ns-sym - (with-meta module-name-or-ns - {:doc (doc this-module)}) - this-module) - (let [refer-symbols - (cond - ;; include everything into the current namespace, - ;; ignore __all__ directive - (or (refer :all) - (and (not (py/has-attr? this-module "__all__")) - (refer :*))) - (keys public-data) - ;; only include that specfied by __all__ attribute - ;; of python module if specified, else same as :all - (refer :*) - (->> (py/get-attr this-module "__all__") - (map (fn [item-name] - (let [item-sym (symbol item-name)] - (when (contains? public-data item-sym) - item-sym)))) - (remove nil?)) - - ;; [.. :refer [..]] behavior - :else - (do - (when-let [missing (->> refer - (remove (partial contains? public-data)) - seq)] - (throw (Exception. - (format "'refer' symbols not found: %s" - (vec missing))))) - refer))] - (doseq [[s v] (select-keys public-data refer-symbols)] - (intern current-ns-sym - (with-meta s (meta v)) - (deref v))))))) +(defn ^:private load-python-lib [req] + (let [{:keys [supported-flags + flags + etc + reload? + no-arglists? + module-name + module-name-or-ns + exclude + refer + current-ns + current-ns-sym + python-namespace + this-module + python-namespace + public-data + refer-symbols] + :as lib-config} + (preload-python-lib! req)] + + (intern-public-and-refer-symbols! lib-config))) (defn require-python "## Basic usage ## @@ -438,3 +498,4 @@ (load-python-lib (vector reqs)) (vector? reqs) (load-python-lib reqs))) + diff --git a/test/libpython_clj/require_python_test.clj b/test/libpython_clj/require_python_test.clj index 6ff616e..0076a50 100644 --- a/test/libpython_clj/require_python_test.clj +++ b/test/libpython_clj/require_python_test.clj @@ -1,16 +1,17 @@ (ns libpython-clj.require-python-test - (:require [libpython-clj.require :as req :refer [require-python]] + (:require [libpython-clj.require :as req :refer [require-python] + :reload true] [libpython-clj.python :as py] [clojure.test :refer :all])) - -;;Since this test mutates the global environment we have to accept that -;;it may not always work. +;; Since this test mutates the global environment we have to accept that +;; it may not always work. (require-python '[math :refer :* :exclude [sin cos] - :as pymath]) + :as pymath + :reload true]) (deftest base-require-test @@ -22,7 +23,6 @@ (is (= pymath (py/import-module "math"))) (is (thrown? Throwable (require-python '[math :refer [blah]]))))) - (deftest parse-flags-test ;; sanity check (is (= #{:reload :foo} @@ -50,3 +50,69 @@ (is (= #{:b} (parse-flags #{:a :b} '[:b true :a false]))) (is (= #{:b} (parse-flags #{:a :b} '[:b :a false]))) (is (= #{:b} (parse-flags #{:a :b} '[:b :a false :a true]))))) + +(deftest python-lib-configuration-test + (let [python-lib-configuration #'req/python-lib-configuration + simple-req '[csv] + simple-spec (python-lib-configuration simple-req) + {:keys [exclude + supported-flags + current-ns-sym + module-name + module-name-or-ns + reload? + no-arglists? + etc + current-ns + this-module + python-namespace + refer]} simple-spec + csv-module (py/import-module "csv")] + + ;; no exclusions + (is (= #{} exclude)) + (is (= #{:no-arglists :reload} supported-flags)) + (is (= 'libpython-clj.require-python-test + current-ns-sym + (symbol (str current-ns)))) + (is (= 'csv module-name module-name-or-ns)) + (is (nil? reload?)) + (is (nil? no-arglists?)) + (is (= {} etc)) + (is (= csv-module this-module))) + + + (let [python-lib-configuration #'req/python-lib-configuration + simple-req '[requests + :reload true + :refer [get] + :no-arglists] + simple-spec (python-lib-configuration simple-req) + {:keys [exclude + supported-flags + current-ns-sym + module-name + module-name-or-ns + reload? + no-arglists? + etc + current-ns + this-module + python-namespace + refer]} simple-spec + requests-module (py/import-module "requests")] + + ;; no exclusions + (is (= #{} exclude)) + (is (= #{:no-arglists :reload} supported-flags)) + (is (= 'libpython-clj.require-python-test + current-ns-sym + (symbol (str current-ns)))) + (is (= 'requests module-name module-name-or-ns)) + (is (= reload? :reload)) + (is (= no-arglists? :no-arglists)) + (is (= { :refer '[get]}) etc) + (is (= requests-module this-module)) + (is (= #{'get} refer)) + (is (nil? python-namespace)))) + From b26a4857a9a83abc6b31eefd9adb1f818ab1a097 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 15 Dec 2019 09:15:37 -0700 Subject: [PATCH 088/456] Getting CI going. --- .travis.yml | 8 ++++++++ src/libpython_clj/python/bridge.clj | 2 +- src/libpython_clj/python/object.clj | 3 ++- test/libpython_clj/python_test.clj | 16 ++++++++++++++-- 4 files changed, 25 insertions(+), 4 deletions(-) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..0cc5d0c --- /dev/null +++ b/.travis.yml @@ -0,0 +1,8 @@ +language: clojure +lein: 2.8.2 +before_install: + - sudo apt-get -y install python3 + - sudo python3 -mpip install numpy +addons: + apt: + update: true diff --git a/src/libpython_clj/python/bridge.clj b/src/libpython_clj/python/bridge.clj index 9dd4e82..5566e90 100644 --- a/src/libpython_clj/python/bridge.clj +++ b/src/libpython_clj/python/bridge.clj @@ -115,7 +115,7 @@ (defn as-python "Bridge a jvm object into python" [item & [options]] - (if (not item) + (if (nil? item) (py-none) (py-proto/as-python item options))) diff --git a/src/libpython_clj/python/object.clj b/src/libpython_clj/python/object.clj index ecf7e42..8c9dbc7 100644 --- a/src/libpython_clj/python/object.clj +++ b/src/libpython_clj/python/object.clj @@ -790,7 +790,8 @@ (with-gil (-> (cond (seq kw-arg-map) - (libpy/PyObject_Call callable (->py-tuple arglist) + (libpy/PyObject_Call callable + (->py-tuple arglist) (->py-dict kw-arg-map)) (seq arglist) (libpy/PyObject_CallObject callable (->py-tuple arglist)) diff --git a/test/libpython_clj/python_test.clj b/test/libpython_clj/python_test.clj index 306e1cb..57e9497 100644 --- a/test/libpython_clj/python_test.clj +++ b/test/libpython_clj/python_test.clj @@ -1,16 +1,18 @@ (ns libpython-clj.python-test (:require [libpython-clj.python :as py] + [libpython-clj.jna :as libpy] [tech.v2.datatype :as dtype] [tech.v2.datatype.functional :as dfn] [tech.v2.tensor :as dtt] + [tech.jna :as jna] [clojure.test :refer :all]) (:import [java.io StringWriter] - [java.util Map List])) + [java.util Map List] + [com.sun.jna Pointer])) (py/initialize!) - (deftest stdout-and-stderr (is (= "hey\n" (with-out-str (py/run-simple-string "print('hey')")))) @@ -307,3 +309,13 @@ (py/$a array (range 10)))] (is (dfn/equals (dtt/->tensor (range 10)) (py/as-tensor ary-data))))) + + +(deftest false-is-always-py-false + (let [py-false (libpy/Py_False) + ->false (py/->python false) + as-false (py/as-python false)] + (is (= (Pointer/nativeValue (jna/as-ptr py-false)) + (Pointer/nativeValue (jna/as-ptr ->false)))) + (is (= (Pointer/nativeValue (jna/as-ptr py-false)) + (Pointer/nativeValue (jna/as-ptr as-false)))))) From 28e3949b5eca75c1405b72c2114a48996941e6c8 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 15 Dec 2019 09:33:50 -0700 Subject: [PATCH 089/456] fixing unit tests. Testing travis. --- src/libpython_clj/require.clj | 9 ++++++--- test/libpython_clj/require_python_test.clj | 11 +++++++---- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/libpython_clj/require.clj b/src/libpython_clj/require.clj index f738dac..5e1932c 100644 --- a/src/libpython_clj/require.clj +++ b/src/libpython_clj/require.clj @@ -279,7 +279,11 @@ (vec missing))))) refer))) -(defn ^:private python-lib-configuration [req] + +(defn- python-lib-configuration + "Build a configuration map of a python library. Current ns is option and used + during testing but unnecessary during normal running events." + [req & [current-ns]] (let [supported-flags #{:reload :no-arglists} [module-name & etc] req flags (parse-flags supported-flags etc) @@ -300,7 +304,7 @@ :else (into #{} (:refer etc))) - current-ns *ns* + current-ns (or current-ns *ns*) current-ns-sym (symbol (str current-ns)) python-namespace (find-ns module-name-or-ns) this-module (import-module (str module-name))] @@ -498,4 +502,3 @@ (load-python-lib (vector reqs)) (vector? reqs) (load-python-lib reqs))) - diff --git a/test/libpython_clj/require_python_test.clj b/test/libpython_clj/require_python_test.clj index 0076a50..79de85a 100644 --- a/test/libpython_clj/require_python_test.clj +++ b/test/libpython_clj/require_python_test.clj @@ -54,7 +54,9 @@ (deftest python-lib-configuration-test (let [python-lib-configuration #'req/python-lib-configuration simple-req '[csv] - simple-spec (python-lib-configuration simple-req) + simple-spec (python-lib-configuration + simple-req + (find-ns 'libpython-clj.require-python-test)) {:keys [exclude supported-flags current-ns-sym @@ -87,7 +89,9 @@ :reload true :refer [get] :no-arglists] - simple-spec (python-lib-configuration simple-req) + simple-spec (python-lib-configuration + simple-req + (find-ns 'libpython-clj.require-python-test)) {:keys [exclude supported-flags current-ns-sym @@ -111,8 +115,7 @@ (is (= 'requests module-name module-name-or-ns)) (is (= reload? :reload)) (is (= no-arglists? :no-arglists)) - (is (= { :refer '[get]}) etc) + (is (= {:refer '[get]} etc)) (is (= requests-module this-module)) (is (= #{'get} refer)) (is (nil? python-namespace)))) - From face68f7a154c9c5ba3129d3a85e256e06220e68 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 15 Dec 2019 09:35:25 -0700 Subject: [PATCH 090/456] travis hacking. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0cc5d0c..0d45a34 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: clojure lein: 2.8.2 before_install: - - sudo apt-get -y install python3 + - sudo apt-get -y install python3 python3-pip - sudo python3 -mpip install numpy addons: apt: From 33ed81ea57fea429fd3360b6ca374c1d3129cf84 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 15 Dec 2019 09:39:48 -0700 Subject: [PATCH 091/456] adding travis. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 3cde429..7b222ce 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ JNA libpython bindings to the tech ecosystem. [![Clojars Project](https://img.shields.io/clojars/v/cnuernber/libpython-clj.svg)](https://clojars.org/cnuernber/libpython-clj) +[![travis integration](https://travis-ci.com/cnuernber/libpython-clj.svg?branch=master)](https://travis-ci.com/cnuernber/libpython-clj) * Bridge between JVM objects and Python objects easily; use Python in your Java and use some Java in your Python. From 081cc7e2c51f7b5dd49f3e12b79235bcce58cd66 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 15 Dec 2019 09:45:47 -0700 Subject: [PATCH 092/456] 1.27 --- CHANGELOG.md | 10 +++++++++- project.clj | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c48d224..ed73d5f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,19 @@ # Time for a ChangeLog! +## 1.27 + +* Fixed bug where (as-python {:is_train false}) results in a dictionary with a none + value instead of a false value. This was found through hours of debugging why + mxnet's forward function call was returning different values in clojure than in + python. + + ## 1.26 * [python startup work](https://github.com/cnuernber/libpython-clj/commit/16da3d885f29bde59ea219c9438b9d3654387971) -* [python generates & clojure transducers](https://github.com/cnuernber/libpython-clj/pull/27) +* [python generates & clojure transducers](https://github.com/cnuernber/libpython-clj/pull/27) * [requre-python reload fix](https://github.com/cnuernber/libpython-clj/pull/24) * Bugfix with require-python :reload semantics. diff --git a/project.clj b/project.clj index 5831f26..daee8b1 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.27-SNAPSHOT" +(defproject cnuernber/libpython-clj "1.27" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" From e29cf80d767bdb28411c827fbe13ec490724fa80 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 15 Dec 2019 09:45:58 -0700 Subject: [PATCH 093/456] snap --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index daee8b1..b82d48e 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.27" +(defproject cnuernber/libpython-clj "1.28-SNAPSHOT" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" From 0223778e93ef81f5f81160baef8f6229cda432ec Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 15 Dec 2019 10:35:07 -0700 Subject: [PATCH 094/456] Using new pathway that queries python executable for lib info. --- src/libpython_clj/python/interpreter.clj | 82 +++++++++++------------- 1 file changed, 38 insertions(+), 44 deletions(-) diff --git a/src/libpython_clj/python/interpreter.clj b/src/libpython_clj/python/interpreter.clj index bee7435..aa560bb 100644 --- a/src/libpython_clj/python/interpreter.clj +++ b/src/libpython_clj/python/interpreter.clj @@ -28,7 +28,7 @@ (defn python-system-data [executable] - (let [{out :out err :err} + (let [{:keys [out err exit]} (sh executable "-c" "import sys, json; print(json.dumps( {\"platform\": sys.platform, @@ -38,7 +38,7 @@ print(json.dumps( \"base_exec_prefix\": sys.base_exec_prefix, \"exec_prefix\": sys.exec_prefix, \"version\": list(sys.version_info)[:3]}))")] - (when (clojure.string/blank? err) + (when (= 0 exit) (json/parse-string out true)))) (defn python-system-info @@ -136,6 +136,41 @@ print(json.dumps( ;;=> ["/usr/lib/x86_64-linux-gnu/libpython3.7m.so" "/usr/lib/python3.7/config-3.7m-x86_64-linux-gnu/libpython3.7m.so"] ) +(defn- ignore-shell-errors + [& args] + (try + (apply sh args) + (catch Throwable e nil))) + + +(defn detect-startup-info + [{:keys [library-path python-home]}] + (let [executable "python3" + system-info (python-system-info executable) + python-home (cond + python-home + python-home + (seq (System/getenv "PYTHONHOME")) + (System/getenv "PYTHONHOME") + :else + (:prefix system-info)) + java-library-path-addendum (when python-home + (-> (Paths/get python-home + (into-array String ["lib"])) + (.toString))) + [ver-maj ver-med _ver-min] (:version system-info) + lib-version (format "%s.%s" ver-maj ver-med) + libname (or library-path + (when (seq lib-version) + (str "python" lib-version "m"))) + retval + {:python-home python-home + :lib-version lib-version + :libname libname + :java-library-path-addendum java-library-path-addendum}] + (log/infof "Startup info detected: %s" retval) + retval)) + ;;All interpreters share the same type symbol table as types are uniform ;;across initializations. So given an unknown item, we can in constant time @@ -358,27 +393,6 @@ print(json.dumps( - -(defn- ignore-shell-errors - [& args] - (try - (apply sh args) - (catch Throwable e nil))) - - -(defn- find-python-home - [& [python-home]] - (cond - python-home - python-home - (seq (System/getenv "PYTHONHOME")) - (System/getenv "PYTHONHOME") - :else - (let [{:keys [out err exit]} (ignore-shell-errors "python3-config" "--prefix")] - (when (= 0 exit) - (s/trimr out))))) - - (defn- find-python-lib-version [] (let [{:keys [out err exit]} (ignore-shell-errors "python3" "--version")] @@ -422,29 +436,9 @@ print(json.dumps( (catch Exception e))) -(defn detect-startup-info - [{:keys [library-path python-home]}] - (let [python-home (find-python-home python-home) - java-library-path-addendum - (when python-home - (-> (Paths/get python-home (into-array String ["lib"])) - (.toString))) - lib-version (find-python-lib-version) - libname (when (seq lib-version) - (str "python" lib-version "m")) - retval - {:python-home python-home - :lib-version lib-version - :libname libname - :java-library-path-addendum java-library-path-addendum}] - (log/infof "Startup info detected: %s" retval) - retval)) - - (defn initialize! [& {:keys [program-name - library-path - python-home] + library-path] :as options}] (when-not @*main-interpreter* (log-info "Executing python initialize!") From ef1ec57cfc83b47279b5f0cdae6ccd47ee53dbe5 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 15 Dec 2019 10:35:54 -0700 Subject: [PATCH 095/456] Adding comments to repl starter --- scripts/conda-repl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/conda-repl b/scripts/conda-repl index c39c4b3..94e04c0 100755 --- a/scripts/conda-repl +++ b/scripts/conda-repl @@ -2,6 +2,8 @@ source activate pyclj +## This is absolutely necessary. +## https://github.com/conda/conda/issues/9500#issuecomment-565753807 export LD_LIBRARY_PATH="$(python3-config --prefix)/lib" lein update-in :dependencies conj \[nrepl\ \"0.6.0\"\]\ From 87be94fb8ee5da2a67cd85f9efb477acb689bee8 Mon Sep 17 00:00:00 2001 From: "J.J. Tolton" Date: Sun, 15 Dec 2019 14:29:35 -0500 Subject: [PATCH 096/456] remove failing unit tests (#31) --- test/libpython_clj/require_python_test.clj | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/libpython_clj/require_python_test.clj b/test/libpython_clj/require_python_test.clj index 79de85a..17240d5 100644 --- a/test/libpython_clj/require_python_test.clj +++ b/test/libpython_clj/require_python_test.clj @@ -74,9 +74,6 @@ ;; no exclusions (is (= #{} exclude)) (is (= #{:no-arglists :reload} supported-flags)) - (is (= 'libpython-clj.require-python-test - current-ns-sym - (symbol (str current-ns)))) (is (= 'csv module-name module-name-or-ns)) (is (nil? reload?)) (is (nil? no-arglists?)) @@ -109,9 +106,6 @@ ;; no exclusions (is (= #{} exclude)) (is (= #{:no-arglists :reload} supported-flags)) - (is (= 'libpython-clj.require-python-test - current-ns-sym - (symbol (str current-ns)))) (is (= 'requests module-name module-name-or-ns)) (is (= reload? :reload)) (is (= no-arglists? :no-arglists)) From a2adb18186633376bc2e1caeeb692ad8bf0f864a Mon Sep 17 00:00:00 2001 From: "J.J. Tolton" Date: Sun, 15 Dec 2019 17:14:08 -0500 Subject: [PATCH 097/456] Issue 29 - class ns syntax (#32) * added alpha framework for ns-class syntax * cleanup * fix tests * fix merge bug in tests --- src/libpython_clj/require.clj | 118 ++++++++++++++++++++- test/libpython_clj/require_python_test.clj | 4 +- 2 files changed, 115 insertions(+), 7 deletions(-) diff --git a/src/libpython_clj/require.clj b/src/libpython_clj/require.clj index 5e1932c..a340f42 100644 --- a/src/libpython_clj/require.clj +++ b/src/libpython_clj/require.clj @@ -47,6 +47,8 @@ (def ^:private vars (py/get-attr builtins "vars")) +(def ^:private pyclass? (py/get-attr inspect "isclass")) + (defn ^:private py-fn-argspec [f] (if-let [spec (try (argspec f) (catch Throwable e nil))] {:args (py/->jvm (py/get-attr spec "args")) @@ -181,8 +183,11 @@ options] (let [fn-ns (symbol (str fn-module-name-or-ns)) fn-sym (symbol fn-name)] - (intern fn-ns (with-meta fn-sym (py-fn-metadata fn-name f - options)) f))) + (intern fn-ns + (with-meta + fn-sym + (py-fn-metadata fn-name f + options)) f))) (defn- parse-flags "FSM style parser for flags. Designed to support both @@ -279,12 +284,11 @@ (vec missing))))) refer))) - (defn- python-lib-configuration "Build a configuration map of a python library. Current ns is option and used during testing but unnecessary during normal running events." [req & [current-ns]] - (let [supported-flags #{:reload :no-arglists} + (let [supported-flags #{:reload :no-arglists :alpha-load-ns-classes} [module-name & etc] req flags (parse-flags supported-flags etc) etc (into {} @@ -296,6 +300,7 @@ etc) reload? (:reload flags) no-arglists? (:no-arglists flags) + load-ns-classes? (:alpha-load-ns-classes flags) module-name-or-ns (:as etc module-name) exclude (into #{} (:exclude etc)) refer (cond @@ -312,6 +317,7 @@ :etc etc :reload? reload? :no-arglists? no-arglists? + :load-ns-classes? load-ns-classes? :module-name module-name :module-name-or-ns module-name-or-ns :exclude exclude @@ -389,6 +395,100 @@ {:doc (doc this-module)}) this-module)) +(defn- generate-class-namespace-configs + [{:keys [module-name-or-ns this-module] :as lib-config}] + (letfn [(pyclass-pair? [[attr attr-val]] (pyclass? attr-val)) + (pyclass-ns-config [[attr attr-val]] + {:namespace module-name-or-ns + :attribute-type :class + :classname attr + :class-symbol (symbol attr) + :class attr-val + :attributes ((comp + (py/get-attr builtins "dict") + vars) attr-val)}) + (pyclass-attribute-fanout + [{:keys [namespace + classname + class-symbol + class + attributes] + :as attr-config}] + (into [attr-config] + (for [[attr attr-val] (seq attributes) + :let + [attr-type + (if (and (not (nil? attr-val)) + (py/callable? attr-val)) + + :method + :attribute)]] + (merge + (dissoc attr-config :attributes :type) + {:attribute-type attr-type + :type :class-attribute + :attribute attr-val + :attribute-name attr}))))] + (into [] + (comp + (filter pyclass-pair?) + (map pyclass-ns-config) + (map pyclass-attribute-fanout) + cat) + (seq (vars this-module))))) + +(defmulti class-sort :attribute-type) +(defmethod class-sort :class [_] 0) +(defmethod class-sort :method [_] 1) +(defmethod class-sort :attribute [_] 2) + +(defmulti intern-ns-class :attribute-type) + +(defmethod intern-ns-class :class + [{original-namespace :namespace + cls-name :classname + cls-sym :class-symbol + cls :class + :as cls-ns-config}] + (let [cls-ns (symbol (str original-namespace "." cls-sym))] + (create-ns cls-ns) + cls-ns-config)) + +(defmethod intern-ns-class :method + [{original-namespace :namespace + cls-name :classname + cls-sym :class-symbol + cls :class + method-name :attribute-name + method :attribute + :as cls-ns-config}] + (let [cls-ns (symbol (str original-namespace "." cls-sym))] + (load-py-fn method (symbol method-name) + cls-ns {}) + cls-ns-config)) + +(defmethod intern-ns-class :attribute + [{original-namespace :namespace + cls-name :classname + cls-sym :class-symbol + cls :class + attribute-name :attribute-name + attribute :attribute + :as cls-ns-config}] + (let [cls-ns (symbol (str original-namespace "." cls-sym))] + (intern cls-ns (symbol attribute-name) attribute) + cls-ns-config)) + +(defn- bind-class-namespaces! + [lib-config] + (let [class-namespace-configs + (->> + (generate-class-namespace-configs + lib-config) + (sort-by class-sort))] + (doseq [cls-namespace-config class-namespace-configs] + (intern-ns-class cls-namespace-config)))) + (defn- intern-public-and-refer-symbols! [{:keys [public-data refer-symbols current-ns-sym]}] (doseq [[s v] (select-keys public-data refer-symbols)] @@ -409,6 +509,7 @@ etc reload? no-arglists? + load-ns-classes? module-name module-name-or-ns exclude @@ -423,7 +524,13 @@ :as lib-config} (preload-python-lib! req)] - (intern-public-and-refer-symbols! lib-config))) + (intern-public-and-refer-symbols! lib-config) + + ;; alpha + ;; TODO: does not respect reloading or much of the other + ;; ..: syntax (yet) + (when load-ns-classes? + (bind-class-namespaces! lib-config)))) (defn require-python "## Basic usage ## @@ -502,3 +609,4 @@ (load-python-lib (vector reqs)) (vector? reqs) (load-python-lib reqs))) + diff --git a/test/libpython_clj/require_python_test.clj b/test/libpython_clj/require_python_test.clj index 17240d5..68771b5 100644 --- a/test/libpython_clj/require_python_test.clj +++ b/test/libpython_clj/require_python_test.clj @@ -73,7 +73,8 @@ ;; no exclusions (is (= #{} exclude)) - (is (= #{:no-arglists :reload} supported-flags)) + (is (= #{:no-arglists :reload :alpha-load-ns-classes} + supported-flags)) (is (= 'csv module-name module-name-or-ns)) (is (nil? reload?)) (is (nil? no-arglists?)) @@ -105,7 +106,6 @@ ;; no exclusions (is (= #{} exclude)) - (is (= #{:no-arglists :reload} supported-flags)) (is (= 'requests module-name module-name-or-ns)) (is (= reload? :reload)) (is (= no-arglists? :no-arglists)) From c39f25077c2e20ee98bceb868d762f35ebf3ebf7 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 17 Dec 2019 13:52:32 -0700 Subject: [PATCH 098/456] Major changes. jvm-as-python derive from abc classes. ranges get translated into ranges if possible --- src/libpython_clj/python.clj | 1 + src/libpython_clj/python/bridge.clj | 333 ++++++++++++++++++----- src/libpython_clj/python/object.clj | 71 +++-- test/libpython_clj/iter_gen_seq_test.clj | 24 ++ test/libpython_clj/python_test.clj | 15 + 5 files changed, 353 insertions(+), 91 deletions(-) create mode 100644 test/libpython_clj/iter_gen_seq_test.clj diff --git a/src/libpython_clj/python.clj b/src/libpython_clj/python.clj index e33c5d7..1eaa601 100644 --- a/src/libpython_clj/python.clj +++ b/src/libpython_clj/python.clj @@ -9,6 +9,7 @@ [tech.jna :as jna]) (:import [com.sun.jna Pointer] [com.sun.jna.ptr PointerByReference] + [java.lang.reflect Field] [java.io Writer] [libpython_clj.jna PyObject CFunction$KeyWordFunction diff --git a/src/libpython_clj/python/bridge.clj b/src/libpython_clj/python/bridge.clj index 5566e90..2e695ad 100644 --- a/src/libpython_clj/python/bridge.clj +++ b/src/libpython_clj/python/bridge.clj @@ -48,10 +48,12 @@ [tech.v2.datatype.functional :as dtype-fn] [tech.v2.datatype :as dtype] [tech.resource :as resource] - [clojure.set :as c-set]) - (:import [java.util Map RandomAccess List Map$Entry Iterator] + [clojure.set :as c-set] + [clojure.tools.logging :as log]) + (:import [java.util Map RandomAccess List Map$Entry Iterator UUID] + [java.util.concurrent ConcurrentHashMap] [clojure.lang IFn Symbol Keyword Seqable - Fn MapEntry] + Fn MapEntry Range LongRange] [tech.v2.datatype ObjectReader ObjectWriter ObjectMutable ObjectIter MutableRemove] [tech.v2.datatype.typed_buffer TypedBuffer] @@ -201,6 +203,12 @@ (->jvm pyobj)) +(defmethod pyobject-as-jvm :range + [pyobj] + (->jvm pyobj)) + + + (defn- mostly-copy-arg "This is the dirty boundary between the languages. Copying as often faster for simple things but we have to be careful not to attempt a copy of things that @@ -714,28 +722,6 @@ (st/print-stack-trace e#))))))) -(defn jvm-iterator-as-python - ^Pointer [^Iterator item] - (with-gil nil - (let [next-fn #(if (.hasNext item) - (-> (.next item) - ;;As python tracks the object in a jvm context - (as-python) - (jna/->ptr-backing-store) - ;;But we are handing the object back to python which is expecting - ;;a new reference. - (pyobj/incref)) - (do - (libpy/PyErr_SetNone - (err/PyExc_StopIteration)) - nil)) - att-map - {"__next__" (pyobj/make-tuple-fn next-fn - :arg-converter nil - :result-converter nil)}] - (create-bridge-from-att-map item att-map)))) - - (defn as-python-incref "Convert to python and add a reference. Necessary for return values from functions as python expects a new reference and the as-python pathway @@ -752,65 +738,260 @@ :result-converter as-python-incref)) +(defonce ^:private jvm-handle-map (ConcurrentHashMap.)) + +(defn- make-jvm-object-handle + ^long [item] + (let [^ConcurrentHashMap hash-map jvm-handle-map] + (loop [handle (pyinterp/get-object-handle item)] + (let [handle (long handle)] + (if (not (.containsKey hash-map handle)) + (do + (.put hash-map handle item) + handle) + (recur (.hashCode (UUID/randomUUID)))))))) + +(defn- get-jvm-object + [handle] + (.get ^ConcurrentHashMap jvm-handle-map (long handle))) + +(defn- remove-jvm-object + [handle] + (.remove ^ConcurrentHashMap jvm-handle-map (long handle)) + nil) + +(defn- py-self->jvm-handle + [self] + (->jvm (py-proto/get-attr self "jvm_handle"))) + +(defn- py-self->jvm-obj + ^Object [self] + (-> (py-self->jvm-handle self) + get-jvm-object)) + +(defn- as-tuple-instance-fn + [fn-obj] + (pyobj/make-tuple-instance-fn fn-obj :result-converter as-python-incref)) + + +(defn- self->map + ^Map [self] + (py-self->jvm-obj self)) + + +(defonce mapping-type + (delay + (with-gil + (let [mod (pyinterop/import-module "collections.abc") + map-type (py-proto/get-attr mod "MutableMapping")] + ;;In order to make things work ingeneral + (pyobj/create-class + "jvm-map-as-python" + [map-type] + {"__init__" (as-tuple-instance-fn + (fn [self jvm-handle] + (py-proto/set-attr! self "jvm_handle" jvm-handle) + nil)) + "__del__" (as-tuple-instance-fn + #(try + (remove-jvm-object (py-self->jvm-handle %)) + (catch Throwable e + (log/warnf e "Error removing object")))) + "__contains__" (as-tuple-instance-fn #(.containsKey (self->map %1) (as-jvm %2))) + "__eq__" (as-tuple-instance-fn #(.equals (self->map %1) (as-jvm %2))) + "__getitem__" (as-tuple-instance-fn + #(do (println "getting" (as-jvm %2)) + (.get (self->map %1) (as-jvm %2)))) + "__setitem__" (as-tuple-instance-fn #(.put (self->map %1) (as-jvm %2) %3)) + "__delitem__" (as-tuple-instance-fn #(.remove (self->map %1) (as-jvm %2))) + "__hash__" (as-tuple-instance-fn #(.hashCode (self->map %1))) + "__iter__" (as-tuple-instance-fn #(.iterator ^Iterable (keys (self->map %1)))) + "__len__" (as-tuple-instance-fn #(.size (self->map %1))) + "__str__" (as-tuple-instance-fn #(.toString (self->map %1))) + "clear" (as-tuple-instance-fn #(.clear (self->map %1))) + "keys" (as-tuple-instance-fn #(seq (.keySet (self->map %1)))) + "values" (as-tuple-instance-fn #(seq (.values (self->map %1)))) + "pop" (as-tuple-instance-fn #(.remove (self->map %1) (as-jvm %2)))}))))) + + (defn jvm-map-as-python - ^Pointer [^Map jvm-data] + [^Map jvm-data] (with-gil - (let [att-map - {"__contains__" (as-py-fn #(.containsKey jvm-data %)) - "__eq__" (as-py-fn #(.equals jvm-data %)) - "__getitem__" (as-py-fn #(.get jvm-data %)) - "__setitem__" (as-py-fn #(.put jvm-data %1 %2)) - "__delitem__" (as-py-fn #(.remove jvm-data %)) - "__hash__" (as-py-fn #(.hashCode jvm-data)) - "__iter__" (as-py-fn #(.iterator ^Iterable (keys jvm-data))) - "__len__" (as-py-fn #(.size jvm-data)) - "__str__" (as-py-fn #(.toString jvm-data)) - "clear" (as-py-fn #(.clear jvm-data)) - "keys" (as-py-fn #(seq (.keySet jvm-data))) - "values" (as-py-fn #(seq (.values jvm-data))) - "pop" (as-py-fn #(.remove jvm-data %))}] - (create-bridge-from-att-map jvm-data att-map)))) + (py-proto/call (jna/as-ptr (deref mapping-type)) (make-jvm-object-handle jvm-data)))) + + +(defmethod py-proto/pyobject->jvm :jvm-map-as-python + [pyobj] + (py-self->jvm-obj pyobj)) + + +(defmethod pyobject-as-jvm :jvm-map-as-python + [pyobj] + (->jvm pyobj)) + + +(defn- self->list + ^List [self] + (py-self->jvm-obj self)) + +(defonce sequence-type + (delay + (let [mod (pyinterop/import-module "collections.abc") + seq-type (py-proto/get-attr mod "MutableSequence")] + (pyobj/create-class + "jvm-list-as-python" + [seq-type] + {"__init__" (as-tuple-instance-fn + (fn [self jvm-handle] + (py-proto/set-attr! self "jvm_handle" jvm-handle) + nil)) + "__del__" (as-tuple-instance-fn + #(try + (remove-jvm-object (py-self->jvm-handle %)) + (catch Throwable e + (log/warnf e "Error removing object")))) + "__contains__" (as-tuple-instance-fn #(.contains (self->list %1) %2)) + "__eq__" (as-tuple-instance-fn #(.equals (self->list %1) (->jvm %2))) + "__getitem__" (as-tuple-instance-fn #(.get (self->list %1) (int (->jvm %2)))) + "__setitem__" (as-tuple-instance-fn #(.set (self->list %1) + (int (->jvm %2)) + (as-jvm %3))) + "__delitem__" (as-tuple-instance-fn #(.remove (self->list %1) + (int (->jvm %2)))) + "__hash__" (as-tuple-instance-fn #(.hashCode (self->list %1))) + "__iter__" (as-tuple-instance-fn #(.iterator (self->list %1))) + "__len__" (as-tuple-instance-fn #(.size (self->list %1))) + "__str__" (as-tuple-instance-fn #(.toString (self->list %1))) + "clear" (as-tuple-instance-fn #(.clear (self->list %1))) + "sort" (as-tuple-instance-fn #(.sort (self->list %1) nil)) + "append" (as-tuple-instance-fn #(.add (self->list %1) %2)) + "insert" (as-tuple-instance-fn #(.add (self->list %1) (int (->jvm %2)) %3)) + "pop" (as-tuple-instance-fn + (fn [self & args] + (let [jvm-data (self->list self) + args (map ->jvm args) + index (int (if (first args) + (first args) + -1)) + index (if (< index 0) + (- (.size jvm-data) index) + index)] + #(.remove jvm-data index))))})))) (defn jvm-list-as-python - ^Pointer [^List jvm-data] + [^List jvm-data] (with-gil - (let [att-map - {"__contains__" (as-py-fn #(.contains jvm-data %)) - "__eq__" (as-py-fn #(.equals jvm-data %)) - "__getitem__" (as-py-fn #(.get jvm-data (int %))) - "__setitem__" (as-py-fn #(.set jvm-data (int %1) %2)) - "__delitem__" (as-py-fn #(.remove jvm-data (int %))) - "__hash__" (as-py-fn #(.hashCode jvm-data)) - "__iter__" (as-py-fn #(.iterator jvm-data)) - "__len__" (as-py-fn #(.size jvm-data)) - "__str__" (as-py-fn #(.toString jvm-data)) - "clear" (as-py-fn #(.clear jvm-data)) - "sort" (as-py-fn #(.sort jvm-data nil)) - "append" (as-py-fn #(.add jvm-data %)) - "insert" (as-py-fn #(.add jvm-data (int %1) %2)) - "pop" (as-py-fn (fn [& args] - (let [index (int (if (first args) - (first args) - -1)) - index (if (< index 0) - (- (.size jvm-data) index) - index)] - #(.remove jvm-data index))))}] - (create-bridge-from-att-map jvm-data att-map)))) + (py-proto/call (jna/as-ptr (deref sequence-type)) (make-jvm-object-handle jvm-data)))) + + +(defmethod py-proto/pyobject->jvm :jvm-list-as-python + [pyobj] + (py-self->jvm-obj pyobj)) + + +(defmethod pyobject-as-jvm :jvm-list-as-python + [pyobj] + (->jvm pyobj)) + + +(defonce iterable-type + (delay + (with-gil + (let [mod (pyinterop/import-module "collections.abc") + iter-base-cls (py-proto/get-attr mod "Iterable")] + (pyobj/create-class + "jvm-iterable-as-python" + [iter-base-cls] + {"__init__" (as-tuple-instance-fn + (fn [self jvm-handle] + (py-proto/set-attr! self "jvm_handle" jvm-handle) + nil)) + "__del__" (as-tuple-instance-fn + #(try + (remove-jvm-object (py-self->jvm-handle %)) + (catch Throwable e + (log/warnf e "Error removing object")))) + "__iter__" (as-tuple-instance-fn + #(.iterator ^Iterable (py-self->jvm-obj %))) + "__eq__" (as-tuple-instance-fn #(.equals (py-self->jvm-obj %1) + (as-jvm %2))) + "__hash__" (as-tuple-instance-fn + #(.hashCode (py-self->jvm-obj %))) + "__str__" (as-tuple-instance-fn + #(.toString (py-self->jvm-obj %)))}))))) (defn jvm-iterable-as-python - ^Pointer [^Iterable jvm-data] + [^Iterable jvm-data] (with-gil - (let [att-map - {"__iter__" (as-py-fn #(.iterator jvm-data)) - "__eq__" (as-py-fn #(.equals jvm-data %)) - "__hash__" (as-py-fn #(.hashCode jvm-data)) - "__str__" (as-py-fn #(.toString jvm-data))}] - (create-bridge-from-att-map jvm-data att-map)))) + (py-proto/call (jna/as-ptr (deref iterable-type)) (make-jvm-object-handle jvm-data)))) +(defmethod py-proto/pyobject->jvm :jvm-iterable-as-python + [pyobj] + (py-self->jvm-obj pyobj)) + + +(defmethod pyobject-as-jvm :jvm-iterable-as-python + [pyobj] + (->jvm pyobj)) + + +(defonce iterator-type + (delay + (with-gil + (let [mod (pyinterop/import-module "collections.abc") + iter-base-cls (py-proto/get-attr mod "Iterator") + next-fn (fn [self] + (let [^Iterator item (py-self->jvm-obj self)] + (if (.hasNext item) + (-> (.next item) + ;;As python tracks the object in a jvm context + (as-python) + (jna/->ptr-backing-store) + ;;But we are handing the object back to python which is expecting + ;;a new reference. + (pyobj/incref)) + (do + (libpy/PyErr_SetNone + (err/PyExc_StopIteration)) + nil))))] + (pyobj/create-class + "jvm-iterator-as-python" + [iter-base-cls] + {"__init__" (as-tuple-instance-fn + (fn [self jvm-handle] + (py-proto/set-attr! self "jvm_handle" jvm-handle) + nil)) + "__del__" (as-tuple-instance-fn + #(try + (remove-jvm-object (py-self->jvm-handle %)) + (catch Throwable e + (log/warnf e "Error removing object")))) + "__next__" (pyobj/make-tuple-instance-fn + next-fn + ;;In this case we are explicitly taking care of all conversions + ;;to python and back so we do not ask for any converters. + :arg-converter nil + :result-converter nil)}))))) + + +(defn jvm-iterator-as-python + [^Iterator jvm-data] + (with-gil + (py-proto/call (jna/as-ptr (deref iterator-type)) (make-jvm-object-handle jvm-data)))) + + +(defmethod py-proto/pyobject->jvm :jvm-iterator-as-python + [pyobj] + (py-self->jvm-obj pyobj)) + + +(defmethod pyobject-as-jvm :jvm-iterator-as-python + [pyobj] + (->jvm pyobj)) + (extend-protocol py-proto/PBridgeToPython Number (as-python [item options] (->python item)) @@ -822,6 +1003,14 @@ (as-python [item options] (->python item)) Boolean (as-python [item options] (->python item)) + Range + (as-python [item options] + (if (casting/integer-type? (dtype/get-datatype item)) + (->python item options) + (jvm-iterable-as-python item))) + LongRange + (as-python [item options] + (->python item options)) Iterable (as-python [item options] (cond diff --git a/src/libpython_clj/python/object.clj b/src/libpython_clj/python/object.clj index 8c9dbc7..147a99c 100644 --- a/src/libpython_clj/python/object.clj +++ b/src/libpython_clj/python/object.clj @@ -44,6 +44,7 @@ [clojure.tools.logging :as log]) (:import [com.sun.jna Pointer CallbackReference] [com.sun.jna.ptr PointerByReference] + [java.lang.reflect Field] [libpython_clj.jna PyObject CFunction$KeyWordFunction @@ -58,7 +59,8 @@ [clojure.lang Symbol Keyword IPersistentMap IPersistentVector - IPersistentSet])) + IPersistentSet + Range LongRange])) (set! *warn-on-reflection* true) @@ -574,6 +576,13 @@ new-cls (py-proto/call (libpy/PyType_Type) name bases cls-dict)] (py-proto/as-jvm new-cls nil)))) +(def ^:private lr-step-field (doto (.getDeclaredField ^Class LongRange "step") + (.setAccessible true))) + + +(def ^:private r-step-field (doto (.getDeclaredField ^Class Range "step") + (.setAccessible true))) + (extend-protocol py-proto/PCopyToPython Number @@ -595,6 +604,20 @@ (if item (py-true) (py-false))) + Range + (->python [item options] + (if (casting/integer-type? (dtype/get-datatype item)) + (let [start (first item) + step (.get ^Field r-step-field item) + stop (+ start (* step (count item)))] + (py-proto/call (libpy/PyRange_Type) start stop step)) + (->py-list item))) + LongRange + (->python [item options] + (let [start (first item) + step (.get ^Field lr-step-field item) + stop (+ start (* step (count item)))] + (py-proto/call (libpy/PyRange_Type) start stop step))) Iterable (->python [item options] (cond @@ -1033,26 +1056,36 @@ (numpy-scalar->jvm pyobj)) +(defmethod pyobject->jvm :range + [pyobj] + (with-gil + (let [start (->jvm (py-proto/get-attr pyobj "start")) + step (->jvm (py-proto/get-attr pyobj "step")) + stop (->jvm (py-proto/get-attr pyobj "stop"))] + (range start stop step)))) + + (defmethod pyobject->jvm :default [pyobj] - (cond - ;;Things could implement mapping and sequence logically so mapping - ;;takes precedence - (= 1 (libpy/PyMapping_Check pyobj)) - (if-let [map-items (try (-> (libpy/PyMapping_Items pyobj) - wrap-pyobject) - (catch Throwable e nil))] - (python->jvm-copy-hashmap pyobj map-items) - (do - ;;Ignore error. The mapping check isn't thorough enough to work. - (libpy/PyErr_Clear) - (python->jvm-copy-persistent-vector pyobj))) - ;;Sequences become persistent vectors - (= 1 (libpy/PySequence_Check pyobj)) - (python->jvm-copy-persistent-vector pyobj) - :else - {:type (python-type pyobj) - :value (Pointer/nativeValue (jna/as-ptr pyobj))})) + (with-gil + (cond + ;;Things could implement mapping and sequence logically so mapping + ;;takes precedence + (= 1 (libpy/PyMapping_Check pyobj)) + (if-let [map-items (try (-> (libpy/PyMapping_Items pyobj) + wrap-pyobject) + (catch Throwable e nil))] + (python->jvm-copy-hashmap pyobj map-items) + (do + ;;Ignore error. The mapping check isn't thorough enough to work. + (libpy/PyErr_Clear) + (python->jvm-copy-persistent-vector pyobj))) + ;;Sequences become persistent vectors + (= 1 (libpy/PySequence_Check pyobj)) + (python->jvm-copy-persistent-vector pyobj) + :else + {:type (python-type pyobj) + :value (Pointer/nativeValue (jna/as-ptr pyobj))}))) (defn is-instance? diff --git a/test/libpython_clj/iter_gen_seq_test.clj b/test/libpython_clj/iter_gen_seq_test.clj new file mode 100644 index 0000000..433f520 --- /dev/null +++ b/test/libpython_clj/iter_gen_seq_test.clj @@ -0,0 +1,24 @@ +(ns libpython-clj.iter-gen-seq-test + "Iterators/sequences and such are crucial to handling lots of forms + of data and thus they have to work correctly between the languages." + (:require [libpython-clj.python :as py] + [libpython-clj.require :refer [require-python]] + [clojure.test :refer :all])) + + +(require-python '[builtins :as python]) + +(deftest generate-sequence-test + (let [{{:strs [fortwice]} :globals} + (py/run-simple-string " +def fortwice(xs): + for x in xs: + yield x + for x in xs: + yield x")] + (is (= [1 2 3 4 5 6 7 8 9 10] + (vec (fortwice (python/map inc (range 10)))))) ;;=> [1 2 3 4 5 6 7 8 9 10] + (is (= [1 2 3 4 5 6 7 8 9 10 1 2 3 4 5 6 7 8 9 10] + (vec (fortwice (map inc (range 10)))))) ;;=> [1 2 3 4 5 6 7 8 9 10 1 2 3 4 5 6 7 8 9 10] + (is (= [1 2 3 4 5 6 7 8 9 10 1 2 3 4 5 6 7 8 9 10] + (vec (fortwice (vec (python/map inc (range 10))))))))) diff --git a/test/libpython_clj/python_test.clj b/test/libpython_clj/python_test.clj index 57e9497..e5983e1 100644 --- a/test/libpython_clj/python_test.clj +++ b/test/libpython_clj/python_test.clj @@ -319,3 +319,18 @@ (Pointer/nativeValue (jna/as-ptr ->false)))) (is (= (Pointer/nativeValue (jna/as-ptr py-false)) (Pointer/nativeValue (jna/as-ptr as-false)))))) + + +(deftest instance-abc-classes + (let [py-dict (py/->python {"a" 1 "b" 2}) + bridged-dict (py/as-python {"a" 1 "b" 2}) + bridged-iter (py/as-python (repeat 5 1)) + bridged-list (py/as-python (vec (range 10))) + pycol (py/import-module "collections") + mapping-type (py/get-attr pycol "Mapping") + iter-type (py/get-attr pycol "Iterable") + sequence-type (py/get-attr pycol "Sequence")] + (is (py/is-instance? py-dict mapping-type)) + (is (py/is-instance? bridged-dict mapping-type)) + (is (py/is-instance? bridged-iter iter-type)) + (is (py/is-instance? bridged-list sequence-type)))) From 4e35ffe5fd2ca08a19473268159ef9f7dc5cd616 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 17 Dec 2019 13:58:54 -0700 Subject: [PATCH 099/456] 1.28 --- CHANGELOG.md | 11 +++++++++++ project.clj | 2 +- test/libpython_clj/iter_gen_seq_test.clj | 4 ++-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed73d5f..94c425f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,17 @@ # Time for a ChangeLog! +## 1.28 + + +* `(range 5)` - clojure ranges <-> python ranges when possible. +* bridged types derive from collections.abc.* so that they pass instance checks in + libraries that are checking for generic types. +* Really interesting unit test for + [generators, ranges and sequences](test/libpython-clj.iter_gen_seq_test.clj). + + + ## 1.27 * Fixed bug where (as-python {:is_train false}) results in a dictionary with a none diff --git a/project.clj b/project.clj index b82d48e..0d82858 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.28-SNAPSHOT" +(defproject cnuernber/libpython-clj "1.28" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" diff --git a/test/libpython_clj/iter_gen_seq_test.clj b/test/libpython_clj/iter_gen_seq_test.clj index 433f520..dfa8006 100644 --- a/test/libpython_clj/iter_gen_seq_test.clj +++ b/test/libpython_clj/iter_gen_seq_test.clj @@ -17,8 +17,8 @@ def fortwice(xs): for x in xs: yield x")] (is (= [1 2 3 4 5 6 7 8 9 10] - (vec (fortwice (python/map inc (range 10)))))) ;;=> [1 2 3 4 5 6 7 8 9 10] + (vec (fortwice (python/map inc (range 10)))))) (is (= [1 2 3 4 5 6 7 8 9 10 1 2 3 4 5 6 7 8 9 10] - (vec (fortwice (map inc (range 10)))))) ;;=> [1 2 3 4 5 6 7 8 9 10 1 2 3 4 5 6 7 8 9 10] + (vec (fortwice (map inc (range 10)))))) (is (= [1 2 3 4 5 6 7 8 9 10 1 2 3 4 5 6 7 8 9 10] (vec (fortwice (vec (python/map inc (range 10))))))))) From 54965e0a0cf42738e7d907da180b9d7995781f8c Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 17 Dec 2019 13:59:02 -0700 Subject: [PATCH 100/456] snap --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 0d82858..b069c92 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.28" +(defproject cnuernber/libpython-clj "1.29-SNAPSHOT" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" From 547ab6dae5f4bea580166da978ffc9f2bd06a5a1 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 17 Dec 2019 16:02:44 -0700 Subject: [PATCH 101/456] editing --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94c425f..1fcbf06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,7 @@ * [python startup work](https://github.com/cnuernber/libpython-clj/commit/16da3d885f29bde59ea219c9438b9d3654387971) -* [python generates & clojure transducers](https://github.com/cnuernber/libpython-clj/pull/27) +* [python generators & clojure transducers](https://github.com/cnuernber/libpython-clj/pull/27) * [requre-python reload fix](https://github.com/cnuernber/libpython-clj/pull/24) * Bugfix with require-python :reload semantics. From 043b868cde92ef3c286d6d6b129e7ebdcbd6650c Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 17 Dec 2019 16:03:35 -0700 Subject: [PATCH 102/456] editing --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1fcbf06..f9cb3c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ * bridged types derive from collections.abc.* so that they pass instance checks in libraries that are checking for generic types. * Really interesting unit test for - [generators, ranges and sequences](test/libpython-clj.iter_gen_seq_test.clj). + [generators, ranges and sequences](test/libpython_clj.iter_gen_seq_test.clj). From bc4eb1329bc7b97bc1db999ebe4c2ee02cfc0c1e Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 17 Dec 2019 16:03:56 -0700 Subject: [PATCH 103/456] editing --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9cb3c6..86e0e6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ * bridged types derive from collections.abc.* so that they pass instance checks in libraries that are checking for generic types. * Really interesting unit test for - [generators, ranges and sequences](test/libpython_clj.iter_gen_seq_test.clj). + [generators, ranges and sequences](test/libpython_clj/iter_gen_seq_test.clj). From 84d704e84a02cd48bf8cc5e28151bcc9462b5a47 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Wed, 18 Dec 2019 07:26:09 -0700 Subject: [PATCH 104/456] Added error checking for misspelled flags --- src/libpython_clj/require.clj | 16 ++++++++++++++-- test/libpython_clj/require_python_test.clj | 1 - 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/libpython_clj/require.clj b/src/libpython_clj/require.clj index a340f42..fd68b5c 100644 --- a/src/libpython_clj/require.clj +++ b/src/libpython_clj/require.clj @@ -49,6 +49,8 @@ (def ^:private pyclass? (py/get-attr inspect "isclass")) +(def ^:private pymodule? (py/get-attr inspect "ismodule")) + (defn ^:private py-fn-argspec [f] (if-let [spec (try (argspec f) (catch Throwable e nil))] {:args (py/->jvm (py/get-attr spec "args")) @@ -195,7 +197,17 @@ boolean flags '[foo :reload true] to support Clojure style 'require syntax. Possibly overengineered." [supported-flags reqs] - + ;; Ensure we error out when flags passed in are mistyped. + ;; First attempt is to filter keywords and make sure any keywords are + ;; in supported-flags + (let [total-flags (set (concat supported-flags [:as :refer :exclude + :* :all]))] + (when-let [missing-flags (->> reqs + (filter #(and (not (total-flags %)) + (keyword? %))) + seq)] + (throw (Exception. (format "Unsupported flags: %s" + (set missing-flags)))))) (letfn [(supported-flag-item ;; scanned a supported tag token [supported-flags flag results item items] @@ -479,6 +491,7 @@ (intern cls-ns (symbol attribute-name) attribute) cls-ns-config)) + (defn- bind-class-namespaces! [lib-config] (let [class-namespace-configs @@ -609,4 +622,3 @@ (load-python-lib (vector reqs)) (vector? reqs) (load-python-lib reqs))) - diff --git a/test/libpython_clj/require_python_test.clj b/test/libpython_clj/require_python_test.clj index 68771b5..95e0cec 100644 --- a/test/libpython_clj/require_python_test.clj +++ b/test/libpython_clj/require_python_test.clj @@ -31,7 +31,6 @@ (is (= #{:reload} (parse-flags #{:reload} '[:reload foo]))) - (is (= #{} (parse-flags #{} '[:reload foo]))) (is (= #{:reload} (parse-flags #{:reload} '[:reload true]))) (is (= #{:reload} (parse-flags #{:reload} '[:reload :as foo]))) (is (= #{:reload} (parse-flags #{:reload} '[:reload foo :as]))) From dd5b40912c3d01eb9a2f6ea79d33a4ecf5c3b2f7 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Thu, 19 Dec 2019 10:15:26 -0700 Subject: [PATCH 105/456] Instructions to install pip3 packages and cleaning up comments. --- dockerfiles/CondaDockerfile | 10 +++++++--- scripts/run-conda-docker | 4 +--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/dockerfiles/CondaDockerfile b/dockerfiles/CondaDockerfile index 1320a77..906bf88 100644 --- a/dockerfiles/CondaDockerfile +++ b/dockerfiles/CondaDockerfile @@ -10,7 +10,6 @@ RUN apt-get -qq update && apt-get -qq -y install curl wget bzip2 openjdk-8-jdk-h && rm -rf /tmp/miniconda.sh \ && conda install -y python=3 \ && conda update conda \ - && conda install -y mxnet numpy \ && curl -O https://download.clojure.org/install/linux-install-1.10.1.492.sh \ && chmod +x linux-install-1.10.1.492.sh \ && ./linux-install-1.10.1.492.sh && rm linux-install-1.10.1.492.sh \ @@ -34,5 +33,10 @@ RUN groupadd -g $GROUPID $USERNAME RUN useradd -u $USERID -g $GROUPID $USERNAME RUN mkdir /home/$USERNAME && chown $USERNAME:$USERNAME /home/$USERNAME USER $USERNAME -RUN conda create -n pyclj python=3.6 && conda install -n pyclj numpy mxnet\ - && echo "source activate pyclj" > /home/$USERNAME/.bashrc \ No newline at end of file + +RUN conda create -n pyclj python=3.6 && conda install -n pyclj numpy mxnet \ + && echo "source activate pyclj" > /home/$USERNAME/.bashrc + + +## To install pip packages into the pyclj environment do +RUN conda run -n pyclj python3 -mpip install xyz \ No newline at end of file diff --git a/scripts/run-conda-docker b/scripts/run-conda-docker index 0e1688c..479e83e 100755 --- a/scripts/run-conda-docker +++ b/scripts/run-conda-docker @@ -1,8 +1,6 @@ #!/bin/bash -## This is incomplete, you still have to mess around with pythonpath and such. The -## goal at this stage is to get this docker container to launch a repl that allows -## initialize! to work out of the box. +scripts/build-conda-docker docker run --rm -it -u $(id -u):$(id -g) \ -e LEIN_REPL_HOST="0.0.0.0" \ From c9bd7aa9c984fcc707e0d926f6b4c1e1a421b47c Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Thu, 19 Dec 2019 10:21:24 -0700 Subject: [PATCH 106/456] working container --- dockerfiles/CondaDockerfile | 2 +- scripts/conda-repl | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dockerfiles/CondaDockerfile b/dockerfiles/CondaDockerfile index 906bf88..385da77 100644 --- a/dockerfiles/CondaDockerfile +++ b/dockerfiles/CondaDockerfile @@ -39,4 +39,4 @@ RUN conda create -n pyclj python=3.6 && conda install -n pyclj numpy mxnet \ ## To install pip packages into the pyclj environment do -RUN conda run -n pyclj python3 -mpip install xyz \ No newline at end of file +RUN conda run -n pyclj python3 -mpip install numpy \ No newline at end of file diff --git a/scripts/conda-repl b/scripts/conda-repl index 94e04c0..17a78c8 100755 --- a/scripts/conda-repl +++ b/scripts/conda-repl @@ -6,6 +6,7 @@ source activate pyclj ## https://github.com/conda/conda/issues/9500#issuecomment-565753807 export LD_LIBRARY_PATH="$(python3-config --prefix)/lib" + lein update-in :dependencies conj \[nrepl\ \"0.6.0\"\]\ -- update-in :plugins conj \[cider/cider-nrepl\ \"0.22.4\"\]\ -- repl :headless :host localhost From 6819ccb22e17262be022e4797c17a912d0a6ab99 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Thu, 19 Dec 2019 11:30:42 -0700 Subject: [PATCH 107/456] 1.29 --- CHANGELOG.md | 7 ++++++- project.clj | 2 +- src/libpython_clj/python/object.clj | 14 +++++++++----- test/libpython_clj/python_test.clj | 8 ++++++++ 4 files changed, 24 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 86e0e6e..4a2297b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,18 @@ # Time for a ChangeLog! +## 1.20 + +* Found/fixed issue with ->jvm and large python dictionaries. + + ## 1.28 * `(range 5)` - clojure ranges <-> python ranges when possible. * bridged types derive from collections.abc.* so that they pass instance checks in libraries that are checking for generic types. -* Really interesting unit test for +* Really interesting unit test for [generators, ranges and sequences](test/libpython_clj/iter_gen_seq_test.clj). diff --git a/project.clj b/project.clj index b069c92..256dd38 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.29-SNAPSHOT" +(defproject cnuernber/libpython-clj "1.29" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" diff --git a/src/libpython_clj/python/object.clj b/src/libpython_clj/python/object.clj index 147a99c..f5f65b7 100644 --- a/src/libpython_clj/python/object.clj +++ b/src/libpython_clj/python/object.clj @@ -980,15 +980,19 @@ (let [ppos (jna/size-t-ref 0) pkey (PointerByReference.) pvalue (PointerByReference.) - retval (transient {})] + retval (java.util.ArrayList.)] + ;;Dictionary iteration doesn't appear to be reentrant so we have + ;;to do 2 passes. (loop [next-retval (libpy/PyDict_Next pyobj ppos pkey pvalue)] (if (not= 0 next-retval) (do - (assoc! retval - (->jvm (jna/as-ptr pkey)) - (->jvm (jna/as-ptr pvalue))) + (.add retval [(jna/as-ptr pkey) + (jna/as-ptr pvalue)]) (recur (libpy/PyDict_Next pyobj ppos pkey pvalue))) - (persistent! retval)))))) + (->> retval + (map (fn [[k v]] + [(->jvm k) (->jvm v)])) + (into {}))))))) (defmethod pyobject->jvm :set diff --git a/test/libpython_clj/python_test.clj b/test/libpython_clj/python_test.clj index e5983e1..2553fbf 100644 --- a/test/libpython_clj/python_test.clj +++ b/test/libpython_clj/python_test.clj @@ -334,3 +334,11 @@ (is (py/is-instance? bridged-dict mapping-type)) (is (py/is-instance? bridged-iter iter-type)) (is (py/is-instance? bridged-list sequence-type)))) + + +(deftest nested-map-and-back + (let [py-dict (-> (py/run-simple-string "testdata={'camera_id': 'CODOT-10106-12067', 'country': 'United States', 'state': 'Colorado', 'city': 'Fountain', 'provider': 'CO DOT', 'description': '0.6 mi N of Ray Nixon Rd Int', 'direction': 'North', 'video': False, 'links': {'jpeg': {'url': 'https://www.cotrip.org/dimages/camera?imageURL=remote/CTMCCAM025S125-20-N.jpg'}}, 'tags': ['auto_rerated'], 'ratings': {'road_weather': 4, 'visibility': 4}, 'health': {}, 'created_at': '2016-04-19T20:27:45.030Z', 'time_zone_offset': -25200}") + (get-in [:globals "testdata"])) + jvm-dict (py/->jvm py-dict)] + (is (= (get jvm-dict "ratings") + (py/->jvm (py/get-item py-dict "ratings")))))) From dd41a2a48e3ac3456f654b5a2b5f87b050c65f91 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Thu, 19 Dec 2019 11:32:57 -0700 Subject: [PATCH 108/456] snap --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 256dd38..f3a93f5 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.29" +(defproject cnuernber/libpython-clj "1.30-SNAPSHOT" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" From b6c0bc69d12c39760d3f43449a84b45e8df1dfe9 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Thu, 19 Dec 2019 15:20:42 -0700 Subject: [PATCH 109/456] editing --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a2297b..0cceae8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Time for a ChangeLog! -## 1.20 +## 1.29 * Found/fixed issue with ->jvm and large python dictionaries. From a11cdcd4d3c499d62ba236c95480bbb3d40842fb Mon Sep 17 00:00:00 2001 From: "J.J. Tolton" Date: Fri, 20 Dec 2019 10:37:02 -0500 Subject: [PATCH 110/456] Issue-35a: (require-python 'module.class) and '[module.class :as module-alias.class-alias] (#37) * add require module.class syntax * wrap findspec to prevent bootstrapping issue on importlib.util * wrap findspec to prevent bootstrapping issue on importlib.util --- src/libpython_clj/require.clj | 236 ++++++++++++++++++--- test/libpython_clj/require_python_test.clj | 32 +++ 2 files changed, 236 insertions(+), 32 deletions(-) diff --git a/src/libpython_clj/require.clj b/src/libpython_clj/require.clj index fd68b5c..b7e1363 100644 --- a/src/libpython_clj/require.clj +++ b/src/libpython_clj/require.clj @@ -1,10 +1,14 @@ (ns libpython-clj.require (:refer-clojure :exclude [fn? doc]) (:require [libpython-clj.python :as py] - [clojure.tools.logging :as log])) + [clojure.tools.logging :as log] + [clojure.java.io :as io])) (py/initialize!) +;; for hot reloading multimethod in development +(ns-unmap 'libpython-clj.require 'intern-ns-class) + (def ^:private builtins (py/import-module "builtins")) (def ^:private inspect (py/import-module "inspect")) @@ -38,6 +42,8 @@ (def ^:private importlib (py/import-module "importlib")) +(def ^:private importlib_util (py/import-module "importlib.util")) + (def ^:private reload-module (py/get-attr importlib "reload")) (def ^:priviate import-module (py/get-attr importlib "import_module")) @@ -51,6 +57,45 @@ (def ^:private pymodule? (py/get-attr inspect "ismodule")) +(defn ^:private findspec [x] + (let [-findspec + (-> importlib_util (py/get-attr "find_spec"))] + (-findspec x))) + +(def ^:private hasattr (py/get-attr builtins "hasattr")) + +(defn ^:private module-string? [x] + (try + (findspec x) + (catch Throwable e + false))) + +(defn module-path-string + "Given a.b, return a + Given a.b.c, return a.b + Given a.b.c.d, return a.b.c etc." + [x] + (clojure.string/join + "." + (pop (clojure.string/split (str x) #"[.]")))) + +(defn module-path-last-string + "Given a.b.c.d, return d" + [x] + (last (clojure.string/split (str x) #"[.]"))) + +(defn ^:private possible-class? + "The presumption here is that, given a symbol like 'builtins.list, + if 'builtins.list does not have a __path__ attribute but 'builtins does, + it's possible that 'list is a class of 'builtins. + + This is a slightly cheaper test than a try/except wrapper around + a module." + [pos-class] + (let [module? (module-path-string pos-class)] + (and (not (module-string? pos-class)) + (boolean (module-string? module?))))) + (defn ^:private py-fn-argspec [f] (if-let [spec (try (argspec f) (catch Throwable e nil))] {:args (py/->jvm (py/get-attr spec "args")) @@ -296,6 +341,21 @@ (vec missing))))) refer))) +(defn import-module-or-cls! [x] + (if (possible-class? x) + (let [module? (module-path-string x) + class? (module-path-last-string x)] + (try + (let [m (import-module module?) + cls (py/get-attr m class?)] + (if (nil? cls) + (throw (Exception. "Unable to load " x))) + [m cls]) + (catch Exception e + (log/error (str "Unable to load " x)) + (throw e)))) + [(import-module (str x)) nil])) + (defn- python-lib-configuration "Build a configuration map of a python library. Current ns is option and used during testing but unnecessary during normal running events." @@ -324,20 +384,52 @@ current-ns (or current-ns *ns*) current-ns-sym (symbol (str current-ns)) python-namespace (find-ns module-name-or-ns) - this-module (import-module (str module-name))] + [this-module cls] (import-module-or-cls! module-name)] + {:supported-flags supported-flags :etc etc :reload? reload? :no-arglists? no-arglists? - :load-ns-classes? load-ns-classes? - :module-name module-name - :module-name-or-ns module-name-or-ns + :load-ns-classes? true + + ;; if something like 'builtins.list was requested, + ;; the module name is 'builtins + :module-name (if (pyclass? cls) + (module-path-string module-name) + module-name) + + ;; if something like 'builtins.list was requested, + ;; the module-name is 'builtins + ;; or in the situation of + ;; '[builtins.list :as python.pylist] + ;; 'python would be bound to :module-name-or-ns + :module-name-or-ns (if (and (pyclass? cls) + (or (symbol? module-name-or-ns) + (string? module-name-or-ns))) + (module-path-string module-name-or-ns) + module-name-or-ns) + + ;; presumes possibilty of alias in situations like + ;; '[builtins.list :as python.pylist] + ;; 'pylist would be bound to :class-name-or-ns in this case + + :class-name-or-ns (if (and (pyclass? cls) + (or (symbol? module-name-or-ns) + (string? module-name-or-ns))) + (module-path-last-string module-name-or-ns) + nil) + :exclude exclude :refer refer :current-ns current-ns :current-ns-sym current-ns-sym :python-namespace python-namespace - :this-module this-module})) + :this-module this-module + :module? (pymodule? this-module) + :class? (pyclass? cls) + :class-name (when (pyclass? cls) + (module-path-last-string + module-name))})) (defn- extract-public-data [{:keys [exclude python-namespace module-name-or-ns]}] @@ -364,8 +456,13 @@ this-module module-name module-name-or-ns] + cls? :class? python-namespace :python-namespace}] + + ;; in the event of '[builtins.list :as python.pylist] + ;; this function is a no-op (cond + cls? nil reload? (reload-python-ns! module-name this-module module-name-or-ns) @@ -373,21 +470,29 @@ (defn enhanced-python-lib-configuration [{:keys [python-namespace exclude this-module] + cls? :class? :as lib-config}] - (let [public-data (extract-public-data lib-config)] - (merge - lib-config - {:public-data public-data - :refer-symbols (extract-refer-symbols lib-config - public-data)}))) + ;; in the even of '[builtins.list :as python.pylist] + ;; or something similar, we do not provide enhanced + ;; data + (if cls? + lib-config + (let [public-data (extract-public-data lib-config)] + (merge + lib-config + {:public-data public-data + :refer-symbols (extract-refer-symbols + lib-config + public-data)})))) (defn- bind-py-symbols-to-ns! [{:keys [reload? python-namespace this-module module-name-or-ns - no-arglists?]}] - (when (or reload? (not python-namespace)) + no-arglists?] + cls? :class?}] + (when (and (or reload? (not python-namespace)) (not cls?)) ;;Mutably define the root namespace. (doseq [[att-name v] (vars this-module)] (try @@ -401,20 +506,59 @@ (log/warnf e "Failed to require symbol %s" att-name)))))) (defn- bind-module-ns! - [{:keys [current-ns-sym module-name-or-ns this-module]}] - (intern current-ns-sym - (with-meta module-name-or-ns - {:doc (doc this-module)}) - this-module)) + [{:keys [current-ns-sym + module-name-or-ns + class-name-or-ns + this-module] + cls? :class?}] + + ;; in event of '[builtins.list :as python.pylist] + ;; this is a no-op + (when (not cls?) + (intern current-ns-sym + (with-meta module-name-or-ns + {:doc (doc this-module)}) + this-module))) (defn- generate-class-namespace-configs - [{:keys [module-name-or-ns this-module] :as lib-config}] - (letfn [(pyclass-pair? [[attr attr-val]] (pyclass? attr-val)) + [{:keys [module-name-or-ns + this-module + class-name + class-name-or-ns + ] + cls? :class? + :as lib-config}] + (letfn [(pyclass-pair? [[attr attr-val]] + + (or + + ;; in the event that something like + ;; builtins.list was requested, + ;; we do not want to bind every + ;; class in the module to the ns, only the + ;; requested class + + (and cls? + (pyclass? attr-val) + (and (hasattr attr-val "__name__") + (= class-name + (py/get-attr + attr-val + "__name__")))) + + ;; if a module like 'builtins was specified + ;; without a class like 'builtins.list, + ;; we bind every class in the module to the module-ns + + (and (not cls?) + (pyclass? attr-val)))) (pyclass-ns-config [[attr attr-val]] {:namespace module-name-or-ns :attribute-type :class - :classname attr + :classname attr :class-symbol (symbol attr) + :class-binding (or class-name-or-ns + class-name) :class attr-val :attributes ((comp (py/get-attr builtins "dict") @@ -439,6 +583,8 @@ (dissoc attr-config :attributes :type) {:attribute-type attr-type :type :class-attribute + :class-binding (or class-name-or-ns + class-name) :attribute attr-val :attribute-name attr}))))] (into [] @@ -461,8 +607,17 @@ cls-name :classname cls-sym :class-symbol cls :class - :as cls-ns-config}] - (let [cls-ns (symbol (str original-namespace "." cls-sym))] + cls-binding :class-binding + :as cls-ns-config}] + + ;; cls-binding percolates down to here in the situation where + ;; '[builtins.list :as python.pylist] occurs + + (let [cls-ns (symbol (str original-namespace + "." + (or + cls-binding + cls-sym)))] (create-ns cls-ns) cls-ns-config)) @@ -471,10 +626,16 @@ cls-name :classname cls-sym :class-symbol cls :class + cls-binding :class-binding method-name :attribute-name method :attribute - :as cls-ns-config}] - (let [cls-ns (symbol (str original-namespace "." cls-sym))] + :as cls-ns-config}] + + ;; cls-binding percolates down to here in the situation where + ;; '[builtins.list :as python.pylist] occurs + + (let [cls-ns (symbol (str original-namespace "." + (or cls-binding cls-sym)))] (load-py-fn method (symbol method-name) cls-ns {}) cls-ns-config)) @@ -484,16 +645,23 @@ cls-name :classname cls-sym :class-symbol cls :class + cls-binding :class-binding attribute-name :attribute-name attribute :attribute - :as cls-ns-config}] - (let [cls-ns (symbol (str original-namespace "." cls-sym))] + :as cls-ns-config}] + + ;; cls-binding percolates down to here in the situation where + ;; '[builtins.list :as python.pylist] occurs + + (let [cls-ns (symbol (str original-namespace "." + (or cls-binding cls-sym)))] (intern cls-ns (symbol attribute-name) attribute) cls-ns-config)) - (defn- bind-class-namespaces! - [lib-config] + [{cls? :class? + module :module? + :as lib-config}] (let [class-namespace-configs (->> (generate-class-namespace-configs @@ -533,11 +701,14 @@ this-module python-namespace public-data - refer-symbols] + refer-symbols + class? + module?] :as lib-config} (preload-python-lib! req)] - (intern-public-and-refer-symbols! lib-config) + (when (not class?) + (intern-public-and-refer-symbols! lib-config)) ;; alpha ;; TODO: does not respect reloading or much of the other @@ -622,3 +793,4 @@ (load-python-lib (vector reqs)) (vector? reqs) (load-python-lib reqs))) + diff --git a/test/libpython_clj/require_python_test.clj b/test/libpython_clj/require_python_test.clj index 95e0cec..1147735 100644 --- a/test/libpython_clj/require_python_test.clj +++ b/test/libpython_clj/require_python_test.clj @@ -112,3 +112,35 @@ (is (= requests-module this-module)) (is (= #{'get} refer)) (is (nil? python-namespace)))) + + +(require-python '([builtins :as python] + [builtins.list :as python.pylist])) + +;; NOTE -- even though builtins.list has been aliased to +;; to python.pylist, you are still required to require +;; "builtins as python" in order to construct a list + +(deftest require-python-classes-with-alias-test + (let [l (python/list)] + (is (= (vec l) [])) + (python.pylist/append l 1) + (is (= (vec l) [1])) + (python.pylist/append l 3) + (is (= (vec l) [1 3])) + (python.pylist/append l 5) + (is (= (vec l) [1 3 5])) + (is (= python.pylist/append python.list/append)) + (python.pylist/clear l) + (is (= (python/list) (vec l))))) + + +(require-python 'csv.DictWriter) + +(deftest require-python-classes + ;; simple creation/recall test + (is csv.DictWriter/__init__) + (is csv.DictWriter/writeheader) + (is csv.DictWriter/writerow) + (is csv.DictWriter/writerows)) + From 461e07eb9c76a64364662760213a6bb18524e9f8 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 22 Dec 2019 10:41:44 -0700 Subject: [PATCH 111/456] Fixed potential crashes related to delay and added error on bad refcount. --- src/libpython_clj/python/bridge.clj | 21 +++++++++--- src/libpython_clj/python/object.clj | 50 ++++++++++++++++------------- 2 files changed, 44 insertions(+), 27 deletions(-) diff --git a/src/libpython_clj/python/bridge.clj b/src/libpython_clj/python/bridge.clj index 2e695ad..de258c4 100644 --- a/src/libpython_clj/python/bridge.clj +++ b/src/libpython_clj/python/bridge.clj @@ -779,8 +779,21 @@ (py-self->jvm-obj self)) +(defmacro pydelay + "Create a delay object that uses only gc reference semantics. If stack reference + semantics happen to be in effect when this delay executes the object may still be + reachable by your program when it's reference counts are released leading to + bad/crashy behavior. This ensures that can't happen at the cost of possibly an object + sticking around." + [& body] + `(delay + (with-gil + (with-bindings {#'pyobj/*pyobject-tracking-flags* [:gc]} + ~@body)))) + + (defonce mapping-type - (delay + (pydelay (with-gil (let [mod (pyinterop/import-module "collections.abc") map-type (py-proto/get-attr mod "MutableMapping")] @@ -835,7 +848,7 @@ (py-self->jvm-obj self)) (defonce sequence-type - (delay + (pydelay (let [mod (pyinterop/import-module "collections.abc") seq-type (py-proto/get-attr mod "MutableSequence")] (pyobj/create-class @@ -896,7 +909,7 @@ (defonce iterable-type - (delay + (pydelay (with-gil (let [mod (pyinterop/import-module "collections.abc") iter-base-cls (py-proto/get-attr mod "Iterable")] @@ -939,7 +952,7 @@ (defonce iterator-type - (delay + (pydelay (with-gil (let [mod (pyinterop/import-module "collections.abc") iter-base-cls (py-proto/get-attr mod "Iterator") diff --git a/src/libpython_clj/python/object.clj b/src/libpython_clj/python/object.clj index f5f65b7..33dad65 100644 --- a/src/libpython_clj/python/object.clj +++ b/src/libpython_clj/python/object.clj @@ -106,6 +106,19 @@ (def ^:dynamic *pyobject-tracking-flags* [:gc]) +(defn incref + "Incref and return object" + [pyobj] + (let [pyobj (jna/as-ptr pyobj)] + (libpy/Py_IncRef pyobj) + pyobj)) + + +(defn refcount + ^long [pyobj] + (long (.ob_refcnt (PyObject. (jna/as-ptr pyobj))))) + + (defn wrap-pyobject "Wrap object such that when it is no longer accessible via the program decref is called. Used for new references. This is some of the meat of the issue, however, @@ -135,16 +148,20 @@ (resource/track pyobj #(with-interpreter interpreter (try - (when *object-reference-logging* - (let [obj-data (PyObject. (Pointer. pyobj-value))] - (println (format "releasing object - 0x%x:%4d:%s" - pyobj-value - (.ob_refcnt obj-data) - py-type-name)))) - (when *object-reference-tracker* - (swap! *object-reference-tracker* - update pyobj-value (fn [arg] - (dec (or arg 0))))) + (let [refcount (refcount pyobj) + obj-data (PyObject. (Pointer. pyobj-value))] + (if (< refcount 1) + (log/errorf "Fatal error -- releasing object - 0x%x:%4d:%s +Object's refcount is bad. Crash is imminent" pyobj-value refcount py-type-name) + (when *object-reference-logging* + (println (format "releasing object - 0x%x:%4d:%s" + pyobj-value + (.ob_refcnt obj-data) + py-type-name)))) + (when *object-reference-tracker* + (swap! *object-reference-tracker* + update pyobj-value (fn [arg] + (dec (or arg 0)))))) (libpy/Py_DecRef (Pointer. pyobj-value)) (catch Throwable e (log/error e "Exception while releasing object")))) @@ -172,19 +189,6 @@ (wrap-pyobject pyobj)))) -(defn incref - "Incref and return object" - [pyobj] - (let [pyobj (jna/as-ptr pyobj)] - (libpy/Py_IncRef pyobj) - pyobj)) - - -(defn refcount - [pyobj] - (.ob_refcnt (PyObject. (jna/as-ptr pyobj)))) - - (defn py-true [] (libpy/Py_True)) From 2f69fed89b2be7e38882b0f747e30ef7676ce23c Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 22 Dec 2019 10:43:04 -0700 Subject: [PATCH 112/456] editing changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0cceae8..06a3d40 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Time for a ChangeLog! +## 1.30-SNAPSHOT + + +* Fixed potential crash related to use of delay mechanism and stack based gc. +* Added logging to complain loudly if refcounts appear to be bad. + ## 1.29 From e655f80a2669c8163e538f9ce9f74b7521c87454 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Wed, 25 Dec 2019 08:47:33 -0700 Subject: [PATCH 113/456] Deeper datatype integration (#38) * Some numpy functionality if datatype is upgraded. * added unit test for basic binary op. * Differentiating between python object ptrs and data ptrs * numpy <-> datatype integration --- project.clj | 2 +- src/libpython_clj/jna.clj | 9 +- src/libpython_clj/jna/base.clj | 11 ++ src/libpython_clj/jna/concrete/bytes.clj | 3 +- src/libpython_clj/jna/concrete/cfunction.clj | 3 +- src/libpython_clj/jna/interpreter.clj | 1 + src/libpython_clj/jna/protocols/object.clj | 5 +- src/libpython_clj/python.clj | 27 +-- src/libpython_clj/python/bridge.clj | 39 ++-- src/libpython_clj/python/interop.clj | 4 +- src/libpython_clj/python/interpreter.clj | 2 +- src/libpython_clj/python/np_array.clj | 180 +++++++++++++++++++ src/libpython_clj/python/object.clj | 20 +-- test/libpython_clj/python/numpy_test.clj | 21 +++ 14 files changed, 263 insertions(+), 64 deletions(-) create mode 100644 src/libpython_clj/python/np_array.clj create mode 100644 test/libpython_clj/python/numpy_test.clj diff --git a/project.clj b/project.clj index f3a93f5..c4414a3 100644 --- a/project.clj +++ b/project.clj @@ -5,7 +5,7 @@ :url "https://www.eclipse.org/legal/epl-2.0/"} :dependencies [[org.clojure/clojure "1.10.1"] [camel-snake-kebab "0.4.0"] - [techascent/tech.datatype "4.61"] + [techascent/tech.datatype "4.65"] [cheshire "5.9.0"]] :repl-options {:init-ns user} :java-source-paths ["java"]) diff --git a/src/libpython_clj/jna.clj b/src/libpython_clj/jna.clj index 2121f5d..9ae0c1e 100644 --- a/src/libpython_clj/jna.clj +++ b/src/libpython_clj/jna.clj @@ -5,10 +5,7 @@ [tech.v2.datatype.typecast :as typecast] [camel-snake-kebab.core :refer [->kebab-case]] [tech.parallel.utils :refer [export-symbols]] - [libpython-clj.jna.base - :refer [def-pylib-fn - ensure-pyobj] - :as libpy-base] + [libpython-clj.jna.base :as libpy-base] [libpython-clj.jna.base] [libpython-clj.jna.interpreter :as jnainterp] [libpython-clj.jna.protocols.object] @@ -34,7 +31,9 @@ (export-symbols libpython-clj.jna.base - find-pylib-symbol) + find-pylib-symbol + as-pyobj + ensure-pyobj) (export-symbols libpython-clj.jna.interpreter diff --git a/src/libpython_clj/jna/base.clj b/src/libpython_clj/jna/base.clj index b179fd6..9cdf94b 100644 --- a/src/libpython_clj/jna/base.clj +++ b/src/libpython_clj/jna/base.clj @@ -33,6 +33,17 @@ (convertible-to-pyobject-ptr? [item] true) (->py-object-ptr [item] item)) +(extend-type Object + PToPyObjectPtr + (convertible-to-pyobject-ptr? [item] (jna/is-jna-ptr-convertible? item)) + (->py-object-ptr [item] (jna/->ptr-backing-store item))) + + +(defn as-pyobj + [item] + (when (and item (convertible-to-pyobject-ptr? item)) + (->py-object-ptr item))) + (defn ensure-pyobj [item] diff --git a/src/libpython_clj/jna/concrete/bytes.clj b/src/libpython_clj/jna/concrete/bytes.clj index 8082853..2c510ef 100644 --- a/src/libpython_clj/jna/concrete/bytes.clj +++ b/src/libpython_clj/jna/concrete/bytes.clj @@ -1,6 +1,7 @@ (ns libpython-clj.jna.concrete.bytes (:require [libpython-clj.jna.base :refer [def-pylib-fn + as-pyobj ensure-pyobj ensure-pytuple ensure-pydict @@ -80,7 +81,7 @@ Integer [obj ensure-pyobj] [ptr-to-buffer jna/ensure-ptr-ptr] - [length jna/as-ptr]) + [length as-pyobj]) diff --git a/src/libpython_clj/jna/concrete/cfunction.clj b/src/libpython_clj/jna/concrete/cfunction.clj index d078a9e..05d9765 100644 --- a/src/libpython_clj/jna/concrete/cfunction.clj +++ b/src/libpython_clj/jna/concrete/cfunction.clj @@ -2,6 +2,7 @@ (:require [libpython-clj.jna.base :refer [def-pylib-fn ensure-pyobj + as-pyobj ensure-pytuple ensure-pydict size-t-type @@ -54,7 +55,7 @@ "Create a new callable from an item." Pointer [method-def (partial jna/ensure-type PyMethodDef)] - [self jna/as-ptr]) + [self as-pyobj]) (def-pylib-fn PyInstanceMethod_New diff --git a/src/libpython_clj/jna/interpreter.clj b/src/libpython_clj/jna/interpreter.clj index 9a6bacc..59dcabe 100644 --- a/src/libpython_clj/jna/interpreter.clj +++ b/src/libpython_clj/jna/interpreter.clj @@ -2,6 +2,7 @@ (:require [libpython-clj.jna.base :refer [def-pylib-fn def-no-gil-pylib-fn + as-pyobj ensure-pyobj ensure-pytuple ensure-pydict diff --git a/src/libpython_clj/jna/protocols/object.clj b/src/libpython_clj/jna/protocols/object.clj index e2f9691..32c2847 100644 --- a/src/libpython_clj/jna/protocols/object.clj +++ b/src/libpython_clj/jna/protocols/object.clj @@ -2,6 +2,7 @@ (:require [libpython-clj.jna.base :refer [def-pylib-fn ensure-pyobj + as-pyobj ensure-pytuple ensure-pydict size-t-type @@ -288,7 +289,7 @@ Pointer [callable ensure-pyobj] [args ensure-pytuple] - [kwargs jna/as-ptr]) + [kwargs as-pyobj]) (def-pylib-fn PyObject_CallObject "Return value: New reference. @@ -301,7 +302,7 @@ This is the equivalent of the Python expression: callable(*args)." Pointer [callable ensure-pyobj] - [args jna/as-ptr]) + [args as-pyobj]) (def-pylib-fn PyObject_Hash diff --git a/src/libpython_clj/python.clj b/src/libpython_clj/python.clj index 1eaa601..0cb0c75 100644 --- a/src/libpython_clj/python.clj +++ b/src/libpython_clj/python.clj @@ -5,7 +5,9 @@ :refer [with-interpreter]] [libpython-clj.python.object :as pyobj] [libpython-clj.python.bridge :as pybridge] - [libpython-clj.jna :as pyjna] + [libpython-clj.jna :as libpy] + ;;Protocol implementations purely for nd-ness + [libpython-clj.python.np-array] [tech.jna :as jna]) (:import [com.sun.jna Pointer] [com.sun.jna.ptr PointerByReference] @@ -246,20 +248,9 @@ :ok) -(defn jvm-type-test - [] - (initialize! :no-io-redirect? true) - (let [retval - (-> (add-module "libpython_clj") - (get-attr "jvm_bridge_type") - (jna/as-ptr))] - (println (.ob_refcnt (libpython_clj.jna.PyObject. retval))) - retval)) - - (defn ptr-refcnt [item] - (-> (jna/as-ptr item) + (-> (libpy/as-pyobj item) (libpython_clj.jna.PyObject. ) (.ob_refcnt))) @@ -277,7 +268,7 @@ (let [ptype# (PointerByReference.) pvalue# (PointerByReference.) ptraceback# (PointerByReference.) - _# (pyjna/PyErr_Fetch ptype# pvalue# ptraceback#) + _# (libpy/PyErr_Fetch ptype# pvalue# ptraceback#) ptype# (-> (jna/->ptr-backing-store ptype#) (pyobj/wrap-pyobject true)) pvalue# (-> (jna/->ptr-backing-store pvalue#) @@ -308,10 +299,10 @@ (do ;;Manual incref here because we cannot detach the object ;;from our gc decref hook added during earlier pyerr-fetch handler. - (pyjna/Py_IncRef ptype) - (pyjna/Py_IncRef pvalue) - (pyjna/Py_IncRef ptraceback) - (pyjna/PyErr_Restore ptype pvalue ptraceback) + (libpy/Py_IncRef ptype) + (libpy/Py_IncRef pvalue) + (libpy/Py_IncRef ptraceback) + (libpy/PyErr_Restore ptype pvalue ptraceback) (pyinterp/check-error-throw)))) (do (call-attr with-var "__exit__" nil nil nil) diff --git a/src/libpython_clj/python/bridge.clj b/src/libpython_clj/python/bridge.clj index de258c4..a385f70 100644 --- a/src/libpython_clj/python/bridge.clj +++ b/src/libpython_clj/python/bridge.clj @@ -209,14 +209,14 @@ -(defn- mostly-copy-arg +(defn mostly-copy-arg "This is the dirty boundary between the languages. Copying as often faster for simple things but we have to be careful not to attempt a copy of things that are only iterable (and not random access)." [arg] (cond - (jna/as-ptr arg) - (jna/as-ptr arg) + (libpy/as-pyobj arg) + (libpy/as-pyobj arg) (dtype/reader? arg) (->python arg) (instance? Map arg) @@ -233,12 +233,9 @@ interpreter# ~interpreter] (with-meta (reify - jna/PToPtr - (is-jna-ptr-convertible? [item#] true) - (->ptr-backing-store [item#] (jna/as-ptr pyobj#)) libpy-base/PToPyObjectPtr (convertible-to-pyobject-ptr? [item#] true) - (->py-object-ptr [item#] (jna/as-ptr pyobj#)) + (->py-object-ptr [item#] (libpy/as-pyobj pyobj#)) py-proto/PPythonType (get-python-type [item] (with-interpreter interpreter# @@ -513,6 +510,10 @@ (defn obj-dtype->dtype [py-dtype] + (when-let [fields (get-attr py-dtype "fields")] + (throw (ex-info (format "Cannot convert numpy object with fields: %s" + (call-attr fields "__str__")) + {}))) (if-let [retval (->> (py-proto/get-attr py-dtype "name") (get py-dtype->dtype-map))] retval @@ -524,18 +525,10 @@ (defn numpy->desc [np-obj] (with-gil - (let [np-obj (as-jvm np-obj) - np (-> (pyinterop/import-module "numpy") - (as-jvm)) - ctypes (get-attr np-obj "ctypes") - ptr-dtype (-> (call-attr np "dtype" "p") - obj-dtype->dtype) - obj-dtype (get-attr np-obj "dtype") - np-dtype (obj-dtype->dtype obj-dtype) - _ (when-let [fields (get-attr obj-dtype "fields")] - (throw (ex-info (format "Cannot convert numpy object with fields: %s" - (call-attr fields "__str__")) - {}))) + (let [np (pyinterop/import-module "numpy") + ctypes (py-proto/as-jvm (get-attr np-obj "ctypes") {}) + np-dtype (-> (py-proto/as-jvm (get-attr np-obj "dtype") {}) + (obj-dtype->dtype)) shape (-> (get-attr ctypes "shape") (as-list) vec) @@ -830,7 +823,7 @@ (defn jvm-map-as-python [^Map jvm-data] (with-gil - (py-proto/call (jna/as-ptr (deref mapping-type)) (make-jvm-object-handle jvm-data)))) + (py-proto/call (libpy/as-pyobj (deref mapping-type)) (make-jvm-object-handle jvm-data)))) (defmethod py-proto/pyobject->jvm :jvm-map-as-python @@ -895,7 +888,7 @@ (defn jvm-list-as-python [^List jvm-data] (with-gil - (py-proto/call (jna/as-ptr (deref sequence-type)) (make-jvm-object-handle jvm-data)))) + (py-proto/call (libpy/as-pyobj (deref sequence-type)) (make-jvm-object-handle jvm-data)))) (defmethod py-proto/pyobject->jvm :jvm-list-as-python @@ -938,7 +931,7 @@ (defn jvm-iterable-as-python [^Iterable jvm-data] (with-gil - (py-proto/call (jna/as-ptr (deref iterable-type)) (make-jvm-object-handle jvm-data)))) + (py-proto/call (libpy/as-pyobj (deref iterable-type)) (make-jvm-object-handle jvm-data)))) (defmethod py-proto/pyobject->jvm :jvm-iterable-as-python @@ -993,7 +986,7 @@ (defn jvm-iterator-as-python [^Iterator jvm-data] (with-gil - (py-proto/call (jna/as-ptr (deref iterator-type)) (make-jvm-object-handle jvm-data)))) + (py-proto/call (libpy/as-pyobj (deref iterator-type)) (make-jvm-object-handle jvm-data)))) (defmethod py-proto/pyobject->jvm :jvm-iterator-as-python diff --git a/src/libpython_clj/python/interop.clj b/src/libpython_clj/python/interop.clj index 935eed6..55713a3 100644 --- a/src/libpython_clj/python/interop.clj +++ b/src/libpython_clj/python/interop.clj @@ -340,7 +340,7 @@ (pyinvoke [this self] (if-let [attr (-> (pybridge->bridge self) (.getAttr "__iter__"))] - (libpy/PyObject_CallObject (jna/as-ptr attr) nil) + (libpy/PyObject_CallObject (libpy/as-pyobj attr) nil) (do (libpy/PyErr_SetNone (libpy/PyExc_Exception)) nil)))) @@ -348,7 +348,7 @@ (pyinvoke [this self] (if-let [next (-> (pybridge->bridge self) (.getAttr "__next__"))] - (libpy/PyObject_CallObject (jna/as-ptr next) nil) + (libpy/PyObject_CallObject (libpy/as-pyobj next) nil) (do (libpy/PyErr_SetNone (libpy/PyExc_Exception)) nil))))})))) diff --git a/src/libpython_clj/python/interpreter.clj b/src/libpython_clj/python/interpreter.clj index aa560bb..ec023f8 100644 --- a/src/libpython_clj/python/interpreter.clj +++ b/src/libpython_clj/python/interpreter.clj @@ -328,7 +328,7 @@ print(json.dumps( "Get a keyword that corresponds to the current type. Uses global type symbol table. Add the type to the symbol table if it does not exist already." [typeobj] - (let [type-addr (Pointer/nativeValue (jna/as-ptr typeobj)) + (let [type-addr (Pointer/nativeValue (libpy/as-pyobj typeobj)) interpreter (ensure-bound-interpreter) symbol-table (-> (swap! (:shared-state* interpreter) (fn [{:keys [type-symbol-table] :as shared-state}] diff --git a/src/libpython_clj/python/np_array.clj b/src/libpython_clj/python/np_array.clj new file mode 100644 index 0000000..a9076cd --- /dev/null +++ b/src/libpython_clj/python/np_array.clj @@ -0,0 +1,180 @@ +(ns libpython-clj.python.np-array + (:require [tech.v2.datatype.protocols :as dtype-proto] + [tech.v2.datatype.operation-provider :as op-provider] + [tech.v2.tensor :as dtt] + [tech.v2.datatype.argtypes :as argtypes] + [libpython-clj.python.interpreter :as py-interp] + [libpython-clj.python.protocols :as py-proto] + [libpython-clj.python.bridge :as py-bridge] + [libpython-clj.python.object :as py-object] + [libpython-clj.python.interop :as py-interop] + [tech.jna :as jna])) + + + +(defmethod py-proto/pyobject->jvm :ndarray + [pyobj] + (-> (py-bridge/numpy->desc pyobj) + (dtt/buffer-descriptor->tensor) + (dtt/clone))) + + +(defmethod py-proto/pyobject-as-jvm :ndarray + [pyobj] + (py-interp/with-gil + (let [interpreter (py-interp/ensure-bound-interpreter)] + (py-bridge/bridge-pyobject + pyobj + interpreter + Iterable + (iterator [this] + (py-proto/python-obj-iterator pyobj interpreter)) + py-proto/PPyObjectBridgeToMap + (as-map [item] + (py-bridge/generic-python-as-map pyobj)) + py-proto/PPyObjectBridgeToList + (as-list [item] + (py-bridge/generic-python-as-list pyobj)) + py-proto/PPyObjectBridgeToTensor + (as-tensor [item] + (-> (py-bridge/numpy->desc item) + dtt/buffer-descriptor->tensor)) + dtype-proto/PDatatype + (get-datatype + [this] + (-> (py-proto/get-attr pyobj "dtype") + (py-proto/as-jvm {}) + (py-bridge/obj-dtype->dtype))) + + + dtype-proto/POperationType + (operation-type + [this] + :numpy-array) + + dtype-proto/PCountable + (ecount [this] (apply * (dtype-proto/shape this))) + + dtype-proto/PShape + (shape + [this] + (-> (py-proto/get-attr pyobj "shape") + (py-proto/->jvm {}))) + + dtype-proto/PToNioBuffer + (convertible-to-nio-buffer? [item] true) + (->buffer-backing-store + [item] + (-> (py-proto/as-tensor item) + (dtype-proto/->buffer-backing-store))) + + dtype-proto/PToJNAPointer + (convertible-to-data-ptr? [item] true) + (->jna-ptr + [item] + (:ptr (py-bridge/numpy->desc item))) + + dtype-proto/PBuffer + (sub-buffer + [buffer offset length] + (-> (py-proto/as-tensor buffer) + (dtype-proto/sub-buffer offset length))) + + dtype-proto/PToBufferDesc + (convertible-to-buffer-desc? [item] true) + (->buffer-descriptor + [item] + (py-bridge/numpy->desc item)) + + dtype-proto/PToReader + (convertible-to-reader? [item] true) + (->reader + [item options] + (-> (py-proto/as-tensor item) + (dtype-proto/->reader options))) + + dtype-proto/PToWriter + (convertible-to-writer? [item] true) + (->writer + [item options] + (-> (py-proto/as-tensor item) + (dtype-proto/->writer options))))))) + + +(def np-mod + (py-bridge/pydelay + (-> (py-interop/import-module "numpy") + (py-proto/as-jvm {})))) + + +(defn- dispatch-unary-op + [op lhs options] + (if (keyword? op) + (if (= op :-) + (py-proto/call-attr lhs "__neg__") + (py-proto/call-attr @np-mod (name op) lhs)) + (throw (Exception. "Unimplemented")))) + + +(defmethod op-provider/half-dispatch-unary-op :numpy-array + [op lhs options] + (dispatch-unary-op op lhs options)) + + +(defmethod op-provider/half-dispatch-boolean-unary-op :numpy-array + [op lhs options] + (dispatch-unary-op op lhs options)) + + +(defn- dispatch-binary-op + [op lhs rhs options] + (case op + :max (py-proto/call-attr @np-mod "max" lhs rhs) + :min (py-proto/call-attr @np-mod "min" lhs rhs) + :+ (py-proto/call-attr @np-mod "add" lhs rhs) + :- (py-proto/call-attr @np-mod "subtract" lhs rhs) + :div (py-proto/call-attr @np-mod "divide" lhs rhs) + :* (py-proto/call-attr @np-mod "multiply" lhs rhs) + :pow (py-proto/call-attr @np-mod "power" lhs rhs) + :quot (py-proto/call-attr @np-mod "floor_divide" lhs rhs) + :rem (py-proto/call-attr @np-mod "mod" lhs rhs) + :bit-and (py-proto/call-attr @np-mod "bitwise_and" lhs rhs) + :bit-flip (py-proto/call-attr @np-mod "bitwise_not" lhs rhs) + :bit-or (py-proto/call-attr @np-mod "bitwise_or" lhs rhs) + :bit-xor (py-proto/call-attr @np-mod "bitwise_xor" lhs rhs) + :bit-shift-left (py-proto/call-attr @np-mod "left_shift" lhs rhs) + :bit-shift-right (py-proto/call-attr @np-mod "right_shift" lhs rhs) + :and (py-proto/call-attr @np-mod "logical_and" lhs rhs) + :or (py-proto/call-attr @np-mod "logical_or" lhs rhs) + :not (py-proto/call-attr @np-mod "logical_not" lhs rhs) + :xor (py-proto/call-attr @np-mod "logical_xor" lhs rhs) + :< (py-proto/call-attr @np-mod "less" lhs rhs) + :<= (py-proto/call-attr @np-mod "less_equal" lhs rhs) + :eq (py-proto/call-attr @np-mod "equal" lhs rhs) + :> (py-proto/call-attr @np-mod "greater" lhs rhs) + :>= (py-proto/call-attr @np-mod "greater_equal" lhs rhs))) + + +(def binary-op-dispatch-table + [[:numpy-array :scalar] + [:scalar :numpy-array] + [:numpy-array :numpy-array]]) + + +(defmacro implement-binary-ops + [] + `(do + ~@(for [targets binary-op-dispatch-table] + `(do + (defmethod op-provider/half-dispatch-binary-op + ~targets + [op# lhs# rhs# options#] + (dispatch-binary-op op# lhs# rhs# options#)) + (defmethod op-provider/half-dispatch-boolean-binary-op + ~targets + [op# lhs# rhs# options#] + (dispatch-binary-op op# lhs# rhs# options#)) + )))) + + +(implement-binary-ops) diff --git a/src/libpython_clj/python/object.clj b/src/libpython_clj/python/object.clj index 33dad65..2c6f8be 100644 --- a/src/libpython_clj/python/object.clj +++ b/src/libpython_clj/python/object.clj @@ -109,14 +109,14 @@ (defn incref "Incref and return object" [pyobj] - (let [pyobj (jna/as-ptr pyobj)] + (let [pyobj (libpy/as-pyobj pyobj)] (libpy/Py_IncRef pyobj) pyobj)) (defn refcount ^long [pyobj] - (long (.ob_refcnt (PyObject. (jna/as-ptr pyobj))))) + (long (.ob_refcnt (PyObject. (libpy/as-pyobj pyobj))))) (defn wrap-pyobject @@ -129,10 +129,10 @@ (check-error-throw)) ;;We don't wrap pynone (if (and pyobj - (not= (Pointer/nativeValue (jna/as-ptr pyobj)) - (Pointer/nativeValue (jna/as-ptr (libpy/Py_None))))) + (not= (Pointer/nativeValue (libpy/as-pyobj pyobj)) + (Pointer/nativeValue (libpy/as-pyobj (libpy/Py_None))))) (let [interpreter (ensure-bound-interpreter) - pyobj-value (Pointer/nativeValue (jna/as-ptr pyobj)) + pyobj-value (Pointer/nativeValue (libpy/as-pyobj pyobj)) py-type-name (name (python-type pyobj))] (when *object-reference-logging* (let [obj-data (PyObject. (Pointer. pyobj-value))] @@ -184,7 +184,7 @@ Object's refcount is bad. Crash is imminent" pyobj-value refcount py-type-name) references that need to escape the current scope." [pyobj] (with-gil - (let [pyobj (jna/as-ptr pyobj)] + (let [pyobj (libpy/as-pyobj pyobj)] (libpy/Py_IncRef pyobj) (wrap-pyobject pyobj)))) @@ -220,7 +220,7 @@ Object's refcount is bad. Crash is imminent" pyobj-value refcount py-type-name) (defn py-raw-type ^Pointer [pyobj] - (let [pyobj (PyObject. (jna/as-ptr pyobj))] + (let [pyobj (PyObject. (libpy/as-pyobj pyobj))] (.ob_type pyobj))) @@ -990,8 +990,8 @@ Object's refcount is bad. Crash is imminent" pyobj-value refcount py-type-name) (loop [next-retval (libpy/PyDict_Next pyobj ppos pkey pvalue)] (if (not= 0 next-retval) (do - (.add retval [(jna/as-ptr pkey) - (jna/as-ptr pvalue)]) + (.add retval [(libpy/as-pyobj pkey) + (libpy/as-pyobj pvalue)]) (recur (libpy/PyDict_Next pyobj ppos pkey pvalue))) (->> retval (map (fn [[k v]] @@ -1093,7 +1093,7 @@ Object's refcount is bad. Crash is imminent" pyobj-value refcount py-type-name) (python->jvm-copy-persistent-vector pyobj) :else {:type (python-type pyobj) - :value (Pointer/nativeValue (jna/as-ptr pyobj))}))) + :value (Pointer/nativeValue (libpy/as-pyobj pyobj))}))) (defn is-instance? diff --git a/test/libpython_clj/python/numpy_test.clj b/test/libpython_clj/python/numpy_test.clj new file mode 100644 index 0000000..0d98463 --- /dev/null +++ b/test/libpython_clj/python/numpy_test.clj @@ -0,0 +1,21 @@ +(ns libpython-clj.python.numpy-test + (:require [libpython-clj.python :as py] + [tech.v2.datatype :as dtype] + [tech.v2.datatype.functional :as dfn] + [tech.v2.tensor :as dtt] + [clojure.test :refer :all])) + +(py/initialize!) + +(def np-mod (py/import-module "numpy")) + + +;;The basic math operations of the dfn namespace must work on numpy objects. +(deftest basic-numpy + (let [test-data [[1 2] [3 4]] + np-ary (py/$a np-mod array test-data) + tens (dtt/->tensor test-data)] + (is (dfn/equals (dfn/- np-ary 4) + (dfn/- tens 4))) + (is (dfn/equals [1 2 3 4] + (dtype/make-container :java-array :int64 np-ary))))) From 961377cfbc118641ec837f9e04127cecd1054a88 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Wed, 25 Dec 2019 09:10:30 -0700 Subject: [PATCH 114/456] Readying changelog --- CHANGELOG.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06a3d40..c47f429 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,27 @@ ## 1.30-SNAPSHOT +This release is a big one. With finalizing `require-python` we have a clear way +to use python in daily use and make it look good in normal clojure usage. There +is a demo of [facial recognition](https://github.com/cnuernber/facial-rec) using some +of the best open systems for doing this; this demo would absolutely not be possible +without this library due to the extensive use of numpy and cython to implement the +face detection. We can now interact with even very complex python systems with +roughly the same performance as a pure-python system. + +#### Finalized `require-python` + +Lots of work put in to make the require-python pathway work with +classes and some serious refactoring overall. + +#### Better Numpy Support + +* Most of the datatype libraries math operators supported by numpy objects (+,-,etc). +* Numpy objects can be used in datatype library functions (like copy, make-container) + and work in optimized ways. + + +#### Bugs Fixed * Fixed potential crash related to use of delay mechanism and stack based gc. * Added logging to complain loudly if refcounts appear to be bad. From a519172f59202126221ee4ba19d6465ff47c6762 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Wed, 25 Dec 2019 09:47:04 -0700 Subject: [PATCH 115/456] editing --- CHANGELOG.md | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c47f429..08a3494 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,11 +3,11 @@ ## 1.30-SNAPSHOT This release is a big one. With finalizing `require-python` we have a clear way -to use python in daily use and make it look good in normal clojure usage. There -is a demo of [facial recognition](https://github.com/cnuernber/facial-rec) using some -of the best open systems for doing this; this demo would absolutely not be possible -without this library due to the extensive use of numpy and cython to implement the -face detection. We can now interact with even very complex python systems with +to use python in daily use and make it look good in normal clojure usage. There +is a demo of [facial recognition](https://github.com/cnuernber/facial-rec) using some +of the best open systems for doing this; this demo would absolutely not be possible +without this library due to the extensive use of numpy and cython to implement the +face detection. We can now interact with even very complex python systems with roughly the same performance as a pure-python system. #### Finalized `require-python` @@ -21,6 +21,25 @@ classes and some serious refactoring overall. * Numpy objects can be used in datatype library functions (like copy, make-container) and work in optimized ways. +```clojure +libpython-clj.python.numpy-test> (def test-ary (py/$a np-mod array (->> (range 9) + (partition 3) + (mapv vec)))) +#'libpython-clj.python.numpy-test/test-ary +libpython-clj.python.numpy-test> test-ary +[[0 1 2] + [3 4 5] + [6 7 8]] +libpython-clj.python.numpy-test> (dfn/+ test-ary 2) +[[ 2 3 4] + [ 5 6 7] + [ 8 9 10]] +libpython-clj.python.numpy-test> (dfn/> test-ary 4) +[[False False False] + [False False True] + [ True True True]] +``` + #### Bugs Fixed From b9a45aa8bf686f9c950a6c83fce0140d2c63a0c5 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Wed, 25 Dec 2019 15:29:33 -0700 Subject: [PATCH 116/456] Aligning require-python with require --- src/libpython_clj/require.clj | 14 +++++++------- test/libpython_clj/require_python_test.clj | 4 +--- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/libpython_clj/require.clj b/src/libpython_clj/require.clj index b7e1363..0e491b0 100644 --- a/src/libpython_clj/require.clj +++ b/src/libpython_clj/require.clj @@ -360,7 +360,8 @@ "Build a configuration map of a python library. Current ns is option and used during testing but unnecessary during normal running events." [req & [current-ns]] - (let [supported-flags #{:reload :no-arglists :alpha-load-ns-classes} + (let [supported-flags #{:reload + :no-arglists} [module-name & etc] req flags (parse-flags supported-flags etc) etc (into {} @@ -372,7 +373,6 @@ etc) reload? (:reload flags) no-arglists? (:no-arglists flags) - load-ns-classes? (:alpha-load-ns-classes flags) module-name-or-ns (:as etc module-name) exclude (into #{} (:exclude etc)) refer (cond @@ -384,13 +384,14 @@ current-ns (or current-ns *ns*) current-ns-sym (symbol (str current-ns)) python-namespace (find-ns module-name-or-ns) - [this-module cls] (import-module-or-cls! module-name)] + [this-module cls] (import-module-or-cls! module-name) + load-ns-classes? (pyclass? cls)] {:supported-flags supported-flags :etc etc :reload? reload? :no-arglists? no-arglists? - :load-ns-classes? true + :load-ns-classes? load-ns-classes? ;; if something like 'builtins.list was requested, ;; the module name is 'builtins @@ -555,7 +556,7 @@ (pyclass-ns-config [[attr attr-val]] {:namespace module-name-or-ns :attribute-type :class - :classname attr + :classname attr :class-symbol (symbol attr) :class-binding (or class-name-or-ns class-name) @@ -612,7 +613,7 @@ ;; cls-binding percolates down to here in the situation where ;; '[builtins.list :as python.pylist] occurs - + (let [cls-ns (symbol (str original-namespace "." (or @@ -793,4 +794,3 @@ (load-python-lib (vector reqs)) (vector? reqs) (load-python-lib reqs))) - diff --git a/test/libpython_clj/require_python_test.clj b/test/libpython_clj/require_python_test.clj index 1147735..7c35457 100644 --- a/test/libpython_clj/require_python_test.clj +++ b/test/libpython_clj/require_python_test.clj @@ -72,7 +72,7 @@ ;; no exclusions (is (= #{} exclude)) - (is (= #{:no-arglists :reload :alpha-load-ns-classes} + (is (= #{:no-arglists :reload} supported-flags)) (is (= 'csv module-name module-name-or-ns)) (is (nil? reload?)) @@ -130,7 +130,6 @@ (is (= (vec l) [1 3])) (python.pylist/append l 5) (is (= (vec l) [1 3 5])) - (is (= python.pylist/append python.list/append)) (python.pylist/clear l) (is (= (python/list) (vec l))))) @@ -143,4 +142,3 @@ (is csv.DictWriter/writeheader) (is csv.DictWriter/writerow) (is csv.DictWriter/writerows)) - From 0c73da1f4f3c3934f7f00775d5690466e379c50c Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Fri, 27 Dec 2019 07:54:52 -0700 Subject: [PATCH 117/456] editing --- README.md | 407 +++++++++++--------------------------------------- docs/Usage.md | 306 +++++++++++++++++++++++++++++++++++++ 2 files changed, 393 insertions(+), 320 deletions(-) create mode 100644 docs/Usage.md diff --git a/README.md b/README.md index 7b222ce..3f88b1c 100644 --- a/README.md +++ b/README.md @@ -10,350 +10,109 @@ JNA libpython bindings to the tech ecosystem. * Python objects are linked to the JVM GC such that when they are no longer reachable from the JVM their references are released. Scope based resource contexts are [also available](https://github.com/cnuernber/libpython-clj/blob/master/docs/scopes-and-gc.md). -* The exact same binary can run top of on multiple version of python reducing version - dependency chain management issues. -* Development of new functionality is faster because it can be done from purely from the - REPL. +* Finding the python libraries is done dynamically allowing one system to run on multiple versions + of python. +* REPL oriented design means fast, smooth, iterative development. -#### Clojure Conj 2019 +## Vision -* [video](https://www.youtube.com/watch?v=vQPW16_jixs) -* [slides](https://docs.google.com/presentation/d/1uegYhpS6P2AtEfhpg6PlgBmTSIPqCXvFTWcGYG_Qk2o/edit?usp=sharing) +We aim to integrate Python into Clojure at a deep level. This means that we want to +be able to load/use python modules almost as if they were Clojure namespaces. We +also want to be able to use Clojure to extend Python objects. I gave a +[talk at Clojure Conj 2019](https://www.youtube.com/watch?v=vQPW16_jixs) that +outlines more of what is going on. - -We have a [video](https://www.youtube.com/watch?v=ajDiGS73i2o) up of a scicloj community -discussion with demos. - - -TechAscent has a [blog post](http://www.techascent.com/blog/functions-across-languages.html) -up about how the inter-language bindings actually work at a very low level. - - -A walkthough using libpython-clj to render some graphs via matplotlib is on -[nextjournal](https://nextjournal.com/chrisn/fun-with-matplotlib). - - -If you want to quickly start using python from clojure your fastest path is probably -the [panthera](https://github.com/alanmarazzi/panthera) library in addition to -learning how the primitives in this library work. - - -New to Clojure or the JVM? Try remixing the nextjournal entry and playing around -there. For more resources on learning and getting more comfortable with Clojure, -we have an [introductory document](docs/new-to-clojure.md). - - -## Usage - - -Python objects are essentially two dictionaries, one for 'attributes' and one for -'items'. When you use python and use the '.' operator, you are referencing attributes. -If you use the '[]' operator, then you are referencing items. Attributes are built in, -item access is optional and happens via the `__getitem__` and `__setitem__` attributes. -This is important to realize in that the code below doesn't look like python because we are -referencing the item and attribute systems by name and not via '.' or '[]'. - - -### Installation - -#### Ubuntu - -```console -sudo apt install libpython3.6 -# numpy and pandas are required for unit tests. Numpy is required for -# zero copy support. -python3.6 -m pip install numpy pandas --user -``` - -#### MacOSX - -Python installation instructions [here](https://docs.python-guide.org/starting/install3/osx/). - - - -### Initialize python +This code is a concrete example that generates an +[embedding for faces](https://github.com/cnuernber/facial-rec): ```clojure -user> - -user> (require '[libpython-clj.python - :refer [as-python as-jvm - ->python ->jvm - get-attr call-attr call-attr-kw - get-item att-type-map - call call-kw initialize! - as-numpy as-tensor ->numpy - run-simple-string - add-module module-dict - import-module - python-type]]) -nil - - -user> (initialize!) -Jun 30, 2019 4:47:39 PM clojure.tools.logging$eval7369$fn__7372 invoke -INFO: executing python initialize! -Jun 30, 2019 4:47:39 PM clojure.tools.logging$eval7369$fn__7372 invoke -INFO: Library python3.6m found at [:system "python3.6m"] -Jun 30, 2019 4:47:39 PM clojure.tools.logging$eval7369$fn__7372 invoke -INFO: Reference thread starting -:ok +(ns facial-rec.face-feature + (:require [libpython-clj.require :refer [require-python]] + [libpython-clj.python :as py] + [tech.v2.datatype :as dtype])) + + +(require-python '(mxnet mxnet.ndarray mxnet.module mxnet.io mxnet.model)) +(require-python 'cv2) +(require-python '[numpy :as np]) + +(defn load-model + [& {:keys [model-path checkpoint] + :or {model-path "models/recognition/model" + checkpoint 0}}] + (let [[sym arg-params aux-params] (mxnet.model/load_checkpoint model-path checkpoint) + all-layers (py/$a sym get_internals) + target-layer (py/get-item all-layers "fc1_output") + ;;TODO - We haven't overloaded enough of the IFn invoke methods for + ;;this to work without using call-kw + model (py/call-kw mxnet.module/Module [] {:symbol target-layer :context (mxnet/cpu) :label_names nil})] + (py/$a model bind :data_shapes [["data" [1 3 112 112]]]) + (py/$a model set_params arg-params aux-params) + model)) + +(defonce model (load-model)) + +(defn face->feature + [img-path] + (py/with-gil-stack-rc-context + (if-let [new-img (cv2/imread img-path)] + (let [new-img (cv2/cvtColor new-img cv2/COLOR_BGR2RGB) + new-img (np/transpose new-img [2 0 1]) + input-blob (np/expand_dims new-img :axis 0) + data (mxnet.ndarray/array input-blob) + batch (mxnet.io/DataBatch :data [data])] + (py/$a model forward batch :is_train false) + (-> (py/$a model get_outputs) + first + (py/$a asnumpy) + (#(dtype/make-container :java-array :float32 %)))) + (throw (Exception. (format "Failed to load img: %s" img-path)))))) ``` -This dynamically finds the python shared library and loads it. If you desire a -different shared library you can override -[here](https://github.com/cnuernber/libpython-clj/blob/master/src/libpython_clj/jna/base.clj#L12). - - -### Execute Some Python - -`*out*` and `*err*` capture python stdout and stderr respectively. - -```clojure -user> (run-simple-string "print('hey')") -hey -{:globals - {'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': , '__spec__': None, '__annotations__': {}, '__builtins__': }, - :locals - {'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': , '__spec__': None, '__annotations__': {}, '__builtins__': }} -``` - -The results have been 'bridged' into java meaning they are still python objects but -there are java wrappers over the top of them. For instance, `Object.toString` forwards -its implementation to the python function `__str__`. - -```clojure -(def bridged (run-simple-string "print('hey')")) -(instance? java.util.Map (:globals bridged)) -true -user> (:globals bridged) -{'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': , '__spec__': None, '__annotations__': {}, '__builtins__': } -``` - -We can get and set global variables here. If we run another string, these are in the -environment. The globals map itself is the global dict of the main module: +## Usage ```clojure -(def main-globals (-> (add-module "__main__") - (module-dict))) -#'user/main-globals - -user> main-globals -{'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': , '__spec__': None, '__annotations__': {}, '__builtins__': } -user> (keys main-globals) -("__name__" - "__doc__" - "__package__" - "__loader__" - "__spec__" - "__annotations__" - "__builtins__") -user> (get main-globals "__name__") -"__main__" -user> (.put main-globals "my_var" 200) +user> (require '[libpython-clj.require :refer [require-python]]) +...logging info.... nil - -user> (run-simple-string "print('your variable is:' + str(my_var))") -your variable is:200 -{:globals - {'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': , '__spec__': None, '__annotations__': {}, '__builtins__': , 'my_var': 200}, - :locals - {'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': , '__spec__': None, '__annotations__': {}, '__builtins__': , 'my_var': 200}} -``` - -Running python isn't ever really necessary, however, although it may at times be -convenient. You can call attributes from clojure easily: - -```clojure -user> (def np (import-module "numpy")) -#'user/np -user> (def ones-ary (call-attr np "ones" [2 3])) -#'user/ones-ary -user> ones-ary -[[1. 1. 1.] - [1. 1. 1.]] -user> (call-attr ones-ary "__len__") -2 -user> (vec ones-ary) -[[1. 1. 1.] [1. 1. 1.]] -user> (type (first *1)) -:pyobject -user> (get-attr ones-ary "shape") -(2, 3) -user> (vec (get-attr ones-ary "shape")) -[2 3] - -user> (att-type-map ones-ary) -{"T" :ndarray, - "__abs__" :method-wrapper, - "__add__" :method-wrapper, - "__and__" :method-wrapper, - "__array__" :builtin-function-or-method, - "__array_finalize__" :none-type, - "__array_function__" :builtin-function-or-method, - "__array_interface__" :dict, - "__array_prepare__" :builtin-function-or-method, - "__array_priority__" :float, - "__array_struct__" :py-capsule, - "__array_ufunc__" :builtin-function-or-method, - "__array_wrap__" :builtin-function-or-method, - "__bool__" :method-wrapper, - "__class__" :type, - "__complex__" :builtin-function-or-method, - "__contains__" :method-wrapper, - ... - "std" :builtin-function-or-method, - "strides" :tuple, - "sum" :builtin-function-or-method, - "swapaxes" :builtin-function-or-method, - "take" :builtin-function-or-method, - "tobytes" :builtin-function-or-method, - "tofile" :builtin-function-or-method, - "tolist" :builtin-function-or-method, - "tostring" :builtin-function-or-method, - "trace" :builtin-function-or-method, - "transpose" :builtin-function-or-method, - "var" :builtin-function-or-method, - "view" :builtin-function-or-method} -``` - - -### att-type-map - -It can be extremely helpful to print out the attribute name->attribute type map: - -```clojure -user> (att-type-map ones-ary) -{"T" :ndarray, - "__abs__" :method-wrapper, - "__add__" :method-wrapper, - "__and__" :method-wrapper, - "__array__" :builtin-function-or-method, - "__array_finalize__" :none-type, - "__array_function__" :builtin-function-or-method, - "__array_interface__" :dict, - ... - "real" :ndarray, - "repeat" :builtin-function-or-method, - "reshape" :builtin-function-or-method, - "resize" :builtin-function-or-method, - "round" :builtin-function-or-method, - "searchsorted" :builtin-function-or-method, - "setfield" :builtin-function-or-method, - "setflags" :builtin-function-or-method, - "shape" :tuple, - "size" :int, - "sort" :builtin-function-or-method, - ... -} -``` - - -### Errors - - -Errors are caught and an exception is thrown. The error text is saved verbatim -in the exception: - - -```clojure -user> (run-simple-string "print('syntax errrr") -Execution error (ExceptionInfo) at libpython-clj.python.interpreter/check-error-throw (interpreter.clj:260). - File "", line 1 - print('syntax errrr - ^ -SyntaxError: EOL while scanning string literal -``` - -### Some Syntax Sugar - -```clojure -user> (py/from-import numpy linspace) -#'user/linspace -user> (linspace 2 3 :num 10) -[2. 2.11111111 2.22222222 2.33333333 2.44444444 2.55555556 - 2.66666667 2.77777778 2.88888889 3. ] -user> (doc linspace) -------------------------- -user/linspace - - Return evenly spaced numbers over a specified interval. - - Returns `num` evenly spaced samples, calculated over the - interval [`start`, `stop`]. - -``` - -* `from-import` - sugar around python `from a import b`. Takes multiple b's. -* `import-as` - surgar around python `import a as b`. -* `$a` - call an attribute using symbol att name. Keywords map to kwargs -* `$c` - call an object mapping keywords to kwargs - - -#### Experimental Sugar - -We are trying to find the best way to handle attributes in order to shorten -generic python notebook-type usage. The currently implemented direction is: - -* `$.` - get an attribute. Can pass in symbol, string, or keyword -* `$..` - get an attribute. If more args are present, get the attribute on that -result. - -```clojure -user> (py/$. numpy linspace) - -user> (py/$.. numpy random shuffle) - +user> (require-python '[numpy :as np]) +nil +user> (def test-ary (np/array [[1 2][3 4]])) +#'user/test-ary +user> test-ary +[[1 2] + [3 4]] ``` +We have a [document](docs/Usage.md) on all the features but beginning usage is +pretty simple. Import your modules, use the things from Clojure. We have put +effort into making sure things like sequences and ranges transfer between the two +languages. -### Numpy - -Speaking of numpy, you can move data between numpy and java easily. - -```clojure -user> (def tens-data (as-tensor ones-ary)) -#'user/tens-data -user> (println tens-data) -#tech.v2.tensor[2 3] -[[1.000 1.000 1.000] - [1.000 1.000 1.000]] -nil +#### Environments -user> (require '[tech.v2.datatype :as dtype]) -nil -user> (def ignored (dtype/copy! (repeat 6 5) tens-data)) -#'user/ignored -user> (.put main-globals "ones_ary" ones_ary) -Syntax error compiling at (*cider-repl cnuernber/libpython-clj:localhost:39019(clj)*:191:7). -Unable to resolve symbol: ones_ary in this context -user> (.put main-globals "ones_ary" ones-ary) -nil -user> (run-simple-string "print(ones_ary)") -[[5. 5. 5.] - [5. 5. 5.]] -{:globals - {'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': , '__spec__': None, '__annotations__': {}, '__builtins__': , 'my_var': 200, 'ones_ary': array([[5., 5., 5.], - [5., 5., 5.]])}, - :locals - {'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': , '__spec__': None, '__annotations__': {}, '__builtins__': , 'my_var': 200, 'ones_ary': array([[5., 5., 5.], - [5., 5., 5.]])}} -``` +One very complimentary aspect of Python with respect to Clojure is it's integration +with cutting edge native libraries. Our support isn't perfect so some understanding +of the mechanism is important to diagnose errors and issues. -So heavy data has a zero-copy route. Anything backed by a `:native-buffer` has a -zero copy pathway to and from numpy. For more information on how this happens, -please refer to the datatype library [documentation](https://github.com/techascent/tech.datatype/tree/master/docs). +Current, we launch the python3 executable and print out various different bits of +configuration as json. We parse the json and use the output to attempt to find +the `libpython3.Xm.so` shared library so for example if we are loading python +3.6 we look for `libpython3.6m.so` on Linux or `libpython3.6m.dylib` on the Mac. -Just keep in mind, careless usage of zero copy is going to cause spooky action at a -distance. +This pathway has allowed us support Conda albeit with some work. For examples +using Conda, check out the facial rec repository above or look into how we build +our test docker containers in [this project](scripts/run-conda-docker). ## Further Information +* Clojure Conj 2019 [video](https://www.youtube.com/watch?v=vQPW16_jixs) and + [slides](https://docs.google.com/presentation/d/1uegYhpS6P2AtEfhpg6PlgBmTSIPqCXvFTWcGYG_Qk2o/edit?usp=sharing). * [development discussion forum](https://clojurians.zulipchat.com/#narrow/stream/215609-libpython-clj-dev) * [design documentation](docs/design.md) * [scope and garbage collection docs](https://github.com/cnuernber/libpython-clj/blob/master/docs/scopes-and-gc.md) @@ -365,6 +124,14 @@ distance. * [Clojure/Python interop technical blog post](www.techascent.com/blogs/functions-across-languages.html) * [persistent datastructures in python](https://github.com/tobgu/pyrsistent) + +## New To Clojure + +New to Clojure or the JVM? Try remixing the nextjournal entry and playing around +there. For more resources on learning and getting more comfortable with Clojure, +we have an [introductory document](docs/new-to-clojure.md). + + ## Resources * [libpython C api](https://docs.python.org/3.7/c-api/index.html#c-api-index) diff --git a/docs/Usage.md b/docs/Usage.md new file mode 100644 index 0000000..1edb379 --- /dev/null +++ b/docs/Usage.md @@ -0,0 +1,306 @@ +# Usage + + +Python objects are essentially two dictionaries, one for 'attributes' and one for +'items'. When you use python and use the '.' operator, you are referencing attributes. +If you use the '[]' operator, then you are referencing items. Attributes are built in, +item access is optional and happens via the `__getitem__` and `__setitem__` attributes. +This is important to realize in that the code below doesn't look like python because we are +referencing the item and attribute systems by name and not via '.' or '[]'. + + +### Installation + +#### Ubuntu + +```console +sudo apt install libpython3.6 +# numpy and pandas are required for unit tests. Numpy is required for +# zero copy support. +python3.6 -m pip install numpy pandas --user +``` + +#### MacOSX + +Python installation instructions [here](https://docs.python-guide.org/starting/install3/osx/). + + + +### Initialize python + +```clojure +user> + +user> (require '[libpython-clj.python + :refer [as-python as-jvm + ->python ->jvm + get-attr call-attr call-attr-kw + get-item att-type-map + call call-kw initialize! + as-numpy as-tensor ->numpy + run-simple-string + add-module module-dict + import-module + python-type]]) +nil + + +user> (initialize!) +Jun 30, 2019 4:47:39 PM clojure.tools.logging$eval7369$fn__7372 invoke +INFO: executing python initialize! +Jun 30, 2019 4:47:39 PM clojure.tools.logging$eval7369$fn__7372 invoke +INFO: Library python3.6m found at [:system "python3.6m"] +Jun 30, 2019 4:47:39 PM clojure.tools.logging$eval7369$fn__7372 invoke +INFO: Reference thread starting +:ok +``` + +This dynamically finds the python shared library and loads it. If you desire a +different shared library you can override +[here](https://github.com/cnuernber/libpython-clj/blob/master/src/libpython_clj/jna/base.clj#L12). + + +### Execute Some Python + +`*out*` and `*err*` capture python stdout and stderr respectively. + +```clojure + +user> (run-simple-string "print('hey')") +hey +{:globals + {'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': , '__spec__': None, '__annotations__': {}, '__builtins__': }, + :locals + {'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': , '__spec__': None, '__annotations__': {}, '__builtins__': }} +``` + +The results have been 'bridged' into java meaning they are still python objects but +there are java wrappers over the top of them. For instance, `Object.toString` forwards +its implementation to the python function `__str__`. + +```clojure +(def bridged (run-simple-string "print('hey')")) +(instance? java.util.Map (:globals bridged)) +true +user> (:globals bridged) +{'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': , '__spec__': None, '__annotations__': {}, '__builtins__': } +``` + +We can get and set global variables here. If we run another string, these are in the +environment. The globals map itself is the global dict of the main module: + +```clojure +(def main-globals (-> (add-module "__main__") + (module-dict))) +#'user/main-globals + +user> main-globals +{'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': , '__spec__': None, '__annotations__': {}, '__builtins__': } +user> (keys main-globals) +("__name__" + "__doc__" + "__package__" + "__loader__" + "__spec__" + "__annotations__" + "__builtins__") +user> (get main-globals "__name__") +"__main__" +user> (.put main-globals "my_var" 200) +nil + +user> (run-simple-string "print('your variable is:' + str(my_var))") +your variable is:200 +{:globals + {'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': , '__spec__': None, '__annotations__': {}, '__builtins__': , 'my_var': 200}, + :locals + {'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': , '__spec__': None, '__annotations__': {}, '__builtins__': , 'my_var': 200}} +``` + +Running python isn't ever really necessary, however, although it may at times be +convenient. You can call attributes from clojure easily: + +```clojure +user> (def np (import-module "numpy")) +#'user/np +user> (def ones-ary (call-attr np "ones" [2 3])) +#'user/ones-ary +user> ones-ary +[[1. 1. 1.] + [1. 1. 1.]] +user> (call-attr ones-ary "__len__") +2 +user> (vec ones-ary) +[[1. 1. 1.] [1. 1. 1.]] +user> (type (first *1)) +:pyobject +user> (get-attr ones-ary "shape") +(2, 3) +user> (vec (get-attr ones-ary "shape")) +[2 3] + +user> (att-type-map ones-ary) +{"T" :ndarray, + "__abs__" :method-wrapper, + "__add__" :method-wrapper, + "__and__" :method-wrapper, + "__array__" :builtin-function-or-method, + "__array_finalize__" :none-type, + "__array_function__" :builtin-function-or-method, + "__array_interface__" :dict, + "__array_prepare__" :builtin-function-or-method, + "__array_priority__" :float, + "__array_struct__" :py-capsule, + "__array_ufunc__" :builtin-function-or-method, + "__array_wrap__" :builtin-function-or-method, + "__bool__" :method-wrapper, + "__class__" :type, + "__complex__" :builtin-function-or-method, + "__contains__" :method-wrapper, + ... + "std" :builtin-function-or-method, + "strides" :tuple, + "sum" :builtin-function-or-method, + "swapaxes" :builtin-function-or-method, + "take" :builtin-function-or-method, + "tobytes" :builtin-function-or-method, + "tofile" :builtin-function-or-method, + "tolist" :builtin-function-or-method, + "tostring" :builtin-function-or-method, + "trace" :builtin-function-or-method, + "transpose" :builtin-function-or-method, + "var" :builtin-function-or-method, + "view" :builtin-function-or-method} +``` + + +### att-type-map + +It can be extremely helpful to print out the attribute name->attribute type map: + +```clojure +user> (att-type-map ones-ary) +{"T" :ndarray, + "__abs__" :method-wrapper, + "__add__" :method-wrapper, + "__and__" :method-wrapper, + "__array__" :builtin-function-or-method, + "__array_finalize__" :none-type, + "__array_function__" :builtin-function-or-method, + "__array_interface__" :dict, + ... + "real" :ndarray, + "repeat" :builtin-function-or-method, + "reshape" :builtin-function-or-method, + "resize" :builtin-function-or-method, + "round" :builtin-function-or-method, + "searchsorted" :builtin-function-or-method, + "setfield" :builtin-function-or-method, + "setflags" :builtin-function-or-method, + "shape" :tuple, + "size" :int, + "sort" :builtin-function-or-method, + ... +} +``` + + +### Errors + + +Errors are caught and an exception is thrown. The error text is saved verbatim +in the exception: + + +```clojure +user> (run-simple-string "print('syntax errrr") +Execution error (ExceptionInfo) at libpython-clj.python.interpreter/check-error-throw (interpreter.clj:260). + File "", line 1 + print('syntax errrr + ^ +SyntaxError: EOL while scanning string literal +``` + +### Some Syntax Sugar + +```clojure +user> (py/from-import numpy linspace) +#'user/linspace +user> (linspace 2 3 :num 10) +[2. 2.11111111 2.22222222 2.33333333 2.44444444 2.55555556 + 2.66666667 2.77777778 2.88888889 3. ] +user> (doc linspace) +------------------------- +user/linspace + + Return evenly spaced numbers over a specified interval. + + Returns `num` evenly spaced samples, calculated over the + interval [`start`, `stop`]. + +``` + +* `from-import` - sugar around python `from a import b`. Takes multiple b's. +* `import-as` - surgar around python `import a as b`. +* `$a` - call an attribute using symbol att name. Keywords map to kwargs +* `$c` - call an object mapping keywords to kwargs + + +#### Experimental Sugar + +We are trying to find the best way to handle attributes in order to shorten +generic python notebook-type usage. The currently implemented direction is: + +* `$.` - get an attribute. Can pass in symbol, string, or keyword +* `$..` - get an attribute. If more args are present, get the attribute on that +result. + +```clojure +user> (py/$. numpy linspace) + +user> (py/$.. numpy random shuffle) + +``` + + +### Numpy + +Speaking of numpy, you can move data between numpy and java easily. + +```clojure +user> (def tens-data (as-tensor ones-ary)) +#'user/tens-data +user> (println tens-data) +#tech.v2.tensor[2 3] +[[1.000 1.000 1.000] + [1.000 1.000 1.000]] +nil + + +user> (require '[tech.v2.datatype :as dtype]) +nil +user> (def ignored (dtype/copy! (repeat 6 5) tens-data)) +#'user/ignored +user> (.put main-globals "ones_ary" ones_ary) +Syntax error compiling at (*cider-repl cnuernber/libpython-clj:localhost:39019(clj)*:191:7). +Unable to resolve symbol: ones_ary in this context +user> (.put main-globals "ones_ary" ones-ary) +nil + +user> (run-simple-string "print(ones_ary)") +[[5. 5. 5.] + [5. 5. 5.]] +{:globals + {'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': , '__spec__': None, '__annotations__': {}, '__builtins__': , 'my_var': 200, 'ones_ary': array([[5., 5., 5.], + [5., 5., 5.]])}, + :locals + {'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': , '__spec__': None, '__annotations__': {}, '__builtins__': , 'my_var': 200, 'ones_ary': array([[5., 5., 5.], + [5., 5., 5.]])}} +``` + +So heavy data has a zero-copy route. Anything backed by a `:native-buffer` has a +zero copy pathway to and from numpy. For more information on how this happens, +please refer to the datatype library [documentation](https://github.com/techascent/tech.datatype/tree/master/docs). + +Just keep in mind, careless usage of zero copy is going to cause spooky action at a +distance. From 8110dc829503d724836ed1d5c775f9966071adff Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Fri, 27 Dec 2019 07:59:17 -0700 Subject: [PATCH 118/456] Updating usage --- docs/Usage.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/Usage.md b/docs/Usage.md index 1edb379..18be975 100644 --- a/docs/Usage.md +++ b/docs/Usage.md @@ -55,9 +55,10 @@ INFO: Reference thread starting :ok ``` -This dynamically finds the python shared library and loads it. If you desire a -different shared library you can override -[here](https://github.com/cnuernber/libpython-clj/blob/master/src/libpython_clj/jna/base.clj#L12). +This dynamically finds the python shared library and loads it using output from +the python3 executable on your system. For information about how that works, +please checkout the code +[here](https://github.com/cnuernber/libpython-clj/blob/master/src/libpython_clj/python/interpreter.clj#L30). ### Execute Some Python From aadb781eacb2eebce79fb3f5f88b2c960a5a3619 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Fri, 27 Dec 2019 08:01:42 -0700 Subject: [PATCH 119/456] editing --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3f88b1c..941dbf9 100644 --- a/README.md +++ b/README.md @@ -105,8 +105,9 @@ the `libpython3.Xm.so` shared library so for example if we are loading python 3.6 we look for `libpython3.6m.so` on Linux or `libpython3.6m.dylib` on the Mac. This pathway has allowed us support Conda albeit with some work. For examples -using Conda, check out the facial rec repository above or look into how we build -our test docker containers in [this project](scripts/run-conda-docker). +using Conda, check out the facial rec repository above or look into how we +[build](scripts/build-conda-docker) +our test [docker containers](dockerfiles/CondaDockerfile). ## Further Information From 7a01196bba77bf7f0aaee59b7ba8825fa60b1d50 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Fri, 27 Dec 2019 14:07:57 -0700 Subject: [PATCH 120/456] making stress tests mandatory --- test/libpython_clj/stress_test.clj | 54 +++++++++++++----------------- 1 file changed, 24 insertions(+), 30 deletions(-) diff --git a/test/libpython_clj/stress_test.clj b/test/libpython_clj/stress_test.clj index 939d2d0..e4eecd1 100644 --- a/test/libpython_clj/stress_test.clj +++ b/test/libpython_clj/stress_test.clj @@ -67,9 +67,8 @@ def getmultidata(): (alter-var-root #'tech.resource.stack/*resource-context* (constantly nil)) -(defn forever-test - [] - (doseq [items (partition 999 (get-data))] +(deftest forever-test + (doseq [items (take 10 (partition 999 (get-data)))] ;;One way is to use the GC (time (do @@ -90,60 +89,57 @@ def getmultidata(): items)))))) -(defn multidata-test - [] - (doseq [items (partition 1000 (get-multi-data))] +(deftest multidata-test + (doseq [items (take 5 (partition 1000 (get-multi-data)))] (time (do (py/with-gil-stack-rc-context (mapv py/->jvm items)) :ok)))) -(defn str-marshal-test - [] +(deftest str-marshal-test (let [test-str (py/->python "a nice string to work with")] (time (py/with-gil-stack-rc-context - (dotimes [iter 100000] + (dotimes [iter 1000] (py/->jvm test-str)))))) -(defn dict-marshal-test - [] +(deftest dict-marshal-test (let [test-item (py/->python {:a 1 :b 2})] (time (py/with-gil-stack-rc-context - (dotimes [iter 10000] + (dotimes [iter 100] (py/->jvm test-item)))))) -(defn print-stress-test - [] - (dotimes [iter 10000] +(deftest print-stress-test + (dotimes [iter 100] (with-out-str (print-data)))) -(defn new-cls-stress-test - [] - (dotimes [iter 1000] +(deftest new-cls-stress-test + (dotimes [iter 100] (py/with-gil-stack-rc-context (let [test-cls (py/create-class "testcls" nil - {"__init__" (py/make-tuple-fn + {"__init__" (py/make-tuple-instance-fn (fn [self name shares price] (py/set-attr! self "name" name) (py/set-attr! self "shares" shares) (py/set-attr! self "price" price) nil)) - "cost" (py/make-tuple-fn + "cost" (py/make-tuple-instance-fn (fn [self] - (* (py/$. self shares) - (py/$. self price)))) - "__str__" (py/make-tuple-fn + (let [self (py/as-jvm self)] + (* (py/$. self shares) + (py/$. self price))))) + "__str__" (py/make-tuple-instance-fn (fn [self] - ;;Self is just a dict so it converts to a hashmap - (pr-str {"name" (py/$. self name) - "shares" (py/$. self shares) - "price" (py/$. self price)}))) + (let [self (py/as-jvm self)] + ;;Self is just a dict so it converts to a hashmap + (pr-str {"name" (py/$. self name) + "shares" (py/$. self shares) + "price" (py/$. self price)})))) "testvar" 55} ) new-inst (test-cls "ACME" 50 90)] @@ -152,6 +148,4 @@ def getmultidata(): (is (= 55 (py/$. new-inst testvar))) (is (= {"name" "ACME", "shares" 50, "price" 90} - (edn/read-string (.toString new-inst)))) - - )))) + (edn/read-string (.toString new-inst)))))))) From b1ea809db27782ede88174eb856b27a6dcdb95d7 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Wed, 1 Jan 2020 16:06:01 -0700 Subject: [PATCH 121/456] support for java characters --- src/libpython_clj/python/bridge.clj | 2 ++ src/libpython_clj/python/object.clj | 8 ++++++++ test/libpython_clj/python_test.clj | 5 +++++ 3 files changed, 15 insertions(+) diff --git a/src/libpython_clj/python/bridge.clj b/src/libpython_clj/python/bridge.clj index a385f70..0a6d8dc 100644 --- a/src/libpython_clj/python/bridge.clj +++ b/src/libpython_clj/python/bridge.clj @@ -1003,6 +1003,8 @@ (as-python [item options] (->python item)) String (as-python [item options] (->python item)) + Character + (as-python [item options] (->python item)) Symbol (as-python [item options] (->python item)) Keyword diff --git a/src/libpython_clj/python/object.clj b/src/libpython_clj/python/object.clj index 2c6f8be..10ab9d7 100644 --- a/src/libpython_clj/python/object.clj +++ b/src/libpython_clj/python/object.clj @@ -358,6 +358,11 @@ Object's refcount is bad. Crash is imminent" pyobj-value refcount py-type-name) (->py-object-ptr [item] (with-gil (->python item))) + Character + (convertible-to-pyobject-ptr? [item] true) + (->py-object-ptr [item] + (with-gil + (->python (str item)))) String (convertible-to-pyobject-ptr? [item] true) (->py-object-ptr [item] @@ -597,6 +602,9 @@ Object's refcount is bad. Crash is imminent" pyobj-value refcount py-type-name) String (->python [item options] (->py-string item)) + Character + (->python [item options] + (->py-string (str item))) Symbol (->python [item options] (->py-string (name item))) diff --git a/test/libpython_clj/python_test.clj b/test/libpython_clj/python_test.clj index 2553fbf..8151c5f 100644 --- a/test/libpython_clj/python_test.clj +++ b/test/libpython_clj/python_test.clj @@ -342,3 +342,8 @@ jvm-dict (py/->jvm py-dict)] (is (= (get jvm-dict "ratings") (py/->jvm (py/get-item py-dict "ratings")))))) + + +(deftest characters + (is (= (py/->jvm (py/->python "c")) + (py/->jvm (py/->python \c))))) From c22efe5d057a0048941e3a7fd93ac6aeff5e9920 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Thu, 2 Jan 2020 12:07:29 -0700 Subject: [PATCH 122/456] editing --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08a3494..f313b2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,9 @@ classes and some serious refactoring overall. * Most of the datatype libraries math operators supported by numpy objects (+,-,etc). * Numpy objects can be used in datatype library functions (like copy, make-container) and work in optimized ways. + +#### Bugs +* Support for java character <-> py string ```clojure libpython-clj.python.numpy-test> (def test-ary (py/$a np-mod array (->> (range 9) From 1576bf280ec04dffeafdd140a38d35e6d165d686 Mon Sep 17 00:00:00 2001 From: "J.J. Tolton" Date: Fri, 3 Jan 2020 14:03:05 -0500 Subject: [PATCH 123/456] Changelog edit (#40) * minor edits * minor edits * minor edits * minor edits --- CHANGELOG.md | 81 ++++++++++++++++++++++++---------------------------- 1 file changed, 38 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f313b2d..95d267e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,27 +3,24 @@ ## 1.30-SNAPSHOT This release is a big one. With finalizing `require-python` we have a clear way -to use python in daily use and make it look good in normal clojure usage. There +to use Python in daily use and make it look good in normal Clojure usage. There is a demo of [facial recognition](https://github.com/cnuernber/facial-rec) using some of the best open systems for doing this; this demo would absolutely not be possible without this library due to the extensive use of numpy and cython to implement the -face detection. We can now interact with even very complex python systems with -roughly the same performance as a pure-python system. +face detection. We can now interact with even very complex Python systems with +roughly the same performance as a pure Python system. #### Finalized `require-python` -Lots of work put in to make the require-python pathway work with +Lots of work put in to make the `require-python` pathway work with classes and some serious refactoring overall. #### Better Numpy Support * Most of the datatype libraries math operators supported by numpy objects (+,-,etc). -* Numpy objects can be used in datatype library functions (like copy, make-container) +* Numpy objects can be used in datatype library functions (like `copy`, `make-container`) and work in optimized ways. -#### Bugs -* Support for java character <-> py string - ```clojure libpython-clj.python.numpy-test> (def test-ary (py/$a np-mod array (->> (range 9) (partition 3) @@ -43,6 +40,10 @@ libpython-clj.python.numpy-test> (dfn/> test-ary 4) [ True True True]] ``` + +#### Bugs +* Support for java character <-> py string + #### Bugs Fixed @@ -52,56 +53,49 @@ libpython-clj.python.numpy-test> (dfn/> test-ary 4) ## 1.29 -* Found/fixed issue with ->jvm and large python dictionaries. +* Found/fixed issue with `->jvm` and large Python dictionaries. ## 1.28 - -* `(range 5)` - clojure ranges <-> python ranges when possible. -* bridged types derive from collections.abc.* so that they pass instance checks in +* `(range 5)` - Clojure ranges <-> Python ranges when possible. +* bridged types derive from `collections.abc.*` so that they pass instance checks in libraries that are checking for generic types. * Really interesting unit test for [generators, ranges and sequences](test/libpython_clj/iter_gen_seq_test.clj). - ## 1.27 -* Fixed bug where (as-python {:is_train false}) results in a dictionary with a none +* Fixed bug where `(as-python {:is_train false})` results in a dictionary with a none value instead of a false value. This was found through hours of debugging why - mxnet's forward function call was returning different values in clojure than in - python. + mxnet's forward function call was returning different values in Clojure than in + Python. ## 1.26 - * [python startup work](https://github.com/cnuernber/libpython-clj/commit/16da3d885f29bde59ea219c9438b9d3654387971) * [python generators & clojure transducers](https://github.com/cnuernber/libpython-clj/pull/27) * [requre-python reload fix](https://github.com/cnuernber/libpython-clj/pull/24) -* Bugfix with require-python :reload semantics. - +* Bugfix with `require-python` :reload semantics. ## 1.25 - -Fixed (with tests) major issue with require-python. +Fixed (with tests) major issue with `require-python`. ## 1.24 - Clojure's range is now respected in two different ways: -* `(range)` - bridges to a python iterable -* `(range 5)` - copies to a python list +* `(range)` - bridges to a Python iterable +* `(range 5)` - copies to a Python list ## 1.23 - -Equals, hashcode, nice default .toString of python types: +Equals, hashcode, nice default `.toString` of Python types: ```clojure user> (require '[libpython-clj.python :as py]) @@ -141,39 +135,39 @@ user> (py/python-type *1) ## 1.22 -Working to make more python environments work out of the box. Currently have a +Working to make more Python environments work out of the box. Currently have a testcase for conda working in a clean install of a docker container. There is now a new method: `libpython-clj.python.interpreter/detect-startup-info` that attempts call `python3-config --prefix` and `python3 --version` in order to automagically -configure the python library. +configure the Python library. ## 1.21 -Bugfix release. Passing infinite sequences to python functions was +Bugfix release. Passing infinite sequences to Python functions was causing a hang as libpython-clj attempted to copy the sequence. The current calling convention does a shallow copy of things that are list-like or map-like, while bridging things that are iterable or don't fall into the above categories. This exposed a bug that caused reference counting to be subtly wrong when -python iterated through a bridged object. And that was my life for a day. +Python iterated through a bridged object. And that was my life for a day. ## 1.20 -With two many huge things we had to skip a few versions! +With too many huge things we had to skip a few versions! #### require-python -`require-python` works like require but it works on python modules. +`require-python` works like require but it works on Python modules. `require-python` dynamically loads the module and exports it's symbols into -a clojure namespace. There are many options available for this pathway. +a Clojure namespace. There are many options available for this pathway. -This implements a big step towards embedding python in Clojure in a simple, +This implements a big step towards embedding Python in Clojure in a simple, clear, and easy to use way. One important thing to consider is the require -has a `:reload:` option to allow you to actively develop a python module and -test it via clojure. +has a `:reload:` option to allow you to actively develop a Python module and +test it via Clojure. This excellent work was in large part done by [James Tolton](https://github.com/jjtolton). @@ -184,7 +178,7 @@ This excellent work was in large part done by [James Tolton](https://github.com/ #### Clojure-defined Python classes -You can now extend a tuple of python classes (or implement a new one). This system +You can now extend a tuple of Python classes (or implement a new one). This system allows, among many things, us to use frameworks that use derivation as part of their public API. Please see [classes-test](test/libpython_clj/classes_test.clj) for a documented example of a simple pathway through the new API. Note that if you use vanilla @@ -201,13 +195,13 @@ a refcount to their return values. ## 1.16 -Fixed a bug where the system would load multiple python libraries, not stopping +Fixed a bug where the system would load multiple Python libraries, not stopping after the first valid library loaded. There are two ways to control the system's -python library loading mechanism: +Python library loading mechanism: -1. Pass in a library name in initialize! -2. alter-var-root the list of libraries in libpython-clj.jna.base before - calling initialize!. +1. Pass in a library name in `initialize!` +2. `alter-var-root` the list of libraries in `libpython-clj.jna.base` before + calling `initialize!`. ## 1.15 @@ -225,11 +219,12 @@ user> (py/$.. numpy random shuffle) libpython-clj now searches for several shared libraries instead of being hardcoded to just one of them. Because of this, there is now: + ```clojure libpython-clj.jna.base/*python-library-names* ``` This is a sequence of library names that will be tried in order. -You can also pass in the desired library name as part of the initialize! call and +You can also pass in the desired library name as part of the `initialize!` call and only this name will be tried. From cf27606166fd3e3891411fdbbd0f06a45bb5b77e Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Fri, 3 Jan 2020 12:05:10 -0700 Subject: [PATCH 124/456] editing --- CHANGELOG.md | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95d267e..d9a2459 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,14 +39,9 @@ libpython-clj.python.numpy-test> (dfn/> test-ary 4) [False False True] [ True True True]] ``` - -#### Bugs -* Support for java character <-> py string - - #### Bugs Fixed - +* Support for java character <-> py string * Fixed potential crash related to use of delay mechanism and stack based gc. * Added logging to complain loudly if refcounts appear to be bad. From c59ac238f839e4e571dafbb8e22b7c47c6993106 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sat, 4 Jan 2020 11:47:39 -0700 Subject: [PATCH 125/456] upgrade dtype --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index c4414a3..bc0b495 100644 --- a/project.clj +++ b/project.clj @@ -5,7 +5,7 @@ :url "https://www.eclipse.org/legal/epl-2.0/"} :dependencies [[org.clojure/clojure "1.10.1"] [camel-snake-kebab "0.4.0"] - [techascent/tech.datatype "4.65"] + [techascent/tech.datatype "4.67"] [cheshire "5.9.0"]] :repl-options {:init-ns user} :java-source-paths ["java"]) From 4a7a0cb3a0ab0b75d69478e57ead451e5b8c06c1 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 5 Jan 2020 12:37:10 -0700 Subject: [PATCH 126/456] upgrade dtype --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index bc0b495..8272704 100644 --- a/project.clj +++ b/project.clj @@ -5,7 +5,7 @@ :url "https://www.eclipse.org/legal/epl-2.0/"} :dependencies [[org.clojure/clojure "1.10.1"] [camel-snake-kebab "0.4.0"] - [techascent/tech.datatype "4.67"] + [techascent/tech.datatype "4.68"] [cheshire "5.9.0"]] :repl-options {:init-ns user} :java-source-paths ["java"]) From dc0fcd559e144ffc51c68755f11727b168f0e2dd Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 5 Jan 2020 13:06:08 -0700 Subject: [PATCH 127/456] upgrade dtype-fix binary-search --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 8272704..70bb98a 100644 --- a/project.clj +++ b/project.clj @@ -5,7 +5,7 @@ :url "https://www.eclipse.org/legal/epl-2.0/"} :dependencies [[org.clojure/clojure "1.10.1"] [camel-snake-kebab "0.4.0"] - [techascent/tech.datatype "4.68"] + [techascent/tech.datatype "4.69"] [cheshire "5.9.0"]] :repl-options {:init-ns user} :java-source-paths ["java"]) From 3b22a653cdc11b0519928208b15b31b959989274 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 7 Jan 2020 11:18:39 -0700 Subject: [PATCH 128/456] moved to clojure.data.json --- project.clj | 2 +- src/libpython_clj/python/interpreter.clj | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/project.clj b/project.clj index 70bb98a..36ae7b2 100644 --- a/project.clj +++ b/project.clj @@ -6,6 +6,6 @@ :dependencies [[org.clojure/clojure "1.10.1"] [camel-snake-kebab "0.4.0"] [techascent/tech.datatype "4.69"] - [cheshire "5.9.0"]] + [org.clojure/data.json "0.2.7"]] :repl-options {:init-ns user} :java-source-paths ["java"]) diff --git a/src/libpython_clj/python/interpreter.clj b/src/libpython_clj/python/interpreter.clj index ec023f8..e59cf56 100644 --- a/src/libpython_clj/python/interpreter.clj +++ b/src/libpython_clj/python/interpreter.clj @@ -9,7 +9,7 @@ [clojure.java.io :as io] [clojure.string :as s] [clojure.tools.logging :as log] - [cheshire.core :as json]) + [clojure.data.json :as json]) (:import [libpython_clj.jna JVMBridge PyObject] @@ -39,7 +39,7 @@ print(json.dumps( \"exec_prefix\": sys.exec_prefix, \"version\": list(sys.version_info)[:3]}))")] (when (= 0 exit) - (json/parse-string out true)))) + (json/read-str out :key-fn keyword)))) (defn python-system-info "An information map about the Python system information provided From d8f5c315bd61c91821bed38ec9e41fc963d756ad Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 7 Jan 2020 11:20:51 -0700 Subject: [PATCH 129/456] 1.30 --- CHANGELOG.md | 2 +- project.clj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9a2459..00ef5a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Time for a ChangeLog! -## 1.30-SNAPSHOT +## 1.30 This release is a big one. With finalizing `require-python` we have a clear way to use Python in daily use and make it look good in normal Clojure usage. There diff --git a/project.clj b/project.clj index 36ae7b2..6cf5177 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.30-SNAPSHOT" +(defproject cnuernber/libpython-clj "1.30" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" From f19ce98bef7ac6f8505d926fac1d3711003b52e3 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 7 Jan 2020 11:20:59 -0700 Subject: [PATCH 130/456] snap --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 6cf5177..4219e1c 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.30" +(defproject cnuernber/libpython-clj "1.31-SNAPSHOT" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" From 6dcdaaa95c177d7dbb682d8ec757263dc29db693 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 7 Jan 2020 13:23:59 -0700 Subject: [PATCH 131/456] Attempt at datafy/nav metadata pathway --- src/libpython_clj/python/bridge.clj | 4 +- src/libpython_clj/python/metadata.clj | 234 ++++++++++++++++++++++++++ src/libpython_clj/require.clj | 3 +- 3 files changed, 238 insertions(+), 3 deletions(-) create mode 100644 src/libpython_clj/python/metadata.clj diff --git a/src/libpython_clj/python/bridge.clj b/src/libpython_clj/python/bridge.clj index 0a6d8dc..d8ec7d0 100644 --- a/src/libpython_clj/python/bridge.clj +++ b/src/libpython_clj/python/bridge.clj @@ -418,7 +418,9 @@ (let [arglist (vec arglist)] (case (count arglist) 1 (.get this (first arglist)) - 2 (.put this (first arglist) (second arglist))))))))) + 2 (.put this (first arglist) (second arglist))))) + py-proto/PPyObjectBridgeToMap + (as-map [item] item))))) (defn generic-python-as-list diff --git a/src/libpython_clj/python/metadata.clj b/src/libpython_clj/python/metadata.clj new file mode 100644 index 0000000..62b186f --- /dev/null +++ b/src/libpython_clj/python/metadata.clj @@ -0,0 +1,234 @@ +(ns libpython-clj.python.metadata + (:refer-clojure :exclude [fn? doc]) + (:require [libpython-clj.python + :refer [import-module as-jvm get-attr call-attr callable? has-attr? + ->jvm with-gil] + :as py] + [libpython-clj.python.protocols :as py-proto] + [clojure.core.protocols :as clj-proto] + [clojure.tools.logging :as log]) + (:import [libpython_clj.python.protocols PPyObject])) + + +(py/initialize!) + + +(def ^:private builtins (as-jvm (import-module "builtins") {})) +(def ^:private inspect (as-jvm (import-module "inspect") {})) +(def ^:private argspec (get-attr inspect "getfullargspec")) +(def ^:private py-source (get-attr inspect "getsource")) +(def ^:private types (import-module "types")) +(def ^:private fn-type + (call-attr builtins "tuple" + [(get-attr types "FunctionType") + (get-attr types "BuiltinFunctionType")])) + +(def ^:private method-type + (call-attr builtins "tuple" + [(get-attr types "MethodType") + (get-attr types "BuiltinMethodType")])) + +(def ^:private isinstance? (get-attr builtins "isinstance")) +(def ^:private fn? #(isinstance? % fn-type)) +(def ^:private method? #(isinstance? % method-type)) +(def ^:private doc #(try + (get-attr % "__doc__") + (catch Exception e + ""))) +(def ^{:private false :public true} + get-pydoc doc) +(def ^:private vars (get-attr builtins "vars")) +(def ^:private pyclass? (get-attr inspect "isclass")) +(def ^:private pymodule? (get-attr inspect "ismodule")) +(def ^:private importlib_util (import-module "importlib.util")) +(defn ^:private findspec [x] + (let [-findspec + (-> importlib_util (get-attr "find_spec"))] + (-findspec x))) + +(defn ^:private py-fn-argspec [f] + (if-let [spec (try (when-not (pyclass? f) + (argspec f)) + (catch Throwable e nil))] + {:args (->jvm (get-attr spec "args") {}) + :varargs (->jvm (get-attr spec "varargs") {}) + :varkw (->jvm (get-attr spec "varkw") {}) + :defaults (->jvm (get-attr spec "defaults") {}) + :kwonlyargs (->jvm (get-attr spec "kwonlyargs") {}) + :kwonlydefaults (->jvm (get-attr spec "kwonlydefaults") {}) + :annotations (->jvm (get-attr spec "annotations") {})} + (py-fn-argspec (get-attr f "__init__")))) + +(defn ^:private py-class-argspec [class] + (let [constructor (get-attr class "__init__")] + (py-fn-argspec constructor))) + + +(defn pyargspec [x] + ;; TODO: certain builtin functions have + ;; ..: signatures that are found in the first line + ;; ..: of their docstring, aka, print. + ;; ..: These seem to be uniform enough that + ;; ..: most IDEs have a way of creating stubs + ;; ..: for the signature. If there is a uniform way + ;; ..: to do this that doesn't simply involve an + ;; ..: army of devs doing transcription I'd like to + ;; ..: pull that in here + (cond + (fn? x) (py-fn-argspec x) + (method? x) (py-fn-argspec x) + ;; (builtin-function? x) (py-builtin-fn-argspec x) + ;; (builtin-method? x) (py-builtin-method-argspec x) + (string? x) "" + (number? x) "" + :else (py-class-argspec x))) + + +(defn pyarglists + ([argspec] (pyarglists argspec + (if-let [defaults + (not-empty (:defaults argspec))] + defaults + []))) + ([argspec defaults] (pyarglists argspec defaults [])) + ([{:as argspec + args :args + varkw :varkw + varargs :varargs + kwonlydefaults :kwonlydefaults + kwonlyargs :kwonlyargs} + defaults res] + (let [n-args (count args) + n-defaults (count defaults) + n-pos-args (- n-args n-defaults) + pos-args (transduce + (comp + (take n-pos-args) + (map symbol)) + (completing conj) + [] + args) + kw-default-args (transduce + (comp + (drop n-pos-args) + (map symbol)) + (completing conj) + [] + args) + or-map (transduce + (comp + (partition-all 2) + (map vec) + (map (fn [[k v]] [(symbol k) v]))) + (completing (partial apply assoc)) + {} + (concat + (interleave kw-default-args defaults) + (flatten (seq kwonlydefaults)))) + + as-varkw (when (not (nil? varkw)) + {:as (symbol varkw)}) + default-map (transduce + (comp + (partition-all 2) + (map vec) + (map (fn [[k v]] [(symbol k) (keyword k)]))) + (completing (partial apply assoc)) + {} + (concat + (interleave kw-default-args defaults) + (flatten (seq kwonlydefaults)))) + + kwargs-map (merge default-map + (when (not-empty or-map) + {:or or-map}) + (when (not-empty as-varkw) + as-varkw)) + opt-args + (cond + (and (empty? kwargs-map) + (nil? varargs)) '() + (empty? kwargs-map) (list '& [(symbol varargs)]) + (nil? varargs) (list '& [kwargs-map]) + :else (list '& [(symbol varargs) + kwargs-map])) + + arglist ((comp vec concat) (list* pos-args) opt-args)] + (let [arglists (conj res arglist) + defaults' (if (not-empty defaults) (pop defaults) []) + argspec' (update argspec :args + (fn [args] + (if (not-empty args) + (pop args) + args)))] + + (if (and (empty? defaults) (empty? defaults')) + arglists + (recur argspec' defaults' arglists)))))) + +(defn py-fn-metadata [fn-name x {:keys [no-arglists?]}] + (let [fn-argspec (pyargspec x) + fn-docstr (get-pydoc x)] + (merge + fn-argspec + {:doc fn-docstr + :name fn-name} + (when (and (callable? x) + (not no-arglists?)) + (try + {:arglists (pyarglists fn-argspec)} + (catch Throwable e + nil)))))) + + +(extend-type PPyObject + clj-proto/Datafiable + (datafy [item] + (with-meta + (with-gil + (->> (vars item) + (py-proto/as-map) + (map (fn [[att-name att-val]] + (when att-val + (try + (let [att-type (py-proto/python-type att-val)] + [att-name + (merge {:type att-type + :doc (doc att-val) + :str (.toString att-val) + :flags (->> {:pyclass? pyclass? + :callable? callable? + :fn? fn? + :method? method? + :pymodule? pymodule?} + (map (fn [[kwd f]] + (when (f att-val) + kwd))) + (remove nil?) + set)} + + (when (callable? att-val) + (py-fn-metadata att-name att-val {})) + (when (has-attr? att-val "__module__") + {:module (get-attr att-val "__module__")}) + (when (has-attr? att-val "__name__") + {:name (get-attr att-val "__name__")}))]) + (catch Throwable e + (log/warnf "Metadata generation failed for %s:%s" + (.toString item) + att-name) + nil))))) + (remove nil?) + (into {}))) + {`clj-proto/nav + (fn nav-pyval + [coll f val] + (cond + (= :module (:type val)) + (as-jvm (import-module (:name val)) {}) + (= :type (:type val)) + (let [mod (as-jvm (import-module (:module val)) {}) + cls-obj (get-attr mod (:name val))] + cls-obj) + :else + val))}))) diff --git a/src/libpython_clj/require.clj b/src/libpython_clj/require.clj index 0e491b0..029d5d3 100644 --- a/src/libpython_clj/require.clj +++ b/src/libpython_clj/require.clj @@ -1,8 +1,7 @@ (ns libpython-clj.require (:refer-clojure :exclude [fn? doc]) (:require [libpython-clj.python :as py] - [clojure.tools.logging :as log] - [clojure.java.io :as io])) + [clojure.tools.logging :as log])) (py/initialize!) From 459551e68582604c63e56141c8d31ce211a73aaa Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 7 Jan 2020 20:34:36 -0700 Subject: [PATCH 132/456] Moved metadata and moving towards modularized metadata <-> namespace --- src/libpython_clj/{python => }/metadata.clj | 77 +++++++-------- src/libpython_clj/require.clj | 102 +++++++------------- 2 files changed, 70 insertions(+), 109 deletions(-) rename src/libpython_clj/{python => }/metadata.clj (81%) diff --git a/src/libpython_clj/python/metadata.clj b/src/libpython_clj/metadata.clj similarity index 81% rename from src/libpython_clj/python/metadata.clj rename to src/libpython_clj/metadata.clj index 62b186f..5343e05 100644 --- a/src/libpython_clj/python/metadata.clj +++ b/src/libpython_clj/metadata.clj @@ -1,4 +1,4 @@ -(ns libpython-clj.python.metadata +(ns libpython-clj.metadata (:refer-clojure :exclude [fn? doc]) (:require [libpython-clj.python :refer [import-module as-jvm get-attr call-attr callable? has-attr? @@ -101,43 +101,30 @@ (let [n-args (count args) n-defaults (count defaults) n-pos-args (- n-args n-defaults) - pos-args (transduce - (comp - (take n-pos-args) - (map symbol)) - (completing conj) - [] - args) - kw-default-args (transduce - (comp - (drop n-pos-args) - (map symbol)) - (completing conj) - [] - args) - or-map (transduce - (comp - (partition-all 2) - (map vec) - (map (fn [[k v]] [(symbol k) v]))) - (completing (partial apply assoc)) - {} - (concat - (interleave kw-default-args defaults) - (flatten (seq kwonlydefaults)))) - + pos-args (->> args + (take n-pos-args) + (map symbol) + (into [])) + kw-default-args (->> args + (drop n-pos-args) + (map symbol) + (into [])) + or-map (->> (concat + (interleave kw-default-args defaults) + (flatten (seq kwonlydefaults))) + (partition-all 2) + (map vec) + (map (fn [[k v]] [(symbol k) v])) + (into {})) as-varkw (when (not (nil? varkw)) {:as (symbol varkw)}) - default-map (transduce - (comp - (partition-all 2) - (map vec) - (map (fn [[k v]] [(symbol k) (keyword k)]))) - (completing (partial apply assoc)) - {} - (concat - (interleave kw-default-args defaults) - (flatten (seq kwonlydefaults)))) + default-map (->> (concat + (interleave kw-default-args defaults) + (flatten (seq kwonlydefaults))) + (partition-all 2) + (map vec) + (map (fn [[k v]] [(symbol k) (keyword k)])) + (into {})) kwargs-map (merge default-map (when (not-empty or-map) @@ -223,12 +210,14 @@ {`clj-proto/nav (fn nav-pyval [coll f val] - (cond - (= :module (:type val)) - (as-jvm (import-module (:name val)) {}) - (= :type (:type val)) - (let [mod (as-jvm (import-module (:module val)) {}) - cls-obj (get-attr mod (:name val))] - cls-obj) - :else + (if (map? val) + (cond + (= :module (:type val)) + (as-jvm (import-module (:name val)) {}) + (= :type (:type val)) + (let [mod (as-jvm (import-module (:module val)) {}) + cls-obj (get-attr mod (:name val))] + cls-obj) + :else + val) val))}))) diff --git a/src/libpython_clj/require.clj b/src/libpython_clj/require.clj index 029d5d3..edccaa9 100644 --- a/src/libpython_clj/require.clj +++ b/src/libpython_clj/require.clj @@ -1,6 +1,9 @@ (ns libpython-clj.require (:refer-clojure :exclude [fn? doc]) (:require [libpython-clj.python :as py] + [clojure.datafy :refer [datafy nav]] + ;;Binds datafy/nav to python objects + [libpython-clj.metadata] [clojure.tools.logging :as log])) (py/initialize!) @@ -252,63 +255,32 @@ seq)] (throw (Exception. (format "Unsupported flags: %s" (set missing-flags)))))) - (letfn [(supported-flag-item - ;; scanned a supported tag token - [supported-flags flag results item items] - (cond - ;; add flag, continue scanning - (true? item) (next-flag-item - supported-flags - (conj results flag) - (first items) - (rest items)) - - ;; don't add flag, continue scanning - (false? item) (next-flag-item - supported-flags - results - (first items) - (rest items)) - :else - ;; unary flag -- add flag but scan current item/s - (next-flag-item - supported-flags - (conj results flag) - item - items))) - - ;; scan flags - (next-flag-item [supported-flags results item items] - (cond - ;; supported flag scanned, begin FSM parse - (get supported-flags item) - (let [flag (get supported-flags item) - remaining-flags (clojure.set/difference - supported-flags #{flag})] - (supported-flag-item - remaining-flags - flag - results - (first items) - (rest items))) - - ;; FSM complete - (nil? item) (into #{} results) - - ;; no flag scanned, continue scanning - :else (recur - supported-flags - results - (first items) - (rest items)))) - - ;; entrypoint - (get-flags [supported-flags reqs] - (next-flag-item supported-flags - [] - (first reqs) - (rest reqs)))] - (trampoline get-flags supported-flags reqs))) + ;;Loop through reqs. If a keyword is found and it is a supported flag, + ;;see if the next thing is a boolean with a default to true. + ;;If the flag is enabled (as false could be passed in), conj (or disj) to flag set + ;;Return reqs minus flags and booleans. + (loop [reqs reqs + retval-reqs [] + retval-flags #{}] + (if (seq reqs) + (let [next-item (first reqs) + reqs (rest reqs) + [bool-flag reqs] + (if (and (supported-flags next-item) + (boolean? (first reqs))) + [(first reqs) (rest reqs)] + [true reqs]) + retval-flags (if (supported-flags next-item) + (if bool-flag + (conj retval-flags next-item) + (disj retval-flags next-item)) + retval-flags) + retval-reqs (if (not (supported-flags next-item)) + (conj retval-reqs next-item) + retval-reqs)] + (recur reqs retval-reqs retval-flags)) + {:reqs retval-reqs + :flags retval-flags}))) (defn- extract-refer-symbols [{:keys [refer this-module]} public-data] @@ -362,14 +334,14 @@ (let [supported-flags #{:reload :no-arglists} [module-name & etc] req - flags (parse-flags supported-flags etc) - etc (into {} - (comp - (remove supported-flags) - (remove boolean?) - (partition-all 2) - (map vec)) - etc) + {flags :flags + etc :reqs} (parse-flags supported-flags etc) + etc (->> etc + (remove supported-flags) + (remove boolean?) + (partition-all 2) + (map vec) + (into {})) reload? (:reload flags) no-arglists? (:no-arglists flags) module-name-or-ns (:as etc module-name) From 78f56317ef1e4b18fe3eafaf3db67932826fd9d9 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Thu, 9 Jan 2020 13:01:08 -0700 Subject: [PATCH 133/456] Working through better metadata generation --- src/libpython_clj/jna/base.clj | 2 +- src/libpython_clj/metadata.clj | 122 ++++++++++++++++----- src/libpython_clj/require.clj | 6 +- test/libpython_clj/require_python_test.clj | 14 +-- 4 files changed, 107 insertions(+), 37 deletions(-) diff --git a/src/libpython_clj/jna/base.clj b/src/libpython_clj/jna/base.clj index 9cdf94b..6bd6c14 100644 --- a/src/libpython_clj/jna/base.clj +++ b/src/libpython_clj/jna/base.clj @@ -47,7 +47,7 @@ (defn ensure-pyobj [item] - (if-let [retval (->py-object-ptr item)] + (if-let [retval (as-pyobj item)] retval (throw (ex-info "Failed to get a pyobject pointer from object." {})))) diff --git a/src/libpython_clj/metadata.clj b/src/libpython_clj/metadata.clj index 5343e05..0439c19 100644 --- a/src/libpython_clj/metadata.clj +++ b/src/libpython_clj/metadata.clj @@ -167,46 +167,65 @@ (catch Throwable e nil)))))) +(defn pyobj-flags + [item] + (->> {:pyclass? pyclass? + :callable? callable? + :fn? fn? + :method? method? + :pymodule? pymodule?} + (map (fn [[kwd f]] + (when (f item) + kwd))) + (remove nil?) + set)) + + +(defn base-pyobj-map + [item] + (merge {:type (py/python-type item) + :doc (doc item) + :str (.toString item) + :flags (pyobj-flags item)} + (when (has-attr? item "__module__") + {:module (get-attr item "__module__")}) + (when (has-attr? item "__name__") + {:name (get-attr item "__name__")}))) + + +(defn scalar? + [att-val] + (or (string? att-val) + (number? att-val))) + (extend-type PPyObject clj-proto/Datafiable (datafy [item] (with-meta (with-gil - (->> (vars item) - (py-proto/as-map) + (->> (if (or (pyclass? item) + (pymodule? item)) + (-> (vars item) + (py-proto/as-map)) + (->> (py/dir item) + (map (juxt identity #(get-attr item %))))) (map (fn [[att-name att-val]] (when att-val (try - (let [att-type (py-proto/python-type att-val)] - [att-name - (merge {:type att-type - :doc (doc att-val) - :str (.toString att-val) - :flags (->> {:pyclass? pyclass? - :callable? callable? - :fn? fn? - :method? method? - :pymodule? pymodule?} - (map (fn [[kwd f]] - (when (f att-val) - kwd))) - (remove nil?) - set)} - - (when (callable? att-val) - (py-fn-metadata att-name att-val {})) - (when (has-attr? att-val "__module__") - {:module (get-attr att-val "__module__")}) - (when (has-attr? att-val "__name__") - {:name (get-attr att-val "__name__")}))]) + [att-name + (merge (base-pyobj-map att-val) + (when (callable? att-val) + (py-fn-metadata att-name att-val {})) + (when (scalar? att-val) + {:value att-val}))] (catch Throwable e (log/warnf "Metadata generation failed for %s:%s" (.toString item) att-name) nil))))) (remove nil?) - (into {}))) + (into (base-pyobj-map item)))) {`clj-proto/nav (fn nav-pyval [coll f val] @@ -221,3 +240,56 @@ :else val) val))}))) + + +(defn metadata-map->py-obj + [metadata-map] + (case (:type metadata-map) + :module (import-module (:name metadata-map)) + :type (-> (import-module (:module metadata-map)) + (get-attr (:name metadata-map))))) + + +(defn get-or-create-namespace! + [ns-symbol ns-doc] + (if-let [ns-obj (find-ns ns-symbol)] + ns-obj + (create-ns (with-meta ns-symbol + (merge (meta ns-symbol) + {:doc ns-doc}))))) + + +(defn apply-static-metadata-to-namespace! + "Given a metadata map, find the item associated with the map and for each + string keyword apply it to the namespace. Namespace is created if it does not + already exist. Returns the namespace symbol." + [ns-symbol metadata-map] + (let [target-item (metadata-map->py-obj metadata-map)] + (get-or-create-namespace! ns-symbol (:doc metadata-map)) + (doseq [[k v] metadata-map] + (when (has-attr? target-item k) + (let [att-val (get-attr target-item k)] + (intern ns-symbol (with-meta (symbol k) v) att-val)))) + ns-symbol)) + + +(defn apply-instance-metadata-to-namespace! + "In this case we have at some point in the past generated metadata from an instance + and we want to create an namespace to get intellisense on objects of that type. + The use case for this is you get a generic 'thing' back and you export its metadata + to a resource edn file. You can then always create a namespace from this metadata + and use that namespace to explore/use the instance and this will work regardless + of if a factory returns a derived object. + Returns the namespace symbol." + [ns-symbol metadata-map] + (get-or-create-namespace! ns-symbol (:doc metadata-map)) + (doseq [[k v] metadata-map] + (when (map? v) + (cond + (contains? (:flags v) :callable?) + (intern ns-symbol (with-meta (symbol k) v) + (fn [inst & args] + (apply (get-attr inst k) args))) + (contains? v :value) + (intern ns-symbol (with-meta (symbol k) v) (:value v))))) + ns-symbol) diff --git a/src/libpython_clj/require.clj b/src/libpython_clj/require.clj index edccaa9..2cde861 100644 --- a/src/libpython_clj/require.clj +++ b/src/libpython_clj/require.clj @@ -279,8 +279,7 @@ (conj retval-reqs next-item) retval-reqs)] (recur reqs retval-reqs retval-flags)) - {:reqs retval-reqs - :flags retval-flags}))) + retval-flags))) (defn- extract-refer-symbols [{:keys [refer this-module]} public-data] @@ -334,8 +333,7 @@ (let [supported-flags #{:reload :no-arglists} [module-name & etc] req - {flags :flags - etc :reqs} (parse-flags supported-flags etc) + flags (parse-flags supported-flags etc) etc (->> etc (remove supported-flags) (remove boolean?) diff --git a/test/libpython_clj/require_python_test.clj b/test/libpython_clj/require_python_test.clj index 7c35457..c1a8d27 100644 --- a/test/libpython_clj/require_python_test.clj +++ b/test/libpython_clj/require_python_test.clj @@ -36,19 +36,19 @@ (is (= #{:reload} (parse-flags #{:reload} '[:reload foo :as]))) (is (= #{:reload} (parse-flags #{:reload} '[foo :reload :as bar]))) (is (= #{} (parse-flags #{:reload} '[:reload false]))) - (is (= #{} (parse-flags #{:reload} '[:reload false :reload]))) - (is (= #{} (parse-flags #{:reload} '[:reload false :reload true]))) - (is (= #{:reload} (parse-flags #{:reload} '[:reload :reload false]))) - (is (= #{:reload} (parse-flags #{:reload} '[:reload true :reload false]))) - (is (= #{:a :b}) (parse-flags #{:a :b} '[:a true :b])) - (is (= #{:a :b}) (parse-flags #{:a :b} '[:a :b])) + (is (= #{:reload} (parse-flags #{:reload} '[:reload false :reload]))) + (is (= #{:reload} (parse-flags #{:reload} '[:reload false :reload true]))) + (is (= #{} (parse-flags #{:reload} '[:reload :reload false]))) + (is (= #{} (parse-flags #{:reload} '[:reload true :reload false]))) + (is (= #{:a :b} (parse-flags #{:a :b} '[:a true :b]))) + (is (= #{:a :b} (parse-flags #{:a :b} '[:a :b]))) (is (= #{:a :b} (parse-flags #{:a :b} '[:a true :b]))) (is (= #{:a :b} (parse-flags #{:a :b} '[:a :b true]))) (is (= #{:a :b} (parse-flags #{:a :b} '[:a true :b true]))) (is (= #{:b} (parse-flags #{:a :b} '[:a false :b true]))) (is (= #{:b} (parse-flags #{:a :b} '[:b true :a false]))) (is (= #{:b} (parse-flags #{:a :b} '[:b :a false]))) - (is (= #{:b} (parse-flags #{:a :b} '[:b :a false :a true]))))) + (is (= #{:b :a} (parse-flags #{:a :b} '[:b :a false :a true]))))) (deftest python-lib-configuration-test (let [python-lib-configuration #'req/python-lib-configuration From 760c9fd72472a7544e35bf0cb8aaf5e12bdf3b0a Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sat, 11 Jan 2020 15:37:46 -0700 Subject: [PATCH 134/456] completed the require code, now need to test/try it for a bit. --- src/libpython_clj/jna/interpreter.clj | 16 +- src/libpython_clj/metadata.clj | 94 +++- src/libpython_clj/require.clj | 637 ++------------------------ 3 files changed, 117 insertions(+), 630 deletions(-) diff --git a/src/libpython_clj/jna/interpreter.clj b/src/libpython_clj/jna/interpreter.clj index 59dcabe..b2ce582 100644 --- a/src/libpython_clj/jna/interpreter.clj +++ b/src/libpython_clj/jna/interpreter.clj @@ -13,7 +13,8 @@ [tech.jna :as jna] [libpython-clj.jna.protocols.object :as pyobj] [libpython-clj.jna.concrete.unicode :as pyuni] - [camel-snake-kebab.core :refer [->kebab-case]]) + [camel-snake-kebab.core :refer [->kebab-case]] + [clojure.tools.logging :as log]) (:import [com.sun.jna Pointer Native NativeLibrary] [com.sun.jna.ptr PointerByReference] [libpython_clj.jna PyObject])) @@ -215,11 +216,14 @@ (defn get-type-name [type-pyobj] - (-> (pyobj/PyObject_GetAttrString type-pyobj "__name__") - (pyuni/PyUnicode_AsUTF8) - (jna/variable-byte-ptr->string) - ->kebab-case - keyword)) + (if-let [obj-name (pyobj/PyObject_GetAttrString type-pyobj "__name__")] + (-> (pyuni/PyUnicode_AsUTF8 obj-name) + (jna/variable-byte-ptr->string) + ->kebab-case + keyword) + (do + (log/warn "Failed to get typename for object") + :typename-lookup-failure))) (defn lookup-type-symbols diff --git a/src/libpython_clj/metadata.clj b/src/libpython_clj/metadata.clj index 0439c19..b6c3ef7 100644 --- a/src/libpython_clj/metadata.clj +++ b/src/libpython_clj/metadata.clj @@ -13,40 +13,41 @@ (py/initialize!) -(def ^:private builtins (as-jvm (import-module "builtins") {})) -(def ^:private inspect (as-jvm (import-module "inspect") {})) -(def ^:private argspec (get-attr inspect "getfullargspec")) -(def ^:private py-source (get-attr inspect "getsource")) -(def ^:private types (import-module "types")) -(def ^:private fn-type +(def builtins (as-jvm (import-module "builtins") {})) +(def inspect (as-jvm (import-module "inspect") {})) +(def argspec (get-attr inspect "getfullargspec")) +(def py-source (get-attr inspect "getsource")) +(def types (import-module "types")) +(def fn-type (call-attr builtins "tuple" [(get-attr types "FunctionType") (get-attr types "BuiltinFunctionType")])) -(def ^:private method-type +(def method-type (call-attr builtins "tuple" [(get-attr types "MethodType") (get-attr types "BuiltinMethodType")])) -(def ^:private isinstance? (get-attr builtins "isinstance")) -(def ^:private fn? #(isinstance? % fn-type)) -(def ^:private method? #(isinstance? % method-type)) -(def ^:private doc #(try - (get-attr % "__doc__") - (catch Exception e - ""))) -(def ^{:private false :public true} - get-pydoc doc) -(def ^:private vars (get-attr builtins "vars")) -(def ^:private pyclass? (get-attr inspect "isclass")) -(def ^:private pymodule? (get-attr inspect "ismodule")) -(def ^:private importlib_util (import-module "importlib.util")) -(defn ^:private findspec [x] +(def isinstance? (get-attr builtins "isinstance")) +(def fn? #(isinstance? % fn-type)) +(def method? #(isinstance? % method-type)) +(def doc #(try + (get-attr % "__doc__") + (catch Exception e + ""))) +(def get-pydoc doc) +(def vars (get-attr builtins "vars")) +(def pyclass? (get-attr inspect "isclass")) +(def pymodule? (get-attr inspect "ismodule")) +(def importlib (py/import-module "importlib")) +(def importlib_util (import-module "importlib.util")) +(def reload-module (py/get-attr importlib "reload")) +(defn findspec [x] (let [-findspec (-> importlib_util (get-attr "find_spec"))] (-findspec x))) -(defn ^:private py-fn-argspec [f] +(defn py-fn-argspec [f] (if-let [spec (try (when-not (pyclass? f) (argspec f)) (catch Throwable e nil))] @@ -59,7 +60,7 @@ :annotations (->jvm (get-attr spec "annotations") {})} (py-fn-argspec (get-attr f "__init__")))) -(defn ^:private py-class-argspec [class] +(defn py-class-argspec [class] (let [constructor (get-attr class "__init__")] (py-fn-argspec constructor))) @@ -153,6 +154,12 @@ arglists (recur argspec' defaults' arglists)))))) + +(defn py-class-argspec [class] + (let [constructor (py/get-attr class "__init__")] + (py-fn-argspec constructor))) + + (defn py-fn-metadata [fn-name x {:keys [no-arglists?]}] (let [fn-argspec (pyargspec x) fn-docstr (get-pydoc x)] @@ -242,6 +249,38 @@ val))}))) +(defn module-path-string + "Given a.b, return a + Given a.b.c, return a.b + Given a.b.c.d, return a.b.c etc." + [x] + (clojure.string/join + "." + (pop (clojure.string/split (str x) #"[.]")))) + + +(defn module-path-last-string + "Given a.b.c.d, return d" + [x] + (last (clojure.string/split (str x) #"[.]"))) + + +(defn path->py-obj + [item-path & {:keys [reload?]}] + (when (seq item-path) + (if-let [module-retval (try + (import-module item-path) + (catch Throwable e nil))] + (if reload? + (reload-module module-retval) + module-retval) + (let [butlast (module-path-string item-path)] + (if-let [parent-mod (path->py-obj butlast :reload? reload?)] + (get-attr parent-mod (module-path-last-string item-path)) + (throw (Exception. (format "Failed to find module or class %s" + item-path)))))))) + + (defn metadata-map->py-obj [metadata-map] (case (:type metadata-map) @@ -263,13 +302,18 @@ "Given a metadata map, find the item associated with the map and for each string keyword apply it to the namespace. Namespace is created if it does not already exist. Returns the namespace symbol." - [ns-symbol metadata-map] + [ns-symbol metadata-map & {:keys [no-arglists?]}] (let [target-item (metadata-map->py-obj metadata-map)] (get-or-create-namespace! ns-symbol (:doc metadata-map)) (doseq [[k v] metadata-map] (when (has-attr? target-item k) (let [att-val (get-attr target-item k)] - (intern ns-symbol (with-meta (symbol k) v) att-val)))) + (intern ns-symbol + (with-meta (symbol k) + (if no-arglists? + (dissoc v :arglists) + v)) + att-val)))) ns-symbol)) diff --git a/src/libpython_clj/require.clj b/src/libpython_clj/require.clj index 2cde861..ab50ce9 100644 --- a/src/libpython_clj/require.clj +++ b/src/libpython_clj/require.clj @@ -1,242 +1,15 @@ (ns libpython-clj.require (:refer-clojure :exclude [fn? doc]) (:require [libpython-clj.python :as py] + [libpython-clj.metadata :as pymeta] [clojure.datafy :refer [datafy nav]] ;;Binds datafy/nav to python objects [libpython-clj.metadata] [clojure.tools.logging :as log])) -(py/initialize!) - ;; for hot reloading multimethod in development (ns-unmap 'libpython-clj.require 'intern-ns-class) -(def ^:private builtins (py/import-module "builtins")) - -(def ^:private inspect (py/import-module "inspect")) - -(def ^:private argspec (py/get-attr inspect "getfullargspec")) - -(def ^:private py-source (py/get-attr inspect "getsource")) - -(def ^:private types (py/import-module "types")) - -(def ^:private fn-type - (py/call-attr builtins "tuple" - [(py/get-attr types "FunctionType") - (py/get-attr types "BuiltinFunctionType")])) - -(def ^:private method-type - (py/call-attr builtins "tuple" - [(py/get-attr types "MethodType") - (py/get-attr types "BuiltinMethodType")])) - -(def ^:private isinstance? (py/get-attr builtins "isinstance")) - -(def ^:private fn? #(isinstance? % fn-type)) - -(def ^:private method? #(isinstance? % method-type)) - -(def ^:private doc #(try - (py/get-attr % "__doc__") - (catch Exception e - ""))) - -(def ^:private importlib (py/import-module "importlib")) - -(def ^:private importlib_util (py/import-module "importlib.util")) - -(def ^:private reload-module (py/get-attr importlib "reload")) - -(def ^:priviate import-module (py/get-attr importlib "import_module")) - -(def ^{:private false :public true} - get-pydoc doc) - -(def ^:private vars (py/get-attr builtins "vars")) - -(def ^:private pyclass? (py/get-attr inspect "isclass")) - -(def ^:private pymodule? (py/get-attr inspect "ismodule")) - -(defn ^:private findspec [x] - (let [-findspec - (-> importlib_util (py/get-attr "find_spec"))] - (-findspec x))) - -(def ^:private hasattr (py/get-attr builtins "hasattr")) - -(defn ^:private module-string? [x] - (try - (findspec x) - (catch Throwable e - false))) - -(defn module-path-string - "Given a.b, return a - Given a.b.c, return a.b - Given a.b.c.d, return a.b.c etc." - [x] - (clojure.string/join - "." - (pop (clojure.string/split (str x) #"[.]")))) - -(defn module-path-last-string - "Given a.b.c.d, return d" - [x] - (last (clojure.string/split (str x) #"[.]"))) - -(defn ^:private possible-class? - "The presumption here is that, given a symbol like 'builtins.list, - if 'builtins.list does not have a __path__ attribute but 'builtins does, - it's possible that 'list is a class of 'builtins. - - This is a slightly cheaper test than a try/except wrapper around - a module." - [pos-class] - (let [module? (module-path-string pos-class)] - (and (not (module-string? pos-class)) - (boolean (module-string? module?))))) - -(defn ^:private py-fn-argspec [f] - (if-let [spec (try (argspec f) (catch Throwable e nil))] - {:args (py/->jvm (py/get-attr spec "args")) - :varargs (py/->jvm (py/get-attr spec "varargs")) - :varkw (py/->jvm (py/get-attr spec "varkw")) - :defaults (py/->jvm (py/get-attr spec "defaults")) - :kwonlyargs (py/->jvm (py/get-attr spec "kwonlyargs")) - :kwonlydefaults (py/->jvm (py/get-attr spec "kwonlydefaults")) - :annotations (py/->jvm (py/get-attr spec "annotations"))} - (py-fn-argspec (py/get-attr f "__init__")))) - -(defn ^:private py-class-argspec [class] - (let [constructor (py/get-attr class "__init__")] - (py-fn-argspec constructor))) - -(defn pyargspec [x] - ;; TODO: certain builtin functions have - ;; ..: signatures that are found in the first line - ;; ..: of their docstring, aka, print. - ;; ..: These seem to be uniform enough that - ;; ..: most IDEs have a way of creating stubs - ;; ..: for the signature. If there is a uniform way - ;; ..: to do this that doesn't simply involve an - ;; ..: army of devs doing transcription I'd like to - ;; ..: pull that in here - (cond - (fn? x) (py-fn-argspec x) - (method? x) (py-fn-argspec x) - ;; (builtin-function? x) (py-builtin-fn-argspec x) - ;; (builtin-method? x) (py-builtin-method-argspec x) - (string? x) "" - (number? x) "" - :else (py-class-argspec x))) - -(defn pyarglists - ([argspec] (pyarglists argspec - (if-let [defaults - (not-empty (:defaults argspec))] - defaults - []))) - ([argspec defaults] (pyarglists argspec defaults [])) - ([{:as argspec - args :args - varkw :varkw - varargs :varargs - kwonlydefaults :kwonlydefaults - kwonlyargs :kwonlyargs} - defaults res] - (let [n-args (count args) - n-defaults (count defaults) - n-pos-args (- n-args n-defaults) - pos-args (transduce - (comp - (take n-pos-args) - (map symbol)) - (completing conj) - [] - args) - kw-default-args (transduce - (comp - (drop n-pos-args) - (map symbol)) - (completing conj) - [] - args) - or-map (transduce - (comp - (partition-all 2) - (map vec) - (map (fn [[k v]] [(symbol k) v]))) - (completing (partial apply assoc)) - {} - (concat - (interleave kw-default-args defaults) - (flatten (seq kwonlydefaults)))) - - as-varkw (when (not (nil? varkw)) - {:as (symbol varkw)}) - default-map (transduce - (comp - (partition-all 2) - (map vec) - (map (fn [[k v]] [(symbol k) (keyword k)]))) - (completing (partial apply assoc)) - {} - (concat - (interleave kw-default-args defaults) - (flatten (seq kwonlydefaults)))) - - kwargs-map (merge default-map - (when (not-empty or-map) - {:or or-map}) - (when (not-empty as-varkw) - as-varkw)) - opt-args - (cond - (and (empty? kwargs-map) - (nil? varargs)) '() - (empty? kwargs-map) (list '& [(symbol varargs)]) - (nil? varargs) (list '& [kwargs-map]) - :else (list '& [(symbol varargs) - kwargs-map])) - - arglist ((comp vec concat) (list* pos-args) opt-args)] - (let [arglists (conj res arglist) - defaults' (if (not-empty defaults) (pop defaults) []) - argspec' (update argspec :args - (fn [args] - (if (not-empty args) - (pop args) - args)))] - - (if (and (empty? defaults) (empty? defaults')) - arglists - (recur argspec' defaults' arglists)))))) - -(defn py-fn-metadata [fn-name x {:keys [no-arglists?]}] - (let [fn-argspec (pyargspec x) - fn-docstr (get-pydoc x)] - (merge - fn-argspec - {:doc fn-docstr - :name fn-name} - (when (and (py/callable? x) - (not no-arglists?)) - (try - {:arglists (pyarglists fn-argspec)} - (catch Throwable e - nil)))))) - -(defn ^:private load-py-fn [f fn-name fn-module-name-or-ns - options] - (let [fn-ns (symbol (str fn-module-name-or-ns)) - fn-sym (symbol fn-name)] - (intern fn-ns - (with-meta - fn-sym - (py-fn-metadata fn-name f - options)) f))) (defn- parse-flags "FSM style parser for flags. Designed to support both @@ -281,6 +54,7 @@ (recur reqs retval-reqs retval-flags)) retval-flags))) + (defn- extract-refer-symbols [{:keys [refer this-module]} public-data] (cond @@ -299,7 +73,6 @@ (when (contains? public-data item-sym) item-sym)))) (remove nil?)) - ;; [.. :refer [..]] behavior :else (do @@ -311,33 +84,18 @@ (vec missing))))) refer))) -(defn import-module-or-cls! [x] - (if (possible-class? x) - (let [module? (module-path-string x) - class? (module-path-last-string x)] - (try - (let [m (import-module module?) - cls (py/get-attr m class?)] - (if (nil? cls) - (throw (Exception. "Unable to load " x))) - [m cls]) - (catch Exception e - (log/error (str "Unable to load " x)) - (throw e)))) - [(import-module (str x)) nil])) - -(defn- python-lib-configuration - "Build a configuration map of a python library. Current ns is option and used - during testing but unnecessary during normal running events." - [req & [current-ns]] - (let [supported-flags #{:reload - :no-arglists} - [module-name & etc] req - flags (parse-flags supported-flags etc) - etc (->> etc - (remove supported-flags) - (remove boolean?) - (partition-all 2) + +(defn- do-require-python + [reqs-vec] + (let [[module-name & etc] reqs-vec + supported-flags #{:reload :no-arglists} + flags (parse-flags supported-flags etc) + etc (->> etc + (remove supported-flags) + (remove boolean?)) + _ (when-not (= 0 (rem (count etc) 2)) + (throw (Exception. "Must have even number of entries"))) + etc (->> etc (partition-all 2) (map vec) (into {})) reload? (:reload flags) @@ -345,346 +103,25 @@ module-name-or-ns (:as etc module-name) exclude (into #{} (:exclude etc)) refer (cond - (= :all (:refer etc)) #{:all} - (= :* (:refer etc)) #{:*} - :else (into - #{} - (:refer etc))) - current-ns (or current-ns *ns*) - current-ns-sym (symbol (str current-ns)) - python-namespace (find-ns module-name-or-ns) - [this-module cls] (import-module-or-cls! module-name) - load-ns-classes? (pyclass? cls)] - - {:supported-flags supported-flags - :etc etc - :reload? reload? - :no-arglists? no-arglists? - :load-ns-classes? load-ns-classes? - - ;; if something like 'builtins.list was requested, - ;; the module name is 'builtins - :module-name (if (pyclass? cls) - (module-path-string module-name) - module-name) - - ;; if something like 'builtins.list was requested, - ;; the module-name is 'builtins - ;; or in the situation of - ;; '[builtins.list :as python.pylist] - ;; 'python would be bound to :module-name-or-ns - :module-name-or-ns (if (and (pyclass? cls) - (or (symbol? module-name-or-ns) - (string? module-name-or-ns))) - (module-path-string module-name-or-ns) - module-name-or-ns) - - ;; presumes possibilty of alias in situations like - ;; '[builtins.list :as python.pylist] - ;; 'pylist would be bound to :class-name-or-ns in this case - - :class-name-or-ns (if (and (pyclass? cls) - (or (symbol? module-name-or-ns) - (string? module-name-or-ns))) - (module-path-last-string module-name-or-ns) - nil) - - :exclude exclude - :refer refer - :current-ns current-ns - :current-ns-sym current-ns-sym - :python-namespace python-namespace - :this-module this-module - :module? (pymodule? this-module) - :class? (pyclass? cls) - :class-name (when (pyclass? cls) - (module-path-last-string - module-name))})) - -(defn- extract-public-data - [{:keys [exclude python-namespace module-name-or-ns]}] - (let [python-namespace - (or python-namespace - (find-ns module-name-or-ns))] - (->> (ns-publics python-namespace) - (remove #(exclude (first %))) - (into {})))) - -(defn- reload-python-ns! - [module-name this-module module-name-or-ns] - (do - (remove-ns module-name) - (reload-module this-module) - (create-ns module-name-or-ns))) - -(defn- create-python-ns! - [module-name-or-ns] - (create-ns module-name-or-ns)) - -(defn ^:private maybe-reload-or-create-ns! - [{:keys [reload? - this-module - module-name - module-name-or-ns] - cls? :class? - python-namespace :python-namespace}] - - ;; in the event of '[builtins.list :as python.pylist] - ;; this function is a no-op - (cond - cls? nil - reload? (reload-python-ns! module-name - this-module - module-name-or-ns) - (not python-namespace) (create-python-ns! module-name-or-ns))) - -(defn enhanced-python-lib-configuration - [{:keys [python-namespace exclude this-module] - cls? :class? - :as lib-config}] - ;; in the even of '[builtins.list :as python.pylist] - ;; or something similar, we do not provide enhanced - ;; data - (if cls? - lib-config - (let [public-data (extract-public-data lib-config)] - (merge - lib-config - {:public-data public-data - :refer-symbols (extract-refer-symbols - lib-config - public-data)})))) - -(defn- bind-py-symbols-to-ns! - [{:keys [reload? - python-namespace - this-module - module-name-or-ns - no-arglists?] - cls? :class?}] - (when (and (or reload? (not python-namespace)) (not cls?)) - ;;Mutably define the root namespace. - (doseq [[att-name v] (vars this-module)] - (try - (when v - (if (py/callable? v) - (load-py-fn v (symbol att-name) module-name-or-ns - {:no-arglists? - no-arglists?}) - (intern module-name-or-ns (symbol att-name) v))) - (catch Throwable e - (log/warnf e "Failed to require symbol %s" att-name)))))) - -(defn- bind-module-ns! - [{:keys [current-ns-sym - module-name-or-ns - class-name-or-ns - this-module] - cls? :class?}] - - ;; in event of '[builtins.list :as python.pylist] - ;; this is a no-op - (when (not cls?) - (intern current-ns-sym - (with-meta module-name-or-ns - {:doc (doc this-module)}) - this-module))) - -(defn- generate-class-namespace-configs - [{:keys [module-name-or-ns - this-module - class-name - class-name-or-ns - ] - cls? :class? - :as lib-config}] - (letfn [(pyclass-pair? [[attr attr-val]] - - (or - - ;; in the event that something like - ;; builtins.list was requested, - ;; we do not want to bind every - ;; class in the module to the ns, only the - ;; requested class - - (and cls? - (pyclass? attr-val) - (and (hasattr attr-val "__name__") - (= class-name - (py/get-attr - attr-val - "__name__")))) - - ;; if a module like 'builtins was specified - ;; without a class like 'builtins.list, - ;; we bind every class in the module to the module-ns - - (and (not cls?) - (pyclass? attr-val)))) - (pyclass-ns-config [[attr attr-val]] - {:namespace module-name-or-ns - :attribute-type :class - :classname attr - :class-symbol (symbol attr) - :class-binding (or class-name-or-ns - class-name) - :class attr-val - :attributes ((comp - (py/get-attr builtins "dict") - vars) attr-val)}) - (pyclass-attribute-fanout - [{:keys [namespace - classname - class-symbol - class - attributes] - :as attr-config}] - (into [attr-config] - (for [[attr attr-val] (seq attributes) - :let - [attr-type - (if (and (not (nil? attr-val)) - (py/callable? attr-val)) - - :method - :attribute)]] - (merge - (dissoc attr-config :attributes :type) - {:attribute-type attr-type - :type :class-attribute - :class-binding (or class-name-or-ns - class-name) - :attribute attr-val - :attribute-name attr}))))] - (into [] - (comp - (filter pyclass-pair?) - (map pyclass-ns-config) - (map pyclass-attribute-fanout) - cat) - (seq (vars this-module))))) - -(defmulti class-sort :attribute-type) -(defmethod class-sort :class [_] 0) -(defmethod class-sort :method [_] 1) -(defmethod class-sort :attribute [_] 2) - -(defmulti intern-ns-class :attribute-type) - -(defmethod intern-ns-class :class - [{original-namespace :namespace - cls-name :classname - cls-sym :class-symbol - cls :class - cls-binding :class-binding - :as cls-ns-config}] - - ;; cls-binding percolates down to here in the situation where - ;; '[builtins.list :as python.pylist] occurs - - (let [cls-ns (symbol (str original-namespace - "." - (or - cls-binding - cls-sym)))] - (create-ns cls-ns) - cls-ns-config)) - -(defmethod intern-ns-class :method - [{original-namespace :namespace - cls-name :classname - cls-sym :class-symbol - cls :class - cls-binding :class-binding - method-name :attribute-name - method :attribute - :as cls-ns-config}] - - ;; cls-binding percolates down to here in the situation where - ;; '[builtins.list :as python.pylist] occurs - - (let [cls-ns (symbol (str original-namespace "." - (or cls-binding cls-sym)))] - (load-py-fn method (symbol method-name) - cls-ns {}) - cls-ns-config)) - -(defmethod intern-ns-class :attribute - [{original-namespace :namespace - cls-name :classname - cls-sym :class-symbol - cls :class - cls-binding :class-binding - attribute-name :attribute-name - attribute :attribute - :as cls-ns-config}] - - ;; cls-binding percolates down to here in the situation where - ;; '[builtins.list :as python.pylist] occurs - - (let [cls-ns (symbol (str original-namespace "." - (or cls-binding cls-sym)))] - (intern cls-ns (symbol attribute-name) attribute) - cls-ns-config)) - -(defn- bind-class-namespaces! - [{cls? :class? - module :module? - :as lib-config}] - (let [class-namespace-configs - (->> - (generate-class-namespace-configs - lib-config) - (sort-by class-sort))] - (doseq [cls-namespace-config class-namespace-configs] - (intern-ns-class cls-namespace-config)))) - -(defn- intern-public-and-refer-symbols! - [{:keys [public-data refer-symbols current-ns-sym]}] - (doseq [[s v] (select-keys public-data refer-symbols)] - (intern current-ns-sym - (with-meta s (meta v)) - (deref v)))) - -(defn preload-python-lib! [req] - (let [lib-config (python-lib-configuration req)] - (maybe-reload-or-create-ns! lib-config) - (bind-py-symbols-to-ns! lib-config) - (bind-module-ns! lib-config) - (enhanced-python-lib-configuration lib-config))) - -(defn ^:private load-python-lib [req] - (let [{:keys [supported-flags - flags - etc - reload? - no-arglists? - load-ns-classes? - module-name - module-name-or-ns - exclude - refer - current-ns - current-ns-sym - python-namespace - this-module - python-namespace - public-data - refer-symbols - class? - module?] - :as lib-config} - (preload-python-lib! req)] - - (when (not class?) - (intern-public-and-refer-symbols! lib-config)) - - ;; alpha - ;; TODO: does not respect reloading or much of the other - ;; ..: syntax (yet) - (when load-ns-classes? - (bind-class-namespaces! lib-config)))) + (= :all (:refer etc)) #{:all} + (= :* (:refer etc)) #{:*} + :else (into #{} (:refer etc))) + pyobj (pymeta/path->py-obj (str module-name) :reload? reload?) + existing-py-ns? (find-ns module-name)] + (when (and reload? existing-py-ns?) + (remove-ns module-name)) + (create-ns module-name) + (when (or (not existing-py-ns?) + reload?) + (pymeta/apply-static-metadata-to-namespace! module-name (datafy pyobj) + :no-arglists? no-arglists?)) + (refer module-name + :exclude exclude + :only (extract-refer-symbols {:refer refer + :this-module pyobj} + (ns-publics (find-ns module-name)))) + (alias module-name-or-ns module-name))) + (defn require-python "## Basic usage ## @@ -758,8 +195,10 @@ (cond (list? reqs) - (doseq [req (vec reqs)] (require-python req)) + (doseq [req reqs] (require-python req)) (symbol? reqs) - (load-python-lib (vector reqs)) + (require-python (vector reqs)) (vector? reqs) - (load-python-lib reqs))) + (do-require-python reqs) + :else + (throw (Exception. "Invalid argument: %s" reqs)))) From be6ecdd82a6126a918db38421fd01cfcde93031a Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 12 Jan 2020 13:11:03 -0700 Subject: [PATCH 135/456] Reworked require-python to work like clojure's require --- src/libpython_clj/metadata.clj | 4 +- src/libpython_clj/require.clj | 29 +++++---- test/libpython_clj/require_python_test.clj | 69 +--------------------- 3 files changed, 18 insertions(+), 84 deletions(-) diff --git a/src/libpython_clj/metadata.clj b/src/libpython_clj/metadata.clj index b6c3ef7..b38669f 100644 --- a/src/libpython_clj/metadata.clj +++ b/src/libpython_clj/metadata.clj @@ -306,7 +306,9 @@ (let [target-item (metadata-map->py-obj metadata-map)] (get-or-create-namespace! ns-symbol (:doc metadata-map)) (doseq [[k v] metadata-map] - (when (has-attr? target-item k) + (when (and (string? k) + (map? v) + (has-attr? target-item k)) (let [att-val (get-attr target-item k)] (intern ns-symbol (with-meta (symbol k) diff --git a/src/libpython_clj/require.clj b/src/libpython_clj/require.clj index ab50ce9..25e3ba5 100644 --- a/src/libpython_clj/require.clj +++ b/src/libpython_clj/require.clj @@ -100,27 +100,26 @@ (into {})) reload? (:reload flags) no-arglists? (:no-arglists flags) - module-name-or-ns (:as etc module-name) + alias-name (:as etc) exclude (into #{} (:exclude etc)) - refer (cond - (= :all (:refer etc)) #{:all} - (= :* (:refer etc)) #{:*} - :else (into #{} (:refer etc))) + refer-data (cond + (= :all (:refer etc)) #{:all} + (= :* (:refer etc)) #{:*} + :else (into #{} (:refer etc))) pyobj (pymeta/path->py-obj (str module-name) :reload? reload?) existing-py-ns? (find-ns module-name)] - (when (and reload? existing-py-ns?) - (remove-ns module-name)) (create-ns module-name) - (when (or (not existing-py-ns?) - reload?) + (when (or (not existing-py-ns?) reload?) (pymeta/apply-static-metadata-to-namespace! module-name (datafy pyobj) :no-arglists? no-arglists?)) - (refer module-name - :exclude exclude - :only (extract-refer-symbols {:refer refer - :this-module pyobj} - (ns-publics (find-ns module-name)))) - (alias module-name-or-ns module-name))) + (when-let [refer-symbols (->> (extract-refer-symbols {:refer refer-data + :this-module pyobj} + (ns-publics + (find-ns module-name))) + seq)] + (refer module-name :exclude exclude :only refer-symbols)) + (when alias-name + (alias alias-name module-name)))) (defn require-python diff --git a/test/libpython_clj/require_python_test.clj b/test/libpython_clj/require_python_test.clj index c1a8d27..453b579 100644 --- a/test/libpython_clj/require_python_test.clj +++ b/test/libpython_clj/require_python_test.clj @@ -10,17 +10,13 @@ (require-python '[math :refer :* :exclude [sin cos] - :as pymath - :reload true]) + :as pymath]) (deftest base-require-test (let [publics (ns-publics (find-ns 'libpython-clj.require-python-test))] - (is (contains? publics 'acos)) - (is (contains? publics 'floor)) (is (not (contains? publics 'sin))) (is (= 10.0 (double (floor 10.1)))) - (is (= pymath (py/import-module "math"))) (is (thrown? Throwable (require-python '[math :refer [blah]]))))) (deftest parse-flags-test @@ -50,69 +46,6 @@ (is (= #{:b} (parse-flags #{:a :b} '[:b :a false]))) (is (= #{:b :a} (parse-flags #{:a :b} '[:b :a false :a true]))))) -(deftest python-lib-configuration-test - (let [python-lib-configuration #'req/python-lib-configuration - simple-req '[csv] - simple-spec (python-lib-configuration - simple-req - (find-ns 'libpython-clj.require-python-test)) - {:keys [exclude - supported-flags - current-ns-sym - module-name - module-name-or-ns - reload? - no-arglists? - etc - current-ns - this-module - python-namespace - refer]} simple-spec - csv-module (py/import-module "csv")] - - ;; no exclusions - (is (= #{} exclude)) - (is (= #{:no-arglists :reload} - supported-flags)) - (is (= 'csv module-name module-name-or-ns)) - (is (nil? reload?)) - (is (nil? no-arglists?)) - (is (= {} etc)) - (is (= csv-module this-module))) - - - (let [python-lib-configuration #'req/python-lib-configuration - simple-req '[requests - :reload true - :refer [get] - :no-arglists] - simple-spec (python-lib-configuration - simple-req - (find-ns 'libpython-clj.require-python-test)) - {:keys [exclude - supported-flags - current-ns-sym - module-name - module-name-or-ns - reload? - no-arglists? - etc - current-ns - this-module - python-namespace - refer]} simple-spec - requests-module (py/import-module "requests")] - - ;; no exclusions - (is (= #{} exclude)) - (is (= 'requests module-name module-name-or-ns)) - (is (= reload? :reload)) - (is (= no-arglists? :no-arglists)) - (is (= {:refer '[get]} etc)) - (is (= requests-module this-module)) - (is (= #{'get} refer)) - (is (nil? python-namespace)))) - (require-python '([builtins :as python] [builtins.list :as python.pylist])) From 65296443e68ec273ea45d1a607904ab40809c4a0 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 12 Jan 2020 13:16:36 -0700 Subject: [PATCH 136/456] editing --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 00ef5a6..3f319aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Time for a ChangeLog! +## 1.31-SNAPSHOT + +* Python objects are now datafy-able and nav-igable. `require-python` + is now rebuilt using datafy. + + ## 1.30 This release is a big one. With finalizing `require-python` we have a clear way From 1a97bc646b411cf8d81e5c7b249f279f11d1d4e9 Mon Sep 17 00:00:00 2001 From: "J.J. Tolton" Date: Fri, 17 Jan 2020 09:51:21 -0500 Subject: [PATCH 137/456] ISSUE-44 (syntax sugar) (#45) * Add syntax sugar for issue 44 https://github.com/cnuernber/libpython-clj/issues/44 * Fix failing tests * more convincing tests * whitespace * update Usage and changelog * fix typo --- CHANGELOG.md | 4 ++ docs/Usage.md | 74 +++++++++++++++++++++++- src/libpython_clj/python.clj | 42 ++++++++++++++ test/libpython_clj/python_test.clj | 93 ++++++++++++++++++++---------- 4 files changed, 179 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f319aa..b6dc3b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ * Python objects are now datafy-able and nav-igable. `require-python` is now rebuilt using datafy. +* `py.`, `py.-`, and `py..` added to the `libpython-clj` APIs + to allow method/attribute access more consistent with idiomatic + Clojure forms. + ## 1.30 diff --git a/docs/Usage.md b/docs/Usage.md index 18be975..ce30b01 100644 --- a/docs/Usage.md +++ b/docs/Usage.md @@ -118,7 +118,7 @@ your variable is:200 {'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': , '__spec__': None, '__annotations__': {}, '__builtins__': , 'my_var': 200}} ``` -Running python isn't ever really necessary, however, although it may at times be +Running Python isn't ever really necessary, however, although it may at times be convenient. You can call attributes from clojure easily: ```clojure @@ -263,6 +263,76 @@ user> (py/$.. numpy random shuffle) ``` +##### New sugar (fixme) + +`libpython-clj` offers syntactic forms similar to those offered by +Clojure for interacting with Python classes and objects. + +**Class/object methods** +Where in Clojure you would use `(. obj method arg1 arg2 ... argN)`, +you can use `(py. pyobj method arg1 arg2 ... argN)`. + +In Python, this is equivalent to `pyobj.method(arg1, arg2, ..., argN)`. +Concrete examples are shown below. + +**Class/object attributes** +Where in Clojure you would use `(.- obj attr)`, you can use +`(py.- pyobj attr)`. + +In Python, this is equivalent to `pyobj.attr`. +Concrete examples shown below. + +**Nested attribute access** +To achieve a chain of method/attribute access, use the `py..` for. + +```clojure +(py.. (requests/get "http://www.google.com") + -content + (decode "latin-1")) +``` +(**Note**: requires Python `requests` module installled) + +**Examples** + +```clojure +user=> (require '[libpython-clj.python :as py :refer [py. py.. py.-]]) +nil +user=> (require '[libpython-clj.require :refer [require-python]]) + +... debug info ... + +user=> (require-python '[builtins :as python]) +WARNING: AssertionError already refers to: class java.lang.AssertionError in namespace: builtins, being replaced by: #'builtins/AssertionError +WARNING: Exception already refers to: class java.lang.Exception in namespace: builtins, being replaced by: #'builtins/Exception +nil +user=> (def xs (python/list)) +#'user/xs +user=> (py. xs append 1) +nil +user=> xs +[1] +user=> (py. xs extend [1 2 3]) +nil +user=> xs +[1, 1, 2, 3] +user=> (py. xs __len__) +4 +user=> ((py.- xs __len__)) ;; attribute syntax to get then call method +4 +user=> (py. xs pop) +3 +user=> (py. xs clear) +nil +;; requires Python requests module installed +user=> (require-python 'requests) +nil +user=> (def requests (py/import-module "requests")) +#'user/requests +user=> (py.. requests (get "http://www.google.com") -content (decode "latin-1")) +"python {:a 1 :b 2})] (is (= :dict (py/python-type py-dict))) @@ -41,7 +39,6 @@ (is (= {"a" 1 "b" 2} (into {} bridge-dict)))))) - (deftest lists (let [py-list (py/->py-list [4 3 2 1])] (is (= :list (py/python-type py-list))) @@ -58,11 +55,10 @@ (.sort ^List bridge-list nil) (is (= [1 2 3 4] (into [] bridge-list)))))) - (deftest global-dict (let [main-module (py/add-module "__main__") ^Map globals (-> (py/module-dict main-module) - (py/as-jvm))] + (py/as-jvm))] (is (instance? Map globals)) (.put globals "item" 100) (py/set-item! globals "item2" 200) @@ -71,7 +67,6 @@ (py/run-simple-string "item3 = item + item2") (is (= 300 (globals "item3"))))) - (deftest numpy-and-back (let [jvm-tens (dtt/->tensor (->> (range 9) (partition 3)))] @@ -85,11 +80,11 @@ ;;This operation is in-place (let [py-trans (py/call-attr py-tens "transpose" [1 0])] (is (= [[0.0 1.0 2.0] [3.0 4.0 5.0] [6.0 7.0 8.0]] - (-> (py/as-tensor py-tens) - dtt/->jvm))) + (-> (py/as-tensor py-tens) + dtt/->jvm))) (is (= [[0.0 3.0 6.0] [1.0 4.0 7.0] [2.0 5.0 8.0]] - (-> (py/as-tensor py-trans) - dtt/->jvm))) + (-> (py/as-tensor py-trans) + dtt/->jvm))) ;;But they are sharing backing store, so mutation will travel both ;;ways. Creepy action at a distance indeed (dtype/copy! [5 6 7] (py/as-tensor py-trans)) @@ -120,7 +115,6 @@ double)) (str "Item type " constructor))))) - (deftest dict-with-complex-key (let [py-dict (py/->python {["a" "b"] 1 ["c" "d"] 2}) @@ -132,12 +126,10 @@ (map vec) set))))) - (deftest simple-print-crashed (let [numpy (py/import-module "numpy")] (println (py/as-tensor (py/call-attr numpy "ones" [3 3]))))) - (deftest true-false-list (is (= [false true] (-> '(false true) @@ -151,7 +143,6 @@ a) vec))))) - (deftest aspy-iter (let [testcode-module (py/import-module "testcode")] (is (= [1 2 3 4 5] @@ -163,7 +154,6 @@ "for_iter" (py/as-python {"a" 1 "b" 2 "c" 3})) (py/->jvm)))))) - (deftest basic-with-test (let [testcode-module (py/import-module "testcode")] (let [fn-list (py/->py-list [])] @@ -192,18 +182,15 @@ (is (= ["enter" "exit: None"] (py/->jvm fn-list)))))) - (deftest arrow-as-fns-with-nil (is (= nil (py/->jvm nil))) (is (= nil (py/as-jvm nil)))) - (deftest pydict-nil-get (let [dict (py/->python {:a 1 :b {:a 1 :b 2}}) bridged (py/as-jvm dict)] (is (= nil (bridged nil))))) - (deftest custom-clojure-item (let [att-map {"clojure_fn" (py/->python #(vector 1 2 3))} my-python-item (py/create-bridge-from-att-map @@ -213,8 +200,7 @@ att-map ;;second is the list of attributes. In this case, since this ;;object isn't iterable or anything, this function will do. - att-map - ) + att-map) py-mod (py/import-module "testcode")] (is (= [1 2 3] (py/call-attr py-mod "calling_custom_clojure_fn" my-python-item))) @@ -234,21 +220,18 @@ my-obj ;;second is the list of attributes. In this case, since this ;;object isn't iterable or anything, this function will do. - att-map - )] + att-map)] (is (= [4 5 6] (vec my-obj))) (is (= [4 5 6] (vec (py/call-attr py-mod "for_iter" my-python-item))))))) - (deftest bridged-dict-to-jvm (let [py-dict (py/->py-dict {:a 1 :b 2}) bridged (py/as-jvm py-dict) copied-back (py/->jvm bridged)] (is (instance? clojure.lang.PersistentArrayMap copied-back)))) - (deftest calling-conventions (let [np (py/import-module "numpy") linspace (py/$. np linspace)] @@ -267,8 +250,60 @@ (str (py/get-attr np "linspace")))) (is (= (str (py/$.. np random shuffle)) (str (-> (py/get-attr np "random") - (py/get-attr "shuffle"))))))) + (py/get-attr "shuffle")))))) + + (let [builtins (py/import-module "builtins") + l (py/call-attr builtins "list")] + (is (= (py/py. l __len__) 0)) + (py/py. l append 1) + (is (= (py/py. l __len__) 1)) + (py/py.. l (extend [1 2 3])) + (is (= ((py/py.- l __len__)) 4))) + + (let [sys (py/import-module "sys")] + (is (int? (py/py.. sys -path __len__)))) + + + (let [{{Foo :Foo} :globals} + (py/run-simple-string " +class Foo: + + def __init__(self, a, b): + self.a = a + self.b = b + self._res = [] + + def res(self): + return self._res + + def extend(self, arg): + self._res.extend(arg) + return self + + def count(self): + return len(self._res) + + def append(self, *args): + return self.extend(args) + + + def math(self, c): + return self.append(self.a + self.b + c) + +") + f (Foo 1 2)] + (is (= [] ((py/py.- f res)) (py/py. f res))) + (is (= 6 (py/py.. f (append 1) + (append 2) + (math 3) + (extend [4 5 6]) count))) + (is (= [1 2 6 4 5 6] ((py/py.- f res)) (py/py. f res))) + (is (= 6 + (py/py.. f res __len__) + ((py/py.. f res -__len__)) + (py/py.. f count) + ((py/py.. f -count)))))) (deftest infinite-seq (let [islice (-> (py/import-module "itertools") @@ -280,7 +315,6 @@ (islice 0 10) (vec)))))) - (deftest persistent-vector-nparray (testing "Create numpy array from nested persistent vectors" (let [ary-data (-> (py/import-module "numpy") @@ -290,7 +324,6 @@ [4 5 6]]) (py/as-tensor ary-data)))))) - (deftest python-tuple-equals (testing "Python tuples have nice equal semantics." (let [lhs (py/->py-tuple [1 2]) @@ -303,14 +336,12 @@ (is (not= (.hashCode lhs) (.hashCode not-same)))))) - (deftest range-nparray (let [ary-data (-> (py/import-module "numpy") (py/$a array (range 10)))] (is (dfn/equals (dtt/->tensor (range 10)) (py/as-tensor ary-data))))) - (deftest false-is-always-py-false (let [py-false (libpy/Py_False) ->false (py/->python false) @@ -320,7 +351,6 @@ (is (= (Pointer/nativeValue (jna/as-ptr py-false)) (Pointer/nativeValue (jna/as-ptr as-false)))))) - (deftest instance-abc-classes (let [py-dict (py/->python {"a" 1 "b" 2}) bridged-dict (py/as-python {"a" 1 "b" 2}) @@ -335,7 +365,6 @@ (is (py/is-instance? bridged-iter iter-type)) (is (py/is-instance? bridged-list sequence-type)))) - (deftest nested-map-and-back (let [py-dict (-> (py/run-simple-string "testdata={'camera_id': 'CODOT-10106-12067', 'country': 'United States', 'state': 'Colorado', 'city': 'Fountain', 'provider': 'CO DOT', 'description': '0.6 mi N of Ray Nixon Rd Int', 'direction': 'North', 'video': False, 'links': {'jpeg': {'url': 'https://www.cotrip.org/dimages/camera?imageURL=remote/CTMCCAM025S125-20-N.jpg'}}, 'tags': ['auto_rerated'], 'ratings': {'road_weather': 4, 'visibility': 4}, 'health': {}, 'created_at': '2016-04-19T20:27:45.030Z', 'time_zone_offset': -25200}") (get-in [:globals "testdata"])) @@ -343,7 +372,7 @@ (is (= (get jvm-dict "ratings") (py/->jvm (py/get-item py-dict "ratings")))))) - (deftest characters (is (= (py/->jvm (py/->python "c")) (py/->jvm (py/->python \c))))) + From 933b38c0fc1e2ede97d5e5f9cddbbd9029443dad Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 19 Jan 2020 12:34:57 -0700 Subject: [PATCH 138/456] 1.31 --- CHANGELOG.md | 2 +- project.clj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6dc3b9..37c82e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Time for a ChangeLog! -## 1.31-SNAPSHOT +## 1.31 * Python objects are now datafy-able and nav-igable. `require-python` is now rebuilt using datafy. diff --git a/project.clj b/project.clj index 4219e1c..18c48ef 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.31-SNAPSHOT" +(defproject cnuernber/libpython-clj "1.31" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" From dd9c27a78913d35d35f66dffaa0c88ccd8bd26d8 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 19 Jan 2020 12:35:17 -0700 Subject: [PATCH 139/456] snap --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 18c48ef..f4c6014 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.31" +(defproject cnuernber/libpython-clj "1.32-SNAPSHOT" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" From 216adaa6c2022f6dcff683fa9c4003331e557acb Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Mon, 20 Jan 2020 15:55:18 -0700 Subject: [PATCH 140/456] some simplification/speed improvements (#51) --- src/libpython_clj/jna/base.clj | 21 ++- src/libpython_clj/python.clj | 5 +- src/libpython_clj/python/bridge.clj | 49 +++---- src/libpython_clj/python/interop.clj | 4 +- src/libpython_clj/python/interpreter.clj | 179 +++++++++-------------- src/libpython_clj/python/object.clj | 7 +- 6 files changed, 120 insertions(+), 145 deletions(-) diff --git a/src/libpython_clj/jna/base.clj b/src/libpython_clj/jna/base.clj index 6bd6c14..ad77dea 100644 --- a/src/libpython_clj/jna/base.clj +++ b/src/libpython_clj/jna/base.clj @@ -3,9 +3,10 @@ [tech.jna.base :as jna-base] [camel-snake-kebab.core :refer [->kebab-case]]) (:import [com.sun.jna Pointer NativeLibrary] - [libpython_clj.jna PyObject])) - + [libpython_clj.jna PyObject] + [java.util.concurrent.atomic AtomicLong])) +(set! *warn-on-reflection* true) (def ^:dynamic *python-library* "python3.6m") @@ -64,7 +65,19 @@ (ensure-pyobj item)) -(def ^:dynamic *gil-captured* false) +(definline current-thread-id + ^long [] + (-> (Thread/currentThread) + (.getId))) + +(def gil-thread-id (AtomicLong. Long/MAX_VALUE)) + + +(defn set-gil-thread-id! + ^long [^long expected ^long new-value] + (when-not (.compareAndSet ^AtomicLong gil-thread-id expected new-value) + (throw (Exception. "Failed to set gil thread id"))) + new-value) (defmacro def-no-gil-pylib-fn @@ -78,7 +91,7 @@ `(defn ~fn-name ~docstring ~(mapv first argpairs) - (when-not *gil-captured* + (when-not (== (current-thread-id) (.get ^AtomicLong gil-thread-id)) (throw (Exception. "Failure to capture gil when calling into libpython"))) (let [~'tvm-fn (jna/find-function ~(str fn-name) *python-library*) ~'fn-args (object-array diff --git a/src/libpython_clj/python.clj b/src/libpython_clj/python.clj index afefb0b..922e790 100644 --- a/src/libpython_clj/python.clj +++ b/src/libpython_clj/python.clj @@ -1,8 +1,7 @@ (ns libpython-clj.python (:require [tech.parallel.utils :refer [export-symbols]] [libpython-clj.python.interop :as pyinterop] - [libpython-clj.python.interpreter :as pyinterp - :refer [with-interpreter]] + [libpython-clj.python.interpreter :as pyinterp] [libpython-clj.python.object :as pyobj] [libpython-clj.python.bridge :as pybridge] [libpython-clj.jna :as libpy] @@ -236,7 +235,7 @@ library-path python-home no-io-redirect?]}] - (when-not @pyinterp/*main-interpreter* + (when-not @pyinterp/main-interpreter* (pyinterp/initialize! :program-name program-name :library-path library-path :python-home python-home) diff --git a/src/libpython_clj/python/bridge.clj b/src/libpython_clj/python/bridge.clj index d8ec7d0..c52183d 100644 --- a/src/libpython_clj/python/bridge.clj +++ b/src/libpython_clj/python/bridge.clj @@ -13,7 +13,6 @@ [libpython-clj.python.interpreter :refer [with-gil - with-interpreter ensure-bound-interpreter check-error-throw initialize!] @@ -238,7 +237,7 @@ (->py-object-ptr [item#] (libpy/as-pyobj pyobj#)) py-proto/PPythonType (get-python-type [item] - (with-interpreter interpreter# + (with-gil (py-proto/get-python-type pyobj#))) py-proto/PCopyToPython (->python [item# options#] pyobj#) @@ -248,43 +247,43 @@ (as-jvm [item# options#] item#) py-proto/PCopyToJVM (->jvm [item# options#] - (with-interpreter interpreter# + (with-gil (->jvm pyobj# options#))) py-proto/PPyObject (dir [item#] - (with-interpreter interpreter# + (with-gil (py-proto/dir pyobj#))) (has-attr? [item# item-name#] - (with-interpreter interpreter# + (with-gil (py-proto/has-attr? pyobj# item-name#))) (get-attr [item# item-name#] - (with-interpreter interpreter# + (with-gil (-> (py-proto/get-attr pyobj# item-name#) as-jvm))) (set-attr! [item# item-name# item-value#] - (with-interpreter interpreter# + (with-gil (py-proto/set-attr! pyobj# item-name# (as-python item-value#)))) (callable? [item#] - (with-interpreter interpreter# + (with-gil (py-proto/callable? pyobj#))) (has-item? [item# item-name#] - (with-interpreter interpreter# + (with-gil (py-proto/has-item? pyobj# item-name#))) (get-item [item# item-name#] - (with-interpreter interpreter# + (with-gil (-> (py-proto/get-item pyobj# item-name#) as-jvm))) (set-item! [item# item-name# item-value#] - (with-interpreter interpreter# + (with-gil (py-proto/set-item! pyobj# item-name# (as-python item-value#)))) py-proto/PPyAttMap (att-type-map [item#] - (with-interpreter interpreter# + (with-gil (py-proto/att-type-map pyobj#))) py-proto/PyCall (do-call-fn [callable# arglist# kw-arg-map#] - (with-interpreter interpreter# + (with-gil (let [arglist# (mapv mostly-copy-arg arglist#) kw-arg-map# (->> kw-arg-map# (map (fn [[k# v#]] @@ -294,7 +293,7 @@ (as-jvm))))) Object (toString [this#] - (with-interpreter interpreter# + (with-gil (if (= 1 (libpy/PyObject_IsInstance pyobj# (libpy/PyType_Type))) (format "%s.%s" (->jvm (py-proto/get-attr pyobj# "__module__")) @@ -363,7 +362,7 @@ (map (juxt identity (partial py-proto/get-attr pyobj))) (into {})) py-call (fn [fn-name & args] - (with-interpreter interpreter + (with-gil (py-impl-call-as fn-name dict-att-map args)))] (bridge-pyobject @@ -405,7 +404,7 @@ (->> (raw-python-iterator dict-att-map) iterator-seq (map (fn [pyobj-key] - (with-interpreter interpreter + (with-gil (let [k (as-jvm pyobj-key) v (.get this pyobj-key) tuple [k v]] @@ -435,7 +434,7 @@ (map (juxt identity (partial py-proto/get-attr pyobj))) (into {})) py-call (fn [fn-name & args] - (with-interpreter interpreter + (with-gil (py-impl-call-as fn-name dict-att-map args)))] (bridge-pyobject pyobj @@ -550,7 +549,7 @@ (defmethod py-proto/python-obj-iterator :default [pyobj interpreter] - (with-interpreter interpreter + (with-gil (let [iter-fn (py-proto/get-attr pyobj "__iter__")] (python->jvm-iterator iter-fn as-jvm)))) @@ -647,32 +646,32 @@ IFn ;;uggh (invoke [this] - (with-interpreter interpreter + (with-gil (cfn this))) (invoke [this arg0] - (with-interpreter interpreter + (with-gil (cfn this arg0))) (invoke [this arg0 arg1] - (with-interpreter interpreter + (with-gil (cfn this arg0 arg1))) (invoke [this arg0 arg1 arg2] - (with-interpreter interpreter + (with-gil (cfn this arg0 arg1 arg2))) (invoke [this arg0 arg1 arg2 arg3] - (with-interpreter interpreter + (with-gil (cfn this arg0 arg1 arg2 arg3))) (invoke [this arg0 arg1 arg2 arg3 arg4] - (with-interpreter interpreter + (with-gil (cfn this arg0 arg1 arg2 arg3 arg4))) (applyTo [this arglist] - (with-interpreter interpreter + (with-gil (apply cfn this arglist))) ;;Mark this as executable Fn diff --git a/src/libpython_clj/python/interop.clj b/src/libpython_clj/python/interop.clj index 55713a3..e1a9efc 100644 --- a/src/libpython_clj/python/interop.clj +++ b/src/libpython_clj/python/interop.clj @@ -10,7 +10,7 @@ [libpython-clj.python.interpreter :refer [get-object-handle - with-gil with-interpreter + with-gil ensure-bound-interpreter ensure-interpreter find-jvm-bridge-entry @@ -357,7 +357,7 @@ (defn expose-bridge-to-python! "Create a python object for this bridge." [^JVMBridge bridge & [libpython-module]] - (with-interpreter (.interpreter bridge) + (with-gil (let [libpython-module (or libpython-module (add-module libpython-clj-module-name)) ^Pointer bridge-type-ptr (get-attr libpython-module diff --git a/src/libpython_clj/python/interpreter.clj b/src/libpython_clj/python/interpreter.clj index e59cf56..dbea6dd 100644 --- a/src/libpython_clj/python/interpreter.clj +++ b/src/libpython_clj/python/interpreter.clj @@ -1,5 +1,5 @@ (ns libpython-clj.python.interpreter - (:require [libpython-clj.jna :as libpy] + (:require [libpython-clj.jna :as libpy ] [libpython-clj.jna.base :as libpy-base] [tech.resource :as resource] [libpython-clj.python.logging @@ -13,6 +13,7 @@ (:import [libpython_clj.jna JVMBridge PyObject] + [java.util.concurrent.atomic AtomicLong] [com.sun.jna Pointer] [com.sun.jna.ptr PointerByReference] [java.io StringWriter] @@ -181,39 +182,19 @@ print(json.dumps( ]) -;;Map of interpreter handle to interpreter -(defonce ^:dynamic *interpreters* (atom {})) - - ;; Main interpreter booted up during initialize! -(defonce ^:dynamic *main-interpreter* (atom nil)) - - - -(defn add-interpreter-handle! - [interpreter] - (swap! *interpreters* assoc - (get-object-handle interpreter) - interpreter)) - +;; * in the right to indicate atom +(def main-interpreter* (atom nil)) +(defn main-interpreter + ^Interpreter [] + @main-interpreter*) -(defn remove-interpreter-handle! - [interpreter] - (swap! *interpreters* dissoc - (get-object-handle interpreter))) -(defn handle->interpreter - [interpreter-handle] - (if-let [retval (get @*interpreters* interpreter-handle)] - retval - (throw (ex-info "Failed to convert from handle to interpreter" - {})))) - (defn handle-or-interpreter->interpreter [hdl-or-interp] (if (number? hdl-or-interp) - (handle->interpreter hdl-or-interp) + (throw (Exception. "Interpreters are no long handles")) hdl-or-interp)) @@ -229,7 +210,7 @@ print(json.dumps( (defn get-jvm-bridge ^JVMBridge [handle interpreter] - (if-let [bridge-obj (find-jvm-bridge-entry handle interpreter)] + (if-let [bridge-obj (find-jvm-bridge-entry handle (main-interpreter))] (:jvm-bridge bridge-obj) (throw (Exception. (format "Unable to find bridge for interpreter %s and handle %s" @@ -262,18 +243,22 @@ print(json.dumps( (defn- construct-main-interpreter! [thread-state type-symbol-table] - (when @*main-interpreter* - (throw (ex-info "Main interpreter is already constructed" {}))) - (let [retval (->Interpreter (atom {:thread-state thread-state - :bridge-objects {} - :sub-interpreters []}) - ;;This that have to live as long as the main - ;;interpreter does - (atom {:type-symbol-table type-symbol-table - :forever []}))] - (reset! *main-interpreter* retval) - (add-interpreter-handle! retval) - :ok)) + (swap! + main-interpreter* + (fn [existing-interpreter] + (when existing-interpreter + (throw (Exception. "Main interpreter is already constructed"))) + + (let [retval (->Interpreter + (atom {:thread-state thread-state + :bridge-objects {} + :sub-interpreters []}) + ;;This that have to live as long as the main + ;;interpreter does + (atom {:type-symbol-table type-symbol-table + :forever []}))] + retval))) + :ok) (defn- python-thread-state @@ -285,13 +270,15 @@ print(json.dumps( "non-reentrant pathway to release the gil. It must not be held by this thread." [interpreter] (let [thread-state (libpy/PyEval_SaveThread)] + (libpy-base/set-gil-thread-id! (libpy-base/current-thread-id) Long/MAX_VALUE) (assoc @(:interpreter-state* interpreter) :thread-state thread-state))) (defn acquire-gil! "Non-reentrant pathway to acquire gil. It must not be held by this thread." [interpreter] - (libpy/PyEval_RestoreThread (python-thread-state interpreter))) + (libpy/PyEval_RestoreThread (python-thread-state interpreter)) + (libpy-base/set-gil-thread-id! Long/MAX_VALUE (libpy-base/current-thread-id))) (defn swap-interpreters! @@ -302,26 +289,27 @@ print(json.dumps( (python-thread-state new-interp))) +(defn main-interpreter-thread-id + ^long [] + (.get ^AtomicLong libpy-base/gil-thread-id)) -;;Interpreter for current thread that holds the gil -(defonce ^:dynamic *current-thread-interpreter* nil) - (defn ensure-interpreter - [] - (if-let [retval (or @*main-interpreter* - *current-thread-interpreter*)] - retval - (throw (ex-info "No interpreters found, perhaps an initialize! call is missing?" - {})))) + ^Interpreter [] + (let [retval (main-interpreter)] + (when-not retval + (throw (Exception. "No interpreters found, perhaps an initialize! call is missing?"))) + retval)) (defn ensure-bound-interpreter [] - (when-not *current-thread-interpreter* - (throw (ex-info "No interpreter bound to current thread" {}))) - *current-thread-interpreter*) + (let [interp (main-interpreter)] + (if (and interp + (= (libpy-base/current-thread-id) (main-interpreter-thread-id))) + interp + (throw (Exception. "No interpreters found, perhaps an initialize! call is missing?"))))) (defn py-type-keyword @@ -343,50 +331,25 @@ print(json.dumps( (get-in symbol-table [type-addr :typename]))) -(defn with-gil-fn - "Run a function with the gil aquired. If you acquired the gil, release - it when finished. Note we also lock the interpreter so that even - if some code releases thegil, this interpreter cannot be entered." - ([interpreter body-fn] - (let [interpreter (or interpreter (ensure-interpreter))] - (cond - ;;No interpreters bound - (not *current-thread-interpreter*) - (locking interpreter - (with-bindings {#'*current-thread-interpreter* interpreter} - (acquire-gil! interpreter) - (with-bindings {#'libpy-base/*gil-captured* true} - (try - (body-fn) - (finally - (release-gil! interpreter)))))) - ;;Switch interpreters in the current thread...deadlock - ;;is possible here. - (not (identical? interpreter *current-thread-interpreter*)) - (locking interpreter - (let [old-interp *current-thread-interpreter*] - (try - (with-bindings {#'*current-thread-interpreter* interpreter} - (swap-interpreters! old-interp interpreter) - (body-fn)) - (finally - (swap-interpreters! interpreter old-interp))))) - :else - (do - (assert (identical? interpreter *current-thread-interpreter*)) - (body-fn)))))) - (defmacro with-gil - "See with-gil-fn" + "Grab the gil and use the main interpreter. Do not grab gil if already grabbed" [& body] - `(with-gil-fn nil (fn [] (do ~@body)))) - - -(defmacro with-interpreter - "See with-gil-fn" - [interp & body] - `(with-gil-fn ~interp (fn [] (do ~@body)))) + `(do + (let [interp# (ensure-interpreter) + ^AtomicLong bound-thread# libpy-base/gil-thread-id + thread-id# (libpy-base/current-thread-id)] + (locking interp# + (let [new-binding?# (if-not (= thread-id# (.get bound-thread#)) + (do + (acquire-gil! interp#) + true) + false)] + (try + ~@body + (finally + (when new-binding?# + (release-gil! interp#))))))))) (defonce ^:dynamic *program-name* "") @@ -440,7 +403,7 @@ print(json.dumps( [& {:keys [program-name library-path] :as options}] - (when-not @*main-interpreter* + (when-not (main-interpreter) (log-info "Executing python initialize!") (let [{:keys [python-home libname java-library-path-addendum] :as startup-info} (detect-startup-info options) @@ -474,8 +437,11 @@ print(json.dumps( (libpy/PySys_SetArgv 0 (-> program-name (jna/string->wide-ptr))))) (let [type-symbols (libpy/lookup-type-symbols) - context (with-bindings {#'libpy-base/*gil-captured* true} - (libpy/PyEval_SaveThread))] + context (do + (libpy-base/set-gil-thread-id! Long/MAX_VALUE (libpy-base/current-thread-id)) + (let [retval (libpy/PyEval_SaveThread)] + (libpy-base/set-gil-thread-id! (libpy-base/current-thread-id) Long/MAX_VALUE) + retval))] (construct-main-interpreter! context type-symbols)))) @@ -509,18 +475,17 @@ print(json.dumps( (defn finalize! [] - (when *current-thread-interpreter* - (throw (ex-info "There cannot be an interpreter bound when finalize! is called" - {}))) - (check-error-throw) - (when-let [main-interpreter (first (swap-vals! *main-interpreter* (constantly nil)))] - (log-info "executing python finalize!") - (with-bindings {#'*current-thread-interpreter* main-interpreter} - (acquire-gil! main-interpreter) + (when-not (== Long/MAX_VALUE (.get ^AtomicLong libpy-base/gil-thread-id)) + (throw (Exception. (format "A thread still owns the interpreter: " + (.get ^AtomicLong libpy-base/gil-thread-id))))) + (let [interp (ensure-interpreter)] + (locking interp + (check-error-throw) + (log-info "executing python finalize!") + (acquire-gil! interp) (let [finalize-value (libpy/Py_FinalizeEx)] (when-not (= 0 finalize-value) - (log-error (format "PyFinalize returned nonzero value: %s" finalize-value))))) - (remove-interpreter-handle! main-interpreter))) + (log-error (format "PyFinalize returned nonzero value: %s" finalize-value))))))) (defn conj-forever! diff --git a/src/libpython_clj/python/object.clj b/src/libpython_clj/python/object.clj index 10ab9d7..97ab6cb 100644 --- a/src/libpython_clj/python/object.clj +++ b/src/libpython_clj/python/object.clj @@ -23,7 +23,6 @@ get pointers back *but* they don't have to manage the gil." (:require [libpython-clj.python.interpreter :refer [with-gil - with-interpreter ensure-interpreter ensure-bound-interpreter check-error-throw] @@ -146,7 +145,7 @@ ;;We ask the garbage collector to track the python object and notify ;;us when it is released. We then decref on that event. (resource/track pyobj - #(with-interpreter interpreter + #(with-gil (try (let [refcount (refcount pyobj) obj-data (PyObject. (Pointer. pyobj-value))] @@ -461,7 +460,7 @@ Object's refcount is bad. Crash is imminent" pyobj-value refcount py-type-name) (reify ObjectReader (lsize [_] n-items) (read [_ idx] - (with-interpreter interpreter + (with-gil (libpy/PyTuple_GetItem tuple idx))))))) @@ -884,7 +883,7 @@ Object's refcount is bad. Crash is imminent" pyobj-value refcount py-type-name) (let [py-iter (py-proto/call iter-fn) py-next-fn (when py-iter (py-proto/get-attr py-iter "__next__")) next-fn (fn [last-item] - (with-interpreter interpreter + (with-gil (let [retval (libpy/PyObject_CallObject py-next-fn nil)] (if (libpy/PyErr_Occurred) (let [ptype (PointerByReference.) From c17c7a0ddf78b2450c9ce565d62d227f82e82775 Mon Sep 17 00:00:00 2001 From: "J.J. Tolton" Date: Tue, 21 Jan 2020 09:07:41 -0500 Subject: [PATCH 141/456] ISSUE-52 (specify python executable) (#53) * Ability to specify python-executable * default to "python3" --- src/libpython_clj/python.clj | 6 ++-- src/libpython_clj/python/interpreter.clj | 42 +++++++++++++----------- 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/src/libpython_clj/python.clj b/src/libpython_clj/python.clj index 922e790..f026df0 100644 --- a/src/libpython_clj/python.clj +++ b/src/libpython_clj/python.clj @@ -234,11 +234,13 @@ [& {:keys [program-name library-path python-home - no-io-redirect?]}] + no-io-redirect? + python-executable]}] (when-not @pyinterp/main-interpreter* (pyinterp/initialize! :program-name program-name :library-path library-path - :python-home python-home) + :python-home python-home + :python-executable python-executable) ;;setup bridge mechansim and io redirection (pyinterop/register-bridge-type!) (when-not no-io-redirect? diff --git a/src/libpython_clj/python/interpreter.clj b/src/libpython_clj/python/interpreter.clj index dbea6dd..08ae981 100644 --- a/src/libpython_clj/python/interpreter.clj +++ b/src/libpython_clj/python/interpreter.clj @@ -145,29 +145,32 @@ print(json.dumps( (defn detect-startup-info - [{:keys [library-path python-home]}] - (let [executable "python3" - system-info (python-system-info executable) - python-home (cond - python-home - python-home - (seq (System/getenv "PYTHONHOME")) - (System/getenv "PYTHONHOME") - :else - (:prefix system-info)) + [{:keys [library-path python-home python-executable]}] + (log-info + (str "Detecting startup-info for Python executable: " + python-executable)) + (let [executable (or python-executable "python3") + system-info (python-system-info executable) + python-home (cond + python-home + python-home + (seq (System/getenv "PYTHONHOME")) + (System/getenv "PYTHONHOME") + :else + (:prefix system-info)) java-library-path-addendum (when python-home (-> (Paths/get python-home (into-array String ["lib"])) (.toString))) [ver-maj ver-med _ver-min] (:version system-info) - lib-version (format "%s.%s" ver-maj ver-med) - libname (or library-path - (when (seq lib-version) - (str "python" lib-version "m"))) + lib-version (format "%s.%s" ver-maj ver-med) + libname (or library-path + (when (seq lib-version) + (str "python" lib-version "m"))) retval - {:python-home python-home - :lib-version lib-version - :libname libname + {:python-home python-home + :lib-version lib-version + :libname libname :java-library-path-addendum java-library-path-addendum}] (log/infof "Startup info detected: %s" retval) retval)) @@ -401,10 +404,11 @@ print(json.dumps( (defn initialize! [& {:keys [program-name - library-path] + library-path + python-executable] :as options}] (when-not (main-interpreter) - (log-info "Executing python initialize!") + (log-info (str "Executing python initialize with options:" options) ) (let [{:keys [python-home libname java-library-path-addendum] :as startup-info} (detect-startup-info options) library-names (cond From 3ee5f8695ba0110556c5fdaee8326575b12c5877 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 21 Jan 2020 07:09:57 -0700 Subject: [PATCH 142/456] Python gc in python thread (#54) * some simplification/speed improvements * This aint workin' yet * all tests pass with cooperative reference cleanup * editing comments * simplification after reading more of concurrent-deque api * This aint workin' yet * all tests pass with cooperative reference cleanup * editing comments * simplification after reading more of concurrent-deque api * updated tech.resource --- project.clj | 2 +- src/libpython_clj/python/bridge.clj | 12 +- src/libpython_clj/python/gc.clj | 67 +++++++++++ src/libpython_clj/python/interpreter.clj | 5 +- src/libpython_clj/python/object.clj | 146 +++++++++++------------ test/libpython_clj/stress_test.clj | 4 - 6 files changed, 150 insertions(+), 86 deletions(-) create mode 100644 src/libpython_clj/python/gc.clj diff --git a/project.clj b/project.clj index f4c6014..42725a0 100644 --- a/project.clj +++ b/project.clj @@ -5,7 +5,7 @@ :url "https://www.eclipse.org/legal/epl-2.0/"} :dependencies [[org.clojure/clojure "1.10.1"] [camel-snake-kebab "0.4.0"] - [techascent/tech.datatype "4.69"] + [techascent/tech.datatype "4.70"] [org.clojure/data.json "0.2.7"]] :repl-options {:init-ns user} :java-source-paths ["java"]) diff --git a/src/libpython_clj/python/bridge.clj b/src/libpython_clj/python/bridge.clj index c52183d..bb3234a 100644 --- a/src/libpython_clj/python/bridge.clj +++ b/src/libpython_clj/python/bridge.clj @@ -40,17 +40,17 @@ [expose-bridge-to-python! pybridge->bridge create-bridge-from-att-map]] + [libpython-clj.python.gc :as pygc] [clojure.stacktrace :as st] [tech.jna :as jna] [tech.v2.tensor :as dtt] [tech.v2.datatype.casting :as casting] [tech.v2.datatype.functional :as dtype-fn] [tech.v2.datatype :as dtype] - [tech.resource :as resource] [clojure.set :as c-set] [clojure.tools.logging :as log]) (:import [java.util Map RandomAccess List Map$Entry Iterator UUID] - [java.util.concurrent ConcurrentHashMap] + [java.util.concurrent ConcurrentHashMap ConcurrentLinkedQueue] [clojure.lang IFn Symbol Keyword Seqable Fn MapEntry Range LongRange] [tech.v2.datatype ObjectReader ObjectWriter ObjectMutable @@ -58,6 +58,7 @@ [tech.v2.datatype.typed_buffer TypedBuffer] [tech.v2.tensor.protocols PTensor] [com.sun.jna Pointer] + [tech.resource GCReference] [java.io Writer] [libpython_clj.jna JVMBridge CFunction$KeyWordFunction @@ -539,8 +540,7 @@ long-addr (get-attr ctypes "data") hash-ary {:ctypes-map ctypes} ptr-val (-> (Pointer. long-addr) - (resource/track #(get hash-ary :ctypes-map) - pyobj/*pyobject-tracking-flags*))] + (pygc/track #(get hash-ary :ctypes-map)))] {:ptr ptr-val :datatype np-dtype :shape shape @@ -782,7 +782,7 @@ [& body] `(delay (with-gil - (with-bindings {#'pyobj/*pyobject-tracking-flags* [:gc]} + (with-bindings {#'pygc/*stack-gc-context* nil} ~@body)))) @@ -1094,7 +1094,7 @@ initial-buffer (->py-tuple shape) (->py-tuple strides))] - (resource/track retval #(get buffer-desc :ptr) pyobj/*pyobject-tracking-flags*)))) + (pygc/track retval #(get buffer-desc :ptr))))) (extend-type Object diff --git a/src/libpython_clj/python/gc.clj b/src/libpython_clj/python/gc.clj new file mode 100644 index 0000000..23b47af --- /dev/null +++ b/src/libpython_clj/python/gc.clj @@ -0,0 +1,67 @@ +(ns libpython-clj.python.gc + "Binding of various sort of gc semantics optimized specifically for + libpython-clj. For general bindings, see tech.resource" + (:import [java.util.concurrent ConcurrentHashMap ConcurrentLinkedDeque] + [java.lang.ref ReferenceQueue] + [tech.resource GCReference])) + + +(set! *warn-on-reflection* true) + + +(defonce ^:dynamic *stack-gc-context* nil) +(defn stack-context + ^ConcurrentLinkedDeque [] + *stack-gc-context*) + + +(defonce reference-queue-var (ReferenceQueue.)) +(defn reference-queue + ^ReferenceQueue [] + reference-queue-var) + + +(defonce ptr-set-var (ConcurrentHashMap/newKeySet)) +(defn ptr-set + ^java.util.Set [] + ptr-set-var) + + +(defn track + [item dispose-fn] + (let [ptr-val (GCReference. item (reference-queue) (fn [ptr-val] + (.remove (ptr-set) ptr-val) + (dispose-fn))) + ^ConcurrentLinkedDeque stack-context (stack-context)] + ;;We have to keep track of the pointer. If we do not the pointer gets gc'd then + ;;it will not be put on the reference queue when the object itself is gc'd. + ;;Nice little gotcha there. + (if stack-context + (.add stack-context ptr-val) + ;;Ensure we don't lose track of the weak reference. If it gets cleaned up + ;;the gc system will fail. + (.add (ptr-set) ptr-val)) + item)) + + +(defn clear-reference-queue + [] + (when-let [next-ref (.poll (reference-queue))] + (.run ^Runnable next-ref) + (recur))) + + +(defn clear-stack-context + [] + (when-let [next-ref (.pollLast (stack-context))] + (.run ^Runnable next-ref) + (recur))) + + +(defmacro with-stack-context + [& body] + `(with-bindings {#'*stack-gc-context* (ConcurrentLinkedDeque.)} + (try + ~@body + (finally + (clear-stack-context))))) diff --git a/src/libpython_clj/python/interpreter.clj b/src/libpython_clj/python/interpreter.clj index 08ae981..26aecf3 100644 --- a/src/libpython_clj/python/interpreter.clj +++ b/src/libpython_clj/python/interpreter.clj @@ -1,7 +1,7 @@ (ns libpython-clj.python.interpreter (:require [libpython-clj.jna :as libpy ] [libpython-clj.jna.base :as libpy-base] - [tech.resource :as resource] + [libpython-clj.python.gc :as pygc] [libpython-clj.python.logging :refer [log-error log-warn log-info]] [tech.jna :as jna] @@ -351,6 +351,7 @@ print(json.dumps( (try ~@body (finally + (pygc/clear-reference-queue) (when new-binding?# (release-gil! interp#))))))))) @@ -437,7 +438,7 @@ print(json.dumps( (recur library-names)))) ;;Set program name (when-let [program-name (or program-name *program-name* "")] - (resource/stack-resource-context + (pygc/with-stack-context (libpy/PySys_SetArgv 0 (-> program-name (jna/string->wide-ptr))))) (let [type-symbols (libpy/lookup-type-symbols) diff --git a/src/libpython_clj/python/object.clj b/src/libpython_clj/python/object.clj index 97ab6cb..9b0be72 100644 --- a/src/libpython_clj/python/object.clj +++ b/src/libpython_clj/python/object.clj @@ -33,12 +33,13 @@ :as py-proto] [libpython-clj.jna.base :as libpy-base] [libpython-clj.jna :as libpy] + [libpython-clj.python.gc :as pygc] [clojure.stacktrace :as st] [tech.jna :as jna] - [tech.resource :as resource] [tech.v2.datatype :as dtype] [tech.v2.datatype.protocols :as dtype-proto] [tech.v2.datatype.casting :as casting] + [tech.resource :as resource] [tech.v2.tensor] [clojure.tools.logging :as log]) (:import [com.sun.jna Pointer CallbackReference] @@ -96,13 +97,10 @@ (py-proto/->jvm item options))) -(def ^:dynamic *object-reference-logging* false) - +(def object-reference-logging nil) -(def ^:dynamic *object-reference-tracker* nil) - -(def ^:dynamic *pyobject-tracking-flags* [:gc]) +(def object-reference-tracker nil) (defn incref @@ -130,41 +128,44 @@ (if (and pyobj (not= (Pointer/nativeValue (libpy/as-pyobj pyobj)) (Pointer/nativeValue (libpy/as-pyobj (libpy/Py_None))))) - (let [interpreter (ensure-bound-interpreter) - pyobj-value (Pointer/nativeValue (libpy/as-pyobj pyobj)) - py-type-name (name (python-type pyobj))] - (when *object-reference-logging* - (let [obj-data (PyObject. (Pointer. pyobj-value))] - (println (format "tracking object - 0x%x:%4d:%s" - pyobj-value - (.ob_refcnt obj-data) - py-type-name)))) - (when *object-reference-tracker* - (swap! *object-reference-tracker* - update pyobj-value #(inc (or % 0)))) - ;;We ask the garbage collector to track the python object and notify - ;;us when it is released. We then decref on that event. - (resource/track pyobj - #(with-gil - (try - (let [refcount (refcount pyobj) - obj-data (PyObject. (Pointer. pyobj-value))] - (if (< refcount 1) - (log/errorf "Fatal error -- releasing object - 0x%x:%4d:%s + (do + (ensure-bound-interpreter) + (let [pyobj-value (Pointer/nativeValue (libpy/as-pyobj pyobj)) + py-type-name (name (python-type pyobj))] + (when object-reference-logging + (let [obj-data (PyObject. (Pointer. pyobj-value))] + (println (format "tracking object - 0x%x:%4d:%s" + pyobj-value + (.ob_refcnt obj-data) + py-type-name)))) + (when object-reference-tracker + (swap! object-reference-tracker + update pyobj-value #(inc (or % 0)))) + ;;We ask the garbage collector to track the python object and notify + ;;us when it is released. We then decref on that event. + (pygc/track pyobj + ;;No longer with-gil. Because cleanup is cooperative, the gil is + ;;guaranteed to be captured here already. + #(try + ;;Intentionally overshadow pyobj. We cannot access it here. + (let [pyobj (Pointer. pyobj-value) + refcount (refcount pyobj) + obj-data (PyObject. pyobj)] + (if (< refcount 1) + (log/errorf "Fatal error -- releasing object - 0x%x:%4d:%s Object's refcount is bad. Crash is imminent" pyobj-value refcount py-type-name) - (when *object-reference-logging* - (println (format "releasing object - 0x%x:%4d:%s" - pyobj-value - (.ob_refcnt obj-data) - py-type-name)))) - (when *object-reference-tracker* - (swap! *object-reference-tracker* - update pyobj-value (fn [arg] - (dec (or arg 0)))))) - (libpy/Py_DecRef (Pointer. pyobj-value)) - (catch Throwable e - (log/error e "Exception while releasing object")))) - *pyobject-tracking-flags*)) + (when object-reference-logging + (println (format "releasing object - 0x%x:%4d:%s" + pyobj-value + (.ob_refcnt obj-data) + py-type-name)))) + (when object-reference-tracker + (swap! object-reference-tracker + update pyobj-value (fn [arg] + (dec (or arg 0)))))) + (libpy/Py_DecRef (Pointer. pyobj-value)) + (catch Throwable e + (log/error e "Exception while releasing object")))))) (do ;;Special handling for PyNone types (libpy/Py_DecRef pyobj) @@ -173,9 +174,8 @@ Object's refcount is bad. Crash is imminent" pyobj-value refcount py-type-name) (defmacro stack-resource-context [& body] - `(with-bindings {#'*pyobject-tracking-flags* [:stack :gc]} - (resource/stack-resource-context - ~@body))) + `(pygc/with-stack-context + ~@body)) (defn incref-wrap-pyobject @@ -391,37 +391,37 @@ Object's refcount is bad. Crash is imminent" pyobj-value refcount py-type-name) doc function] :as method-data}] - (resource/stack-resource-context - (when-not (cfunc-instance? function) - (throw (Exception. - (format "Callbacks must implement one of the CFunction interfaces: + ;;Here we really do need a resource stack context + (when-not (cfunc-instance? function) + (throw (Exception. + (format "Callbacks must implement one of the CFunction interfaces: %s" (type function))))) - (let [meth-flags (long (cond - (instance? CFunction$NoArgFunction function) - @libpy/METH_NOARGS - - (instance? CFunction$TupleFunction function) - @libpy/METH_VARARGS - - (instance? CFunction$KeyWordFunction function) - (bit-or @libpy/METH_KEYWORDS @libpy/METH_VARARGS) - :else - (throw (ex-info (format "Failed due to type: %s" - (type function)) - {})))) - name-ptr (jna/string->ptr name) - doc-ptr (jna/string->ptr doc)] - (set! (.ml_name method-def) name-ptr) - (set! (.ml_meth method-def) (CallbackReference/getFunctionPointer function)) - (set! (.ml_flags method-def) (int meth-flags)) - (set! (.ml_doc method-def) doc-ptr) - (.write method-def) - (pyinterp/conj-forever! (assoc method-data - :name-ptr name-ptr - :doc-ptr doc-ptr - :callback-object function - :method-definition method-def)) - method-def))) + (let [meth-flags (long (cond + (instance? CFunction$NoArgFunction function) + @libpy/METH_NOARGS + + (instance? CFunction$TupleFunction function) + @libpy/METH_VARARGS + + (instance? CFunction$KeyWordFunction function) + (bit-or @libpy/METH_KEYWORDS @libpy/METH_VARARGS) + :else + (throw (ex-info (format "Failed due to type: %s" + (type function)) + {})))) + name-ptr (jna/string->ptr-untracked name) + doc-ptr (jna/string->ptr-untracked doc)] + (set! (.ml_name method-def) name-ptr) + (set! (.ml_meth method-def) (CallbackReference/getFunctionPointer function)) + (set! (.ml_flags method-def) (int meth-flags)) + (set! (.ml_doc method-def) doc-ptr) + (.write method-def) + (pyinterp/conj-forever! (assoc method-data + :name-ptr name-ptr + :doc-ptr doc-ptr + :callback-object function + :method-definition method-def)) + method-def)) (defn method-def-data->method-def diff --git a/test/libpython_clj/stress_test.clj b/test/libpython_clj/stress_test.clj index e4eecd1..615c709 100644 --- a/test/libpython_clj/stress_test.clj +++ b/test/libpython_clj/stress_test.clj @@ -56,10 +56,6 @@ def getmultidata(): (gd-fn))) -;;If you want to see how the sausage is made... -(alter-var-root #'libpython-clj.python.object/*object-reference-logging* - (constantly false)) - ;;Ensure that failure to open resource context before tracking for stack ;;related things causes immediate failure. Unless you feel like being pedantic, ;;this isn't necessary. The python library automatically switches to normal gc-only From 2088b9f062dc01317467ae23bb7c5d65d8a10078 Mon Sep 17 00:00:00 2001 From: "J.J. Tolton" Date: Tue, 21 Jan 2020 09:11:27 -0500 Subject: [PATCH 143/456] ISSUE-49: Namespace bindings for modules (#55) * bind namespace to symbol * throw error if invalid symbol name --- src/libpython_clj/require.clj | 54 +++++++++++++++++++++++++---------- 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/src/libpython_clj/require.clj b/src/libpython_clj/require.clj index 25e3ba5..7b33685 100644 --- a/src/libpython_clj/require.clj +++ b/src/libpython_clj/require.clj @@ -21,7 +21,7 @@ ;; First attempt is to filter keywords and make sure any keywords are ;; in supported-flags (let [total-flags (set (concat supported-flags [:as :refer :exclude - :* :all]))] + :* :all :bind-ns]))] (when-let [missing-flags (->> reqs (filter #(and (not (total-flags %)) (keyword? %))) @@ -88,31 +88,48 @@ (defn- do-require-python [reqs-vec] (let [[module-name & etc] reqs-vec - supported-flags #{:reload :no-arglists} - flags (parse-flags supported-flags etc) - etc (->> etc - (remove supported-flags) - (remove boolean?)) - _ (when-not (= 0 (rem (count etc) 2)) - (throw (Exception. "Must have even number of entries"))) - etc (->> etc (partition-all 2) - (map vec) - (into {})) + supported-flags #{:reload :no-arglists :bind-ns} + flags (parse-flags supported-flags etc) + etc (->> etc + (remove supported-flags) + (remove boolean?)) + _ (when-not (= 0 (rem (count etc) 2)) + (throw (Exception. "Must have even number of entries"))) + etc (->> etc (partition-all 2) + (map vec) + (into {})) reload? (:reload flags) no-arglists? (:no-arglists flags) + bind-ns? (:bind-ns flags) alias-name (:as etc) exclude (into #{} (:exclude etc)) + refer-data (cond (= :all (:refer etc)) #{:all} (= :* (:refer etc)) #{:*} - :else (into #{} (:refer etc))) - pyobj (pymeta/path->py-obj (str module-name) :reload? reload?) - existing-py-ns? (find-ns module-name)] + :else (into #{} (:refer etc))) + pyobj (pymeta/path->py-obj (str module-name) :reload? reload?) + existing-py-ns? (find-ns module-name)] (create-ns module-name) + + (when bind-ns? + (let [import-name (or (not-empty (str alias-name)) + (str module-name)) + ns-dots (re-find #"[.]" import-name)] + (when (not (zero? (count ns-dots))) + (throw (Exception. (str "Cannot have periods in module/class" + "name. Please :alias " + import-name + " to something without periods.")))) + (intern + (symbol (str *ns*)) + (symbol import-name) + pyobj))) + (when (or (not existing-py-ns?) reload?) (pymeta/apply-static-metadata-to-namespace! module-name (datafy pyobj) :no-arglists? no-arglists?)) - (when-let [refer-symbols (->> (extract-refer-symbols {:refer refer-data + (when-let [refer-symbols (->> (extract-refer-symbols {:refer refer-data :this-module pyobj} (ns-publics (find-ns module-name))) @@ -150,6 +167,12 @@ (require-python '[numpy :refer [linspace] :no-arglists :as np]) + If you would like to bind the Python module to the namespace, use + the :bind-ns flag. + + (require-python '[requests :bind-ns true]) or + (require-python '[requests :bind-ns]) + ## Use with custom modules ## For use with a custom namespace foo.py while developing, you can @@ -201,3 +224,4 @@ (do-require-python reqs) :else (throw (Exception. "Invalid argument: %s" reqs)))) + From 12e5eb600b815e56ce78d9cab27108ae4ba18793 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 21 Jan 2020 09:29:31 -0700 Subject: [PATCH 144/456] Final round of perf opts. Next step is jna direct mapping. --- project.clj | 2 +- src/libpython_clj/python/object.clj | 111 ++++++++++++++-------------- 2 files changed, 58 insertions(+), 55 deletions(-) diff --git a/project.clj b/project.clj index 42725a0..88c046a 100644 --- a/project.clj +++ b/project.clj @@ -5,7 +5,7 @@ :url "https://www.eclipse.org/legal/epl-2.0/"} :dependencies [[org.clojure/clojure "1.10.1"] [camel-snake-kebab "0.4.0"] - [techascent/tech.datatype "4.70"] + [techascent/tech.datatype "4.71"] [org.clojure/data.json "0.2.7"]] :repl-options {:init-ns user} :java-source-paths ["java"]) diff --git a/src/libpython_clj/python/object.clj b/src/libpython_clj/python/object.clj index 9b0be72..8ca7428 100644 --- a/src/libpython_clj/python/object.clj +++ b/src/libpython_clj/python/object.clj @@ -97,10 +97,7 @@ (py-proto/->jvm item options))) -(def object-reference-logging nil) - - -(def object-reference-tracker nil) +(def object-reference-logging (atom false)) (defn incref @@ -116,60 +113,66 @@ (long (.ob_refcnt (PyObject. (libpy/as-pyobj pyobj))))) +(def ^:private non-native-val + (delay (Pointer/nativeValue (libpy/as-pyobj (libpy/Py_None))))) + + (defn wrap-pyobject "Wrap object such that when it is no longer accessible via the program decref is called. Used for new references. This is some of the meat of the issue, however, in that getting the two system's garbage collectors to play nice is kind - of tough." - [pyobj & [skip-check-error?]] - (when-not skip-check-error? - (check-error-throw)) - ;;We don't wrap pynone - (if (and pyobj - (not= (Pointer/nativeValue (libpy/as-pyobj pyobj)) - (Pointer/nativeValue (libpy/as-pyobj (libpy/Py_None))))) - (do - (ensure-bound-interpreter) - (let [pyobj-value (Pointer/nativeValue (libpy/as-pyobj pyobj)) - py-type-name (name (python-type pyobj))] - (when object-reference-logging - (let [obj-data (PyObject. (Pointer. pyobj-value))] - (println (format "tracking object - 0x%x:%4d:%s" - pyobj-value - (.ob_refcnt obj-data) - py-type-name)))) - (when object-reference-tracker - (swap! object-reference-tracker - update pyobj-value #(inc (or % 0)))) - ;;We ask the garbage collector to track the python object and notify - ;;us when it is released. We then decref on that event. - (pygc/track pyobj - ;;No longer with-gil. Because cleanup is cooperative, the gil is - ;;guaranteed to be captured here already. - #(try - ;;Intentionally overshadow pyobj. We cannot access it here. - (let [pyobj (Pointer. pyobj-value) - refcount (refcount pyobj) - obj-data (PyObject. pyobj)] - (if (< refcount 1) - (log/errorf "Fatal error -- releasing object - 0x%x:%4d:%s -Object's refcount is bad. Crash is imminent" pyobj-value refcount py-type-name) - (when object-reference-logging - (println (format "releasing object - 0x%x:%4d:%s" - pyobj-value - (.ob_refcnt obj-data) - py-type-name)))) - (when object-reference-tracker - (swap! object-reference-tracker - update pyobj-value (fn [arg] - (dec (or arg 0)))))) - (libpy/Py_DecRef (Pointer. pyobj-value)) - (catch Throwable e - (log/error e "Exception while releasing object")))))) - (do - ;;Special handling for PyNone types - (libpy/Py_DecRef pyobj) - nil))) + of tough. + This is a hot path; it is called quite a lot from a lot of places." + ([pyobj skip-check-error?] + ;;We don't wrap pynone + (if pyobj + (let [pyobj-value (Pointer/nativeValue (libpy/as-pyobj pyobj)) + ^PyObject obj-data (when @object-reference-logging + (PyObject. (Pointer. pyobj-value)))] + (if (not= pyobj-value + (long @non-native-val)) + (do + (ensure-bound-interpreter) + (when @object-reference-logging + (println (format "tracking object - 0x%x:%4d:%s" + pyobj-value + (.ob_refcnt obj-data) + (name (python-type pyobj))))) + ;;We ask the garbage collector to track the python object and notify + ;;us when it is released. We then decref on that event. + (pygc/track + pyobj + ;;No longer with-gil. Because cleanup is cooperative, the gil is + ;;guaranteed to be captured here already. + #(try + ;;Intentionally overshadow pyobj. We cannot access it here. + (let [pyobj (Pointer. pyobj-value)] + (when @object-reference-logging + (let [_ (.read obj-data) + refcount (.ob_refcnt obj-data)] + (if (< refcount 1) + (log/errorf "Fatal error -- releasing object - 0x%x:%4d:%s +Object's refcount is bad. Crash is imminent" + pyobj-value + refcount + (name (python-type pyobj))) + (println (format "releasing object - 0x%x:%4d:%s" + pyobj-value + refcount + (name (python-type pyobj)))))))) + (libpy/Py_DecRef pyobj) + (catch Throwable e + (log/error e "Exception while releasing object")))) + (when-not skip-check-error? (check-error-throw)) + pyobj) + (do + ;;Special handling for PyNone types + (libpy/Py_DecRef pyobj) + (when-not skip-check-error? (check-error-throw)) + nil))) + (when-not skip-check-error? (check-error-throw)))) + ([pyobj] + (wrap-pyobject pyobj false))) (defmacro stack-resource-context From 78d8660dfbabfe25b49914563d034fe3b9cd1241 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 21 Jan 2020 16:40:40 -0700 Subject: [PATCH 145/456] Smallest possible step towards directmapping --- java/libpython_clj/jna/DirectMapped.java | 10 ++++++++++ src/libpython_clj/jna/protocols/object.clj | 14 +++++++------- src/libpython_clj/python.clj | 2 +- src/libpython_clj/python/interpreter.clj | 13 +++++++++---- 4 files changed, 27 insertions(+), 12 deletions(-) create mode 100644 java/libpython_clj/jna/DirectMapped.java diff --git a/java/libpython_clj/jna/DirectMapped.java b/java/libpython_clj/jna/DirectMapped.java new file mode 100644 index 0000000..e861ae7 --- /dev/null +++ b/java/libpython_clj/jna/DirectMapped.java @@ -0,0 +1,10 @@ +package libpython_clj.jna; + +import com.sun.jna.Pointer; + + +public class DirectMapped +{ + public static native void Py_IncRef(Pointer ptr); + public static native void Py_DecRef(Pointer ptr); +} diff --git a/src/libpython_clj/jna/protocols/object.clj b/src/libpython_clj/jna/protocols/object.clj index 32c2847..a4cf4c5 100644 --- a/src/libpython_clj/jna/protocols/object.clj +++ b/src/libpython_clj/jna/protocols/object.clj @@ -13,21 +13,21 @@ [camel-snake-kebab.core :refer [->kebab-case]]) (:import [com.sun.jna Pointer Native NativeLibrary] [com.sun.jna.ptr PointerByReference] - [libpython_clj.jna PyObject])) + [libpython_clj.jna PyObject DirectMapped])) ;; Object Protocol -(def-pylib-fn Py_DecRef +(defn Py_DecRef "Decrement the refference count on an object" - nil - [py-obj ensure-pyobj]) + [py-obj] + (DirectMapped/Py_DecRef (ensure-pyobj py-obj))) -(def-pylib-fn Py_IncRef +(defn Py_IncRef "Increment the reference count on an object" - nil - [py-obj ensure-pyobj]) + [py-obj] + (DirectMapped/Py_IncRef (ensure-pyobj py-obj))) ;; object.h:937 diff --git a/src/libpython_clj/python.clj b/src/libpython_clj/python.clj index f026df0..9568953 100644 --- a/src/libpython_clj/python.clj +++ b/src/libpython_clj/python.clj @@ -12,7 +12,7 @@ [com.sun.jna.ptr PointerByReference] [java.lang.reflect Field] [java.io Writer] - [libpython_clj.jna PyObject + [libpython_clj.jna PyObject DirectMapped CFunction$KeyWordFunction CFunction$TupleFunction CFunction$NoArgFunction])) diff --git a/src/libpython_clj/python/interpreter.clj b/src/libpython_clj/python/interpreter.clj index 26aecf3..2c8d8a5 100644 --- a/src/libpython_clj/python/interpreter.clj +++ b/src/libpython_clj/python/interpreter.clj @@ -10,9 +10,7 @@ [clojure.string :as s] [clojure.tools.logging :as log] [clojure.data.json :as json]) - (:import [libpython_clj.jna - JVMBridge - PyObject] + (:import [libpython_clj.jna JVMBridge PyObject DirectMapped] [java.util.concurrent.atomic AtomicLong] [com.sun.jna Pointer] [com.sun.jna.ptr PointerByReference] @@ -403,6 +401,12 @@ print(json.dumps( (catch Exception e))) +(defn- setup-direct-mapping! + [] + (let [library (jna/load-library libpy-base/*python-library*)] + (com.sun.jna.Native/register DirectMapped library))) + + (defn initialize! [& {:keys [program-name library-path @@ -435,7 +439,8 @@ print(json.dumps( (not (try-load-python-library! library-name @python-home-wide-ptr* @python-path-wide-ptr*))) - (recur library-names)))) + (recur library-names))) + (setup-direct-mapping!)) ;;Set program name (when-let [program-name (or program-name *program-name* "")] (pygc/with-stack-context From 6b3b1bcf1d682eefbe1a39ae9fef25437215fcfa Mon Sep 17 00:00:00 2001 From: "J.J. Tolton" Date: Tue, 21 Jan 2020 18:42:02 -0500 Subject: [PATCH 146/456] CHANGELOG update for 1.32-SNAPSHOT (#56) * update changelog * fix typo * fix formatting * fix formatting * caveats --- CHANGELOG.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 37c82e1..2ce5583 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,52 @@ # Time for a ChangeLog! +## 1.32 + +* Python executables can now be specified directly using the syntax + ```clojure + (py/initialize! :python-executable ) + ``` + where **executable** can be a system installation of Python + such as `"python3"`, `"python3.7"`; it can also be a fully qualified + path such as `"/usr/bin/python3.7"`; or any Python executable along + your discoverable system path. + +* Python virtual environments can now be used instead of system + installations! This has been tested on Linux/Ubuntu variants + with virtual environments installed with + ```bash + virtualenv -p $(which ) env + ``` + and then invoked using + ```clojure + (py/initialize! :python-executable "/abs/path/to/env/bin/python") + ``` + + Tested on Python 3.6.8 and Python 3.7. + + **WARNING**: This is suitable for casual hacking and exploratory + development -- however, at this time, we still strongly recommend + using Docker and a system installation of Python in production + environments. + +* **breaking change** (and remediation): `require-python` no longer + automatically binds the Python module to the Clojure the namespace + symbol. If you wish to bind the module to the namespace symbol, + you need to use the `:bind-ns` flag. Example: + + ```clojure + (require-python 'requests) ;;=> nil + requests ;;=> throws Exception + + (require-python '[requests :bind-ns]) ;;=> nil + (py.. requests + (get "https://www.google.com) + -content + (decode "latin-1)) ;; works + ``` + + + ## 1.31 * Python objects are now datafy-able and nav-igable. `require-python` From 76708c286d0a8696fc3aa683fc43ade19da8a927 Mon Sep 17 00:00:00 2001 From: "J.J. Tolton" Date: Wed, 22 Jan 2020 11:51:43 -0500 Subject: [PATCH 147/456] ISSUE-46: programmatic method helper syntax (#57) * ISSUE-49 changes * ISSUE-49 changes * revert bad test * fix spacing --- src/libpython_clj/python.clj | 69 +++++++++++++++++++++++++++++- src/libpython_clj/require.clj | 12 ++++++ test/libpython_clj/python_test.clj | 39 +++++++++++++++-- 3 files changed, 115 insertions(+), 5 deletions(-) diff --git a/src/libpython_clj/python.clj b/src/libpython_clj/python.clj index 9568953..cb10838 100644 --- a/src/libpython_clj/python.clj +++ b/src/libpython_clj/python.clj @@ -432,6 +432,37 @@ [x & args] (list* (into (vector #'$a x) args))) +(defmacro py* + "Special syntax for passing along *args and **kwargs style arguments + to methods. + + Usage: + + (py* obj method args kwargs) + + Example: + + (def d (python/dict)) + d ;;=> {} + (def iterable [[:a 1] [:b 2]]) + (def kwargs {:cat \"dog\" :name \"taco\"}) + (py* d update [iterable] kwargs) + d ;;=> {\"a\": 1, \"b\": 2, \"cat\": \"dog\", \"name\": \"taco\"}" + ([x method args] + (list #'call-attr-kw x (str method) args nil)) + ([x method args kwargs] + (list #'call-attr-kw x (str method) args kwargs))) + +(defmacro py** + "Like py*, but it is assumed that the LAST argument is kwargs." + ([x method kwargs] + (list #'call-attr-kw x (str method) nil kwargs)) + ([x method arg & args] + (let [args (into [arg] args) + kwargs (last args) + args (vec (pop args))] + (list #'call-attr-kw x (str method) args kwargs)))) + (defn ^:private handle-pydotdot ([x form] @@ -439,8 +470,18 @@ (let [form-data (vec form) [instance-member & args] form-data symbol-str (str instance-member)] - (if (clojure.string/starts-with? symbol-str "-") + (cond + (clojure.string/starts-with? symbol-str "-") (list #'py.- x (symbol (subs symbol-str 1 (count symbol-str)))) + + (clojure.string/starts-with? symbol-str "**") + (list* #'py** x (symbol (subs symbol-str 2 (count symbol-str))) args) + + (clojure.string/starts-with? symbol-str "*") + (list* #'py* x (symbol (subs symbol-str 1 (count symbol-str))) args) + + :else ;; assumed to be method invocation + (list* (into (vector #'py. x instance-member) args)))) (handle-pydotdot x (list form)))) ([x form & more] @@ -456,6 +497,30 @@ is equivalent to Python's import sys - sys.path.append('/home/user/bin')" + sys.path.append('/home/user/bin') + + SPECIAL SYNTAX for programmatic *args and **kwargs + + Special syntax is provided to meet the needs required by + Python's *args and **kwargs syntax programmatically. + + + (= (py.. obj (*method args)) + (py* obj methods args)) + + (= (py.. obj (*methods args kwargs)) + (py* obj method args kwargs)) + + (= (py.. obj (**method kwargs)) + (py** obj kwargs)) + + (= (py.. obj (**method arg1 arg2 arg3 ... argN kwargs)) + (py** obj method arg1 arg2 arg3 ... argN kwargs) + (py* obj method [arg1 arg2 arg3 ... argN] kwargs)) + + + These forms exist for when you need to pass in a map of options + in the same way you would use the f(*args, **kwargs) forms in + Python." [x & args] (apply handle-pydotdot x args)) diff --git a/src/libpython_clj/require.clj b/src/libpython_clj/require.clj index 7b33685..9773371 100644 --- a/src/libpython_clj/require.clj +++ b/src/libpython_clj/require.clj @@ -225,3 +225,15 @@ :else (throw (Exception. "Invalid argument: %s" reqs)))) +(defn import-python + "Loads python, python.list, python.dict, python.set, python.tuple, + and python.frozenset." + [] + (require-python + '([builtins :as python] + [builtins.list :as python.list] + [builtins.dict :as python.dict] + [builtins.set :as python.set] + [builtins.tuple :as python.tuple] + [builtins.frozenset :as python.frozenset]))) + diff --git a/test/libpython_clj/python_test.clj b/test/libpython_clj/python_test.clj index 903d325..a2de61b 100644 --- a/test/libpython_clj/python_test.clj +++ b/test/libpython_clj/python_test.clj @@ -1,5 +1,5 @@ (ns libpython-clj.python-test - (:require [libpython-clj.python :as py] + (:require [libpython-clj.python :as py :refer [py. py.. py.- py* py**]] [libpython-clj.jna :as libpy] [tech.v2.datatype :as dtype] [tech.v2.datatype.functional :as dfn] @@ -303,7 +303,41 @@ class Foo: (py/py.. f res __len__) ((py/py.. f res -__len__)) (py/py.. f count) - ((py/py.. f -count)))))) + ((py/py.. f -count))))) + + + (let [builtins (py/import-module "builtins") + dict (py/get-attr builtins "dict") + d (dict)] + (py* d update nil {:a 1}) + (is (= 1 (py* d get [:a]))) + (let [iterable [[:a 2]]] + (py* d update [iterable])) + (is (= 2 (py* d get [:a]))) + (let [iterable [[:a 1] [:b 2]] + kwargs {:name "taco"}] + (py* d update [iterable] kwargs)) + (is (= {"name" "taco" "a" 1 "b" 2} d)) + + (py. d clear) + + (py** d update {:a 1}) + (is (= d {"a" 1})) + + (py** d update [[:a 2]] {:c 3}) + (is (= d {"a" 2 "c" 3})) + + + (py. d clear) + + (doto d + (py.. (*update nil {:a 1})) + (py.. (*update [[[:b 2]]])) + (py.. (**update {:c 3})) + (py.. (**update [[:d 4]] {:e 5}))) + + (is (= d {"a" 1 "b" 2 "c" 3 "d" 4 "e" 5}))) + ) (deftest infinite-seq (let [islice (-> (py/import-module "itertools") @@ -375,4 +409,3 @@ class Foo: (deftest characters (is (= (py/->jvm (py/->python "c")) (py/->jvm (py/->python \c))))) - From 02cdd56823d211a814de68c2a892824f71db5fb9 Mon Sep 17 00:00:00 2001 From: "J.J. Tolton" Date: Wed, 22 Jan 2020 11:52:00 -0500 Subject: [PATCH 148/456] update changelog (#58) --- CHANGELOG.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ce5583..b8685e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,25 @@ (decode "latin-1)) ;; works ``` +* Python method helper syntax for programmatic passing of maps + to satisfy `*args`, `**kwargs` situations on the `py.` family of + macros. Two new macros have been introduced to address this + + ```clojure + (py* obj method args) + (py* obj method args kwargs) + (py** obj method kwargs) + (py** obj method arg1 arg2 arg3 ... argN kwargs) + ``` + and the `py..` syntax has been extended to accomodate these + conventions as well. + + ```clojure + (py.. obj (*method args)) + (py.. obj (*method args kwargs)) + (py.. obj (**method kwargs)) + (py.. obj (**method arg1 arg2 arg3 ... argN kwargs)) + ``` ## 1.31 From 215595ae305f4360ae2680269ad66d20b3c2611f Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Wed, 22 Jan 2020 13:09:46 -0700 Subject: [PATCH 149/456] Direct mapping (#59) * major push into direct mapping * Another factor of 2 with direct mapping. * More direct mapped work to hit some more common cases. --- java/libpython_clj/jna/DirectMapped.java | 35 +++++++++++++++ project.clj | 3 +- src/libpython_clj/jna/concrete/dict.clj | 14 +++--- src/libpython_clj/jna/concrete/err.clj | 7 +-- .../jna/concrete/numeric/float.clj | 15 +++---- .../jna/concrete/numeric/integer.clj | 15 ++++--- src/libpython_clj/jna/concrete/tuple.clj | 25 ++++++----- src/libpython_clj/jna/concrete/unicode.clj | 22 +++++----- src/libpython_clj/jna/interpreter.clj | 13 +++--- src/libpython_clj/jna/protocols/object.clj | 43 ++++++++++--------- src/libpython_clj/jna/protocols/sequence.clj | 21 +++++---- src/libpython_clj/python/interop.clj | 4 +- src/libpython_clj/python/object.clj | 11 ++--- 13 files changed, 133 insertions(+), 95 deletions(-) diff --git a/java/libpython_clj/jna/DirectMapped.java b/java/libpython_clj/jna/DirectMapped.java index e861ae7..d38ccd8 100644 --- a/java/libpython_clj/jna/DirectMapped.java +++ b/java/libpython_clj/jna/DirectMapped.java @@ -1,10 +1,45 @@ package libpython_clj.jna; import com.sun.jna.Pointer; +import com.sun.jna.ptr.LongByReference; +import com.sun.jna.ptr.PointerByReference; +import java.nio.ByteBuffer; public class DirectMapped { public static native void Py_IncRef(Pointer ptr); public static native void Py_DecRef(Pointer ptr); + public static native void PyEval_RestoreThread(Pointer tstate); + public static native Pointer PyEval_SaveThread(); + public static native Pointer PyFloat_FromDouble(double y); + public static native double PyFloat_AsDouble(Pointer val); + public static native Pointer PyLong_FromLongLong(long v); + public static native long PyLong_AsLongLong(Pointer v); + public static native int PyDict_Next(Pointer p, LongByReference ppos, + PointerByReference pkey, + PointerByReference pvalue); + //FIXME - how to change signatures if 32 bit. + public static native Pointer PyTuple_New(long size); + public static native Pointer PyTuple_GetItem(Pointer p, long size); + public static native int PyTuple_SetItem(Pointer p, long pos, Pointer o); + public static native Pointer PyErr_Occurred(); + public static native Pointer PyObject_CallObject(Pointer callable, Pointer argtuple); + public static native Pointer PyObject_Call(Pointer callable, + Pointer argtuple, + Pointer kwargs); + public static native int PyObject_HasAttrString(Pointer obj, String attrName); + public static native Pointer PyObject_GetAttrString(Pointer obj, String attrName); + public static native int PyCallable_Check(Pointer p); + public static native Pointer PyUnicode_AsUTF8AndSize(Pointer pyobj, + LongByReference reference); + public static native Pointer PyUnicode_Decode(ByteBuffer bytedata, + long numchars, //size-t + String encoding, + String flags); + public static native int PySequence_Check(Pointer val); + //returns size-t + public static native long PySequence_Length(Pointer val); + //takes size-t + public static native Pointer PySequence_GetItem(Pointer val, long idx); } diff --git a/project.clj b/project.clj index 88c046a..d0bd0e2 100644 --- a/project.clj +++ b/project.clj @@ -5,7 +5,8 @@ :url "https://www.eclipse.org/legal/epl-2.0/"} :dependencies [[org.clojure/clojure "1.10.1"] [camel-snake-kebab "0.4.0"] - [techascent/tech.datatype "4.71"] + [techascent/tech.datatype "4.72"] [org.clojure/data.json "0.2.7"]] + :profiles {:dev {:dependencies [[criterium "0.4.5"]]}} :repl-options {:init-ns user} :java-source-paths ["java"]) diff --git a/src/libpython_clj/jna/concrete/dict.clj b/src/libpython_clj/jna/concrete/dict.clj index 93548a0..b194f6a 100644 --- a/src/libpython_clj/jna/concrete/dict.clj +++ b/src/libpython_clj/jna/concrete/dict.clj @@ -9,7 +9,7 @@ :as libpy-base] [tech.jna :as jna]) (:import [com.sun.jna Pointer] - [libpython_clj.jna PyObject])) + [libpython_clj.jna PyObject DirectMapped])) @@ -181,7 +181,7 @@ [p ensure-pyobj]) -(def-pylib-fn PyDict_Next +(defn PyDict_Next "Iterate over all key-value pairs in the dictionary p. The Py_ssize_t referred to by ppos must be initialized to 0 prior to the first call to this function to start the iteration; the function returns true for each pair in the dictionary, and false once @@ -190,11 +190,11 @@ may be NULL. Any references returned through them are borrowed. ppos should not be altered during iteration. Its value represents offsets within the internal dictionary structure, and since the structure is sparse, the offsets are not consecutive." - Integer - [p ensure-pyobj] - [ppos (partial jna/ensure-type jna/size-t-ref-type)] - [pkey jna/ensure-ptr-ptr] - [pvalue jna/ensure-ptr-ptr]) + ^long [p ppos pkey pvalue] + (long (DirectMapped/PyDict_Next (ensure-pyobj p) + (jna/ensure-type jna/size-t-ref-type ppos) + (jna/ensure-ptr-ptr pkey) + (jna/ensure-ptr-ptr pvalue)))) (def-pylib-fn PyDict_Merge diff --git a/src/libpython_clj/jna/concrete/err.clj b/src/libpython_clj/jna/concrete/err.clj index 27bdc18..7df5a24 100644 --- a/src/libpython_clj/jna/concrete/err.clj +++ b/src/libpython_clj/jna/concrete/err.clj @@ -10,12 +10,13 @@ :as libpy-base] [tech.jna :as jna]) (:import [com.sun.jna Pointer] - [libpython_clj.jna PyObject PyTypeObject])) + [libpython_clj.jna PyObject PyTypeObject DirectMapped])) -(def-pylib-fn PyErr_Occurred +(defn PyErr_Occurred "Check if the error indicator is set. If so, return the exception type." - Pointer) + ^Pointer [] + (DirectMapped/PyErr_Occurred)) (def-pylib-fn PyErr_Clear diff --git a/src/libpython_clj/jna/concrete/numeric/float.clj b/src/libpython_clj/jna/concrete/numeric/float.clj index 56eccc8..dee3276 100644 --- a/src/libpython_clj/jna/concrete/numeric/float.clj +++ b/src/libpython_clj/jna/concrete/numeric/float.clj @@ -8,7 +8,7 @@ :as libpy-base] [tech.jna :as jna]) (:import [com.sun.jna Pointer] - [libpython_clj.jna PyObject])) + [libpython_clj.jna PyObject DirectMapped])) (def-pylib-fn PyFloat_Check @@ -31,22 +31,21 @@ [str ensure-pyobj]) -(def-pylib-fn PyFloat_FromDouble +(defn PyFloat_FromDouble "Return value: New reference. Create a PyFloatObject object from v, or NULL on failure." - Pointer - [v double]) + ^Pointer [v] + (DirectMapped/PyFloat_FromDouble (double v))) -(def-pylib-fn PyFloat_AsDouble +(defn PyFloat_AsDouble "Return a C double representation of the contents of pyfloat. If pyfloat is not a Python floating point object but has a __float__() method, this method will first be called to convert pyfloat into a float. This method returns -1.0 upon failure, so one should call PyErr_Occurred() to check for errors." - Double - [v ensure-pyobj]) - + ^double [v] + (DirectMapped/PyFloat_AsDouble (ensure-pyobj v))) (def-pylib-fn PyFloat_GetInfo diff --git a/src/libpython_clj/jna/concrete/numeric/integer.clj b/src/libpython_clj/jna/concrete/numeric/integer.clj index c04e60b..3c8af96 100644 --- a/src/libpython_clj/jna/concrete/numeric/integer.clj +++ b/src/libpython_clj/jna/concrete/numeric/integer.clj @@ -8,7 +8,7 @@ :as libpy-base] [tech.jna :as jna]) (:import [com.sun.jna Pointer] - [libpython_clj.jna PyObject])) + [libpython_clj.jna PyObject DirectMapped])) (def-pylib-fn PyLong_Check @@ -53,12 +53,13 @@ [v jna/size-t]) -(def-pylib-fn PyLong_FromLongLong +(defn PyLong_FromLongLong "Return value: New reference. Return a new PyLongObject object from a C long long, or NULL on failure." - Pointer - [v long]) + ^Pointer [v] + (DirectMapped/PyLong_FromLongLong (long v))) + (def-pylib-fn PyLong_FromUnsignedLongLong "Return value: New reference. @@ -87,7 +88,7 @@ [obj ensure-pyobj]) -(def-pylib-fn PyLong_AsLongLong +(defn PyLong_AsLongLong "Return a C long long representation of obj. If obj is not an instance of PyLongObject, first call its __int__() method (if present) to convert it to a PyLongObject. @@ -95,5 +96,5 @@ Raise OverflowError if the value of obj is out of range for a long. Returns -1 on error. Use PyErr_Occurred() to disambiguate." - Long - [obj ensure-pyobj]) + ^long [obj] + (DirectMapped/PyLong_AsLongLong (ensure-pyobj obj))) diff --git a/src/libpython_clj/jna/concrete/tuple.clj b/src/libpython_clj/jna/concrete/tuple.clj index f129e06..df726a3 100644 --- a/src/libpython_clj/jna/concrete/tuple.clj +++ b/src/libpython_clj/jna/concrete/tuple.clj @@ -9,7 +9,7 @@ :as libpy-base] [tech.jna :as jna]) (:import [com.sun.jna Pointer] - [libpython_clj.jna PyObject])) + [libpython_clj.jna PyObject DirectMapped])) (def-pylib-fn PyTuple_Check @@ -18,22 +18,21 @@ [p ensure-pyobj]) -(def-pylib-fn PyTuple_New +(defn PyTuple_New "Return value: New reference. Return a new tuple object of size len, or NULL on failure." - Pointer - [len jna/size-t]) + ^Pointer [^long len] + (DirectMapped/PyTuple_New (long len))) -(def-pylib-fn PyTuple_GetItem +(defn PyTuple_GetItem "Return value: Borrowed reference. Return the object at position pos in the tuple pointed to by p. If pos is out of bounds, return NULL and sets an IndexError exception." - Pointer - [p ensure-pyobj] - [pos jna/size-t]) + ^Pointer [p pos] + (DirectMapped/PyTuple_GetItem (ensure-pyobj p) (long pos))) (def-pylib-fn PyTuple_GetSlice @@ -47,14 +46,14 @@ [high jna/size-t]) -(def-pylib-fn PyTuple_SetItem +(defn PyTuple_SetItem "Insert a reference to object o at position pos of the tuple pointed to by p. Return 0 on success. Note This function “steals” a reference to o" - Integer - [p ensure-pyobj] - [pos jna/size-t] - [o ensure-pyobj]) + ^long [p pos o] + (long (DirectMapped/PyTuple_SetItem (ensure-pyobj p) + (long pos) + (ensure-pyobj o)))) diff --git a/src/libpython_clj/jna/concrete/unicode.clj b/src/libpython_clj/jna/concrete/unicode.clj index af2c863..7a22d1f 100644 --- a/src/libpython_clj/jna/concrete/unicode.clj +++ b/src/libpython_clj/jna/concrete/unicode.clj @@ -12,11 +12,11 @@ (:import [com.sun.jna Pointer] [com.sun.jna.ptr PointerByReference LongByReference IntByReference] - [libpython_clj.jna PyObject])) + [libpython_clj.jna PyObject DirectMapped])) -(def-pylib-fn PyUnicode_Decode +(defn PyUnicode_Decode "Return value: New reference. Create a Unicode object by decoding size bytes of the encoded string s. encoding and @@ -26,11 +26,11 @@ Signature: PyObject* (const char *s, Py_ssize_t size, const char *encoding, const char *errors)" - Pointer - [s dtype/as-nio-buffer] - [size jna/size-t] - [encoding str] - [errors str]) + ^Pointer [s size encoding errors] + (DirectMapped/PyUnicode_Decode (dtype/as-nio-buffer s) + (jna/size-t size) + (str encoding) + (str errors))) (def-pylib-fn PyUnicode_AsEncodedString @@ -55,7 +55,7 @@ IntByReference)) -(def-pylib-fn PyUnicode_AsUTF8AndSize +(defn PyUnicode_AsUTF8AndSize "Return a pointer to the UTF-8 encoding of the Unicode object, and store the size of the encoded representation (in bytes) in size. The size argument can be NULL; in this case no size will be stored. The returned buffer always has an extra null byte @@ -71,9 +71,9 @@ New in version 3.3. Changed in version 3.7: The return type is now const char * rather of char *." - Pointer - [py-obj ensure-pyobj] - [size-ptr (partial jna/ensure-type (size-t-by-reference-type))]) + ^Pointer [py-obj size-ptr] + (DirectMapped/PyUnicode_AsUTF8AndSize (ensure-pyobj py-obj) + ^LongByReference size-ptr)) (def-pylib-fn PyUnicode_AsUTF8 diff --git a/src/libpython_clj/jna/interpreter.clj b/src/libpython_clj/jna/interpreter.clj index b2ce582..ae7b437 100644 --- a/src/libpython_clj/jna/interpreter.clj +++ b/src/libpython_clj/jna/interpreter.clj @@ -17,7 +17,7 @@ [clojure.tools.logging :as log]) (:import [com.sun.jna Pointer Native NativeLibrary] [com.sun.jna.ptr PointerByReference] - [libpython_clj.jna PyObject])) + [libpython_clj.jna PyObject DirectMapped])) @@ -264,7 +264,7 @@ ;;Acquire the GIL of the given thread state -(def-no-gil-pylib-fn PyEval_RestoreThread +(defn PyEval_RestoreThread "Acquire the global interpreter lock (if it has been created and thread support is enabled) and set the thread state to tstate, which must not be NULL. If the lock has been created, the current thread must not have acquired it, otherwise deadlock ensues. @@ -275,17 +275,18 @@ thread, even if the thread was not created by Python. You can use _Py_IsFinalizing() or sys.is_finalizing() to check if the interpreter is in process of being finalized before calling this function to avoid unwanted termination." - nil - [tstate ensure-pyobj]) + [tstate] + (DirectMapped/PyEval_RestoreThread (ensure-pyobj tstate))) ;;Release the GIL, return thread state -(def-pylib-fn PyEval_SaveThread +(defn PyEval_SaveThread "Release the global interpreter lock (if it has been created and thread support is enabled) and reset the thread state to NULL, returning the previous thread state (which is not NULL). If the lock has been created, the current thread must have acquired it." - Pointer) + ^Pointer [] + (DirectMapped/PyEval_SaveThread)) ;;Get current thread state for interpreter diff --git a/src/libpython_clj/jna/protocols/object.clj b/src/libpython_clj/jna/protocols/object.clj index a4cf4c5..061b949 100644 --- a/src/libpython_clj/jna/protocols/object.clj +++ b/src/libpython_clj/jna/protocols/object.clj @@ -110,16 +110,17 @@ [attr-name ensure-pyobj]) -(def-pylib-fn PyObject_HasAttrString +(defn PyObject_HasAttrString "Returns 1 if o has the attribute attr_name, and 0 otherwise. This is equivalent to the Python expression hasattr(o, attr_name). This function always succeeds. Note that exceptions which occur while calling __getattr__() and __getattribute__() methods and creating a temporary string object will get suppressed. To get error - reporting use Pointer_GetAttrString() instead." - Integer - [pyobj ensure-pyobj] - [attr-name str]) + reporting use Object_GetAttrString() instead." + ^long [pyobj attr-name] + (long + (DirectMapped/PyObject_HasAttrString (ensure-pyobj pyobj) + (str attr-name)))) (def-pylib-fn PyObject_GetAttr @@ -133,15 +134,15 @@ [attr-name ensure-pyobj]) -(def-pylib-fn PyObject_GetAttrString +(defn PyObject_GetAttrString "Return value: New reference. Retrieve an attribute named attr_name from object o. Returns the attribute value on success, or NULL on failure. This is the equivalent of the Python expression o.attr_name." - Pointer - [pyobj ensure-pyobj] - [attr-name str]) + ^Pointer [pyobj attr-name] + (DirectMapped/PyObject_GetAttrString (ensure-pyobj pyobj) + (str attr-name))) (def-pylib-fn PyObject_GenericGetAttr @@ -267,14 +268,14 @@ [opid bool-fn-constant]) -(def-pylib-fn PyCallable_Check +(defn PyCallable_Check "Determine if the object o is callable. Return 1 if the object is callable and 0 otherwise. This function always succeeds." - Integer - [pyobj ensure-pyobj]) + ^long [pyobj] + (long (DirectMapped/PyCallable_Check (ensure-pyobj pyobj)))) -(def-pylib-fn PyObject_Call +(defn PyObject_Call "Return value: New reference. Call a callable Python object callable, with arguments given by the tuple args, and @@ -286,12 +287,12 @@ Returns the result of the call on success, or NULL on failure. This is the equivalent of the Python expression: callable(*args, **kwargs)." - Pointer - [callable ensure-pyobj] - [args ensure-pytuple] - [kwargs as-pyobj]) + ^Pointer [callable args kwargs] + (DirectMapped/PyObject_Call (ensure-pyobj callable) + (ensure-pytuple args) + (as-pyobj kwargs))) -(def-pylib-fn PyObject_CallObject +(defn PyObject_CallObject "Return value: New reference. Call a callable Python object callable, with arguments given by the tuple args. If no @@ -300,9 +301,9 @@ Returns the result of the call on success, or NULL on failure. This is the equivalent of the Python expression: callable(*args)." - Pointer - [callable ensure-pyobj] - [args as-pyobj]) + ^Pointer [callable args] + (DirectMapped/PyObject_CallObject (ensure-pyobj callable) + (as-pyobj args))) (def-pylib-fn PyObject_Hash diff --git a/src/libpython_clj/jna/protocols/sequence.clj b/src/libpython_clj/jna/protocols/sequence.clj index 7e9bc76..b3d67b9 100644 --- a/src/libpython_clj/jna/protocols/sequence.clj +++ b/src/libpython_clj/jna/protocols/sequence.clj @@ -9,24 +9,24 @@ :as libpy-base] [tech.jna :as jna]) (:import [com.sun.jna Pointer] - [libpython_clj.jna PyObject])) + [libpython_clj.jna PyObject DirectMapped])) -(def-pylib-fn PySequence_Check +(defn PySequence_Check "Return 1 if the object provides sequence protocol, and 0 otherwise. Note that it returns 1 for Python classes with a __getitem__() method unless they are dict subclasses since in general case it is impossible to determine what the type of keys it supports. This function always succeeds." - Integer - [o ensure-pyobj]) + ^long [o] + (long (DirectMapped/PySequence_Check (ensure-pyobj o)))) -(def-pylib-fn PySequence_Length +(defn PySequence_Length "Returns the number of objects in sequence o on success, and -1 on failure. This is equivalent to the Python expression len(o)." - size-t-type - [o ensure-pyobj]) + ^long [o] + (long (DirectMapped/PySequence_Length (ensure-pyobj o)))) (def-pylib-fn PySequence_Concat @@ -71,14 +71,13 @@ [count jna/size-t]) -(def-pylib-fn PySequence_GetItem +(defn PySequence_GetItem "Return value: New reference. Return the ith element of o, or NULL on failure. This is the equivalent of the Python expression o[i]." - Pointer - [o ensure-pyobj] - [i jna/size-t]) + ^Pointer [o i] + (DirectMapped/PySequence_GetItem (ensure-pyobj o) (jna/size-t i))) (def-pylib-fn PySequence_GetSlice diff --git a/src/libpython_clj/python/interop.clj b/src/libpython_clj/python/interop.clj index e1a9efc..3cad57f 100644 --- a/src/libpython_clj/python/interop.clj +++ b/src/libpython_clj/python/interop.clj @@ -220,8 +220,8 @@ docstring-ptr (when docstring (jna/string->ptr-untracked docstring)) type-name-ptr (jna/string->ptr-untracked (str module-name "." type-name)) tp_flags (long (or tp_flags - (bit-or @libpy/Py_TPFLAGS_DEFAULT - @libpy/Py_TPFLAGS_BASETYPE))) + (bit-or libpy/Py_TPFLAGS_DEFAULT + libpy/Py_TPFLAGS_BASETYPE))) ;;We allocate our memory manually here else the system will gc the ;;type object memory when the type goes out of scope. new-mem (jna/malloc-untracked type-obj-size) diff --git a/src/libpython_clj/python/object.clj b/src/libpython_clj/python/object.clj index 8ca7428..19016d1 100644 --- a/src/libpython_clj/python/object.clj +++ b/src/libpython_clj/python/object.clj @@ -222,8 +222,9 @@ Object's refcount is bad. Crash is imminent" (defn py-raw-type ^Pointer [pyobj] - (let [pyobj (PyObject. (libpy/as-pyobj pyobj))] - (.ob_type pyobj))) + (let [^Pointer pyobj (libpy/as-pyobj pyobj)] + (jna/size-t-compile-time-switch + (.getPointer pyobj 4) (.getPointer pyobj 8)))) (extend-protocol py-proto/PPythonType @@ -401,13 +402,13 @@ Object's refcount is bad. Crash is imminent" %s" (type function))))) (let [meth-flags (long (cond (instance? CFunction$NoArgFunction function) - @libpy/METH_NOARGS + libpy/METH_NOARGS (instance? CFunction$TupleFunction function) - @libpy/METH_VARARGS + libpy/METH_VARARGS (instance? CFunction$KeyWordFunction function) - (bit-or @libpy/METH_KEYWORDS @libpy/METH_VARARGS) + (bit-or libpy/METH_KEYWORDS libpy/METH_VARARGS) :else (throw (ex-info (format "Failed due to type: %s" (type function)) From ad8a1407377b3a77a60443214ea808b70afb6dbd Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Thu, 23 Jan 2020 09:22:16 -0700 Subject: [PATCH 150/456] fix for #50 and added gc! call --- src/libpython_clj/python.clj | 12 +++- src/libpython_clj/python/bridge.clj | 87 ++++++++++++++++++++++++++++- src/libpython_clj/python/object.clj | 4 +- test/libpython_clj/python_test.clj | 21 ++++++- 4 files changed, 118 insertions(+), 6 deletions(-) diff --git a/src/libpython_clj/python.clj b/src/libpython_clj/python.clj index cb10838..b28de7f 100644 --- a/src/libpython_clj/python.clj +++ b/src/libpython_clj/python.clj @@ -333,6 +333,14 @@ (catch Throwable e# (with-exit-error-handler ~varname e#)))))))) + +(defn gc! + "Run the system garbage collection facility and then call the cooperative + 'cleanup python objects' queue" + [] + (System/gc) + (with-gil)) + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -476,12 +484,12 @@ (clojure.string/starts-with? symbol-str "**") (list* #'py** x (symbol (subs symbol-str 2 (count symbol-str))) args) - + (clojure.string/starts-with? symbol-str "*") (list* #'py* x (symbol (subs symbol-str 1 (count symbol-str))) args) :else ;; assumed to be method invocation - + (list* (into (vector #'py. x instance-member) args)))) (handle-pydotdot x (list form)))) ([x form & more] diff --git a/src/libpython_clj/python/bridge.clj b/src/libpython_clj/python/bridge.clj index bb3234a..030224e 100644 --- a/src/libpython_clj/python/bridge.clj +++ b/src/libpython_clj/python/bridge.clj @@ -665,11 +665,96 @@ (with-gil (cfn this arg0 arg1 arg2 arg3))) - (invoke [this arg0 arg1 arg2 arg3 arg4] (with-gil (cfn this arg0 arg1 arg2 arg3 arg4))) + (invoke [this arg0 arg1 arg2 arg3 arg4 arg5] + (with-gil + (cfn this arg0 arg1 arg2 arg3 arg4 arg5))) + + (invoke [this arg0 arg1 arg2 arg3 arg4 arg5 arg6] + (with-gil + (cfn this arg0 arg1 arg2 arg3 arg4 arg5 arg6))) + + (invoke [this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7] + (with-gil + (cfn this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7))) + + (invoke [this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8] + (with-gil + (cfn this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8))) + + (invoke [this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9] + (with-gil + (cfn this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9))) + + (invoke [this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10] + (with-gil + (cfn this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10))) + + (invoke [this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 + arg11] + (with-gil + (cfn this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 + arg11))) + + (invoke [this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 + arg11 arg12] + (with-gil + (cfn this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 + arg11 arg12))) + + (invoke [this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 + arg11 arg12 arg13] + (with-gil + (cfn this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 + arg11 arg12 arg13))) + + (invoke [this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 + arg11 arg12 arg13 arg14] + (with-gil + (cfn this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 + arg11 arg12 arg13 arg14))) + + (invoke [this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 + arg11 arg12 arg13 arg14 arg15] + (with-gil + (cfn this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 + arg11 arg12 arg13 arg14 arg15))) + + (invoke [this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 + arg11 arg12 arg13 arg14 arg15 arg16] + (with-gil + (cfn this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 + arg11 arg12 arg13 arg14 arg15 arg16))) + + (invoke [this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 + arg11 arg12 arg13 arg14 arg15 arg16 arg17] + (with-gil + (cfn this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 + arg11 arg12 arg13 arg14 arg15 arg16 arg17))) + + (invoke [this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 + arg11 arg12 arg13 arg14 arg15 arg16 arg17 arg18] + (with-gil + (cfn this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 + arg11 arg12 arg13 arg14 arg15 arg16 arg17 arg18))) + + (invoke [this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 + arg11 arg12 arg13 arg14 arg15 arg16 arg17 arg18 arg19] + (with-gil + (cfn this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 + arg11 arg12 arg13 arg14 arg15 arg16 arg17 arg18 arg19))) + + (invoke [this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 + arg11 arg12 arg13 arg14 arg15 arg16 arg17 arg18 arg19 arg20-obj-array] + (with-gil + (apply + cfn this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 + arg11 arg12 arg13 arg14 arg15 arg16 arg17 arg18 arg19 + arg20-obj-array))) + (applyTo [this arglist] (with-gil (apply cfn this arglist))) diff --git a/src/libpython_clj/python/object.clj b/src/libpython_clj/python/object.clj index 19016d1..5495ec8 100644 --- a/src/libpython_clj/python/object.clj +++ b/src/libpython_clj/python/object.clj @@ -159,8 +159,8 @@ Object's refcount is bad. Crash is imminent" (println (format "releasing object - 0x%x:%4d:%s" pyobj-value refcount - (name (python-type pyobj)))))))) - (libpy/Py_DecRef pyobj) + (name (python-type pyobj))))))) + (libpy/Py_DecRef pyobj)) (catch Throwable e (log/error e "Exception while releasing object")))) (when-not skip-check-error? (check-error-throw)) diff --git a/test/libpython_clj/python_test.clj b/test/libpython_clj/python_test.clj index a2de61b..3d9a499 100644 --- a/test/libpython_clj/python_test.clj +++ b/test/libpython_clj/python_test.clj @@ -252,7 +252,7 @@ (str (-> (py/get-attr np "random") (py/get-attr "shuffle")))))) - + (let [builtins (py/import-module "builtins") l (py/call-attr builtins "list")] (is (= (py/py. l __len__) 0)) @@ -409,3 +409,22 @@ class Foo: (deftest characters (is (= (py/->jvm (py/->python "c")) (py/->jvm (py/->python \c))))) + + +(comment + (require '[libpython-clj.require :refer [require-python]]) + + (require-python '[pandas :as pd]) + (require-python '[plotly.express :as px]) + (def px (py/import-module "plotly.express")) + + (let [data (doto (pd/DataFrame {:index [1 2] :value [2 3] :variable [1 1]}) + (py. melt :id_vars "index"))] + (py. px line :data_frame data :x "index" :y "value" :color "variable")) + + + (let [data (doto (pd/DataFrame {:index [1 2] :value [2 3] :variable [1 1]}) + (py. melt :id_vars "index"))] + ((py.- px line) :data_frame data :x "index" :y "value" :color "variable")) + + ) From 95cdd6939d367453dedcfc77e9221309f89454d5 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Thu, 23 Jan 2020 10:08:34 -0700 Subject: [PATCH 151/456] edit --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 941dbf9..3b31194 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,8 @@ JNA libpython bindings to the tech ecosystem. * Finding the python libraries is done dynamically allowing one system to run on multiple versions of python. * REPL oriented design means fast, smooth, iterative development. +* Carin Meier has written excellent posts on [plotting](http://gigasquidsoftware.com/blog/2020/01/18/parens-for-pyplot/) and + [advanced text generation](http://gigasquidsoftware.com/blog/2020/01/10/hugging-face-gpt-with-clojure/). ## Vision From 3f5553e31a0d9f4a8b2f1b0b96e248154304ec0b Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Thu, 23 Jan 2020 10:22:13 -0700 Subject: [PATCH 152/456] Updated Changelog --- CHANGELOG.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b8685e2..f5cdfe1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,18 @@ # Time for a ChangeLog! ## 1.32 +* DecRef now happens cooperatively in python thread. We used to use separate threads + in order to do decrement the refcount on objects that aren't reachable any more. Now + it happens at the end of the `with-gil` macro and thus it is possible to have all + python access confined to a single thread if this is desired for stability. It is + also quite a bit faster as the GIL is captured once and all decrefs happen after + that. + +* Major performance and stability enhancements. + 1. Doubled down on single-interpreter design. This simplified some important aspects + and led to a bit of perf gain. + 2. Implemented JNA [DirectMapping](https://github.com/java-native-access/jna/blob/master/www/DirectMapping.md) for quite a few hotspots found via profiling some simple + examples. Lots of people helped out with this (John Collins, Tom Poole (joinr)). * Python executables can now be specified directly using the syntax ```clojure @@ -64,6 +76,14 @@ (py.. obj (**method kwargs)) (py.. obj (**method arg1 arg2 arg3 ... argN kwargs)) ``` + +### Bugs Fixed: + +* [attribute calls with argument given in map](https://github.com/cnuernber/libpython-clj/issues/46) +* [allow specification of python executable](https://github.com/cnuernber/libpython-clj/issues/52) +* [difference in calling conventions leads to strange behavior in pandas](https://github.com/cnuernber/libpython-clj/issues/50) with [screencast of fix](https://drive.google.com/file/d/1PTXzWqNaRAiIDDZWqkeffIK2KESRWSRh/view?usp=sharing) +* [Allow single threaded use of Python](https://github.com/cnuernber/libpython-clj/issues/48) +* [Simplify interpreter design for only one interpreter](https://github.com/cnuernber/libpython-clj/issues/47) ## 1.31 From 6c63efcc21a30838a0809bb0d3428f1220a058f2 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Thu, 23 Jan 2020 10:23:14 -0700 Subject: [PATCH 153/456] 1.32 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index d0bd0e2..58cb88a 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.32-SNAPSHOT" +(defproject cnuernber/libpython-clj "1.32" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" From a0765ed5299f4fe3aa0ad442ca5d9b9c29a2bbaa Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Thu, 23 Jan 2020 10:23:35 -0700 Subject: [PATCH 154/456] snap --- CHANGELOG.md | 60 +++++++++++++++++++++++++++------------------------- project.clj | 2 +- 2 files changed, 32 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5cdfe1..3682aed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Time for a ChangeLog! +## 1.33-SNAPSHOT + ## 1.32 * DecRef now happens cooperatively in python thread. We used to use separate threads in order to do decrement the refcount on objects that aren't reachable any more. Now @@ -7,7 +9,7 @@ python access confined to a single thread if this is desired for stability. It is also quite a bit faster as the GIL is captured once and all decrefs happen after that. - + * Major performance and stability enhancements. 1. Doubled down on single-interpreter design. This simplified some important aspects and led to a bit of perf gain. @@ -15,68 +17,68 @@ examples. Lots of people helped out with this (John Collins, Tom Poole (joinr)). * Python executables can now be specified directly using the syntax - ```clojure + ```clojure (py/initialize! :python-executable ) ``` where **executable** can be a system installation of Python such as `"python3"`, `"python3.7"`; it can also be a fully qualified path such as `"/usr/bin/python3.7"`; or any Python executable along your discoverable system path. - -* Python virtual environments can now be used instead of system + +* Python virtual environments can now be used instead of system installations! This has been tested on Linux/Ubuntu variants - with virtual environments installed with - ```bash + with virtual environments installed with + ```bash virtualenv -p $(which ) env ``` - and then invoked using - ```clojure + and then invoked using + ```clojure (py/initialize! :python-executable "/abs/path/to/env/bin/python") ``` - + Tested on Python 3.6.8 and Python 3.7. - + **WARNING**: This is suitable for casual hacking and exploratory - development -- however, at this time, we still strongly recommend + development -- however, at this time, we still strongly recommend using Docker and a system installation of Python in production environments. - + * **breaking change** (and remediation): `require-python` no longer - automatically binds the Python module to the Clojure the namespace + automatically binds the Python module to the Clojure the namespace symbol. If you wish to bind the module to the namespace symbol, you need to use the `:bind-ns` flag. Example: - - ```clojure + + ```clojure (require-python 'requests) ;;=> nil requests ;;=> throws Exception - + (require-python '[requests :bind-ns]) ;;=> nil - (py.. requests + (py.. requests (get "https://www.google.com) - -content + -content (decode "latin-1)) ;; works ``` * Python method helper syntax for programmatic passing of maps to satisfy `*args`, `**kwargs` situations on the `py.` family of - macros. Two new macros have been introduced to address this - - ```clojure + macros. Two new macros have been introduced to address this + + ```clojure (py* obj method args) (py* obj method args kwargs) (py** obj method kwargs) (py** obj method arg1 arg2 arg3 ... argN kwargs) ``` - and the `py..` syntax has been extended to accomodate these + and the `py..` syntax has been extended to accomodate these conventions as well. - - ```clojure + + ```clojure (py.. obj (*method args)) (py.. obj (*method args kwargs)) (py.. obj (**method kwargs)) (py.. obj (**method arg1 arg2 arg3 ... argN kwargs)) ``` - + ### Bugs Fixed: * [attribute calls with argument given in map](https://github.com/cnuernber/libpython-clj/issues/46) @@ -90,11 +92,11 @@ * Python objects are now datafy-able and nav-igable. `require-python` is now rebuilt using datafy. - + * `py.`, `py.-`, and `py..` added to the `libpython-clj` APIs to allow method/attribute access more consistent with idiomatic Clojure forms. - + ## 1.30 @@ -116,7 +118,7 @@ classes and some serious refactoring overall. * Most of the datatype libraries math operators supported by numpy objects (+,-,etc). * Numpy objects can be used in datatype library functions (like `copy`, `make-container`) and work in optimized ways. - + ```clojure libpython-clj.python.numpy-test> (def test-ary (py/$a np-mod array (->> (range 9) (partition 3) @@ -135,7 +137,7 @@ libpython-clj.python.numpy-test> (dfn/> test-ary 4) [False False True] [ True True True]] ``` - + #### Bugs Fixed * Support for java character <-> py string * Fixed potential crash related to use of delay mechanism and stack based gc. diff --git a/project.clj b/project.clj index 58cb88a..c067e42 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.32" +(defproject cnuernber/libpython-clj "1.33-SNAPSHOT" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" From 526eda23df9b756ad6f2ba90912725fa650b67ac Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Thu, 23 Jan 2020 13:42:55 -0700 Subject: [PATCH 155/456] Adding in PyGILState_Check --- java/libpython_clj/jna/DirectMapped.java | 1 + src/libpython_clj/jna.clj | 1 + src/libpython_clj/jna/interpreter.clj | 11 +++++++++++ src/libpython_clj/python/interpreter.clj | 22 +++++++++++++++------- 4 files changed, 28 insertions(+), 7 deletions(-) diff --git a/java/libpython_clj/jna/DirectMapped.java b/java/libpython_clj/jna/DirectMapped.java index d38ccd8..4439e9e 100644 --- a/java/libpython_clj/jna/DirectMapped.java +++ b/java/libpython_clj/jna/DirectMapped.java @@ -10,6 +10,7 @@ public class DirectMapped { public static native void Py_IncRef(Pointer ptr); public static native void Py_DecRef(Pointer ptr); + public static native int PyGILState_Check(); public static native void PyEval_RestoreThread(Pointer tstate); public static native Pointer PyEval_SaveThread(); public static native Pointer PyFloat_FromDouble(double y); diff --git a/src/libpython_clj/jna.clj b/src/libpython_clj/jna.clj index 9ae0c1e..0cf0803 100644 --- a/src/libpython_clj/jna.clj +++ b/src/libpython_clj/jna.clj @@ -49,6 +49,7 @@ Py_None Py_NotImplemented PyMem_Free + PyGILState_Check PyEval_RestoreThread PyEval_SaveThread PyThreadState_Get diff --git a/src/libpython_clj/jna/interpreter.clj b/src/libpython_clj/jna/interpreter.clj index ae7b437..5cbe405 100644 --- a/src/libpython_clj/jna/interpreter.clj +++ b/src/libpython_clj/jna/interpreter.clj @@ -263,6 +263,17 @@ [data-ptr jna/as-ptr]) +(defn PyGILState_Check + "Return 1 if the current thread is holding the GIL and 0 otherwise. This function + can be called from any thread at any time. Only if it has had its Python thread + state initialized and currently is holding the GIL will it return 1. This is + mainly a helper/diagnostic function. It can be useful for example in callback + contexts or memory allocation functions when knowing that the GIL is locked can + allow the caller to perform sensitive actions or otherwise behave differently." + ^long [] + (DirectMapped/PyGILState_Check)) + + ;;Acquire the GIL of the given thread state (defn PyEval_RestoreThread "Acquire the global interpreter lock (if it has been created and thread support is diff --git a/src/libpython_clj/python/interpreter.clj b/src/libpython_clj/python/interpreter.clj index 2c8d8a5..ec68f01 100644 --- a/src/libpython_clj/python/interpreter.clj +++ b/src/libpython_clj/python/interpreter.clj @@ -339,19 +339,27 @@ print(json.dumps( `(do (let [interp# (ensure-interpreter) ^AtomicLong bound-thread# libpy-base/gil-thread-id - thread-id# (libpy-base/current-thread-id)] + thread-id# (libpy-base/current-thread-id) + bound-thread-id# (.get bound-thread#)] (locking interp# (let [new-binding?# (if-not (= thread-id# (.get bound-thread#)) - (do - (acquire-gil! interp#) - true) - false)] + (if (== 1 (libpy/PyGILState_Check)) + (do + (.set bound-thread# thread-id#) + :external-entry) + (do + (acquire-gil! interp#) + :internal-entry)) + nil)] (try ~@body (finally (pygc/clear-reference-queue) - (when new-binding?# - (release-gil! interp#))))))))) + (cond + (= new-binding?# :internal-entry) + (release-gil! interp#) + (= new-binding?# :external-entry) + (.set bound-thread# bound-thread-id#))))))))) (defonce ^:dynamic *program-name* "") From 2a6b3d9ab2c491634c38c81c3fad7a8710e9778c Mon Sep 17 00:00:00 2001 From: Tom Date: Fri, 24 Jan 2020 11:09:00 -0600 Subject: [PATCH 156/456] Added autogenerated arities and macro (#63) bridge-callable-pyobject macro autogenerates IFn implementation if no IFn specs are defined.Otherwise, respects existing IFn implementationas per bridge-object. Macroexpansion will automatically generate all invoke method implementations, as well as applyTo. replaced brute-force expansion in generic-python-as-jvm call site with new macro. --- src/libpython_clj/python/bridge.clj | 159 ++++++++-------------------- 1 file changed, 43 insertions(+), 116 deletions(-) diff --git a/src/libpython_clj/python/bridge.clj b/src/libpython_clj/python/bridge.clj index 030224e..e882487 100644 --- a/src/libpython_clj/python/bridge.clj +++ b/src/libpython_clj/python/bridge.clj @@ -621,6 +621,47 @@ (py-proto/call-attr-kw item (key-sym-str->str attr) pos-args kw-args))) +;;utility fn to generate IFn arities +(defn- emit-args + [bodyf varf] + (let [argify (fn [n argfn bodyf] + (let [raw `[~'this ~@(map #(symbol (str "arg" %)) + (range n))]] + `~(bodyf (argfn raw))))] + (concat (for [i (range 21)] + (argify i identity bodyf)) + [(argify 21 (fn [xs] + `[~@(butlast xs) ~'arg20-obj-array]) + varf)]))) + +;;Python specific interop wrapper for IFn invocations. +(defn- emit-py-args [] + (emit-args (fn [args] `(~'invoke [~@args] + (~'with-gil + (~'cfn ~@args)))) + (fn [args] + `(~'invoke [~@args] + (~'with-gil + (~'apply ~'cfn ~@(butlast args) ~(last args))))))) + +(defmacro bridge-callable-pyobject + "Like bridge-pyobject, except it populates the implementation of IFn + for us, where all arg permutations are supplied, as well as applyTo, + and the function invocation is of the form + (invoke [this arg] (with-gil (cfn this arg))). + If caller supplies an implementation for clojure.lang.IFn or aliased + Fn, the macro will use that instead (allowing more control but + requiring caller to specify implementations for all desired arities)." + [pyobj interpreter & body] + (let [fn-specs (when-not (some #{'IFn 'clojure.lang.IFn} body) + `(~'IFn + ~@(emit-py-args) + (~'applyTo [~'this ~'arglist] + (~'with-gil + (~'apply ~'cfn ~'this ~'arglist)))))] + `(bridge-pyobject ~pyobj ~interpreter + ~@fn-specs + ~@body))) (defn generic-python-as-jvm "Given a generic pyobject, wrap it in a read-only map interface @@ -631,7 +672,7 @@ nil (let [interpreter (ensure-bound-interpreter)] (if (py-proto/callable? pyobj) - (bridge-pyobject + (bridge-callable-pyobject pyobj interpreter Iterable @@ -643,121 +684,7 @@ py-proto/PPyObjectBridgeToList (as-list [item] (generic-python-as-list pyobj)) - IFn - ;;uggh - (invoke [this] - (with-gil - (cfn this))) - - (invoke [this arg0] - (with-gil - (cfn this arg0))) - - (invoke [this arg0 arg1] - (with-gil - (cfn this arg0 arg1))) - - (invoke [this arg0 arg1 arg2] - (with-gil - (cfn this arg0 arg1 arg2))) - - (invoke [this arg0 arg1 arg2 arg3] - (with-gil - (cfn this arg0 arg1 arg2 arg3))) - - (invoke [this arg0 arg1 arg2 arg3 arg4] - (with-gil - (cfn this arg0 arg1 arg2 arg3 arg4))) - - (invoke [this arg0 arg1 arg2 arg3 arg4 arg5] - (with-gil - (cfn this arg0 arg1 arg2 arg3 arg4 arg5))) - - (invoke [this arg0 arg1 arg2 arg3 arg4 arg5 arg6] - (with-gil - (cfn this arg0 arg1 arg2 arg3 arg4 arg5 arg6))) - - (invoke [this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7] - (with-gil - (cfn this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7))) - - (invoke [this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8] - (with-gil - (cfn this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8))) - - (invoke [this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9] - (with-gil - (cfn this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9))) - - (invoke [this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10] - (with-gil - (cfn this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10))) - - (invoke [this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 - arg11] - (with-gil - (cfn this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 - arg11))) - - (invoke [this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 - arg11 arg12] - (with-gil - (cfn this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 - arg11 arg12))) - - (invoke [this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 - arg11 arg12 arg13] - (with-gil - (cfn this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 - arg11 arg12 arg13))) - - (invoke [this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 - arg11 arg12 arg13 arg14] - (with-gil - (cfn this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 - arg11 arg12 arg13 arg14))) - - (invoke [this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 - arg11 arg12 arg13 arg14 arg15] - (with-gil - (cfn this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 - arg11 arg12 arg13 arg14 arg15))) - - (invoke [this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 - arg11 arg12 arg13 arg14 arg15 arg16] - (with-gil - (cfn this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 - arg11 arg12 arg13 arg14 arg15 arg16))) - - (invoke [this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 - arg11 arg12 arg13 arg14 arg15 arg16 arg17] - (with-gil - (cfn this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 - arg11 arg12 arg13 arg14 arg15 arg16 arg17))) - - (invoke [this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 - arg11 arg12 arg13 arg14 arg15 arg16 arg17 arg18] - (with-gil - (cfn this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 - arg11 arg12 arg13 arg14 arg15 arg16 arg17 arg18))) - - (invoke [this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 - arg11 arg12 arg13 arg14 arg15 arg16 arg17 arg18 arg19] - (with-gil - (cfn this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 - arg11 arg12 arg13 arg14 arg15 arg16 arg17 arg18 arg19))) - - (invoke [this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 - arg11 arg12 arg13 arg14 arg15 arg16 arg17 arg18 arg19 arg20-obj-array] - (with-gil - (apply - cfn this arg0 arg1 arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 - arg11 arg12 arg13 arg14 arg15 arg16 arg17 arg18 arg19 - arg20-obj-array))) - - (applyTo [this arglist] - (with-gil - (apply cfn this arglist))) + ;;IFn is now supplied for us by the macro. ;;Mark this as executable Fn PyFunction From f3547cfd59111019010a88835796eedce19f0426 Mon Sep 17 00:00:00 2001 From: "J.J. Tolton" Date: Fri, 24 Jan 2020 12:09:26 -0500 Subject: [PATCH 157/456] ISSUE-60: Add support for prefix lists and fix require-python/require syntax conformity (#62) * prefix list support in require-python * remove TODOs, whitespace * better string test --- CHANGELOG.md | 32 +++++++ src/libpython_clj/require.clj | 97 +++++++++++++++++----- test/libpython_clj/require_python_test.clj | 38 +++++++-- 3 files changed, 142 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3682aed..597b25d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,38 @@ ## 1.33-SNAPSHOT +* **BREAKING CHANGE** `require-python` now respects prefix lists -- + unfortunately, the previous syntax was incorrect. + ```clojure + ;; WRONG (syntax version < 1.33) + (require-python '(os math)) + ``` + would be equivalent to + ```clojure + ;; (do (require-python 'os) (require-python 'math)) + ``` + the correct syntax for this SHOULD have been + ```clojure + (require-python 'os 'math) + ``` + + 1.33 fixes this mistake, and provides support for prefix lists, + for example: + + ```clojure + (require-python + '[builtins :as python] + '(builtins + [list :as python.list] + [dict :as python.dict] + [tuple :as python.tuple] + [set :as python.set] + [frozenset :as python.frozenset])) + ``` + (**Note**: this is done for you by the function `libpython-clj.require/import-python`) + + + ## 1.32 * DecRef now happens cooperatively in python thread. We used to use separate threads in order to do decrement the refcount on objects that aren't reachable any more. Now diff --git a/src/libpython_clj/require.clj b/src/libpython_clj/require.clj index 9773371..11399fc 100644 --- a/src/libpython_clj/require.clj +++ b/src/libpython_clj/require.clj @@ -139,6 +139,37 @@ (alias alias-name module-name)))) +(defn ^:private req-transform + ([req] + (if (symbol? req) + req + (let [base (first req) + reqs (rest req)] + (map (partial req-transform base) reqs)))) + ([prefix req] + (cond + (and (symbol? req) + (clojure.string/includes? (name req) ".") ) + (throw (Exception. + (str "After removing prefix list, requirements cannot " + "contain periods. Please remove periods from " req))) + (symbol? req) + (symbol (str prefix "." req)) + :else + (let [base (first req) + reqs (rest req) + alias? (reduce (fn [res [a b]] + (cond + (not b) nil + (= :as a) (reduced b) + :else nil)) + nil + (partition 2 1 reqs))] + (if alias? + (into [(symbol (str prefix "." base))] reqs) + (into [(req-transform prefix base)] reqs)))))) + + (defn require-python "## Basic usage ## @@ -148,12 +179,13 @@ (require-python '[math :as maaaath]) (maaaath/sin 1.0) ;;=> 0.8414709848078965 - - (require-python '(math csv)) - (require-python '([math :as pymath] csv)) - (require-python '([math :as pymath] [csv :as py-csv]) + + (require-python 'math 'csv) + (require-python '[math :as pymath] 'csv)) + (require-python '[math :as pymath] '[csv :as py-csv]) (require-python 'concurrent.futures) (require-python '[concurrent.futures :as fs]) + (require-python '(concurrent [futures :as fs])) (require-python '[requests :refer [get post]]) @@ -201,6 +233,22 @@ (os/chdir \"/path/to/foodir\") + ## prefix lists ## + + For convenience, if you are loading multiple Python modules + with the same prefix, you can use the following notation: + + (require-python '(a b c)) + is equivalent to + (require-python 'a.b 'a.c) + + (require-python '(foo [bar :as baz :refer [qux]])) + is equivalent to + (require-python '[foo.bar :as baz :refer [qux]]) + + (require-python '(foo [bar :as baz :refer [qux]] buster)) + (require-python '[foo.bar :as baz :refer [qux]] 'foo.buster)) + ## For library developers ## If you want to intern all symbols to your current namespace, @@ -213,27 +261,36 @@ you can do (require-python '[operators :refer :*])" - [reqs] + ([req] + (cond + (list? req) ;; prefix list + (let [prefix-lists (req-transform req)] + (doseq [req prefix-lists] (require-python req))) + (symbol? req) + (require-python (vector req)) + (vector? req) + (do-require-python req) + :else + (throw (Exception. "Invalid argument: %s" req))) + :ok) + ([req & reqs] + (require-python req) + (when (not-empty reqs) + (apply require-python reqs)) + :ok)) - (cond - (list? reqs) - (doseq [req reqs] (require-python req)) - (symbol? reqs) - (require-python (vector reqs)) - (vector? reqs) - (do-require-python reqs) - :else - (throw (Exception. "Invalid argument: %s" reqs)))) (defn import-python "Loads python, python.list, python.dict, python.set, python.tuple, and python.frozenset." [] (require-python - '([builtins :as python] - [builtins.list :as python.list] - [builtins.dict :as python.dict] - [builtins.set :as python.set] - [builtins.tuple :as python.tuple] - [builtins.frozenset :as python.frozenset]))) + '(builtins + [list :as python.list] + [dict :as python.dict] + [set :as python.set] + [tuple :as python.tuple] + [frozenset :as python.frozenset]) + '[builtins :as python]) + :ok) diff --git a/test/libpython_clj/require_python_test.clj b/test/libpython_clj/require_python_test.clj index 453b579..e2c5c85 100644 --- a/test/libpython_clj/require_python_test.clj +++ b/test/libpython_clj/require_python_test.clj @@ -12,7 +12,6 @@ :exclude [sin cos] :as pymath]) - (deftest base-require-test (let [publics (ns-publics (find-ns 'libpython-clj.require-python-test))] (is (not (contains? publics 'sin))) @@ -46,9 +45,8 @@ (is (= #{:b} (parse-flags #{:a :b} '[:b :a false]))) (is (= #{:b :a} (parse-flags #{:a :b} '[:b :a false :a true]))))) - -(require-python '([builtins :as python] - [builtins.list :as python.pylist])) +(require-python '[builtins :as python] + '[builtins.list :as python.pylist]) ;; NOTE -- even though builtins.list has been aliased to ;; to python.pylist, you are still required to require @@ -66,7 +64,6 @@ (python.pylist/clear l) (is (= (python/list) (vec l))))) - (require-python 'csv.DictWriter) (deftest require-python-classes @@ -75,3 +72,34 @@ (is csv.DictWriter/writeheader) (is csv.DictWriter/writerow) (is csv.DictWriter/writerows)) + +(deftest test-req-transform + (let [req-transform #'libpython-clj.require/req-transform] + + (is (= (req-transform 'a) 'a)) + + (is (= (req-transform '(a b c)) + '(a.b a.c))) + + (is (= (req-transform '(a [b :as c :refer [blah]])) + (list '[a.b :as c :refer [blah]]))) + + (is (= (req-transform '(a [b :as c :refer [x] :flagA] d)) + (list '[a.b :as c :refer [x] :flagA] 'a.d))) + + (is (= (req-transform 'a 'b) + 'a.b)) + + (is (= (req-transform 'a.b 'c) + 'a.b.c)) + + (is (thrown? Exception (req-transform 'a.b 'c.d))) + + (is (= (req-transform 'a '[b]) '[a.b])) + + (is (= (req-transform 'a '[b :as c :refer [blah] :flagA]) + '[a.b :as c :refer [blah] :flagA])))) + + +(deftest import-python-test + (is (= :ok (libpython-clj.require/import-python)))) From 09813762f58c87b9e2c41092b79dd28c08a8c59d Mon Sep 17 00:00:00 2001 From: fong Date: Sat, 25 Jan 2020 18:24:42 -0500 Subject: [PATCH 158/456] Fix Issue #43: (#64) * Change with-gil (acquire-gil! and release-gil!) to reentrant PyGILState api. * Remove unnecessary acquire-gil! in interpreter/finalize! * Remove unnecessary nil from with-wil in (defn generic-python-as-jvm) --- java/libpython_clj/jna/DirectMapped.java | 2 ++ src/libpython_clj/jna.clj | 2 ++ src/libpython_clj/jna/interpreter.clj | 29 +++++++++++++++++++ src/libpython_clj/python/bridge.clj | 2 +- src/libpython_clj/python/interpreter.clj | 37 +++++++++--------------- 5 files changed, 47 insertions(+), 25 deletions(-) diff --git a/java/libpython_clj/jna/DirectMapped.java b/java/libpython_clj/jna/DirectMapped.java index 4439e9e..718ab90 100644 --- a/java/libpython_clj/jna/DirectMapped.java +++ b/java/libpython_clj/jna/DirectMapped.java @@ -10,6 +10,8 @@ public class DirectMapped { public static native void Py_IncRef(Pointer ptr); public static native void Py_DecRef(Pointer ptr); + public static native int PyGILState_Ensure(); + public static native void PyGILState_Release(int gilState); public static native int PyGILState_Check(); public static native void PyEval_RestoreThread(Pointer tstate); public static native Pointer PyEval_SaveThread(); diff --git a/src/libpython_clj/jna.clj b/src/libpython_clj/jna.clj index 0cf0803..def6530 100644 --- a/src/libpython_clj/jna.clj +++ b/src/libpython_clj/jna.clj @@ -49,6 +49,8 @@ Py_None Py_NotImplemented PyMem_Free + PyGILState_Ensure + PyGILState_Release PyGILState_Check PyEval_RestoreThread PyEval_SaveThread diff --git a/src/libpython_clj/jna/interpreter.clj b/src/libpython_clj/jna/interpreter.clj index 5cbe405..9f58c3a 100644 --- a/src/libpython_clj/jna/interpreter.clj +++ b/src/libpython_clj/jna/interpreter.clj @@ -263,6 +263,35 @@ [data-ptr jna/as-ptr]) +(defn PyGILState_Ensure + "Ensure that the current thread is ready to call the Python C API regardless of + the current state of Python, or of the global interpreter lock. This may be called + as many times as desired by a thread as long as each call is matched with a call + to PyGILState_Release(). In general, other thread-related APIs may be used between + PyGILState_Ensure() and PyGILState_Release() calls as long as the thread state is + restored to its previous state before the Release(). For example, normal usage of + the Py_BEGIN_ALLOW_THREADS and Py_END_ALLOW_THREADS macros is acceptable. + + The return value is an opaque “handle” to the thread state when PyGILState_Ensure() + was called, and must be passed to PyGILState_Release() to ensure Python is left in + the same state. Even though recursive calls are allowed, these handles cannot be shared - + each unique call to PyGILState_Ensure() must save the handle for its call to PyGILState_Release(). + + When the function returns, the current thread will hold the GIL and be able to call + arbitrary Python code. Failure is a fatal error." + ^long [] + (DirectMapped/PyGILState_Ensure)) + +(defn PyGILState_Release + "Release any resources previously acquired. After this call, Python’s state will be + the same as it was prior to the corresponding PyGILState_Ensure() call (but generally + this state will be unknown to the caller, hence the use of the GILState API). + + Every call to PyGILState_Ensure() must be matched by a call to PyGILState_Release() + on the same thread." + [^long s] + (DirectMapped/PyGILState_Release s)) + (defn PyGILState_Check "Return 1 if the current thread is holding the GIL and 0 otherwise. This function can be called from any thread at any time. Only if it has had its Python thread diff --git a/src/libpython_clj/python/bridge.clj b/src/libpython_clj/python/bridge.clj index e882487..5668a1d 100644 --- a/src/libpython_clj/python/bridge.clj +++ b/src/libpython_clj/python/bridge.clj @@ -667,7 +667,7 @@ "Given a generic pyobject, wrap it in a read-only map interface where the keys are the attributes." [pyobj] - (with-gil nil + (with-gil (if (= :none-type (python-type pyobj)) nil (let [interpreter (ensure-bound-interpreter)] diff --git a/src/libpython_clj/python/interpreter.clj b/src/libpython_clj/python/interpreter.clj index ec68f01..5a5309c 100644 --- a/src/libpython_clj/python/interpreter.clj +++ b/src/libpython_clj/python/interpreter.clj @@ -268,18 +268,17 @@ print(json.dumps( (defn release-gil! - "non-reentrant pathway to release the gil. It must not be held by this thread." - [interpreter] - (let [thread-state (libpy/PyEval_SaveThread)] - (libpy-base/set-gil-thread-id! (libpy-base/current-thread-id) Long/MAX_VALUE) - (assoc @(:interpreter-state* interpreter) :thread-state thread-state))) + "Reentrant pathway to release the gil. It must not be held by this thread." + [git-state] + (libpy-base/set-gil-thread-id! (libpy-base/current-thread-id) Long/MAX_VALUE) + (libpy/PyGILState_Release git-state)) (defn acquire-gil! - "Non-reentrant pathway to acquire gil. It must not be held by this thread." - [interpreter] - (libpy/PyEval_RestoreThread (python-thread-state interpreter)) - (libpy-base/set-gil-thread-id! Long/MAX_VALUE (libpy-base/current-thread-id))) + "Reentrant pathway to acquire gil. It must not be held by this thread." + [] + (libpy-base/set-gil-thread-id! Long/MAX_VALUE (libpy-base/current-thread-id)) + (libpy/PyGILState_Ensure)) (defn swap-interpreters! @@ -342,24 +341,15 @@ print(json.dumps( thread-id# (libpy-base/current-thread-id) bound-thread-id# (.get bound-thread#)] (locking interp# - (let [new-binding?# (if-not (= thread-id# (.get bound-thread#)) - (if (== 1 (libpy/PyGILState_Check)) - (do - (.set bound-thread# thread-id#) - :external-entry) - (do - (acquire-gil! interp#) - :internal-entry)) - nil)] + (let [gil-state# (if-not (= thread-id# (.get bound-thread#)) + (acquire-gil!) + nil)] (try ~@body (finally (pygc/clear-reference-queue) - (cond - (= new-binding?# :internal-entry) - (release-gil! interp#) - (= new-binding?# :external-entry) - (.set bound-thread# bound-thread-id#))))))))) + (when-not (nil? gil-state#) + (release-gil! gil-state#))))))))) (defonce ^:dynamic *program-name* "") @@ -500,7 +490,6 @@ print(json.dumps( (locking interp (check-error-throw) (log-info "executing python finalize!") - (acquire-gil! interp) (let [finalize-value (libpy/Py_FinalizeEx)] (when-not (= 0 finalize-value) (log-error (format "PyFinalize returned nonzero value: %s" finalize-value))))))) From de2af0f86ca93138c47bcdeffecdb624da14a9ef Mon Sep 17 00:00:00 2001 From: fong Date: Sun, 26 Jan 2020 17:13:07 -0500 Subject: [PATCH 159/456] make release-gil! exception safe (#65) --- src/libpython_clj/python/interpreter.clj | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libpython_clj/python/interpreter.clj b/src/libpython_clj/python/interpreter.clj index 5a5309c..5a3323d 100644 --- a/src/libpython_clj/python/interpreter.clj +++ b/src/libpython_clj/python/interpreter.clj @@ -268,14 +268,14 @@ print(json.dumps( (defn release-gil! - "Reentrant pathway to release the gil. It must not be held by this thread." - [git-state] - (libpy-base/set-gil-thread-id! (libpy-base/current-thread-id) Long/MAX_VALUE) - (libpy/PyGILState_Release git-state)) + "Reentrant pathway to release the gil." + [gil-state] + (libpy/PyGILState_Release gil-state) + (libpy-base/set-gil-thread-id! (libpy-base/current-thread-id) Long/MAX_VALUE)) (defn acquire-gil! - "Reentrant pathway to acquire gil. It must not be held by this thread." + "Reentrant pathway to acquire gil." [] (libpy-base/set-gil-thread-id! Long/MAX_VALUE (libpy-base/current-thread-id)) (libpy/PyGILState_Ensure)) From 5be0e7ab4f26e663864c0530e6a7560fcfe21f8b Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Mon, 27 Jan 2020 11:26:36 -0700 Subject: [PATCH 160/456] Cleaning up gil state managment a slight bit. --- src/libpython_clj/jna/base.clj | 3 +-- src/libpython_clj/python/interpreter.clj | 26 ++++++++++++------------ 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/src/libpython_clj/jna/base.clj b/src/libpython_clj/jna/base.clj index ad77dea..49427f5 100644 --- a/src/libpython_clj/jna/base.clj +++ b/src/libpython_clj/jna/base.clj @@ -1,7 +1,6 @@ (ns libpython-clj.jna.base (:require [tech.jna :as jna] - [tech.jna.base :as jna-base] - [camel-snake-kebab.core :refer [->kebab-case]]) + [tech.jna.base :as jna-base]) (:import [com.sun.jna Pointer NativeLibrary] [libpython_clj.jna PyObject] [java.util.concurrent.atomic AtomicLong])) diff --git a/src/libpython_clj/python/interpreter.clj b/src/libpython_clj/python/interpreter.clj index 5a3323d..e1bc8e9 100644 --- a/src/libpython_clj/python/interpreter.clj +++ b/src/libpython_clj/python/interpreter.clj @@ -269,16 +269,17 @@ print(json.dumps( (defn release-gil! "Reentrant pathway to release the gil." - [gil-state] - (libpy/PyGILState_Release gil-state) - (libpy-base/set-gil-thread-id! (libpy-base/current-thread-id) Long/MAX_VALUE)) + [gil-state ^long original-thread-id] + (.set ^AtomicLong libpy-base/gil-thread-id original-thread-id) + (libpy/PyGILState_Release gil-state)) (defn acquire-gil! "Reentrant pathway to acquire gil." [] - (libpy-base/set-gil-thread-id! Long/MAX_VALUE (libpy-base/current-thread-id)) - (libpy/PyGILState_Ensure)) + (let [retval (libpy/PyGILState_Ensure)] + (.set ^AtomicLong libpy-base/gil-thread-id (libpy-base/current-thread-id)) + retval)) (defn swap-interpreters! @@ -333,23 +334,22 @@ print(json.dumps( (defmacro with-gil - "Grab the gil and use the main interpreter. Do not grab gil if already grabbed" + "Grab the gil and use the main interpreter using reentrant acquire-gil pathway." [& body] `(do (let [interp# (ensure-interpreter) ^AtomicLong bound-thread# libpy-base/gil-thread-id thread-id# (libpy-base/current-thread-id) - bound-thread-id# (.get bound-thread#)] + previously-bound-thread-id# (.get bound-thread#)] (locking interp# - (let [gil-state# (if-not (= thread-id# (.get bound-thread#)) - (acquire-gil!) - nil)] + (let [gil-state# (when-not (== thread-id# previously-bound-thread-id#) + (acquire-gil!))] (try ~@body (finally - (pygc/clear-reference-queue) - (when-not (nil? gil-state#) - (release-gil! gil-state#))))))))) + (when gil-state# + (pygc/clear-reference-queue) + (release-gil! gil-state# previously-bound-thread-id#))))))))) (defonce ^:dynamic *program-name* "") From e658148bca0fe0b4c291911d0fc7dadc0ba7a8e3 Mon Sep 17 00:00:00 2001 From: Oliver Rolle Date: Tue, 28 Jan 2020 16:07:50 +0100 Subject: [PATCH 161/456] Added support for anaconda python in windows (#67) --- docs/Usage.md | 13 ++++++- src/libpython_clj/python.clj | 10 ++++-- src/libpython_clj/python/interpreter.clj | 28 +++++++-------- src/libpython_clj/python/windows.clj | 43 ++++++++++++++++++++++++ 4 files changed, 76 insertions(+), 18 deletions(-) create mode 100644 src/libpython_clj/python/windows.clj diff --git a/docs/Usage.md b/docs/Usage.md index ce30b01..bc646e7 100644 --- a/docs/Usage.md +++ b/docs/Usage.md @@ -44,7 +44,7 @@ user> (require '[libpython-clj.python python-type]]) nil - +; Mac and Linux user> (initialize!) Jun 30, 2019 4:47:39 PM clojure.tools.logging$eval7369$fn__7372 invoke INFO: executing python initialize! @@ -53,6 +53,17 @@ INFO: Library python3.6m found at [:system "python3.6m"] Jun 30, 2019 4:47:39 PM clojure.tools.logging$eval7369$fn__7372 invoke INFO: Reference thread starting :ok + +; Windows with Anaconda +(initialize! ; Python executable + :python-executable "C:\\Users\\USER\\AppData\\Local\\Continuum\\anaconda3\\python.exe" + ; Python Library + :library-path "C:\\Users\\USER\\AppData\\Local\\Continuum\\anaconda3\\python37.dll" + ; Anacondas PATH environment to load native dlls of modules (numpy, etc.) + :windows-anaconda-activate-bat "C:\\Users\\USER\\AppData\\Local\\Continuum\\anaconda3\\Scripts\\activate.bat" + ) +... +:ok ``` This dynamically finds the python shared library and loads it using output from diff --git a/src/libpython_clj/python.clj b/src/libpython_clj/python.clj index b28de7f..6e113fb 100644 --- a/src/libpython_clj/python.clj +++ b/src/libpython_clj/python.clj @@ -4,6 +4,7 @@ [libpython-clj.python.interpreter :as pyinterp] [libpython-clj.python.object :as pyobj] [libpython-clj.python.bridge :as pybridge] + [libpython-clj.python.windows :as win] [libpython-clj.jna :as libpy] ;;Protocol implementations purely for nd-ness [libpython-clj.python.np-array] @@ -235,7 +236,8 @@ library-path python-home no-io-redirect? - python-executable]}] + python-executable + windows-anaconda-activate-bat]}] (when-not @pyinterp/main-interpreter* (pyinterp/initialize! :program-name program-name :library-path library-path @@ -245,14 +247,16 @@ (pyinterop/register-bridge-type!) (when-not no-io-redirect? (pyinterop/setup-std-writer #'*err* "stderr") - (pyinterop/setup-std-writer #'*out* "stdout"))) + (pyinterop/setup-std-writer #'*out* "stdout")) + (if-not (nil? windows-anaconda-activate-bat) + (win/setup-windows-conda! windows-anaconda-activate-bat))) :ok) (defn ptr-refcnt [item] (-> (libpy/as-pyobj item) - (libpython_clj.jna.PyObject. ) + (libpython_clj.jna.PyObject.) (.ob_refcnt))) diff --git a/src/libpython_clj/python/interpreter.clj b/src/libpython_clj/python/interpreter.clj index e1bc8e9..e4d7ca5 100644 --- a/src/libpython_clj/python/interpreter.clj +++ b/src/libpython_clj/python/interpreter.clj @@ -1,5 +1,5 @@ (ns libpython-clj.python.interpreter - (:require [libpython-clj.jna :as libpy ] + (:require [libpython-clj.jna :as libpy] [libpython-clj.jna.base :as libpy-base] [libpython-clj.python.gc :as pygc] [libpython-clj.python.logging @@ -30,13 +30,13 @@ (let [{:keys [out err exit]} (sh executable "-c" "import sys, json; print(json.dumps( -{\"platform\": sys.platform, - \"prefix\": sys.prefix, - \"base_prefix\": sys.base_prefix, - \"executable\": sys.executable, - \"base_exec_prefix\": sys.base_exec_prefix, - \"exec_prefix\": sys.exec_prefix, - \"version\": list(sys.version_info)[:3]}))")] +{'platform': sys.platform, + 'prefix': sys.prefix, + 'base_prefix': sys.base_prefix, + 'executable': sys.executable, + 'base_exec_prefix': sys.base_exec_prefix, + 'exec_prefix': sys.exec_prefix, + 'version': list(sys.version_info)[:3]}))")] (when (= 0 exit) (json/read-str out :key-fn keyword)))) @@ -93,7 +93,7 @@ print(json.dumps( ;; ..: mac and windows are for sys.platform :linux "libpython%s.%sm.so$" :mac "libpython%s.%sm.dylib$" - :windows "python%s.%sm.dll$") + :win32 "python%s%s.dll$") major minor)))) (defn python-library-paths @@ -131,9 +131,9 @@ print(json.dumps( (let [executable "python3.7" system-info (python-system-info executable) pyregex (python-library-regex system-info)] - (python-library-paths system-info pyregex)) + (python-library-paths system-info pyregex))) ;;=> ["/usr/lib/x86_64-linux-gnu/libpython3.7m.so" "/usr/lib/python3.7/config-3.7m-x86_64-linux-gnu/libpython3.7m.so"] - ) + (defn- ignore-shell-errors [& args] @@ -179,8 +179,8 @@ print(json.dumps( ;;get the type of that item if we have seen it before. (defrecord Interpreter [ interpreter-state* ;;Thread state, per interpreter - shared-state* ;;state shared among all interpreters - ]) + shared-state*]) ;;state shared among all interpreters + ;; Main interpreter booted up during initialize! @@ -411,7 +411,7 @@ print(json.dumps( python-executable] :as options}] (when-not (main-interpreter) - (log-info (str "Executing python initialize with options:" options) ) + (log-info (str "Executing python initialize with options:" options)) (let [{:keys [python-home libname java-library-path-addendum] :as startup-info} (detect-startup-info options) library-names (cond diff --git a/src/libpython_clj/python/windows.clj b/src/libpython_clj/python/windows.clj new file mode 100644 index 0000000..d9ad4bf --- /dev/null +++ b/src/libpython_clj/python/windows.clj @@ -0,0 +1,43 @@ +(ns libpython-clj.python.windows + (:require [libpython-clj.python.interop :refer [run-simple-string]] + [clojure.java.shell :refer [sh]] + [clojure.java.io :as io] + [clojure.string :as s])) + +(defn create-echo-path-bat! [] + "Creates temporary file to extract condas PATH environment variable" + (let [tmp (java.io.File/createTempFile "echo-path" ".bat")] + (spit tmp "echo %PATH%") + (.toString tmp))) + +(defn delete-echo-path-bat! [tmp] + "Deletes temporary file" + (io/delete-file tmp)) + +(defn- get-windows-anaconda-env-path [activate-bat echo-bat] + "Get anacondas windows PATH environment variable to load native dlls for numpy etc. like python with anaconda does." + (-> (sh "cmd.exe" "/K" (str activate-bat " & " echo-bat)) + :out + (s/split #"\r\n") + reverse + (nth 2))) + + +(defn- generate-python-set-env-path [path] + "Double quote windows path separator \\ -> \\\\" + (let [quoted (s/replace path "\\" "\\\\")] + (str + "import os;\n" + "path = '" quoted "';\n" + "os.environ['PATH'] = path;\n"))) + +(defn setup-windows-conda! [windows-conda-activate-bat] + "Setup python PATH environment variable like in anaconda to be able to load native dlls for numpy etc. like anaconda does." + (let [echo-bat (create-echo-path-bat!)] + (->> (get-windows-anaconda-env-path + windows-conda-activate-bat + echo-bat) + generate-python-set-env-path + run-simple-string) + (delete-echo-path-bat! echo-bat))) + From b0f853179d6b286fedef6e5cbbca8b503a5d0ab3 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 28 Jan 2020 10:47:35 -0700 Subject: [PATCH 162/456] 1.33 --- CHANGELOG.md | 11 ++++++++++- project.clj | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 597b25d..7b14bb4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,14 @@ # Time for a ChangeLog! -## 1.33-SNAPSHOT +## 1.33 + +* Better [windows anaconda support](https://github.com/cnuernber/libpython-clj/pull/67) + thanks to [orolle](https://github.com/orolle). + +* Moved to PyGILState* functions for GIL management. This mainly due to + [FongHou](https://github.com/cnuernber/libpython-clj/commits?author=FongHou) in + PRs [here](https://github.com/cnuernber/libpython-clj/pull/64) and + [here](https://github.com/cnuernber/libpython-clj/pull/65). * **BREAKING CHANGE** `require-python` now respects prefix lists -- unfortunately, the previous syntax was incorrect. @@ -32,6 +40,7 @@ ``` (**Note**: this is done for you by the function `libpython-clj.require/import-python`) + This fix brought to you by [jjtolten](https://github.com/jjtolton). ## 1.32 diff --git a/project.clj b/project.clj index c067e42..7128ac2 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.33-SNAPSHOT" +(defproject cnuernber/libpython-clj "1.33" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" From 3a8d65508c8f7a446a66363a7cce22c6d223082e Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 28 Jan 2020 10:47:50 -0700 Subject: [PATCH 163/456] snap --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b14bb4..9644ff3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Time for a ChangeLog! +## 1.34-SNAPSHOT + ## 1.33 * Better [windows anaconda support](https://github.com/cnuernber/libpython-clj/pull/67) From 7768da6d4f323ce57fdbb68a88d62751dd609d65 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 28 Jan 2020 11:37:03 -0700 Subject: [PATCH 164/456] Less maintenance. Gigasquid's examples are cleaner --- CHANGELOG.md | 2 + README.md | 30 ++++--- example/.gitignore | 3 - example/README.md | 36 -------- example/project.clj | 7 -- example/scripts/get-data.sh | 4 - example/src/keras_simple.clj | 97 --------------------- example/src/matplotlib.clj | 73 ---------------- example/src/mxnet_autograd.clj | 63 -------------- example/src/new_to_clojure.clj | 155 --------------------------------- example/src/walkthrough.clj | 55 ------------ 11 files changed, 19 insertions(+), 506 deletions(-) delete mode 100644 example/.gitignore delete mode 100644 example/README.md delete mode 100644 example/project.clj delete mode 100755 example/scripts/get-data.sh delete mode 100644 example/src/keras_simple.clj delete mode 100644 example/src/matplotlib.clj delete mode 100644 example/src/mxnet_autograd.clj delete mode 100644 example/src/new_to_clojure.clj delete mode 100644 example/src/walkthrough.clj diff --git a/CHANGELOG.md b/CHANGELOG.md index 9644ff3..24763ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## 1.34-SNAPSHOT +* [Examples are now done by gigasquid](https://github.com/gigasquid/libpython-clj-examples) + ## 1.33 * Better [windows anaconda support](https://github.com/cnuernber/libpython-clj/pull/67) diff --git a/README.md b/README.md index 3b31194..65e7f01 100644 --- a/README.md +++ b/README.md @@ -31,30 +31,34 @@ This code is a concrete example that generates an ```clojure (ns facial-rec.face-feature (:require [libpython-clj.require :refer [require-python]] - [libpython-clj.python :as py] + [libpython-clj.python :refer [py. py.. py.-] :as py] [tech.v2.datatype :as dtype])) -(require-python '(mxnet mxnet.ndarray mxnet.module mxnet.io mxnet.model)) + +(require-python 'mxnet + '(mxnet ndarray module io model)) (require-python 'cv2) (require-python '[numpy :as np]) + (defn load-model [& {:keys [model-path checkpoint] :or {model-path "models/recognition/model" checkpoint 0}}] (let [[sym arg-params aux-params] (mxnet.model/load_checkpoint model-path checkpoint) - all-layers (py/$a sym get_internals) + all-layers (py. sym get_internals) target-layer (py/get-item all-layers "fc1_output") - ;;TODO - We haven't overloaded enough of the IFn invoke methods for - ;;this to work without using call-kw - model (py/call-kw mxnet.module/Module [] {:symbol target-layer :context (mxnet/cpu) :label_names nil})] - (py/$a model bind :data_shapes [["data" [1 3 112 112]]]) - (py/$a model set_params arg-params aux-params) + model (mxnet.module/Module :symbol target-layer + :context (mxnet/cpu) + :label_names nil)] + (py. model bind :data_shapes [["data" [1 3 112 112]]]) + (py. model set_params arg-params aux-params) model)) (defonce model (load-model)) + (defn face->feature [img-path] (py/with-gil-stack-rc-context @@ -64,10 +68,10 @@ This code is a concrete example that generates an input-blob (np/expand_dims new-img :axis 0) data (mxnet.ndarray/array input-blob) batch (mxnet.io/DataBatch :data [data])] - (py/$a model forward batch :is_train false) - (-> (py/$a model get_outputs) + (py. model forward batch :is_train false) + (-> (py. model get_outputs) first - (py/$a asnumpy) + (py. asnumpy) (#(dtype/make-container :java-array :float32 %)))) (throw (Exception. (format "Failed to load img: %s" img-path)))))) ``` @@ -107,7 +111,7 @@ the `libpython3.Xm.so` shared library so for example if we are loading python 3.6 we look for `libpython3.6m.so` on Linux or `libpython3.6m.dylib` on the Mac. This pathway has allowed us support Conda albeit with some work. For examples -using Conda, check out the facial rec repository above or look into how we +using Conda, check out the facial rec repository above or look into how we [build](scripts/build-conda-docker) our test [docker containers](dockerfiles/CondaDockerfile). @@ -119,7 +123,7 @@ our test [docker containers](dockerfiles/CondaDockerfile). * [development discussion forum](https://clojurians.zulipchat.com/#narrow/stream/215609-libpython-clj-dev) * [design documentation](docs/design.md) * [scope and garbage collection docs](https://github.com/cnuernber/libpython-clj/blob/master/docs/scopes-and-gc.md) -* [examples](example/README.md) +* [examples](https://github.com/gigasquid/libpython-clj-examples) * [docker setup](https://github.com/scicloj/docker-hub) * [pandas bindings (!!)](https://github.com/alanmarazzi/panthera) * [nextjournal notebook](https://nextjournal.com/chrisn/fun-with-matplotlib) diff --git a/example/.gitignore b/example/.gitignore deleted file mode 100644 index fc98b99..0000000 --- a/example/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -*.csv -test.jpg -test.png \ No newline at end of file diff --git a/example/README.md b/example/README.md deleted file mode 100644 index a2fcb51..0000000 --- a/example/README.md +++ /dev/null @@ -1,36 +0,0 @@ -# libpython-clj Examples - - -## Usage - -```console -scripts/get-data.sh -python3.6 -m pip install tensorflow keras matplotlib --user -``` - -Then, potentially check that everything is installed. From python console: - -```console -Python 3.6.7 (default, Oct 22 2018, 11:32:17) -[GCC 8.2.0] on linux -Type "help", "copyright", "credits" or "license" for more information. ->>> import keras -Using TensorFlow backend. ->>> -``` - -## Examples - - -* [walkthrough](src/walkthrough.clj) - Simple, quick exploration of libpython features. -* [keras-simple](src/keras_simple.clj) - Keras example taken from [here](https://machinelearningmastery.com/tutorial-first-neural-network-python-keras/). -* [matplotlib](src/matplotlib.clj) - Quick demo of matplotlib, numpy including zero-copy pathways. - - -## Virtual Environments - - -This was lightly tested with `python3 -m venv venv`. Simply activate the virtual environment in the same console -before you launch the repl and things should work fine assuming the base python -executable versions line up. Again, see readme if they do not in order to override the -version loaded. diff --git a/example/project.clj b/example/project.clj deleted file mode 100644 index 0464bad..0000000 --- a/example/project.clj +++ /dev/null @@ -1,7 +0,0 @@ -(defproject libpython-clj/example "0.1.0-SNAPSHOT" - :description "Examples of python integration" - :url "https://github.com/cnuernber/libpython-clj" - :license {:name "EPL-2.0" - :url "https://www.eclipse.org/legal/epl-2.0/"} - :dependencies [[org.clojure/clojure "1.10.1"] - [cnuernber/libpython-clj "1.15"]]) diff --git a/example/scripts/get-data.sh b/example/scripts/get-data.sh deleted file mode 100755 index f41de9a..0000000 --- a/example/scripts/get-data.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - - -wget https://raw.githubusercontent.com/jbrownlee/Datasets/master/pima-indians-diabetes.data.csv diff --git a/example/src/keras_simple.clj b/example/src/keras_simple.clj deleted file mode 100644 index 44a2236..0000000 --- a/example/src/keras_simple.clj +++ /dev/null @@ -1,97 +0,0 @@ -(ns keras-simple.core - "https://machinelearningmastery.com/tutorial-first-neural-network-python-keras/" - (:require [libpython-clj.python - :refer [import-module - import-as - from-import - a$ ;;compile time call-attr - afn ;;runtime call-attr - get-item - get-attr - python-type - call-attr - call-attr-kw - att-type-map - ->py-dict] - :as py] - [clojure.pprint :as pp])) - - -;;Uncomment this line to load a different version of your python shared library: - - -;;(alter-var-root #'libpython-clj.jna.base/*python-library* (constantly "python3.7m")) - - -(py/initialize!) - - -(import-as numpy np) -(import-as builtins builtins) -(from-import builtins slice) -(import-as keras keras) -(from-import keras.layers Dense) -(import-as keras.models keras-models) -(import-as keras.layers keras-layers) - - -(defonce initial-data (a$ np loadtxt "pima-indians-diabetes.data.csv" :delimiter ",")) - - -(def features (get-item initial-data [(slice nil) (slice 0 8)])) - -(def labels (get-item initial-data [(slice nil) (slice 8 9)])) - -(defn dense-layer - [output-size & args] - (apply py/cfn Dense output-size args)) - - -(defn sequential-model - [] - (a$ keras-models Sequential)) - - -(defn add-layer! - [model layer] - (a$ model add layer) - model) - -(defn compile-model! - [model & args] - (apply py/afn model "compile" args) - model) - - -(def model (-> (sequential-model) - (add-layer! (dense-layer 12 :input_dim 8 :activation :relu)) - (add-layer! (dense-layer 8 :activation :relu)) - (add-layer! (dense-layer 1 :activation :sigmoid)) - (compile-model! :loss :binary_crossentropy - :optimizer :adam - :metrics (py/->py-list [:accuracy])))) - -(defn fit-model - [model features labels & args] - (apply py/afn model "fit" features labels args) - model) - - -(def fitted-model (fit-model model features labels - :epochs 150 - :batch_size 10)) - - -(defn eval-model - [model features lables] - (let [model-names (->> ($. model metrics_names) - (mapv keyword))] - (->> (a$ model evaluate features labels) - (map vector model-names) - (into {})))) - - -(def scores (eval-model fitted-model features labels)) - - -(pp/pprint scores) diff --git a/example/src/matplotlib.clj b/example/src/matplotlib.clj deleted file mode 100644 index 386beab..0000000 --- a/example/src/matplotlib.clj +++ /dev/null @@ -1,73 +0,0 @@ -(ns matplotlib - (:require [libpython-clj.python :as py] - [tech.v2.datatype :as dtype] - [tech.libs.buffered-image :as bufimg] - [tech.v2.tensor :as dtt] - [clojure.java.io :as io] - [clojure.java.shell :as sh])) - -(comment - - ;;First let's use matplotlib normally - - (py/initialize!) - (def mfig (py/import-module "matplotlib.figure")) - (def magg (py/import-module "matplotlib.backends.backend_agg")) - (def np (py/import-module "numpy")) - (def x (py/call-attr np "linspace" 0 2 100)) - (py/python-type x) - (py/get-attr x "shape") - (def plt (py/import-module "matplotlib.pyplot")) - - - ;; plt.plot(x, x, label='linear') - ;; plt.plot(x, x**2, label='quadratic') - ;; plt.plot(x, x**3, label='cubic') - - (defn plot-it - [] - (py/call-attr-kw plt "plot" [x x] {"label" "linear"}) - (py/call-attr-kw plt "plot" [x (py/call-attr x "__pow__" 2)] {"label" "quadratic"}) - (py/call-attr-kw plt "plot" [x (py/call-attr x "__pow__" 3)] {"label" "cubic"}) - (py/call-attr plt "xlabel" "x label") - (py/call-attr plt "ylabel" "y label") - (py/call-attr plt "title" "Simple Plot") - (py/call-attr plt "legend")) - - (plot-it) - - (py/call-attr plt "show") - - - - ;;Now let's get a version of this in memory on the JVM where we - ;;can do something with it. - - (def fig (py/call-attr plt "figure")) - (def canvas (py/get-attr fig "canvas")) - (def agg-canvas (py/call-attr magg "FigureCanvasAgg" fig)) - ;; Redo all the drawing above - (plot-it) - - (py/call-attr agg-canvas "draw") - (def np-data (py/call-attr np "array" - (py/call-attr agg-canvas "buffer_rgba"))) - (def tens (py/as-tensor np-data)) - - (dtype/get-datatype tens) - (dtype/shape tens) - (dtt/tensor->buffer tens) - - (def bufimage (bufimg/new-image 480 640 :byte-abgr)) - - (def ignored (dtype/copy! tens bufimage)) - - ;;Now we fix the ordering (RGBA->ABGR): - - (def ignored (-> (dtt/select tens :all :all (->> (range 4) - reverse)) - (dtype/copy! bufimage))) - - (bufimg/save! bufimage "test.png") - - ) diff --git a/example/src/mxnet_autograd.clj b/example/src/mxnet_autograd.clj deleted file mode 100644 index 33eca31..0000000 --- a/example/src/mxnet_autograd.clj +++ /dev/null @@ -1,63 +0,0 @@ -(ns mxnet-autograd - "https://mxnet.incubator.apache.org/api/python/docs/api/autograd/index.html" - (:require [libpython-clj.python :as py])) - - -(comment - - (py/initialize!) - - (def mx (py/import-module "mxnet")) - - (def mx-nd (py/get-attr mx "nd")) - - (def autograd (py/get-attr mx "autograd")) - - (def x (py/call-attr mx-nd "ones" 1)) - - (py/call-attr x "attach_grad") - - (def z - (py/with - [r (py/call-attr autograd "record")] - (py/call-attr mx-nd "elemwise_add" - (py/call-attr mx-nd "exp" x) - x))) - (def dx (py/call-attr-kw autograd "grad" [z [x]] {:create_graph true})) - - (def x (-> (py/call-attr mx-nd "arange" 4) - (py/call-attr "reshape" [4 1]))) - - (py/call-attr x "attach_grad") - - (def y (py/with - [r (py/call-attr autograd "record")] - (->> (py/call-attr mx-nd "dot" - (py/get-attr x "T") - x) - (py/call-attr mx-nd "multiply" 2)))) - - (py/call-attr-kw y "backward" [] {:retain_graph true}) - - (py/get-attr x "grad") - - x - - - (def mx-rand (py/get-attr mx-nd "random")) - - (def x (py/call-attr-kw mx-rand "uniform" [] {:shape 10})) - - (py/call-attr x "attach_grad") - - (def m (py/with - [r (py/call-attr autograd "record")] - (py/call-attr mx-nd "sigmoid" x))) - - - (py/call-attr m "backward") - - x - (py/get-attr x "grad") - - ) diff --git a/example/src/new_to_clojure.clj b/example/src/new_to_clojure.clj deleted file mode 100644 index 0e08797..0000000 --- a/example/src/new_to_clojure.clj +++ /dev/null @@ -1,155 +0,0 @@ -(ns new-to-clojure - "This is a clojure namespace declaration. This goes at the top, is documented like - such and can contain information about how to interpret the rest of this file. - For now, we leave it at that.") - - -;; This is a clojure line comment. From here on out we will place everything in a -;; comment form which, as we will see, can container evaluatable code. If you -;; are in emacs with clojure, leiningen, and cider installed then you can -;; type C-c,C-e to execute each line. - -(comment - ;;First, let's talk about Clojure. This is a a list - (list 1 2 3) - - ;;So is this - '(list 1 2 3) - - ;;When the reader finds a list it applies the first thing in the list - ;;to everything following the first thing. - - (+ 1 2 3) - - ;;If the first thing isn't executable, that is an error - (1 2 3) - - ;;Clojure arrays are persisent vectors meaning they are immutable - ;;datastructures that have good random access semantics. They are often - ;;used for data because the reader does *not* attempt to execute them. - [1 2 3 4] - - (conj [1 2 3 4] 5) - - (map inc [1 2 3 4]) - - (filter #(> % 2) [1 2 3 4]) - - ;;In Clojure, a dict is called a map and those are also immutable datastructures. - ;;We specify them like such: - {:a 1 :b 2} - - (assoc {:a 1 :b 2} :c 3) - - (dissoc {:a 1 :b 2} :b) - - (map identity {:a 1 :b 2}) - - (into {} (map identity {:a 1 :b 2})) - - (defn gen123plus - [x] - (map #(+ x %) [1 2 3])) - - (interleave (gen123plus 10) (gen123plus 20)) - - (defn zigzag - [period] - (->> (concat (range period) - (range period, 0 -1)) - repeat - (apply concat))) - - ;;When we want another library, we usually use require expects a clojure library. - ;;Because we are on them JVM, we sometimes have to use 'import' which targets a - ;;particular class - - (require '[libpython-clj.python :as py]) - - (py/initialize!) - - (def test-dict (py/->python {:a 1 :b 2})) - - (py/python-type test-dict) - - (py/att-type-map test-dict) - - (def bridged (py/as-jvm test-dict)) - - (def np (py/import-module "numpy")) - - (-> (py/get-attr np "linspace") - (py/get-attr "__doc__") - print) - - ;; Examples - ;; -------- - ;; >>> np.linspace(2.0, 3.0, num=5) - ;; array([ 2. , 2.25, 2.5 , 2.75, 3. ]) - - (def np-ary (py/call-attr-kw np "linspace" [2.0 3.0] {:num 5})) - - (def tens (py/as-tensor np-ary)) - - (map identity tens) - - (require '[tech.v2.datatype :as dtype]) - - (dtype/set-value! tens 2 6677) - - (println np-ary) - - - (def mfig (py/import-module "matplotlib.figure")) - (def magg (py/import-module "matplotlib.backends.backend_agg")) - - (def x (py/call-attr np "linspace" 0 2 100)) - - (def plt (py/import-module "matplotlib.pyplot")) - - (def fig (py/call-attr plt "figure")) - (def canvas (py/get-attr fig "canvas")) - (def agg-canvas (py/call-attr magg "FigureCanvasAgg" fig)) - - (defn plot-it - [x] - (py/call-attr-kw plt "plot" [x x] {"label" "linear"}) - (py/call-attr-kw plt "plot" [x (py/call-attr x "__pow__" 2)] - {"label" "quadratic"}) - (py/call-attr-kw plt "plot" [x (py/call-attr x "__pow__" 3)] - {"label" "cubic"}) - (py/call-attr plt "xlabel" "x label") - (py/call-attr plt "ylabel" "y label") - (py/call-attr plt "title" "Simple Plot") - (py/call-attr plt "legend")) - - (plot-it x) - - (py/call-attr agg-canvas "draw") - (def np-data (py/call-attr np "array" - (py/call-attr agg-canvas "buffer_rgba"))) - (def tens (py/as-tensor np-data)) - - (require '[tech.v2.tensor :as dtt]) - (require '[tech.libs.buffered-image :as bufimg]) - - (def jvm-img (bufimg/new-image 480 640 :byte-abgr)) - - ;;Copy will read the data out in row-major format. - (def ignored (dtype/copy! tens jvm-img)) - - (bufimg/save! jvm-img "test.png") - - ;;Now in emacs, run M-x auto-image-file-mode` - ;;The color of the image is off. The reason is that the canvas produces an - ;;"RGBA" image and java buffered image is ABGR. - ;;The fix is to create a view of the tensor that has the last dimension reversed: - - (def tens-view (dtt/select tens :all :all [3 2 1 0])) - - (def ignored (dtype/copy! tens-view jvm-img)) - - (bufimg/save! jvm-img "test.png") - - ;;now the image will be correct. - ) diff --git a/example/src/walkthrough.clj b/example/src/walkthrough.clj deleted file mode 100644 index 31c4780..0000000 --- a/example/src/walkthrough.clj +++ /dev/null @@ -1,55 +0,0 @@ -(ns walkthough) - - -(comment - ;; First step is to initialize library - (require '[libpython-clj.python :as py]) - (py/initialize!) - - ;;The system uses clojure.tools.logging. - - - ;;'->' means copy, 'as' means bridge. - - (def test-dict (py/->python {:a 1 :b 2})) - (py/python-type test-dict) - - ;;This created a 'dict' object in python and returned - ;;a pointer to that object. test-dict is a jna pointer. - - (py/att-type-map test-dict) - - (def bridged (py/as-jvm test-dict)) - - ;;bridged implements java.util.Map and as such things like - ;;count, get work. - ;;to add an element, use the .put method. Also iteration works - ;;but you get back bridged objects. - (def back-to-clojure (into {} bridged)) - - - ;;That is fun, and those are the basic mechanics of the language. - ;;Now let's actually use something - - (def np (py/import-module "numpy")) - - ;;Lets lookup the function documentation for 'linspace': - (-> (py/get-attr np "linspace") - (py/get-attr "__doc__") - print) - - ;;Examples - ;;-------- - ;;>>> np.linspace(2.0, 3.0, num=5) - (def np-ary (py/call-attr-kw np "linspace" [2.0 3.0] {:num 5})) - - ;;Now we can bridge the numpy object into a tensor - (def tens (py/as-tensor np-ary)) - - ;;This means they share the backing store. - (require '[tech.v2.datatype :as dtype]) - (dtype/set-value! tens 2 6677) - - (vec np-ary) - (vec tens) - ) From 8d4b86079a7593757f0301b06027282ea5308d8e Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 28 Jan 2020 11:39:08 -0700 Subject: [PATCH 165/456] editing --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 65e7f01..30d7407 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,8 @@ JNA libpython bindings to the tech ecosystem. of python. * REPL oriented design means fast, smooth, iterative development. * Carin Meier has written excellent posts on [plotting](http://gigasquidsoftware.com/blog/2020/01/18/parens-for-pyplot/) and - [advanced text generation](http://gigasquidsoftware.com/blog/2020/01/10/hugging-face-gpt-with-clojure/). + [advanced text generation](http://gigasquidsoftware.com/blog/2020/01/10/hugging-face-gpt-with-clojure/). She also has some + great [examples](https://github.com/gigasquid/libpython-clj-examples). ## Vision From c4bacdfa1b441afb98445f15121850b272c86de7 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 28 Jan 2020 11:42:21 -0700 Subject: [PATCH 166/456] editing --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 30d7407..9e04016 100644 --- a/README.md +++ b/README.md @@ -127,7 +127,7 @@ our test [docker containers](dockerfiles/CondaDockerfile). * [examples](https://github.com/gigasquid/libpython-clj-examples) * [docker setup](https://github.com/scicloj/docker-hub) * [pandas bindings (!!)](https://github.com/alanmarazzi/panthera) -* [nextjournal notebook](https://nextjournal.com/chrisn/fun-with-matplotlib) +* [nextjournal notebooks](https://nextjournal.com/kommen) * [scicloj video](https://www.youtube.com/watch?v=ajDiGS73i2o) * [Clojure/Python interop technical blog post](www.techascent.com/blogs/functions-across-languages.html) * [persistent datastructures in python](https://github.com/tobgu/pyrsistent) From 3c22831412e96707b8b094205cb93e78d95483df Mon Sep 17 00:00:00 2001 From: Arsene Rei Date: Wed, 29 Jan 2020 13:41:23 -0800 Subject: [PATCH 167/456] Update technical blog post link (#69) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9e04016..6ec38f6 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,7 @@ our test [docker containers](dockerfiles/CondaDockerfile). * [pandas bindings (!!)](https://github.com/alanmarazzi/panthera) * [nextjournal notebooks](https://nextjournal.com/kommen) * [scicloj video](https://www.youtube.com/watch?v=ajDiGS73i2o) -* [Clojure/Python interop technical blog post](www.techascent.com/blogs/functions-across-languages.html) +* [Clojure/Python interop technical blog post](http://techascent.com/blog/functions-across-languages.html) * [persistent datastructures in python](https://github.com/tobgu/pyrsistent) From 72ff8db10b7d3b7bcaca15a3d766f93c3b1a83cc Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Fri, 31 Jan 2020 07:16:41 -0700 Subject: [PATCH 168/456] snap --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 7128ac2..12c07a3 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.33" +(defproject cnuernber/libpython-clj "1.34-SNAPSHOT" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "EPL-2.0" From 4322de75f48284f0f1fdfef54732c93ec4e7b546 Mon Sep 17 00:00:00 2001 From: Carin Meier Date: Sat, 1 Feb 2020 11:52:39 -0500 Subject: [PATCH 169/456] Issue 71 - Clojure deps (#75) * Initial working of deps template * Get docker running * clean up unneeded lein * Making the docker stuff work right * clean up docker file * Simplify aliases * remove unneeded * Try for travis * try again * Yaml is such a joy * w/o lein deps * Add javac * more docs * Add clean alias --- .gitignore | 3 +- .travis.yml | 9 ++ README.md | 65 +++++++++++ deps.edn | 92 +++++++++++++++ dev/resources/logback.xml | 11 ++ dev/src/build.clj | 217 ++++++++++++++++++++++++++++++++++++ dev/src/user.clj | 9 ++ dockerfiles/CondaDockerfile | 11 +- project.clj | 12 -- scripts/conda-repl | 4 +- scripts/run-conda-docker | 7 +- 11 files changed, 412 insertions(+), 28 deletions(-) create mode 100644 deps.edn create mode 100644 dev/resources/logback.xml create mode 100644 dev/src/build.clj create mode 100644 dev/src/user.clj delete mode 100644 project.clj diff --git a/.gitignore b/.gitignore index e3921c6..217fc6e 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,5 @@ pom.xml.asc .nrepl-port .hgignore .hg/ -__pycache__ \ No newline at end of file +__pycache__ +.cpcache/ \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 0d45a34..9dceeb6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,15 @@ lein: 2.8.2 before_install: - sudo apt-get -y install python3 python3-pip - sudo python3 -mpip install numpy + - curl -O https://download.clojure.org/install/linux-install-1.10.1.507.sh + - chmod +x linux-install-1.10.1.507.sh + - sudo ./linux-install-1.10.1.507.sh + - clojure -Sdescribe addons: apt: update: true +install: clojure -Sdescribe +script: + - clojure -A:javac + - clojure -A:test + diff --git a/README.md b/README.md index 6ec38f6..d7fbb98 100644 --- a/README.md +++ b/README.md @@ -147,6 +147,71 @@ we have an [introductory document](docs/new-to-clojure.md). * [create C ptr from numpy](https://docs.scipy.org/doc/numpy/reference/generated/numpy.ndarray.ctypes.html) +## How to run project tasks +This project has java sources in additon to regular clojure sources. +The build alias have the javac built into the tasks, however for running tests or repl you would +need to run that as a seperate step if you need to recompile the java classes. +Example: + +- `clojure -A:javac` +- `clojure -A:test` + +Also note, that if you have tools installed you may substitue the `clj` command for `clojure` below: + +- `clj -A:test` + + +### Compile Java classes + +```bash +$ clojure -A:javac +``` + +### REPL + +```bash +$ clojure -A:repl +nREPL server started on port 56785 on host localhost - nrepl://localhost:56785 +``` + + +### Tests + +```bash +$ clojure -A:test +``` + +### Package jar + +```bash +$ clojure -A:jar +``` + +### Clean target directory + +```bash +$ clojure -A:clean +``` + + +### Local install + +To install jar to local .m2 : + +```bash +$ clojure -A:install +``` + +### Deploy to clojars + +Put your clojars.org credentials to settings.xml (or uncomment login and password prompt in dev/src/build.clj). + +```bash +$ clojure -A:deploy +``` +This command will sign jar before deploy, using your gpg key. (see dev/src/build.clj for signing options) + + ## License diff --git a/deps.edn b/deps.edn new file mode 100644 index 0000000..389864a --- /dev/null +++ b/deps.edn @@ -0,0 +1,92 @@ +{ + :mvn/repos {"clojars" {:url "https://repo.clojars.org/"} + "central" {:url "https://repo1.maven.org/maven2/"}} + + ;; don't change target/classes (do not remove it from :paths and + ;; do not rename it, otherwise edit build.clj) + ;; uncomment java-src if you have java sources. + :paths ["src" "java" "resources" "target/classes"] + + :deps { + org.clojure/clojure {:mvn/version "1.10.1"} + camel-snake-kebab {:mvn/version "0.4.0"} + techascent/tech.datatype {:mvn/version "4.72"} + org.clojure/data.json {:mvn/version "0.2.7"}} + + :build { + :java-source-paths "java" + :javac-options ["-target" "1.8" "-source" "1.8" "-Xlint:-options"] + + :group-id "cnuernber" + :artifact-id "libpython-clj" + :artifact-version "1.34-SNAPSHOT" + :description "libpython bindings to the techascent ecosystem" + :url "http://github.com/cnuernber/libpython-clj" + :scm {:url "git@github.com:cnuernber/libpython-clj.git"} + :license {:name "EPL-2.0" + :url "https://www.eclipse.org/legal/epl-2.0/"}} + + + :aliases { + ;;; clj -A:repl + :repl {:extra-deps {criterium {:mvn/version "0.4.5"} + nrepl {:mvn/version "0.6.0"} + healthsamurai/matcho {:mvn/version "0.3.3"} + hashp {:mvn/version "0.1.1"} + badigeon/badigeon {:git/url "https://github.com/EwenG/badigeon.git" + :sha "c5d7d8f9c44fee2f193ef924cdf8a485aee539c5"} + ch.qos.logback/logback-classic {:mvn/version "1.1.3"}} + :jvm-opts ["-Duser.timezone=UTC"] + :extra-paths ["dev/src" "resources" "test"] + :main-opts ["--main" "nrepl.cmdline"]} + ;;; clj -A:test + :test {:extra-paths ["test" "src/test/clojure"] + :extra-deps {org.clojure/test.check {:mvn/version "RELEASE"} + com.cognitect/test-runner + {:git/url "https://github.com/cognitect-labs/test-runner" + :sha "3cb0a9daf1cb746259dc8309b218f9211ad3b33b"}} + :main-opts ["-m" "cognitect.test-runner" + "-d" "test" + "-d" "src/test/clojure"]} + + ;; clean : clj -A:clean + :clean {:main-opts ["-e" "(load-file,\"dev/src/build.clj\"),(build/clean)"] + :extra-deps {badigeon/badigeon {:git/url "https://github.com/EwenG/badigeon.git" + :sha "c5d7d8f9c44fee2f193ef924cdf8a485aee539c5"} + ch.qos.logback/logback-classic {:mvn/version "1.1.3"}} + ;;; to supress noisy logs with logback file + :extra-paths ["dev/resources"]} + ;; build jar (compile java, clj files): clj -A:jar + :jar {:main-opts ["-e" "(load-file,\"dev/src/build.clj\"),(build/clean),(build/compile-java),(build/jar)"] + :extra-deps {badigeon/badigeon {:git/url "https://github.com/EwenG/badigeon.git" + :sha "c5d7d8f9c44fee2f193ef924cdf8a485aee539c5"} + ch.qos.logback/logback-classic {:mvn/version "1.1.3"}} + ;;; to supress noisy logs with logback file + :extra-paths ["dev/resources"]} + + ;; compile java classes only: clj -A:javac + :javac {:main-opts ["-e" "(load-file,\"dev/src/build.clj\"),(build/compile-java)"] + :extra-deps {badigeon/badigeon {:git/url "https://github.com/EwenG/badigeon.git" + :sha "c5d7d8f9c44fee2f193ef924cdf8a485aee539c5"} + ch.qos.logback/logback-classic {:mvn/version "1.1.3"}} + ;;; to supress noisy logs with logback file + :extra-paths ["dev/resources"]} + + ;; install to local .m2: clj -A:install + :install {:main-opts ["-e" "(load-file,\"dev/src/build.clj\"),(build/install)"] + :extra-deps {badigeon/badigeon {:git/url "https://github.com/EwenG/badigeon.git" + :sha "c5d7d8f9c44fee2f193ef924cdf8a485aee539c5"} + ch.qos.logback/logback-classic {:mvn/version "1.1.3"}} + ;;; to supress noisy logs with logback file + :extra-paths ["dev/resources"]} + + ;; public to clojars: clj -A:deploy + :deploy {:main-opts ["-e" "(load-file,\"dev/src/build.clj\"),(build/deploy)"] + :extra-deps {badigeon/badigeon {:git/url "https://github.com/EwenG/badigeon.git" + :sha "c5d7d8f9c44fee2f193ef924cdf8a485aee539c5"} + ch.qos.logback/logback-classic {:mvn/version "1.1.3"}} + ;;; to supress noisy logs with logback file + :extra-paths ["dev/resources"]}}} + + + diff --git a/dev/resources/logback.xml b/dev/resources/logback.xml new file mode 100644 index 0000000..b70d039 --- /dev/null +++ b/dev/resources/logback.xml @@ -0,0 +1,11 @@ + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + + diff --git a/dev/src/build.clj b/dev/src/build.clj new file mode 100644 index 0000000..8655466 --- /dev/null +++ b/dev/src/build.clj @@ -0,0 +1,217 @@ +(ns build + (:require [badigeon.clean :as clean] + [badigeon.classpath :as classpath] + [badigeon.javac :as javac] + [badigeon.jar :as jar] + [badigeon.install :as install] + [badigeon.sign :as sign] + [badigeon.deploy :as deploy] + [badigeon.pom :as pom] + [clojure.tools.deps.alpha.reader :as deps-reader] + [clojure.data.xml :as xml] + [clojure.string :as str] + [clojure.data.xml.tree :as tree] + [clojure.data.xml.event :as event] + [clojure.java.io :as io]) + (:import (java.util.jar JarEntry JarOutputStream) + (java.nio.file Path Paths) + [java.io File Reader ByteArrayOutputStream] + [clojure.data.xml.node Element] + (java.net URI))) + + +(def deps-content (deps-reader/slurp-deps "deps.edn")) +(def java-src-folder (-> deps-content :build :java-source-paths)) +(def javac-options (-> deps-content :build :javac-options)) +(def group-id (-> deps-content :build :group-id)) +(def artifact-id (-> deps-content :build :artifact-id)) +(def group-artefact-id (symbol (str group-id "/" artifact-id))) +(def artifact-version (-> deps-content :build :artifact-version)) +(def artifact-description (-> deps-content :build :description)) +(def artifact-url (-> deps-content :build :url)) +(def artifact-scm (-> deps-content :build :scm)) +(def artifact-license (-> deps-content :build :license)) +(def jar-name (str "target/" artifact-id "-" artifact-version ".jar")) + + +(defn clean + [] + (clean/clean "target" + {;; By default, clean does not allow deleting folders outside the target directory, + ;; unless :allow-outside-target? is true + :allow-outside-target? false})) + +(defn make-classpath + [] + ;; Builds a classpath by using the provided deps spec or, by default, the deps.edn file of the current project. + ;; Returns the built classpath as a string. + + (classpath/make-classpath)) + +(defn compile-java + ([emit-path] + ;; Compile java sources under the java-src-folder directory + (when java-src-folder + (javac/javac java-src-folder {;; Emit class files to the target/classes directory + :compile-path emit-path + ;; Additional options used by the javac command + :javac-options javac-options}))) + ([] + (compile-java "target/classes"))) + + +(defn- parse-xml + [^Reader rdr] + (let [roots (tree/seq-tree + event/event-element event/event-exit? event/event-node + (xml/event-seq rdr {:include-node? #{:element :characters :comment}}))] + (first (filter #(instance? Element %) (first roots))))) + + +(xml/alias-uri 'ppom "http://maven.apache.org/POM/4.0.0") + +(defn- add-extra-stuff->pom + [] + (let [pom-file (io/file "pom.xml") + pom (with-open [rdr (io/reader pom-file)] + (-> rdr + parse-xml)) + pom (if artifact-url + (#'badigeon.pom/xml-update pom [::ppom/url] (xml/sexp-as-element [::ppom/url artifact-url])) + pom) + pom (if artifact-description + (#'badigeon.pom/xml-update pom [::ppom/description] (xml/sexp-as-element [::ppom/description artifact-description])) + pom) + pom (if artifact-scm + (#'badigeon.pom/xml-update pom [::ppom/scm] (xml/sexp-as-element [::ppom/scm [::ppom/url (:url artifact-scm)]])) + pom) + pom (if artifact-license + (#'badigeon.pom/xml-update pom [::ppom/license] (xml/sexp-as-element [::ppom/licenses + [::ppom/license + [::ppom/url (or (:url artifact-license) "")] + [::ppom/name (or (:name artifact-license)) ""]]])) + + pom)] + + + (spit pom-file (str/replace (xml/indent-str pom) #"\n\s+\n" "\n")) + (spit pom-file "" :append true))) + +(defn make-pom + [] + (when (.exists (io/file "pom.xml")) + (io/delete-file "pom.xml")) + (let [lib (symbol group-artefact-id) + maven-coords {:mvn/version artifact-version}] + (pom/sync-pom lib maven-coords (-> (deps-reader/slurp-deps "deps.edn"))) + (add-extra-stuff->pom))) + +(defn jar + [] + (when (.exists (io/file "pom.xml")) + (io/delete-file "pom.xml")) + ;; Package the project into a jar file + (jar/jar group-artefact-id {:mvn/version artifact-version} + {;; The jar file produced. + :out-path jar-name + ;;;; Adds a \"Main\" entry to the jar manifest with the value + ;;:main main-ns + ;; Additional key/value pairs to add to the jar manifest. If a value is a collection, a manifest section is built for it. + ;;:manifest {"Project-awesome-level" "super-great" + ;; :my-section-1 [["MyKey1" "MyValue1"] ["MyKey2" "MyValue2"]] + ;; :my-section-2 {"MyKey3" "MyValue3" "MyKey4" "MyValue4"}} + + ;; By default Badigeon add entries for all files in the directory listed in the + ;; :paths section of the deps.edn file. This can be overridden here. + ;;:paths ["src" "target/classes" "java-src"] + + ;; The dependencies to be added to the \"dependencies\" section of the pom.xml file. + ;; When not specified, defaults to the :deps entry of the deps.edn file, without + ;; merging the user-level and system-level deps.edn files + ;;:deps '{org.clojure/clojure {:mvn/version "1.10.1"}} + ;; The repositories to be added to the \"repositories\" section of the pom.xml file. + ;; When not specified, default to nil - even if the deps.edn files contains + ;; a :mvn/repos entry. + :mvn/repos '{"clojars" {:url "https://repo.clojars.org/"}} + ;; A predicate used to excludes files from beeing added to the jar. + ;; The predicate is a function of two parameters: The path of the directory + ;; beeing visited (among the :paths of the project) and the path of the file + ;; beeing visited under this directory. + :exclusion-predicate badigeon.jar/default-exclusion-predicate + ;; A function to add files to the jar that would otherwise not have been added to it. + ;; The function must take two parameters: the path of the root directory of the + ;; project and the file being visited under this directory. When the function + ;; returns a falsy value, the file is not added to the jar. Otherwise the function + ;; must return a string which represents the path within the jar where the file + ;; is copied. + ;; :inclusion-path (partial badigeon.jar/default-inclusion-path "badigeon" "badigeon") + ;; By default git and local dependencies are not allowed. Set allow-all-dependencies? to true to allow them + :allow-all-dependencies? true}) + (add-extra-stuff->pom) + (println "Successfully created jar file: " jar-name)) + + + +(defn install + [] + ;; Install the created jar file into the local maven repository. + (let [local-repo (str (System/getProperty "user.home") "/.m2/repository")] + (install/install group-artefact-id {:mvn/version artifact-version} + ;; The jar file to be installed + jar-name + ;; The pom.xml file to be installed. This file is generated when creating the jar with the badigeon.jar/jar function. + "pom.xml" + {;; The local repository where the jar should be installed. + :local-repo local-repo}) + (println "Successfully installed to local repo: " local-repo))) + +(defn deploy + [] + ;; Deploy the created jar file to a remote repository. + (let [;; Artifacts are maps with a required :file-path key and an optional :extension key + artifacts [{:file-path jar-name :extension "jar"} + {:file-path "pom.xml" :extension "pom"}] + ;; Artifacts must be signed when deploying non-snapshot versions of artifacts. + artifacts (badigeon.sign/sign artifacts {;; The gpg command can be customized + :command "gpg"})] + ;; The gpg key used for signing. Defaults to the first private key found in your keyring. + ;;:gpg-key "root@eruditorum.org" + + + ;; default to reading the credentials from ~/.m2/settings.xml. uncomment to prompt user credentials. + ;;;; Prompt for a username + ;;username (badigeon.prompt/prompt "Username (clojars): ") + ;;;; Prompt for a password using the process standard input and without echoing. + ;;password (badigeon.prompt/prompt-password "Password (clojars): ") + + (badigeon.deploy/deploy + group-artefact-id artifact-version + artifacts + {;; :id is used to match the repository in the ~/.m2/settings.xml for credentials when no credentials are explicitly provided. + :id "clojars" + ;; The URL of the repository to deploy to. + :url "https://repo.clojars.org/"} + { + ;; Take creds from ~/.m2/settings.xml + + ;; The credentials used when authenticating to the remote repository. + ;; When none is provided, default to reading the credentials from ~/.m2/settings.xml + ;;:credentials {:username username :password password + ;; ;;:private-key "/path/to/private-key" :passphrase "passphrase" + ;; } + ;; When allow-unsigned? is false, artifacts must be signed when deploying non-snapshot versions of artifacts. Default to false. + :allow-unsigned? false}) + (println "Success!"))) + +(comment + (def cp (make-classpath)) + (clean) + (compile-java) + (extract-classes-from-deps) + (make-pom) + (add-extra-stuff->pom) + (jar) + (install)) + diff --git a/dev/src/user.clj b/dev/src/user.clj new file mode 100644 index 0000000..94ca101 --- /dev/null +++ b/dev/src/user.clj @@ -0,0 +1,9 @@ +(ns user) + +;; debug print with #p +(require 'hashp.core) + +(def dev-mode true) +(println "***************************") +(println {:msg "dev mode" :status (if (true? dev-mode) "on" "off")}) +(println "***************************") diff --git a/dockerfiles/CondaDockerfile b/dockerfiles/CondaDockerfile index 385da77..bdc956b 100644 --- a/dockerfiles/CondaDockerfile +++ b/dockerfiles/CondaDockerfile @@ -3,6 +3,8 @@ FROM ubuntu:latest # Updating Ubuntu packages +ARG CLOJURE_TOOLS_VERSION=1.10.1.507 + RUN apt-get -qq update && apt-get -qq -y install curl wget bzip2 openjdk-8-jdk-headless \ && curl -sSL https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o /tmp/miniconda.sh \ @@ -10,12 +12,9 @@ RUN apt-get -qq update && apt-get -qq -y install curl wget bzip2 openjdk-8-jdk-h && rm -rf /tmp/miniconda.sh \ && conda install -y python=3 \ && conda update conda \ - && curl -O https://download.clojure.org/install/linux-install-1.10.1.492.sh \ - && chmod +x linux-install-1.10.1.492.sh \ - && ./linux-install-1.10.1.492.sh && rm linux-install-1.10.1.492.sh \ - && wget https://raw.githubusercontent.com/technomancy/leiningen/stable/bin/lein \ - && chmod a+x lein \ - && mv lein /usr/bin \ + && curl -o install-clojure https://download.clojure.org/install/linux-install-${CLOJURE_TOOLS_VERSION}.sh \ + && chmod +x install-clojure \ + && ./install-clojure && rm install-clojure \ && apt-get -qq -y autoremove \ && apt-get autoclean \ && rm -rf /var/lib/apt/lists/* /var/log/dpkg.log \ diff --git a/project.clj b/project.clj deleted file mode 100644 index 12c07a3..0000000 --- a/project.clj +++ /dev/null @@ -1,12 +0,0 @@ -(defproject cnuernber/libpython-clj "1.34-SNAPSHOT" - :description "libpython bindings to the techascent ecosystem" - :url "http://github.com/cnuernber/libpython-clj" - :license {:name "EPL-2.0" - :url "https://www.eclipse.org/legal/epl-2.0/"} - :dependencies [[org.clojure/clojure "1.10.1"] - [camel-snake-kebab "0.4.0"] - [techascent/tech.datatype "4.72"] - [org.clojure/data.json "0.2.7"]] - :profiles {:dev {:dependencies [[criterium "0.4.5"]]}} - :repl-options {:init-ns user} - :java-source-paths ["java"]) diff --git a/scripts/conda-repl b/scripts/conda-repl index 17a78c8..2c68746 100755 --- a/scripts/conda-repl +++ b/scripts/conda-repl @@ -6,7 +6,5 @@ source activate pyclj ## https://github.com/conda/conda/issues/9500#issuecomment-565753807 export LD_LIBRARY_PATH="$(python3-config --prefix)/lib" +clojure -A:repl -b 0.0.0.0 -p 2222 -lein update-in :dependencies conj \[nrepl\ \"0.6.0\"\]\ - -- update-in :plugins conj \[cider/cider-nrepl\ \"0.22.4\"\]\ - -- repl :headless :host localhost diff --git a/scripts/run-conda-docker b/scripts/run-conda-docker index 479e83e..0299e0b 100755 --- a/scripts/run-conda-docker +++ b/scripts/run-conda-docker @@ -3,12 +3,7 @@ scripts/build-conda-docker docker run --rm -it -u $(id -u):$(id -g) \ - -e LEIN_REPL_HOST="0.0.0.0" \ -v /$HOME/.m2:/home/$USER/.m2 \ - -v /$HOME/.lein:/home/$USER/.lein \ -v $(pwd)/:/libpython-clj \ - --net=host -w /libpython-clj \ + -p 2222:2222 -w /libpython-clj \ docker-conda scripts/conda-repl - # lein update-in :dependencies conj \[nrepl\ \"0.6.0\"\]\ - # -- update-in :plugins conj \[cider/cider-nrepl\ \"0.22.4\"\]\ - # -- repl :headless :host localhost From b0417a9f4f0a940e23b0916ad81460adb86aadd1 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sat, 1 Feb 2020 11:13:23 -0700 Subject: [PATCH 170/456] upgraded tech.datatype for fixed tech.parallel --- deps.edn | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/deps.edn b/deps.edn index 389864a..8f34b3b 100644 --- a/deps.edn +++ b/deps.edn @@ -10,7 +10,7 @@ :deps { org.clojure/clojure {:mvn/version "1.10.1"} camel-snake-kebab {:mvn/version "0.4.0"} - techascent/tech.datatype {:mvn/version "4.72"} + techascent/tech.datatype {:mvn/version "4.73"} org.clojure/data.json {:mvn/version "0.2.7"}} :build { @@ -87,6 +87,3 @@ ch.qos.logback/logback-classic {:mvn/version "1.1.3"}} ;;; to supress noisy logs with logback file :extra-paths ["dev/resources"]}}} - - - From 7ead006e67d2675ab4919e487cabb8b3401cba7e Mon Sep 17 00:00:00 2001 From: "J.J. Tolton" Date: Sat, 1 Feb 2020 18:05:56 -0500 Subject: [PATCH 171/456] ISSUE-72: datafy/nav extensibility for arbitrary Python objects (#73) * make datafy/nav extensible * update CHANGELOG.md * fix tests * fix docstring * spacing * no :reload true * dispatch on symbols over strings * dispatch on symbols over strings * add support for datafy for more complex types * refactor for style * eliminate redundancy * cleaner macro syntax * fix url --- CHANGELOG.md | 9 +- src/libpython_clj/metadata.clj | 80 +++++++-------- src/libpython_clj/require.clj | 109 ++++++++++++++++++++- test/libpython_clj/require_python_test.clj | 19 +++- 4 files changed, 169 insertions(+), 48 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 24763ee..ba42fb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ * [Examples are now done by gigasquid](https://github.com/gigasquid/libpython-clj-examples) +* [datafy/nav](https://clojure.github.io/clojure/branch-master/clojure.datafy-api.html) are now extensible for custom Python objects. + Extend `libpython-clj.require/pydafy` and `libpython-clj.require/pynav` + respectively with the symbol of class you want to extend. See + respective docstrings for details. + +* bugfix -- python.str now loaded by `import-python` + ## 1.33 * Better [windows anaconda support](https://github.com/cnuernber/libpython-clj/pull/67) @@ -44,7 +51,7 @@ ``` (**Note**: this is done for you by the function `libpython-clj.require/import-python`) - This fix brought to you by [jjtolten](https://github.com/jjtolton). + This fix brought to you by [jjtolton](https://github.com/jjtolton). ## 1.32 diff --git a/src/libpython_clj/metadata.clj b/src/libpython_clj/metadata.clj index b38669f..9ee4bc0 100644 --- a/src/libpython_clj/metadata.clj +++ b/src/libpython_clj/metadata.clj @@ -205,48 +205,44 @@ (or (string? att-val) (number? att-val))) - -(extend-type PPyObject - clj-proto/Datafiable - (datafy [item] - (with-meta - (with-gil - (->> (if (or (pyclass? item) - (pymodule? item)) - (-> (vars item) - (py-proto/as-map)) - (->> (py/dir item) - (map (juxt identity #(get-attr item %))))) - (map (fn [[att-name att-val]] - (when att-val - (try - [att-name - (merge (base-pyobj-map att-val) - (when (callable? att-val) - (py-fn-metadata att-name att-val {})) - (when (scalar? att-val) - {:value att-val}))] - (catch Throwable e - (log/warnf "Metadata generation failed for %s:%s" - (.toString item) - att-name) - nil))))) - (remove nil?) - (into (base-pyobj-map item)))) - {`clj-proto/nav - (fn nav-pyval - [coll f val] - (if (map? val) - (cond - (= :module (:type val)) - (as-jvm (import-module (:name val)) {}) - (= :type (:type val)) - (let [mod (as-jvm (import-module (:module val)) {}) - cls-obj (get-attr mod (:name val))] - cls-obj) - :else - val) - val))}))) +(defn datafy-module [item] + (with-gil + (->> (if (or (pyclass? item) + (pymodule? item)) + (-> (vars item) + (py-proto/as-map)) + (->> (py/dir item) + (map (juxt identity #(get-attr item %))))) + (map (fn [[att-name att-val]] + (when att-val + (try + [att-name + (merge (base-pyobj-map att-val) + (when (callable? att-val) + (py-fn-metadata att-name att-val {})) + (when (scalar? att-val) + {:value att-val}))] + (catch Throwable e + (log/warnf "Metadata generation failed for %s:%s" + (.toString item) + att-name) + nil))))) + (remove nil?) + (into (base-pyobj-map item))))) + +(defn nav-module [coll f val] + (with-gil + (if (map? val) + (cond + (= :module (:type val)) + (as-jvm (import-module (:name val)) {}) + (= :type (:type val)) + (let [mod (as-jvm (import-module (:module val)) {}) + cls-obj (get-attr mod (:name val))] + cls-obj) + :else + val) + val))) (defn module-path-string diff --git a/src/libpython_clj/require.clj b/src/libpython_clj/require.clj index 11399fc..0513309 100644 --- a/src/libpython_clj/require.clj +++ b/src/libpython_clj/require.clj @@ -4,11 +4,15 @@ [libpython-clj.metadata :as pymeta] [clojure.datafy :refer [datafy nav]] ;;Binds datafy/nav to python objects - [libpython-clj.metadata] - [clojure.tools.logging :as log])) + [libpython-clj.metadata :as metadata] + [clojure.tools.logging :as log] + [libpython-clj.python.protocols :as py-proto] + [clojure.core.protocols :as clj-proto]) + (:import [libpython_clj.python.protocols PPyObject PBridgeToJVM])) ;; for hot reloading multimethod in development (ns-unmap 'libpython-clj.require 'intern-ns-class) +(ns-unmap *ns* 'pydafy) (defn- parse-flags @@ -290,7 +294,106 @@ [dict :as python.dict] [set :as python.set] [tuple :as python.tuple] - [frozenset :as python.frozenset]) + [frozenset :as python.frozenset] + [str :as python.str]) '[builtins :as python]) :ok) + +(def ^:private builtins (py/import-module "builtins")) + + +(def ^:private pytype (comp symbol str (py/get-attr builtins "type"))) + + +(defmulti pydafy + "Turn a Python object into Clojure data. Metadata of Clojure + data will automatically be merged with nav protocol. + + Extend this method to convert a custom Python object into Clojure data. + Extend pynav if you would like to nav the resulting Clojure data. + + Note: we don't have a way yet to use a 'python type' as a 'jvm class', + so you need to extend with the symbol of the object name. I know it's + an ugly hack. Sorry. See examples in libpython-clj.require." + pytype) + +(defmulti pynav + "Nav data from a Python object. + + Extend this method to nav a custom Python object into Clojure data. + Extend pydafy if you would like to datafy a Python object. + + Note: we don't have a way yet to use a 'python type' as a 'jvm class', + so you need to extend with the symbol of the object name. I know it's + an ugly hack. Sorry. See examples in libpython-clj.require." + (fn [coll k v] (pytype coll))) + + +(defmethod pydafy :default [x] + (if (metadata/pyclass? x) + (metadata/datafy-module x) + (throw (ex-info (str "datafy not implemented for " (pytype x)) + {:type (pytype x)})))) + + +(defmethod pynav :default [x] + (if (metadata/pyclass? x) + (metadata/nav-module x) + (throw (ex-info (str "nav not implemented for " (pytype x)) + {:type (pytype x)})))) + + +(defmethod pydafy 'builtins.module [m] + (metadata/datafy-module m)) + + +(defmethod pynav 'builtins.module [coll k v] + (metadata/nav-module coll k v)) + + +(defmethod pydafy 'builtins.dict [x] + (py/->jvm x)) + + +(defn ^:private py-datafy [item] + (let [res (pydafy item) + m (meta res)] + (try + (with-meta + res + (merge + {'clj-proto/nav pynav} + m)) + (catch ClassCastException _ + ;; presumably metadata doesn't work for this type + res)))) + + +(defn ^:private py-nav [coll k v] + (let [res (pynav coll k v) + m (meta res)] + (try + (with-meta + res + (merge + {'clj-proto/datafy pydafy} + m)) + (catch ClassCastException _ + ;; presumably metadata doesn't work for this type + res)))) + + +(defmacro pydatafy [& types] + ;; credit: Tom Spoon + ;; https://clojurians.zulipchat.com/#narrow/stream/215609-libpython-clj-dev/topic/feature-requests/near/187056819 + `(do ~@(for [t types] + `(extend-type ~t + clj-proto/Datafiable + (datafy [item#] (py-datafy item#)) + clj-proto/Navigable + (nav [coll# k# v#] (py-nav coll# k# v#)))))) + + +(pydatafy PPyObject PBridgeToJVM) + diff --git a/test/libpython_clj/require_python_test.clj b/test/libpython_clj/require_python_test.clj index e2c5c85..a79b39b 100644 --- a/test/libpython_clj/require_python_test.clj +++ b/test/libpython_clj/require_python_test.clj @@ -1,8 +1,10 @@ (ns libpython-clj.require-python-test - (:require [libpython-clj.require :as req :refer [require-python] + (:require [libpython-clj.require :as req + :refer [require-python pydafy pynav] :reload true] [libpython-clj.python :as py] - [clojure.test :refer :all])) + [clojure.test :refer :all] + [clojure.datafy :refer [datafy nav]])) ;; Since this test mutates the global environment we have to accept that ;; it may not always work. @@ -100,6 +102,19 @@ (is (= (req-transform 'a '[b :as c :refer [blah] :flagA]) '[a.b :as c :refer [blah] :flagA])))) +(deftest datafy-test + + (defmethod pydafy 'builtins.list [x] + (vec x)) + + (is (vector? (datafy (python/list [1 2 3])))) + + (defmethod pydafy 'builtins.frozenset [x] + (set x)) + + (is (set? (datafy (python/frozenset [1 2 3]))))) + + (deftest import-python-test (is (= :ok (libpython-clj.require/import-python)))) From e6be3a9e8fa24f112d0ff13b60da0fbe76f49aee Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 2 Feb 2020 09:45:13 -0700 Subject: [PATCH 172/456] Adding back project.clj for a more complete build system --- project.clj | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 project.clj diff --git a/project.clj b/project.clj new file mode 100644 index 0000000..1d379fd --- /dev/null +++ b/project.clj @@ -0,0 +1,11 @@ +(defproject cnuernber/libpython-clj "1.34-SNAPSHOT" + :description "libpython bindings to the techascent ecosystem" + :url "http://github.com/cnuernber/libpython-clj" + :license {:name "Eclipse Public License" + :url "http://www.eclipse.org/legal/epl-v10.html"} + :plugins [[lein-tools-deps "0.4.5"]] + :middleware [lein-tools-deps.plugin/resolve-dependencies-with-deps-edn] + :lein-tools-deps/config {:config-files [:install :user :project]} + :profiles {:dev {:lein-tools-deps/config {:resolve-aliases [:test]}} + :uberjar {:aot :all}} + :java-source-paths ["java"]) From 13403cf6bff1e7427e89eaa7a71a53bb949b8aa1 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 2 Feb 2020 09:50:52 -0700 Subject: [PATCH 173/456] 1.34 --- deps.edn | 2 +- project.clj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deps.edn b/deps.edn index 8f34b3b..2bfa4a8 100644 --- a/deps.edn +++ b/deps.edn @@ -5,7 +5,7 @@ ;; don't change target/classes (do not remove it from :paths and ;; do not rename it, otherwise edit build.clj) ;; uncomment java-src if you have java sources. - :paths ["src" "java" "resources" "target/classes"] + :paths ["src" "java"] :deps { org.clojure/clojure {:mvn/version "1.10.1"} diff --git a/project.clj b/project.clj index 1d379fd..9d83c90 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.34-SNAPSHOT" +(defproject cnuernber/libpython-clj "1.34" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "Eclipse Public License" From 6f9dfde064c9d975c8fa321e4147fddba5beb533 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 2 Feb 2020 09:51:22 -0700 Subject: [PATCH 174/456] snap --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 9d83c90..4c40ea6 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.34" +(defproject cnuernber/libpython-clj "1.35-SNAPSHOT" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "Eclipse Public License" From e4b080bbf11eaaaab08c185823d8b39463c68073 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 2 Feb 2020 09:52:24 -0700 Subject: [PATCH 175/456] 1.35 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 4c40ea6..0c6cbc3 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.35-SNAPSHOT" +(defproject cnuernber/libpython-clj "1.35" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "Eclipse Public License" From f5c73698c139adde85fd52c33d1e4512d90eb50c Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 2 Feb 2020 09:52:31 -0700 Subject: [PATCH 176/456] snap --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 0c6cbc3..9f4cda1 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.35" +(defproject cnuernber/libpython-clj "1.36-SNAPSHOT" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "Eclipse Public License" From 70dc24f78a28cb5136511a39b4d80de927954e8f Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 2 Feb 2020 10:20:26 -0700 Subject: [PATCH 177/456] upgraded core.async to latest --- deps.edn | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deps.edn b/deps.edn index 2bfa4a8..c5c879b 100644 --- a/deps.edn +++ b/deps.edn @@ -5,12 +5,12 @@ ;; don't change target/classes (do not remove it from :paths and ;; do not rename it, otherwise edit build.clj) ;; uncomment java-src if you have java sources. - :paths ["src" "java"] + :paths ["src" "java" "resources" "target/classes"] :deps { org.clojure/clojure {:mvn/version "1.10.1"} camel-snake-kebab {:mvn/version "0.4.0"} - techascent/tech.datatype {:mvn/version "4.73"} + techascent/tech.datatype {:mvn/version "4.74"} org.clojure/data.json {:mvn/version "0.2.7"}} :build { @@ -19,7 +19,7 @@ :group-id "cnuernber" :artifact-id "libpython-clj" - :artifact-version "1.34-SNAPSHOT" + :artifact-version "1.36-SNAPSHOT" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :scm {:url "git@github.com:cnuernber/libpython-clj.git"} From 9c6622a6f03d752961cea64c44a78ba0aeb86add Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 2 Feb 2020 10:34:20 -0700 Subject: [PATCH 178/456] 1.36 --- CHANGELOG.md | 11 ++++++++++- deps.edn | 2 +- project.clj | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba42fb1..c37f2ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ # Time for a ChangeLog! -## 1.34-SNAPSHOT +## 1.36 + +* clojure.core.async upgrade + + +## 1.35 * [Examples are now done by gigasquid](https://github.com/gigasquid/libpython-clj-examples) @@ -11,6 +16,10 @@ * bugfix -- python.str now loaded by `import-python` + +## 1.34 + * Skipped due to build system issues. + ## 1.33 * Better [windows anaconda support](https://github.com/cnuernber/libpython-clj/pull/67) diff --git a/deps.edn b/deps.edn index c5c879b..ba9d381 100644 --- a/deps.edn +++ b/deps.edn @@ -5,7 +5,7 @@ ;; don't change target/classes (do not remove it from :paths and ;; do not rename it, otherwise edit build.clj) ;; uncomment java-src if you have java sources. - :paths ["src" "java" "resources" "target/classes"] + :paths ["src" "java"] :deps { org.clojure/clojure {:mvn/version "1.10.1"} diff --git a/project.clj b/project.clj index 9f4cda1..5325ab6 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.36-SNAPSHOT" +(defproject cnuernber/libpython-clj "1.36" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "Eclipse Public License" From 374284ef181f640e9dfca8bfa155dd015ce1498f Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 2 Feb 2020 10:34:51 -0700 Subject: [PATCH 179/456] snap --- deps.edn | 2 +- project.clj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deps.edn b/deps.edn index ba9d381..c5c879b 100644 --- a/deps.edn +++ b/deps.edn @@ -5,7 +5,7 @@ ;; don't change target/classes (do not remove it from :paths and ;; do not rename it, otherwise edit build.clj) ;; uncomment java-src if you have java sources. - :paths ["src" "java"] + :paths ["src" "java" "resources" "target/classes"] :deps { org.clojure/clojure {:mvn/version "1.10.1"} diff --git a/project.clj b/project.clj index 5325ab6..fbe8e7d 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.36" +(defproject cnuernber/libpython-clj "1.37-SNAPSHOT" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "Eclipse Public License" From 4ab6846531188d826a74b221b559351363602240 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 4 Feb 2020 07:09:24 -0700 Subject: [PATCH 180/456] Moving back to lein on main branch for now --- deps.edn | 89 --------------- dev/resources/logback.xml | 11 -- dev/src/build.clj | 217 ------------------------------------ dev/src/user.clj | 9 -- dockerfiles/CondaDockerfile | 3 + project.clj | 10 +- scripts/conda-repl | 4 +- 7 files changed, 11 insertions(+), 332 deletions(-) delete mode 100644 deps.edn delete mode 100644 dev/resources/logback.xml delete mode 100644 dev/src/build.clj delete mode 100644 dev/src/user.clj diff --git a/deps.edn b/deps.edn deleted file mode 100644 index c5c879b..0000000 --- a/deps.edn +++ /dev/null @@ -1,89 +0,0 @@ -{ - :mvn/repos {"clojars" {:url "https://repo.clojars.org/"} - "central" {:url "https://repo1.maven.org/maven2/"}} - - ;; don't change target/classes (do not remove it from :paths and - ;; do not rename it, otherwise edit build.clj) - ;; uncomment java-src if you have java sources. - :paths ["src" "java" "resources" "target/classes"] - - :deps { - org.clojure/clojure {:mvn/version "1.10.1"} - camel-snake-kebab {:mvn/version "0.4.0"} - techascent/tech.datatype {:mvn/version "4.74"} - org.clojure/data.json {:mvn/version "0.2.7"}} - - :build { - :java-source-paths "java" - :javac-options ["-target" "1.8" "-source" "1.8" "-Xlint:-options"] - - :group-id "cnuernber" - :artifact-id "libpython-clj" - :artifact-version "1.36-SNAPSHOT" - :description "libpython bindings to the techascent ecosystem" - :url "http://github.com/cnuernber/libpython-clj" - :scm {:url "git@github.com:cnuernber/libpython-clj.git"} - :license {:name "EPL-2.0" - :url "https://www.eclipse.org/legal/epl-2.0/"}} - - - :aliases { - ;;; clj -A:repl - :repl {:extra-deps {criterium {:mvn/version "0.4.5"} - nrepl {:mvn/version "0.6.0"} - healthsamurai/matcho {:mvn/version "0.3.3"} - hashp {:mvn/version "0.1.1"} - badigeon/badigeon {:git/url "https://github.com/EwenG/badigeon.git" - :sha "c5d7d8f9c44fee2f193ef924cdf8a485aee539c5"} - ch.qos.logback/logback-classic {:mvn/version "1.1.3"}} - :jvm-opts ["-Duser.timezone=UTC"] - :extra-paths ["dev/src" "resources" "test"] - :main-opts ["--main" "nrepl.cmdline"]} - ;;; clj -A:test - :test {:extra-paths ["test" "src/test/clojure"] - :extra-deps {org.clojure/test.check {:mvn/version "RELEASE"} - com.cognitect/test-runner - {:git/url "https://github.com/cognitect-labs/test-runner" - :sha "3cb0a9daf1cb746259dc8309b218f9211ad3b33b"}} - :main-opts ["-m" "cognitect.test-runner" - "-d" "test" - "-d" "src/test/clojure"]} - - ;; clean : clj -A:clean - :clean {:main-opts ["-e" "(load-file,\"dev/src/build.clj\"),(build/clean)"] - :extra-deps {badigeon/badigeon {:git/url "https://github.com/EwenG/badigeon.git" - :sha "c5d7d8f9c44fee2f193ef924cdf8a485aee539c5"} - ch.qos.logback/logback-classic {:mvn/version "1.1.3"}} - ;;; to supress noisy logs with logback file - :extra-paths ["dev/resources"]} - ;; build jar (compile java, clj files): clj -A:jar - :jar {:main-opts ["-e" "(load-file,\"dev/src/build.clj\"),(build/clean),(build/compile-java),(build/jar)"] - :extra-deps {badigeon/badigeon {:git/url "https://github.com/EwenG/badigeon.git" - :sha "c5d7d8f9c44fee2f193ef924cdf8a485aee539c5"} - ch.qos.logback/logback-classic {:mvn/version "1.1.3"}} - ;;; to supress noisy logs with logback file - :extra-paths ["dev/resources"]} - - ;; compile java classes only: clj -A:javac - :javac {:main-opts ["-e" "(load-file,\"dev/src/build.clj\"),(build/compile-java)"] - :extra-deps {badigeon/badigeon {:git/url "https://github.com/EwenG/badigeon.git" - :sha "c5d7d8f9c44fee2f193ef924cdf8a485aee539c5"} - ch.qos.logback/logback-classic {:mvn/version "1.1.3"}} - ;;; to supress noisy logs with logback file - :extra-paths ["dev/resources"]} - - ;; install to local .m2: clj -A:install - :install {:main-opts ["-e" "(load-file,\"dev/src/build.clj\"),(build/install)"] - :extra-deps {badigeon/badigeon {:git/url "https://github.com/EwenG/badigeon.git" - :sha "c5d7d8f9c44fee2f193ef924cdf8a485aee539c5"} - ch.qos.logback/logback-classic {:mvn/version "1.1.3"}} - ;;; to supress noisy logs with logback file - :extra-paths ["dev/resources"]} - - ;; public to clojars: clj -A:deploy - :deploy {:main-opts ["-e" "(load-file,\"dev/src/build.clj\"),(build/deploy)"] - :extra-deps {badigeon/badigeon {:git/url "https://github.com/EwenG/badigeon.git" - :sha "c5d7d8f9c44fee2f193ef924cdf8a485aee539c5"} - ch.qos.logback/logback-classic {:mvn/version "1.1.3"}} - ;;; to supress noisy logs with logback file - :extra-paths ["dev/resources"]}}} diff --git a/dev/resources/logback.xml b/dev/resources/logback.xml deleted file mode 100644 index b70d039..0000000 --- a/dev/resources/logback.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - - - - - - - diff --git a/dev/src/build.clj b/dev/src/build.clj deleted file mode 100644 index 8655466..0000000 --- a/dev/src/build.clj +++ /dev/null @@ -1,217 +0,0 @@ -(ns build - (:require [badigeon.clean :as clean] - [badigeon.classpath :as classpath] - [badigeon.javac :as javac] - [badigeon.jar :as jar] - [badigeon.install :as install] - [badigeon.sign :as sign] - [badigeon.deploy :as deploy] - [badigeon.pom :as pom] - [clojure.tools.deps.alpha.reader :as deps-reader] - [clojure.data.xml :as xml] - [clojure.string :as str] - [clojure.data.xml.tree :as tree] - [clojure.data.xml.event :as event] - [clojure.java.io :as io]) - (:import (java.util.jar JarEntry JarOutputStream) - (java.nio.file Path Paths) - [java.io File Reader ByteArrayOutputStream] - [clojure.data.xml.node Element] - (java.net URI))) - - -(def deps-content (deps-reader/slurp-deps "deps.edn")) -(def java-src-folder (-> deps-content :build :java-source-paths)) -(def javac-options (-> deps-content :build :javac-options)) -(def group-id (-> deps-content :build :group-id)) -(def artifact-id (-> deps-content :build :artifact-id)) -(def group-artefact-id (symbol (str group-id "/" artifact-id))) -(def artifact-version (-> deps-content :build :artifact-version)) -(def artifact-description (-> deps-content :build :description)) -(def artifact-url (-> deps-content :build :url)) -(def artifact-scm (-> deps-content :build :scm)) -(def artifact-license (-> deps-content :build :license)) -(def jar-name (str "target/" artifact-id "-" artifact-version ".jar")) - - -(defn clean - [] - (clean/clean "target" - {;; By default, clean does not allow deleting folders outside the target directory, - ;; unless :allow-outside-target? is true - :allow-outside-target? false})) - -(defn make-classpath - [] - ;; Builds a classpath by using the provided deps spec or, by default, the deps.edn file of the current project. - ;; Returns the built classpath as a string. - - (classpath/make-classpath)) - -(defn compile-java - ([emit-path] - ;; Compile java sources under the java-src-folder directory - (when java-src-folder - (javac/javac java-src-folder {;; Emit class files to the target/classes directory - :compile-path emit-path - ;; Additional options used by the javac command - :javac-options javac-options}))) - ([] - (compile-java "target/classes"))) - - -(defn- parse-xml - [^Reader rdr] - (let [roots (tree/seq-tree - event/event-element event/event-exit? event/event-node - (xml/event-seq rdr {:include-node? #{:element :characters :comment}}))] - (first (filter #(instance? Element %) (first roots))))) - - -(xml/alias-uri 'ppom "http://maven.apache.org/POM/4.0.0") - -(defn- add-extra-stuff->pom - [] - (let [pom-file (io/file "pom.xml") - pom (with-open [rdr (io/reader pom-file)] - (-> rdr - parse-xml)) - pom (if artifact-url - (#'badigeon.pom/xml-update pom [::ppom/url] (xml/sexp-as-element [::ppom/url artifact-url])) - pom) - pom (if artifact-description - (#'badigeon.pom/xml-update pom [::ppom/description] (xml/sexp-as-element [::ppom/description artifact-description])) - pom) - pom (if artifact-scm - (#'badigeon.pom/xml-update pom [::ppom/scm] (xml/sexp-as-element [::ppom/scm [::ppom/url (:url artifact-scm)]])) - pom) - pom (if artifact-license - (#'badigeon.pom/xml-update pom [::ppom/license] (xml/sexp-as-element [::ppom/licenses - [::ppom/license - [::ppom/url (or (:url artifact-license) "")] - [::ppom/name (or (:name artifact-license)) ""]]])) - - pom)] - - - (spit pom-file (str/replace (xml/indent-str pom) #"\n\s+\n" "\n")) - (spit pom-file "" :append true))) - -(defn make-pom - [] - (when (.exists (io/file "pom.xml")) - (io/delete-file "pom.xml")) - (let [lib (symbol group-artefact-id) - maven-coords {:mvn/version artifact-version}] - (pom/sync-pom lib maven-coords (-> (deps-reader/slurp-deps "deps.edn"))) - (add-extra-stuff->pom))) - -(defn jar - [] - (when (.exists (io/file "pom.xml")) - (io/delete-file "pom.xml")) - ;; Package the project into a jar file - (jar/jar group-artefact-id {:mvn/version artifact-version} - {;; The jar file produced. - :out-path jar-name - ;;;; Adds a \"Main\" entry to the jar manifest with the value - ;;:main main-ns - ;; Additional key/value pairs to add to the jar manifest. If a value is a collection, a manifest section is built for it. - ;;:manifest {"Project-awesome-level" "super-great" - ;; :my-section-1 [["MyKey1" "MyValue1"] ["MyKey2" "MyValue2"]] - ;; :my-section-2 {"MyKey3" "MyValue3" "MyKey4" "MyValue4"}} - - ;; By default Badigeon add entries for all files in the directory listed in the - ;; :paths section of the deps.edn file. This can be overridden here. - ;;:paths ["src" "target/classes" "java-src"] - - ;; The dependencies to be added to the \"dependencies\" section of the pom.xml file. - ;; When not specified, defaults to the :deps entry of the deps.edn file, without - ;; merging the user-level and system-level deps.edn files - ;;:deps '{org.clojure/clojure {:mvn/version "1.10.1"}} - ;; The repositories to be added to the \"repositories\" section of the pom.xml file. - ;; When not specified, default to nil - even if the deps.edn files contains - ;; a :mvn/repos entry. - :mvn/repos '{"clojars" {:url "https://repo.clojars.org/"}} - ;; A predicate used to excludes files from beeing added to the jar. - ;; The predicate is a function of two parameters: The path of the directory - ;; beeing visited (among the :paths of the project) and the path of the file - ;; beeing visited under this directory. - :exclusion-predicate badigeon.jar/default-exclusion-predicate - ;; A function to add files to the jar that would otherwise not have been added to it. - ;; The function must take two parameters: the path of the root directory of the - ;; project and the file being visited under this directory. When the function - ;; returns a falsy value, the file is not added to the jar. Otherwise the function - ;; must return a string which represents the path within the jar where the file - ;; is copied. - ;; :inclusion-path (partial badigeon.jar/default-inclusion-path "badigeon" "badigeon") - ;; By default git and local dependencies are not allowed. Set allow-all-dependencies? to true to allow them - :allow-all-dependencies? true}) - (add-extra-stuff->pom) - (println "Successfully created jar file: " jar-name)) - - - -(defn install - [] - ;; Install the created jar file into the local maven repository. - (let [local-repo (str (System/getProperty "user.home") "/.m2/repository")] - (install/install group-artefact-id {:mvn/version artifact-version} - ;; The jar file to be installed - jar-name - ;; The pom.xml file to be installed. This file is generated when creating the jar with the badigeon.jar/jar function. - "pom.xml" - {;; The local repository where the jar should be installed. - :local-repo local-repo}) - (println "Successfully installed to local repo: " local-repo))) - -(defn deploy - [] - ;; Deploy the created jar file to a remote repository. - (let [;; Artifacts are maps with a required :file-path key and an optional :extension key - artifacts [{:file-path jar-name :extension "jar"} - {:file-path "pom.xml" :extension "pom"}] - ;; Artifacts must be signed when deploying non-snapshot versions of artifacts. - artifacts (badigeon.sign/sign artifacts {;; The gpg command can be customized - :command "gpg"})] - ;; The gpg key used for signing. Defaults to the first private key found in your keyring. - ;;:gpg-key "root@eruditorum.org" - - - ;; default to reading the credentials from ~/.m2/settings.xml. uncomment to prompt user credentials. - ;;;; Prompt for a username - ;;username (badigeon.prompt/prompt "Username (clojars): ") - ;;;; Prompt for a password using the process standard input and without echoing. - ;;password (badigeon.prompt/prompt-password "Password (clojars): ") - - (badigeon.deploy/deploy - group-artefact-id artifact-version - artifacts - {;; :id is used to match the repository in the ~/.m2/settings.xml for credentials when no credentials are explicitly provided. - :id "clojars" - ;; The URL of the repository to deploy to. - :url "https://repo.clojars.org/"} - { - ;; Take creds from ~/.m2/settings.xml - - ;; The credentials used when authenticating to the remote repository. - ;; When none is provided, default to reading the credentials from ~/.m2/settings.xml - ;;:credentials {:username username :password password - ;; ;;:private-key "/path/to/private-key" :passphrase "passphrase" - ;; } - ;; When allow-unsigned? is false, artifacts must be signed when deploying non-snapshot versions of artifacts. Default to false. - :allow-unsigned? false}) - (println "Success!"))) - -(comment - (def cp (make-classpath)) - (clean) - (compile-java) - (extract-classes-from-deps) - (make-pom) - (add-extra-stuff->pom) - (jar) - (install)) - diff --git a/dev/src/user.clj b/dev/src/user.clj deleted file mode 100644 index 94ca101..0000000 --- a/dev/src/user.clj +++ /dev/null @@ -1,9 +0,0 @@ -(ns user) - -;; debug print with #p -(require 'hashp.core) - -(def dev-mode true) -(println "***************************") -(println {:msg "dev mode" :status (if (true? dev-mode) "on" "off")}) -(println "***************************") diff --git a/dockerfiles/CondaDockerfile b/dockerfiles/CondaDockerfile index bdc956b..aad9560 100644 --- a/dockerfiles/CondaDockerfile +++ b/dockerfiles/CondaDockerfile @@ -15,6 +15,9 @@ RUN apt-get -qq update && apt-get -qq -y install curl wget bzip2 openjdk-8-jdk-h && curl -o install-clojure https://download.clojure.org/install/linux-install-${CLOJURE_TOOLS_VERSION}.sh \ && chmod +x install-clojure \ && ./install-clojure && rm install-clojure \ + && wget https://raw.githubusercontent.com/technomancy/leiningen/stable/bin/lein \ + && chmod a+x lein \ + && mv lein /usr/bin \ && apt-get -qq -y autoremove \ && apt-get autoclean \ && rm -rf /var/lib/apt/lists/* /var/log/dpkg.log \ diff --git a/project.clj b/project.clj index fbe8e7d..a819bb7 100644 --- a/project.clj +++ b/project.clj @@ -3,9 +3,9 @@ :url "http://github.com/cnuernber/libpython-clj" :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} - :plugins [[lein-tools-deps "0.4.5"]] - :middleware [lein-tools-deps.plugin/resolve-dependencies-with-deps-edn] - :lein-tools-deps/config {:config-files [:install :user :project]} - :profiles {:dev {:lein-tools-deps/config {:resolve-aliases [:test]}} - :uberjar {:aot :all}} + :dependencies [[org.clojure/clojure "1.10.1"] + [camel-snake-kebab "0.4.0"] + [techascent/tech.datatype "4.74"] + [org.clojure/data.json "0.2.7"]] + :profiles {:dev {:dependencies [[criterium "0.4.5"]]}} :java-source-paths ["java"]) diff --git a/scripts/conda-repl b/scripts/conda-repl index 2c68746..e1af4fc 100755 --- a/scripts/conda-repl +++ b/scripts/conda-repl @@ -6,5 +6,7 @@ source activate pyclj ## https://github.com/conda/conda/issues/9500#issuecomment-565753807 export LD_LIBRARY_PATH="$(python3-config --prefix)/lib" -clojure -A:repl -b 0.0.0.0 -p 2222 +lein update-in :dependencies conj \[nrepl\ \"0.6.0\"\]\ + -- update-in :plugins conj \[cider/cider-nrepl\ \"0.22.4\"\]\ + -- repl :headless :host localhost From dfc9d446c877b72181f543247802194aad68fcdb Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 4 Feb 2020 07:13:03 -0700 Subject: [PATCH 181/456] update travis to newer leingingen --- .travis.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9dceeb6..ad845f9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,17 +1,9 @@ language: clojure -lein: 2.8.2 +lein: 2.9.1 before_install: - sudo apt-get -y install python3 python3-pip - sudo python3 -mpip install numpy - - curl -O https://download.clojure.org/install/linux-install-1.10.1.507.sh - - chmod +x linux-install-1.10.1.507.sh - - sudo ./linux-install-1.10.1.507.sh - - clojure -Sdescribe addons: apt: update: true -install: clojure -Sdescribe -script: - - clojure -A:javac - - clojure -A:test From 2f6cef4eda51c343c20b6a35696ea8bcdeb594f2 Mon Sep 17 00:00:00 2001 From: "J.J. Tolton" Date: Sat, 8 Feb 2020 14:59:37 -0500 Subject: [PATCH 182/456] ISSUE-78: clj-new template and deps.edn parallel support (#79) * makefile changes * update travis to newer leingingen * WIP clj-new template * wip * fix namespace * templates * update changelog * project.clj * travis * fix travis * travis * travis * travis minimal * revert to working, update lein * add dev folders * minified deps.edn * better lein workflow * simplified clj-new template * fix clj-new template * clj-new fix * whitespace Co-authored-by: Chris Nuernberger --- .travis.yml | 6 +++- CHANGELOG.md | 24 +++++++++++++++ deps.edn | 11 +++++++ project.clj | 10 +++--- src/clj/new/libpython_clj.clj | 46 ++++++++++++++++++++++++++++ src/clj/new/libpython_clj/core.clj | 12 ++++++++ src/clj/new/libpython_clj/deps.edn | 11 +++++++ src/clj/new/libpython_clj/python.clj | 9 ++++++ 8 files changed, 123 insertions(+), 6 deletions(-) create mode 100644 deps.edn create mode 100644 src/clj/new/libpython_clj.clj create mode 100644 src/clj/new/libpython_clj/core.clj create mode 100644 src/clj/new/libpython_clj/deps.edn create mode 100644 src/clj/new/libpython_clj/python.clj diff --git a/.travis.yml b/.travis.yml index ad845f9..1002412 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,11 @@ lein: 2.9.1 before_install: - sudo apt-get -y install python3 python3-pip - sudo python3 -mpip install numpy + - curl -O https://download.clojure.org/install/linux-install-1.10.1.507.sh + - chmod +x linux-install-1.10.1.507.sh + - sudo ./linux-install-1.10.1.507.sh + - clojure -Sdescribe addons: apt: update: true - +install: clojure -Sdescribe diff --git a/CHANGELOG.md b/CHANGELOG.md index c37f2ad..5a6baa3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,28 @@ # Time for a ChangeLog! +## 1.37 + +* libpython-clj projects can now be created quickly with + +```bash + + clj -A:new https://github.com/cnuernber/libpython-clj@ +``` + **NOTE**: this assumes you have `clj-new` configured in you `~/.clojure/deps.edn` + profile. If you do not, you can use the following: + +```bash +clj -Sdeps '{:deps + {seancorfield/clj-new + {:mvn/version "0.8.6"}}}' \ + -m clj-new.create \ + https://github.com/cnuernber/libpython-clj@ \ + myname/myapp +``` + +* `deps.edn` now supported in parallel with `project.clj` + + ## 1.36 * clojure.core.async upgrade @@ -7,6 +30,7 @@ ## 1.35 + * [Examples are now done by gigasquid](https://github.com/gigasquid/libpython-clj-examples) * [datafy/nav](https://clojure.github.io/clojure/branch-master/clojure.datafy-api.html) are now extensible for custom Python objects. diff --git a/deps.edn b/deps.edn new file mode 100644 index 0000000..59e8dd5 --- /dev/null +++ b/deps.edn @@ -0,0 +1,11 @@ +{ + :paths ["src"] + + :deps { + org.clojure/clojure {:mvn/version "1.10.1"} + camel-snake-kebab {:mvn/version "0.4.0"} + techascent/tech.datatype {:mvn/version "4.74"} + org.clojure/data.json {:mvn/version "0.2.7"} + seancorfield/clj-new {:mvn/version "0.8.6"} + } + } diff --git a/project.clj b/project.clj index a819bb7..41c1601 100644 --- a/project.clj +++ b/project.clj @@ -3,9 +3,9 @@ :url "http://github.com/cnuernber/libpython-clj" :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} - :dependencies [[org.clojure/clojure "1.10.1"] - [camel-snake-kebab "0.4.0"] - [techascent/tech.datatype "4.74"] - [org.clojure/data.json "0.2.7"]] :profiles {:dev {:dependencies [[criterium "0.4.5"]]}} - :java-source-paths ["java"]) + :java-source-paths ["java"] + :plugins [[lein-tools-deps "0.4.5"]] + :middleware [lein-tools-deps.plugin/resolve-dependencies-with-deps-edn] + :lein-tools-deps/config {:config-files [:install :user :project]}) + diff --git a/src/clj/new/libpython_clj.clj b/src/clj/new/libpython_clj.clj new file mode 100644 index 0000000..8b2a953 --- /dev/null +++ b/src/clj/new/libpython_clj.clj @@ -0,0 +1,46 @@ +(ns clj.new.libpython-clj + (:require [clj.new.templates :refer [renderer project-name name-to-path ->files]])) + +(def render (renderer "libpython_clj")) + +(defn file-map->files [data file-map] + (apply ->files data (seq file-map))) + +(defn libpython-clj-template! [name & {force :force? dir :dir}] + (let [data {:name (project-name name) + :base (clojure.string/replace + (project-name name) + #"(.*?)[.](.*$)" + "$1") + :suffix (clojure.string/replace + (project-name name) + #"(.*?)[.](.*$)" + "$2") + :sanitized (name-to-path name)} + {base :base} data] + + (println (str "Generating libpython-clj template for " + (:name data) "at") (:sanitized data) ".\n\n" + "For the latest information, please check out " + "https://github.com/cnuernber/libpython-clj\n" + "or join us for discussion at " + "https://clojurians.zulipchat.com/#narrow/stream/215609-libpython-clj-dev") + + (with-bindings {#'clj.new.templates/*force?* force + #'clj.new.templates/*dir* dir} + (file-map->files + data + {"deps.edn" (render "deps.edn" data) + (format "src/%s/%s.clj" (:base data) (:suffix data)) (render "core.clj" data) + (format "src/%s/python.clj" (:base data)) (render "python.clj" data)})))) + + +(defn libpython-clj [name] + (libpython-clj-template! name)) + +(comment + (libpython-clj-template! + "mydomain.myapp" + :dir "/tmp/data" + :force? true)) + diff --git a/src/clj/new/libpython_clj/core.clj b/src/clj/new/libpython_clj/core.clj new file mode 100644 index 0000000..a233d2d --- /dev/null +++ b/src/clj/new/libpython_clj/core.clj @@ -0,0 +1,12 @@ +(ns {{base}}.{{suffix}} + (:require [libpython-clj.python :as py :refer [py. py.- py.. py* py**]] + {{base}}.python + [libpython-clj.require :refer [require-python import-python]])) + +(import-python) + +(comment + (require-python 'os) + (os/getcwd)) + + diff --git a/src/clj/new/libpython_clj/deps.edn b/src/clj/new/libpython_clj/deps.edn new file mode 100644 index 0000000..f7545a6 --- /dev/null +++ b/src/clj/new/libpython_clj/deps.edn @@ -0,0 +1,11 @@ +{ + :mvn/repos {"clojars" {:url "https://repo.clojars.org/"} + "central" {:url "https://repo1.maven.org/maven2/"}} + + :paths ["src" "resources"] + + :deps { + org.clojure/clojure {:mvn/version "1.10.1"} + cnuernber/libpython-clj {:mvn/version "1.36"} + } + } diff --git a/src/clj/new/libpython_clj/python.clj b/src/clj/new/libpython_clj/python.clj new file mode 100644 index 0000000..3f3b89d --- /dev/null +++ b/src/clj/new/libpython_clj/python.clj @@ -0,0 +1,9 @@ +(ns {{base}}.python + (:require [libpython-clj.python :as py])) + +(defn initialize-python! + ([] (py/initialize!)) + ([python-executable] + (py/initialize! :python-executable python-executable))) + +(initialize-python! "python3.7") From 96f65498ec535bec794d6075df4255d61325038b Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sat, 8 Feb 2020 13:20:21 -0700 Subject: [PATCH 183/456] Moving files to a clj-template directory to minimize libpy deps --- CHANGELOG.md | 2 +- clj-template/deps.edn | 8 ++++++++ clj-template/project.clj | 8 ++++++++ {src => clj-template/src}/clj/new/libpython_clj.clj | 8 ++++---- {src => clj-template/src}/clj/new/libpython_clj/core.clj | 0 {src => clj-template/src}/clj/new/libpython_clj/deps.edn | 0 .../src}/clj/new/libpython_clj/python.clj | 5 +++++ 7 files changed, 26 insertions(+), 5 deletions(-) create mode 100644 clj-template/deps.edn create mode 100644 clj-template/project.clj rename {src => clj-template/src}/clj/new/libpython_clj.clj (97%) rename {src => clj-template/src}/clj/new/libpython_clj/core.clj (100%) rename {src => clj-template/src}/clj/new/libpython_clj/deps.edn (100%) rename {src => clj-template/src}/clj/new/libpython_clj/python.clj (57%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a6baa3..dae0d4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## 1.37 -* libpython-clj projects can now be created quickly with +* libpython-clj projects can now be created quickly with: ```bash diff --git a/clj-template/deps.edn b/clj-template/deps.edn new file mode 100644 index 0000000..8297390 --- /dev/null +++ b/clj-template/deps.edn @@ -0,0 +1,8 @@ +{ + :paths ["src"] + + :deps { + org.clojure/clojure {:mvn/version "1.10.1"} + seancorfield/clj-new {:mvn/version "0.8.6"} + } + } diff --git a/clj-template/project.clj b/clj-template/project.clj new file mode 100644 index 0000000..07ed992 --- /dev/null +++ b/clj-template/project.clj @@ -0,0 +1,8 @@ +(defproject libpython-clj/clj-template "1.0-SNAPSHOT" + :description "libpython bindings to the techascent ecosystem" + :url "http://github.com/cnuernber/libpython-clj" + :license {:name "Eclipse Public License" + :url "http://www.eclipse.org/legal/epl-v10.html"} + :plugins [[lein-tools-deps "0.4.5"]] + :middleware [lein-tools-deps.plugin/resolve-dependencies-with-deps-edn] + :lein-tools-deps/config {:config-files [:install :user :project]}) diff --git a/src/clj/new/libpython_clj.clj b/clj-template/src/clj/new/libpython_clj.clj similarity index 97% rename from src/clj/new/libpython_clj.clj rename to clj-template/src/clj/new/libpython_clj.clj index 8b2a953..0430dbf 100644 --- a/src/clj/new/libpython_clj.clj +++ b/clj-template/src/clj/new/libpython_clj.clj @@ -18,7 +18,7 @@ "$2") :sanitized (name-to-path name)} {base :base} data] - + (println (str "Generating libpython-clj template for " (:name data) "at") (:sanitized data) ".\n\n" "For the latest information, please check out " @@ -41,6 +41,6 @@ (comment (libpython-clj-template! "mydomain.myapp" - :dir "/tmp/data" - :force? true)) - + :dir "testdir" + :force? true) + ) diff --git a/src/clj/new/libpython_clj/core.clj b/clj-template/src/clj/new/libpython_clj/core.clj similarity index 100% rename from src/clj/new/libpython_clj/core.clj rename to clj-template/src/clj/new/libpython_clj/core.clj diff --git a/src/clj/new/libpython_clj/deps.edn b/clj-template/src/clj/new/libpython_clj/deps.edn similarity index 100% rename from src/clj/new/libpython_clj/deps.edn rename to clj-template/src/clj/new/libpython_clj/deps.edn diff --git a/src/clj/new/libpython_clj/python.clj b/clj-template/src/clj/new/libpython_clj/python.clj similarity index 57% rename from src/clj/new/libpython_clj/python.clj rename to clj-template/src/clj/new/libpython_clj/python.clj index 3f3b89d..ae532c5 100644 --- a/src/clj/new/libpython_clj/python.clj +++ b/clj-template/src/clj/new/libpython_clj/python.clj @@ -6,4 +6,9 @@ ([python-executable] (py/initialize! :python-executable python-executable))) + +;; If you are using environments, you can specialized which python executable +;; you bind to below. Make sure you do this before you call 'require-python' +;; in any file. + (initialize-python! "python3.7") From 2aacdbbe2fffbd8b95910a0ce761b4cf6039a305 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sat, 8 Feb 2020 13:20:32 -0700 Subject: [PATCH 184/456] really minimize deps --- deps.edn | 1 - 1 file changed, 1 deletion(-) diff --git a/deps.edn b/deps.edn index 59e8dd5..4584b1f 100644 --- a/deps.edn +++ b/deps.edn @@ -6,6 +6,5 @@ camel-snake-kebab {:mvn/version "0.4.0"} techascent/tech.datatype {:mvn/version "4.74"} org.clojure/data.json {:mvn/version "0.2.7"} - seancorfield/clj-new {:mvn/version "0.8.6"} } } From 721b03b0ab07f4ceed5a527b29369bdbd385edaf Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sat, 8 Feb 2020 13:21:50 -0700 Subject: [PATCH 185/456] Attempting to keep version numbers same --- clj-template/project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clj-template/project.clj b/clj-template/project.clj index 07ed992..296c472 100644 --- a/clj-template/project.clj +++ b/clj-template/project.clj @@ -1,4 +1,4 @@ -(defproject libpython-clj/clj-template "1.0-SNAPSHOT" +(defproject libpython-clj/clj-template "1.36" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "Eclipse Public License" From 2473ad60d904f87d775a7eeef8555a9b7980ec64 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 9 Feb 2020 10:42:44 -0700 Subject: [PATCH 186/456] 1.36 --- CHANGELOG.md | 4 +++- project.clj | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dae0d4a..1b0f596 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ # Time for a ChangeLog! -## 1.37 +## 1.37-SNAPSHOT + +!! Undergoing work, potentially need a github project !! * libpython-clj projects can now be created quickly with: diff --git a/project.clj b/project.clj index 41c1601..8dc14fd 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cnuernber/libpython-clj "1.37-SNAPSHOT" +(defproject clj-python/libpython-clj "1.36" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "Eclipse Public License" @@ -8,4 +8,3 @@ :plugins [[lein-tools-deps "0.4.5"]] :middleware [lein-tools-deps.plugin/resolve-dependencies-with-deps-edn] :lein-tools-deps/config {:config-files [:install :user :project]}) - From 191ca270e5b63252deb21119a95cf420ab006ae5 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 9 Feb 2020 10:43:02 -0700 Subject: [PATCH 187/456] snap --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 8dc14fd..f448c11 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-python/libpython-clj "1.36" +(defproject clj-python/libpython-clj "1.37-SNAPSHOT" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "Eclipse Public License" From 04972161fc7b574ec9ad8c59d72039f48331ca98 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 9 Feb 2020 10:45:46 -0700 Subject: [PATCH 188/456] Updating readme and removing template --- README.md | 4 +- clj-template/deps.edn | 8 ---- clj-template/project.clj | 8 ---- clj-template/src/clj/new/libpython_clj.clj | 46 ------------------- .../src/clj/new/libpython_clj/core.clj | 12 ----- .../src/clj/new/libpython_clj/deps.edn | 11 ----- .../src/clj/new/libpython_clj/python.clj | 14 ------ 7 files changed, 2 insertions(+), 101 deletions(-) delete mode 100644 clj-template/deps.edn delete mode 100644 clj-template/project.clj delete mode 100644 clj-template/src/clj/new/libpython_clj.clj delete mode 100644 clj-template/src/clj/new/libpython_clj/core.clj delete mode 100644 clj-template/src/clj/new/libpython_clj/deps.edn delete mode 100644 clj-template/src/clj/new/libpython_clj/python.clj diff --git a/README.md b/README.md index d7fbb98..bab0429 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,8 @@ JNA libpython bindings to the tech ecosystem. -[![Clojars Project](https://img.shields.io/clojars/v/cnuernber/libpython-clj.svg)](https://clojars.org/cnuernber/libpython-clj) -[![travis integration](https://travis-ci.com/cnuernber/libpython-clj.svg?branch=master)](https://travis-ci.com/cnuernber/libpython-clj) +[![Clojars Project](https://img.shields.io/clojars/v/clj-python/libpython-clj.svg)](https://clojars.org/cnuernber/libpython-clj) +[![travis integration](https://travis-ci.com/clj-python/libpython-clj.svg?branch=master)](https://travis-ci.com/cnuernber/libpython-clj) * Bridge between JVM objects and Python objects easily; use Python in your Java and use some Java in your Python. diff --git a/clj-template/deps.edn b/clj-template/deps.edn deleted file mode 100644 index 8297390..0000000 --- a/clj-template/deps.edn +++ /dev/null @@ -1,8 +0,0 @@ -{ - :paths ["src"] - - :deps { - org.clojure/clojure {:mvn/version "1.10.1"} - seancorfield/clj-new {:mvn/version "0.8.6"} - } - } diff --git a/clj-template/project.clj b/clj-template/project.clj deleted file mode 100644 index 296c472..0000000 --- a/clj-template/project.clj +++ /dev/null @@ -1,8 +0,0 @@ -(defproject libpython-clj/clj-template "1.36" - :description "libpython bindings to the techascent ecosystem" - :url "http://github.com/cnuernber/libpython-clj" - :license {:name "Eclipse Public License" - :url "http://www.eclipse.org/legal/epl-v10.html"} - :plugins [[lein-tools-deps "0.4.5"]] - :middleware [lein-tools-deps.plugin/resolve-dependencies-with-deps-edn] - :lein-tools-deps/config {:config-files [:install :user :project]}) diff --git a/clj-template/src/clj/new/libpython_clj.clj b/clj-template/src/clj/new/libpython_clj.clj deleted file mode 100644 index 0430dbf..0000000 --- a/clj-template/src/clj/new/libpython_clj.clj +++ /dev/null @@ -1,46 +0,0 @@ -(ns clj.new.libpython-clj - (:require [clj.new.templates :refer [renderer project-name name-to-path ->files]])) - -(def render (renderer "libpython_clj")) - -(defn file-map->files [data file-map] - (apply ->files data (seq file-map))) - -(defn libpython-clj-template! [name & {force :force? dir :dir}] - (let [data {:name (project-name name) - :base (clojure.string/replace - (project-name name) - #"(.*?)[.](.*$)" - "$1") - :suffix (clojure.string/replace - (project-name name) - #"(.*?)[.](.*$)" - "$2") - :sanitized (name-to-path name)} - {base :base} data] - - (println (str "Generating libpython-clj template for " - (:name data) "at") (:sanitized data) ".\n\n" - "For the latest information, please check out " - "https://github.com/cnuernber/libpython-clj\n" - "or join us for discussion at " - "https://clojurians.zulipchat.com/#narrow/stream/215609-libpython-clj-dev") - - (with-bindings {#'clj.new.templates/*force?* force - #'clj.new.templates/*dir* dir} - (file-map->files - data - {"deps.edn" (render "deps.edn" data) - (format "src/%s/%s.clj" (:base data) (:suffix data)) (render "core.clj" data) - (format "src/%s/python.clj" (:base data)) (render "python.clj" data)})))) - - -(defn libpython-clj [name] - (libpython-clj-template! name)) - -(comment - (libpython-clj-template! - "mydomain.myapp" - :dir "testdir" - :force? true) - ) diff --git a/clj-template/src/clj/new/libpython_clj/core.clj b/clj-template/src/clj/new/libpython_clj/core.clj deleted file mode 100644 index a233d2d..0000000 --- a/clj-template/src/clj/new/libpython_clj/core.clj +++ /dev/null @@ -1,12 +0,0 @@ -(ns {{base}}.{{suffix}} - (:require [libpython-clj.python :as py :refer [py. py.- py.. py* py**]] - {{base}}.python - [libpython-clj.require :refer [require-python import-python]])) - -(import-python) - -(comment - (require-python 'os) - (os/getcwd)) - - diff --git a/clj-template/src/clj/new/libpython_clj/deps.edn b/clj-template/src/clj/new/libpython_clj/deps.edn deleted file mode 100644 index f7545a6..0000000 --- a/clj-template/src/clj/new/libpython_clj/deps.edn +++ /dev/null @@ -1,11 +0,0 @@ -{ - :mvn/repos {"clojars" {:url "https://repo.clojars.org/"} - "central" {:url "https://repo1.maven.org/maven2/"}} - - :paths ["src" "resources"] - - :deps { - org.clojure/clojure {:mvn/version "1.10.1"} - cnuernber/libpython-clj {:mvn/version "1.36"} - } - } diff --git a/clj-template/src/clj/new/libpython_clj/python.clj b/clj-template/src/clj/new/libpython_clj/python.clj deleted file mode 100644 index ae532c5..0000000 --- a/clj-template/src/clj/new/libpython_clj/python.clj +++ /dev/null @@ -1,14 +0,0 @@ -(ns {{base}}.python - (:require [libpython-clj.python :as py])) - -(defn initialize-python! - ([] (py/initialize!)) - ([python-executable] - (py/initialize! :python-executable python-executable))) - - -;; If you are using environments, you can specialized which python executable -;; you bind to below. Make sure you do this before you call 'require-python' -;; in any file. - -(initialize-python! "python3.7") From 06a0a27a2c1320cfe3cbed89c29fb88bbdc9f19f Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 9 Feb 2020 10:55:13 -0700 Subject: [PATCH 189/456] updated to reflect moved clj-template repo --- CHANGELOG.md | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b0f596..fc3ad5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,25 +3,6 @@ ## 1.37-SNAPSHOT !! Undergoing work, potentially need a github project !! - -* libpython-clj projects can now be created quickly with: - -```bash - - clj -A:new https://github.com/cnuernber/libpython-clj@ -``` - **NOTE**: this assumes you have `clj-new` configured in you `~/.clojure/deps.edn` - profile. If you do not, you can use the following: - -```bash -clj -Sdeps '{:deps - {seancorfield/clj-new - {:mvn/version "0.8.6"}}}' \ - -m clj-new.create \ - https://github.com/cnuernber/libpython-clj@ \ - myname/myapp -``` - * `deps.edn` now supported in parallel with `project.clj` From 70c97176bc4cb5460d8f35bb7cdb70858b35b1c1 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 9 Feb 2020 13:26:08 -0700 Subject: [PATCH 190/456] updating readme --- README.md | 55 ++----------------------------------------------------- 1 file changed, 2 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index bab0429..599db8f 100644 --- a/README.md +++ b/README.md @@ -147,67 +147,16 @@ we have an [introductory document](docs/new-to-clojure.md). * [create C ptr from numpy](https://docs.scipy.org/doc/numpy/reference/generated/numpy.ndarray.ctypes.html) -## How to run project tasks -This project has java sources in additon to regular clojure sources. -The build alias have the javac built into the tasks, however for running tests or repl you would -need to run that as a seperate step if you need to recompile the java classes. -Example: - -- `clojure -A:javac` -- `clojure -A:test` - -Also note, that if you have tools installed you may substitue the `clj` command for `clojure` below: - -- `clj -A:test` - - -### Compile Java classes - -```bash -$ clojure -A:javac -``` - -### REPL - -```bash -$ clojure -A:repl -nREPL server started on port 56785 on host localhost - nrepl://localhost:56785 -``` - - -### Tests - -```bash -$ clojure -A:test -``` - -### Package jar - -```bash -$ clojure -A:jar -``` - -### Clean target directory - -```bash -$ clojure -A:clean -``` - - -### Local install - To install jar to local .m2 : ```bash -$ clojure -A:install +$ lein install ``` ### Deploy to clojars -Put your clojars.org credentials to settings.xml (or uncomment login and password prompt in dev/src/build.clj). - ```bash -$ clojure -A:deploy +$ lein deploy clojars ``` This command will sign jar before deploy, using your gpg key. (see dev/src/build.clj for signing options) From c732dc65de629d9cd8a350a507795247517857e4 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 9 Feb 2020 13:27:07 -0700 Subject: [PATCH 191/456] editing --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 599db8f..a6144b3 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,8 @@ JNA libpython bindings to the tech ecosystem. -[![Clojars Project](https://img.shields.io/clojars/v/clj-python/libpython-clj.svg)](https://clojars.org/cnuernber/libpython-clj) -[![travis integration](https://travis-ci.com/clj-python/libpython-clj.svg?branch=master)](https://travis-ci.com/cnuernber/libpython-clj) +[![Clojars Project](https://img.shields.io/clojars/v/clj-python/libpython-clj.svg)](https://clojars.org/clj-python/libpython-clj) +[![travis integration](https://travis-ci.com/clj-python/libpython-clj.svg?branch=master)](https://travis-ci.com/clj-python/libpython-clj) * Bridge between JVM objects and Python objects easily; use Python in your Java and use some Java in your Python. From 3e500e354991f315fa9567536a4b16959a63e202 Mon Sep 17 00:00:00 2001 From: fong Date: Thu, 20 Feb 2020 09:05:05 -0500 Subject: [PATCH 192/456] move (.get libpy-base/gil-thread-id) into (locking ...) because interleaving (.get ...) (== ..) (.set ...) may lead to race condition. (#82) --- src/libpython_clj/python/interpreter.clj | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libpython_clj/python/interpreter.clj b/src/libpython_clj/python/interpreter.clj index e4d7ca5..526fded 100644 --- a/src/libpython_clj/python/interpreter.clj +++ b/src/libpython_clj/python/interpreter.clj @@ -337,12 +337,12 @@ print(json.dumps( "Grab the gil and use the main interpreter using reentrant acquire-gil pathway." [& body] `(do - (let [interp# (ensure-interpreter) - ^AtomicLong bound-thread# libpy-base/gil-thread-id - thread-id# (libpy-base/current-thread-id) - previously-bound-thread-id# (.get bound-thread#)] + (let [interp# (ensure-interpreter)] (locking interp# - (let [gil-state# (when-not (== thread-id# previously-bound-thread-id#) + (let [^AtomicLong bound-thread# libpy-base/gil-thread-id + previously-bound-thread-id# (.get bound-thread#) + thread-id# (libpy-base/current-thread-id) + gil-state# (when-not (== thread-id# previously-bound-thread-id#) (acquire-gil!))] (try ~@body From 8cf96fbc8afdcc7a96988e3357802ab6e165438e Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Thu, 20 Feb 2020 12:18:36 -0700 Subject: [PATCH 193/456] Fixes #84 (#85) --- src/libpython_clj/metadata.clj | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libpython_clj/metadata.clj b/src/libpython_clj/metadata.clj index 9ee4bc0..947f59c 100644 --- a/src/libpython_clj/metadata.clj +++ b/src/libpython_clj/metadata.clj @@ -209,8 +209,9 @@ (with-gil (->> (if (or (pyclass? item) (pymodule? item)) - (-> (vars item) - (py-proto/as-map)) + (->> (vars item) + (py-proto/as-map) + (into {})) (->> (py/dir item) (map (juxt identity #(get-attr item %))))) (map (fn [[att-name att-val]] From 1aa1d43421d3c9802ad3dca9012a8a2191abea97 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Fri, 28 Feb 2020 18:46:54 -0700 Subject: [PATCH 194/456] 1.37 --- CHANGELOG.md | 5 +++-- project.clj | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc3ad5c..cde4846 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,9 @@ # Time for a ChangeLog! -## 1.37-SNAPSHOT +## 1.37 -!! Undergoing work, potentially need a github project !! +* Fix for metadata generation of sys module that was failing. This needs a deeper fix. +* Race condition and stability fix. * `deps.edn` now supported in parallel with `project.clj` diff --git a/project.clj b/project.clj index f448c11..6eab293 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-python/libpython-clj "1.37-SNAPSHOT" +(defproject clj-python/libpython-clj "1.37" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "Eclipse Public License" From 461e733e5e578bc3ca60ed98073a1bfc3827c3ba Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Fri, 28 Feb 2020 18:57:41 -0700 Subject: [PATCH 195/456] snap --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 6eab293..d11e649 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-python/libpython-clj "1.37" +(defproject clj-python/libpython-clj "1.38-SNAPSHOT" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "Eclipse Public License" From c5f4466cd1d01f4b21e0c813f57420572207bb1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ertu=C4=9Frul=20=C3=87etin?= Date: Tue, 10 Mar 2020 20:02:01 +0300 Subject: [PATCH 196/456] added :repl-options instructions (#90) --- README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/README.md b/README.md index a6144b3..963f059 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,25 @@ This code is a concrete example that generates an ## Usage +#### Config namespace +```clojure +(ns my-py-clj.config + (:require [libpython-clj.python :as py])) + +;; When you use conda, it should look like this. +(py/initialize! :python-executable "/opt/anaconda3/envs/my_env/bin/python3.7" + :library-path "/opt/anaconda3/envs/my_env/lib/libpython3.7m.dylib") +``` + +#### Update project.clj +```clojure +{... + ;; This namespace going to run when the REPL is up. + :repl-options {:init-ns my-py-clj.config} +...} +``` + + ```clojure user> (require '[libpython-clj.require :refer [require-python]]) ...logging info.... From 96d15e7740258d7f13cceb6836abf0f075f57829 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Wed, 18 Mar 2020 15:45:24 -0600 Subject: [PATCH 197/456] upgrade to tech.datatype 4.88 --- deps.edn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps.edn b/deps.edn index 4584b1f..c5b6c82 100644 --- a/deps.edn +++ b/deps.edn @@ -4,7 +4,7 @@ :deps { org.clojure/clojure {:mvn/version "1.10.1"} camel-snake-kebab {:mvn/version "0.4.0"} - techascent/tech.datatype {:mvn/version "4.74"} + techascent/tech.datatype {:mvn/version "4.88"} org.clojure/data.json {:mvn/version "0.2.7"} } } From a9fcc6bcc22a47927cfb0affdac2b6124cbd056c Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Wed, 18 Mar 2020 15:46:45 -0600 Subject: [PATCH 198/456] 1.38 --- CHANGELOG.md | 4 ++++ project.clj | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cde4846..9956718 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## 1.37 +* Update to tech.datatype 4.88 - much faster group-by, lots of small improvements. + +## 1.37 + * Fix for metadata generation of sys module that was failing. This needs a deeper fix. * Race condition and stability fix. * `deps.edn` now supported in parallel with `project.clj` diff --git a/project.clj b/project.clj index d11e649..1fb36af 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-python/libpython-clj "1.38-SNAPSHOT" +(defproject clj-python/libpython-clj "1.38" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "Eclipse Public License" From 371e466646c0ba087d6415b66909e102fb94c961 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Wed, 18 Mar 2020 15:47:13 -0600 Subject: [PATCH 199/456] snap --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 1fb36af..8ec9e4c 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-python/libpython-clj "1.38" +(defproject clj-python/libpython-clj "1.39-SNAPSHOT" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "Eclipse Public License" From 1e828528ba482a27836275b1aa99ab437d6df1f5 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 31 Mar 2020 13:06:34 -0600 Subject: [PATCH 200/456] Upgrade datatype to 5.0 --- deps.edn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps.edn b/deps.edn index c5b6c82..a5b30cf 100644 --- a/deps.edn +++ b/deps.edn @@ -4,7 +4,7 @@ :deps { org.clojure/clojure {:mvn/version "1.10.1"} camel-snake-kebab {:mvn/version "0.4.0"} - techascent/tech.datatype {:mvn/version "4.88"} + techascent/tech.datatype {:mvn/version "5.0-alpha-1"} org.clojure/data.json {:mvn/version "0.2.7"} } } From 8f5efa2b2f1edbef1fcaa2271500f498f709793a Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 31 Mar 2020 13:07:33 -0600 Subject: [PATCH 201/456] changelog update for latest change. --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9956718..9281602 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Time for a ChangeLog! + +## 1.38-SNAPSHOT + +* `tech.datatype` is upgrade to 5.0 (!!). + ## 1.37 * Update to tech.datatype 4.88 - much faster group-by, lots of small improvements. From bade2c477c1afe327eb432b3c226be4d33f0e518 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 31 Mar 2020 13:11:16 -0600 Subject: [PATCH 202/456] simple changes to clean up file --- src/libpython_clj/python/np_array.clj | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/libpython_clj/python/np_array.clj b/src/libpython_clj/python/np_array.clj index a9076cd..8f33a27 100644 --- a/src/libpython_clj/python/np_array.clj +++ b/src/libpython_clj/python/np_array.clj @@ -2,14 +2,10 @@ (:require [tech.v2.datatype.protocols :as dtype-proto] [tech.v2.datatype.operation-provider :as op-provider] [tech.v2.tensor :as dtt] - [tech.v2.datatype.argtypes :as argtypes] [libpython-clj.python.interpreter :as py-interp] [libpython-clj.python.protocols :as py-proto] [libpython-clj.python.bridge :as py-bridge] - [libpython-clj.python.object :as py-object] - [libpython-clj.python.interop :as py-interop] - [tech.jna :as jna])) - + [libpython-clj.python.interop :as py-interop])) (defmethod py-proto/pyobject->jvm :ndarray From f178642187cef8613fbad2ed1db226c1ad18ff55 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sun, 5 Apr 2020 07:11:38 -0600 Subject: [PATCH 203/456] Fixes #88 --- docs/slicing.md | 67 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 docs/slicing.md diff --git a/docs/slicing.md b/docs/slicing.md new file mode 100644 index 0000000..5516ed6 --- /dev/null +++ b/docs/slicing.md @@ -0,0 +1,67 @@ +# Slicing And Slices + + +The way Python implements slicing is via overloading the `get-item` function call. +This is the call the Python interpreter makes under the covers whenever you use the +square bracket `[]` syntax. + + +For quite a few objects, that function call take a tuple of arguments. The trick to +numpy slicing is to create builtin slice objects with the appropriate arguments and +pass them into the `get-item` call in a tuple. + + +```clojure +user> (require '[libpython-clj.python :as py]) +nil +user> (require '[libpython-clj.require :refer [require-python]]) +... lotta logs ... +user> (require-python '[builtins]) +WARNING: AssertionError already refers to: class java.lang.AssertionError in namespace: builtins, being replaced by: #'builtins/AssertionError +WARNING: Exception already refers to: class java.lang.Exception in namespace: builtins, being replaced by: #'builtins/Exception +:ok +user> (doc builtins/slice) +------------------------- +builtins/slice +[[self & [args {:as kwargs}]]] + slice(stop) +slice(start, stop[, step]) + +Create a slice object. This is used for extended slicing (e.g. a[0:10:2]). +nil +user> (require-python '[numpy :as np]) +:ok + +user> (require-python '[numpy :as np]) +:ok +user> (def ary (-> (np/arange 9) + (np/reshape [3 3]))) +#'user/ary +user> ary +[[0 1 2] + [3 4 5] + [6 7 8]] + +user> (py/get-item ary [(builtins/slice 1 nil) (builtins/slice 1 nil)]) +[[4 5] + [7 8]] +user> (py/get-item ary [(builtins/slice -1) (builtins/slice 1 nil)]) +[[1 2] + [4 5]] +user> (py/get-item ary [(builtins/slice nil) (builtins/slice 1 nil)]) +[[1 2] + [4 5] + [7 8]] +user> (py/get-item ary [(builtins/slice nil) (builtins/slice 1 2)]) +[[1] + [4] + [7]] +user> (py/get-item ary [(builtins/slice nil) (builtins/slice 1 3)]) +[[1 2] + [4 5] + [7 8]] +user> (py/get-item ary [(builtins/slice nil) (builtins/slice 1 4)]) +[[1 2] + [4 5] + [7 8]] +``` From eba7aa02be63c06d8ef27c7f571e78e2e7855c22 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sat, 18 Apr 2020 16:02:50 -0600 Subject: [PATCH 204/456] 1.39 --- CHANGELOG.md | 43 ++++++++++++++++++++++--------------------- deps.edn | 2 +- project.clj | 2 +- 3 files changed, 24 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9281602..069d8d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ # Time for a ChangeLog! - +## 1.39 +* `tech.datatype` upgrade to version that supports datetime types. ## 1.38-SNAPSHOT @@ -16,7 +17,7 @@ * `deps.edn` now supported in parallel with `project.clj` -## 1.36 +## 1.36 * clojure.core.async upgrade @@ -26,11 +27,11 @@ * [Examples are now done by gigasquid](https://github.com/gigasquid/libpython-clj-examples) -* [datafy/nav](https://clojure.github.io/clojure/branch-master/clojure.datafy-api.html) are now extensible for custom Python objects. - Extend `libpython-clj.require/pydafy` and `libpython-clj.require/pynav` - respectively with the symbol of class you want to extend. See +* [datafy/nav](https://clojure.github.io/clojure/branch-master/clojure.datafy-api.html) are now extensible for custom Python objects. + Extend `libpython-clj.require/pydafy` and `libpython-clj.require/pynav` + respectively with the symbol of class you want to extend. See respective docstrings for details. - + * bugfix -- python.str now loaded by `import-python` @@ -42,33 +43,33 @@ * Better [windows anaconda support](https://github.com/cnuernber/libpython-clj/pull/67) thanks to [orolle](https://github.com/orolle). -* Moved to PyGILState* functions for GIL management. This mainly due to +* Moved to PyGILState* functions for GIL management. This mainly due to [FongHou](https://github.com/cnuernber/libpython-clj/commits?author=FongHou) in - PRs [here](https://github.com/cnuernber/libpython-clj/pull/64) and + PRs [here](https://github.com/cnuernber/libpython-clj/pull/64) and [here](https://github.com/cnuernber/libpython-clj/pull/65). * **BREAKING CHANGE** `require-python` now respects prefix lists -- - unfortunately, the previous syntax was incorrect. - ```clojure + unfortunately, the previous syntax was incorrect. + ```clojure ;; WRONG (syntax version < 1.33) - (require-python '(os math)) + (require-python '(os math)) ``` - would be equivalent to - ```clojure + would be equivalent to + ```clojure ;; (do (require-python 'os) (require-python 'math)) ``` the correct syntax for this SHOULD have been - ```clojure + ```clojure (require-python 'os 'math) ``` - - 1.33 fixes this mistake, and provides support for prefix lists, + + 1.33 fixes this mistake, and provides support for prefix lists, for example: - + ```clojure - (require-python + (require-python '[builtins :as python] - '(builtins + '(builtins [list :as python.list] [dict :as python.dict] [tuple :as python.tuple] @@ -76,9 +77,9 @@ [frozenset :as python.frozenset])) ``` (**Note**: this is done for you by the function `libpython-clj.require/import-python`) - + This fix brought to you by [jjtolton](https://github.com/jjtolton). - + ## 1.32 * DecRef now happens cooperatively in python thread. We used to use separate threads diff --git a/deps.edn b/deps.edn index a5b30cf..64856af 100644 --- a/deps.edn +++ b/deps.edn @@ -4,7 +4,7 @@ :deps { org.clojure/clojure {:mvn/version "1.10.1"} camel-snake-kebab {:mvn/version "0.4.0"} - techascent/tech.datatype {:mvn/version "5.0-alpha-1"} + techascent/tech.datatype {:mvn/version "5.0-beta-9"} org.clojure/data.json {:mvn/version "0.2.7"} } } diff --git a/project.clj b/project.clj index 8ec9e4c..dd87fd0 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-python/libpython-clj "1.39-SNAPSHOT" +(defproject clj-python/libpython-clj "1.39" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "Eclipse Public License" From 4ca44fd9c2017efeb478f60518847c54be2b5a80 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sat, 18 Apr 2020 16:02:59 -0600 Subject: [PATCH 205/456] snap --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index dd87fd0..7b40ea2 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-python/libpython-clj "1.39" +(defproject clj-python/libpython-clj "1.40-SNAPSHOT" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "Eclipse Public License" From e762878b220fb51b38defe65007b06ab575f704d Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sat, 18 Apr 2020 16:39:28 -0600 Subject: [PATCH 206/456] 1.40 --- CHANGELOG.md | 3 +++ deps.edn | 2 +- project.clj | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 069d8d6..2e89716 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ # Time for a ChangeLog! +## 1.40 +* `tech.datatype` fixed bug in argsort. + ## 1.39 * `tech.datatype` upgrade to version that supports datetime types. diff --git a/deps.edn b/deps.edn index 64856af..c835d1a 100644 --- a/deps.edn +++ b/deps.edn @@ -4,7 +4,7 @@ :deps { org.clojure/clojure {:mvn/version "1.10.1"} camel-snake-kebab {:mvn/version "0.4.0"} - techascent/tech.datatype {:mvn/version "5.0-beta-9"} + techascent/tech.datatype {:mvn/version "5.0-beta-10"} org.clojure/data.json {:mvn/version "0.2.7"} } } diff --git a/project.clj b/project.clj index 7b40ea2..2c04c8c 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-python/libpython-clj "1.40-SNAPSHOT" +(defproject clj-python/libpython-clj "1.40" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "Eclipse Public License" From 497f2f3421b79c9620110e68052b5f84693a61fc Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sat, 18 Apr 2020 16:39:35 -0600 Subject: [PATCH 207/456] snap --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 2c04c8c..d566b92 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-python/libpython-clj "1.40" +(defproject clj-python/libpython-clj "1.41-SNAPSHOT" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "Eclipse Public License" From 118843293d0adb4edcad1987736c6344a5d6a250 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sat, 25 Apr 2020 12:12:07 -0600 Subject: [PATCH 208/456] 1.41 --- CHANGELOG.md | 3 +++ deps.edn | 2 +- project.clj | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e89716..21dbc16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ # Time for a ChangeLog! +## 1.41 + * `tech.datatype` - latest version to work with `tech.ml.dataset`. + ## 1.40 * `tech.datatype` fixed bug in argsort. diff --git a/deps.edn b/deps.edn index c835d1a..7819577 100644 --- a/deps.edn +++ b/deps.edn @@ -4,7 +4,7 @@ :deps { org.clojure/clojure {:mvn/version "1.10.1"} camel-snake-kebab {:mvn/version "0.4.0"} - techascent/tech.datatype {:mvn/version "5.0-beta-10"} + techascent/tech.datatype {:mvn/version "5.0-beta-16"} org.clojure/data.json {:mvn/version "0.2.7"} } } diff --git a/project.clj b/project.clj index d566b92..8b0f8a2 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-python/libpython-clj "1.41-SNAPSHOT" +(defproject clj-python/libpython-clj "1.41" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "Eclipse Public License" From 49ab6e231f074e3f93dfabf867254c66145e6ff1 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sat, 25 Apr 2020 12:12:20 -0600 Subject: [PATCH 209/456] snap --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 8b0f8a2..b65ac31 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-python/libpython-clj "1.41" +(defproject clj-python/libpython-clj "1.42-SNAPSHOT" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "Eclipse Public License" From 2f206261034a970b52f4c4002b3cf18f562375f2 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Wed, 29 Apr 2020 16:32:14 -0600 Subject: [PATCH 210/456] 1.42 --- CHANGELOG.md | 3 +++ deps.edn | 2 +- project.clj | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21dbc16..bcfbee3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ # Time for a ChangeLog! +## 1.42 + * `tech.datatype` - latest version to work with `tech.ml.dataset`. + ## 1.41 * `tech.datatype` - latest version to work with `tech.ml.dataset`. diff --git a/deps.edn b/deps.edn index 7819577..a601101 100644 --- a/deps.edn +++ b/deps.edn @@ -4,7 +4,7 @@ :deps { org.clojure/clojure {:mvn/version "1.10.1"} camel-snake-kebab {:mvn/version "0.4.0"} - techascent/tech.datatype {:mvn/version "5.0-beta-16"} + techascent/tech.datatype {:mvn/version "5.0-beta-20"} org.clojure/data.json {:mvn/version "0.2.7"} } } diff --git a/project.clj b/project.clj index b65ac31..c350866 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-python/libpython-clj "1.42-SNAPSHOT" +(defproject clj-python/libpython-clj "1.42" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "Eclipse Public License" From a2cb7ba907405077e6be8eb202a9a4f32eb4431e Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Wed, 29 Apr 2020 16:32:23 -0600 Subject: [PATCH 211/456] snap --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index c350866..7fe5b1a 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-python/libpython-clj "1.42" +(defproject clj-python/libpython-clj "1.43-SNAPSHOT" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "Eclipse Public License" From aafc20f4efa9aed2329d7bf08b2d7fbb7e04170d Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 12 May 2020 12:25:50 -0600 Subject: [PATCH 212/456] 1.43 --- CHANGELOG.md | 3 +++ deps.edn | 2 +- project.clj | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bcfbee3..1574e41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ # Time for a ChangeLog! +## 1.43 + * `tech.datatype` - latest version to work with `tech.ml.dataset`. + ## 1.42 * `tech.datatype` - latest version to work with `tech.ml.dataset`. diff --git a/deps.edn b/deps.edn index a601101..b0b0a7d 100644 --- a/deps.edn +++ b/deps.edn @@ -4,7 +4,7 @@ :deps { org.clojure/clojure {:mvn/version "1.10.1"} camel-snake-kebab {:mvn/version "0.4.0"} - techascent/tech.datatype {:mvn/version "5.0-beta-20"} + techascent/tech.datatype {:mvn/version "5.0-beta-30"} org.clojure/data.json {:mvn/version "0.2.7"} } } diff --git a/project.clj b/project.clj index 7fe5b1a..7e0aac6 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-python/libpython-clj "1.43-SNAPSHOT" +(defproject clj-python/libpython-clj "1.43" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "Eclipse Public License" From b12b8acb672fe22855f6c81c83b04a0d0a2d7000 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Tue, 12 May 2020 12:26:04 -0600 Subject: [PATCH 213/456] snap --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 7e0aac6..4d1537b 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-python/libpython-clj "1.43" +(defproject clj-python/libpython-clj "1.44-SNAPSHOT" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "Eclipse Public License" From dfd4cf58bf5311ef437b1d87194cb6a2836bfda6 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sat, 16 May 2020 10:39:15 -0600 Subject: [PATCH 214/456] Fixes #98 - first attempt at making things really work for python 3.8 --- dockerfiles/CondaDockerfile | 3 + dockerfiles/Py38Dockerfile | 32 ++++++++ scripts/build-py38-docker | 7 ++ scripts/py38-repl | 6 ++ scripts/run-conda-docker | 2 +- scripts/run-py38-docker | 9 +++ src/libpython_clj/jna/base.clj | 4 +- src/libpython_clj/python/interpreter.clj | 93 +++++++++++++++--------- 8 files changed, 119 insertions(+), 37 deletions(-) create mode 100644 dockerfiles/Py38Dockerfile create mode 100755 scripts/build-py38-docker create mode 100755 scripts/py38-repl create mode 100755 scripts/run-py38-docker diff --git a/dockerfiles/CondaDockerfile b/dockerfiles/CondaDockerfile index aad9560..fdba41d 100644 --- a/dockerfiles/CondaDockerfile +++ b/dockerfiles/CondaDockerfile @@ -39,6 +39,9 @@ USER $USERNAME RUN conda create -n pyclj python=3.6 && conda install -n pyclj numpy mxnet \ && echo "source activate pyclj" > /home/$USERNAME/.bashrc +## cause leiningen to install +RUN lein -v + ## To install pip packages into the pyclj environment do RUN conda run -n pyclj python3 -mpip install numpy \ No newline at end of file diff --git a/dockerfiles/Py38Dockerfile b/dockerfiles/Py38Dockerfile new file mode 100644 index 0000000..41ec6a5 --- /dev/null +++ b/dockerfiles/Py38Dockerfile @@ -0,0 +1,32 @@ +# We will use Ubuntu for our image +FROM ubuntu:latest + +# Updating Ubuntu packages + +ARG CLOJURE_TOOLS_VERSION=1.10.1.507 + + +RUN apt-get -qq update && apt-get -qq -y install curl wget bzip2 openjdk-8-jdk-headless python3.8 libpython3.8 python3-pip \ + && curl -o install-clojure https://download.clojure.org/install/linux-install-${CLOJURE_TOOLS_VERSION}.sh \ + && chmod +x install-clojure \ + && ./install-clojure && rm install-clojure \ + && wget https://raw.githubusercontent.com/technomancy/leiningen/stable/bin/lein \ + && chmod a+x lein \ + && mv lein /usr/bin \ + && apt-get -qq -y autoremove \ + && apt-get autoclean \ + && rm -rf /var/lib/apt/lists/* /var/log/dpkg.log + + +ARG USERID +ARG GROUPID +ARG USERNAME + +RUN groupadd -g $GROUPID $USERNAME +RUN useradd -u $USERID -g $GROUPID $USERNAME +RUN mkdir /home/$USERNAME && chown $USERNAME:$USERNAME /home/$USERNAME +USER $USERNAME + +# Install leiningen during build process +RUN lein -v +RUN python3.8 -mpip install numpy \ No newline at end of file diff --git a/scripts/build-py38-docker b/scripts/build-py38-docker new file mode 100755 index 0000000..32fab3d --- /dev/null +++ b/scripts/build-py38-docker @@ -0,0 +1,7 @@ +#!/bin/bash + +set -e + +pushd dockerfiles +docker build -t docker-py38 -f Py38Dockerfile --build-arg USERID=$(id -u) --build-arg GROUPID=$(id -u) --build-arg USERNAME=$USER . +popd diff --git a/scripts/py38-repl b/scripts/py38-repl new file mode 100755 index 0000000..fe9624f --- /dev/null +++ b/scripts/py38-repl @@ -0,0 +1,6 @@ +#!/bin/bash + +lein update-in :dependencies conj \[nrepl\ \"0.6.0\"\]\ + -- update-in :plugins conj \[cider/cider-nrepl\ \"0.22.4\"\]\ + -- repl :headless :host localhost + diff --git a/scripts/run-conda-docker b/scripts/run-conda-docker index 0299e0b..b8837a1 100755 --- a/scripts/run-conda-docker +++ b/scripts/run-conda-docker @@ -5,5 +5,5 @@ scripts/build-conda-docker docker run --rm -it -u $(id -u):$(id -g) \ -v /$HOME/.m2:/home/$USER/.m2 \ -v $(pwd)/:/libpython-clj \ - -p 2222:2222 -w /libpython-clj \ + --net=host -w /libpython-clj \ docker-conda scripts/conda-repl diff --git a/scripts/run-py38-docker b/scripts/run-py38-docker new file mode 100755 index 0000000..dff5bf4 --- /dev/null +++ b/scripts/run-py38-docker @@ -0,0 +1,9 @@ +#!/bin/bash + +scripts/build-py38-docker + +docker run --rm -it -u $(id -u):$(id -g) \ + -v /$HOME/.m2:/home/$USER/.m2 \ + -v $(pwd)/:/libpython-clj \ + --net=host -w /libpython-clj \ + docker-py38 scripts/py38-repl diff --git a/src/libpython_clj/jna/base.clj b/src/libpython_clj/jna/base.clj index 49427f5..45a9a3f 100644 --- a/src/libpython_clj/jna/base.clj +++ b/src/libpython_clj/jna/base.clj @@ -69,7 +69,7 @@ (-> (Thread/currentThread) (.getId))) -(def gil-thread-id (AtomicLong. Long/MAX_VALUE)) +(defonce gil-thread-id (AtomicLong. Long/MAX_VALUE)) (defn set-gil-thread-id! @@ -106,7 +106,7 @@ `(.invoke (jna-base/to-typed-fn ~'tvm-fn) ~'fn-args))))) -(def size-t-type (type (jna/size-t 0))) +(defonce size-t-type (type (jna/size-t 0))) (defn find-pylib-symbol diff --git a/src/libpython_clj/python/interpreter.clj b/src/libpython_clj/python/interpreter.clj index 526fded..1ecb42a 100644 --- a/src/libpython_clj/python/interpreter.clj +++ b/src/libpython_clj/python/interpreter.clj @@ -9,7 +9,8 @@ [clojure.java.io :as io] [clojure.string :as s] [clojure.tools.logging :as log] - [clojure.data.json :as json]) + [clojure.data.json :as json] + [clojure.pprint :as pp]) (:import [libpython_clj.jna JVMBridge PyObject DirectMapped] [java.util.concurrent.atomic AtomicLong] [com.sun.jna Pointer] @@ -91,11 +92,12 @@ print(json.dumps( (condp (partial =) (keyword platform) ;; TODO: not sure what the strings returned by ;; ..: mac and windows are for sys.platform - :linux "libpython%s.%sm.so$" - :mac "libpython%s.%sm.dylib$" + :linux "libpython%s.%sm?.so$" + :mac "libpython%s.%sm?.dylib$" :win32 "python%s%s.dll$") major minor)))) + (defn python-library-paths "Returns vector of matching python libraries in order of: - virtual-env (library) @@ -105,6 +107,7 @@ print(json.dumps( - installation prefix (executable) - default executable location" [system-info python-regex] + (println system-info) (transduce (comp (map io/file) @@ -128,7 +131,7 @@ print(json.dumps( (comment ;; library paths workflow - (let [executable "python3.7" + (let [executable "python3.6" system-info (python-system-info executable) pyregex (python-library-regex system-info)] (python-library-paths system-info pyregex))) @@ -142,13 +145,28 @@ print(json.dumps( (catch Throwable e nil))) +(def default-python-executables + ["python3" "python3.6" "python3.7" "python3.8" "python3.9" "python"]) + + (defn detect-startup-info [{:keys [library-path python-home python-executable]}] (log-info (str "Detecting startup-info for Python executable: " python-executable)) - (let [executable (or python-executable "python3") - system-info (python-system-info executable) + (let [executable-seq (concat + (when python-executable + [python-executable]) + default-python-executables) + system-info (->> executable-seq + (map #(try (python-system-info %) + (catch Throwable e + nil))) + (remove nil?) + (first)) + _ (when-not system-info + (throw (Exception. (format "Python executable was not found. Tried: %s" + (vec executable-seq))))) python-home (cond python-home python-home @@ -165,12 +183,21 @@ print(json.dumps( libname (or library-path (when (seq lib-version) (str "python" lib-version "m"))) + libnames (concat [libname] + ;;Make sure we try without the 'm' suffix + (when lib-version + [(str "python" lib-version)])) retval - {:python-home python-home - :lib-version lib-version - :libname libname - :java-library-path-addendum java-library-path-addendum}] - (log/infof "Startup info detected: %s" retval) + (merge + system-info + {:python-home python-home + :lib-version lib-version + :libname libname + :libnames libnames + :java-library-path-addendum java-library-path-addendum})] + (log/infof "Startup info detected:\n%s" + (with-out-str + (pp/pprint (dissoc retval :libname)))) retval)) @@ -185,7 +212,8 @@ print(json.dumps( ;; Main interpreter booted up during initialize! ;; * in the right to indicate atom -(def main-interpreter* (atom nil)) +(defonce main-interpreter* (atom nil)) + (defn main-interpreter ^Interpreter [] @main-interpreter*) @@ -295,7 +323,6 @@ print(json.dumps( (.get ^AtomicLong libpy-base/gil-thread-id)) - (defn ensure-interpreter ^Interpreter [] (let [retval (main-interpreter)] @@ -355,7 +382,6 @@ print(json.dumps( (defonce ^:dynamic *program-name* "") - (defn- find-python-lib-version [] (let [{:keys [out err exit]} (ignore-shell-errors "python3" "--version")] @@ -407,24 +433,21 @@ print(json.dumps( (defn initialize! [& {:keys [program-name - library-path - python-executable] + library-path] :as options}] (when-not (main-interpreter) (log-info (str "Executing python initialize with options:" options)) - (let [{:keys [python-home libname java-library-path-addendum] :as startup-info} + (let [{:keys [python-home libnames java-library-path-addendum + executable] :as startup-info} (detect-startup-info options) - library-names (cond - library-path - [library-path] - libname - (concat - [libname] - (libpy-base/library-names)) - :else - (libpy-base/library-names))] + library-names (concat + (when library-path + [library-path]) + libnames + (libpy-base/library-names))] (reset! python-home-wide-ptr* nil) (reset! python-path-wide-ptr* nil) + (log/infof "Trying python library names %s" (vec library-names)) (when python-home (append-java-library-path! java-library-path-addendum) ;;This can never be released if load-library succeeeds @@ -438,17 +461,19 @@ print(json.dumps( @python-home-wide-ptr* @python-path-wide-ptr*))) (recur library-names))) - (setup-direct-mapping!)) - ;;Set program name - (when-let [program-name (or program-name *program-name* "")] - (pygc/with-stack-context - (libpy/PySys_SetArgv 0 (-> program-name - (jna/string->wide-ptr))))) + (setup-direct-mapping!) + ;;Set program name + (when-let [program-name (or program-name executable "")] + (pygc/with-stack-context + (libpy/PySys_SetArgv 0 (-> program-name + (jna/string->wide-ptr)))))) (let [type-symbols (libpy/lookup-type-symbols) context (do - (libpy-base/set-gil-thread-id! Long/MAX_VALUE (libpy-base/current-thread-id)) + (libpy-base/set-gil-thread-id! + Long/MAX_VALUE (libpy-base/current-thread-id)) (let [retval (libpy/PyEval_SaveThread)] - (libpy-base/set-gil-thread-id! (libpy-base/current-thread-id) Long/MAX_VALUE) + (libpy-base/set-gil-thread-id! + (libpy-base/current-thread-id) Long/MAX_VALUE) retval))] (construct-main-interpreter! context type-symbols)))) From 62f146d86bad08bf4aa2a4d5f08e0f61192149ba Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sat, 16 May 2020 10:45:36 -0600 Subject: [PATCH 215/456] 1.44 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 4d1537b..2186bbb 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-python/libpython-clj "1.44-SNAPSHOT" +(defproject clj-python/libpython-clj "1.44" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "Eclipse Public License" From 601b04b721e0a930dae2a22aaf23c6bae487537d Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sat, 16 May 2020 10:47:39 -0600 Subject: [PATCH 216/456] snap --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 2186bbb..4d1537b 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-python/libpython-clj "1.44" +(defproject clj-python/libpython-clj "1.44-SNAPSHOT" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "Eclipse Public License" From 22b934353bb27dcda77d6f9832ea7c9d7da23c32 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Wed, 20 May 2020 10:58:12 -0600 Subject: [PATCH 217/456] Upgrading to dtype beta-32 --- deps.edn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps.edn b/deps.edn index b0b0a7d..daf93eb 100644 --- a/deps.edn +++ b/deps.edn @@ -4,7 +4,7 @@ :deps { org.clojure/clojure {:mvn/version "1.10.1"} camel-snake-kebab {:mvn/version "0.4.0"} - techascent/tech.datatype {:mvn/version "5.0-beta-30"} + techascent/tech.datatype {:mvn/version "5.0-beta-32"} org.clojure/data.json {:mvn/version "0.2.7"} } } From 0fc0eefb75bbf63671037134c0e1ee3536c379d1 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Sat, 13 Jun 2020 12:45:12 -0600 Subject: [PATCH 218/456] Updated to latest datatype lib. --- deps.edn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps.edn b/deps.edn index daf93eb..559e174 100644 --- a/deps.edn +++ b/deps.edn @@ -4,7 +4,7 @@ :deps { org.clojure/clojure {:mvn/version "1.10.1"} camel-snake-kebab {:mvn/version "0.4.0"} - techascent/tech.datatype {:mvn/version "5.0-beta-32"} + techascent/tech.datatype {:mvn/version "5.0-beta-40"} org.clojure/data.json {:mvn/version "0.2.7"} } } From db7c7ce9b2dd67a1b015af273b004776ad49c065 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Mon, 15 Jun 2020 10:31:33 -0600 Subject: [PATCH 219/456] 1.45 --- CHANGELOG.md | 3 +++ deps.edn | 2 +- project.clj | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1574e41..551e384 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ # Time for a ChangeLog! +## 1.45 + * tech.datatype - 5.0 release + ## 1.43 * `tech.datatype` - latest version to work with `tech.ml.dataset`. diff --git a/deps.edn b/deps.edn index 559e174..e1ebe25 100644 --- a/deps.edn +++ b/deps.edn @@ -4,7 +4,7 @@ :deps { org.clojure/clojure {:mvn/version "1.10.1"} camel-snake-kebab {:mvn/version "0.4.0"} - techascent/tech.datatype {:mvn/version "5.0-beta-40"} + techascent/tech.datatype {:mvn/version "5.0"} org.clojure/data.json {:mvn/version "0.2.7"} } } diff --git a/project.clj b/project.clj index 4d1537b..8597108 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-python/libpython-clj "1.44-SNAPSHOT" +(defproject clj-python/libpython-clj "1.45" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "Eclipse Public License" From cdeb60662aadd68252be82a7930937689b93bf7f Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Mon, 15 Jun 2020 10:31:41 -0600 Subject: [PATCH 220/456] snap --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 8597108..1a0ea28 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-python/libpython-clj "1.45" +(defproject clj-python/libpython-clj "1.46-SNAPSHOT" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "Eclipse Public License" From fbf719a2cb46390f3e410aaf0cda5e5802d3dc05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Madis=20N=C3=B5mme?= Date: Fri, 10 Jul 2020 15:51:32 +0300 Subject: [PATCH 221/456] Add full example on DataFrame usage & access (#100) Add example of Python vs Clojure doing the same thing - import dependencies - create DataFrame - access its rows To clarify how the `get-item` and `get-attr` work together. After spending long time to figure out how the use the attributes vs items access in Clojure to access DataFrame rows and columns, I thought it would be useful to document full example how to do it Python & Clojure side by side. --- docs/Usage.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/docs/Usage.md b/docs/Usage.md index bc646e7..596d339 100644 --- a/docs/Usage.md +++ b/docs/Usage.md @@ -8,6 +8,15 @@ item access is optional and happens via the `__getitem__` and `__setitem__` attr This is important to realize in that the code below doesn't look like python because we are referencing the item and attribute systems by name and not via '.' or '[]'. +This would result in the following analogous code (full example [further on](#dataframe-access-full-example)): + +```python +table.loc[row_date] +``` + +```clojure +(get-item (get-attr table :loc) row-date) +``` ### Installation @@ -216,6 +225,31 @@ user> (att-type-map ones-ary) } ``` +### DataFrame access full example + +Here's how to create Pandas DataFrame and accessing its rows via `loc` in both Python and Clojure: + +```python +# Python +import numpy as np +import pandas as pan + +dates = pan.date_range("1/1/2000", periods=8) +table = pan.DataFrame(np.random.randn(8, 4), index=dates, columns=["A", "B", "C", "D"]) +row_date = pan.date_range(start="2000-01-01", end="2000-01-01") +table.loc[row_date] +``` + +```clojure +; Clojure +(require-python '[numpy :as np]) +(require-python '[pandas :as pan]) + +(def dates (pan/date_range "1/1/2000" :periods 8)) +(def table (pan/DataFrame (call-attr np/random :randn 8 4) :index dates :columns ["A" "B" "C" "D"])) +(def row-date (pan/date_range :start "2000-01-01" :end "2000-01-01")) +(get-item (get-attr table :loc) row-date) +``` ### Errors From ab270cd95bfb5b97d790352333209b63431b828f Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Fri, 10 Jul 2020 06:54:48 -0600 Subject: [PATCH 222/456] upgrade datatype. --- deps.edn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps.edn b/deps.edn index e1ebe25..5b20205 100644 --- a/deps.edn +++ b/deps.edn @@ -4,7 +4,7 @@ :deps { org.clojure/clojure {:mvn/version "1.10.1"} camel-snake-kebab {:mvn/version "0.4.0"} - techascent/tech.datatype {:mvn/version "5.0"} + techascent/tech.datatype {:mvn/version "5.05"} org.clojure/data.json {:mvn/version "0.2.7"} } } From b5772690a351229d5539d1dde1099f7718182e7c Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Thu, 16 Jul 2020 07:29:53 -0600 Subject: [PATCH 223/456] Attempting to fix travis. --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 1002412..9b592bd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,9 @@ language: clojure lein: 2.9.1 +sudo: required before_install: - sudo apt-get -y install python3 python3-pip + - sudo python3 -mpip install setuptools - sudo python3 -mpip install numpy - curl -O https://download.clojure.org/install/linux-install-1.10.1.507.sh - chmod +x linux-install-1.10.1.507.sh From a846a4902d9b109ebfa93ea9c23eba5a2f8f8efe Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Thu, 16 Jul 2020 07:33:52 -0600 Subject: [PATCH 224/456] travis hacking... --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 9b592bd..3883953 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,5 @@ language: clojure +dist: bionic lein: 2.9.1 sudo: required before_install: From a85be74de9385ba2ef0c01c78eb0c4045b9abd42 Mon Sep 17 00:00:00 2001 From: "J.J. Tolton" Date: Mon, 7 Sep 2020 16:57:42 -0400 Subject: [PATCH 225/456] add pathing metadata; fix metatype bug (#118) --- src/libpython_clj/metadata.clj | 53 +++++++++++++++------- src/libpython_clj/require.clj | 4 +- test/libpython_clj/require_python_test.clj | 32 ++++++++++++- 3 files changed, 70 insertions(+), 19 deletions(-) diff --git a/src/libpython_clj/metadata.clj b/src/libpython_clj/metadata.clj index 947f59c..4f5f7c7 100644 --- a/src/libpython_clj/metadata.clj +++ b/src/libpython_clj/metadata.clj @@ -17,6 +17,8 @@ (def inspect (as-jvm (import-module "inspect") {})) (def argspec (get-attr inspect "getfullargspec")) (def py-source (get-attr inspect "getsource")) +(def py-sourcelines (get-attr inspect "getsourcelines")) +(def py-file (get-attr inspect "getfile")) (def types (import-module "types")) (def fn-type (call-attr builtins "tuple" @@ -42,11 +44,25 @@ (def importlib (py/import-module "importlib")) (def importlib_util (import-module "importlib.util")) (def reload-module (py/get-attr importlib "reload")) + (defn findspec [x] (let [-findspec (-> importlib_util (get-attr "find_spec"))] (-findspec x))) + +(defn find-lineno [x] + (try + (-> x py-sourcelines last) + (catch Exception _ + nil))) + +(defn find-file [x] + (try + (py-file x) + (catch Exception _ + nil))) + (defn py-fn-argspec [f] (if-let [spec (try (when-not (pyclass? f) (argspec f)) @@ -155,10 +171,6 @@ (recur argspec' defaults' arglists)))))) -(defn py-class-argspec [class] - (let [constructor (py/get-attr class "__init__")] - (py-fn-argspec constructor))) - (defn py-fn-metadata [fn-name x {:keys [no-arglists?]}] (let [fn-argspec (pyargspec x) @@ -190,14 +202,18 @@ (defn base-pyobj-map [item] - (merge {:type (py/python-type item) - :doc (doc item) - :str (.toString item) - :flags (pyobj-flags item)} - (when (has-attr? item "__module__") - {:module (get-attr item "__module__")}) - (when (has-attr? item "__name__") - {:name (get-attr item "__name__")}))) + (cond-> {:type (py/python-type item) + :doc (doc item) + :str (.toString item) + :flags (pyobj-flags item) + :line (find-lineno item) + :file (find-file item)} + (has-attr? item "__module__") + (assoc :module (get-attr item "__module__")) + (has-attr? item "__name__") + (assoc :name (get-attr item "__name__")) + (and (find-lineno item) (find-file item)) + (assoc :line (find-lineno item) :file (find-file item)))) (defn scalar? @@ -280,10 +296,15 @@ (defn metadata-map->py-obj [metadata-map] - (case (:type metadata-map) - :module (import-module (:name metadata-map)) - :type (-> (import-module (:module metadata-map)) - (get-attr (:name metadata-map))))) + (try + (case (:type metadata-map) + :module (import-module (:name metadata-map)) + :type (-> (import-module (:module metadata-map)) + (get-attr (:name metadata-map)))) + (catch Exception _ + ;; metatypes -- e.g. socket.SocketIO + (-> (import-module (:module metadata-map)) + (get-attr (:name metadata-map)))))) (defn get-or-create-namespace! diff --git a/src/libpython_clj/require.clj b/src/libpython_clj/require.clj index 0513309..3cc4cca 100644 --- a/src/libpython_clj/require.clj +++ b/src/libpython_clj/require.clj @@ -127,7 +127,9 @@ " to something without periods.")))) (intern (symbol (str *ns*)) - (symbol import-name) + (with-meta (symbol import-name) + {:file (metadata/find-file pyobj) + :line 1}) pyobj))) (when (or (not existing-py-ns?) reload?) diff --git a/test/libpython_clj/require_python_test.clj b/test/libpython_clj/require_python_test.clj index a79b39b..42afbea 100644 --- a/test/libpython_clj/require_python_test.clj +++ b/test/libpython_clj/require_python_test.clj @@ -114,7 +114,35 @@ (is (set? (datafy (python/frozenset [1 2 3]))))) - - (deftest import-python-test (is (= :ok (libpython-clj.require/import-python)))) + +(require-python '[socket :bind-ns true]) +(require-python 'socket.SocketIO) +(deftest metadata-test + (testing "metadata generation" + ;; module meta + (let [{line :line file :file} (meta #'socket)] + (is (= 1 line) + "Modules have line numbers") + (is (string? file) + "Modules have file paths")) + ;; class meta + (let [{line :line file :file} (meta #'socket/SocketIO)] + (is (int? line) + "Classes have line numbers") + (is (string? file) + "Classes have file paths")) + ;; function meta + (let [{line :line file :file} (meta #'socket/_intenum_converter)] + (is (int? line) + "Functions have line numbers") + (is (string? file) + "Functions have file paths")) + ;; method meta + (let [{line :line file :file} (meta #'socket.SocketIO/readable)] + (is (int? line) + "Methods have line numbers") + (is (string? file) + "Methods have file paths")))) + From 03fab5f5d7b6a29e3975342db8c47ac1fd1a5b69 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Mon, 7 Sep 2020 15:24:04 -0600 Subject: [PATCH 226/456] Support for Python3.9 - cfunction_new -> cfunction_newex --- src/libpython_clj/jna.clj | 2 +- src/libpython_clj/jna/concrete/cfunction.clj | 5 +++-- src/libpython_clj/python/object.clj | 11 ++++++----- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/libpython_clj/jna.clj b/src/libpython_clj/jna.clj index def6530..b3eee91 100644 --- a/src/libpython_clj/jna.clj +++ b/src/libpython_clj/jna.clj @@ -292,7 +292,7 @@ METH_O METH_STATIC METH_VARARGS - PyCFunction_New + PyCFunction_NewEx PyInstanceMethod_New) diff --git a/src/libpython_clj/jna/concrete/cfunction.clj b/src/libpython_clj/jna/concrete/cfunction.clj index 05d9765..afa1017 100644 --- a/src/libpython_clj/jna/concrete/cfunction.clj +++ b/src/libpython_clj/jna/concrete/cfunction.clj @@ -51,11 +51,12 @@ ;; #endif -(def-pylib-fn PyCFunction_New +(def-pylib-fn PyCFunction_NewEx "Create a new callable from an item." Pointer [method-def (partial jna/ensure-type PyMethodDef)] - [self as-pyobj]) + [self as-pyobj] + [module as-pyobj]) (def-pylib-fn PyInstanceMethod_New diff --git a/src/libpython_clj/python/object.clj b/src/libpython_clj/python/object.clj index 5495ec8..d339fc1 100644 --- a/src/libpython_clj/python/object.clj +++ b/src/libpython_clj/python/object.clj @@ -444,11 +444,12 @@ Object's refcount is bad. Crash is imminent" ;;This is a nice little tidbit, cfunction_new ;;steals the reference. (let [py-self (when py-self (incref (->python py-self)))] - (-> (libpy/PyCFunction_New (method-def-data->method-def - {:name method-name - :doc documentation - :function cfunc}) - py-self) + (-> (libpy/PyCFunction_NewEx (method-def-data->method-def + {:name method-name + :doc documentation + :function cfunc}) + py-self + nil) (wrap-pyobject))))) From 466b53bd8def26ca77ba75cb33005df24b0078c4 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Mon, 7 Sep 2020 15:24:28 -0600 Subject: [PATCH 227/456] Adding a minimal pathway to test on python3.9 --- dockerfiles/Py39Dockerfile | 35 +++++++++++++++++++++++++++++++++++ scripts/build-py39-docker | 7 +++++++ scripts/run-py39-docker | 9 +++++++++ 3 files changed, 51 insertions(+) create mode 100644 dockerfiles/Py39Dockerfile create mode 100755 scripts/build-py39-docker create mode 100755 scripts/run-py39-docker diff --git a/dockerfiles/Py39Dockerfile b/dockerfiles/Py39Dockerfile new file mode 100644 index 0000000..f071037 --- /dev/null +++ b/dockerfiles/Py39Dockerfile @@ -0,0 +1,35 @@ +# We will use Ubuntu for our image +FROM ubuntu:latest + +# Updating Ubuntu packages + +ARG CLOJURE_TOOLS_VERSION=1.10.1.507 + +RUN apt-get -qq update \ + && apt install -y software-properties-common \ + && add-apt-repository -y ppa:deadsnakes/ppa + +RUN apt-get -qq update && apt-get -qq -y install curl wget bzip2 openjdk-8-jdk-headless python3.9 libpython3.9 python3-pip python3-distutils\ + && curl -o install-clojure https://download.clojure.org/install/linux-install-${CLOJURE_TOOLS_VERSION}.sh \ + && chmod +x install-clojure \ + && ./install-clojure && rm install-clojure \ + && wget https://raw.githubusercontent.com/technomancy/leiningen/stable/bin/lein \ + && chmod a+x lein \ + && mv lein /usr/bin \ + && apt-get -qq -y autoremove \ + && apt-get autoclean \ + && rm -rf /var/lib/apt/lists/* /var/log/dpkg.log + + +ARG USERID +ARG GROUPID +ARG USERNAME + +RUN groupadd -g $GROUPID $USERNAME +RUN useradd -u $USERID -g $GROUPID $USERNAME +RUN mkdir /home/$USERNAME && chown $USERNAME:$USERNAME /home/$USERNAME +USER $USERNAME + +# Install leiningen during build process +RUN lein -v +# RUN python3.9 -mpip install numpy \ No newline at end of file diff --git a/scripts/build-py39-docker b/scripts/build-py39-docker new file mode 100755 index 0000000..c9b099a --- /dev/null +++ b/scripts/build-py39-docker @@ -0,0 +1,7 @@ +#!/bin/bash + +set -e + +pushd dockerfiles +docker build -t docker-py39 -f Py39Dockerfile --build-arg USERID=$(id -u) --build-arg GROUPID=$(id -u) --build-arg USERNAME=$USER . +popd diff --git a/scripts/run-py39-docker b/scripts/run-py39-docker new file mode 100755 index 0000000..39358ad --- /dev/null +++ b/scripts/run-py39-docker @@ -0,0 +1,9 @@ +#!/bin/bash + +scripts/build-py39-docker + +docker run --rm -it -u $(id -u):$(id -g) \ + -v /$HOME/.m2:/home/$USER/.m2 \ + -v $(pwd)/:/libpython-clj \ + --net=host -w /libpython-clj \ + docker-py39 scripts/py38-repl From e129cf14690d1d5602016293a8a7241c36910555 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Mon, 7 Sep 2020 15:29:46 -0600 Subject: [PATCH 228/456] 1.46 --- deps.edn | 4 ++-- project.clj | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deps.edn b/deps.edn index 5b20205..111c78b 100644 --- a/deps.edn +++ b/deps.edn @@ -4,7 +4,7 @@ :deps { org.clojure/clojure {:mvn/version "1.10.1"} camel-snake-kebab {:mvn/version "0.4.0"} - techascent/tech.datatype {:mvn/version "5.05"} - org.clojure/data.json {:mvn/version "0.2.7"} + techascent/tech.datatype {:mvn/version "5.18"} + org.clojure/data.json {:mvn/version "1.0.0"} } } diff --git a/project.clj b/project.clj index 1a0ea28..def8b8d 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-python/libpython-clj "1.46-SNAPSHOT" +(defproject clj-python/libpython-clj "1.46" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "Eclipse Public License" From 69c4c0c11098e7352aed9d48d606ab402525af11 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Mon, 7 Sep 2020 15:29:56 -0600 Subject: [PATCH 229/456] snap --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index def8b8d..857e6b9 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clj-python/libpython-clj "1.46" +(defproject clj-python/libpython-clj "1.47-SNAPSHOT" :description "libpython bindings to the techascent ecosystem" :url "http://github.com/cnuernber/libpython-clj" :license {:name "Eclipse Public License" From 73f9c7fbf743f85cc69559b88174e61f82a1a27e Mon Sep 17 00:00:00 2001 From: Alan Thompson Date: Mon, 19 Oct 2020 15:49:38 -0700 Subject: [PATCH 230/456] Added warning re outdated Clojars version (#128) Hi - Just had some problems due to referencing the outdated `cnuernber` aritfacts on Clojars. This name is still widely referenced in blog entries & videos and crashes when trying to run with Python 3.8, for example. I thought a small warning on the README may help others to avoid this trap. Thanks for a great library! Alan --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 963f059..8791f3a 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,14 @@ -# libpython-clj +# JNA libpython bindings to the tech ecosystem. -JNA libpython bindings to the tech ecosystem. +### Version Info [![Clojars Project](https://img.shields.io/clojars/v/clj-python/libpython-clj.svg)](https://clojars.org/clj-python/libpython-clj) [![travis integration](https://travis-ci.com/clj-python/libpython-clj.svg?branch=master)](https://travis-ci.com/clj-python/libpython-clj) + - Note - Please avoid deprecated versions such as `[cnuernber/libpython-clj "1.36"]` (***note name change***). + +## libpython-clj features + * Bridge between JVM objects and Python objects easily; use Python in your Java and use some Java in your Python. * Python objects are linked to the JVM GC such that when they are no longer reachable From 0c5c7443753d29cc0ee365de0417cf66e868b800 Mon Sep 17 00:00:00 2001 From: Jay Date: Mon, 19 Oct 2020 20:34:00 -0400 Subject: [PATCH 231/456] fix docstring --- src/libpython_clj/python.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libpython_clj/python.clj b/src/libpython_clj/python.clj index 6e113fb..e6d3970 100644 --- a/src/libpython_clj/python.clj +++ b/src/libpython_clj/python.clj @@ -163,7 +163,7 @@ (defn run-simple-string "Run a string expression returning a map of - {:globals :locals :result}. + {:globals :locals}. This uses the global __main__ dict under the covers so it matches the behavior of the cpython implementation with the exception of returning the various maps used. From fb6d793675ba7f3e54c2c16cc65ed923f6fd8855 Mon Sep 17 00:00:00 2001 From: Chris Nuernberger Date: Thu, 22 Oct 2020 13:21:07 -0600 Subject: [PATCH 232/456] Dtype next support- faster loading and support for newest version of tech stack moving forward (#131) * initial dtype next changes. * Initialize! works again * [test] Passing. * Attempting to get API docs. * bumping version before merge to master * Test with latest. Co-authored-by: Harold --- deps.edn | 10 - docs/Usage.html | 321 ++++++++++ docs/css/default.css | 649 +++++++++++++++++++++ docs/design.html | 96 +++ docs/highlight/highlight.min.js | 2 + docs/highlight/solarized-light.css | 84 +++ docs/index.html | 3 + docs/js/jquery.min.js | 4 + docs/js/page_effects.js | 113 ++++ docs/libpython-clj.python.html | 26 + docs/new-to-clojure.html | 49 ++ docs/scopes-and-gc.html | 42 ++ docs/slicing.html | 58 ++ project.clj | 28 +- src/libpython_clj/jna.clj | 8 +- src/libpython_clj/jna/concrete/bytes.clj | 4 +- src/libpython_clj/jna/concrete/unicode.clj | 4 +- src/libpython_clj/jna/protocols/buffer.clj | 2 +- src/libpython_clj/python.clj | 5 +- src/libpython_clj/python/bridge.clj | 86 ++- src/libpython_clj/python/interop.clj | 5 +- src/libpython_clj/python/np_array.clj | 99 +--- src/libpython_clj/python/object.clj | 30 +- src/libpython_clj/python/protocols.clj | 5 - test/libpython_clj/python/numpy_test.clj | 14 +- test/libpython_clj/python_test.clj | 33 +- {docs => topics}/Usage.md | 0 {docs => topics}/design.md | 0 {docs => topics}/new-to-clojure.md | 0 {docs => topics}/scopes-and-gc.md | 0 {docs => topics}/slicing.md | 0 31 files changed, 1578 insertions(+), 202 deletions(-) delete mode 100644 deps.edn create mode 100644 docs/Usage.html create mode 100644 docs/css/default.css create mode 100644 docs/design.html create mode 100644 docs/highlight/highlight.min.js create mode 100644 docs/highlight/solarized-light.css create mode 100644 docs/index.html create mode 100644 docs/js/jquery.min.js create mode 100644 docs/js/page_effects.js create mode 100644 docs/libpython-clj.python.html create mode 100644 docs/new-to-clojure.html create mode 100644 docs/scopes-and-gc.html create mode 100644 docs/slicing.html rename {docs => topics}/Usage.md (100%) rename {docs => topics}/design.md (100%) rename {docs => topics}/new-to-clojure.md (100%) rename {docs => topics}/scopes-and-gc.md (100%) rename {docs => topics}/slicing.md (100%) diff --git a/deps.edn b/deps.edn deleted file mode 100644 index 111c78b..0000000 --- a/deps.edn +++ /dev/null @@ -1,10 +0,0 @@ -{ - :paths ["src"] - - :deps { - org.clojure/clojure {:mvn/version "1.10.1"} - camel-snake-kebab {:mvn/version "0.4.0"} - techascent/tech.datatype {:mvn/version "5.18"} - org.clojure/data.json {:mvn/version "1.0.0"} - } - } diff --git a/docs/Usage.html b/docs/Usage.html new file mode 100644 index 0000000..ae328bd --- /dev/null +++ b/docs/Usage.html @@ -0,0 +1,321 @@ + +Usage

Usage

+

Python objects are essentially two dictionaries, one for ‘attributes’ and one for ‘items’. When you use python and use the ‘.’ operator, you are referencing attributes. If you use the ‘[]’ operator, then you are referencing items. Attributes are built in, item access is optional and happens via the __getitem__ and __setitem__ attributes. This is important to realize in that the code below doesn’t look like python because we are referencing the item and attribute systems by name and not via ‘.’ or ‘[]’.

+

This would result in the following analogous code (full example further on):

+
table.loc[row_date]
+
+
(get-item (get-attr table :loc) row-date)
+
+

Installation

+

Ubuntu

+
sudo apt install libpython3.6
+# numpy and pandas are required for unit tests.  Numpy is required for
+# zero copy support.
+python3.6 -m pip install numpy pandas --user
+
+

MacOSX

+

Python installation instructions here.

+

Initialize python

+
user>
+
+user> (require '[libpython-clj.python
+                 :refer [as-python as-jvm
+                         ->python ->jvm
+                         get-attr call-attr call-attr-kw
+                         get-item att-type-map
+                         call call-kw initialize!
+                         as-numpy as-tensor ->numpy
+                         run-simple-string
+                         add-module module-dict
+                         import-module
+                         python-type]])
+nil
+
+; Mac and Linux
+user> (initialize!)
+Jun 30, 2019 4:47:39 PM clojure.tools.logging$eval7369$fn__7372 invoke
+INFO: executing python initialize!
+Jun 30, 2019 4:47:39 PM clojure.tools.logging$eval7369$fn__7372 invoke
+INFO: Library python3.6m found at [:system "python3.6m"]
+Jun 30, 2019 4:47:39 PM clojure.tools.logging$eval7369$fn__7372 invoke
+INFO: Reference thread starting
+:ok
+
+; Windows with Anaconda
+(initialize! ; Python executable
+             :python-executable "C:\\Users\\USER\\AppData\\Local\\Continuum\\anaconda3\\python.exe"
+             ; Python Library
+             :library-path "C:\\Users\\USER\\AppData\\Local\\Continuum\\anaconda3\\python37.dll"
+             ; Anacondas PATH environment to load native dlls of modules (numpy, etc.)
+             :windows-anaconda-activate-bat "C:\\Users\\USER\\AppData\\Local\\Continuum\\anaconda3\\Scripts\\activate.bat"
+             )
+...
+:ok
+
+

This dynamically finds the python shared library and loads it using output from the python3 executable on your system. For information about how that works, please checkout the code here.

+

Execute Some Python

+

*out* and *err* capture python stdout and stderr respectively.

+

user> (run-simple-string "print('hey')") +hey +{:globals + {'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': <class '_frozen_importlib.BuiltinImporter'>, '__spec__': None, '__annotations__': {}, '__builtins__': <module 'builtins' (built-in)>}, + :locals + {'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': <class '_frozen_importlib.BuiltinImporter'>, '__spec__': None, '__annotations__': {}, '__builtins__': <module 'builtins' (built-in)>}} +
+

The results have been ‘bridged’ into java meaning they are still python objects but there are java wrappers over the top of them. For instance, Object.toString forwards its implementation to the python function __str__.

+
(def bridged (run-simple-string "print('hey')"))
+(instance? java.util.Map (:globals bridged))
+true
+user> (:globals bridged)
+{'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': <class '_frozen_importlib.BuiltinImporter'>, '__spec__': None, '__annotations__': {}, '__builtins__': <module 'builtins' (built-in)>}
+
+

We can get and set global variables here. If we run another string, these are in the environment. The globals map itself is the global dict of the main module:

+
(def main-globals (-> (add-module "__main__")
+                            (module-dict)))
+#'user/main-globals
+
+user> main-globals
+{'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': <class '_frozen_importlib.BuiltinImporter'>, '__spec__': None, '__annotations__': {}, '__builtins__': <module 'builtins' (built-in)>}
+user> (keys main-globals)
+("__name__"
+ "__doc__"
+ "__package__"
+ "__loader__"
+ "__spec__"
+ "__annotations__"
+ "__builtins__")
+user> (get main-globals "__name__")
+"__main__"
+user> (.put main-globals "my_var" 200)
+nil
+
+user> (run-simple-string "print('your variable is:' + str(my_var))")
+your variable is:200
+{:globals
+ {'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': <class '_frozen_importlib.BuiltinImporter'>, '__spec__': None, '__annotations__': {}, '__builtins__': <module 'builtins' (built-in)>, 'my_var': 200},
+ :locals
+ {'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': <class '_frozen_importlib.BuiltinImporter'>, '__spec__': None, '__annotations__': {}, '__builtins__': <module 'builtins' (built-in)>, 'my_var': 200}}
+
+

Running Python isn’t ever really necessary, however, although it may at times be convenient. You can call attributes from clojure easily:

+
user> (def np (import-module "numpy"))
+#'user/np
+user> (def ones-ary (call-attr np "ones" [2 3]))
+#'user/ones-ary
+user> ones-ary
+[[1. 1. 1.]
+ [1. 1. 1.]]
+user> (call-attr ones-ary "__len__")
+2
+user> (vec ones-ary)
+[[1. 1. 1.] [1. 1. 1.]]
+user> (type (first *1))
+:pyobject
+user> (get-attr ones-ary "shape")
+(2, 3)
+user> (vec (get-attr ones-ary "shape"))
+[2 3]
+
+user> (att-type-map ones-ary)
+{"T" :ndarray,
+ "__abs__" :method-wrapper,
+ "__add__" :method-wrapper,
+ "__and__" :method-wrapper,
+ "__array__" :builtin-function-or-method,
+ "__array_finalize__" :none-type,
+ "__array_function__" :builtin-function-or-method,
+ "__array_interface__" :dict,
+ "__array_prepare__" :builtin-function-or-method,
+ "__array_priority__" :float,
+ "__array_struct__" :py-capsule,
+ "__array_ufunc__" :builtin-function-or-method,
+ "__array_wrap__" :builtin-function-or-method,
+ "__bool__" :method-wrapper,
+ "__class__" :type,
+ "__complex__" :builtin-function-or-method,
+ "__contains__" :method-wrapper,
+ ...
+ "std" :builtin-function-or-method,
+ "strides" :tuple,
+ "sum" :builtin-function-or-method,
+ "swapaxes" :builtin-function-or-method,
+ "take" :builtin-function-or-method,
+ "tobytes" :builtin-function-or-method,
+ "tofile" :builtin-function-or-method,
+ "tolist" :builtin-function-or-method,
+ "tostring" :builtin-function-or-method,
+ "trace" :builtin-function-or-method,
+ "transpose" :builtin-function-or-method,
+ "var" :builtin-function-or-method,
+ "view" :builtin-function-or-method}
+
+

att-type-map

+

It can be extremely helpful to print out the attribute name->attribute type map:

+
user> (att-type-map ones-ary)
+{"T" :ndarray,
+ "__abs__" :method-wrapper,
+ "__add__" :method-wrapper,
+ "__and__" :method-wrapper,
+ "__array__" :builtin-function-or-method,
+ "__array_finalize__" :none-type,
+ "__array_function__" :builtin-function-or-method,
+ "__array_interface__" :dict,
+ ...
+  "real" :ndarray,
+ "repeat" :builtin-function-or-method,
+ "reshape" :builtin-function-or-method,
+ "resize" :builtin-function-or-method,
+ "round" :builtin-function-or-method,
+ "searchsorted" :builtin-function-or-method,
+ "setfield" :builtin-function-or-method,
+ "setflags" :builtin-function-or-method,
+ "shape" :tuple,
+ "size" :int,
+ "sort" :builtin-function-or-method,
+ ...
+}
+
+

DataFrame access full example

+

Here’s how to create Pandas DataFrame and accessing its rows via loc in both Python and Clojure:

+
# Python
+import numpy as np
+import pandas as pan
+
+dates = pan.date_range("1/1/2000", periods=8)
+table = pan.DataFrame(np.random.randn(8, 4), index=dates, columns=["A", "B", "C", "D"])
+row_date = pan.date_range(start="2000-01-01", end="2000-01-01")
+table.loc[row_date]
+
+
; Clojure
+(require-python '[numpy :as np])
+(require-python '[pandas :as pan])
+
+(def dates (pan/date_range "1/1/2000" :periods 8))
+(def table (pan/DataFrame (call-attr np/random :randn 8 4) :index dates :columns ["A" "B" "C" "D"]))
+(def row-date (pan/date_range :start "2000-01-01" :end "2000-01-01"))
+(get-item (get-attr table :loc) row-date)
+
+

Errors

+

Errors are caught and an exception is thrown. The error text is saved verbatim in the exception:

+
user> (run-simple-string "print('syntax errrr")
+Execution error (ExceptionInfo) at libpython-clj.python.interpreter/check-error-throw (interpreter.clj:260).
+  File "<string>", line 1
+    print('syntax errrr
+                      ^
+SyntaxError: EOL while scanning string literal
+
+

Some Syntax Sugar

+
user> (py/from-import numpy linspace)
+#'user/linspace
+user> (linspace 2 3 :num 10)
+[2.         2.11111111 2.22222222 2.33333333 2.44444444 2.55555556
+ 2.66666667 2.77777778 2.88888889 3.        ]
+user> (doc linspace)
+-------------------------
+user/linspace
+
+    Return evenly spaced numbers over a specified interval.
+
+    Returns `num` evenly spaced samples, calculated over the
+    interval [`start`, `stop`].
+
+
+
    +
  • from-import - sugar around python from a import b. Takes multiple b’s.
  • +
  • import-as - surgar around python import a as b.
  • +
  • $a - call an attribute using symbol att name. Keywords map to kwargs
  • +
  • $c - call an object mapping keywords to kwargs
  • +
+

Experimental Sugar

+

We are trying to find the best way to handle attributes in order to shorten generic python notebook-type usage. The currently implemented direction is:

+
    +
  • $. - get an attribute. Can pass in symbol, string, or keyword
  • +
  • $.. - get an attribute. If more args are present, get the attribute on that result.
  • +
+
user> (py/$. numpy linspace)
+<function linspace at 0x7fa6642766a8>
+user> (py/$.. numpy random shuffle)
+<built-in method shuffle of numpy.random.mtrand.RandomState object at 0x7fa66410cca8>
+
+
New sugar (fixme)
+

libpython-clj offers syntactic forms similar to those offered by Clojure for interacting with Python classes and objects.

+

Class/object methods Where in Clojure you would use (. obj method arg1 arg2 ... argN), you can use (py. pyobj method arg1 arg2 ... argN).

+

In Python, this is equivalent to pyobj.method(arg1, arg2, ..., argN). Concrete examples are shown below.

+

Class/object attributes Where in Clojure you would use (.- obj attr), you can use (py.- pyobj attr).

+

In Python, this is equivalent to pyobj.attr. Concrete examples shown below.

+

Nested attribute access To achieve a chain of method/attribute access, use the py.. for.

+
(py.. (requests/get "http://www.google.com") 
+      -content
+      (decode "latin-1"))
+
+

(**Note**: requires Python requests module installled)

+

Examples

+
user=> (require '[libpython-clj.python :as py :refer [py. py.. py.-]])
+nil
+user=> (require '[libpython-clj.require :refer [require-python]])
+
+... debug info ...
+
+user=> (require-python '[builtins :as python])
+WARNING: AssertionError already refers to: class java.lang.AssertionError in namespace: builtins, being replaced by: #'builtins/AssertionError
+WARNING: Exception already refers to: class java.lang.Exception in namespace: builtins, being replaced by: #'builtins/Exception
+nil
+user=> (def xs (python/list))
+#'user/xs
+user=> (py. xs append 1)
+nil
+user=> xs
+[1]
+user=> (py. xs extend [1 2 3])
+nil
+user=> xs
+[1, 1, 2, 3]
+user=> (py. xs __len__)
+4
+user=> ((py.- xs __len__)) ;; attribute syntax to get then call method
+4
+user=> (py. xs pop)
+3
+user=> (py. xs clear)
+nil
+;; requires Python requests module installed
+user=> (require-python 'requests)
+nil
+user=> (def requests (py/import-module "requests"))
+#'user/requests
+user=> (py.. requests (get "http://www.google.com") -content (decode "latin-1"))
+"<!doctype html><html itemscope=\"\" ... snip ... "
+
+

Numpy

+

Speaking of numpy, you can move data between numpy and java easily.

+
user> (def tens-data (as-tensor ones-ary))
+#'user/tens-data
+user> (println tens-data)
+#tech.v2.tensor<float64>[2 3]
+[[1.000 1.000 1.000]
+ [1.000 1.000 1.000]]
+nil
+
+
+user> (require '[tech.v2.datatype :as dtype])
+nil
+user> (def ignored (dtype/copy! (repeat 6 5) tens-data))
+#'user/ignored
+user> (.put main-globals "ones_ary" ones_ary)
+Syntax error compiling at (*cider-repl cnuernber/libpython-clj:localhost:39019(clj)*:191:7).
+Unable to resolve symbol: ones_ary in this context
+user> (.put main-globals "ones_ary" ones-ary)
+nil
+
+user> (run-simple-string "print(ones_ary)")
+[[5. 5. 5.]
+ [5. 5. 5.]]
+{:globals
+ {'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': <class '_frozen_importlib.BuiltinImporter'>, '__spec__': None, '__annotations__': {}, '__builtins__': <module 'builtins' (built-in)>, 'my_var': 200, 'ones_ary': array([[5., 5., 5.],
+       [5., 5., 5.]])},
+ :locals
+ {'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': <class '_frozen_importlib.BuiltinImporter'>, '__spec__': None, '__annotations__': {}, '__builtins__': <module 'builtins' (built-in)>, 'my_var': 200, 'ones_ary': array([[5., 5., 5.],
+       [5., 5., 5.]])}}
+
+

So heavy data has a zero-copy route. Anything backed by a :native-buffer has a zero copy pathway to and from numpy. For more information on how this happens, please refer to the datatype library documentation.

+

Just keep in mind, careless usage of zero copy is going to cause spooky action at a distance.

\ No newline at end of file diff --git a/docs/css/default.css b/docs/css/default.css new file mode 100644 index 0000000..c78d040 --- /dev/null +++ b/docs/css/default.css @@ -0,0 +1,649 @@ +@import url('https://fonts.googleapis.com/css?family=PT+Sans'); + +body { + font-family: 'PT Sans', Helvetica, sans-serif; + font-size: 14px; +} + +a { + color: #337ab7; + text-decoration: none; +} + +a:hover { + color: #30426a; + text-decoration: underline; +} + +pre, code { + font-family: Monaco, DejaVu Sans Mono, Consolas, monospace; + font-size: 9pt; + margin: 15px 0; +} + +h1 { + font-weight: normal; + font-size: 29px; + margin: 10px 0 2px 0; + padding: 0; +} + +h2 { + font-weight: normal; + font-size: 25px; +} + +h3 > a:hover { + text-decoration: none; +} + +.document h1, .namespace-index h1 { + font-size: 32px; + margin-top: 12px; +} + +#header, #content, .sidebar { + position: fixed; +} + +#header { + top: 0; + left: 0; + right: 0; + height: 22px; + color: #f5f5f5; + padding: 5px 7px; +} + +#content { + top: 32px; + right: 0; + bottom: 0; + overflow: auto; + background: #fff; + color: #333; + padding: 0 18px; +} + +.sidebar { + position: fixed; + top: 32px; + bottom: 0; + overflow: auto; +} + +.sidebar.primary { + background: #30426a; + border-right: solid 1px #cccccc; + left: 0; + width: 250px; + color: white; + font-size: 110%; +} + +.sidebar.secondary { + background: #f2f2f2; + border-right: solid 1px #d7d7d7; + left: 251px; + width: 200px; + font-size: 110%; +} + +#content.namespace-index, #content.document { + left: 251px; +} + +#content.namespace-docs { + left: 452px; +} + +#content.document { + padding-bottom: 10%; +} + +#header { + background: #2d3e63; + box-shadow: 0 0 8px rgba(0, 0, 0, 0.4); + z-index: 100; +} + +#header h1 { + margin: 0; + padding: 0; + font-size: 18px; + font-weight: lighter; + text-shadow: -1px -1px 0px #333; +} + +#header h1 .project-version { + font-weight: normal; +} + +.project-version { + padding-left: 0.15em; +} + +#header a, .sidebar a { + display: block; + text-decoration: none; +} + +#header a { + color: #f5f5f5; +} + +.sidebar.primary, .sidebar.primary a { + color: #b2bfdc; +} + +.sidebar.primary a:hover { + color: white; +} + +.sidebar.secondary, .sidebar.secondary a { + color: #738bc0; +} + +.sidebar.secondary a:hover { + color: #2d3e63; +} + +#header h2 { + float: right; + font-size: 9pt; + font-weight: normal; + margin: 4px 3px; + padding: 0; + color: #bbb; +} + +#header h2 a { + display: inline; +} + +.sidebar h3 { + margin: 0; + padding: 10px 13px 0 13px; + font-size: 19px; + font-weight: lighter; +} + +.sidebar.primary h3.no-link { + text-transform: uppercase; + font-size: 12px; + color: #738bc0; +} + +.sidebar.secondary h3 a { + text-transform: uppercase; + font-size: 12px; + color: #2d3e63; +} + +.sidebar ul { + padding: 7px 0 6px 0; + margin: 0; +} + +.sidebar ul.index-link { + padding-bottom: 4px; +} + +.sidebar li { + display: block; + vertical-align: middle; +} + +.sidebar li a, .sidebar li .no-link { + border-left: 3px solid transparent; + padding: 0 10px; + white-space: nowrap; +} + +.sidebar li .inner { + display: inline-block; + padding-top: 7px; + height: 24px; +} + +.sidebar li a, .sidebar li .tree { + height: 31px; +} + +.depth-1 .inner { padding-left: 2px; } +.depth-2 .inner { padding-left: 6px; } +.depth-3 .inner { padding-left: 20px; } +.depth-4 .inner { padding-left: 34px; } +.depth-5 .inner { padding-left: 48px; } +.depth-6 .inner { padding-left: 62px; } + +.sidebar li .tree { + display: block; + float: left; + position: relative; + top: -10px; + margin: 0 4px 0 0; + padding: 0; +} + +.sidebar li.depth-1 .tree { + display: none; +} + +.sidebar li .tree .top, .sidebar li .tree .bottom { + display: block; + margin: 0; + padding: 0; + width: 7px; +} + +.sidebar li .tree .top { + border-left: 1px solid #aaa; + border-bottom: 1px solid #aaa; + height: 19px; +} + +.sidebar li .tree .bottom { + height: 22px; +} + +.sidebar li.branch .tree .bottom { + border-left: 1px solid #aaa; +} + +.sidebar.primary li.current a { + border-left: 3px solid #e99d1a; + color: white; +} + +.sidebar.secondary li.current a { + border-left: 3px solid #2d3e63; + color: #33a; +} + +.namespace-index h2 { + margin: 30px 0 0 0; +} + +.namespace-index h3 { + font-size: 16px; + font-weight: bold; + margin-bottom: 0; + letter-spacing: 0.05em; + border-bottom: solid 1px #ddd; + max-width: 680px; + background-color: #fafafa; + padding: 0.5em; +} + +.namespace-index .topics { + padding-left: 30px; + margin: 11px 0 0 0; +} + +.namespace-index .topics li { + padding: 5px 0; +} + +.namespace-docs h3 { + font-size: 18px; + font-weight: bold; +} + +.public h3 { + margin: 0; + float: left; +} + +.usage { + clear: both; +} + +.public { + margin: 0; + border-top: 1px solid #e0e0e0; + padding-top: 14px; + padding-bottom: 6px; +} + +.public:last-child { + margin-bottom: 20%; +} + +.members .public:last-child { + margin-bottom: 0; +} + +.members { + margin: 15px 0; +} + +.members h4 { + color: #555; + font-weight: normal; + font-variant: small-caps; + margin: 0 0 5px 0; +} + +.members .inner { + padding-top: 5px; + padding-left: 12px; + margin-top: 2px; + margin-left: 7px; + border-left: 1px solid #bbb; +} + +#content .members .inner h3 { + font-size: 12pt; +} + +.members .public { + border-top: none; + margin-top: 0; + padding-top: 6px; + padding-bottom: 0; +} + +.members .public:first-child { + padding-top: 0; +} + +h4.type, +h4.dynamic, +h4.added, +h4.deprecated { + float: left; + margin: 3px 10px 15px 0; + font-size: 15px; + font-weight: bold; + font-variant: small-caps; +} + +.public h4.type, +.public h4.dynamic, +.public h4.added, +.public h4.deprecated { + font-size: 13px; + font-weight: bold; + margin: 3px 0 0 10px; +} + +.members h4.type, +.members h4.added, +.members h4.deprecated { + margin-top: 1px; +} + +h4.type { + color: #717171; +} + +h4.dynamic { + color: #9933aa; +} + +h4.added { + color: #508820; +} + +h4.deprecated { + color: #880000; +} + +.namespace { + margin-bottom: 30px; +} + +.namespace:last-child { + margin-bottom: 10%; +} + +.index { + padding: 0; + font-size: 80%; + margin: 15px 0; + line-height: 1.6em; +} + +.index * { + display: inline; +} + +.index p { + padding-right: 3px; +} + +.index li { + padding-right: 5px; +} + +.index ul { + padding-left: 0; +} + +.type-sig { + clear: both; + color: #088; +} + +.type-sig pre { + padding-top: 10px; + margin: 0; +} + +.usage code { + display: block; + color: #008; + margin: 2px 0; +} + +.usage code:first-child { + padding-top: 10px; +} + +p { + margin: 15px 0; +} + +.public p:first-child, .public pre.plaintext { + margin-top: 12px; +} + +.doc { + margin: 0 0 26px 0; + clear: both; +} + +.public .doc { + margin: 0; +} + +.namespace-index { + font-size: 120%; +} + +.namespace-index .doc { + margin-bottom: 20px; +} + +.namespace-index .namespace .doc { + margin-bottom: 10px; +} + +.markdown p, .markdown li, .markdown dt, .markdown dd, .markdown td { + line-height: 1.6em; +} + +.markdown h2 { + font-weight: normal; + font-size: 25px; +} + +#content .markdown h3 { + font-size: 20px; +} + +.markdown h4 { + font-size: 15px; +} + +.doc, .public, .namespace .index { + max-width: 680px; + overflow-x: visible; +} + +.markdown pre > code { + display: block; + padding: 10px; +} + +.markdown pre > code, .src-link a { + border: 1px solid #e4e4e4; + border-radius: 2px; +} + +.src-link a { + background: #f6f6f6; +} + +.markdown code:not(.hljs) { + color: #c7254e; + background-color: #f9f2f4; + border-radius: 4px; + font-size: 90%; + padding: 2px 4px; +} + +pre.deps { + display: inline-block; + margin: 0 10px; + border: 1px solid #e4e4e4; + border-radius: 2px; + padding: 10px; + background-color: #f6f6f6; +} + +.markdown hr { + border-style: solid; + border-top: none; + color: #ccc; +} + +.doc ul, .doc ol { + padding-left: 30px; +} + +.doc table { + border-collapse: collapse; + margin: 0 10px; +} + +.doc table td, .doc table th { + border: 1px solid #dddddd; + padding: 4px 6px; +} + +.doc table th { + background: #f2f2f2; +} + +.doc dl { + margin: 0 10px 20px 10px; +} + +.doc dl dt { + font-weight: bold; + margin: 0; + padding: 3px 0; + border-bottom: 1px solid #ddd; +} + +.doc dl dd { + padding: 5px 0; + margin: 0 0 5px 10px; +} + +.doc abbr { + border-bottom: 1px dotted #333; + font-variant: none; + cursor: help; +} + +.src-link { + margin-bottom: 15px; +} + +.src-link a { + font-size: 70%; + padding: 1px 4px; + text-decoration: none; + color: #5555bb; + background-color: #f6f6f6; +} + +blockquote { + opacity: 0.6; + border-left: solid 2px #ddd; + margin-left: 0; + padding-left: 1em; +} + +/* Responsiveness Theme */ + +@media (max-device-width: 480px) { + .sidebar { + display:none; + } + + #content { + position: relative; + left: initial !important; + top: 110px; + padding: 0 1em; + } + + #header { + display: flex; + flex-direction: column-reverse; + height: 100px; + } + + #header > h1 { + font-size: 52px; + } + + #header h2 { + float: none; + font-size: 20px; + } + + .namespace-index > h1 { + display: none; + } + + .public, .doc, .namespace > .index, .namespace > .doc, .namespace > h3 { + max-width: initial; + } + + .doc { + text-align: justify; + } + + .public { + padding-top: 2em; + padding-bottom: 2em; + } + + .public > h3 { + font-size: 300%; + } + + .public > h4.type, .public > h4.added, .public > h4.deprecated { + font-size: 150%; + margin-top: 1em; + } + + pre > code { + font-size: 200%; + } +} diff --git a/docs/design.html b/docs/design.html new file mode 100644 index 0000000..c6a2285 --- /dev/null +++ b/docs/design.html @@ -0,0 +1,96 @@ + +LibPython-CLJ Design Notes

LibPython-CLJ Design Notes

+

Key Design Points

+

Code Organization

+

There are 3 rough sections of code: 1. A JNA layer which is a 1-1 mapping most of the C API with no changes and full documentation. The docstrings on the functions match the documentation if you lookup the 3.7 API documentation. Users must manually manage the GIL when using this API layer.

+
    +
  1. +

    An API implementation layer which knows about things like interpreters and type symbol tables. Users must know how to manipulate the GIL and how the two garbage collectors work with each other to add to this layer.

  2. +
  3. +

    A public API layer. Details like managing the GIL or messing with garbage collection in general do not leak out to this layer.

  4. +
+

Interpreters

+

Interpreters are global objects. After initialize!, there is a main interpreter which is used automatically from any thread. Access to the interpreters relies on thread local variables and interpreter locking so you can do things like:

+
(with-interpreter interpreter
+  some code)
+
+

You don’t have to do this, however. If you don’t care what context you run in you can just use the public API which under the covers simple does:

+
(with-gil
+  some code)
+
+

This type of thing grabs the GIL if it hasn’t already been claimed by the current thread and off you go. When the code is finished, it saves the interpreter thread state back into a global atom and thus releases the GIL. Interpreters have both shared and per-interpreter state named :shared-state and :interpreter-state respectively. Shared state would be the global type symbol table. Interpreter state contains things like a map of objects to their per-interpreter python bridging class.

+

If the interpreter isn’t specified it uses the main interpreter. The simplest way to ensure you have the gil is with-gil. You may also use (ensure-bound-interpreter) if you wish to penalize users for using a function incorrectly. This returns the interpreter currently bound to this thread or throws.

+

Garbage Collection

+

The system uses the tech.resource library to attach a GC hook to appropriate java object that releases the associated python object if the java object goes out of scope. Bridges use a similar technique to unregister the bridge on destruction of their python counterpart. There should be no need for manual addref/release calls in any user code aside from potentially (and damn rarely) a (System/gc) call.

+

Copying Vs. Bridging

+

Objects either in python or in java may be either copied or bridged into the other ecosystem. Bridging allows sharing complex and potentially changing datastructures while copying allows a cleaner partitioning of concerns and frees both garbage collection systems to act more independently. Numeric buffers that have a direct representation as a C-ptr (the datatype native-buffer type) have a zero-copy pathway via numpy. If you want access to object functionality that object needs to be bridged; so for example if you want to call numpy functions then you need to bridge that object. Tensors are always represented in python as numpy objects using zero-copy where possible in all cases.

+

Bridging

+

Python Callables implement clojure.lang.IFn along with a python specific interface so in general you can call them like any other function but you also can use keyword arguments if you know you are dealing with a python function. Python dicts implement java.util.Map and clojure.lang.IFn and lists are java.util.List, java.util.RandomAccess, and clojure.lang.IFn. This allows fluid manipulation of the datastructures (even mutation) from both languages.

+

You can create a python function from a clojure function with create-function. You can create a new bridge type by implementing the libpython_clj.jna.JVMBridge interface:

+
public interface JVMBridge extends AutoCloseable
+{
+  public Pointer getAttr(String name);
+  public void setAttr(String name, Pointer val);
+  public String[] dir();
+  public Object interpreter();
+  public Object wrappedObject();
+  // Called from python when the python mirror is deleted.
+  // This had better not throw anything.
+  public default void close() {}
+}
+
+

If all you want to do is override the attribute map that is simple. Here is the bridge for java.util.Map:

+
(defn jvm-map->python
+  ^Pointer [^Map jvm-data]
+  (with-gil
+    (let [att-map
+          {"__contains__" (jvm-fn->python #(.containsKey jvm-data %))
+           "__eq__" (jvm-fn->python #(.equals jvm-data %))
+           "__getitem__" (jvm-fn->python #(.get jvm-data %))
+           "__setitem__" (jvm-fn->python #(.put jvm-data %1 %2))
+           "__hash__" (jvm-fn->python #(.hashCode jvm-data))
+           "__iter__" (jvm-fn->python #(.iterator ^Iterable jvm-data))
+           "__len__" (jvm-fn->python #(.size jvm-data))
+           "__str__" (jvm-fn->python #(.toString jvm-data))
+           "clear" (jvm-fn->python #(.clear jvm-data))
+           "keys" (jvm-fn->python #(seq (.keySet jvm-data)))
+           "values" (jvm-fn->python #(seq (.values jvm-data)))
+           "pop" (jvm-fn->python #(.remove jvm-data %))}]
+      (create-bridge-from-att-map jvm-data att-map))))
+
+

IO

+

sys.stdout and sys.stderr are redirected to *out* and *err* respectively. This rerouting is done using a bridge that is then set as sys.stdout or sys.stderr:

+
(defn create-var-writer
+  "Returns an unregistered bridge"
+  ^Pointer [writer-var]
+  (with-gil
+    (create-bridge-from-att-map
+     writer-var
+     {"write" (->python (fn [& args]
+                          (.write ^Writer @writer-var (str (first args)))))
+      "flush" (->python (fn [& args]))
+      "isatty" (->python (fn [& args]
+                           (libpy/Py_False)))
+      })))
+
+(defn setup-std-writer
+  [writer-var sys-mod-attname]
+  (with-gil
+    (let [sys-module (import-module "sys")
+          std-out-writer (get-or-create-var-writer writer-var)]
+      (py-proto/set-attr! sys-module sys-mod-attname std-out-writer)
+      :ok)))
+
+
+(defn initialize!
+  [& {:keys [program-name no-io-redirect?]}]
+  (when-not @pyinterp/*main-interpreter*
+    (pyinterp/initialize! program-name)
+    ;;setup bridge mechansim and io redirection
+    (pyinterop/register-bridge-type!)
+    (when-not no-io-redirect?
+      (pyinterop/setup-std-writer #'*err* "stderr")
+      (pyinterop/setup-std-writer #'*out* "stdout")))
+  :ok)
+
\ No newline at end of file diff --git a/docs/highlight/highlight.min.js b/docs/highlight/highlight.min.js new file mode 100644 index 0000000..6486ffd --- /dev/null +++ b/docs/highlight/highlight.min.js @@ -0,0 +1,2 @@ +/*! highlight.js v9.6.0 | BSD3 License | git.io/hljslicense */ +!function(e){var n="object"==typeof window&&window||"object"==typeof self&&self;"undefined"!=typeof exports?e(exports):n&&(n.hljs=e({}),"function"==typeof define&&define.amd&&define([],function(){return n.hljs}))}(function(e){function n(e){return e.replace(/[&<>]/gm,function(e){return I[e]})}function t(e){return e.nodeName.toLowerCase()}function r(e,n){var t=e&&e.exec(n);return t&&0===t.index}function a(e){return k.test(e)}function i(e){var n,t,r,i,o=e.className+" ";if(o+=e.parentNode?e.parentNode.className:"",t=B.exec(o))return R(t[1])?t[1]:"no-highlight";for(o=o.split(/\s+/),n=0,r=o.length;r>n;n++)if(i=o[n],a(i)||R(i))return i}function o(e,n){var t,r={};for(t in e)r[t]=e[t];if(n)for(t in n)r[t]=n[t];return r}function u(e){var n=[];return function r(e,a){for(var i=e.firstChild;i;i=i.nextSibling)3===i.nodeType?a+=i.nodeValue.length:1===i.nodeType&&(n.push({event:"start",offset:a,node:i}),a=r(i,a),t(i).match(/br|hr|img|input/)||n.push({event:"stop",offset:a,node:i}));return a}(e,0),n}function c(e,r,a){function i(){return e.length&&r.length?e[0].offset!==r[0].offset?e[0].offset"}function u(e){l+=""}function c(e){("start"===e.event?o:u)(e.node)}for(var s=0,l="",f=[];e.length||r.length;){var g=i();if(l+=n(a.substr(s,g[0].offset-s)),s=g[0].offset,g===e){f.reverse().forEach(u);do c(g.splice(0,1)[0]),g=i();while(g===e&&g.length&&g[0].offset===s);f.reverse().forEach(o)}else"start"===g[0].event?f.push(g[0].node):f.pop(),c(g.splice(0,1)[0])}return l+n(a.substr(s))}function s(e){function n(e){return e&&e.source||e}function t(t,r){return new RegExp(n(t),"m"+(e.cI?"i":"")+(r?"g":""))}function r(a,i){if(!a.compiled){if(a.compiled=!0,a.k=a.k||a.bK,a.k){var u={},c=function(n,t){e.cI&&(t=t.toLowerCase()),t.split(" ").forEach(function(e){var t=e.split("|");u[t[0]]=[n,t[1]?Number(t[1]):1]})};"string"==typeof a.k?c("keyword",a.k):E(a.k).forEach(function(e){c(e,a.k[e])}),a.k=u}a.lR=t(a.l||/\w+/,!0),i&&(a.bK&&(a.b="\\b("+a.bK.split(" ").join("|")+")\\b"),a.b||(a.b=/\B|\b/),a.bR=t(a.b),a.e||a.eW||(a.e=/\B|\b/),a.e&&(a.eR=t(a.e)),a.tE=n(a.e)||"",a.eW&&i.tE&&(a.tE+=(a.e?"|":"")+i.tE)),a.i&&(a.iR=t(a.i)),null==a.r&&(a.r=1),a.c||(a.c=[]);var s=[];a.c.forEach(function(e){e.v?e.v.forEach(function(n){s.push(o(e,n))}):s.push("self"===e?a:e)}),a.c=s,a.c.forEach(function(e){r(e,a)}),a.starts&&r(a.starts,i);var l=a.c.map(function(e){return e.bK?"\\.?("+e.b+")\\.?":e.b}).concat([a.tE,a.i]).map(n).filter(Boolean);a.t=l.length?t(l.join("|"),!0):{exec:function(){return null}}}}r(e)}function l(e,t,a,i){function o(e,n){var t,a;for(t=0,a=n.c.length;a>t;t++)if(r(n.c[t].bR,e))return n.c[t]}function u(e,n){if(r(e.eR,n)){for(;e.endsParent&&e.parent;)e=e.parent;return e}return e.eW?u(e.parent,n):void 0}function c(e,n){return!a&&r(n.iR,e)}function g(e,n){var t=N.cI?n[0].toLowerCase():n[0];return e.k.hasOwnProperty(t)&&e.k[t]}function h(e,n,t,r){var a=r?"":y.classPrefix,i='',i+n+o}function p(){var e,t,r,a;if(!E.k)return n(B);for(a="",t=0,E.lR.lastIndex=0,r=E.lR.exec(B);r;)a+=n(B.substr(t,r.index-t)),e=g(E,r),e?(M+=e[1],a+=h(e[0],n(r[0]))):a+=n(r[0]),t=E.lR.lastIndex,r=E.lR.exec(B);return a+n(B.substr(t))}function d(){var e="string"==typeof E.sL;if(e&&!x[E.sL])return n(B);var t=e?l(E.sL,B,!0,L[E.sL]):f(B,E.sL.length?E.sL:void 0);return E.r>0&&(M+=t.r),e&&(L[E.sL]=t.top),h(t.language,t.value,!1,!0)}function b(){k+=null!=E.sL?d():p(),B=""}function v(e){k+=e.cN?h(e.cN,"",!0):"",E=Object.create(e,{parent:{value:E}})}function m(e,n){if(B+=e,null==n)return b(),0;var t=o(n,E);if(t)return t.skip?B+=n:(t.eB&&(B+=n),b(),t.rB||t.eB||(B=n)),v(t,n),t.rB?0:n.length;var r=u(E,n);if(r){var a=E;a.skip?B+=n:(a.rE||a.eE||(B+=n),b(),a.eE&&(B=n));do E.cN&&(k+=C),E.skip||(M+=E.r),E=E.parent;while(E!==r.parent);return r.starts&&v(r.starts,""),a.rE?0:n.length}if(c(n,E))throw new Error('Illegal lexeme "'+n+'" for mode "'+(E.cN||"")+'"');return B+=n,n.length||1}var N=R(e);if(!N)throw new Error('Unknown language: "'+e+'"');s(N);var w,E=i||N,L={},k="";for(w=E;w!==N;w=w.parent)w.cN&&(k=h(w.cN,"",!0)+k);var B="",M=0;try{for(var I,j,O=0;;){if(E.t.lastIndex=O,I=E.t.exec(t),!I)break;j=m(t.substr(O,I.index-O),I[0]),O=I.index+j}for(m(t.substr(O)),w=E;w.parent;w=w.parent)w.cN&&(k+=C);return{r:M,value:k,language:e,top:E}}catch(T){if(T.message&&-1!==T.message.indexOf("Illegal"))return{r:0,value:n(t)};throw T}}function f(e,t){t=t||y.languages||E(x);var r={r:0,value:n(e)},a=r;return t.filter(R).forEach(function(n){var t=l(n,e,!1);t.language=n,t.r>a.r&&(a=t),t.r>r.r&&(a=r,r=t)}),a.language&&(r.second_best=a),r}function g(e){return y.tabReplace||y.useBR?e.replace(M,function(e,n){return y.useBR&&"\n"===e?"
":y.tabReplace?n.replace(/\t/g,y.tabReplace):void 0}):e}function h(e,n,t){var r=n?L[n]:t,a=[e.trim()];return e.match(/\bhljs\b/)||a.push("hljs"),-1===e.indexOf(r)&&a.push(r),a.join(" ").trim()}function p(e){var n,t,r,o,s,p=i(e);a(p)||(y.useBR?(n=document.createElementNS("http://www.w3.org/1999/xhtml","div"),n.innerHTML=e.innerHTML.replace(/\n/g,"").replace(//g,"\n")):n=e,s=n.textContent,r=p?l(p,s,!0):f(s),t=u(n),t.length&&(o=document.createElementNS("http://www.w3.org/1999/xhtml","div"),o.innerHTML=r.value,r.value=c(t,u(o),s)),r.value=g(r.value),e.innerHTML=r.value,e.className=h(e.className,p,r.language),e.result={language:r.language,re:r.r},r.second_best&&(e.second_best={language:r.second_best.language,re:r.second_best.r}))}function d(e){y=o(y,e)}function b(){if(!b.called){b.called=!0;var e=document.querySelectorAll("pre code");w.forEach.call(e,p)}}function v(){addEventListener("DOMContentLoaded",b,!1),addEventListener("load",b,!1)}function m(n,t){var r=x[n]=t(e);r.aliases&&r.aliases.forEach(function(e){L[e]=n})}function N(){return E(x)}function R(e){return e=(e||"").toLowerCase(),x[e]||x[L[e]]}var w=[],E=Object.keys,x={},L={},k=/^(no-?highlight|plain|text)$/i,B=/\blang(?:uage)?-([\w-]+)\b/i,M=/((^(<[^>]+>|\t|)+|(?:\n)))/gm,C="
",y={classPrefix:"hljs-",tabReplace:null,useBR:!1,languages:void 0},I={"&":"&","<":"<",">":">"};return e.highlight=l,e.highlightAuto=f,e.fixMarkup=g,e.highlightBlock=p,e.configure=d,e.initHighlighting=b,e.initHighlightingOnLoad=v,e.registerLanguage=m,e.listLanguages=N,e.getLanguage=R,e.inherit=o,e.IR="[a-zA-Z]\\w*",e.UIR="[a-zA-Z_]\\w*",e.NR="\\b\\d+(\\.\\d+)?",e.CNR="(-?)(\\b0[xX][a-fA-F0-9]+|(\\b\\d+(\\.\\d*)?|\\.\\d+)([eE][-+]?\\d+)?)",e.BNR="\\b(0b[01]+)",e.RSR="!|!=|!==|%|%=|&|&&|&=|\\*|\\*=|\\+|\\+=|,|-|-=|/=|/|:|;|<<|<<=|<=|<|===|==|=|>>>=|>>=|>=|>>>|>>|>|\\?|\\[|\\{|\\(|\\^|\\^=|\\||\\|=|\\|\\||~",e.BE={b:"\\\\[\\s\\S]",r:0},e.ASM={cN:"string",b:"'",e:"'",i:"\\n",c:[e.BE]},e.QSM={cN:"string",b:'"',e:'"',i:"\\n",c:[e.BE]},e.PWM={b:/\b(a|an|the|are|I'm|isn't|don't|doesn't|won't|but|just|should|pretty|simply|enough|gonna|going|wtf|so|such|will|you|your|like)\b/},e.C=function(n,t,r){var a=e.inherit({cN:"comment",b:n,e:t,c:[]},r||{});return a.c.push(e.PWM),a.c.push({cN:"doctag",b:"(?:TODO|FIXME|NOTE|BUG|XXX):",r:0}),a},e.CLCM=e.C("//","$"),e.CBCM=e.C("/\\*","\\*/"),e.HCM=e.C("#","$"),e.NM={cN:"number",b:e.NR,r:0},e.CNM={cN:"number",b:e.CNR,r:0},e.BNM={cN:"number",b:e.BNR,r:0},e.CSSNM={cN:"number",b:e.NR+"(%|em|ex|ch|rem|vw|vh|vmin|vmax|cm|mm|in|pt|pc|px|deg|grad|rad|turn|s|ms|Hz|kHz|dpi|dpcm|dppx)?",r:0},e.RM={cN:"regexp",b:/\//,e:/\/[gimuy]*/,i:/\n/,c:[e.BE,{b:/\[/,e:/\]/,r:0,c:[e.BE]}]},e.TM={cN:"title",b:e.IR,r:0},e.UTM={cN:"title",b:e.UIR,r:0},e.METHOD_GUARD={b:"\\.\\s*"+e.UIR,r:0},e});hljs.registerLanguage("clojure",function(e){var t={"builtin-name":"def defonce cond apply if-not if-let if not not= = < > <= >= == + / * - rem quot neg? pos? delay? symbol? keyword? true? false? integer? empty? coll? list? set? ifn? fn? associative? sequential? sorted? counted? reversible? number? decimal? class? distinct? isa? float? rational? reduced? ratio? odd? even? char? seq? vector? string? map? nil? contains? zero? instance? not-every? not-any? libspec? -> ->> .. . inc compare do dotimes mapcat take remove take-while drop letfn drop-last take-last drop-while while intern condp case reduced cycle split-at split-with repeat replicate iterate range merge zipmap declare line-seq sort comparator sort-by dorun doall nthnext nthrest partition eval doseq await await-for let agent atom send send-off release-pending-sends add-watch mapv filterv remove-watch agent-error restart-agent set-error-handler error-handler set-error-mode! error-mode shutdown-agents quote var fn loop recur throw try monitor-enter monitor-exit defmacro defn defn- macroexpand macroexpand-1 for dosync and or when when-not when-let comp juxt partial sequence memoize constantly complement identity assert peek pop doto proxy defstruct first rest cons defprotocol cast coll deftype defrecord last butlast sigs reify second ffirst fnext nfirst nnext defmulti defmethod meta with-meta ns in-ns create-ns import refer keys select-keys vals key val rseq name namespace promise into transient persistent! conj! assoc! dissoc! pop! disj! use class type num float double short byte boolean bigint biginteger bigdec print-method print-dup throw-if printf format load compile get-in update-in pr pr-on newline flush read slurp read-line subvec with-open memfn time re-find re-groups rand-int rand mod locking assert-valid-fdecl alias resolve ref deref refset swap! reset! set-validator! compare-and-set! alter-meta! reset-meta! commute get-validator alter ref-set ref-history-count ref-min-history ref-max-history ensure sync io! new next conj set! to-array future future-call into-array aset gen-class reduce map filter find empty hash-map hash-set sorted-map sorted-map-by sorted-set sorted-set-by vec vector seq flatten reverse assoc dissoc list disj get union difference intersection extend extend-type extend-protocol int nth delay count concat chunk chunk-buffer chunk-append chunk-first chunk-rest max min dec unchecked-inc-int unchecked-inc unchecked-dec-inc unchecked-dec unchecked-negate unchecked-add-int unchecked-add unchecked-subtract-int unchecked-subtract chunk-next chunk-cons chunked-seq? prn vary-meta lazy-seq spread list* str find-keyword keyword symbol gensym force rationalize"},r="a-zA-Z_\\-!.?+*=<>&#'",n="["+r+"]["+r+"0-9/;:]*",a="[-+]?\\d+(\\.\\d+)?",o={b:n,r:0},s={cN:"number",b:a,r:0},i=e.inherit(e.QSM,{i:null}),c=e.C(";","$",{r:0}),d={cN:"literal",b:/\b(true|false|nil)\b/},l={b:"[\\[\\{]",e:"[\\]\\}]"},m={cN:"comment",b:"\\^"+n},p=e.C("\\^\\{","\\}"),u={cN:"symbol",b:"[:]{1,2}"+n},f={b:"\\(",e:"\\)"},h={eW:!0,r:0},y={k:t,l:n,cN:"name",b:n,starts:h},b=[f,i,m,p,c,u,l,s,d,o];return f.c=[e.C("comment",""),y,h],h.c=b,l.c=b,{aliases:["clj"],i:/\S/,c:[f,i,m,p,c,u,l,s,d]}});hljs.registerLanguage("clojure-repl",function(e){return{c:[{cN:"meta",b:/^([\w.-]+|\s*#_)=>/,starts:{e:/$/,sL:"clojure"}}]}}); \ No newline at end of file diff --git a/docs/highlight/solarized-light.css b/docs/highlight/solarized-light.css new file mode 100644 index 0000000..fdcfcc7 --- /dev/null +++ b/docs/highlight/solarized-light.css @@ -0,0 +1,84 @@ +/* + +Orginal Style from ethanschoonover.com/solarized (c) Jeremy Hull + +*/ + +.hljs { + display: block; + overflow-x: auto; + padding: 0.5em; + background: #fdf6e3; + color: #657b83; +} + +.hljs-comment, +.hljs-quote { + color: #93a1a1; +} + +/* Solarized Green */ +.hljs-keyword, +.hljs-selector-tag, +.hljs-addition { + color: #859900; +} + +/* Solarized Cyan */ +.hljs-number, +.hljs-string, +.hljs-meta .hljs-meta-string, +.hljs-literal, +.hljs-doctag, +.hljs-regexp { + color: #2aa198; +} + +/* Solarized Blue */ +.hljs-title, +.hljs-section, +.hljs-name, +.hljs-selector-id, +.hljs-selector-class { + color: #268bd2; +} + +/* Solarized Yellow */ +.hljs-attribute, +.hljs-attr, +.hljs-variable, +.hljs-template-variable, +.hljs-class .hljs-title, +.hljs-type { + color: #b58900; +} + +/* Solarized Orange */ +.hljs-symbol, +.hljs-bullet, +.hljs-subst, +.hljs-meta, +.hljs-meta .hljs-keyword, +.hljs-selector-attr, +.hljs-selector-pseudo, +.hljs-link { + color: #cb4b16; +} + +/* Solarized Red */ +.hljs-built_in, +.hljs-deletion { + color: #dc322f; +} + +.hljs-formula { + background: #eee8d5; +} + +.hljs-emphasis { + font-style: italic; +} + +.hljs-strong { + font-weight: bold; +} diff --git a/docs/index.html b/docs/index.html new file mode 100644 index 0000000..eb81ef0 --- /dev/null +++ b/docs/index.html @@ -0,0 +1,3 @@ + +tech.ml.dataset 1.47-SNAPSHOT \ No newline at end of file diff --git a/docs/js/jquery.min.js b/docs/js/jquery.min.js new file mode 100644 index 0000000..73f33fb --- /dev/null +++ b/docs/js/jquery.min.js @@ -0,0 +1,4 @@ +/*! jQuery v1.11.0 | (c) 2005, 2014 jQuery Foundation, Inc. | jquery.org/license */ +!function(a,b){"object"==typeof module&&"object"==typeof module.exports?module.exports=a.document?b(a,!0):function(a){if(!a.document)throw new Error("jQuery requires a window with a document");return b(a)}:b(a)}("undefined"!=typeof window?window:this,function(a,b){var c=[],d=c.slice,e=c.concat,f=c.push,g=c.indexOf,h={},i=h.toString,j=h.hasOwnProperty,k="".trim,l={},m="1.11.0",n=function(a,b){return new n.fn.init(a,b)},o=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g,p=/^-ms-/,q=/-([\da-z])/gi,r=function(a,b){return b.toUpperCase()};n.fn=n.prototype={jquery:m,constructor:n,selector:"",length:0,toArray:function(){return d.call(this)},get:function(a){return null!=a?0>a?this[a+this.length]:this[a]:d.call(this)},pushStack:function(a){var b=n.merge(this.constructor(),a);return b.prevObject=this,b.context=this.context,b},each:function(a,b){return n.each(this,a,b)},map:function(a){return this.pushStack(n.map(this,function(b,c){return a.call(b,c,b)}))},slice:function(){return this.pushStack(d.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},eq:function(a){var b=this.length,c=+a+(0>a?b:0);return this.pushStack(c>=0&&b>c?[this[c]]:[])},end:function(){return this.prevObject||this.constructor(null)},push:f,sort:c.sort,splice:c.splice},n.extend=n.fn.extend=function(){var a,b,c,d,e,f,g=arguments[0]||{},h=1,i=arguments.length,j=!1;for("boolean"==typeof g&&(j=g,g=arguments[h]||{},h++),"object"==typeof g||n.isFunction(g)||(g={}),h===i&&(g=this,h--);i>h;h++)if(null!=(e=arguments[h]))for(d in e)a=g[d],c=e[d],g!==c&&(j&&c&&(n.isPlainObject(c)||(b=n.isArray(c)))?(b?(b=!1,f=a&&n.isArray(a)?a:[]):f=a&&n.isPlainObject(a)?a:{},g[d]=n.extend(j,f,c)):void 0!==c&&(g[d]=c));return g},n.extend({expando:"jQuery"+(m+Math.random()).replace(/\D/g,""),isReady:!0,error:function(a){throw new Error(a)},noop:function(){},isFunction:function(a){return"function"===n.type(a)},isArray:Array.isArray||function(a){return"array"===n.type(a)},isWindow:function(a){return null!=a&&a==a.window},isNumeric:function(a){return a-parseFloat(a)>=0},isEmptyObject:function(a){var b;for(b in a)return!1;return!0},isPlainObject:function(a){var b;if(!a||"object"!==n.type(a)||a.nodeType||n.isWindow(a))return!1;try{if(a.constructor&&!j.call(a,"constructor")&&!j.call(a.constructor.prototype,"isPrototypeOf"))return!1}catch(c){return!1}if(l.ownLast)for(b in a)return j.call(a,b);for(b in a);return void 0===b||j.call(a,b)},type:function(a){return null==a?a+"":"object"==typeof a||"function"==typeof a?h[i.call(a)]||"object":typeof a},globalEval:function(b){b&&n.trim(b)&&(a.execScript||function(b){a.eval.call(a,b)})(b)},camelCase:function(a){return a.replace(p,"ms-").replace(q,r)},nodeName:function(a,b){return a.nodeName&&a.nodeName.toLowerCase()===b.toLowerCase()},each:function(a,b,c){var d,e=0,f=a.length,g=s(a);if(c){if(g){for(;f>e;e++)if(d=b.apply(a[e],c),d===!1)break}else for(e in a)if(d=b.apply(a[e],c),d===!1)break}else if(g){for(;f>e;e++)if(d=b.call(a[e],e,a[e]),d===!1)break}else for(e in a)if(d=b.call(a[e],e,a[e]),d===!1)break;return a},trim:k&&!k.call("\ufeff\xa0")?function(a){return null==a?"":k.call(a)}:function(a){return null==a?"":(a+"").replace(o,"")},makeArray:function(a,b){var c=b||[];return null!=a&&(s(Object(a))?n.merge(c,"string"==typeof a?[a]:a):f.call(c,a)),c},inArray:function(a,b,c){var d;if(b){if(g)return g.call(b,a,c);for(d=b.length,c=c?0>c?Math.max(0,d+c):c:0;d>c;c++)if(c in b&&b[c]===a)return c}return-1},merge:function(a,b){var c=+b.length,d=0,e=a.length;while(c>d)a[e++]=b[d++];if(c!==c)while(void 0!==b[d])a[e++]=b[d++];return a.length=e,a},grep:function(a,b,c){for(var d,e=[],f=0,g=a.length,h=!c;g>f;f++)d=!b(a[f],f),d!==h&&e.push(a[f]);return e},map:function(a,b,c){var d,f=0,g=a.length,h=s(a),i=[];if(h)for(;g>f;f++)d=b(a[f],f,c),null!=d&&i.push(d);else for(f in a)d=b(a[f],f,c),null!=d&&i.push(d);return e.apply([],i)},guid:1,proxy:function(a,b){var c,e,f;return"string"==typeof b&&(f=a[b],b=a,a=f),n.isFunction(a)?(c=d.call(arguments,2),e=function(){return a.apply(b||this,c.concat(d.call(arguments)))},e.guid=a.guid=a.guid||n.guid++,e):void 0},now:function(){return+new Date},support:l}),n.each("Boolean Number String Function Array Date RegExp Object Error".split(" "),function(a,b){h["[object "+b+"]"]=b.toLowerCase()});function s(a){var b=a.length,c=n.type(a);return"function"===c||n.isWindow(a)?!1:1===a.nodeType&&b?!0:"array"===c||0===b||"number"==typeof b&&b>0&&b-1 in a}var t=function(a){var b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s="sizzle"+-new Date,t=a.document,u=0,v=0,w=eb(),x=eb(),y=eb(),z=function(a,b){return a===b&&(j=!0),0},A="undefined",B=1<<31,C={}.hasOwnProperty,D=[],E=D.pop,F=D.push,G=D.push,H=D.slice,I=D.indexOf||function(a){for(var b=0,c=this.length;c>b;b++)if(this[b]===a)return b;return-1},J="checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped",K="[\\x20\\t\\r\\n\\f]",L="(?:\\\\.|[\\w-]|[^\\x00-\\xa0])+",M=L.replace("w","w#"),N="\\["+K+"*("+L+")"+K+"*(?:([*^$|!~]?=)"+K+"*(?:(['\"])((?:\\\\.|[^\\\\])*?)\\3|("+M+")|)|)"+K+"*\\]",O=":("+L+")(?:\\(((['\"])((?:\\\\.|[^\\\\])*?)\\3|((?:\\\\.|[^\\\\()[\\]]|"+N.replace(3,8)+")*)|.*)\\)|)",P=new RegExp("^"+K+"+|((?:^|[^\\\\])(?:\\\\.)*)"+K+"+$","g"),Q=new RegExp("^"+K+"*,"+K+"*"),R=new RegExp("^"+K+"*([>+~]|"+K+")"+K+"*"),S=new RegExp("="+K+"*([^\\]'\"]*?)"+K+"*\\]","g"),T=new RegExp(O),U=new RegExp("^"+M+"$"),V={ID:new RegExp("^#("+L+")"),CLASS:new RegExp("^\\.("+L+")"),TAG:new RegExp("^("+L.replace("w","w*")+")"),ATTR:new RegExp("^"+N),PSEUDO:new RegExp("^"+O),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+K+"*(even|odd|(([+-]|)(\\d*)n|)"+K+"*(?:([+-]|)"+K+"*(\\d+)|))"+K+"*\\)|)","i"),bool:new RegExp("^(?:"+J+")$","i"),needsContext:new RegExp("^"+K+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+K+"*((?:-\\d)?\\d*)"+K+"*\\)|)(?=[^-]|$)","i")},W=/^(?:input|select|textarea|button)$/i,X=/^h\d$/i,Y=/^[^{]+\{\s*\[native \w/,Z=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,$=/[+~]/,_=/'|\\/g,ab=new RegExp("\\\\([\\da-f]{1,6}"+K+"?|("+K+")|.)","ig"),bb=function(a,b,c){var d="0x"+b-65536;return d!==d||c?b:0>d?String.fromCharCode(d+65536):String.fromCharCode(d>>10|55296,1023&d|56320)};try{G.apply(D=H.call(t.childNodes),t.childNodes),D[t.childNodes.length].nodeType}catch(cb){G={apply:D.length?function(a,b){F.apply(a,H.call(b))}:function(a,b){var c=a.length,d=0;while(a[c++]=b[d++]);a.length=c-1}}}function db(a,b,d,e){var f,g,h,i,j,m,p,q,u,v;if((b?b.ownerDocument||b:t)!==l&&k(b),b=b||l,d=d||[],!a||"string"!=typeof a)return d;if(1!==(i=b.nodeType)&&9!==i)return[];if(n&&!e){if(f=Z.exec(a))if(h=f[1]){if(9===i){if(g=b.getElementById(h),!g||!g.parentNode)return d;if(g.id===h)return d.push(g),d}else if(b.ownerDocument&&(g=b.ownerDocument.getElementById(h))&&r(b,g)&&g.id===h)return d.push(g),d}else{if(f[2])return G.apply(d,b.getElementsByTagName(a)),d;if((h=f[3])&&c.getElementsByClassName&&b.getElementsByClassName)return G.apply(d,b.getElementsByClassName(h)),d}if(c.qsa&&(!o||!o.test(a))){if(q=p=s,u=b,v=9===i&&a,1===i&&"object"!==b.nodeName.toLowerCase()){m=ob(a),(p=b.getAttribute("id"))?q=p.replace(_,"\\$&"):b.setAttribute("id",q),q="[id='"+q+"'] ",j=m.length;while(j--)m[j]=q+pb(m[j]);u=$.test(a)&&mb(b.parentNode)||b,v=m.join(",")}if(v)try{return G.apply(d,u.querySelectorAll(v)),d}catch(w){}finally{p||b.removeAttribute("id")}}}return xb(a.replace(P,"$1"),b,d,e)}function eb(){var a=[];function b(c,e){return a.push(c+" ")>d.cacheLength&&delete b[a.shift()],b[c+" "]=e}return b}function fb(a){return a[s]=!0,a}function gb(a){var b=l.createElement("div");try{return!!a(b)}catch(c){return!1}finally{b.parentNode&&b.parentNode.removeChild(b),b=null}}function hb(a,b){var c=a.split("|"),e=a.length;while(e--)d.attrHandle[c[e]]=b}function ib(a,b){var c=b&&a,d=c&&1===a.nodeType&&1===b.nodeType&&(~b.sourceIndex||B)-(~a.sourceIndex||B);if(d)return d;if(c)while(c=c.nextSibling)if(c===b)return-1;return a?1:-1}function jb(a){return function(b){var c=b.nodeName.toLowerCase();return"input"===c&&b.type===a}}function kb(a){return function(b){var c=b.nodeName.toLowerCase();return("input"===c||"button"===c)&&b.type===a}}function lb(a){return fb(function(b){return b=+b,fb(function(c,d){var e,f=a([],c.length,b),g=f.length;while(g--)c[e=f[g]]&&(c[e]=!(d[e]=c[e]))})})}function mb(a){return a&&typeof a.getElementsByTagName!==A&&a}c=db.support={},f=db.isXML=function(a){var b=a&&(a.ownerDocument||a).documentElement;return b?"HTML"!==b.nodeName:!1},k=db.setDocument=function(a){var b,e=a?a.ownerDocument||a:t,g=e.defaultView;return e!==l&&9===e.nodeType&&e.documentElement?(l=e,m=e.documentElement,n=!f(e),g&&g!==g.top&&(g.addEventListener?g.addEventListener("unload",function(){k()},!1):g.attachEvent&&g.attachEvent("onunload",function(){k()})),c.attributes=gb(function(a){return a.className="i",!a.getAttribute("className")}),c.getElementsByTagName=gb(function(a){return a.appendChild(e.createComment("")),!a.getElementsByTagName("*").length}),c.getElementsByClassName=Y.test(e.getElementsByClassName)&&gb(function(a){return a.innerHTML="
",a.firstChild.className="i",2===a.getElementsByClassName("i").length}),c.getById=gb(function(a){return m.appendChild(a).id=s,!e.getElementsByName||!e.getElementsByName(s).length}),c.getById?(d.find.ID=function(a,b){if(typeof b.getElementById!==A&&n){var c=b.getElementById(a);return c&&c.parentNode?[c]:[]}},d.filter.ID=function(a){var b=a.replace(ab,bb);return function(a){return a.getAttribute("id")===b}}):(delete d.find.ID,d.filter.ID=function(a){var b=a.replace(ab,bb);return function(a){var c=typeof a.getAttributeNode!==A&&a.getAttributeNode("id");return c&&c.value===b}}),d.find.TAG=c.getElementsByTagName?function(a,b){return typeof b.getElementsByTagName!==A?b.getElementsByTagName(a):void 0}:function(a,b){var c,d=[],e=0,f=b.getElementsByTagName(a);if("*"===a){while(c=f[e++])1===c.nodeType&&d.push(c);return d}return f},d.find.CLASS=c.getElementsByClassName&&function(a,b){return typeof b.getElementsByClassName!==A&&n?b.getElementsByClassName(a):void 0},p=[],o=[],(c.qsa=Y.test(e.querySelectorAll))&&(gb(function(a){a.innerHTML="",a.querySelectorAll("[t^='']").length&&o.push("[*^$]="+K+"*(?:''|\"\")"),a.querySelectorAll("[selected]").length||o.push("\\["+K+"*(?:value|"+J+")"),a.querySelectorAll(":checked").length||o.push(":checked")}),gb(function(a){var b=e.createElement("input");b.setAttribute("type","hidden"),a.appendChild(b).setAttribute("name","D"),a.querySelectorAll("[name=d]").length&&o.push("name"+K+"*[*^$|!~]?="),a.querySelectorAll(":enabled").length||o.push(":enabled",":disabled"),a.querySelectorAll("*,:x"),o.push(",.*:")})),(c.matchesSelector=Y.test(q=m.webkitMatchesSelector||m.mozMatchesSelector||m.oMatchesSelector||m.msMatchesSelector))&&gb(function(a){c.disconnectedMatch=q.call(a,"div"),q.call(a,"[s!='']:x"),p.push("!=",O)}),o=o.length&&new RegExp(o.join("|")),p=p.length&&new RegExp(p.join("|")),b=Y.test(m.compareDocumentPosition),r=b||Y.test(m.contains)?function(a,b){var c=9===a.nodeType?a.documentElement:a,d=b&&b.parentNode;return a===d||!(!d||1!==d.nodeType||!(c.contains?c.contains(d):a.compareDocumentPosition&&16&a.compareDocumentPosition(d)))}:function(a,b){if(b)while(b=b.parentNode)if(b===a)return!0;return!1},z=b?function(a,b){if(a===b)return j=!0,0;var d=!a.compareDocumentPosition-!b.compareDocumentPosition;return d?d:(d=(a.ownerDocument||a)===(b.ownerDocument||b)?a.compareDocumentPosition(b):1,1&d||!c.sortDetached&&b.compareDocumentPosition(a)===d?a===e||a.ownerDocument===t&&r(t,a)?-1:b===e||b.ownerDocument===t&&r(t,b)?1:i?I.call(i,a)-I.call(i,b):0:4&d?-1:1)}:function(a,b){if(a===b)return j=!0,0;var c,d=0,f=a.parentNode,g=b.parentNode,h=[a],k=[b];if(!f||!g)return a===e?-1:b===e?1:f?-1:g?1:i?I.call(i,a)-I.call(i,b):0;if(f===g)return ib(a,b);c=a;while(c=c.parentNode)h.unshift(c);c=b;while(c=c.parentNode)k.unshift(c);while(h[d]===k[d])d++;return d?ib(h[d],k[d]):h[d]===t?-1:k[d]===t?1:0},e):l},db.matches=function(a,b){return db(a,null,null,b)},db.matchesSelector=function(a,b){if((a.ownerDocument||a)!==l&&k(a),b=b.replace(S,"='$1']"),!(!c.matchesSelector||!n||p&&p.test(b)||o&&o.test(b)))try{var d=q.call(a,b);if(d||c.disconnectedMatch||a.document&&11!==a.document.nodeType)return d}catch(e){}return db(b,l,null,[a]).length>0},db.contains=function(a,b){return(a.ownerDocument||a)!==l&&k(a),r(a,b)},db.attr=function(a,b){(a.ownerDocument||a)!==l&&k(a);var e=d.attrHandle[b.toLowerCase()],f=e&&C.call(d.attrHandle,b.toLowerCase())?e(a,b,!n):void 0;return void 0!==f?f:c.attributes||!n?a.getAttribute(b):(f=a.getAttributeNode(b))&&f.specified?f.value:null},db.error=function(a){throw new Error("Syntax error, unrecognized expression: "+a)},db.uniqueSort=function(a){var b,d=[],e=0,f=0;if(j=!c.detectDuplicates,i=!c.sortStable&&a.slice(0),a.sort(z),j){while(b=a[f++])b===a[f]&&(e=d.push(f));while(e--)a.splice(d[e],1)}return i=null,a},e=db.getText=function(a){var b,c="",d=0,f=a.nodeType;if(f){if(1===f||9===f||11===f){if("string"==typeof a.textContent)return a.textContent;for(a=a.firstChild;a;a=a.nextSibling)c+=e(a)}else if(3===f||4===f)return a.nodeValue}else while(b=a[d++])c+=e(b);return c},d=db.selectors={cacheLength:50,createPseudo:fb,match:V,attrHandle:{},find:{},relative:{">":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(a){return a[1]=a[1].replace(ab,bb),a[3]=(a[4]||a[5]||"").replace(ab,bb),"~="===a[2]&&(a[3]=" "+a[3]+" "),a.slice(0,4)},CHILD:function(a){return a[1]=a[1].toLowerCase(),"nth"===a[1].slice(0,3)?(a[3]||db.error(a[0]),a[4]=+(a[4]?a[5]+(a[6]||1):2*("even"===a[3]||"odd"===a[3])),a[5]=+(a[7]+a[8]||"odd"===a[3])):a[3]&&db.error(a[0]),a},PSEUDO:function(a){var b,c=!a[5]&&a[2];return V.CHILD.test(a[0])?null:(a[3]&&void 0!==a[4]?a[2]=a[4]:c&&T.test(c)&&(b=ob(c,!0))&&(b=c.indexOf(")",c.length-b)-c.length)&&(a[0]=a[0].slice(0,b),a[2]=c.slice(0,b)),a.slice(0,3))}},filter:{TAG:function(a){var b=a.replace(ab,bb).toLowerCase();return"*"===a?function(){return!0}:function(a){return a.nodeName&&a.nodeName.toLowerCase()===b}},CLASS:function(a){var b=w[a+" "];return b||(b=new RegExp("(^|"+K+")"+a+"("+K+"|$)"))&&w(a,function(a){return b.test("string"==typeof a.className&&a.className||typeof a.getAttribute!==A&&a.getAttribute("class")||"")})},ATTR:function(a,b,c){return function(d){var e=db.attr(d,a);return null==e?"!="===b:b?(e+="","="===b?e===c:"!="===b?e!==c:"^="===b?c&&0===e.indexOf(c):"*="===b?c&&e.indexOf(c)>-1:"$="===b?c&&e.slice(-c.length)===c:"~="===b?(" "+e+" ").indexOf(c)>-1:"|="===b?e===c||e.slice(0,c.length+1)===c+"-":!1):!0}},CHILD:function(a,b,c,d,e){var f="nth"!==a.slice(0,3),g="last"!==a.slice(-4),h="of-type"===b;return 1===d&&0===e?function(a){return!!a.parentNode}:function(b,c,i){var j,k,l,m,n,o,p=f!==g?"nextSibling":"previousSibling",q=b.parentNode,r=h&&b.nodeName.toLowerCase(),t=!i&&!h;if(q){if(f){while(p){l=b;while(l=l[p])if(h?l.nodeName.toLowerCase()===r:1===l.nodeType)return!1;o=p="only"===a&&!o&&"nextSibling"}return!0}if(o=[g?q.firstChild:q.lastChild],g&&t){k=q[s]||(q[s]={}),j=k[a]||[],n=j[0]===u&&j[1],m=j[0]===u&&j[2],l=n&&q.childNodes[n];while(l=++n&&l&&l[p]||(m=n=0)||o.pop())if(1===l.nodeType&&++m&&l===b){k[a]=[u,n,m];break}}else if(t&&(j=(b[s]||(b[s]={}))[a])&&j[0]===u)m=j[1];else while(l=++n&&l&&l[p]||(m=n=0)||o.pop())if((h?l.nodeName.toLowerCase()===r:1===l.nodeType)&&++m&&(t&&((l[s]||(l[s]={}))[a]=[u,m]),l===b))break;return m-=e,m===d||m%d===0&&m/d>=0}}},PSEUDO:function(a,b){var c,e=d.pseudos[a]||d.setFilters[a.toLowerCase()]||db.error("unsupported pseudo: "+a);return e[s]?e(b):e.length>1?(c=[a,a,"",b],d.setFilters.hasOwnProperty(a.toLowerCase())?fb(function(a,c){var d,f=e(a,b),g=f.length;while(g--)d=I.call(a,f[g]),a[d]=!(c[d]=f[g])}):function(a){return e(a,0,c)}):e}},pseudos:{not:fb(function(a){var b=[],c=[],d=g(a.replace(P,"$1"));return d[s]?fb(function(a,b,c,e){var f,g=d(a,null,e,[]),h=a.length;while(h--)(f=g[h])&&(a[h]=!(b[h]=f))}):function(a,e,f){return b[0]=a,d(b,null,f,c),!c.pop()}}),has:fb(function(a){return function(b){return db(a,b).length>0}}),contains:fb(function(a){return function(b){return(b.textContent||b.innerText||e(b)).indexOf(a)>-1}}),lang:fb(function(a){return U.test(a||"")||db.error("unsupported lang: "+a),a=a.replace(ab,bb).toLowerCase(),function(b){var c;do if(c=n?b.lang:b.getAttribute("xml:lang")||b.getAttribute("lang"))return c=c.toLowerCase(),c===a||0===c.indexOf(a+"-");while((b=b.parentNode)&&1===b.nodeType);return!1}}),target:function(b){var c=a.location&&a.location.hash;return c&&c.slice(1)===b.id},root:function(a){return a===m},focus:function(a){return a===l.activeElement&&(!l.hasFocus||l.hasFocus())&&!!(a.type||a.href||~a.tabIndex)},enabled:function(a){return a.disabled===!1},disabled:function(a){return a.disabled===!0},checked:function(a){var b=a.nodeName.toLowerCase();return"input"===b&&!!a.checked||"option"===b&&!!a.selected},selected:function(a){return a.parentNode&&a.parentNode.selectedIndex,a.selected===!0},empty:function(a){for(a=a.firstChild;a;a=a.nextSibling)if(a.nodeType<6)return!1;return!0},parent:function(a){return!d.pseudos.empty(a)},header:function(a){return X.test(a.nodeName)},input:function(a){return W.test(a.nodeName)},button:function(a){var b=a.nodeName.toLowerCase();return"input"===b&&"button"===a.type||"button"===b},text:function(a){var b;return"input"===a.nodeName.toLowerCase()&&"text"===a.type&&(null==(b=a.getAttribute("type"))||"text"===b.toLowerCase())},first:lb(function(){return[0]}),last:lb(function(a,b){return[b-1]}),eq:lb(function(a,b,c){return[0>c?c+b:c]}),even:lb(function(a,b){for(var c=0;b>c;c+=2)a.push(c);return a}),odd:lb(function(a,b){for(var c=1;b>c;c+=2)a.push(c);return a}),lt:lb(function(a,b,c){for(var d=0>c?c+b:c;--d>=0;)a.push(d);return a}),gt:lb(function(a,b,c){for(var d=0>c?c+b:c;++db;b++)d+=a[b].value;return d}function qb(a,b,c){var d=b.dir,e=c&&"parentNode"===d,f=v++;return b.first?function(b,c,f){while(b=b[d])if(1===b.nodeType||e)return a(b,c,f)}:function(b,c,g){var h,i,j=[u,f];if(g){while(b=b[d])if((1===b.nodeType||e)&&a(b,c,g))return!0}else while(b=b[d])if(1===b.nodeType||e){if(i=b[s]||(b[s]={}),(h=i[d])&&h[0]===u&&h[1]===f)return j[2]=h[2];if(i[d]=j,j[2]=a(b,c,g))return!0}}}function rb(a){return a.length>1?function(b,c,d){var e=a.length;while(e--)if(!a[e](b,c,d))return!1;return!0}:a[0]}function sb(a,b,c,d,e){for(var f,g=[],h=0,i=a.length,j=null!=b;i>h;h++)(f=a[h])&&(!c||c(f,d,e))&&(g.push(f),j&&b.push(h));return g}function tb(a,b,c,d,e,f){return d&&!d[s]&&(d=tb(d)),e&&!e[s]&&(e=tb(e,f)),fb(function(f,g,h,i){var j,k,l,m=[],n=[],o=g.length,p=f||wb(b||"*",h.nodeType?[h]:h,[]),q=!a||!f&&b?p:sb(p,m,a,h,i),r=c?e||(f?a:o||d)?[]:g:q;if(c&&c(q,r,h,i),d){j=sb(r,n),d(j,[],h,i),k=j.length;while(k--)(l=j[k])&&(r[n[k]]=!(q[n[k]]=l))}if(f){if(e||a){if(e){j=[],k=r.length;while(k--)(l=r[k])&&j.push(q[k]=l);e(null,r=[],j,i)}k=r.length;while(k--)(l=r[k])&&(j=e?I.call(f,l):m[k])>-1&&(f[j]=!(g[j]=l))}}else r=sb(r===g?r.splice(o,r.length):r),e?e(null,g,r,i):G.apply(g,r)})}function ub(a){for(var b,c,e,f=a.length,g=d.relative[a[0].type],i=g||d.relative[" "],j=g?1:0,k=qb(function(a){return a===b},i,!0),l=qb(function(a){return I.call(b,a)>-1},i,!0),m=[function(a,c,d){return!g&&(d||c!==h)||((b=c).nodeType?k(a,c,d):l(a,c,d))}];f>j;j++)if(c=d.relative[a[j].type])m=[qb(rb(m),c)];else{if(c=d.filter[a[j].type].apply(null,a[j].matches),c[s]){for(e=++j;f>e;e++)if(d.relative[a[e].type])break;return tb(j>1&&rb(m),j>1&&pb(a.slice(0,j-1).concat({value:" "===a[j-2].type?"*":""})).replace(P,"$1"),c,e>j&&ub(a.slice(j,e)),f>e&&ub(a=a.slice(e)),f>e&&pb(a))}m.push(c)}return rb(m)}function vb(a,b){var c=b.length>0,e=a.length>0,f=function(f,g,i,j,k){var m,n,o,p=0,q="0",r=f&&[],s=[],t=h,v=f||e&&d.find.TAG("*",k),w=u+=null==t?1:Math.random()||.1,x=v.length;for(k&&(h=g!==l&&g);q!==x&&null!=(m=v[q]);q++){if(e&&m){n=0;while(o=a[n++])if(o(m,g,i)){j.push(m);break}k&&(u=w)}c&&((m=!o&&m)&&p--,f&&r.push(m))}if(p+=q,c&&q!==p){n=0;while(o=b[n++])o(r,s,g,i);if(f){if(p>0)while(q--)r[q]||s[q]||(s[q]=E.call(j));s=sb(s)}G.apply(j,s),k&&!f&&s.length>0&&p+b.length>1&&db.uniqueSort(j)}return k&&(u=w,h=t),r};return c?fb(f):f}g=db.compile=function(a,b){var c,d=[],e=[],f=y[a+" "];if(!f){b||(b=ob(a)),c=b.length;while(c--)f=ub(b[c]),f[s]?d.push(f):e.push(f);f=y(a,vb(e,d))}return f};function wb(a,b,c){for(var d=0,e=b.length;e>d;d++)db(a,b[d],c);return c}function xb(a,b,e,f){var h,i,j,k,l,m=ob(a);if(!f&&1===m.length){if(i=m[0]=m[0].slice(0),i.length>2&&"ID"===(j=i[0]).type&&c.getById&&9===b.nodeType&&n&&d.relative[i[1].type]){if(b=(d.find.ID(j.matches[0].replace(ab,bb),b)||[])[0],!b)return e;a=a.slice(i.shift().value.length)}h=V.needsContext.test(a)?0:i.length;while(h--){if(j=i[h],d.relative[k=j.type])break;if((l=d.find[k])&&(f=l(j.matches[0].replace(ab,bb),$.test(i[0].type)&&mb(b.parentNode)||b))){if(i.splice(h,1),a=f.length&&pb(i),!a)return G.apply(e,f),e;break}}}return g(a,m)(f,b,!n,e,$.test(a)&&mb(b.parentNode)||b),e}return c.sortStable=s.split("").sort(z).join("")===s,c.detectDuplicates=!!j,k(),c.sortDetached=gb(function(a){return 1&a.compareDocumentPosition(l.createElement("div"))}),gb(function(a){return a.innerHTML="","#"===a.firstChild.getAttribute("href")})||hb("type|href|height|width",function(a,b,c){return c?void 0:a.getAttribute(b,"type"===b.toLowerCase()?1:2)}),c.attributes&&gb(function(a){return a.innerHTML="",a.firstChild.setAttribute("value",""),""===a.firstChild.getAttribute("value")})||hb("value",function(a,b,c){return c||"input"!==a.nodeName.toLowerCase()?void 0:a.defaultValue}),gb(function(a){return null==a.getAttribute("disabled")})||hb(J,function(a,b,c){var d;return c?void 0:a[b]===!0?b.toLowerCase():(d=a.getAttributeNode(b))&&d.specified?d.value:null}),db}(a);n.find=t,n.expr=t.selectors,n.expr[":"]=n.expr.pseudos,n.unique=t.uniqueSort,n.text=t.getText,n.isXMLDoc=t.isXML,n.contains=t.contains;var u=n.expr.match.needsContext,v=/^<(\w+)\s*\/?>(?:<\/\1>|)$/,w=/^.[^:#\[\.,]*$/;function x(a,b,c){if(n.isFunction(b))return n.grep(a,function(a,d){return!!b.call(a,d,a)!==c});if(b.nodeType)return n.grep(a,function(a){return a===b!==c});if("string"==typeof b){if(w.test(b))return n.filter(b,a,c);b=n.filter(b,a)}return n.grep(a,function(a){return n.inArray(a,b)>=0!==c})}n.filter=function(a,b,c){var d=b[0];return c&&(a=":not("+a+")"),1===b.length&&1===d.nodeType?n.find.matchesSelector(d,a)?[d]:[]:n.find.matches(a,n.grep(b,function(a){return 1===a.nodeType}))},n.fn.extend({find:function(a){var b,c=[],d=this,e=d.length;if("string"!=typeof a)return this.pushStack(n(a).filter(function(){for(b=0;e>b;b++)if(n.contains(d[b],this))return!0}));for(b=0;e>b;b++)n.find(a,d[b],c);return c=this.pushStack(e>1?n.unique(c):c),c.selector=this.selector?this.selector+" "+a:a,c},filter:function(a){return this.pushStack(x(this,a||[],!1))},not:function(a){return this.pushStack(x(this,a||[],!0))},is:function(a){return!!x(this,"string"==typeof a&&u.test(a)?n(a):a||[],!1).length}});var y,z=a.document,A=/^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]*))$/,B=n.fn.init=function(a,b){var c,d;if(!a)return this;if("string"==typeof a){if(c="<"===a.charAt(0)&&">"===a.charAt(a.length-1)&&a.length>=3?[null,a,null]:A.exec(a),!c||!c[1]&&b)return!b||b.jquery?(b||y).find(a):this.constructor(b).find(a);if(c[1]){if(b=b instanceof n?b[0]:b,n.merge(this,n.parseHTML(c[1],b&&b.nodeType?b.ownerDocument||b:z,!0)),v.test(c[1])&&n.isPlainObject(b))for(c in b)n.isFunction(this[c])?this[c](b[c]):this.attr(c,b[c]);return this}if(d=z.getElementById(c[2]),d&&d.parentNode){if(d.id!==c[2])return y.find(a);this.length=1,this[0]=d}return this.context=z,this.selector=a,this}return a.nodeType?(this.context=this[0]=a,this.length=1,this):n.isFunction(a)?"undefined"!=typeof y.ready?y.ready(a):a(n):(void 0!==a.selector&&(this.selector=a.selector,this.context=a.context),n.makeArray(a,this))};B.prototype=n.fn,y=n(z);var C=/^(?:parents|prev(?:Until|All))/,D={children:!0,contents:!0,next:!0,prev:!0};n.extend({dir:function(a,b,c){var d=[],e=a[b];while(e&&9!==e.nodeType&&(void 0===c||1!==e.nodeType||!n(e).is(c)))1===e.nodeType&&d.push(e),e=e[b];return d},sibling:function(a,b){for(var c=[];a;a=a.nextSibling)1===a.nodeType&&a!==b&&c.push(a);return c}}),n.fn.extend({has:function(a){var b,c=n(a,this),d=c.length;return this.filter(function(){for(b=0;d>b;b++)if(n.contains(this,c[b]))return!0})},closest:function(a,b){for(var c,d=0,e=this.length,f=[],g=u.test(a)||"string"!=typeof a?n(a,b||this.context):0;e>d;d++)for(c=this[d];c&&c!==b;c=c.parentNode)if(c.nodeType<11&&(g?g.index(c)>-1:1===c.nodeType&&n.find.matchesSelector(c,a))){f.push(c);break}return this.pushStack(f.length>1?n.unique(f):f)},index:function(a){return a?"string"==typeof a?n.inArray(this[0],n(a)):n.inArray(a.jquery?a[0]:a,this):this[0]&&this[0].parentNode?this.first().prevAll().length:-1},add:function(a,b){return this.pushStack(n.unique(n.merge(this.get(),n(a,b))))},addBack:function(a){return this.add(null==a?this.prevObject:this.prevObject.filter(a))}});function E(a,b){do a=a[b];while(a&&1!==a.nodeType);return a}n.each({parent:function(a){var b=a.parentNode;return b&&11!==b.nodeType?b:null},parents:function(a){return n.dir(a,"parentNode")},parentsUntil:function(a,b,c){return n.dir(a,"parentNode",c)},next:function(a){return E(a,"nextSibling")},prev:function(a){return E(a,"previousSibling")},nextAll:function(a){return n.dir(a,"nextSibling")},prevAll:function(a){return n.dir(a,"previousSibling")},nextUntil:function(a,b,c){return n.dir(a,"nextSibling",c)},prevUntil:function(a,b,c){return n.dir(a,"previousSibling",c)},siblings:function(a){return n.sibling((a.parentNode||{}).firstChild,a)},children:function(a){return n.sibling(a.firstChild)},contents:function(a){return n.nodeName(a,"iframe")?a.contentDocument||a.contentWindow.document:n.merge([],a.childNodes)}},function(a,b){n.fn[a]=function(c,d){var e=n.map(this,b,c);return"Until"!==a.slice(-5)&&(d=c),d&&"string"==typeof d&&(e=n.filter(d,e)),this.length>1&&(D[a]||(e=n.unique(e)),C.test(a)&&(e=e.reverse())),this.pushStack(e)}});var F=/\S+/g,G={};function H(a){var b=G[a]={};return n.each(a.match(F)||[],function(a,c){b[c]=!0}),b}n.Callbacks=function(a){a="string"==typeof a?G[a]||H(a):n.extend({},a);var b,c,d,e,f,g,h=[],i=!a.once&&[],j=function(l){for(c=a.memory&&l,d=!0,f=g||0,g=0,e=h.length,b=!0;h&&e>f;f++)if(h[f].apply(l[0],l[1])===!1&&a.stopOnFalse){c=!1;break}b=!1,h&&(i?i.length&&j(i.shift()):c?h=[]:k.disable())},k={add:function(){if(h){var d=h.length;!function f(b){n.each(b,function(b,c){var d=n.type(c);"function"===d?a.unique&&k.has(c)||h.push(c):c&&c.length&&"string"!==d&&f(c)})}(arguments),b?e=h.length:c&&(g=d,j(c))}return this},remove:function(){return h&&n.each(arguments,function(a,c){var d;while((d=n.inArray(c,h,d))>-1)h.splice(d,1),b&&(e>=d&&e--,f>=d&&f--)}),this},has:function(a){return a?n.inArray(a,h)>-1:!(!h||!h.length)},empty:function(){return h=[],e=0,this},disable:function(){return h=i=c=void 0,this},disabled:function(){return!h},lock:function(){return i=void 0,c||k.disable(),this},locked:function(){return!i},fireWith:function(a,c){return!h||d&&!i||(c=c||[],c=[a,c.slice?c.slice():c],b?i.push(c):j(c)),this},fire:function(){return k.fireWith(this,arguments),this},fired:function(){return!!d}};return k},n.extend({Deferred:function(a){var b=[["resolve","done",n.Callbacks("once memory"),"resolved"],["reject","fail",n.Callbacks("once memory"),"rejected"],["notify","progress",n.Callbacks("memory")]],c="pending",d={state:function(){return c},always:function(){return e.done(arguments).fail(arguments),this},then:function(){var a=arguments;return n.Deferred(function(c){n.each(b,function(b,f){var g=n.isFunction(a[b])&&a[b];e[f[1]](function(){var a=g&&g.apply(this,arguments);a&&n.isFunction(a.promise)?a.promise().done(c.resolve).fail(c.reject).progress(c.notify):c[f[0]+"With"](this===d?c.promise():this,g?[a]:arguments)})}),a=null}).promise()},promise:function(a){return null!=a?n.extend(a,d):d}},e={};return d.pipe=d.then,n.each(b,function(a,f){var g=f[2],h=f[3];d[f[1]]=g.add,h&&g.add(function(){c=h},b[1^a][2].disable,b[2][2].lock),e[f[0]]=function(){return e[f[0]+"With"](this===e?d:this,arguments),this},e[f[0]+"With"]=g.fireWith}),d.promise(e),a&&a.call(e,e),e},when:function(a){var b=0,c=d.call(arguments),e=c.length,f=1!==e||a&&n.isFunction(a.promise)?e:0,g=1===f?a:n.Deferred(),h=function(a,b,c){return function(e){b[a]=this,c[a]=arguments.length>1?d.call(arguments):e,c===i?g.notifyWith(b,c):--f||g.resolveWith(b,c)}},i,j,k;if(e>1)for(i=new Array(e),j=new Array(e),k=new Array(e);e>b;b++)c[b]&&n.isFunction(c[b].promise)?c[b].promise().done(h(b,k,c)).fail(g.reject).progress(h(b,j,i)):--f;return f||g.resolveWith(k,c),g.promise()}});var I;n.fn.ready=function(a){return n.ready.promise().done(a),this},n.extend({isReady:!1,readyWait:1,holdReady:function(a){a?n.readyWait++:n.ready(!0)},ready:function(a){if(a===!0?!--n.readyWait:!n.isReady){if(!z.body)return setTimeout(n.ready);n.isReady=!0,a!==!0&&--n.readyWait>0||(I.resolveWith(z,[n]),n.fn.trigger&&n(z).trigger("ready").off("ready"))}}});function J(){z.addEventListener?(z.removeEventListener("DOMContentLoaded",K,!1),a.removeEventListener("load",K,!1)):(z.detachEvent("onreadystatechange",K),a.detachEvent("onload",K))}function K(){(z.addEventListener||"load"===event.type||"complete"===z.readyState)&&(J(),n.ready())}n.ready.promise=function(b){if(!I)if(I=n.Deferred(),"complete"===z.readyState)setTimeout(n.ready);else if(z.addEventListener)z.addEventListener("DOMContentLoaded",K,!1),a.addEventListener("load",K,!1);else{z.attachEvent("onreadystatechange",K),a.attachEvent("onload",K);var c=!1;try{c=null==a.frameElement&&z.documentElement}catch(d){}c&&c.doScroll&&!function e(){if(!n.isReady){try{c.doScroll("left")}catch(a){return setTimeout(e,50)}J(),n.ready()}}()}return I.promise(b)};var L="undefined",M;for(M in n(l))break;l.ownLast="0"!==M,l.inlineBlockNeedsLayout=!1,n(function(){var a,b,c=z.getElementsByTagName("body")[0];c&&(a=z.createElement("div"),a.style.cssText="border:0;width:0;height:0;position:absolute;top:0;left:-9999px;margin-top:1px",b=z.createElement("div"),c.appendChild(a).appendChild(b),typeof b.style.zoom!==L&&(b.style.cssText="border:0;margin:0;width:1px;padding:1px;display:inline;zoom:1",(l.inlineBlockNeedsLayout=3===b.offsetWidth)&&(c.style.zoom=1)),c.removeChild(a),a=b=null)}),function(){var a=z.createElement("div");if(null==l.deleteExpando){l.deleteExpando=!0;try{delete a.test}catch(b){l.deleteExpando=!1}}a=null}(),n.acceptData=function(a){var b=n.noData[(a.nodeName+" ").toLowerCase()],c=+a.nodeType||1;return 1!==c&&9!==c?!1:!b||b!==!0&&a.getAttribute("classid")===b};var N=/^(?:\{[\w\W]*\}|\[[\w\W]*\])$/,O=/([A-Z])/g;function P(a,b,c){if(void 0===c&&1===a.nodeType){var d="data-"+b.replace(O,"-$1").toLowerCase();if(c=a.getAttribute(d),"string"==typeof c){try{c="true"===c?!0:"false"===c?!1:"null"===c?null:+c+""===c?+c:N.test(c)?n.parseJSON(c):c}catch(e){}n.data(a,b,c)}else c=void 0}return c}function Q(a){var b;for(b in a)if(("data"!==b||!n.isEmptyObject(a[b]))&&"toJSON"!==b)return!1;return!0}function R(a,b,d,e){if(n.acceptData(a)){var f,g,h=n.expando,i=a.nodeType,j=i?n.cache:a,k=i?a[h]:a[h]&&h;if(k&&j[k]&&(e||j[k].data)||void 0!==d||"string"!=typeof b)return k||(k=i?a[h]=c.pop()||n.guid++:h),j[k]||(j[k]=i?{}:{toJSON:n.noop}),("object"==typeof b||"function"==typeof b)&&(e?j[k]=n.extend(j[k],b):j[k].data=n.extend(j[k].data,b)),g=j[k],e||(g.data||(g.data={}),g=g.data),void 0!==d&&(g[n.camelCase(b)]=d),"string"==typeof b?(f=g[b],null==f&&(f=g[n.camelCase(b)])):f=g,f +}}function S(a,b,c){if(n.acceptData(a)){var d,e,f=a.nodeType,g=f?n.cache:a,h=f?a[n.expando]:n.expando;if(g[h]){if(b&&(d=c?g[h]:g[h].data)){n.isArray(b)?b=b.concat(n.map(b,n.camelCase)):b in d?b=[b]:(b=n.camelCase(b),b=b in d?[b]:b.split(" ")),e=b.length;while(e--)delete d[b[e]];if(c?!Q(d):!n.isEmptyObject(d))return}(c||(delete g[h].data,Q(g[h])))&&(f?n.cleanData([a],!0):l.deleteExpando||g!=g.window?delete g[h]:g[h]=null)}}}n.extend({cache:{},noData:{"applet ":!0,"embed ":!0,"object ":"clsid:D27CDB6E-AE6D-11cf-96B8-444553540000"},hasData:function(a){return a=a.nodeType?n.cache[a[n.expando]]:a[n.expando],!!a&&!Q(a)},data:function(a,b,c){return R(a,b,c)},removeData:function(a,b){return S(a,b)},_data:function(a,b,c){return R(a,b,c,!0)},_removeData:function(a,b){return S(a,b,!0)}}),n.fn.extend({data:function(a,b){var c,d,e,f=this[0],g=f&&f.attributes;if(void 0===a){if(this.length&&(e=n.data(f),1===f.nodeType&&!n._data(f,"parsedAttrs"))){c=g.length;while(c--)d=g[c].name,0===d.indexOf("data-")&&(d=n.camelCase(d.slice(5)),P(f,d,e[d]));n._data(f,"parsedAttrs",!0)}return e}return"object"==typeof a?this.each(function(){n.data(this,a)}):arguments.length>1?this.each(function(){n.data(this,a,b)}):f?P(f,a,n.data(f,a)):void 0},removeData:function(a){return this.each(function(){n.removeData(this,a)})}}),n.extend({queue:function(a,b,c){var d;return a?(b=(b||"fx")+"queue",d=n._data(a,b),c&&(!d||n.isArray(c)?d=n._data(a,b,n.makeArray(c)):d.push(c)),d||[]):void 0},dequeue:function(a,b){b=b||"fx";var c=n.queue(a,b),d=c.length,e=c.shift(),f=n._queueHooks(a,b),g=function(){n.dequeue(a,b)};"inprogress"===e&&(e=c.shift(),d--),e&&("fx"===b&&c.unshift("inprogress"),delete f.stop,e.call(a,g,f)),!d&&f&&f.empty.fire()},_queueHooks:function(a,b){var c=b+"queueHooks";return n._data(a,c)||n._data(a,c,{empty:n.Callbacks("once memory").add(function(){n._removeData(a,b+"queue"),n._removeData(a,c)})})}}),n.fn.extend({queue:function(a,b){var c=2;return"string"!=typeof a&&(b=a,a="fx",c--),arguments.lengthh;h++)b(a[h],c,g?d:d.call(a[h],h,b(a[h],c)));return e?a:j?b.call(a):i?b(a[0],c):f},X=/^(?:checkbox|radio)$/i;!function(){var a=z.createDocumentFragment(),b=z.createElement("div"),c=z.createElement("input");if(b.setAttribute("className","t"),b.innerHTML="
a",l.leadingWhitespace=3===b.firstChild.nodeType,l.tbody=!b.getElementsByTagName("tbody").length,l.htmlSerialize=!!b.getElementsByTagName("link").length,l.html5Clone="<:nav>"!==z.createElement("nav").cloneNode(!0).outerHTML,c.type="checkbox",c.checked=!0,a.appendChild(c),l.appendChecked=c.checked,b.innerHTML="",l.noCloneChecked=!!b.cloneNode(!0).lastChild.defaultValue,a.appendChild(b),b.innerHTML="",l.checkClone=b.cloneNode(!0).cloneNode(!0).lastChild.checked,l.noCloneEvent=!0,b.attachEvent&&(b.attachEvent("onclick",function(){l.noCloneEvent=!1}),b.cloneNode(!0).click()),null==l.deleteExpando){l.deleteExpando=!0;try{delete b.test}catch(d){l.deleteExpando=!1}}a=b=c=null}(),function(){var b,c,d=z.createElement("div");for(b in{submit:!0,change:!0,focusin:!0})c="on"+b,(l[b+"Bubbles"]=c in a)||(d.setAttribute(c,"t"),l[b+"Bubbles"]=d.attributes[c].expando===!1);d=null}();var Y=/^(?:input|select|textarea)$/i,Z=/^key/,$=/^(?:mouse|contextmenu)|click/,_=/^(?:focusinfocus|focusoutblur)$/,ab=/^([^.]*)(?:\.(.+)|)$/;function bb(){return!0}function cb(){return!1}function db(){try{return z.activeElement}catch(a){}}n.event={global:{},add:function(a,b,c,d,e){var f,g,h,i,j,k,l,m,o,p,q,r=n._data(a);if(r){c.handler&&(i=c,c=i.handler,e=i.selector),c.guid||(c.guid=n.guid++),(g=r.events)||(g=r.events={}),(k=r.handle)||(k=r.handle=function(a){return typeof n===L||a&&n.event.triggered===a.type?void 0:n.event.dispatch.apply(k.elem,arguments)},k.elem=a),b=(b||"").match(F)||[""],h=b.length;while(h--)f=ab.exec(b[h])||[],o=q=f[1],p=(f[2]||"").split(".").sort(),o&&(j=n.event.special[o]||{},o=(e?j.delegateType:j.bindType)||o,j=n.event.special[o]||{},l=n.extend({type:o,origType:q,data:d,handler:c,guid:c.guid,selector:e,needsContext:e&&n.expr.match.needsContext.test(e),namespace:p.join(".")},i),(m=g[o])||(m=g[o]=[],m.delegateCount=0,j.setup&&j.setup.call(a,d,p,k)!==!1||(a.addEventListener?a.addEventListener(o,k,!1):a.attachEvent&&a.attachEvent("on"+o,k))),j.add&&(j.add.call(a,l),l.handler.guid||(l.handler.guid=c.guid)),e?m.splice(m.delegateCount++,0,l):m.push(l),n.event.global[o]=!0);a=null}},remove:function(a,b,c,d,e){var f,g,h,i,j,k,l,m,o,p,q,r=n.hasData(a)&&n._data(a);if(r&&(k=r.events)){b=(b||"").match(F)||[""],j=b.length;while(j--)if(h=ab.exec(b[j])||[],o=q=h[1],p=(h[2]||"").split(".").sort(),o){l=n.event.special[o]||{},o=(d?l.delegateType:l.bindType)||o,m=k[o]||[],h=h[2]&&new RegExp("(^|\\.)"+p.join("\\.(?:.*\\.|)")+"(\\.|$)"),i=f=m.length;while(f--)g=m[f],!e&&q!==g.origType||c&&c.guid!==g.guid||h&&!h.test(g.namespace)||d&&d!==g.selector&&("**"!==d||!g.selector)||(m.splice(f,1),g.selector&&m.delegateCount--,l.remove&&l.remove.call(a,g));i&&!m.length&&(l.teardown&&l.teardown.call(a,p,r.handle)!==!1||n.removeEvent(a,o,r.handle),delete k[o])}else for(o in k)n.event.remove(a,o+b[j],c,d,!0);n.isEmptyObject(k)&&(delete r.handle,n._removeData(a,"events"))}},trigger:function(b,c,d,e){var f,g,h,i,k,l,m,o=[d||z],p=j.call(b,"type")?b.type:b,q=j.call(b,"namespace")?b.namespace.split("."):[];if(h=l=d=d||z,3!==d.nodeType&&8!==d.nodeType&&!_.test(p+n.event.triggered)&&(p.indexOf(".")>=0&&(q=p.split("."),p=q.shift(),q.sort()),g=p.indexOf(":")<0&&"on"+p,b=b[n.expando]?b:new n.Event(p,"object"==typeof b&&b),b.isTrigger=e?2:3,b.namespace=q.join("."),b.namespace_re=b.namespace?new RegExp("(^|\\.)"+q.join("\\.(?:.*\\.|)")+"(\\.|$)"):null,b.result=void 0,b.target||(b.target=d),c=null==c?[b]:n.makeArray(c,[b]),k=n.event.special[p]||{},e||!k.trigger||k.trigger.apply(d,c)!==!1)){if(!e&&!k.noBubble&&!n.isWindow(d)){for(i=k.delegateType||p,_.test(i+p)||(h=h.parentNode);h;h=h.parentNode)o.push(h),l=h;l===(d.ownerDocument||z)&&o.push(l.defaultView||l.parentWindow||a)}m=0;while((h=o[m++])&&!b.isPropagationStopped())b.type=m>1?i:k.bindType||p,f=(n._data(h,"events")||{})[b.type]&&n._data(h,"handle"),f&&f.apply(h,c),f=g&&h[g],f&&f.apply&&n.acceptData(h)&&(b.result=f.apply(h,c),b.result===!1&&b.preventDefault());if(b.type=p,!e&&!b.isDefaultPrevented()&&(!k._default||k._default.apply(o.pop(),c)===!1)&&n.acceptData(d)&&g&&d[p]&&!n.isWindow(d)){l=d[g],l&&(d[g]=null),n.event.triggered=p;try{d[p]()}catch(r){}n.event.triggered=void 0,l&&(d[g]=l)}return b.result}},dispatch:function(a){a=n.event.fix(a);var b,c,e,f,g,h=[],i=d.call(arguments),j=(n._data(this,"events")||{})[a.type]||[],k=n.event.special[a.type]||{};if(i[0]=a,a.delegateTarget=this,!k.preDispatch||k.preDispatch.call(this,a)!==!1){h=n.event.handlers.call(this,a,j),b=0;while((f=h[b++])&&!a.isPropagationStopped()){a.currentTarget=f.elem,g=0;while((e=f.handlers[g++])&&!a.isImmediatePropagationStopped())(!a.namespace_re||a.namespace_re.test(e.namespace))&&(a.handleObj=e,a.data=e.data,c=((n.event.special[e.origType]||{}).handle||e.handler).apply(f.elem,i),void 0!==c&&(a.result=c)===!1&&(a.preventDefault(),a.stopPropagation()))}return k.postDispatch&&k.postDispatch.call(this,a),a.result}},handlers:function(a,b){var c,d,e,f,g=[],h=b.delegateCount,i=a.target;if(h&&i.nodeType&&(!a.button||"click"!==a.type))for(;i!=this;i=i.parentNode||this)if(1===i.nodeType&&(i.disabled!==!0||"click"!==a.type)){for(e=[],f=0;h>f;f++)d=b[f],c=d.selector+" ",void 0===e[c]&&(e[c]=d.needsContext?n(c,this).index(i)>=0:n.find(c,this,null,[i]).length),e[c]&&e.push(d);e.length&&g.push({elem:i,handlers:e})}return h]","i"),ib=/^\s+/,jb=/<(?!area|br|col|embed|hr|img|input|link|meta|param)(([\w:]+)[^>]*)\/>/gi,kb=/<([\w:]+)/,lb=/\s*$/g,sb={option:[1,""],legend:[1,"
","
"],area:[1,"",""],param:[1,"",""],thead:[1,"","
"],tr:[2,"","
"],col:[2,"","
"],td:[3,"","
"],_default:l.htmlSerialize?[0,"",""]:[1,"X
","
"]},tb=eb(z),ub=tb.appendChild(z.createElement("div"));sb.optgroup=sb.option,sb.tbody=sb.tfoot=sb.colgroup=sb.caption=sb.thead,sb.th=sb.td;function vb(a,b){var c,d,e=0,f=typeof a.getElementsByTagName!==L?a.getElementsByTagName(b||"*"):typeof a.querySelectorAll!==L?a.querySelectorAll(b||"*"):void 0;if(!f)for(f=[],c=a.childNodes||a;null!=(d=c[e]);e++)!b||n.nodeName(d,b)?f.push(d):n.merge(f,vb(d,b));return void 0===b||b&&n.nodeName(a,b)?n.merge([a],f):f}function wb(a){X.test(a.type)&&(a.defaultChecked=a.checked)}function xb(a,b){return n.nodeName(a,"table")&&n.nodeName(11!==b.nodeType?b:b.firstChild,"tr")?a.getElementsByTagName("tbody")[0]||a.appendChild(a.ownerDocument.createElement("tbody")):a}function yb(a){return a.type=(null!==n.find.attr(a,"type"))+"/"+a.type,a}function zb(a){var b=qb.exec(a.type);return b?a.type=b[1]:a.removeAttribute("type"),a}function Ab(a,b){for(var c,d=0;null!=(c=a[d]);d++)n._data(c,"globalEval",!b||n._data(b[d],"globalEval"))}function Bb(a,b){if(1===b.nodeType&&n.hasData(a)){var c,d,e,f=n._data(a),g=n._data(b,f),h=f.events;if(h){delete g.handle,g.events={};for(c in h)for(d=0,e=h[c].length;e>d;d++)n.event.add(b,c,h[c][d])}g.data&&(g.data=n.extend({},g.data))}}function Cb(a,b){var c,d,e;if(1===b.nodeType){if(c=b.nodeName.toLowerCase(),!l.noCloneEvent&&b[n.expando]){e=n._data(b);for(d in e.events)n.removeEvent(b,d,e.handle);b.removeAttribute(n.expando)}"script"===c&&b.text!==a.text?(yb(b).text=a.text,zb(b)):"object"===c?(b.parentNode&&(b.outerHTML=a.outerHTML),l.html5Clone&&a.innerHTML&&!n.trim(b.innerHTML)&&(b.innerHTML=a.innerHTML)):"input"===c&&X.test(a.type)?(b.defaultChecked=b.checked=a.checked,b.value!==a.value&&(b.value=a.value)):"option"===c?b.defaultSelected=b.selected=a.defaultSelected:("input"===c||"textarea"===c)&&(b.defaultValue=a.defaultValue)}}n.extend({clone:function(a,b,c){var d,e,f,g,h,i=n.contains(a.ownerDocument,a);if(l.html5Clone||n.isXMLDoc(a)||!hb.test("<"+a.nodeName+">")?f=a.cloneNode(!0):(ub.innerHTML=a.outerHTML,ub.removeChild(f=ub.firstChild)),!(l.noCloneEvent&&l.noCloneChecked||1!==a.nodeType&&11!==a.nodeType||n.isXMLDoc(a)))for(d=vb(f),h=vb(a),g=0;null!=(e=h[g]);++g)d[g]&&Cb(e,d[g]);if(b)if(c)for(h=h||vb(a),d=d||vb(f),g=0;null!=(e=h[g]);g++)Bb(e,d[g]);else Bb(a,f);return d=vb(f,"script"),d.length>0&&Ab(d,!i&&vb(a,"script")),d=h=e=null,f},buildFragment:function(a,b,c,d){for(var e,f,g,h,i,j,k,m=a.length,o=eb(b),p=[],q=0;m>q;q++)if(f=a[q],f||0===f)if("object"===n.type(f))n.merge(p,f.nodeType?[f]:f);else if(mb.test(f)){h=h||o.appendChild(b.createElement("div")),i=(kb.exec(f)||["",""])[1].toLowerCase(),k=sb[i]||sb._default,h.innerHTML=k[1]+f.replace(jb,"<$1>")+k[2],e=k[0];while(e--)h=h.lastChild;if(!l.leadingWhitespace&&ib.test(f)&&p.push(b.createTextNode(ib.exec(f)[0])),!l.tbody){f="table"!==i||lb.test(f)?""!==k[1]||lb.test(f)?0:h:h.firstChild,e=f&&f.childNodes.length;while(e--)n.nodeName(j=f.childNodes[e],"tbody")&&!j.childNodes.length&&f.removeChild(j)}n.merge(p,h.childNodes),h.textContent="";while(h.firstChild)h.removeChild(h.firstChild);h=o.lastChild}else p.push(b.createTextNode(f));h&&o.removeChild(h),l.appendChecked||n.grep(vb(p,"input"),wb),q=0;while(f=p[q++])if((!d||-1===n.inArray(f,d))&&(g=n.contains(f.ownerDocument,f),h=vb(o.appendChild(f),"script"),g&&Ab(h),c)){e=0;while(f=h[e++])pb.test(f.type||"")&&c.push(f)}return h=null,o},cleanData:function(a,b){for(var d,e,f,g,h=0,i=n.expando,j=n.cache,k=l.deleteExpando,m=n.event.special;null!=(d=a[h]);h++)if((b||n.acceptData(d))&&(f=d[i],g=f&&j[f])){if(g.events)for(e in g.events)m[e]?n.event.remove(d,e):n.removeEvent(d,e,g.handle);j[f]&&(delete j[f],k?delete d[i]:typeof d.removeAttribute!==L?d.removeAttribute(i):d[i]=null,c.push(f))}}}),n.fn.extend({text:function(a){return W(this,function(a){return void 0===a?n.text(this):this.empty().append((this[0]&&this[0].ownerDocument||z).createTextNode(a))},null,a,arguments.length)},append:function(){return this.domManip(arguments,function(a){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var b=xb(this,a);b.appendChild(a)}})},prepend:function(){return this.domManip(arguments,function(a){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var b=xb(this,a);b.insertBefore(a,b.firstChild)}})},before:function(){return this.domManip(arguments,function(a){this.parentNode&&this.parentNode.insertBefore(a,this)})},after:function(){return this.domManip(arguments,function(a){this.parentNode&&this.parentNode.insertBefore(a,this.nextSibling)})},remove:function(a,b){for(var c,d=a?n.filter(a,this):this,e=0;null!=(c=d[e]);e++)b||1!==c.nodeType||n.cleanData(vb(c)),c.parentNode&&(b&&n.contains(c.ownerDocument,c)&&Ab(vb(c,"script")),c.parentNode.removeChild(c));return this},empty:function(){for(var a,b=0;null!=(a=this[b]);b++){1===a.nodeType&&n.cleanData(vb(a,!1));while(a.firstChild)a.removeChild(a.firstChild);a.options&&n.nodeName(a,"select")&&(a.options.length=0)}return this},clone:function(a,b){return a=null==a?!1:a,b=null==b?a:b,this.map(function(){return n.clone(this,a,b)})},html:function(a){return W(this,function(a){var b=this[0]||{},c=0,d=this.length;if(void 0===a)return 1===b.nodeType?b.innerHTML.replace(gb,""):void 0;if(!("string"!=typeof a||nb.test(a)||!l.htmlSerialize&&hb.test(a)||!l.leadingWhitespace&&ib.test(a)||sb[(kb.exec(a)||["",""])[1].toLowerCase()])){a=a.replace(jb,"<$1>");try{for(;d>c;c++)b=this[c]||{},1===b.nodeType&&(n.cleanData(vb(b,!1)),b.innerHTML=a);b=0}catch(e){}}b&&this.empty().append(a)},null,a,arguments.length)},replaceWith:function(){var a=arguments[0];return this.domManip(arguments,function(b){a=this.parentNode,n.cleanData(vb(this)),a&&a.replaceChild(b,this)}),a&&(a.length||a.nodeType)?this:this.remove()},detach:function(a){return this.remove(a,!0)},domManip:function(a,b){a=e.apply([],a);var c,d,f,g,h,i,j=0,k=this.length,m=this,o=k-1,p=a[0],q=n.isFunction(p);if(q||k>1&&"string"==typeof p&&!l.checkClone&&ob.test(p))return this.each(function(c){var d=m.eq(c);q&&(a[0]=p.call(this,c,d.html())),d.domManip(a,b)});if(k&&(i=n.buildFragment(a,this[0].ownerDocument,!1,this),c=i.firstChild,1===i.childNodes.length&&(i=c),c)){for(g=n.map(vb(i,"script"),yb),f=g.length;k>j;j++)d=i,j!==o&&(d=n.clone(d,!0,!0),f&&n.merge(g,vb(d,"script"))),b.call(this[j],d,j);if(f)for(h=g[g.length-1].ownerDocument,n.map(g,zb),j=0;f>j;j++)d=g[j],pb.test(d.type||"")&&!n._data(d,"globalEval")&&n.contains(h,d)&&(d.src?n._evalUrl&&n._evalUrl(d.src):n.globalEval((d.text||d.textContent||d.innerHTML||"").replace(rb,"")));i=c=null}return this}}),n.each({appendTo:"append",prependTo:"prepend",insertBefore:"before",insertAfter:"after",replaceAll:"replaceWith"},function(a,b){n.fn[a]=function(a){for(var c,d=0,e=[],g=n(a),h=g.length-1;h>=d;d++)c=d===h?this:this.clone(!0),n(g[d])[b](c),f.apply(e,c.get());return this.pushStack(e)}});var Db,Eb={};function Fb(b,c){var d=n(c.createElement(b)).appendTo(c.body),e=a.getDefaultComputedStyle?a.getDefaultComputedStyle(d[0]).display:n.css(d[0],"display");return d.detach(),e}function Gb(a){var b=z,c=Eb[a];return c||(c=Fb(a,b),"none"!==c&&c||(Db=(Db||n("