From ffaf9e00ed6945181418c6b5f0bbc262827a5ef6 Mon Sep 17 00:00:00 2001 From: Dmytro Katyukha Date: Sun, 10 Sep 2023 18:33:12 +0300 Subject: [PATCH] [REF] Dynamically load python, thus become more portable. --- dub.selections.json | 2 + .../cli/source/odood/cli/commands/addons.d | 30 ++- subpackages/lib/source/odood/lib/odoo/test.d | 4 +- subpackages/utils/dub.sdl | 2 + .../utils/source/odood/utils/addons/addon.d | 16 +- .../odood/utils/addons/addon_manifest.d | 221 ++++++++++++------ .../utils/source/odood/utils/tipy/package.d | 190 +++++++++++++++ .../utils/source/odood/utils/tipy/python.d | 82 +++++++ 8 files changed, 462 insertions(+), 85 deletions(-) create mode 100644 subpackages/utils/source/odood/utils/tipy/package.d create mode 100644 subpackages/utils/source/odood/utils/tipy/python.d diff --git a/dub.selections.json b/dub.selections.json index e7e42ae9..a7270912 100644 --- a/dub.selections.json +++ b/dub.selections.json @@ -1,6 +1,8 @@ { "fileVersion": 1, "versions": { + "bindbc-common": "~master", + "bindbc-loader": "1.1.2", "cachetools": "0.3.1", "color": "0.0.9", "colored": "0.0.31", diff --git a/subpackages/cli/source/odood/cli/commands/addons.d b/subpackages/cli/source/odood/cli/commands/addons.d index e09d06bf..59f1c151 100644 --- a/subpackages/cli/source/odood/cli/commands/addons.d +++ b/subpackages/cli/source/odood/cli/commands/addons.d @@ -199,9 +199,36 @@ class CommandAddonsList: OdoodCommand { ]; foreach(field; fields) { switch(field) { + case "name": + row ~= [addon.manifest.name]; + break; case "version": row ~= [addon.manifest.module_version.toString]; break; + case "author": + row ~= [addon.manifest.author]; + break; + case "category": + row ~= [addon.manifest.category]; + break; + case "license": + row ~= [addon.manifest.license]; + break; + case "maintainer": + row ~= [addon.manifest.maintainer]; + break; + case "auto_install": + row ~= [addon.manifest.auto_install.to!string]; + break; + case "application": + row ~= [addon.manifest.application.to!string]; + break; + case "installable": + row ~= [addon.manifest.installable.to!string]; + break; + case "tags": + row ~= [addon.manifest.tags.join(", ")]; + break; case "price": if (addon.manifest.price.is_set) row ~= [ @@ -212,8 +239,7 @@ class CommandAddonsList: OdoodCommand { row ~= ["", ""]; break; default: - row ~= [addon.manifest[field]]; - break; + throw new OdoodCLIException("Unsupported manifest field %s".format(field)); } } return row; diff --git a/subpackages/lib/source/odood/lib/odoo/test.d b/subpackages/lib/source/odood/lib/odoo/test.d index b6784711..fe2247ec 100644 --- a/subpackages/lib/source/odood/lib/odoo/test.d +++ b/subpackages/lib/source/odood/lib/odoo/test.d @@ -337,7 +337,7 @@ struct OdooTestRunner { /// Add new module to test run auto ref addModule(in OdooAddon addon) { - if (!addon.getManifest.installable) { + if (!addon.manifest.installable) { warningf("Addon %s is not installable. Skipping", addon.name); return this; } @@ -358,7 +358,7 @@ struct OdooTestRunner { /// Add new additional module to install before test auto ref addAdditionalModule(in OdooAddon addon) { - if (!addon.getManifest.installable) { + if (!addon.manifest.installable) { warningf("Additional addon %s is not installable. Skipping", addon.name); return this; } diff --git a/subpackages/utils/dub.sdl b/subpackages/utils/dub.sdl index 8e313bdd..bea57d31 100644 --- a/subpackages/utils/dub.sdl +++ b/subpackages/utils/dub.sdl @@ -10,6 +10,8 @@ dependency "theprocess" version=">=0.0.5" dependency "zipper" version=">=0.0.3" dependency "semver" version=">=0.4.0" dependency "pyd" version=">=0.14.4" +dependency "bindbc-loader" version="~>1.1.2" +dependency "bindbc-common" version="~master" targetPath "build" targetType "library" diff --git a/subpackages/utils/source/odood/utils/addons/addon.d b/subpackages/utils/source/odood/utils/addons/addon.d index 0354a9f1..3aa2afbb 100644 --- a/subpackages/utils/source/odood/utils/addons/addon.d +++ b/subpackages/utils/source/odood/utils/addons/addon.d @@ -20,8 +20,7 @@ private import odood.utils.addons.addon_manifest; final class OdooAddon { private immutable string _name; private immutable Path _path; - private immutable Path _manifest_path; - private Nullable!OdooAddonManifest _manifest; + private OdooAddonManifest _manifest; @disable this(); @@ -32,10 +31,9 @@ final class OdooAddon { * path = Path to addon on filesystem **/ this(in Path path) { - // TODO: there is no need to specify name here. It have to be computed based on path. this._path = path.toAbsolute; this._name = _path.baseName; - this._manifest_path = getAddonManifestPath(_path).get; + this._manifest = parseOdooManifest(getAddonManifestPath(_path).get); } /// name of the addon @@ -45,15 +43,7 @@ final class OdooAddon { auto path() const => _path; /// module manifest - auto manifest() { - if (_manifest.isNull) - _manifest = OdooAddonManifest(_manifest_path).nullable; - - return _manifest.get; - } - - /// Get module manifest - auto getManifest() const => OdooAddonManifest(_manifest_path); + auto manifest() const => _manifest; /// Addons are comparable by name pure nothrow int opCmp(in OdooAddon other) const { diff --git a/subpackages/utils/source/odood/utils/addons/addon_manifest.d b/subpackages/utils/source/odood/utils/addons/addon_manifest.d index b32aa91f..68256e30 100644 --- a/subpackages/utils/source/odood/utils/addons/addon_manifest.d +++ b/subpackages/utils/source/odood/utils/addons/addon_manifest.d @@ -1,13 +1,13 @@ module odood.utils.addons.addon_manifest; -private import std.typecons: Nullable, nullable, tuple; -private import std.conv: to; +private import std.format; +private import std.string; +private import std.typecons; -private import pyd.embedded: py_eval; -private import pyd.pydobject: PydObject; -private import pyd.make_object: PydConversionException; +private import thepath; -private import thepath: Path; +private import odood.utils.tipy; +private import odood.utils.tipy.python; private import odood.utils.addons.addon_version; @@ -15,84 +15,169 @@ private import odood.utils.addons.addon_version; /** Struct designed to read addons manifest **/ struct OdooAddonManifest { - private PydObject _manifest; - this(in Path path) { - _manifest = py_eval(path.readFileText); - } - - /// Allows to access manifest as pyd object - auto raw_manifest() { - return _manifest; - } - - /// Is addon installable - bool installable() { - return _manifest.get("installable", true).to_d!bool; - } - - /// Is this addon application - bool application() { - return _manifest.get("application", false).to_d!bool; - } - - /** Price info for this addon + /** Use separate struct to handle prices + * The default currency is EUR * - * Returns: - * tuple with following fields: - * - currency - * - price - * - is_set + * Additionally, this struct contains field is_set, that determines + * if price was set in manifest or not (even if price is 0). **/ - auto price() { - string currency = _manifest.get("currency","EUR").to_d!string; + private struct ManifestPrice { + string currency; float price; - if (_manifest.has_key("price")) { - try - price = _manifest["price"].to_d!float; - catch (PydConversionException) - price = _manifest["price"].to_d!(string).to!float; - return tuple!( - "currency", "price", "is_set" - )(currency, price, true); + bool is_set = false; + + string toString() const { + if (is_set) + return "%s %s".format(price, currency); + return ""; } - return tuple!( - "currency", "price", "is_set" - )(currency, price, false); } - /// Return list of dependencies of addon - string[] dependencies() { - if (_manifest.has_key("depends")) - return _manifest["depends"].to_d!(string[]); - return []; - } + string name; + OdooAddonVersion module_version = OdooAddonVersion("1.0"); + string author; + string category; + string description; + string license; + string maintainer; + + bool auto_install=false; + bool application=false; + bool installable=true; + + // Dependencies + string[] dependencies; + string[] python_dependencies; + string[] bin_dependencies; + + // CR&D Extensions + string[] tags; - /// Return list of python dependencies - string[] python_dependencies() { - if (!_manifest.has_key("external_dependencies")) - return []; - if (!_manifest["external_dependencies"].has_key("python")) - return []; - return _manifest["external_dependencies"]["python"].to_d!(string[]); + ManifestPrice price; + + /// Return string representation of manifest + string toString() const { + return "AddonManifest: %s (%s)".format(name, module_version); } +} - /// Returns parsed module version - auto module_version() { - // If version is not specified, then return "1.0" - return OdooAddonVersion(_manifest.get("version", "1.0").to_d!string); +/** Parse Odoo manifest file + **/ +auto parseOdooManifest(in string manifest_content) { + OdooAddonManifest manifest; + + auto parsed = callPyFunc(_fn_literal_eval, manifest_content); + scope(exit) Py_DecRef(parsed); + + // PyDict_GetItemString returns borrowed reference, + // thus there is no need to call Py_DecRef from our side + if (auto val = PyDict_GetItemString(parsed, "name".toStringz)) + manifest.name = val.convertPyToD!string; + if (auto val = PyDict_GetItemString(parsed, "version".toStringz)) + manifest.module_version = OdooAddonVersion(val.convertPyToD!string); + if (auto val = PyDict_GetItemString(parsed, "author".toStringz)) + manifest.author = val.convertPyToD!string; + if (auto val = PyDict_GetItemString(parsed, "category".toStringz)) + manifest.category = val.convertPyToD!string; + if (auto val = PyDict_GetItemString(parsed, "description".toStringz)) + manifest.description = val.convertPyToD!string; + if (auto val = PyDict_GetItemString(parsed, "license".toStringz)) + manifest.license = val.convertPyToD!string; + if (auto val = PyDict_GetItemString(parsed, "maintainer".toStringz)) + manifest.maintainer = val.convertPyToD!string; + + if (auto val = PyDict_GetItemString(parsed, "auto_install".toStringz)) + manifest.auto_install = val.convertPyToD!bool; + if (auto val = PyDict_GetItemString(parsed, "application".toStringz)) + manifest.application = val.convertPyToD!bool; + if (auto val = PyDict_GetItemString(parsed, "installable".toStringz)) + manifest.installable = val.convertPyToD!bool; + + if (auto val = PyDict_GetItemString(parsed, "depends".toStringz)) + manifest.dependencies = val.convertPyToD!(string[]); + if (auto external_deps = PyDict_GetItemString(parsed, "external_dependencies".toStringz)) { + if (auto val = PyDict_GetItemString(external_deps, "python".toStringz)) + manifest.python_dependencies = val.convertPyToD!(string[]); + if (auto val = PyDict_GetItemString(external_deps, "bin".toStringz)) + manifest.bin_dependencies = val.convertPyToD!(string[]); } - /// Access manifest item as string: - string opIndex(in string index) { - return _manifest.get(index, "").to_d!string; + if (auto val = PyDict_GetItemString(parsed, "tags".toStringz)) + manifest.tags = val.convertPyToD!(string[]); + + + if (auto py_price = PyDict_GetItemString(parsed, "price".toStringz)) { + manifest.price.price = py_price.convertPyToD!float; + + if (auto py_currency = PyDict_GetItemString(parsed, "currency".toStringz)) { + manifest.price.currency = py_currency.convertPyToD!string; + } else { + manifest.price.currency = "EUR"; + } + + manifest.price.is_set = true; } + return manifest; } +/// ditto +auto parseOdooManifest(in Path path) { + return parseOdooManifest(path.readFileText); +} + +// Module level link to ast module +private PyObject* _fn_literal_eval; -// Initialize pyd as early as possible. +// Initialize python interpreter (import ast.literal_eval) shared static this() { - import pyd.def: py_init; - py_init(); + loadPyLib; + + Py_Initialize(); + + auto mod_ast = PyImport_ImportModule("ast"); + scope(exit) Py_DecRef(mod_ast); + + // Save function literal_eval from ast on module level + _fn_literal_eval = PyObject_GetAttrString( + mod_ast, "literal_eval".toStringz + ).pyEnforce; + + +} + +// Finalize python interpreter (do clean up) +shared static ~this() { + if (_fn_literal_eval) Py_DecRef(_fn_literal_eval); + Py_Finalize(); +} + + +// Tests +unittest { + auto manifest = parseOdooManifest(`{ + 'name': "A Module", + 'version': '1.0', + 'depends': ['base'], + 'author': "Author Name", + 'category': 'Category', + 'description': """ + Description text + """, + # data files always loaded at installation + 'data': [ + 'views/mymodule_view.xml', + ], + # data files containing optionally loaded demonstration data + 'demo': [ + 'demo/demo_data.xml', + ], +}`); + + assert(manifest.name == "A Module"); + assert(manifest.module_version.isStandard == false); + assert(manifest.module_version.toString == "1.0"); + assert(manifest.module_version.rawVersion == "1.0"); + assert(manifest.dependencies == ["base"]); } diff --git a/subpackages/utils/source/odood/utils/tipy/package.d b/subpackages/utils/source/odood/utils/tipy/package.d new file mode 100644 index 00000000..adcc4466 --- /dev/null +++ b/subpackages/utils/source/odood/utils/tipy/package.d @@ -0,0 +1,190 @@ +module odood.utils.tipy; + +private import std.string; +private import std.traits: + isSomeString, isScalarType, isIntegral, isBoolean, isFloatingPoint, isArray; +private import std.range: ElementType; + +private static import bindbc.loader; + +private import odood.utils.tipy.python; + + +private bindbc.loader.SharedLib pylib; + +private enum supported_lib_names = [ + "libpython3.11.so", + "libpython3.10.so", + "libpython3.9.so", + "libpython3.8.so", + "libpython3.7.so", + "libpython3.6.so", + "libpython3.5.so", + "libpython3.4.so", + "libpython3.3.so", +]; + + +// Load python library +bool loadPyLib() { + foreach(libname; supported_lib_names) { + pylib = bindbc.loader.load(libname.ptr); + if (pylib == bindbc.loader.invalidHandle) { + continue; + } + + auto err_count = bindbc.loader.errorCount; + odood.utils.tipy.python.bindModuleSymbols(pylib); + if (bindbc.loader.errorCount == err_count) + return true; + } + + // Cannot load library + return false; +} + + +/** Ensure no python error is set. + * Checks python error indicator and throw error if such indicator is set. + * + * Throws: + * Exception when python object is not valid and some error occured. + **/ +void pyEnsureNoError() { + if (PyErr_Occurred()) { + PyObject* etype=null, evalue=null, etraceback=null; + PyErr_Fetch(&etype, &evalue, &etraceback); + scope(exit) { + if (etype) Py_DecRef(etype); + if (evalue) Py_DecRef(evalue); + if (etraceback) Py_DecRef(etraceback); + } + + // TODO: Think about avoiding usage of convertPyToD + // to avoid possible infinite recursion + string msg = etype.convertPyToD!string; + if (evalue) msg ~= ": " ~ evalue.convertPyToD!string; + throw new Exception(msg); + } +} + +/** Ensure that pyobjec is valid and no error is produced + * + * Params: + * value = python object to validate + * + * Returns: + * value if it is valid, otherwise throws error + * + * Throws: + * Exception when python object is not valid and some error occured. + **/ +auto pyEnforce(PyObject* value) { + if (!value) + pyEnsureNoError(); + return value; +} + + +/** Convert python object to D representation. + **/ +T convertPyToD(T)(PyObject* o) +if (isSomeString!T) { + auto unicode = PyObject_Str(o).pyEnforce; + scope(exit) Py_DecRef(unicode); + auto str = PyUnicode_AsUTF8String(unicode).pyEnforce; + scope(exit) Py_DecRef(str); + return PyBytes_AsString(str).fromStringz.idup; +} + +/// ditto +T convertPyToD(T)(PyObject* o) +if (isBoolean!T) { + int result = PyObject_IsTrue(o); + switch(result) { + case 1: return true; + case 0: return false; + case -1: + // Error. Pass null explicitely to enforce checking error + // info and raising exception. + pyEnsureNoError(); + throw new Exception("Cannot convert py object to bool for unknown reason"); + default: + assert(0, "Unsupported return value from PyObject_IsTrue"); + } +} + +/// ditto +T convertPyToD(T)(PyObject* o) +if (isArray!T && !isSomeString!T) { + T result; + auto iterator = PyObject_GetIter(o).pyEnforce; + scope(exit) Py_DecRef(iterator); + + while(auto item = PyIter_Next(iterator)) { + scope(exit) Py_DecRef(item); + result ~= convertPyToD!(ElementType!T)(item); + } + + // Ensure python error indicatori is not set + pyEnsureNoError(); + + return result; +} + +/// ditto +T convertPyToD(T)(PyObject* o) +if (isFloatingPoint!T) { + auto number = PyNumber_Float(o).pyEnforce; + double result = PyFloat_AsDouble(number); + pyEnsureNoError; + return result; +} + + +/** Convert to python object + * + * Params: + * val = value to convert to python object + * Returns: + * Pointer to new reference to PyObject. + **/ +PyObject* convertToPy(T)(T val) { + static if(isIntegral!T) + return PyLong_FromLongLong(val); + else static if (isSomeString!T) + return PyUnicode_FromString(val.toStringz); + else + static assert(0, "Unsupported type"); +} + + +/** Call python function with arguments + * + * Params: + * fn = python object that represents function to call + * params = D variadic parameters to pass to function + * Returns: + * PyObject pointer that represents result of function execution + **/ +auto callPyFunc(T...)(PyObject* fn, T params) { + import std.range: iota; + //import std.stdio; + + auto args = PyTuple_New(T.length); + scope(exit) Py_DecRef(args); + + auto kwargs = PyDict_New(); + scope(exit) Py_DecRef(kwargs); + + static foreach(i; iota(0, T.length)) { + // There is no nees to decref, because SetItem steals ref + PyTuple_SetItem(args, i, params[i].convertToPy); + } + + //writefln("Call fn %s with args %s and kwargs %s", convertPyToString(fn), convertPyToString(args), convertPyToString(kwargs)); + auto res = PyObject_Call(fn, args, kwargs); + pyEnforce(res); + //writefln("Result: %s", res.convertPyToString); + return res; +} diff --git a/subpackages/utils/source/odood/utils/tipy/python.d b/subpackages/utils/source/odood/utils/tipy/python.d new file mode 100644 index 00000000..7d0e162f --- /dev/null +++ b/subpackages/utils/source/odood/utils/tipy/python.d @@ -0,0 +1,82 @@ +module odood.utils.tipy.python; + +private import std.string; +private import std.stdio; +private import std.path; + +private import bindbc.common.codegen: joinFnBinds, FnBind; + + +// Copied from python +enum Py_single_input = 256; +enum Py_file_input = 257; +enum Py_eval_input = 258; +enum Py_func_type_input = 345; + +struct PyObject {}; + +alias Py_ssize_t = size_t; + + +mixin(joinFnBinds!false((){ + FnBind[] ret = [ + // General + {q{void}, q{Py_Initialize}}, + {q{void}, q{Py_Finalize}}, + {q{void}, q{Py_DecRef}, q{PyObject* o}}, + + // Import & module + {q{PyObject*}, q{PyImport_ImportModule}, q{const char* name}}, + {q{PyObject*}, q{PyModule_GetDict}, q{PyObject* mod}}, + + //PyErr + {q{PyObject*}, q{PyErr_Occurred}}, + {q{void}, q{PyErr_Fetch}, q{PyObject **ptype, PyObject **pvalue, PyObject **ptraceback}}, + + // PyDict + {q{PyObject*}, q{PyDict_New}}, + {q{PyObject*}, q{PyDict_GetItemString}, q{PyObject* p, const char* key}}, // returns Borrowed reference + + // PyTuple + {q{PyObject*}, q{PyTuple_New}, q{Py_ssize_t len}}, + {q{PyObject*}, q{PyTuple_SetItem}, q{PyObject* p, Py_ssize_t pos, PyObject* o}}, + + // PyObject + {q{PyObject*}, q{PyObject_Str}, q{PyObject* o}}, + {q{PyObject*}, q{PyObject_Call}, q{PyObject* callable, PyObject* args, PyObject* kwargs}}, + {q{PyObject*}, q{PyObject_CallObject}, q{PyObject* callable, PyObject* args}}, + {q{PyObject*}, q{PyObject_Type}, q{PyObject* o}}, + {q{PyObject*}, q{PyObject_GetAttrString}, q{PyObject* o, const char* attr_name}}, + {q{PyObject*}, q{PyObject_GetIter}, q{PyObject* o}}, + {q{int}, q{PyObject_IsTrue}, q{PyObject* o}}, + {q{int}, q{PyObject_IsInstance}, q{PyObject* inst, PyObject* cls}}, + + // PyIter + {q{PyObject*}, q{PyIter_Next}, q{PyObject* o}}, + + // PyList + {q{PyObject*}, q{PyList_AsTuple}, q{PyObject* list}}, + + // PyBytes + {q{char*}, q{PyBytes_AsString}, q{PyObject* o}}, + + // PyUnicode + {q{PyObject*}, q{PyUnicode_AsUTF8String}, q{PyObject* unicode}}, + {q{PyObject*}, q{PyUnicode_FromString}, q{const char*}}, + + // PyFloat + {q{double}, q{PyFloat_AsDouble}, q{PyObject* pyfloat}}, + + // PyLong + {q{double}, q{PyLong_AsDouble}, q{PyObject* pylong}}, + {q{PyObject*}, q{PyLong_FromLongLong}, q{long v}}, + {q{long}, q{PyLong_AsLongLong}, q{PyObject* pylong}}, + + // PyNumber + {q{PyObject*}, q{PyNumber_Float}, q{PyObject* o}}, + ]; + // See: https://github.com/BindBC/bindbc-freetype/blob/master/source/ft/advanc.d + return ret; +}())); + +