diff --git a/README.rst b/README.rst index d2affa5..c0b1477 100644 --- a/README.rst +++ b/README.rst @@ -7,13 +7,34 @@ palimport Palimpsest importer for python Palimport allows you to import modules that can be defined in any custom language, as long as you provide a grammar and way to interpret it ( in python !) - This way you can embed multiple DSLs in your python programs. -Supported parsers : +Why ? +----- + +Because managing importer properly, and following python package logic, is not trivial. +So if you want to embed your programming language into python, and leverage all python tools and libraries, better implement your importer in palimport. + +Benefits +-------- + +Here are a few benefits of using palimport to teach new languages to your favorite python interpreter:: + +- Already implemented Python 2/3 importer compatibility (using filefinder2) +- Importer is enabled/disabled as a context manager, so you stay in control over what can be imported or not. +- Compare your importers with other importers right here, and be notified when an importer is changed. + + +Roadmap +------- + + +Currently Supported parsers include :: +- hy [TODO] +- coconut [TODO] - lark -- more to come +- add yours here ! Currently tested with python 3.5:: diff --git a/palimport/__init__.py b/palimport/__init__.py index 5186ccd..b053f7b 100644 --- a/palimport/__init__.py +++ b/palimport/__init__.py @@ -17,6 +17,9 @@ from ._lark import Importer as LarkImporter +from ._hylang import Importer as HyImporter + + from .finder import Finder from .loader import Loader diff --git a/palimport/_hylang/__init__.py b/palimport/_hylang/__init__.py new file mode 100644 index 0000000..3eb6703 --- /dev/null +++ b/palimport/_hylang/__init__.py @@ -0,0 +1,34 @@ +from __future__ import absolute_import + +import sys +import filefinder2 + +# we rely on filefinder2 as a py2/3 wrapper of importlib + +from .finder import HyFinder +from .loader import HyLoader + + +class Importer(filefinder2.Py3Importer): + + def __enter__(self): + super(Importer, self).__enter__() + + # we hook the grammar customized loader + self.path_hook = HyFinder.path_hook((HyLoader, ['.hy']), ) + + if self.path_hook not in sys.path_hooks: + ffidx = sys.path_hooks.index(filefinder2.ff_path_hook) + sys.path_hooks.insert(ffidx, self.path_hook ) + + def __exit__(self, exc_type, exc_val, exc_tb): + + # removing path_hook + sys.path_hooks.pop(sys.path_hooks.index(self.path_hook)) + + super(Importer, self).__exit__(exc_type, exc_val, exc_tb) + + +__all__ = [ + Importer +] diff --git a/palimport/_hylang/finder.py b/palimport/_hylang/finder.py new file mode 100644 index 0000000..52c31d3 --- /dev/null +++ b/palimport/_hylang/finder.py @@ -0,0 +1,72 @@ +from __future__ import absolute_import, division, print_function + +""" +A module to setup custom importer for .hy files + +""" + +# We need to be extra careful with python versions +# Ref : https://docs.python.org/dev/library/importlib.html#importlib.import_module + +import os + +from filefinder2.machinery import FileFinder as filefinder2_FileFinder + +from .._utils import _ImportError + + +class HyFinder(filefinder2_FileFinder): + """PathEntryFinder to handle finding Lark grammars""" + + def __init__(self, path, *loader_details): + super(HyFinder, self).__init__(path, *loader_details) + + def __repr__(self): + return 'HyFinder({!r})'.format(self.path) + + @classmethod + def path_hook(cls, *loader_details): + """A class method which returns a closure to use on sys.path_hook + which will return an instance using the specified loaders and the path + called on the closure. + + If the path called on the closure is not a directory, or doesnt contain + any files with the supported extension, ImportError is raised. + + This is different from default python behavior + but prevent polluting the cache with custom finders + """ + def path_hook_for_HyFinder(path): + """Path hook for importlib.machinery.FileFinder.""" + + if not (os.path.isdir(path)): + raise _ImportError('only directories are supported') + + exts = [x for ld in loader_details for x in ld[1]] + if not any(fname.endswith(ext) for fname in os.listdir(path) for ext in exts): + raise _ImportError( + 'only directories containing {ext} files are supported'.format(ext=", ".join(exts)), + path=path) + return cls(path, *loader_details) + + return path_hook_for_HyFinder + + def find_spec(self, fullname, target=None): + """ + Try to find a spec for the specified module. + :param fullname: the name of the package we are trying to import + :return: the matching spec, or None if not found. + """ + + # We attempt to load a .hy file as a module + tail_module = fullname.rpartition('.')[2] + base_path = os.path.join(self.path, tail_module) + for suffix, loader_class in self._loaders: + full_path = base_path + suffix + if os.path.isfile(full_path): # maybe we need more checks here (importlib filefinder checks its cache...) + return self._get_spec(loader_class, fullname, full_path, None, target) + + # Otherwise, we try find python modules (to be able to embed .lark files within python packages) + return super(HyFinder, self).find_spec(fullname=fullname, target=target) + + diff --git a/palimport/_hylang/loader.py b/palimport/_hylang/loader.py new file mode 100644 index 0000000..aef0989 --- /dev/null +++ b/palimport/_hylang/loader.py @@ -0,0 +1,64 @@ +import __future__ + +import os +import sys +import filefinder2 + +from palimport._utils import _verbose_message, _ImportError + +from hy.compiler import hy_compile, HyTypeError +from hy.models import HyObject, HyExpression, HySymbol, replace_hy_obj +from hy.lex import tokenize, LexException +from hy.importlib.machinery import HyPathFinder + +import marshal + +#from hy._compat import PY3, PY37, MAGIC, builtins, long_type, wr_long +from hy._compat import string_types + + +class HyLoader(filefinder2.machinery.SourceFileLoader): + + def set_data(self, path, data): + """Optional method which writes data (bytes) to a file path (a str). + Implementing this method allows for the writing of bytecode files. + """ + # st = os.stat(path) + # timestamp = long_type(st.st_mtime) + # + # cfile = filefinder2.util.cache_from_source(path) + # try: + # os.makedirs(os.path.dirname(cfile)) + # except (IOError, OSError): + # pass + # + # with builtins.open(cfile, 'wb') as fc: + # fc.write(MAGIC) + # if PY37: + # # With PEP 552, the header structure has a new flags field + # # that we need to fill in. All zeros preserve the legacy + # # behaviour, but should we implement reproducible builds, + # # this is where we'd add the information. + # wr_long(fc, 0) + # wr_long(fc, timestamp) + # if PY3: + # wr_long(fc, st.st_size) + # marshal.dump(data, fc) + + # TODO : investigate : removing get_code breaks loader !!! + def get_code(self, fullname): + source = self.get_source(fullname) + _verbose_message('compiling code for "{0}"'.format(fullname)) + try: + code = self.source_to_code(source, self.get_filename(fullname), fullname) + return code + except TypeError: + raise + + def source_to_code(self, data, path, module_name=None): + """Compile source to HST, then to AST. + module_name parameter has been added compared to python API, to be able to pass it to hy_compile""" + hst = HyExpression([HySymbol("do")] + tokenize(data + "\n")) + ast = hy_compile(hst, module_name) + flags = (__future__.CO_FUTURE_DIVISION | __future__.CO_FUTURE_PRINT_FUNCTION) + return compile(ast, path, "exec", flags) diff --git a/tests/hylang/__init__.py b/tests/hylang/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/hylang/fact.hy b/tests/hylang/fact.hy new file mode 100644 index 0000000..4155eec --- /dev/null +++ b/tests/hylang/fact.hy @@ -0,0 +1,10 @@ +(require [hy.contrib.loop [loop]]) + +(defn factorial [n] + (loop [[i n] [acc 1]] + (if (zero? i) + acc + (recur (dec i) (* acc i))))) + +;; Test +;; (factorial 1000) \ No newline at end of file diff --git a/tests/hylang/test_fact.py b/tests/hylang/test_fact.py new file mode 100644 index 0000000..85959dc --- /dev/null +++ b/tests/hylang/test_fact.py @@ -0,0 +1,12 @@ +import palimport + + +with palimport.HyImporter(): + if __package__: # attempting relative import when possible + from . import fact + else: + import fact + + +def test_fact(): + assert fact.factorial(5) == 120