diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..666bb8d --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.pytest_cache/ +tests/ +.zip diff --git a/README.md b/README.md new file mode 100644 index 0000000..e69de29 diff --git a/hirise_blender/__init__.py b/hirise_blender/__init__.py new file mode 100644 index 0000000..6995358 --- /dev/null +++ b/hirise_blender/__init__.py @@ -0,0 +1,26 @@ +import bpy + +bl_info = { + "name": "Import HIRISE Image", + "blender": (3, 30, 0), + "category": "Object", +} + +if 'bpy' in locals(): + print('HIRISE Reloading') + from importlib import reload + import sys + for k, v in list(sys.modules.items()): + if k.startswith('hirise_blender.'): + reload(v) + +from .hirise_blender import ImportIMGData, menu_func_import + +# Register and add to the "file selector" menu (required to use F3 search "Text Import Operator" for quick access). +def register(): + bpy.utils.register_class(ImportIMGData) + bpy.types.TOPBAR_MT_file_import.append(menu_func_import) + +def unregister(): + bpy.utils.unregister_class(ImportIMGData) + bpy.types.TOPBAR_MT_file_import.remove(menu_func_import) \ No newline at end of file diff --git a/hirise_blender/__pycache__/__init__.cpython-310.pyc b/hirise_blender/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..da5ea62 Binary files /dev/null and b/hirise_blender/__pycache__/__init__.cpython-310.pyc differ diff --git a/hirise_blender/__pycache__/hirise_blender.cpython-310.pyc b/hirise_blender/__pycache__/hirise_blender.cpython-310.pyc new file mode 100644 index 0000000..fbceabe Binary files /dev/null and b/hirise_blender/__pycache__/hirise_blender.cpython-310.pyc differ diff --git a/hirise_blender/hirise_blender.py b/hirise_blender/hirise_blender.py new file mode 100644 index 0000000..b64e9f8 --- /dev/null +++ b/hirise_blender/hirise_blender.py @@ -0,0 +1,166 @@ + +import bpy +from .planetaryimage import PDS3Image +import numpy as np +import os + + +def make_mesh(gridded_data, lower_res_factor = 4, quads = True): + data_width = gridded_data.shape[0]//lower_res_factor + data_height = gridded_data.shape[1]//lower_res_factor + + vertices = np.zeros((data_width*data_height, 3)) + if quads: + faces = np.zeros(((data_width-1)*(data_height-1), 4), dtype = 'int') + else: + faces = np.zeros(((data_width-1)*(data_height-1)*2, 3), dtype = 'int') + + for i in range(data_width): + for j in range(data_height): + vertices[i*data_height+j][0] = i*lower_res_factor + vertices[i*data_height+j][1] = j*lower_res_factor + if i == 0 or i == data_width-1 or j == 0 or j == data_height-1: + vertices[i*data_height+j][2] = 0 + else: + vertices[i*data_height+j][2] = gridded_data[i*lower_res_factor][j*lower_res_factor] + + for i in range(data_width-1): + for j in range(data_height-1): + if quads: + faces[(i*(data_height-1)+j)][0] = i*data_height+j + faces[(i*(data_height-1)+j)][1] = i*data_height+j+1 + faces[(i*(data_height-1)+j)][2] = (i+1)*data_height+j+1 + faces[(i*(data_height-1)+j)][3] = (i+1)*data_height+j + else: + faces[(i*(data_height-1)+j)*2][0] = i*data_height+j + faces[(i*(data_height-1)+j)*2][1] = i*data_height+j+1 + faces[(i*(data_height-1)+j)*2][2] = (i+1)*data_height+j + + faces[(i*(data_height-1)+j)*2+1][0] = i*data_height+j+1 + faces[(i*(data_height-1)+j)*2+1][1] = (i+1)*data_height+j+1 + faces[(i*(data_height-1)+j)*2+1][2] = (i+1)*data_height+j + + mesh_data = bpy.data.meshes.new("cube_mesh_data") + #mesh_data.from_pydata(vertices, [], faces) + + print("adding vertices...") + mesh_data.vertices.add(vertices.shape[0]) + mesh_data.vertices.foreach_set("co", vertices.flatten()) + + print("adding vertex indices...") + mesh_data.loops.add(faces.size) + mesh_data.loops.foreach_set("vertex_index", faces.flatten()) + + print("adding faces...") + mesh_data.polygons.add(faces.shape[0]) + if quads: + mesh_data.polygons.foreach_set("loop_start", range(0, faces.size, 4)) + mesh_data.polygons.foreach_set("loop_total", [4]*faces.shape[0]) + else: + mesh_data.polygons.foreach_set("loop_start", range(0, faces.size, 3)) + mesh_data.polygons.foreach_set("loop_total", [3]*faces.shape[0]) + + print("update mesh...") + mesh_data.update() + + print("validating mesh...") + mesh_data.validate() + + return mesh_data + +def read_img_data(context, filepath, ppgs, safety, quads): + print(context) + filename = os.path.basename(filepath)[:-4] + # Open IMAGE + image = PDS3Image.open(filepath) + + maxval = image.data.max() + minval = image.data.min() + remove_mask = np.where(image.data < -10000, 0, image.data) + minval = remove_mask.min() + + renormalized_masked = np.where(remove_mask != 0, remove_mask-minval, remove_mask) + + npdata = renormalized_masked[0] + + if not safety and ppgs < 4: + ppgs = 4 + + + mesh_data = make_mesh(npdata, ppgs, quads) + + obj = bpy.data.objects.new(filename, mesh_data) + + new_collection = bpy.data.collections.new(filename) + bpy.context.scene.collection.children.link(new_collection) + # add object to scene collection + new_collection.objects.link(obj) + + return {'FINISHED'} + + +# ImportHelper is a helper class, defines filename and +# invoke() function which calls the file selector. +from bpy_extras.io_utils import ImportHelper +from bpy.props import StringProperty, BoolProperty, IntProperty +from bpy.types import Operator + + +class ImportIMGData(Operator, ImportHelper): + """This appears in the tooltip of the operator and in the generated docs""" + bl_idname = "import_test.img_data" # important since its how bpy.ops.import_test.some_data is constructed + bl_label = "Import File" + + # ImportHelper mixin class uses this + filename_ext = ".img" + + filter_glob: StringProperty( + default="*.img", + options={'HIDDEN'}, + maxlen=255, # Max internal buffer length, longer would be clamped. + ) + + ppgs: IntProperty( + name="Pixels Per Grid Square" + , description="Choose How Many Pixels in the IMG file will correspond to a single grid square (quad or tri) in the mesh." + , default=8 + , max = 32 + , min = 1 + ) + + safety: BoolProperty( + name="Safety" + , description="Enable to go below 4 pixels per grid. DO THIS ONLY IF YOU HAVE TONS OF RAM" + , default=False + ) + + quads: BoolProperty( + name = "Use Quads" + , description="Use a quad mesh or a triangle mesh." + , default = True + ) + + def execute(self, context): + return read_img_data(context, self.filepath, self.ppgs, self.safety, self.quads) + + +# Only needed if you want to add into a dynamic menu. +def menu_func_import(self, context): + self.layout.operator(ImportIMGData.bl_idname, text="HIRISE .IMG") + + +# Register and add to the "file selector" menu (required to use F3 search "Text Import Operator" for quick access). +def register(): + bpy.utils.register_class(ImportIMGData) + bpy.types.TOPBAR_MT_file_import.append(menu_func_import) + + +def unregister(): + bpy.utils.unregister_class(ImportIMGData) + bpy.types.TOPBAR_MT_file_import.remove(menu_func_import) + + +if __name__ == "__main__": + register() + # test call + #bpy.ops.import_test.some_data('INVOKE_DEFAULT') diff --git a/hirise_blender/planetaryimage/__init__.py b/hirise_blender/planetaryimage/__init__.py new file mode 100755 index 0000000..7d965a3 --- /dev/null +++ b/hirise_blender/planetaryimage/__init__.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- + +__author__ = 'PlanetaryPy Developers' +__email__ = 'contact@planetarypy.com' +__version__ = '0.4.1' +__all__ = [ + 'PDS3Image', + 'CubeFile', +] + +__all__ = [ + 'CubeFile', + 'PDS3Image', +] + +from .cubefile import CubeFile +from .pds3image import PDS3Image diff --git a/hirise_blender/planetaryimage/__pycache__/__init__.cpython-310.pyc b/hirise_blender/planetaryimage/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..608edd0 Binary files /dev/null and b/hirise_blender/planetaryimage/__pycache__/__init__.cpython-310.pyc differ diff --git a/hirise_blender/planetaryimage/__pycache__/cubefile.cpython-310.pyc b/hirise_blender/planetaryimage/__pycache__/cubefile.cpython-310.pyc new file mode 100644 index 0000000..c327a57 Binary files /dev/null and b/hirise_blender/planetaryimage/__pycache__/cubefile.cpython-310.pyc differ diff --git a/hirise_blender/planetaryimage/__pycache__/decoders.cpython-310.pyc b/hirise_blender/planetaryimage/__pycache__/decoders.cpython-310.pyc new file mode 100644 index 0000000..9b47af1 Binary files /dev/null and b/hirise_blender/planetaryimage/__pycache__/decoders.cpython-310.pyc differ diff --git a/hirise_blender/planetaryimage/__pycache__/image.cpython-310.pyc b/hirise_blender/planetaryimage/__pycache__/image.cpython-310.pyc new file mode 100644 index 0000000..ab5ea8f Binary files /dev/null and b/hirise_blender/planetaryimage/__pycache__/image.cpython-310.pyc differ diff --git a/hirise_blender/planetaryimage/__pycache__/pds3image.cpython-310.pyc b/hirise_blender/planetaryimage/__pycache__/pds3image.cpython-310.pyc new file mode 100644 index 0000000..00b0d26 Binary files /dev/null and b/hirise_blender/planetaryimage/__pycache__/pds3image.cpython-310.pyc differ diff --git a/hirise_blender/planetaryimage/__pycache__/specialpixels.cpython-310.pyc b/hirise_blender/planetaryimage/__pycache__/specialpixels.cpython-310.pyc new file mode 100644 index 0000000..f0ddf2f Binary files /dev/null and b/hirise_blender/planetaryimage/__pycache__/specialpixels.cpython-310.pyc differ diff --git a/hirise_blender/planetaryimage/cubefile.py b/hirise_blender/planetaryimage/cubefile.py new file mode 100644 index 0000000..1266116 --- /dev/null +++ b/hirise_blender/planetaryimage/cubefile.py @@ -0,0 +1,255 @@ +# -*- coding: utf-8 -*- +import numpy + +from .image import PlanetaryImage +from .specialpixels import SPECIAL_PIXELS +from .decoders import BandSequentialDecoder, TileDecoder + + +class CubeFile(PlanetaryImage): + """A Isis Cube file reader. + + Examples + -------- + >>> from planetaryimage import CubeFile + >>> image = CubeFile.open('tests/data/pattern.cub') + >>> # Examples of CubeFile Attributes + >>> image.base + 0.0 + >>> image.multiplier + 1.0 + >>> image.specials['His'] + -3.4028233e+38 + >>> image.tile_lines + 128 + >>> image.tile_samples + 128 + >>> image.tile_shape + (128, 128) + + """ + + PIXEL_TYPES = { + 'UnsignedByte': numpy.dtype('uint8'), + 'SignedByte': numpy.dtype('int8'), + 'UnsignedWord': numpy.dtype('uint16'), + 'SignedWord': numpy.dtype('int16'), + 'UnsignedInteger': numpy.dtype('uint32'), + 'SignedInteger': numpy.dtype('int32'), + 'Real': numpy.dtype('float32'), + 'Double': numpy.dtype('float64') + } + + BYTE_ORDERS = { + 'NoByteOrder': '=', # system + 'Lsb': '<', # little-endian + 'Msb': '>' # big-endian + } + + SPECIAL_PIXELS = SPECIAL_PIXELS + + def _save(self, file_to_write, overwrite): + raise NotImplementedError + + def _create_label(self, array): + raise NotImplementedError + + @property + def _bands(self): + return self.label['IsisCube']['Core']['Dimensions']['Bands'] + + @property + def _lines(self): + return self.label['IsisCube']['Core']['Dimensions']['Lines'] + + @property + def _samples(self): + return self.label['IsisCube']['Core']['Dimensions']['Samples'] + + @property + def _format(self): + return self.label['IsisCube']['Core']['Format'] + + @property + def _start_byte(self): + return self.label['IsisCube']['Core']['StartByte'] - 1 + + @property + def _dtype(self): + return self._pixel_type.newbyteorder(self._byte_order) + + @property + def base(self): + """An additive factor by which to offset pixel DN.""" + return self.label['IsisCube']['Core']['Pixels']['Base'] + + @property + def multiplier(self): + """A multiplicative factor by which to scale pixel DN.""" + return self.label['IsisCube']['Core']['Pixels']['Multiplier'] + + @property + def tile_lines(self): + """Number of lines per tile.""" + if self.format != 'Tile': + return None + return self.label['IsisCube']['Core']['TileLines'] + + @property + def tile_samples(self): + """Number of samples per tile.""" + if self.format != 'Tile': + return None + return self.label['IsisCube']['Core']['TileSamples'] + + @property + def tile_shape(self): + """Shape of tiles.""" + if self.format != 'Tile': + return None + return (self.tile_lines, self.tile_samples) + + @property + def _byte_order(self): + return self.BYTE_ORDERS[self._pixels_group['ByteOrder']] + + @property + def _pixels_group(self): + return self.label['IsisCube']['Core']['Pixels'] + + @property + def _pixel_type(self): + return self.PIXEL_TYPES[self._pixels_group['Type']] + + @property + def specials(self): + """Return the special pixel values""" + pixel_type = self._pixels_group['Type'] + return self.SPECIAL_PIXELS[pixel_type] + + @property + def data_filename(self): + """Return detached filename else None.""" + return self.label['IsisCube']['Core'].get('^Core') + + def apply_scaling(self, copy=True): + """Scale pixel values to there true DN. + + Parameters + ---------- + copy: bool [True] + Whether to apply the scaling to a copy of the pixel data + and leave the original unaffected + + Returns + ------- + Numpy Array + A scaled version of the pixel data + + """ + if copy: + return self.multiplier * self.data + self.base + + if self.multiplier != 1: + self.data *= self.multiplier + + if self.base != 0: + self.data += self.base + + return self.data + + def apply_numpy_specials(self, copy=True): + """Convert isis special pixel values to numpy special pixel values. + + ======= ======= + Isis Numpy + ======= ======= + Null nan + Lrs -inf + Lis -inf + His inf + Hrs inf + ======= ======= + + Parameters + ---------- + copy : bool [True] + Whether to apply the new special values to a copy of the + pixel data and leave the original unaffected + + Returns + ------- + Numpy Array + A numpy array with special values converted to numpy's nan, inf, + and -inf + """ + if copy: + data = self.data.astype(numpy.float64) + + elif self.data.dtype != numpy.float64: + data = self.data = self.data.astype(numpy.float64) + + else: + data = self.data + + data[data == self.specials['Null']] = numpy.nan + data[data < self.specials['Min']] = numpy.NINF + data[data > self.specials['Max']] = numpy.inf + + return data + + def specials_mask(self): + """Create a pixel map for special pixels. + + Returns + ------- + An array where the value is `False` if the pixel is special + and `True` otherwise + """ + mask = self.data >= self.specials['Min'] + mask &= self.data <= self.specials['Max'] + return mask + + def get_image_array(self): + """Create an array for use in making an image. + + Creates a linear stretch of the image and scales it to between `0` and + `255`. `Null`, `Lis` and `Lrs` pixels are set to `0`. `His` and `Hrs` + pixels are set to `255`. + + Usage:: + + from planetaryimage import CubeFile + from PIL import Image + + # Read in the image and create the image data + image = CubeFile.open('test.cub') + data = image.get_image_array() + + # Save the first band to a new file + Image.fromarray(data[0]).save('test.png') + + Returns + ------- + A uint8 array of pixel values. + """ + specials_mask = self.specials_mask() + data = self.data.copy() + + data[specials_mask] -= data[specials_mask].min() + data[specials_mask] *= 255 / data[specials_mask].max() + + data[data == self.specials['His']] = 255 + data[data == self.specials['Hrs']] = 255 + + return data.astype(numpy.uint8) + + @property + def _decoder(self): + if self.format == 'BandSequential': + return BandSequentialDecoder(self.dtype, self.shape) + + if self.format == 'Tile': + return TileDecoder(self.dtype, self.shape, self.tile_shape) + + raise ValueError('Unkown format (%s)' % self.format) diff --git a/hirise_blender/planetaryimage/decoders.py b/hirise_blender/planetaryimage/decoders.py new file mode 100644 index 0000000..5044147 --- /dev/null +++ b/hirise_blender/planetaryimage/decoders.py @@ -0,0 +1,49 @@ +import numpy +#from ..six.moves import range + + +class BandSequentialDecoder(object): + def __init__(self, dtype, shape, compression=None): + self.dtype = dtype + self.shape = shape + self.sample_bytes = dtype.itemsize + self.compression = compression + + @property + def size(self): + return numpy.product(self.shape) + + def decode(self, stream): + if self.compression: + data = numpy.fromstring(stream.read(self.size*self.sample_bytes), self.dtype) + else: + data = numpy.fromfile(stream, self.dtype, self.size) + return data.reshape(self.shape) + + +class TileDecoder(object): + def __init__(self, dtype, shape, tile_shape): + self.dtype = dtype + self.shape = shape + self.tile_shape = tile_shape + + def decode(self, stream): + bands, lines, samples = self.shape + tile_lines, tile_samples = self.tile_shape + tile_size = tile_lines * tile_samples + data = numpy.empty(self.shape, dtype=self.dtype) + + for band in data: + for line in range(0, lines, tile_lines): + for sample in range(0, samples, tile_samples): + sample_end = sample + tile_samples + line_end = line + tile_lines + chunk = band[line:line_end, sample:sample_end] + + tile = numpy.fromfile(stream, self.dtype, tile_size) + tile = tile.reshape((tile_lines, tile_samples)) + + chunk_lines, chunk_samples = chunk.shape + chunk[:] = tile[:chunk_lines, :chunk_samples] + + return data diff --git a/hirise_blender/planetaryimage/image.py b/hirise_blender/planetaryimage/image.py new file mode 100644 index 0000000..e395b05 --- /dev/null +++ b/hirise_blender/planetaryimage/image.py @@ -0,0 +1,217 @@ +# -*- coding: utf-8 -*- +import os +import gzip +import bz2 +from ..six import string_types # not found +from ..pvl import load # not found +import numpy + + +class PlanetaryImage(object): + """A generic image reader. Parent object for PDS3Image and CubeFile + + Parameters + ---------- + + stream + file object to read as an image file + + filename : string + an optional filename to attach to the object + + compression : string + an optional string that indicate the compression type 'bz2' or 'gz' + + Attributes + ---------- + compression : string + Compression type (i.e. 'gz', 'bz2', or None). + + data : numpy array + A numpy array representing the image. + + filename : string + The filename given. + + label : pvl module + The image's label in dictionary form. + + Examples + -------- + >>> from planetaryimage import PDS3Image + >>> testfile = 'tests/mission_data/2p129641989eth0361p2600r8m1.img' + >>> image = PDS3Image.open(testfile) + >>> # Examples of attributes + >>> image.bands + 1 + >>> image.lines + 64 + >>> image.samples + 64 + >>> str(image.format) + 'BAND_SEQUENTIAL' + >>> image.data_filename + >>> image.dtype + dtype('>i2') + >>> image.start_byte + 34304 + >>> image.shape + (1, 64, 64) + >>> image.size + 4096 + + See https://planetaryimage.readthedocs.io/en/latest/usage.html to see how + to open images to view them and make manipulations. + + """ + + @classmethod + def open(cls, filename): + """ Read an image file from disk + + Parameters + ---------- + filename : string + Name of file to read as an image file. This file may be gzip + (``.gz``) or bzip2 (``.bz2``) compressed. + """ + if filename.endswith('.gz'): + fp = gzip.open(filename, 'rb') + try: + return cls(fp, filename, compression='gz') + finally: + fp.close() + elif filename.endswith('.bz2'): + fp = bz2.BZ2File(filename, 'rb') + try: + return cls(fp, filename, compression='bz2') + finally: + fp.close() + else: + with open(filename, 'rb') as fp: + return cls(fp, filename) + + def __init__(self, stream_string_or_array, filename=None, compression=None): + """ + Create an Image object. + """ + + if isinstance(stream_string_or_array, string_types): + error_msg = ( + 'A file like object is expected for stream. ' + 'Use %s.open(filename) to open a image file.' + ) + raise TypeError(error_msg % type(self).__name__) + if isinstance(stream_string_or_array, numpy.ndarray): + self.filename = None + self.compression = None + self.data = stream_string_or_array + self.label = self._create_label(stream_string_or_array) + else: + #: The filename if given, otherwise none. + self.filename = filename + + self.compression = compression + + # TODO: rename to header and add footer? + #: The parsed label header in dictionary form. + self.label = self._load_label(stream_string_or_array) + + #: A numpy array representing the image + self.data = self._load_data(stream_string_or_array) + + def __repr__(self): + # TODO: pick a better repr + return self.filename + + def save(self, file_to_write=None, overwrite=False): + self._save(file_to_write, overwrite) + + @property + def image(self): + """An Image like array of ``self.data`` convenient for image processing tasks + + * 2D array for single band, grayscale image data + * 3D array for three band, RGB image data + + Enables working with ``self.data`` as if it were a PIL image. + + See https://planetaryimage.readthedocs.io/en/latest/usage.html to see + how to open images to view them and make manipulations. + + """ + if self.bands == 1: + return self.data.squeeze() + elif self.bands == 3: + return numpy.dstack(self.data) + # TODO: what about multiband images with 2, and 4+ bands? + + @property + def bands(self): + """Number of image bands.""" + return self._bands + + @property + def lines(self): + """Number of lines per band.""" + return self._lines + + @property + def samples(self): + """Number of samples per line.""" + return self._samples + + @property + def format(self): + """Image format.""" + return self._format + + _data_filename = None + + @property + def data_filename(self): + """Return detached filename else None.""" + return self._data_filename + + @property + def dtype(self): + """Pixel data type.""" + return self._dtype + + @property + def start_byte(self): + """Index of the start of the image data (zero indexed).""" + return self._start_byte + + @property + def shape(self): + """Tuple of images bands, lines and samples.""" + return (self.bands, self.lines, self.samples) + + @property + def size(self): + """Total number of pixels""" + return self.bands * self.lines * self.samples + + def _load_label(self, stream): + return load(stream) + + def _load_data(self, stream): + if self.data_filename is not None: + return self._load_detached_data() + + stream.seek(self.start_byte) + return self._decode(stream) + + def create_label(self, array): + self._create_label(array) + + def _decode(self, stream): + return self._decoder.decode(stream) + + def _load_detached_data(self): + dirpath = os.path.dirname(self.filename) + filename = os.path.abspath(os.path.join(dirpath, self.data_filename)) + + with open(filename, 'rb') as stream: + return self._decode(stream) diff --git a/hirise_blender/planetaryimage/pds3image.py b/hirise_blender/planetaryimage/pds3image.py new file mode 100644 index 0000000..e53fe14 --- /dev/null +++ b/hirise_blender/planetaryimage/pds3image.py @@ -0,0 +1,349 @@ +# -*- coding: utf-8 -*- +import numpy +from ..six import string_types, integer_types +import os +from ..pvl import dump, dumps, encoder, load, PVLModule, Units +import collections + +from .image import PlanetaryImage +from .decoders import BandSequentialDecoder + + +class Pointer(collections.namedtuple('Pointer', ['filename', 'bytes'])): + @staticmethod + def _parse_bytes(value, record_bytes): + if isinstance(value, integer_types): + return (value - 1) * record_bytes + + if isinstance(value, Units) and value.units == 'BYTES': + return value.value + + raise ValueError('Unsupported pointer type') + + @classmethod + def parse(cls, value, record_bytes): + """Parses the pointer label. + + Parameters + ---------- + pointer_data + Supported values for `pointer_data` are:: + + ^PTR = nnn + ^PTR = nnn + ^PTR = "filename" + ^PTR = ("filename") + ^PTR = ("filename", nnn) + ^PTR = ("filename", nnn ) + + record_bytes + Record multiplier value + + Returns + ------- + Pointer object + """ + if isinstance(value, string_types): + return cls(value, 0) + + if isinstance(value, list): + if len(value) == 1: + return cls(value[0], 0) + + if len(value) == 2: + return cls(value[0], cls._parse_bytes(value[1], record_bytes)) + + raise ValueError('Unsupported pointer type') + + return cls(None, cls._parse_bytes(value, record_bytes)) + + +class PDS3Image(PlanetaryImage): + """A PDS3 image reader. + + Examples + -------- + >>> from planetaryimage import PDS3Image + >>> testfile = 'tests/mission_data/2p129641989eth0361p2600r8m1.img' + >>> image = PDS3Image.open(testfile) + >>> # Examples of PDS3Image Attributes + >>> image.dtype + dtype('>i2') + >>> image.record_bytes + 128 + >>> image.data_filename + + """ + + SAMPLE_TYPES = { + 'MSB_INTEGER': '>i', + 'INTEGER': '>i', + 'MAC_INTEGER': '>i', + 'SUN_INTEGER': '>i', + + 'MSB_UNSIGNED_INTEGER': '>u', + 'UNSIGNED_INTEGER': '>u', + 'MAC_UNSIGNED_INTEGER': '>u', + 'SUN_UNSIGNED_INTEGER': '>u', + + 'LSB_INTEGER': 'f', + 'FLOAT': '>f', + 'REAL': '>f', + 'MAC_REAL': '>f', + 'SUN_REAL': '>f', + + 'IEEE_COMPLEX': '>c', + 'COMPLEX': '>c', + 'MAC_COMPLEX': '>c', + 'SUN_COMPLEX': '>c', + + 'PC_REAL': 'S', + 'LSB_BIT_STRING': 'i': 'MSB_INTEGER', + '>u': 'MSB_UNSIGNED_INTEGER', + 'f': 'IEEE_REAL', + '>c': 'IEEE_COMPLEX', + 'S': 'MSB_BIT_STRING', + ' 1 and self._format != 'BAND_SEQUENTIAL'): + raise NotImplementedError + else: + self.data.tofile(stream, format='%' + self.dtype.kind) + stream.close() + + def _create_label(self, array): + """Create sample PDS3 label for NumPy Array. + It is called by 'image.py' to create PDS3Image object + from Numpy Array. + + Returns + ------- + PVLModule label for the given NumPy array. + + Usage: self.label = _create_label(array) + + """ + if len(array.shape) == 3: + bands = array.shape[0] + lines = array.shape[1] + line_samples = array.shape[2] + else: + bands = 1 + lines = array.shape[0] + line_samples = array.shape[1] + record_bytes = line_samples * array.itemsize + label_module = PVLModule([ + ('PDS_VERSION_ID', 'PDS3'), + ('RECORD_TYPE', 'FIXED_LENGTH'), + ('RECORD_BYTES', record_bytes), + ('LABEL_RECORDS', 1), + ('^IMAGE', 1), + ('IMAGE', + {'BANDS': bands, + 'LINES': lines, + 'LINE_SAMPLES': line_samples, + 'MAXIMUM': 0, + 'MEAN': 0, + 'MEDIAN': 0, + 'MINIMUM': 0, + 'SAMPLE_BITS': array.itemsize * 8, + 'SAMPLE_TYPE': 'MSB_INTEGER', + 'STANDARD_DEVIATION': 0}) + ]) + return self._update_label(label_module, array) + + def _update_label(self, label, array): + """Update PDS3 label for NumPy Array. + It is called by '_create_label' to update label values + such as, + - ^IMAGE, RECORD_BYTES + - STANDARD_DEVIATION + - MAXIMUM, MINIMUM + - MEDIAN, MEAN + + Returns + ------- + Update label module for the NumPy array. + + Usage: self.label = self._update_label(label, array) + + """ + maximum = float(numpy.max(array)) + mean = float(numpy.mean(array)) + median = float(numpy.median(array)) + minimum = float(numpy.min(array)) + stdev = float(numpy.std(array, ddof=1)) + + encoder = encoder.PDSLabelEncoder + serial_label = dumps(label, cls=encoder) + label_sz = len(serial_label) + image_pointer = int(label_sz / label['RECORD_BYTES']) + 1 + + label['^IMAGE'] = image_pointer + 1 + label['LABEL_RECORDS'] = image_pointer + label['IMAGE']['MEAN'] = mean + label['IMAGE']['MAXIMUM'] = maximum + label['IMAGE']['MEDIAN'] = median + label['IMAGE']['MINIMUM'] = minimum + label['IMAGE']['STANDARD_DEVIATION'] = stdev + + return label + + @property + def _bands(self): + try: + if len(self.data.shape) == 3: + return self.data.shape[0] + else: + return 1 + except AttributeError: + return self.label['IMAGE'].get('BANDS', 1) + + @property + def _lines(self): + try: + if len(self.data.shape) == 3: + return self.data.shape[1] + else: + return self.data.shape[0] + except AttributeError: + return self.label['IMAGE']['LINES'] + + @property + def _samples(self): + try: + if len(self.data.shape) == 3: + return self.data.shape[2] + else: + return self.data.shape[1] + except AttributeError: + return self.label['IMAGE']['LINE_SAMPLES'] + + @property + def _format(self): + return self.label['IMAGE'].get('BAND_STORAGE_TYPE', 'BAND_SEQUENTIAL') + + @property + def _start_byte(self): + return self._image_pointer.bytes + + @property + def _data_filename(self): + return self._image_pointer.filename + + @property + def _dtype(self): + return self._pixel_type.newbyteorder(self._byte_order) + + @property + def record_bytes(self): + """Number of bytes for fixed length records.""" + return self.label.get('RECORD_BYTES', 0) + + @property + def _image_pointer(self): + return Pointer.parse(self.label['^IMAGE'], self.record_bytes) + + @property + def _sample_type(self): + sample_type = self.label['IMAGE']['SAMPLE_TYPE'] + try: + return self.SAMPLE_TYPES[sample_type] + except KeyError: + raise ValueError('Unsupported sample type: %r' % sample_type) + + @property + def _sample_bytes(self): + # get bytes to match NumPy dtype expressions + try: + return self.data.itemsize + except AttributeError: + return int(self.label['IMAGE']['SAMPLE_BITS'] / 8) + + # FIXME: This dtype overrides the Image.dtype right? Then whats the point + # of _dtype above here ^^, should we just rename this one _dtype and remove + # the other one? + @property + def dtype(self): + """Pixel data type.""" + try: + return self.data.dtype + except AttributeError: + return numpy.dtype('%s%d' % (self._sample_type, self._sample_bytes)) + + @property + def _decoder(self): + if self.format == 'BAND_SEQUENTIAL': + return BandSequentialDecoder( + self.dtype, self.shape, self.compression + ) + raise ValueError('Unkown format (%s)' % self.format) diff --git a/hirise_blender/planetaryimage/specialpixels.py b/hirise_blender/planetaryimage/specialpixels.py new file mode 100644 index 0000000..cfbb8f3 --- /dev/null +++ b/hirise_blender/planetaryimage/specialpixels.py @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- + +""" Constants for Isis Special Pixels. + + Min: The minimum valid value for a pixel. + Null: Pixel has no data available. + Lis: Pixel was lower bound saturated on the instrument. + His: Pixel was higher bound saturated on the instrument. + Lrs: Pixel was lower bound saturated during a computation. + Hrs: Pixel was higher bound saturated during a computation. + Max: The maximum valid value for a pixel. +""" + +import numpy + +__all__ = ['SPECIAL_PIXELS'] + + +def _make_num(num, dtype): + return numpy.fromstring(num, dtype=dtype)[0] + + +SPECIAL_PIXELS = { + + 'UnsignedByte': { + 'Min': 1, + 'Null': 0, + 'Lrs': 0, + 'Lis': 0, + 'His': 255, + 'Hrs': 255, + 'Max': 254 + }, + + 'UnsignedWord': { + 'Min': 3, + 'Null': 0, + 'Lrs': 1, + 'Lis': 2, + 'His': 65534, + 'Hrs': 65535, + 'Max': 65522 + }, + + 'SignedWord': { + 'Min': -32752, + 'Null': -32768, + 'Lrs': -32767, + 'Lis': -32766, + 'His': -32765, + 'Hrs': -32764, + 'Max': 32767 + }, + + 'SignedInteger': { + 'Min': -8388614, + 'Null': -8388613, + 'Lrs': -8388612, + 'Lis': -8388611, + 'His': -8388610, + 'Hrs': -8388609, + 'Max': 2147483647 + }, + + 'Real': { + 'Min': _make_num(b'\xFF\x7F\xFF\xFA', '>f4'), + 'Null': _make_num(b'\xFF\x7F\xFF\xFB', '>f4'), + 'Lrs': _make_num(b'\xFF\x7F\xFF\xFC', '>f4'), + 'Lis': _make_num(b'\xFF\x7F\xFF\xFD', '>f4'), + 'His': _make_num(b'\xFF\x7F\xFF\xFE', '>f4'), + 'Hrs': _make_num(b'\xFF\x7F\xFF\xFF', '>f4'), + 'Max': numpy.finfo('f4').max + }, + + 'Double': { + 'Min': _make_num(b'\xFF\xEF\xFF\xFF\xFF\xFF\xFF\xFA', '>f8'), + 'Null': _make_num(b'\xFF\xEF\xFF\xFF\xFF\xFF\xFF\xFB', '>f8'), + 'Lrs': _make_num(b'\xFF\xEF\xFF\xFF\xFF\xFF\xFF\xFC', '>f8'), + 'Lis': _make_num(b'\xFF\xEF\xFF\xFF\xFF\xFF\xFF\xFD', '>f8'), + 'His': _make_num(b'\xFF\xEF\xFF\xFF\xFF\xFF\xFF\xFE', '>f8'), + 'Hrs': _make_num(b'\xFF\xEF\xFF\xFF\xFF\xFF\xFF\xFF', '>f8'), + 'Max': numpy.finfo('f8').max + } +} diff --git a/hirise_blender/pvl/__init__.py b/hirise_blender/pvl/__init__.py new file mode 100644 index 0000000..8b71efd --- /dev/null +++ b/hirise_blender/pvl/__init__.py @@ -0,0 +1,268 @@ +# -*- coding: utf-8 -*- +"""Python implementation of PVL (Parameter Value Language).""" + +# Copyright 2015, 2017, 2019-2021, ``pvl`` library authors. +# +# Reuse is permitted under the terms of the license. +# The AUTHORS file and the LICENSE file are at the +# top level of this library. + +import inspect +import io +import urllib.request +from pathlib import Path + +from .encoder import PDSLabelEncoder, PVLEncoder +from .parser import PVLParser, OmniParser +from .collections import ( + PVLModule, + PVLGroup, + PVLObject, + Quantity, + Units, +) + +__author__ = "The pvl Developers" +__email__ = "rbeyer@rossbeyer.net" +__version__ = "1.3.2" +__all__ = [ + "load", + "loads", + "dump", + "dumps", + "PVLModule", + "PVLGroup", + "PVLObject", + "Quantity", + "Units", +] + + +def load( + path, + parser=None, + grammar=None, + decoder=None, + encoding=None, + **kwargs +): + """Returns a Python object from parsing the file at *path*. + + :param path: an :class:`os.PathLike` which presumably has a + PVL Module in it to parse. + :param parser: defaults to :class:`pvl.parser.OmniParser()`. + :param grammar: defaults to :class:`pvl.grammar.OmniGrammar()`. + :param decoder: defaults to :class:`pvl.decoder.OmniDecoder()`. + :param encoding: defaults to None, and has the same meaning as + for :py:func:`open()`. + :param ``**kwargs``: the keyword arguments that will be passed + to :func:`loads()` and are described there. + + If *path* is not an :class:`os.PathLike`, it will be assumed to be an + already-opened file object, and ``.read()`` will be applied + to extract the text. + + If the :class:`os.PathLike` or file object contains some bytes + decodable as text, followed by some that is not (e.g. an ISIS + cube file), that's fine, this function will just extract the + decodable text. + """ + return loads( + get_text_from(path, encoding=encoding), + parser=parser, + grammar=grammar, + decoder=decoder, + **kwargs + ) + + +def get_text_from(path, encoding=None) -> str: + try: + p = Path(path) + return p.read_text(encoding=encoding) + except UnicodeDecodeError: + # This may be the result of an ISIS cube file (or anything else) + # where the first set of bytes might be decodable, but once the + # image data starts, they won't be, and the above tidy function + # fails. So open the file as a bytestream, and read until + # we can't decode. We don't want to just run the .read_bytes() + # method of Path, because this could be a giant file. + with open(path, mode="rb") as f: + return decode_by_char(f) + except TypeError: + # Not an os.PathLike, maybe it is an already-opened file object + if path.readable(): + position = path.tell() + try: + s = path.read() + if isinstance(s, bytes): + # Oh, it was opened in 'b' mode, need to rewind and + # decode. Since the 'catch' below already does that, + # we'll just emit a ... contrived ... UnicodeDecodeError + # so we don't have to double-write the code: + raise UnicodeDecodeError( + "utf_8", + "dummy".encode(), + 0, + 1, + "file object in byte mode", + ) + except UnicodeDecodeError: + # All of the bytes weren't decodeable, maybe the initial + # sequence is (as above)? + path.seek(position) # Reset after the previous .read(): + s = decode_by_char(path) + + else: + # Not a path, not an already-opened file. + raise TypeError( + "Expected an os.PathLike or an already-opened " + "file object, but did not get either." + ) + return s + + +def decode_by_char(f: io.RawIOBase) -> str: + """Returns a ``str`` decoded from the characters in *f*. + + :param f: is expected to be a file object which has been + opened in binary mode ('rb') or just read mode ('r'). + + The *f* stream will have one character or byte at a time read from it, + and will attempt to decode each to a string and accumulate + those individual strings together. Once the end of the file is found + or an element can no longer be decoded as UTF, the accumulated string will + be returned. + """ + s = "" + try: + for elem in iter(lambda: f.read(1), b""): + if isinstance(elem, str): + if elem == "": + break + s += elem + else: + s += elem.decode() + + except UnicodeError: + # Expecting this to mean that we got to the end of decodable + # bytes, so we're all done, and pass through to return s. + pass + + return s + + +def loadu(url, parser=None, grammar=None, decoder=None, **kwargs): + """Returns a Python object from parsing *url*. + + :param url: this will be passed to :func:`urllib.request.urlopen` + and can be a string or a :class:`urllib.request.Request` object. + :param parser: defaults to :class:`pvl.parser.OmniParser()`. + :param grammar: defaults to :class:`pvl.grammar.OmniGrammar()`. + :param decoder: defaults to :class:`pvl.decoder.OmniDecoder()`. + :param ``**kwargs``: the keyword arguments that will be passed + to :func:`urllib.request.urlopen` and to :func:`loads()`. + + The ``**kwargs`` will first be scanned for arguments that + can be given to :func:`urllib.request.urlopen`. If any are + found, they are extracted and used. All remaining elements + will be passed on as keyword arguments to :func:`loads()`. + + Note that *url* can be any URL that :func:`urllib.request.urlopen` + takes. Certainly http and https URLs, but also file, ftp, rsync, + sftp and more! + """ + + # Peel off the args for urlopen: + url_args = dict() + for a in inspect.signature(urllib.request.urlopen).parameters.keys(): + if a in kwargs: + url_args[a] = kwargs.pop(a) + + # The object returned from urlopen will always have a .read() + # function that returns bytes, so: + with urllib.request.urlopen(url, **url_args) as resp: + s = decode_by_char(resp) + + return loads(s, parser=parser, grammar=grammar, decoder=decoder, **kwargs) + + +def loads(s: str, parser=None, grammar=None, decoder=None, **kwargs): + """Deserialize the string, *s*, as a Python object. + + :param s: contains some PVL to parse. + :param parser: defaults to :class:`pvl.parser.OmniParser()`. + :param grammar: defaults to :class:`pvl.grammar.OmniGrammar()`. + :param decoder: defaults to :class:`pvl.decoder.OmniDecoder()`. + :param ``**kwargs``: the keyword arguments to pass to the *parser* class + if *parser* is none. + """ + # decoder = __create_decoder(cls, strict, grammar=grammar, **kwargs) + # return decoder.decode(s) + + if isinstance(s, bytes): + # Someone passed us an old-style bytes sequence. Although it isn't + # a string, we can deal with it: + s = s.decode() + + if parser is None: + parser = OmniParser(grammar=grammar, decoder=decoder, **kwargs) + elif not isinstance(parser, PVLParser): + raise TypeError("The parser must be an instance of pvl.PVLParser.") + + return parser.parse(s) + + +def dump(module, path, **kwargs): + """Serialize *module* as PVL text to the provided *path*. + + :param module: a ``PVLModule`` or ``dict``-like object to serialize. + :param path: an :class:`os.PathLike` + :param ``**kwargs``: the keyword arguments to pass to :func:`dumps()`. + + If *path* is an :class:`os.PathLike`, it will attempt to be opened + and the serialized module will be written into that file via + the :func:`pathlib.Path.write_text()` function, and will return + what that function returns. + + If *path* is not an :class:`os.PathLike`, it will be assumed to be an + already-opened file object, and ``.write()`` will be applied + on that object to write the serialized module, and will return + what that function returns. + """ + try: + p = Path(path) + return p.write_text(dumps(module, **kwargs)) + + except TypeError: + # Not an os.PathLike, maybe it is an already-opened file object + try: + if isinstance(path, io.TextIOBase): + return path.write(dumps(module, **kwargs)) + else: + return path.write(dumps(module, **kwargs).encode()) + except AttributeError: + # Not a path, not an already-opened file. + raise TypeError( + "Expected an os.PathLike or an already-opened " + "file object for writing, but got neither." + ) + + +def dumps(module, encoder=None, grammar=None, decoder=None, **kwargs) -> str: + """Returns a string where the *module* object has been serialized + to PVL syntax. + + :param module: a ``PVLModule`` or ``dict`` like object to serialize. + :param encoder: defaults to :class:`pvl.parser.PDSLabelEncoder()`. + :param grammar: defaults to :class:`pvl.grammar.ODLGrammar()`. + :param decoder: defaults to :class:`pvl.decoder.ODLDecoder()`. + :param ``**kwargs``: the keyword arguments to pass to the encoder + class if *encoder* is none. + """ + if encoder is None: + encoder = PDSLabelEncoder(grammar=grammar, decoder=decoder, **kwargs) + elif not isinstance(encoder, PVLEncoder): + raise TypeError("The encoder must be an instance of pvl.PVLEncoder.") + + return encoder.encode(module) diff --git a/hirise_blender/pvl/__pycache__/__init__.cpython-310.pyc b/hirise_blender/pvl/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..eb73bdf Binary files /dev/null and b/hirise_blender/pvl/__pycache__/__init__.cpython-310.pyc differ diff --git a/hirise_blender/pvl/__pycache__/collections.cpython-310.pyc b/hirise_blender/pvl/__pycache__/collections.cpython-310.pyc new file mode 100644 index 0000000..385e7a9 Binary files /dev/null and b/hirise_blender/pvl/__pycache__/collections.cpython-310.pyc differ diff --git a/hirise_blender/pvl/__pycache__/decoder.cpython-310.pyc b/hirise_blender/pvl/__pycache__/decoder.cpython-310.pyc new file mode 100644 index 0000000..fc4bb07 Binary files /dev/null and b/hirise_blender/pvl/__pycache__/decoder.cpython-310.pyc differ diff --git a/hirise_blender/pvl/__pycache__/encoder.cpython-310.pyc b/hirise_blender/pvl/__pycache__/encoder.cpython-310.pyc new file mode 100644 index 0000000..5b24167 Binary files /dev/null and b/hirise_blender/pvl/__pycache__/encoder.cpython-310.pyc differ diff --git a/hirise_blender/pvl/__pycache__/exceptions.cpython-310.pyc b/hirise_blender/pvl/__pycache__/exceptions.cpython-310.pyc new file mode 100644 index 0000000..76ada5c Binary files /dev/null and b/hirise_blender/pvl/__pycache__/exceptions.cpython-310.pyc differ diff --git a/hirise_blender/pvl/__pycache__/grammar.cpython-310.pyc b/hirise_blender/pvl/__pycache__/grammar.cpython-310.pyc new file mode 100644 index 0000000..4d072ac Binary files /dev/null and b/hirise_blender/pvl/__pycache__/grammar.cpython-310.pyc differ diff --git a/hirise_blender/pvl/__pycache__/lexer.cpython-310.pyc b/hirise_blender/pvl/__pycache__/lexer.cpython-310.pyc new file mode 100644 index 0000000..acde577 Binary files /dev/null and b/hirise_blender/pvl/__pycache__/lexer.cpython-310.pyc differ diff --git a/hirise_blender/pvl/__pycache__/parser.cpython-310.pyc b/hirise_blender/pvl/__pycache__/parser.cpython-310.pyc new file mode 100644 index 0000000..2ea09c7 Binary files /dev/null and b/hirise_blender/pvl/__pycache__/parser.cpython-310.pyc differ diff --git a/hirise_blender/pvl/__pycache__/token.cpython-310.pyc b/hirise_blender/pvl/__pycache__/token.cpython-310.pyc new file mode 100644 index 0000000..a50471e Binary files /dev/null and b/hirise_blender/pvl/__pycache__/token.cpython-310.pyc differ diff --git a/hirise_blender/pvl/collections.py b/hirise_blender/pvl/collections.py new file mode 100644 index 0000000..4687f2b --- /dev/null +++ b/hirise_blender/pvl/collections.py @@ -0,0 +1,704 @@ +# -*- coding: utf-8 -*- +"""Parameter Value Language container datatypes providing enhancements +to Python general purpose built-in containers. + +To enable efficient operations on parsed PVL text, we need an object +that acts as both a dict-like Mapping container and a list-like +Sequence container, essentially an ordered multi-dict. There is +no existing object or even an Abstract Base Class in the Python +Standard Library for such an object. So we define the +MutableMappingSequence ABC here, which is (as the name implies) an +abstract base class that implements both the Python MutableMapping +and Mutable Sequence ABCs. We also provide two implementations, the +OrderedMultiDict, and the newer PVLMultiDict. + +Additionally, for PVL Values which also have an associated PVL Units +Expression, they need to be returned as a quantity object which contains +both a notion of a value and the units for that value. Again, there +is no fundamental Python type for a quantity, so we define the Quantity +class (formerly the Units class). +""" +# Copyright 2015, 2017, 2019-2021, ``pvl`` library authors. +# +# Reuse is permitted under the terms of the license. +# The AUTHORS file and the LICENSE file are at the +# top level of this library. + +import pprint +import warnings +from abc import abstractmethod +from collections import namedtuple, abc + + +class MutableMappingSequence(abc.MutableMapping, abc.MutableSequence): + """ABC for a mutable object that has both mapping and + sequence characteristics. + + Must implement `.getall(k)` and `.popall(k)` since a MutableMappingSequence + can have many values for a single key, while `.get(k)` and + `.pop(k)` return and operate on a single value, the *all* + versions return and operate on all values in the MutableMappingSequence + with the key `k`. + + Furthermore, `.pop()` without an argument should function as the + MutableSequence pop() function and pop the last value when considering + the MutableMappingSequence in a list-like manner. + """ + + @abstractmethod + def append(self, key, value): + pass + + @abstractmethod + def getall(self, key): + pass + + @abstractmethod + def popall(self, key): + pass + + +dict_setitem = dict.__setitem__ +dict_getitem = dict.__getitem__ +dict_delitem = dict.__delitem__ +dict_contains = dict.__contains__ +dict_clear = dict.clear + + +class MappingView(object): + def __init__(self, mapping): + self._mapping = mapping + + def __len__(self): + return len(self._mapping) + + def __repr__(self): + return "{!s}({!r})".format(type(self).__name__, self._mapping) + + +class KeysView(MappingView): + def __contains__(self, key): + return key in self._mapping + + def __iter__(self): + for key, _ in self._mapping: + yield key + + def __getitem__(self, index): + return self._mapping[index][0] + + def __repr__(self): + keys = [key for key, _ in self._mapping] + return "{!s}({!r})".format(type(self).__name__, keys) + + def index(self, key): + keys = [k for k, _ in self._mapping] + return keys.index(key) + + +class ItemsView(MappingView): + def __contains__(self, item): + key, value = item + return value in self._mapping.getlist(key) + + def __iter__(self): + for item in self._mapping: + yield item + + def __getitem__(self, index): + return self._mapping[index] + + def index(self, item): + items = [i for i in self._mapping] + return items.index(item) + + +class ValuesView(MappingView): + def __contains__(self, value): + for _, v in self._mapping: + if v == value: + return True + return False + + def __iter__(self): + for _, value in self._mapping: + yield value + + def __getitem__(self, index): + return self._mapping[index][1] + + def __repr__(self): + values = [value for _, value in self._mapping] + return "{!s}({!r})".format(type(self).__name__, values) + + def index(self, value): + values = [val for _, val in self._mapping] + return values.index(value) + + +class OrderedMultiDict(dict, MutableMappingSequence): + """A ``dict`` like container. + + This container preserves the original ordering as well as + allows multiple values for the same key. It provides similar + semantics to a ``list`` of ``tuples`` but with ``dict`` style + access. + + Using ``__setitem__`` syntax overwrites all fields with the + same key and ``__getitem__`` will return the first value with + the key. + """ + + def __init__(self, *args, **kwargs): + self.__items = [] + self.extend(*args, **kwargs) + + def __setitem__(self, key, value): + if key not in self: + return self.append(key, value) + + dict_setitem(self, key, [value]) + iteritems = iter(self.__items) + + for index, (old_key, old_value) in enumerate(iteritems): + if old_key == key: + # replace first occurrence + self.__items[index] = (key, value) + break + + tail = [item for item in iteritems if item[0] != key] + self.__items[index + 1:] = tail + + def __getitem__(self, key): + if isinstance(key, (int, slice)): + return self.__items[key] + return dict_getitem(self, key)[0] + + def __delitem__(self, key): + dict_delitem(self, key) + self.__items = [item for item in self.__items if item[0] != key] + + def __iter__(self): + return iter(self.__items) + + def __len__(self): + return len(self.__items) + + def __eq__(self, other): + if not isinstance(other, type(self)): + return False + + if len(self) != len(other): + return False + + items1 = self.items() + items2 = other.items() + + for ((key1, value1), (key2, value2)) in zip(items1, items2): + if key1 != key2: + return False + + if value1 != value2: + return False + + return True + + def __ne__(self, other): + return not (self == other) + + def __repr__(self): + if not self.__items: + return "{!s}([])".format(type(self).__name__) + + lines = [] + for item in self.__items: + for line in pprint.pformat(item).splitlines(): + lines.append(" " + line) + + return "{!s}([\n{!s}\n])".format(type(self).__name__, "\n".join(lines)) + + get = abc.MutableMapping.get + update = abc.MutableMapping.update + + def keys(self): + return KeysView(self) + + def values(self): + return ValuesView(self) + + def items(self): + return ItemsView(self) + + def clear(self): + dict_clear(self) + self.__items = [] + + def discard(self, key): + + warnings.warn( + "The discard(k) function is deprecated in favor of .popall(k), " + "please begin using it, as .discard(k) may be removed in the " + "next major patch.", + PendingDeprecationWarning, + ) + + try: + del self[key] + except KeyError: + pass + + def append(self, key, value): + """Adds a (name, value) pair, doesn't overwrite the value if + it already exists. + """ + self.__items.append((key, value)) + + try: + dict_getitem(self, key).append(value) + except KeyError: + dict_setitem(self, key, [value]) + + def extend(self, *args, **kwargs): + """Add key value pairs for an iterable.""" + if len(args) > 1: + raise TypeError(f"expected at most 1 arguments, got {len(args)}") + + iterable = args[0] if args else None + if iterable: + if isinstance(iterable, abc.Mapping) or hasattr(iterable, "items"): + for key, value in iterable.items(): + self.append(key, value) + else: + for key, value in iterable: + self.append(key, value) + + for key, value in kwargs.items(): + self.append(key, value) + + def getall(self, key) -> abc.Sequence: + """Returns a list of all the values for a named field. + Returns KeyError if the key doesn't exist. + """ + return list(dict_getitem(self, key)) + + def getlist(self, key) -> abc.Sequence: + """Returns a list of all the values for the named field. + Returns an empty list if the key doesn't exist. + """ + warnings.warn( + "The getlist() function is deprecated in favor of .getall(), " + "please begin using it, as .getlist() may be removed in the " + "next major patch.", + PendingDeprecationWarning, + ) + + try: + return self.getall(key) + except KeyError: + return [] + + # Turns out that this super-class function, even though it doesn't have + # a concept of multiple keys, clears out multiple elements with the key, + # probably because of how __delitem__ is defined: + popall = abc.MutableMapping.pop + + def pop(self, *args, **kwargs): + """Removes all items with the specified *key*.""" + + if len(args) == 0 and len(kwargs) == 0: + if not self: + raise KeyError( + "pop(): {!s} ".format(type(self).__name__) + "is empty" + ) + + key, _ = item = self.__items.pop() + values = dict_getitem(self, key) + values.pop() + + if not values: + dict_delitem(self, key) + + return item + + warnings.warn( + "The pop(k) function removes " + "all keys with value k to remain backwards compatible with the " + "pvl 0.x architecture, this concept of " + "operations for .pop(k) may change in future versions. " + "Consider using .popall(k) instead.", + FutureWarning, + ) + + return self.popall(*args, *kwargs) + + def popitem(self): + + warnings.warn( + "The popitem() function removes " + "and returns the last key, value pair to remain backwards " + "compatible with the pvl 0.x architecture, this concept of " + "operations for .popitem() may change in future versions. " + "Consider using the list-like .pop(), without an argument instead.", + FutureWarning, + ) + return self.pop() + + def copy(self): + return type(self)(self) + + def insert(self, index: int, *args) -> None: + """Inserts at the index given by *index*. + + The first positional argument will always be taken as the + *index* for insertion. + + If three arguments are given, the second will be taken + as the *key*, and the third as the *value* to insert. + + If only two arguments are given, the second must be a sequence. + + If it is a sequence of pairs (such that every item in the sequence is + itself a sequence of length two), that sequence will be inserted + as key, value pairs. + + If it happens to be a sequence of two items (the first of which is + not a sequence), the first will be taken as the *key* and the + second the *value* to insert. + """ + + if not isinstance(index, int): + raise TypeError( + "The first positional argument to pvl.MultiDict.insert()" + "must be an int." + ) + + kvlist = _insert_arg_helper(args) + + for (key, value) in kvlist: + self.__items.insert(index, (key, value)) + index += 1 + + # Make sure indexing works with the new item + if key in self: + value_list = [val for k, val in self.__items if k == key] + dict_setitem(self, key, value_list) + else: + dict_setitem(self, key, [value]) + + return + + def key_index(self, key, instance: int = 0) -> int: + """Get the index of the key to insert before or after.""" + if key not in self: + raise KeyError(str(key)) + + idxs = list() + for idx, k in enumerate(self.keys()): + if key == k: + idxs.append(idx) + + try: + return idxs[instance] + except IndexError: + raise IndexError( + f"There are only {len(idxs)} elements with the key {key}, " + f"the provided index ({instance}) is out of bounds." + ) + + def insert_after(self, key, new_item: abc.Iterable, instance=0): + """Insert an item after a key""" + index = self.key_index(key, instance) + self.insert(index + 1, new_item) + + def insert_before(self, key, new_item: abc.Iterable, instance=0): + """Insert an item before a key""" + index = self.key_index(key, instance) + self.insert(index, new_item) + + +def _insert_arg_helper(args): + # Helper function to un-mangle the many and various ways that + # key, value pairs could be provided to the .insert() functions. + # Takes all of them, and returns a list of key, value pairs, even + # if there is only one. + if len(args) == 1: + if not isinstance(args, (abc.Sequence, abc.Mapping)): + raise TypeError( + "If a single argument is provided to the second positional " + "argument of insert(), it must have a Sequence or Mapping " + f"interface. Instead it was {type(args)}: {args}" + ) + + if isinstance(args[0], abc.Mapping): + return list(args[0].items()) + + else: + if len(args[0]) == 2 and ( + isinstance(args[0][0], str) + or not isinstance(args[0][0], abc.Sequence) + ): + kvlist = (args[0],) + else: + for pair in args[0]: + msg = ( + "One of the elements in the sequence passed to the " + "second argument of insert() " + ) + if not isinstance(pair, abc.Sequence): + raise TypeError( + msg + f"was not itself a sequence, it is: {pair}" + ) + if not len(pair) == 2: + raise TypeError( + msg + f"was not a pair of values, it is: {pair}" + ) + + kvlist = args[0] + + elif len(args) == 2: + kvlist = (args,) + else: + raise TypeError( + f"insert() takes 2 or 3 positional arguments ({len(args)} given)." + ) + + return kvlist + + +try: # noqa: C901 + # In order to access super class attributes for our derived class, we must + # import the native Python version, instead of the default Cython version. + from multidict._multidict_py import MultiDict + + class PVLMultiDict(MultiDict, MutableMappingSequence): + """This is a new class that may be implemented as the default + structure to be returned from the pvl loaders in the future (replacing + OrderedMultiDict). + + Here is a summary of the differences: + + * OrderedMultiDict.getall('k') where k is not in the structure returns + an empty list, PVLMultiDict.getall('k') properly returns a KeyError. + * The .items(), .keys(), and .values() are proper iterators + and don't return sequences like OrderedMultiDict did. + * Calling list() on an OrderedMultiDict returns a list of tuples, which + is like calling list() on the results of a dict.items() iterator. + Calling list() on a PVLMultiDict returns just a list of keys, + which is semantically identical to calling list() on a dict. + * OrderedMultiDict.pop(k) removed all keys that matched k, + PVLMultiDict.pop(k) just removes the first occurrence. + PVLMultiDict.popall(k) would pop all. + * OrderedMultiDict.popitem() removes the last item from the underlying + list, PVLMultiDict.popitem() removes an arbitrary key, value pair, + semantically identical to .popitem() on a dict. + * OrderedMultiDict.__repr__() and .__str__() return identical strings, + PVLMultiDict provides a .__str__() that is pretty-printed similar + to OrderedMultiDict, but also a .__repr__() with a more compact + representation. + * Equality is different: OrderedMultiDict has an isinstance() + check in the __eq__() operator, which I don't think was right, + since equality is about values, not about type. PVLMultiDict + has a value-based notion of equality. So an empty PVLGroup and an + empty PVLObject derived from PVLMultiDict could test equal, + but would fail an isinstance() check. + """ + + # Also evaluated the boltons.OrderedMultiDict, but its semantics were + # too different #52 + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def __getitem__(self, key): + # Allow list-like access of the underlying structure + if isinstance(key, (int, slice)): + return list(self.items())[key] + return super().__getitem__(key) + + def __repr__(self): + if len(self) == 0: + return f"{self.__class__.__name__}()" + + return ( + f"{self.__class__.__name__}(" + str(list(self.items())) + ")" + ) + + def __str__(self): + if len(self) == 0: + return self.__repr__() + + lines = [] + for item in self.items(): + for line in pprint.pformat(item).splitlines(): + lines.append(" " + line) + + return f"{self.__class__.__name__}([\n" + "\n".join(lines) + "\n])" + + def key_index(self, key, ith: int = 0) -> int: + """Returns the index of the item in the underlying list + implementation that is the *ith* value of that *key*. + + Effectively creates a list of all indexes that match *key*, and then + returns the original index of the *ith* element of that list. The + *ith* integer can be any positive or negative integer and follows + the rules for list indexes. + """ + if key not in self: + raise KeyError(str(key)) + idxs = list() + for idx, (k, v) in enumerate(self.items()): + if key == k: + idxs.append(idx) + + try: + return idxs[ith] + except IndexError: + raise IndexError( + f"There are only {len(idxs)} elements with the key {key}, " + f"the provided index ({ith}) is out of bounds." + ) + + def _insert_item( + self, key, new_item: abc.Iterable, instance: int, is_after: bool + ): + """Insert a new item before or after another item.""" + index = self.key_index(key, instance) + index = index + 1 if is_after else index + + if isinstance(new_item, abc.Mapping): + tuple_iter = tuple(new_item.items()) + else: + tuple_iter = new_item + self.insert(index, tuple_iter) + + def insert(self, index: int, *args) -> None: + """Inserts at the index given by *index*. + + The first positional argument will always be taken as the + *index* for insertion. + + If three arguments are given, the second will be taken + as the *key*, and the third as the *value* to insert. + + If only two arguments are given, the second must be a sequence. + + If it is a sequence of pairs (such that every item in the sequence + is itself a sequence of length two), that sequence will be inserted + as key, value pairs. + + If it happens to be a sequence of two items (the first of which is + not a sequence), the first will be taken as the *key* and the + second the *value* to insert. + """ + if not isinstance(index, int): + raise TypeError( + "The first positional argument to pvl.MultiDict.insert()" + "must be an int." + ) + + kvlist = _insert_arg_helper(args) + + for (key, value) in kvlist: + identity = self._title(key) + self._impl._items.insert( + index, (identity, self._key(key), value) + ) + self._impl.incr_version() + index += 1 + return + + def insert_after(self, key, new_item, instance=0): + """Insert an item after a key""" + self._insert_item(key, new_item, instance, True) + + def insert_before(self, key, new_item, instance=0): + """Insert an item before a key""" + self._insert_item(key, new_item, instance, False) + + def pop(self, *args, **kwargs): + """Returns a two-tuple or a single value, depending on how it is + called. + + If no arguments are given, it removes and returns the last key, + value pair (list-like behavior). + + If a *key* is given, the first instance of key is found and its + value is removed and returned. If *default* is not given and + *key* is not in the dictionary, a KeyError is raised, otherwise + *default* is returned (dict-like behavior). + """ + if len(args) == 0 and len(kwargs) == 0: + i, k, v = self._impl._items.pop() + self._impl.incr_version() + return i, v + else: + return super().pop(*args, **kwargs) + + def append(self, key, value): + # Not sure why super() decided to go with the set-like add() instead + # of the more appropriate list-like append(). Fixed it for them. + self.add(key, value) + + # New versions based on PVLMultiDict + class PVLModuleNew(PVLMultiDict): + pass + + class PVLAggregationNew(PVLMultiDict): + pass + + class PVLGroupNew(PVLAggregationNew): + pass + + class PVLObjectNew(PVLAggregationNew): + pass + + +except ImportError: + warnings.warn( + "The multidict library is not present, so the new PVLMultiDict " + "cannot be used. At this time, it is completely optional, and doesn't " + "impact the use of pvl.", + ImportWarning, + ) + + +class PVLModule(OrderedMultiDict): + pass + + +class PVLAggregation(OrderedMultiDict): + pass + + +class PVLGroup(PVLAggregation): + pass + + +class PVLObject(PVLAggregation): + pass + + +class Quantity(namedtuple("Quantity", ["value", "units"])): + """A simple collections.namedtuple object to contain + a value and units parameter. + + If you need more comprehensive units handling, you + may want to use the astropy.units.Quantity object, + the pint.Quantity object, or some other 3rd party + object. Please see the documentation on :doc:`quantities` + for how to use 3rd party Quantity objects with pvl. + """ + + def __int__(self): + return int(self.value) + + def __float__(self): + return float(self.value) + + +class Units(Quantity): + warnings.warn( + "The pvl.collections.Units object is deprecated, and may be removed at " + "the next major patch. Please use pvl.collections.Quantity instead.", + PendingDeprecationWarning, + ) diff --git a/hirise_blender/pvl/decoder.py b/hirise_blender/pvl/decoder.py new file mode 100644 index 0000000..c98dd86 --- /dev/null +++ b/hirise_blender/pvl/decoder.py @@ -0,0 +1,552 @@ +# -*- coding: utf-8 -*- +"""Parameter Value Language decoder. + +The definition of PVL used in this module is based on the Consultive +Committee for Space Data Systems, and their Parameter Value +Language Specification (CCSD0006 and CCSD0008), CCSDS 6441.0-B-2, +referred to as the Blue Book with a date of June 2000. + +A decoder deals with converting strings given to it (typically +by the parser) to the appropriate Python type. +""" +# Copyright 2015, 2017, 2019-2021, ``pvl`` library authors. +# +# Reuse is permitted under the terms of the license. +# The AUTHORS file and the LICENSE file are at the +# top level of this library. + +import re +from datetime import datetime, timedelta, timezone +from decimal import InvalidOperation +from itertools import repeat, chain +from warnings import warn + +from .grammar import PVLGrammar, ODLGrammar, PDSGrammar +from .collections import Quantity +from .exceptions import QuantityError + + +def for_try_except(exception, function, *iterable): + """Return the result of the first successful application of *function* + to an element of *iterable*. If the *function* raises an Exception + of type *exception*, it will continue to the next item of *iterable*. + If there are no successful applications an Exception of type + *exception* will be raised. + + If additional *iterable* arguments are passed, *function* must + take that many arguments and is applied to the items from + all iterables in parallel (like ``map()``). With multiple iterables, + the iterator stops when the shortest iterable is exhausted. + """ + for tup in zip(*iterable): + try: + return function(*tup) + except exception: + pass + + raise exception + + +class PVLDecoder(object): + """A decoder based on the rules in the CCSDS-641.0-B-2 'Blue Book' + which defines the PVL language. + + :param grammar: defaults to a :class:`pvl.grammar.PVLGrammar`, but can + be any object that implements the :class:`pvl.grammar` interface. + + :param quantity_cls: defaults to :class:`pvl.collections.Quantity`, but + could be any class object that takes two arguments, where the + first is the value, and the second is the units value. + + :param real_cls: defaults to :class:`float`, but could be any class object + that can be constructed from a `str` object. + """ + + def __init__(self, grammar=None, quantity_cls=None, real_cls=None): + self.errors = [] + + if grammar is None: + self.grammar = PVLGrammar() + elif isinstance(grammar, PVLGrammar): + self.grammar = grammar + else: + raise Exception + + if quantity_cls is None: + self.quantity_cls = Quantity + else: + self.quantity_cls = quantity_cls + + if real_cls is None: + self.real_cls = float + else: + self.real_cls = real_cls + + def decode(self, value: str): + """Returns a Python object based on *value*.""" + return self.decode_simple_value(value) + + def decode_simple_value(self, value: str): + """Returns a Python object based on *value*, assuming + that *value* can be decoded as a PVL Simple Value:: + + ::= ( | | ) + """ + if value.casefold() == self.grammar.none_keyword.casefold(): + return None + + if value.casefold() == self.grammar.true_keyword.casefold(): + return True + + if value.casefold() == self.grammar.false_keyword.casefold(): + return False + + for d in ( + self.decode_quoted_string, + self.decode_non_decimal, + self.decode_decimal, + self.decode_datetime, + ): + try: + return d(value) + except ValueError: + pass + + return self.decode_unquoted_string(value) + + def decode_unquoted_string(self, value: str) -> str: + """Returns a Python ``str`` if *value* can be decoded + as an unquoted string, based on this decoder's grammar. + Raises a ValueError otherwise. + """ + for coll in ( + ("a comment", chain.from_iterable(self.grammar.comments)), + ("some whitespace", self.grammar.whitespace), + ("a special character", self.grammar.reserved_characters), + ): + for item in coll[1]: + if item in value: + raise ValueError( + "Expected a Simple Value, but encountered " + f'{coll[0]} in "{self}": "{item}".' + ) + + agg_keywords = self.grammar.aggregation_keywords.items() + for kw in chain.from_iterable(agg_keywords): + if kw.casefold() == value.casefold(): + raise ValueError( + "Expected a Simple Value, but encountered " + f'an aggregation keyword: "{value}".' + ) + + for es in self.grammar.end_statements: + if es.casefold() == value.casefold(): + raise ValueError( + "Expected a Simple Value, but encountered " + f'an End-Statement: "{value}".' + ) + + # This try block is going to look illogical. But the decode + # rules for Unquoted Strings spell out the things that they + # cannot be, so if it *can* be a datetime, then it *can't* be + # an Unquoted String, which is why we raise if it succeeds, + # and pass if it fails: + try: + self.decode_datetime(value) + raise ValueError + except ValueError: + pass + + return str(value) + + def decode_quoted_string(self, value: str) -> str: + """Returns a Python ``str`` if *value* begins and ends + with matching quote characters based on this decoder's + grammar. Raises ValueError otherwise. + """ + for q in self.grammar.quotes: + if value.startswith(q) and value.endswith(q) and len(value) > 1: + return str(value[1:-1]) + raise ValueError(f'The object "{value}" is not a PVL Quoted String.') + + def decode_decimal(self, value: str): + """Returns a Python ``int`` or ``self.real_cls`` object, as appropriate + based on *value*. Raises a ValueError otherwise. + """ + # Returns int or real_cls + try: + return int(value, base=10) + except ValueError: + try: + return self.real_cls(str(value)) + except InvalidOperation as err: + raise ValueError from err + + def decode_non_decimal(self, value: str) -> int: + """Returns a Python ``int`` as decoded from *value* + on the assumption that *value* conforms to a + non-decimal integer value as defined by this decoder's + grammar, raises ValueError otherwise. + """ + # Non-Decimal (Binary, Hex, and Octal) + for nd_re in ( + self.grammar.binary_re, + self.grammar.octal_re, + self.grammar.hex_re, + ): + match = nd_re.fullmatch(value) + if match is not None: + d = match.groupdict("") + return int(d["sign"] + d["non_decimal"], base=int(d["radix"])) + raise ValueError + + def decode_datetime(self, value: str): # noqa: C901 + """Takes a string and attempts to convert it to the appropriate + Python ``datetime`` ``time``, ``date``, or ``datetime`` + type based on this decoder's grammar, or in one case, a ``str``. + + The PVL standard allows for the seconds value to range + from zero to 60, so that the 60 can accommodate leap + seconds. However, the Python ``datetime`` classes don't + support second values for more than 59 seconds. + + If a time with 60 seconds is encountered, it will not be + returned as a datetime object (since that is not representable + via Python datetime objects), but simply as a string. + + The user can then then try and use the ``time`` module + to parse this string into a ``time.struct_time``. We + chose not to do this with pvl because ``time.struct_time`` + is a full *datetime* like object, even if it parsed + only a *time* like object, the year, month, and day + values in the ``time.struct_time`` would default, which + could be misleading. + + Alternately, the pvl.grammar.PVLGrammar class contains + two regexes: ``leap_second_Ymd_re`` and ``leap_second_Yj_re`` + which could be used along with the ``re.match`` object's + ``groupdict()`` function to extract the string representations + of the various numerical values, cast them to the appropriate + numerical types, and do something useful with them. + """ + try: + # datetime.date objects will always be naive, so just return: + return for_try_except( + ValueError, + datetime.strptime, + repeat(value), + self.grammar.date_formats, + ).date() + except ValueError: + # datetime.time and datetime.datetime might be either: + d = None + try: + d = for_try_except( + ValueError, + datetime.strptime, + repeat(value), + self.grammar.time_formats, + ).time() + except ValueError: + try: + d = for_try_except( + ValueError, + datetime.strptime, + repeat(value), + self.grammar.datetime_formats, + ) + except ValueError: + pass + if d is not None: + if d.utcoffset() is None: + if value.endswith("Z"): + return d.replace(tzinfo=timezone.utc) + elif self.grammar.default_timezone is not None: + return d.replace(tzinfo=self.grammar.default_timezone) + return d + + # if we can regex a 60-second time, return str + if self.is_leap_seconds(value): + return str(value) + else: + raise ValueError + + def is_leap_seconds(self, value: str) -> bool: + """Returns True if *value* is a time that matches the + grammar's definition of a leap seconds time (a time string with + a value of 60 for the seconds value). False otherwise.""" + for r in ( + self.grammar.leap_second_Ymd_re, + self.grammar.leap_second_Yj_re, + ): + if r is not None and r.fullmatch(value) is not None: + return True + else: + return False + + def decode_quantity(self, value, unit): + """Returns a Python object that represents a value with + an associated unit, based on the values provided via + *value* and *unit*. This function creates an object + based on the decoder's *quantity_cls*. + """ + try: + return self.quantity_cls(value, str(unit)) + except ValueError as err: + raise QuantityError(err) + + +class ODLDecoder(PVLDecoder): + """A decoder based on the rules in the PDS3 Standards Reference + (version 3.8, 27 Feb 2009) Chapter 12: Object Description + Language Specification and Usage. + + Extends PVLDecoder, and if *grammar* is not specified, it will + default to an ODLGrammar() object. + """ + + def __init__(self, grammar=None, quantity_cls=None, real_cls=None): + self.errors = [] + + if grammar is None: + grammar = ODLGrammar() + + super().__init__( + grammar=grammar, + quantity_cls=quantity_cls, + real_cls=real_cls + ) + + def decode_datetime(self, value: str): + """Extends parent function to also deal with datetimes + and times with a time zone offset. + + If it cannot, it will raise a ValueError. + """ + + try: + return super().decode_datetime(value) + except ValueError: + # if there is a +HH:MM or a -HH:MM suffix that + # can be stripped, then we're in business. + # Otherwise ... + match = re.fullmatch( + r"(?P
.+?)" # the part before the sign + r"(?P[+-])" # required sign + r"(?P0?[0-9]|1[0-2])" # 0 to 12 + fr"(?:{self.grammar._M_frag})?", # Minutes + value, + ) + if match is not None: + gd = match.groupdict(default=0) + dt = super().decode_datetime(gd["dt"]) + offset = timedelta( + hours=int(gd["hour"]), minutes=int(gd["minute"]) + ) + if gd["sign"] == "-": + offset = -1 * offset + return dt.replace(tzinfo=timezone(offset)) + raise ValueError + + def decode_non_decimal(self, value: str) -> int: + """Extends parent function by allowing the wider variety of + radix values that ODL permits over PVL. + """ + match = self.grammar.nondecimal_re.fullmatch(value) + if match is not None: + d = match.groupdict("") + return int(d["sign"] + d["non_decimal"], base=int(d["radix"])) + raise ValueError + + def decode_quoted_string(self, value: str) -> str: + """Extends parent function because the + ODL specification allows for a dash (-) line continuation + character that results in the dash, the line end, and any + leading whitespace on the next line to be removed. It also + allows for a sequence of format effectors surrounded by + spacing characters to be collapsed to a single space. + """ + s = super().decode_quoted_string(value) + + # Deal with dash (-) continuation: + # sp = ''.join(self.grammar.spacing_characters) + fe = "".join(self.grammar.format_effectors) + ws = "".join(self.grammar.whitespace) + nodash = re.sub(fr"-[{fe}][{ws}]*", "", s) + + # Originally thought that only format effectors surrounded + # by whitespace was to be collapsed + # foo = re.sub(fr'[{sp}]*[{fe}]+[{sp}]*', ' ', nodash) + + # But really it collapses all whitespace and strips lead and trail. + return re.sub(fr"[{ws}]+", " ", nodash.strip(ws)) + + def decode_unquoted_string(self, value: str) -> str: + """Extends parent function to provide the extra enforcement that only + ODL Identifier text may be unquoted as a value. + """ + s = super().decode_unquoted_string(value) + + if self.is_identifier(s): + return s + else: + raise ValueError( + f"Only text that qualifies as an ODL Identifier may be " + f"unquoted as a value, and '{s}' is not." + ) + + @staticmethod + def is_identifier(value): + """Returns true if *value* is an ODL Identifier, false otherwise. + + An ODL Identifier is composed of letters, digits, and underscores. + The first character must be a letter, and the last must not + be an underscore. + """ + if isinstance(value, str): + if len(value) == 0: + return False + + try: + # Ensure we're dealing with ASCII + value.encode(encoding="ascii") + + # value can't start with a letter or end with an underbar + if not value[0].isalpha() or value.endswith("_"): + return False + + for c in value: + if not (c.isalpha() or c.isdigit() or c == "_"): + return False + else: + return True + + except UnicodeError: + return False + else: + return False + + +class PDSLabelDecoder(ODLDecoder): + """A decoder based on the rules in the PDS3 Standards Reference + (version 3.8, 27 Feb 2009) Chapter 12: Object Description + Language Specification and Usage. + + Extends ODLDecoder, and if *grammar* is not specified, it will + default to a PDS3Grammar() object. + """ + + def __init__(self, grammar=None, quantity_cls=None): + self.errors = [] + + if grammar is None: + super().__init__(grammar=PDSGrammar(), quantity_cls=quantity_cls) + else: + super().__init__(grammar=grammar, quantity_cls=quantity_cls) + + def decode_datetime(self, value: str): + """Overrides parent function since PDS3 forbids a timezone + specification, and times with a precision more than miliseconds. + + If it cannot decode properly, it will raise a ValueError. + """ + + t = super(ODLDecoder, self).decode_datetime(value) + + if ( + hasattr(t, "microsecond") + and t.microsecond != round(t.microsecond / 1000) * 1000 + ): + raise ValueError( + f"The PDS specification does not allow time values with" + f"precision greater than miliseconds, and this has " + f"microsecond precision: {t}." + ) + + return t + + +class OmniDecoder(ODLDecoder): + """A permissive decoder that attempts to parse all forms of + "PVL" that are thrown at it. + + Extends ODLDecoder. + """ + + def decode_non_decimal(self, value: str) -> int: + """Extends parent function by allowing a plus or + minus sign to be in two different positions + in a non-decimal number, since PVL has one + specification, and ODL has another. + """ + # Non-Decimal with a variety of radix values and sign + # positions. + match = self.grammar.nondecimal_re.fullmatch(value) + if match is not None: + d = match.groupdict("") + if "second_sign" in d: + if d["sign"] != "" and d["second_sign"] != "": + raise ValueError( + f'The non-decimal value, "{value}", ' "has two signs." + ) + elif d["sign"] != "": + sign = d["sign"] + else: + sign = d["second_sign"] + else: + sign = d["sign"] + + return int(sign + d["non_decimal"], base=int(d["radix"])) + raise ValueError + + def decode_datetime(self, value: str): + """Returns an appropriate Python datetime time, date, or datetime + object by using the 3rd party dateutil library (if present) + to parse an ISO 8601 datetime string in *value*. If it cannot, + or the dateutil library is not present, it will raise a + ValueError. + """ + + try: + return super().decode_datetime(value) + except ValueError: + try: + from dateutil.parser import isoparser + + isop = isoparser() + + if len(value) > 3 and value[-2] == "+" and value[-1].isdigit(): + # This technically means that we accept slightly more + # formats than ISO 8601 strings, since under that + # specification, two digits after the '+' are required + # for an hour offset, but ODL doesn't have this + # requirement. If we find only one digit, we'll + # just assume it means an hour and insert a zero so + # that it can be parsed. + tokens = value.rpartition("+") + value = tokens[0] + "+0" + tokens[-1] + + try: + return isop.parse_isodate(value) + except ValueError: + try: + return isop.parse_isotime(value) + except ValueError: + return isop.isoparse(value) + + except ImportError: + warn( + "The dateutil library is not present, so more " + "exotic date and time formats beyond the PVL/ODL " + "set cannot be parsed.", + ImportWarning, + ) + + raise ValueError + + def decode_unquoted_string(self, value: str) -> str: + """Overrides parent function since the ODLDecoder has a more narrow + definition of what is allowable as an unquoted string than the + PVLDecoder does. + """ + return super(ODLDecoder, self).decode_unquoted_string(value) diff --git a/hirise_blender/pvl/encoder.py b/hirise_blender/pvl/encoder.py new file mode 100644 index 0000000..2a77ac3 --- /dev/null +++ b/hirise_blender/pvl/encoder.py @@ -0,0 +1,1178 @@ +# -*- coding: utf-8 -*- +"""Parameter Value Langage encoder. + +An encoder deals with converting Python objects into +string values that conform to a PVL specification. +""" + +# Copyright 2015, 2019-2021, ``pvl`` library authors. +# +# Reuse is permitted under the terms of the license. +# The AUTHORS file and the LICENSE file are at the +# top level of this library. + +import datetime +import re +import textwrap + +from collections import abc, namedtuple +from decimal import Decimal +from warnings import warn + +from .collections import PVLObject, PVLGroup, Quantity +from .grammar import PVLGrammar, ODLGrammar, PDSGrammar, ISISGrammar +from .token import Token +from .decoder import PVLDecoder, ODLDecoder, PDSLabelDecoder + + +class QuantTup(namedtuple("QuantTup", ["cls", "value_prop", "units_prop"])): + """ + This class is just a convenient namedtuple for internally keeping track + of quantity classes that encoders can deal with. In general, users + should not be instantiating this, instead use your encoder's + add_quantity_cls() function. + """ + + +class PVLEncoder(object): + """An encoder based on the rules in the CCSDS-641.0-B-2 'Blue Book' + which defines the PVL language. + + :param grammar: A pvl.grammar object, if None or not specified, it will + be set to the grammar parameter of *decoder* (if + *decoder* is not None) or will default to PVLGrammar(). + :param grammar: defaults to pvl.grammar.PVLGrammar(). + :param decoder: defaults to pvl.decoder.PVLDecoder(). + :param indent: specifies the number of spaces that will be used to + indent each level of the PVL document, when Groups or Objects + are encountered, defaults to 2. + :param width: specifies the number of characters in width that each + line should have, defaults to 80. + :param aggregation_end: when True the encoder will print the value + of the aggregation's Block Name in the End Aggregation Statement + (e.g. END_GROUP = foo), and when false, it won't (e.g. END_GROUP). + Defaults to True. + :param end_delimiter: when True the encoder will print the grammar's + delimiter (e.g. ';' for PVL) after each statement, when False + it won't. Defaults to True. + :param newline: is the string that will be placed at the end of each + 'line' of output (and counts against *width*), defaults to '\\\\n'. + :param group_class: must this class will be tested against with + isinstance() to determine if various elements of the dict-like + passed to encode() should be encoded as a PVL Group or PVL Object, + defaults to PVLGroup. + :param object_class: must be a class that can take a *group_class* + object in its constructor (essentially converting a *group_class* + to an *object_class*), otherwise will raise TypeError. Defaults + to PVLObject. + """ + + def __init__( + self, + grammar=None, + decoder=None, + indent: int = 2, + width: int = 80, + aggregation_end: bool = True, + end_delimiter: bool = True, + newline: str = "\n", + group_class=PVLGroup, + object_class=PVLObject, + ): + + if grammar is None: + if decoder is not None: + self.grammar = decoder.grammar + else: + self.grammar = PVLGrammar() + elif isinstance(grammar, PVLGrammar): + self.grammar = grammar + else: + raise Exception + + if decoder is None: + self.decoder = PVLDecoder(self.grammar) + elif isinstance(decoder, PVLDecoder): + self.decoder = decoder + else: + raise Exception + + self.indent = indent + self.width = width + self.end_delimiter = end_delimiter + self.aggregation_end = aggregation_end + self.newline = newline + + # This list of 3-tuples *always* has our own pvl quantity object, + # and should *only* be added to with self.add_quantity_cls(). + self.quantities = [QuantTup(Quantity, "value", "units")] + self._import_quantities() + + if issubclass(group_class, abc.Mapping): + self.grpcls = group_class + else: + raise TypeError("The group_class must be a Mapping type.") + + if issubclass(object_class, abc.Mapping): + self.objcls = object_class + else: + raise TypeError("The object_class must be a Mapping type.") + + try: + self.objcls(self.grpcls()) + except TypeError: + raise TypeError( + f"The object_class type ({object_class}) cannot be " + f"instantiated with an argument that is of type " + f"group_class ({group_class})." + ) + + # Finally, let's keep track of everything we consider "numerical": + self.numeric_types = (int, float, self.decoder.real_cls, Decimal) + + def _import_quantities(self): + warn_str = ( + "The {} library is not present, so {} objects will " + "not be properly encoded." + ) + try: + from astropy import units as u + + self.add_quantity_cls(u.Quantity, "value", "unit") + except ImportError: + warn( + warn_str.format("astropy", "astropy.units.Quantity"), + ImportWarning, + ) + + try: + from pint import Quantity as q + + self.add_quantity_cls(q, "magnitude", "units") + except ImportError: + warn(warn_str.format("pint", "pint.Quantity"), ImportWarning) + + def add_quantity_cls(self, cls, value_prop: str, units_prop: str): + """Adds a quantity class to the list of possible + quantities that this encoder can handle. + + :param cls: The name of a quantity class that can be tested + with ``isinstance()``. + :param value_prop: A string that is the property name of + *cls* that contains the value or magnitude of the quantity + object. + :param units_prop: A string that is the property name of + *cls* that contains the units element of the quantity + object. + """ + if not isinstance(cls, type): + raise TypeError(f"The cls given ({cls}) is not a Python class.") + + # If a quantity object can't encode "one meter" its probably not + # going to work for us. + test_cls = cls(1, "m") + for prop in (value_prop, units_prop): + if not hasattr(test_cls, prop): + raise AttributeError( + f"The class ({cls}) does not have an " + f" attribute named {prop}." + ) + + self.quantities.append(QuantTup(cls, value_prop, units_prop)) + + def format(self, s: str, level: int = 0) -> str: + """Returns a string derived from *s*, which + has leading space characters equal to + *level* times the number of spaces specified + by this encoder's indent property. + + It uses the textwrap library to wrap long lines. + """ + + prefix = level * (self.indent * " ") + + if len(prefix + s + self.newline) > self.width and "=" in s: + (preq, _, posteq) = s.partition("=") + new_prefix = prefix + preq.strip() + " = " + + lines = textwrap.wrap( + posteq.strip(), + width=(self.width - len(self.newline)), + replace_whitespace=False, + initial_indent=new_prefix, + subsequent_indent=(" " * len(new_prefix)), + break_long_words=False, + break_on_hyphens=False, + ) + return self.newline.join(lines) + else: + return prefix + s + + def encode(self, module: abc.Mapping) -> str: + """Returns a ``str`` formatted as a PVL document based + on the dict-like *module* object + according to the rules of this encoder. + """ + lines = list() + lines.append(self.encode_module(module, 0)) + + end_line = self.grammar.end_statements[0] + if self.end_delimiter: + end_line += self.grammar.delimiters[0] + + lines.append(end_line) + + # Final check to ensure we're sending out the right character set: + s = self.newline.join(lines) + + for i, c in enumerate(s): + if not self.grammar.char_allowed(c): + raise ValueError( + "Encountered a character that was not " + "a valid character according to the " + 'grammar: "{}", it is in: ' + '"{}"'.format(c, s[i - 5, i + 5]) + ) + + return self.newline.join(lines) + + def encode_module(self, module: abc.Mapping, level: int = 0) -> str: + """Returns a ``str`` formatted as a PVL module based + on the dict-like *module* object according to the + rules of this encoder, with an indentation level + of *level*. + """ + lines = list() + + # To align things on the equals sign, just need to normalize + # the non-aggregation key length: + + non_agg_key_lengths = list() + for k, v in module.items(): + if not isinstance(v, abc.Mapping): + non_agg_key_lengths.append(len(k)) + longest_key_len = max(non_agg_key_lengths, default=0) + + for k, v in module.items(): + if isinstance(v, abc.Mapping): + lines.append(self.encode_aggregation_block(k, v, level)) + else: + lines.append( + self.encode_assignment(k, v, level, longest_key_len) + ) + return self.newline.join(lines) + + def encode_aggregation_block( + self, key: str, value: abc.Mapping, level: int = 0 + ) -> str: + """Returns a ``str`` formatted as a PVL Aggregation Block with + *key* as its name, and its contents based on the + dict-like *value* object according to the + rules of this encoder, with an indentation level + of *level*. + """ + lines = list() + + if isinstance(value, self.grpcls): + agg_keywords = self.grammar.group_pref_keywords + elif isinstance(value, abc.Mapping): + agg_keywords = self.grammar.object_pref_keywords + else: + raise ValueError("The value {value} is not dict-like.") + + agg_begin = "{} = {}".format(agg_keywords[0], key) + if self.end_delimiter: + agg_begin += self.grammar.delimiters[0] + lines.append(self.format(agg_begin, level)) + + lines.append(self.encode_module(value, (level + 1))) + + agg_end = "" + if self.aggregation_end: + agg_end += "{} = {}".format(agg_keywords[1], key) + else: + agg_end += agg_keywords[1] + if self.end_delimiter: + agg_end += self.grammar.delimiters[0] + lines.append(self.format(agg_end, level)) + + return self.newline.join(lines) + + def encode_assignment( + self, key: str, value, level: int = 0, key_len: int = None + ) -> str: + """Returns a ``str`` formatted as a PVL Assignment Statement + with *key* as its Parameter Name, and its value based + on *value* object according to the rules of this encoder, + with an indentation level of *level*. It also allows for + an optional *key_len* which indicates the width in characters + that the Assignment Statement should be set to, defaults to + the width of *key*. + """ + if key_len is None: + key_len = len(key) + + s = "" + s += "{} = ".format(key.ljust(key_len)) + + enc_val = self.encode_value(value) + + if enc_val.startswith(self.grammar.quotes): + # deal with quoted lines that need to preserve + # newlines + s = self.format(s, level) + s += enc_val + + if self.end_delimiter: + s += self.grammar.delimiters[0] + + return s + else: + s += enc_val + + if self.end_delimiter: + s += self.grammar.delimiters[0] + + return self.format(s, level) + + def encode_value(self, value) -> str: + """Returns a ``str`` formatted as a PVL Value based + on the *value* object according to the rules of this encoder. + """ + try: + return self.encode_quantity(value) + except ValueError: + return self.encode_simple_value(value) + + def encode_quantity(self, value) -> str: + """Returns a ``str`` formatted as a PVL Value followed by + a PVL Units Expression if the *value* object can be + encoded this way, otherwise raise ValueError.""" + for (cls, v_prop, u_prop) in self.quantities: + if isinstance(value, cls): + return self.encode_value_units( + getattr(value, v_prop), getattr(value, u_prop) + ) + + raise ValueError( + f"The value object {value} could not be " + "encoded as a PVL Value followed by a PVL " + f"Units Expression, it is of type {type(value)}" + ) + + def encode_value_units(self, value, units) -> str: + """Returns a ``str`` formatted as a PVL Value from *value* + followed by a PVL Units Expressions from *units*.""" + value_str = self.encode_simple_value(value) + units_str = self.encode_units(str(units)) + return f"{value_str} {units_str}" + + def encode_simple_value(self, value) -> str: + """Returns a ``str`` formatted as a PVL Simple Value based + on the *value* object according to the rules of this encoder. + """ + if value is None: + return self.grammar.none_keyword + elif isinstance(value, (set, frozenset)): + return self.encode_set(value) + elif isinstance(value, list): + return self.encode_sequence(value) + elif isinstance( + value, (datetime.datetime, datetime.date, datetime.time) + ): + return self.encode_datetype(value) + elif isinstance(value, bool): + if value: + return self.grammar.true_keyword + else: + return self.grammar.false_keyword + elif isinstance(value, self.numeric_types): + return str(value) + elif isinstance(value, str): + return self.encode_string(value) + else: + raise TypeError(f"{value!r} is not serializable.") + + def encode_setseq(self, values: abc.Collection) -> str: + """This function provides shared functionality for + encode_sequence() and encode_set(). + """ + return ", ".join([self.encode_value(v) for v in values]) + + def encode_sequence(self, value: abc.Sequence) -> str: + """Returns a ``str`` formatted as a PVL Sequence based + on the *value* object according to the rules of this encoder. + """ + return "(" + self.encode_setseq(value) + ")" + + def encode_set(self, value: abc.Set) -> str: + """Returns a ``str`` formatted as a PVL Set based + on the *value* object according to the rules of this encoder. + """ + return "{" + self.encode_setseq(value) + "}" + + def encode_datetype(self, value) -> str: + """Returns a ``str`` formatted as a PVL Date/Time based + on the *value* object according to the rules of this encoder. + If *value* is not a datetime date, time, or datetime object, + it will raise TypeError. + """ + if isinstance(value, datetime.datetime): + return self.encode_datetime(value) + elif isinstance(value, datetime.date): + return self.encode_date(value) + elif isinstance(value, datetime.time): + return self.encode_time(value) + else: + raise TypeError(f"{value!r} is not a datetime type.") + + @staticmethod + def encode_date(value: datetime.date) -> str: + """Returns a ``str`` formatted as a PVL Date based + on the *value* object according to the rules of this encoder. + """ + return f"{value:%Y-%m-%d}" + + @staticmethod + def encode_time(value: datetime.time) -> str: + """Returns a ``str`` formatted as a PVL Time based + on the *value* object according to the rules of this encoder. + """ + s = f"{value:%H:%M}" + + if value.microsecond: + s += f":{value:%S.%f}" + elif value.second: + s += f":{value:%S}" + + return s + + def encode_datetime(self, value: datetime.datetime) -> str: + """Returns a ``str`` formatted as a PVL Date/Time based + on the *value* object according to the rules of this encoder. + """ + date = self.encode_date(value) + time = self.encode_time(value) + return date + "T" + time + + def needs_quotes(self, s: str) -> bool: + """Returns true if *s* must be quoted according to this + encoder's grammar, false otherwise. + """ + if any(c in self.grammar.whitespace for c in s): + return True + + if s in self.grammar.reserved_keywords: + return True + + tok = Token(s, grammar=self.grammar, decoder=self.decoder) + return not tok.is_unquoted_string() + + def encode_string(self, value) -> str: + """Returns a ``str`` formatted as a PVL String based + on the *value* object according to the rules of this encoder. + """ + s = str(value) + + if self.needs_quotes(s): + for q in self.grammar.quotes: + if q not in s: + return q + s + q + else: + raise ValueError( + "All of the quote characters, " + f"{self.grammar.quotes}, were in the " + f'string ("{s}"), so it could not be quoted.' + ) + else: + return s + + def encode_units(self, value: str) -> str: + """Returns a ``str`` formatted as a PVL Units Value based + on the *value* object according to the rules of this encoder. + """ + return ( + self.grammar.units_delimiters[0] + + value + + self.grammar.units_delimiters[1] + ) + + +class ODLEncoder(PVLEncoder): + """An encoder based on the rules in the PDS3 Standards Reference + (version 3.8, 27 Feb 2009) Chapter 12: Object Description + Language Specification and Usage for ODL only. This is + almost certainly not what you want. There are very rarely + cases where you'd want to use ODL that you wouldn't also want + to use the PDS Label restrictions, so you probably really want + the PDSLabelEncoder class, not this one. Move along. + + It extends PVLEncoder. + + :param grammar: defaults to pvl.grammar.ODLGrammar(). + :param decoder: defaults to pvl.decoder.ODLDecoder(). + :param end_delimiter: defaults to False. + :param newline: defaults to '\\\\r\\\\n'. + """ + + def __init__( + self, + grammar=None, + decoder=None, + indent=2, + width=80, + aggregation_end=True, + end_delimiter=False, + newline="\r\n", + group_class=PVLGroup, + object_class=PVLObject + ): + + if grammar is None: + grammar = ODLGrammar() + + if decoder is None: + decoder = ODLDecoder(grammar) + + if not callable(getattr(decoder, "is_identifier", None)): + raise TypeError( + f"The decoder for an ODLEncoder() must have the " + f"is_identifier() function, and this does not: {decoder}" + ) + + super().__init__( + grammar, + decoder, + indent, + width, + aggregation_end, + end_delimiter, + newline, + group_class=group_class, + object_class=object_class + ) + + def encode(self, module: abc.Mapping) -> str: + """Extends parent function, but ODL requires that there must be + a spacing or format character after the END statement and this + adds the encoder's ``newline`` sequence. + """ + s = super().encode(module) + return s + self.newline + + def is_scalar(self, value) -> bool: + """Returns a boolean indicating whether the *value* object + qualifies as an ODL 'scalar_value'. + + ODL defines a 'scalar-value' as a numeric_value, a + date_time_string, a text_string_value, or a symbol_value. + + For Python, these correspond to the following: + + * numeric_value: any of self.numeric_types, and Quantity whose value + is one of the self.numeric_types. + * date_time_string: datetime objects + * text_string_value: str + * symbol_value: str + + """ + for quant in self.quantities: + if isinstance(value, quant.cls): + if isinstance( + getattr(value, quant.value_prop), self.numeric_types + ): + return True + + scalar_types = ( + *self.numeric_types, + datetime.date, + datetime.datetime, + datetime.time, + str + ) + if isinstance(value, scalar_types): + return True + + return False + + def is_symbol(self, value) -> bool: + """Returns true if *value* is an ODL Symbol String, false otherwise. + + An ODL Symbol String is enclosed by single quotes + and may not contain any of the following characters: + + 1. The apostrophe, which is reserved as the symbol string delimiter. + 2. ODL Format Effectors + 3. Control characters + + This means that an ODL Symbol String is a subset of the PVL + quoted string, and will be represented in Python as a ``str``. + """ + if isinstance(value, str): + if "'" in value: # Item 1 + return False + + for fe in self.grammar.format_effectors: # Item 2 + if fe in value: + return False + + if len(value) > self.width / 2: + # This means that the string is long and it is very + # likely to get wrapped and have carriage returns, + # and thus "ODL Format Effectors" inserted later. + # Unfortunately, without knowing the width of the + # parameter term, and the current indent level, this + # still may end up being incorrect threshhold. + return False + + if value.isprintable() and len(value) > 0: # Item 3 + return True + else: + return False + + def needs_quotes(self, s: str) -> bool: + """Return true if *s* is an ODL Identifier, false otherwise. + + Overrides parent function. + """ + return not self.decoder.is_identifier(s) + + def is_assignment_statement(self, s) -> bool: + """Returns true if *s* is an ODL Assignment Statement, false otherwise. + + An ODL Assignment Statement is either an + element_identifier or a namespace_identifier + joined to an element_identifier with a colon. + """ + if self.decoder.is_identifier(s): + return True + + (ns, _, el) = s.partition(":") + + if self.decoder.is_identifier(ns) and self.decoder.is_identifier(el): + return True + + return False + + def encode_assignment(self, key, value, level=0, key_len=None) -> str: + """Overrides parent function by restricting the length of + keywords and enforcing that they be ODL Identifiers + and uppercasing their characters. + """ + + if key_len is None: + key_len = len(key) + + if len(key) > 30: + raise ValueError( + "ODL keywords must be 30 characters or less " + f"in length, this one is longer: {key}" + ) + + if ( + key.startswith("^") and self.is_assignment_statement(key[1:]) + ) or self.is_assignment_statement(key): + ident = key.upper() + else: + raise ValueError( + f'The keyword "{key}" is not a valid ODL ' "Identifier." + ) + + s = "{} = ".format(ident.ljust(key_len)) + s += self.encode_value(value) + + if self.end_delimiter: + s += self.grammar.delimiters[0] + + return self.format(s, level) + + def encode_sequence(self, value) -> str: + """Extends parent function, as ODL only allows one- and + two-dimensional sequences of ODL scalar_values. + """ + if len(value) == 0: + raise ValueError("ODL does not allow empty Sequences.") + + for v in value: # check the first dimension (list of elements) + if isinstance(v, list): + for i in v: # check the second dimension (list of lists) + if isinstance(i, list): + # Shouldn't be lists of lists of lists. + raise ValueError( + "ODL only allows one- and two- " + "dimensional Sequences, but " + f"this has more: {value}" + ) + elif not self.is_scalar(i): + raise ValueError( + "ODL only allows scalar_values " + f"within sequences: {v}" + ) + + elif not self.is_scalar(v): + raise ValueError( + "ODL only allows scalar_values within " f"sequences: {v}" + ) + + return super().encode_sequence(value) + + def encode_set(self, values) -> str: + """Extends parent function, ODL only allows sets to contain + scalar values. + """ + + if not all(map(self.is_scalar, values)): + raise ValueError( + f"ODL only allows scalar values in sets: {values}" + ) + + return super().encode_set(values) + + def encode_value(self, value): + """Extends parent function by only allowing Units Expressions for + numeric values. + """ + for quant in self.quantities: + if isinstance(value, quant.cls): + if isinstance( + getattr(value, quant.value_prop), + self.numeric_types + ): + return super().encode_value(value) + else: + raise ValueError( + "Unit expressions are only allowed " + f"following numeric values: {value}" + ) + + return super().encode_value(value) + + def encode_string(self, value): + """Extends parent function by appropriately quoting Symbol + Strings. + """ + if self.decoder.is_identifier(value): + return value + elif self.is_symbol(value): + return "'" + value + "'" + else: + return super().encode_string(value) + + def encode_time(self, value: datetime.time) -> str: + """Extends parent function since ODL allows a time zone offset + from UTC to be included, and otherwise recommends that times + be suffixed with a 'Z' to clearly indicate that they are in UTC. + """ + if value.tzinfo is None: + raise ValueError( + f"ODL cannot output local times, and this time does not " + f"have a timezone offset: {value}" + ) + + t = super().encode_time(value) + + if value.utcoffset() == datetime.timedelta(): + return t + "Z" + else: + td_str = str(value.utcoffset()) + (h, m, s) = td_str.split(":") + if s != "00": + raise ValueError( + "The datetime value had a timezone offset " + f"with seconds values ({value}) which is " + "not allowed in ODL." + ) + if m == "00": + return t + f"+{h:0>2}" + else: + return t + f"+{h:0>2}:{m}" + + return t + + def encode_units(self, value) -> str: + """Overrides parent function since ODL limits what characters + and operators can be present in Units Expressions. + """ + + # if self.is_identifier(value.strip('*/()-')): + if self.decoder.is_identifier(re.sub(r"[\s*/()-]", "", value)): + + if "**" in value: + exponents = re.findall(r"\*\*.+?", value) + for e in exponents: + if re.search(r"\*\*-?\d+", e) is None: + raise ValueError( + "The exponentiation operator (**) in " + f'this Units Expression "{value}" ' + "is not a decimal integer." + ) + + return ( + self.grammar.units_delimiters[0] + + value + + self.grammar.units_delimiters[1] + ) + else: + raise ValueError( + f'The value, "{value}", does not conform to ' + "the specification for an ODL Units Expression." + ) + + +class PDSLabelEncoder(ODLEncoder): + """An encoder based on the rules in the PDS3 Standards Reference + (version 3.8, 27 Feb 2009) Chapter 12: Object Description + Language Specification and Usage and writes out labels that + conform to the PDS 3 standards. + + It extends ODLEncoder. + + You are not allowed to chose *end_delimiter* or *newline* + as the parent class allows, because to be PDS-compliant, + those are fixed choices. However, in some cases, the PDS3 + Standards are asymmetric, allowing for a wider variety of + PVL-text on "read" and a more narrow variety of PVL-text + on "write". The default values of the PDSLabelEncoder enforce + those strict "write" rules, but if you wish to alter them, + but still produce PVL-text that would validate against the PDS3 + standard, you may alter them. + + :param convert_group_to_object: Defaults to True, meaning that + if a GROUP does not conform to the PDS definition of a + GROUP, then it will be written out as an OBJECT. If it is + False, then an exception will be thrown if incompatible + GROUPs are encountered. In PVL and ODL, the OBJECT and GROUP + aggregations are interchangeable, but the PDS applies + restrictions to what can appear in a GROUP. + :param tab_replace: Defaults to 4 and indicates the number of + space characters to replace horizontal tab characters with + (since tabs aren't allowed in PDS labels). If this is set + to zero, tabs will not be replaced with spaces. + :param symbol_single_quotes: Defaults to True, and if a Python `str` + object qualifies as a PVL Symbol String, it will be written to + PVL-text as a single-quoted string. If False, no special + handling is done, and any PVL Symbol String will be treated + as a PVL Text String, which is typically enclosed with double-quotes. + :param time_trailing_z: defaults to True, and suffixes a "Z" to + datetimes and times written to PVL-text as the PDS encoding + standard requires. If False, no trailing "Z" is written. + + """ + + def __init__( + self, + grammar=None, + decoder=None, + indent=2, + width=80, + aggregation_end=True, + group_class=PVLGroup, + object_class=PVLObject, + convert_group_to_object=True, + tab_replace=4, + symbol_single_quote=True, + time_trailing_z=True, + ): + + if grammar is None: + grammar = PDSGrammar() + + if decoder is None: + decoder = PDSLabelDecoder(grammar) + + super().__init__( + grammar, + decoder, + indent, + width, + aggregation_end, + end_delimiter=False, + newline="\r\n", + group_class=group_class, + object_class=object_class + ) + + self.convert_group_to_object = convert_group_to_object + self.tab_replace = tab_replace + self.symbol_single_quote = symbol_single_quote + self.time_trailing_z = time_trailing_z + + def count_aggs( + self, module: abc.Mapping, obj_count: int = 0, grp_count: int = 0 + ) -> tuple((int, int)): + """Returns the count of OBJECT and GROUP aggregations + that are contained within the *module* as a two-tuple + in that order. + """ + # This currently just counts the values in the passed + # in module, it does not 'recurse' if those aggregations also + # may contain aggregations. + + for k, v in module.items(): + if isinstance(v, abc.Mapping): + if isinstance(v, self.grpcls): + grp_count += 1 + elif isinstance(v, self.objcls): + obj_count += 1 + else: + # We treat other dict-like Python objects as + # PVL Objects for the purposes of this count, + # because that is how they will be encoded. + obj_count += 1 + + return obj_count, grp_count + + def encode(self, module: abc.MutableMapping) -> str: + """Extends the parent function, by adding a restriction. + For PDS, if there are any GROUP elements, there must be at + least one OBJECT element in the label. Behavior here + depends on the value of this encoder's convert_group_to_object + property. + """ + (obj_count, grp_count) = self.count_aggs(module) + + if grp_count > 0 and obj_count < 1: + if self.convert_group_to_object: + for k, v in module.items(): + # First try to convert any GROUPs that would not + # be valid PDS GROUPs. + if isinstance(v, self.grpcls) and not self.is_PDSgroup(v): + module[k] = self.objcls(v) + break + else: + # Then just convert the first GROUP + for k, v in module.items(): + if isinstance(v, self.grpcls): + module[k] = self.objcls(v) + break + else: + raise ValueError( + "Couldn't convert any of the GROUPs " "to OBJECTs." + ) + else: + raise ValueError( + "This module has a GROUP element, but no " + "OBJECT elements, which is not allowed by " + "the PDS. You could set " + "*convert_group_to_object* to *True* on the " + "encoder to try and convert a GROUP " + "to an OBJECT." + ) + + s = super().encode(module) + if self.tab_replace > 0: + return s.replace("\t", (" " * self.tab_replace)) + else: + return s + + def is_PDSgroup(self, group: abc.Mapping) -> bool: + """Returns true if the dict-like *group* qualifies as a PDS Group, + false otherwise. + + PDS applies the following restrictions to GROUPS: + + 1. The GROUP structure may only be used in a data product + label which also contains one or more data OBJECT definitions. + 2. The GROUP statement must contain only attribute assignment + statements, include pointers, or related information pointers + (i.e., no data location pointers). If there are multiple + values, a single statement must be used with either sequence + or set syntax; no attribute assignment statement or pointer + may be repeated. + 3. GROUP statements may not be nested. + 4. GROUP statements may not contain OBJECT definitions. + 5. Only PSDD elements may appear within a GROUP statement. + *PSDD is not defined anywhere in the PDS document, so don't + know how to test for it.* + 6. The keyword contents associated with a specific GROUP + identifier must be identical across all labels of a single data + set (with the exception of the “PARAMETERS” GROUP, as + explained). + + Use of the GROUP structure must be coordinated with the + responsible PDS discipline Node. + + Items 1 & 6 and the final sentence above, can't really be tested + by examining a single group, but must be dealt with in a larger + context. The ODLEncoder.encode_module() handles #1, at least. + You're on your own for the other two issues. + + Item 5: *PSDD* is not defined anywhere in the ODL PDS document, + so don't know how to test for it. + """ + (obj_count, grp_count) = self.count_aggs(group) + + # Items 3 and 4: + if obj_count != 0 or grp_count != 0: + return False + + # Item 2, no data location pointers: + for k, v in group.items(): + if k.startswith("^"): + if isinstance(v, int): + return False + else: + for quant in self.quantities: + if isinstance(v, quant.cls) and isinstance( + getattr(v, quant.value_prop), int + ): + return False + + # Item 2, no repeated keys: + keys = list(group.keys()) + if len(keys) != len(set(keys)): + return False + + return True + + def encode_aggregation_block(self, key, value, level=0): + """Extends parent function because PDS has restrictions on + what may be in a GROUP. + + If the encoder's *convert_group_to_object* parameter is True, + and a GROUP does not conform to the PDS definition of a GROUP, + then it will be written out as an OBJECT. If it is False, + then an exception will be thrown. + """ + + # print('value at top:') + # print(value) + + if isinstance(value, self.grpcls) and not self.is_PDSgroup(value): + if self.convert_group_to_object: + value = self.objcls(value) + else: + raise ValueError( + "This GROUP element is not a valid PDS " + "GROUP. You could set " + "*convert_group_to_object* to *True* on the " + "encoder to try and convert the GROUP" + "to an OBJECT." + ) + + # print('value at bottom:') + # print(value) + + return super().encode_aggregation_block(key, value, level) + + def encode_set(self, values) -> str: + """Extends parent function because PDS only allows symbol values + and integers within sets. + """ + for v in values: + if not self.is_symbol(v) and not isinstance(v, int): + raise ValueError( + "The PDS only allows integers and symbols " + f"in sets: {values}" + ) + + return super().encode_set(values) + + def encode_string(self, value): + """Extends parent function to treat Symbol Strings as Text Strings, + which typically means that they are double-quoted and not + single-quoted. + """ + if self.decoder.is_identifier(value): + return value + elif self.is_symbol(value) and self.symbol_single_quote: + return "'" + value + "'" + else: + return super(ODLEncoder, self).encode_string(value) + + def encode_time(self, value: datetime.time) -> str: + """Overrides parent's encode_time() function because + even though ODL allows for timezones, PDS does not. + + Not in the section on times, but at the end of the PDS + ODL document, in section 12.7.3, para 14, it indicates that + alternate time zones may not be used in a PDS label, only + these: + 1. YYYY-MM-DDTHH:MM:SS.SSS. + 2. YYYY-DDDTHH:MM:SS.SSS. + + """ + s = f"{value:%H:%M}" + + if value.microsecond: + ms = round(value.microsecond / 1000) + if value.microsecond != ms * 1000: + raise ValueError( + f"PDS labels can only encode time values to the milisecond " + f"precision, and this time ({value}) has too much " + f"precision." + ) + else: + s += f":{value:%S}.{ms}" + elif value.second: + s += f":{value:%S}" + + if ( + value.tzinfo is None or + value.tzinfo.utcoffset(None) == datetime.timedelta(0) + ): + if self.time_trailing_z: + return s + "Z" + else: + return s + else: + raise ValueError( + "PDS labels should only have UTC times, but " + f"this time has a timezone: {value}" + ) + + +class ISISEncoder(PVLEncoder): + """An encoder for writing PVL text that can be parsed by the + ISIS PVL text parser. + + The ISIS3 implementation (as of 3.9) of PVL/ODL (like) does not + strictly follow any of the published standards. It was based + on PDS3 ODL from the 1990s, but has several extensions adopted + from existing and prior data sets from ISIS2, PDS, JAXA, ISRO, + ..., and extensions used only within ISIS files (cub, net). This + is one of the reasons using ISIS cube files or PVL text written by + ISIS as an archive format has been strongly discouraged. + + Since there is no specification, only a detailed analysis of + the ISIS software that parses and writes its PVL text would + yield a strategy for parsing it. This encoder is most likely the + least reliable for that reason. We welcome bug reports to help + extend our coverage of this flavor of PVL text. + + :param grammar: defaults to pvl.grammar.ISISGrammar(). + :param decoder: defaults to pvl.decoder.PVLDecoder(). + :param end_delimiter: defaults to False. + :param newline: defaults to '\\\\n'. + """ + + def __init__( + self, + grammar=None, + decoder=None, + indent=2, + width=80, + aggregation_end=True, + end_delimiter=False, + newline="\n", + group_class=PVLGroup, + object_class=PVLObject + ): + + if grammar is None: + grammar = ISISGrammar() + + if decoder is None: + decoder = PVLDecoder(grammar) + + super().__init__( + grammar, + decoder, + indent, + width, + aggregation_end, + end_delimiter, + newline, + group_class=group_class, + object_class=object_class + ) diff --git a/hirise_blender/pvl/exceptions.py b/hirise_blender/pvl/exceptions.py new file mode 100644 index 0000000..9a87feb --- /dev/null +++ b/hirise_blender/pvl/exceptions.py @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- +""" +Exceptions for the Parameter Value Library. +""" + +# Copyright 2019-2020, ``pvl`` library authors. +# +# Reuse is permitted under the terms of the license. +# The AUTHORS file and the LICENSE file are at the +# top level of this library. + + +def firstpos(sub: str, pos: int): + """On the assumption that *sub* is a substring contained in a longer + string, and *pos* is the index in that longer string of the final + character in sub, returns the position of the first character of + sub in that longer string. + + This is useful in the PVL library when we know the position of the + final character of a token, but want the position of the first + character. + """ + return pos - len(sub) + 1 + + +def linecount(doc: str, end: int, start: int = 0): + """Returns the number of lines (by counting the + number of newline characters \\n, with the first line + being line number one) in the string *doc* between the + positions *start* and *end*. + """ + return doc.count("\n", start, end) + 1 + + +class LexerError(ValueError): + """Subclass of ValueError with the following additional properties: + + msg: The unformatted error message + doc: The PVL text being parsed + pos: The start index in doc where parsing failed + lineno: The line corresponding to pos + colno: The column corresponding to pos + """ + + def __init__(self, msg, doc, pos, lexeme): + self.pos = firstpos(lexeme, pos) + lineno = linecount(doc, self.pos) + colno = self.pos - doc.rfind("\n", 0, self.pos) + # Assemble a context string that consists of whole + # words, using fragments is hard to read. + context_tokens = doc[self.pos - 15: self.pos + 15].split(" ") + context = " ".join(context_tokens[1:-1]) + errmsg = ( + f"{msg}: line {lineno} column {colno} (char {pos}) " + f'near "{context}"' + ) + super().__init__(self, errmsg) + self.msg = msg + self.doc = doc + self.lineno = lineno + self.colno = colno + self.lexeme = lexeme + + def __reduce__(self): + return self.__class__, (self.msg, self.doc, self.pos, self.lexeme) + + +class ParseError(Exception): + """An exception to signal errors in the pvl parser.""" + + def __init__(self, msg, token=None): + super().__init__(self, msg) + self.token = token + + +class QuantityError(Exception): + """A simple exception to distinguish errors from Quantity classes.""" + + pass diff --git a/hirise_blender/pvl/grammar.py b/hirise_blender/pvl/grammar.py new file mode 100755 index 0000000..afa4e53 --- /dev/null +++ b/hirise_blender/pvl/grammar.py @@ -0,0 +1,345 @@ +# -*- coding: utf-8 -*- +"""Describes the language aspects of PVL dialects. + +These grammar objects are not particularly meant to be easily +user-modifiable during running of an external program, which is why +they have no arguments at initiation time, nor are there any methods +or functions to modify them. This is because these grammar objects +are used both for reading and writing PVL-text. As such, objects +like PVLGrammar and ODLGrammar shouldn't be altered, because if +they are, then the PVL-text written out with them wouldn't conform +to the spec. + +Certainly, these objects do have attributes that can be altered, +but unless you've carefully read the code, it isn't recommended. + +Maybe someday we'll add a more user-friendly interface to allow that, +but in the meantime, just leave an Issue on the GitHub repo. +""" + +# Copyright 2019-2021, ``pvl`` library authors. +# +# Reuse is permitted under the terms of the license. +# The AUTHORS file and the LICENSE file are at the +# top level of this library. + +import re +from collections import abc +from datetime import timezone + + +class PVLGrammar: + """Describes a PVL grammar for use by the lexer and parser. + + The reference for this grammar is the CCSDS-641.0-B-2 'Blue Book'. + """ + + spacing_characters = (" ", "\t") + format_effectors = ("\n", "\r", "\v", "\f") + + # Tuple of characters to be recognized as PVL White Space + # (used to separate syntactic elements and promote readability, + # but the amount or presence of White Space may not be used to + # provide different meanings). + whitespace = spacing_characters + format_effectors + + # Tuple of characters that may not occur in Parameter Names, + # Unquoted Strings, nor Block Names. + reserved_characters = ( + "&", + "<", + ">", + "'", + "{", + "}", + ",", + "[", + "]", + "=", + "!", + "#", + "(", + ")", + "%", + "+", + '"', + ";", + "~", + "|", + ) + + # If there are any reserved_characters that might start a number, + # they need to be added to numeric_start_chars, otherwise that + # character will get lexed separately from the rest. + # Technically, since '-' isn't in reserved_characters, it isn't needed, + # but it doesn't hurt to keep it here. + numeric_start_chars = ("+", "-") + + delimiters = (";",) + + # Tuple of two-tuples with each two-tuple containing a pair of character + # sequences that enclose a comment. + comments = (("/*", "*/"),) + + # A note on keywords: they should always be compared with + # the str.casefold() function. + # So 'NULL'.casefold(), 'Null'.casefold(), and 'NuLl".casefold() + # all compare equals to none_keyword.casefold(). + none_keyword = "NULL" + true_keyword = "TRUE" + false_keyword = "FALSE" + group_pref_keywords = ("BEGIN_GROUP", "END_GROUP") + group_keywords = {"GROUP": "END_GROUP", "BEGIN_GROUP": "END_GROUP"} + object_pref_keywords = ("BEGIN_OBJECT", "END_OBJECT") + object_keywords = {"OBJECT": "END_OBJECT", "BEGIN_OBJECT": "END_OBJECT"} + aggregation_keywords = dict() + aggregation_keywords.update(group_keywords) + aggregation_keywords.update(object_keywords) + end_statements = ("END",) + reserved_keywords = set(end_statements) + for p in aggregation_keywords.items(): + reserved_keywords |= set(p) + + quotes = ('"', "'") + set_delimiters = ("{", "}") + sequence_delimiters = ("(", ")") + units_delimiters = ("<", ">") + + # [sign]radix#non_decimal_integer# + _s = r"(?P[+-]?)" + nondecimal_pre_re = re.compile(fr"{_s}(?P2|8|16)#") + binary_re = re.compile(fr"{_s}(?P2)#(?P[01]+)#") + octal_re = re.compile(fr"{_s}(?P8)#(?P[0-7]+)#") + hex_re = re.compile(fr"{_s}(?P16)#(?P[0-9A-Fa-f]+)#") + nondecimal_re = re.compile( + fr"{nondecimal_pre_re.pattern}(?P[0-9|A-Fa-f]+)#" + ) + + # The PVL Blue Book says that all PVl Date/Time Values are represented + # in Universal Coordinated Time + default_timezone = timezone.utc + _d_formats = ("%Y-%m-%d", "%Y-%j") + _t_formats = ("%H:%M", "%H:%M:%S", "%H:%M:%S.%f") + date_formats = _d_formats + tuple(x + "Z" for x in _d_formats) + time_formats = _t_formats + tuple(x + "Z" for x in _t_formats) + datetime_formats = list() + for d in _d_formats: + for t in _t_formats: + datetime_formats.append(f"{d}T{t}") + datetime_formats.append(f"{d}T{t}Z") + + # I really didn't want to write these, because it is so easy to + # make a mistake with time regexes, but they're they only way + # to parse times with 60 seconds in them. The above regexes and + # the datetime library are used for all other time parsing. + _H_frag = r"(?P0\d|1\d|2[0-3])" # 00 to 23 + _M_frag = r"(?P[0-5]\d)" # 00 to 59 + _f_frag = r"(\.(?P\d+))" # 1 or more digits + _Y_frag = r"(?P\d{3}[1-9])" # 0001 to 9999 + _m_frag = r"(?P0[1-9]|1[0-2])" # 01 to 12 + _d_frag = r"(?P0[1-9]|[12]\d|3[01])" # 01 to 31 + _Ymd_frag = fr"{_Y_frag}-{_m_frag}-{_d_frag}" + # 001 to 366: + _j_frag = r"(?P(00[1-9]|0[1-9]\d)|[12]\d{2}|3[0-5]\d|36[0-6])" + _Yj_frag = fr"{_Y_frag}-{_j_frag}" + _time_frag = fr"{_H_frag}:{_M_frag}:60{_f_frag}?Z?" # Only times with 60 s + # _time_frag = fr'{_H_frag}:{_M_frag}]' # Only times with 60 s + leap_second_Ymd_re = re.compile(fr"({_Ymd_frag}T)?{_time_frag}") + leap_second_Yj_re = re.compile(fr"({_Yj_frag}T)?{_time_frag}") + + def char_allowed(self, char): + """Returns true if *char* is allowed in the PVL Character Set. + + This is defined as most of the ISO 8859-1 'latin-1' character + set with some exclusions. + """ + if len(char) != 1: + raise ValueError( + f"This function only takes single characters and it was given " + f"{len(char)} ('{char}')." + ) + + o = ord(char) + + # The vertical tab, ord('\t') = 11, is mistakenly + # shaded on page B-3 of the PVL specification. + if ( + o > 255 + or (0 <= o <= 8) + or + # o == 11 or + (14 <= o <= 31) + or (127 <= o <= 159) + ): + return False + else: + return True + + +class ODLGrammar(PVLGrammar): + """This defines an ODL grammar. + + The reference for this grammar is the PDS3 Standards Reference + (version 3.8, 27 Feb 2009) Chapter 12: Object Description + Language Specification and Usage. + """ + + group_pref_keywords = ("GROUP", "END_GROUP") + object_pref_keywords = ("OBJECT", "END_OBJECT") + + # ODL does not allow times with a seconds value of 60. + leap_second_Ymd_re = None + leap_second_Yj_re = None + + # ODL allows "local" times without a timezone specifier. + default_timezone = None + + # ODL allows the radix to be from 2 to 16, but the optional sign + # must be after the first octothorpe (#). Why ODL thought this was + # an important difference to make from PVL, I have no idea. + # radix#[sign]non_decimal_integer# + nondecimal_pre_re = re.compile(fr"(?P[2-9]|1[0-6])#{PVLGrammar._s}") + nondecimal_re = re.compile( + fr"{nondecimal_pre_re.pattern}(?P[0-9A-Fa-f]+)#" + ) + + def char_allowed(self, char): + """Returns true if *char* is allowed in the ODL Character Set. + + The ODL Character Set is limited to ASCII. This is fewer + characters than PVL, but appears to allow more control + characters to be in quoted strings than PVL does. + """ + super().char_allowed(char) + + try: + char.encode(encoding="ascii") + return True + except UnicodeError: + return False + + +class PDSGrammar(ODLGrammar): + """This defines a PDS3 ODL grammar. + + The reference for this grammar is the PDS3 Standards Reference + (version 3.8, 27 Feb 2009) Chapter 12: Object Description + Language Specification and Usage. + """ + + # The PDS spec only allows allows miliseconds, not microseconds, + # but there is only a %f microseconds time format specifier in + # Python, and no miliseconds format specifier, so dealing with + # only the miliseconds will have to be enforced at the encoder and + # decoder stages. + + # PDSLabels default to UTC: + default_timezone = timezone.utc + + +class ISISGrammar(PVLGrammar): + """This defines the ISIS version of PVL. + + This is valid as of ISIS 3.9, and before, at least. + + The ISIS 'Pvl' object typically writes out parameter + values and keywords in CamelCase (e.g. 'Group', 'End_Group', + 'CenterLatitude', etc.), but it will accept all-uppercase + versions. + + Technically, since the ISIS 'Pvl' object which parses + PVL text into C++ objects for ISIS programs to work with + does not recognize the 'BEGIN_' construction, + this means that ISIS does not parse PVL text that would be + valid according to the PVL, ODL, or PDS3 specs. + """ + + # The other thing that ISIS seems to be doing differently is to + # split any text of all kinds with a dash continuation character. This + # is currently handled in the OmniParser.parse() function. + + # At + # https://astrodiscuss.usgs.gov/t/what-pvl-specification-does-isis-conform-to/ + # + # Stuart Sides, ISIS developer, says: + # The ISIS3 implementation of PVL/ODL (like) does not strictly + # follow any of the published standards. It was based on PDS3 + # ODL from the 1990s, but has several extensions (your example + # of continuation lines) adopted from existing and prior data + # sets from ISIS2, PDS, JAXA, ISRO, ..., and extensions used + # only within ISIS3 files (cub, net). This is one of the + # reasons using ISIS cube files as an archive format has been + # strongly discouraged. So to answer your question, there is + # no published specification for ISIS3 PVL. + + # The ISIS parser (at least <=3.9) doesn't recognize the + # 'BEGIN_' construction, which is why we must + # have a separate grammar object. Since we're at it, we might + # as well use the *_pref_keywords to indicate the CamelCase + # that ISIS folks are expecting. + group_pref_keywords = ("Group", "End_Group") + group_keywords = {"GROUP": "END_GROUP"} + object_pref_keywords = ("Object", "End_Object") + object_keywords = {"OBJECT": "END_OBJECT"} + + # A single-line comment that starts with the octothorpe (#) is not part + # of PVL or ODL, but it is used when ISIS writes out comments. + comments = (("/*", "*/"), ("#", "\n")) + + def __init__(self): + # ISIS allows for + characters in Unquoted String values. + self.reserved_characters = tuple( + self.adjust_reserved_characters(self.reserved_characters) + ) + + @staticmethod + def adjust_reserved_characters(chars: abc.Iterable): + # ISIS allows for + characters in Unquoted String values. + # Removing the plus from the reserved characters allows for + # that, but might lead to other parsing errors, so be on the lookout. + rc = list(chars) + rc.remove("+") + return rc + + +class OmniGrammar(PVLGrammar): + """A broadly permissive grammar. + + This grammar does not follow a specification, but is meant to allow + the broadest possible ingestion of PVL-like text that is found. + + This grammar should not be used to write out Python objects to PVL, + instead please use one of the grammars that follows a published + specification, like the PVLGrammar or the ODLGrammar. + """ + + # Interestingly, a single-line comment that starts with the + # octothorpe (#) is neither part of PVL nor ODL, but people use + # it all the time. + comments = (("/*", "*/"), ("#", "\n")) + + # ODL allows the radix to be from 2 to 16, and allows the sign to be + # 'inside' the octothorpes, so we need to allow for the wide variety + # of radix, and the variational placement of the optional sign: + # [sign]radix#[sign]non_decimal_integer# + _ss = r"(?P[+-]?)" + nondecimal_pre_re = re.compile( + PVLGrammar._s + fr"(?P[2-9]|1[0-6])#{_ss}" + ) + nondecimal_re = re.compile( + nondecimal_pre_re.pattern + r"(?P[0-9A-Fa-f]+)#" + ) + + def __init__(self): + # Handle the fact that ISIS writes out unquoted plus signs. + # See ISISGrammar for details. + # Also add the ASCII NULL ("\0") to the reserved_characters tuple. + self.reserved_characters = tuple( + ISISGrammar.adjust_reserved_characters(self.reserved_characters) + + ["\0", ] + ) + + def char_allowed(self, char): + """Takes all characters, could accept bad things, and the user must + beware.""" + return True diff --git a/hirise_blender/pvl/lexer.py b/hirise_blender/pvl/lexer.py new file mode 100644 index 0000000..8fbba5d --- /dev/null +++ b/hirise_blender/pvl/lexer.py @@ -0,0 +1,435 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +"""Provides lexer functions for PVL.""" + +# Copyright 2019-2020, ``pvl`` library authors. +# +# Reuse is permitted under the terms of the license. +# The AUTHORS file and the LICENSE file are at the +# top level of this library. + + +from enum import Enum, auto + +from .grammar import PVLGrammar +from .token import Token +from .decoder import PVLDecoder +from .exceptions import LexerError, firstpos + + +class Preserve(Enum): + FALSE = auto() + COMMENT = auto() + UNIT = auto() + QUOTE = auto() + NONDECIMAL = auto() + + +def lex_preserve(char: str, lexeme: str, preserve: dict) -> tuple((str, dict)): + """Returns a modified *lexeme* string and a modified *preserve* + dict in a two-tuple. The modified *lexeme* will always be + the concatenation of *lexeme* and *char*. + + This is a lexer() helper function that is responsible for + changing the state of the *preserve* dict, if needed. + + If the value for 'end' in *preserve* is the same as *char*, + then the modified *preserve* will have its 'state' value + set to ``Preserve.FALSE`` and its 'end' value set to None, + otherwise second item in the returned tuple will be *preserve* + unchanged. + """ + # print(f'in preserve: char "{char}", lexeme "{lexeme}, p {preserve}"') + if char == preserve["end"]: + return lexeme + char, dict(state=Preserve.FALSE, end=None) + else: + return lexeme + char, preserve + + +def lex_singlechar_comments( + char: str, lexeme: str, preserve: dict, comments: dict +) -> tuple((str, dict)): + """Returns a modified *lexeme* string and a modified *preserve* + dict in a two-tuple. + + This is a lexer() helper function for determining how to modify + *lexeme* and *preserve* based on the single character in *char* + which may or may not be a comment character. + + If the *preserve* 'state' value is Preserve.COMMENT then + the value of lex_preserve() is returned. + + If *char* is among the keys of the *comments* dict, then the + returned *lexeme* will be the concatenation of *lexeme* and + *char*. returned *preserve* dict will have its 'state' value + set to Preserve.COMMENT and its 'end' value set to the value + of *comments[char]*. + + Otherwise return *lexeme* and *preserve* unchanged in the + two-tuple. + """ + if preserve["state"] == Preserve.COMMENT: + return lex_preserve(char, lexeme, preserve) + elif char in comments: + return ( + lexeme + char, + dict(state=Preserve.COMMENT, end=comments[char]), + ) + + return lexeme, preserve + + +def lex_multichar_comments( + char: str, + prev_char: str, + next_char: str, + lexeme: str, + preserve: dict, + comments: tuple(tuple((str, str))) = PVLGrammar().comments, +) -> tuple((str, dict)): + """Returns a modified *lexeme* string and a modified *preserve* + dict in a two-tuple. + + This is a lexer() helper function for determining how to + modify *lexeme* and *preserve* based on the single character + in *char* which may or may not be part of a multi-character + comment character group. + + This function has an internal list of allowed pairs of + multi-character comments that it can deal with, if the + *comments* tuple contains any two-tuples that cannot be + handled, a NotImplementedError will be raised. + + This function will determine whether to append *char* to + *lexeme* or not, and will set the value of the 'state' and + 'end' values of *preserve* appropriately. + """ + # print(f'lex_multichar got these comments: {comments}') + if len(comments) == 0: + raise ValueError("The variable provided to comments is empty.") + + allowed_pairs = (("/*", "*/"),) + for p in comments: + if p not in allowed_pairs: + raise NotImplementedError( + "Can only handle these " + "multicharacter comments: " + f"{allowed_pairs}. To handle " + "others this class must be extended." + ) + + if ("/*", "*/") in comments: + if char == "*": + if prev_char == "/": + return lexeme + "/*", dict(state=Preserve.COMMENT, end="*/") + elif next_char == "/": + return lexeme + "*/", dict(state=Preserve.FALSE, end=None) + else: + return lexeme + "*", preserve + elif char == "/": + # If part of a comment ignore, and let the char == '*' handler + # above deal with it, otherwise add it to the lexeme. + if prev_char != "*" and next_char != "*": + return lexeme + "/", preserve + + return lexeme, preserve + + +def lex_comment( + char: str, + prev_char: str, + next_char: str, + lexeme: str, + preserve: dict, + c_info: dict, +) -> tuple((str, dict)): + """Returns a modified *lexeme* string and a modified *preserve* + dict in a two-tuple. + + This is a lexer() helper function for determining how to + modify *lexeme* and *preserve* based on the single character + in *char* which may or may not be a comment character. + + This function just makes the decision about whether to call + lex_multichar_comments() or lex_singlechar_comments(), and + then returns what they return. + """ + + if char in c_info["multi_chars"]: + return lex_multichar_comments( + char, + prev_char, + next_char, + lexeme, + preserve, + comments=c_info["multi_comments"], + ) + else: + return lex_singlechar_comments( + char, lexeme, preserve, c_info["single_comments"] + ) + + +def _prev_char(s: str, idx: int): + """Returns the character from *s* at the position before *idx* + or None, if *idx* is zero. + """ + if idx <= 0: + return None + else: + return s[idx - 1] + + +def _next_char(s: str, idx: int): + """Returns the character from *s* at the position after *idx* + or None, if *idx* is the last position in *s*. + """ + try: + return s[idx + 1] + except IndexError: + return None + + +def _prepare_comment_tuples(comments: tuple(tuple((str, str)))) -> dict: + """Returns a dict of information based on the contents + of *comments*. + + This is a lexer() helper function to prepare information + for lexer(). + """ + # I initially tried to avoid this function, if you + # don't pre-compute this stuff, you end up re-computing + # it every time you pass into the lex_comment() function, + # which seemed excessive. + d = dict() + m = list() + d["single_comments"] = dict() + d["multi_chars"] = set() + for pair in comments: + if len(pair[0]) == 1: + d["single_comments"][pair[0]] = pair[1] + else: + m.append(pair) + for p in pair: + d["multi_chars"] |= set(p) + + d["chars"] = set(d["single_comments"].keys()) + d["chars"] |= d["multi_chars"] + d["multi_comments"] = tuple(m) + + # print(d) + return d + + +def lex_char( + char: str, + prev_char: str, + next_char: str, + lexeme: str, + preserve: dict, + g: PVLGrammar, + c_info: dict, +) -> tuple((str, dict)): + """Returns a modified *lexeme* string and a modified *preserve* + dict in a two-tuple. + + This is the main lexer() helper function for determining how + to modify (or not) *lexeme* and *preserve* based on the + single character in *char* and the other values passed into + this function. + """ + + # When we are 'in' a comment or a units expression, + # we want those to consume everything, regardless. + # So we must handle the 'preserve' states first, + # and then after that we can check to see if the char + # should put us into one of those states. + + # print(f'lex_char start: char "{char}", lexeme "{lexeme}", "{preserve}"') + + if preserve["state"] != Preserve.FALSE: + if preserve["state"] == Preserve.COMMENT: + (lexeme, preserve) = lex_comment( + char, prev_char, next_char, lexeme, preserve, c_info + ) + elif preserve["state"] in ( + Preserve.UNIT, + Preserve.QUOTE, + Preserve.NONDECIMAL, + ): + (lexeme, preserve) = lex_preserve(char, lexeme, preserve) + else: + raise ValueError( + "{} is not a ".format(preserve["state"]) + + "recognized preservation state." + ) + elif ( + char == "#" + and g.nondecimal_pre_re.fullmatch(lexeme + char) is not None + ): + lexeme += char + preserve = dict(state=Preserve.NONDECIMAL, end="#") + elif char in c_info["chars"]: + (lexeme, preserve) = lex_comment( + char, prev_char, next_char, lexeme, preserve, c_info + ) + elif char in g.units_delimiters[0]: + lexeme += char + preserve = dict(state=Preserve.UNIT, end=g.units_delimiters[1]) + elif char in g.quotes: + lexeme += char + preserve = dict(state=Preserve.QUOTE, end=char) + else: + if char not in g.whitespace: + lexeme += char # adding a char each time + + # print(f'lex_char end: char "{char}", lexeme "{lexeme}", "{preserve}"') + return lexeme, preserve + + +def lex_continue( + char: str, + next_char: str, + lexeme: str, + token: Token, + preserve: dict, + g: PVLGrammar, +) -> bool: + """Return True if accumulation of *lexeme* should continue based + on the values passed into this function, false otherwise. + + This is a lexer() helper function. + """ + + if next_char is None: + return False + + if not g.char_allowed(next_char): + return False + + if preserve["state"] != Preserve.FALSE: + return True + + # Since Numeric objects can begin with a reserved + # character, the reserved characters may split up + # the lexeme. + if ( + char in g.numeric_start_chars + and Token(char + next_char, grammar=g).is_numeric() + ): + return True + + # Since Non Decimal Numerics can have reserved characters in them. + if g.nondecimal_pre_re.fullmatch(lexeme + next_char) is not None: + return True + + # Since the numeric signs could be in the reserved characters, + # make sure we can parse scientific notation correctly: + if ( + char.lower() == "e" + and next_char in g.numeric_start_chars + and Token(lexeme + next_char + "2", grammar=g).is_numeric() + ): + return True + + # Some datetimes can have trailing numeric tz offsets, + # if the decoder allows it, this means there could be + # a '+' that splits the lexeme that we don't want. + if next_char in g.numeric_start_chars and token.is_datetime(): + return True + + return False + + +def lexer(s: str, g=PVLGrammar(), d=PVLDecoder()): + """This is a generator function that returns pvl.Token objects + based on the passed in string, *s*, when the generator's + next() is called. + + A call to send(*t*) will 'return' the value *t* to the + generator, which will be yielded upon calling next(). + This allows a user to 'peek' at the next token, but return it + if they don't like what they see. + + *g* is expected to be an instance of pvl.grammar, and *d* an + instance of pvl.decoder. The lexer will perform differently, + given different values of *g* and *d*. + """ + c_info = _prepare_comment_tuples(g.comments) + # print(c_info) + + lexeme = "" + preserve = dict(state=Preserve.FALSE, end=None) + for i, char in enumerate(s): + if not g.char_allowed(char): + raise LexerError( + f'The character "{char}" (ord: {ord(char)}) ' + " is not allowed by the grammar.", + s, + i, + lexeme, + ) + + prev_char = _prev_char(s, i) + next_char = _next_char(s, i) + + # print(repr(f'lexeme at top: ->{lexeme}<-, char: {char}, ' + # f'prev: {prev_char}, next: {next_char}, ' + # f'{preserve}')) + + (lexeme, preserve) = lex_char( + char, prev_char, next_char, lexeme, preserve, g, c_info + ) + + # print(repr(f' at bot: ->{lexeme}<-, ' + # f' ' + # f'{preserve}')) + + # Now having dealt with char, decide whether to + # go on continue accumulating the lexeme, or yield it. + + if lexeme == "": + continue + + try: + # The ``while t is not None: yield None; t = yield(t)`` + # construction below allows a user of the lexer to + # yield a token, not like what they see, and then use + # the generator's send() function to put the token + # back into the generator. + # + # The first ``yield None`` in there allows the call to + # send() on this generator to return None, and keep the + # value of *t* ready for the next call of next() on the + # generator. This is the magic that allows a user to + # 'return' a token to the generator. + tok = Token(lexeme, grammar=g, decoder=d, pos=firstpos(lexeme, i)) + + if lex_continue(char, next_char, lexeme, tok, preserve, g): + # Any lexeme state that we want to just allow + # to run around again and don't want to get + # caught by the clause in the elif, should + # test true via lex_continue() + continue + + elif ( + next_char is None + or not g.char_allowed(next_char) + or next_char in g.whitespace + or next_char in g.reserved_characters + or s.startswith(tuple(p[0] for p in g.comments), i + 1) + or lexeme.endswith(tuple(p[1] for p in g.comments)) + or lexeme in g.reserved_characters + or tok.is_quoted_string() + ): + # print(f'yielding {tok}') + t = yield tok + while t is not None: + yield None + t = yield t + lexeme = "" + else: + continue + + except ValueError as err: + raise LexerError(err, s, i, lexeme) diff --git a/hirise_blender/pvl/new.py b/hirise_blender/pvl/new.py new file mode 100755 index 0000000..4d6ac9b --- /dev/null +++ b/hirise_blender/pvl/new.py @@ -0,0 +1,209 @@ +# -*- coding: utf-8 -*- +"""Python implementation of PVL (Parameter Value Language), with upcoming +features. + +If you currently use:: + + import pvl + +you can change to:: + + import pvl.new as pvl + +And then use all of the pvl functions as you usually would. You +will also need to have the 3rd party multidict library +(https://github.com/aio-libs/multidict, conda installable) installed. +But then, any objects that are returned by the load functions will +be the new PVLMultiDict objects. +""" + +# Copyright 2015, 2017, 2019-2021, ``pvl`` library authors. +# +# Reuse is permitted under the terms of the license. +# The AUTHORS file and the LICENSE file are at the +# top level of this library. + +import inspect +import io +import urllib.request +from pathlib import Path + +try: # noqa: C901 + # In order to access super class attributes for our derived class, we must + # import the native Python version, instead of the default Cython version. + from multidict._multidict_py import MultiDict # noqa: F401 +except ImportError as err: + raise ImportError( + "The multidict library is not present, so the new PVLMultiDict is not " + "available, and pvl.new can't be imported. In order to do so, install " + "the multidict package", + ImportWarning, + ) from err + +from pvl import * # noqa: F401,F403 +from pvl import get_text_from, decode_by_char + +from .encoder import PDSLabelEncoder, PVLEncoder +from .parser import PVLParser, OmniParser +from .collections import PVLModuleNew, PVLGroupNew, PVLObjectNew + +__all__ = [ + "PVLModuleNew", + "PVLGroupNew", + "PVLObjectNew", +] + + +def load(path, parser=None, grammar=None, decoder=None, **kwargs): + """Returns a Python object from parsing the file at *path*. + + :param path: an :class:`os.PathLike` which presumably has a + PVL Module in it to parse. + :param parser: defaults to :class:`pvl.parser.OmniParser()`. + :param grammar: defaults to :class:`pvl.grammar.OmniGrammar()`. + :param decoder: defaults to :class:`pvl.decoder.OmniDecoder()`. + :param ``**kwargs``: the keyword arguments that will be passed + to :func:`loads()` and are described there. + + If *path* is not an :class:`os.PathLike`, it will be assumed to be an + already-opened file object, and ``.read()`` will be applied + to extract the text. + + If the :class:`os.PathLike` or file object contains some bytes + decodable as text, followed by some that is not (e.g. an ISIS + cube file), that's fine, this function will just extract the + decodable text. + """ + return loads( + get_text_from(path), + parser=parser, + grammar=grammar, + decoder=decoder, + **kwargs + ) + + +def loadu(url, parser=None, grammar=None, decoder=None, **kwargs): + """Returns a Python object from parsing *url*. + + :param url: this will be passed to :func:`urllib.request.urlopen` + and can be a string or a :class:`urllib.request.Request` object. + :param parser: defaults to :class:`pvl.parser.OmniParser()`. + :param grammar: defaults to :class:`pvl.grammar.OmniGrammar()`. + :param decoder: defaults to :class:`pvl.decoder.OmniDecoder()`. + :param ``**kwargs``: the keyword arguments that will be passed + to :func:`urllib.request.urlopen` and to :func:`loads()`. + + The ``**kwargs`` will first be scanned for arguments that + can be given to :func:`urllib.request.urlopen`. If any are + found, they are extracted and used. All remaining elements + will be passed on as keyword arguments to :func:`loads()`. + + Note that *url* can be any URL that :func:`urllib.request.urlopen` + takes. Certainly http and https URLs, but also file, ftp, rsync, + sftp and more! + """ + + # Peel off the args for urlopen: + url_args = dict() + for a in inspect.signature(urllib.request.urlopen).parameters.keys(): + if a in kwargs: + url_args[a] = kwargs.pop(a) + + # The object returned from urlopen will always have a .read() + # function that returns bytes, so: + with urllib.request.urlopen(url, **url_args) as resp: + s = decode_by_char(resp) + + return loads(s, parser=parser, grammar=grammar, decoder=decoder, **kwargs) + + +def loads(s: str, parser=None, grammar=None, decoder=None, **kwargs): + """Deserialize the string, *s*, as a Python object. + + :param s: contains some PVL to parse. + :param parser: defaults to :class:`pvl.parser.OmniParser() which will + return the new PVLMultiDict-derived objects`. + :param grammar: defaults to :class:`pvl.grammar.OmniGrammar()`. + :param decoder: defaults to :class:`pvl.decoder.OmniDecoder()`. + :param ``**kwargs``: the keyword arguments to pass to the *parser* class + if *parser* is none. + """ + if isinstance(s, bytes): + # Someone passed us an old-style bytes sequence. Although it isn't + # a string, we can deal with it: + s = s.decode() + + if parser is None: + parser = OmniParser( + grammar=grammar, + decoder=decoder, + module_class=PVLModuleNew, + group_class=PVLGroupNew, + object_class=PVLObjectNew, + **kwargs + ) + elif not isinstance(parser, PVLParser): + raise TypeError("The parser must be an instance of pvl.PVLParser.") + + return parser.parse(s) + + +def dump(module, path, **kwargs): + """Serialize *module* as PVL text to the provided *path*. + + :param module: a ``PVLModule`` or ``dict``-like object to serialize. + :param path: an :class:`os.PathLike` + :param ``**kwargs``: the keyword arguments to pass to :func:`dumps()`. + + If *path* is an :class:`os.PathLike`, it will attempt to be opened + and the serialized module will be written into that file via + the :func:`pathlib.Path.write_text()` function, and will return + what that function returns. + + If *path* is not an :class:`os.PathLike`, it will be assumed to be an + already-opened file object, and ``.write()`` will be applied + on that object to write the serialized module, and will return + what that function returns. + """ + try: + p = Path(path) + return p.write_text(dumps(module, **kwargs)) + + except TypeError: + # Not an os.PathLike, maybe it is an already-opened file object + try: + if isinstance(path, io.TextIOBase): + return path.write(dumps(module, **kwargs)) + else: + return path.write(dumps(module, **kwargs).encode()) + except AttributeError: + # Not a path, not an already-opened file. + raise TypeError( + "Expected an os.PathLike or an already-opened " + "file object for writing, but got neither." + ) + + +def dumps(module, encoder=None, grammar=None, decoder=None, **kwargs) -> str: + """Returns a string where the *module* object has been serialized + to PVL syntax. + + :param module: a ``PVLModule`` or ``dict`` like object to serialize. + :param encoder: defaults to :class:`pvl.parser.PDSLabelEncoder()`. + :param grammar: defaults to :class:`pvl.grammar.ODLGrammar()`. + :param decoder: defaults to :class:`pvl.decoder.ODLDecoder()`. + :param ``**kwargs``: the keyword arguments to pass to the encoder + class if *encoder* is none. + """ + if encoder is None: + encoder = PDSLabelEncoder( + grammar=grammar, + decoder=decoder, + group_class=PVLGroupNew, + object_class=PVLObjectNew, + **kwargs) + elif not isinstance(encoder, PVLEncoder): + raise TypeError("The encoder must be an instance of pvl.PVLEncoder.") + + return encoder.encode(module) diff --git a/hirise_blender/pvl/parser.py b/hirise_blender/pvl/parser.py new file mode 100644 index 0000000..c7f312a --- /dev/null +++ b/hirise_blender/pvl/parser.py @@ -0,0 +1,955 @@ +# -*- coding: utf-8 -*- +"""Parameter Value Language parser. + +The definition of PVL used in this module is based on the Consultive +Committee for Space Data Systems, and their Parameter Value +Language Specification (CCSD0006 and CCSD0008), CCSDS 6441.0-B-2, +referred to as the Blue Book with a date of June 2000. + +Some of the documention in this module represents the structure +diagrams from the Blue Book for parsing PVL in a Backus–Naur +form. + +So Figure 1-1 from the Blue Book would be represented as : + + ::= ( [ + | ] )* + +Finally, the Blue Book defines as a possibly empty collection +of white space characters or comments: + + ::= ( | )* + +However, to help remember that could be empty, we will typically +always show it as *. + +Likewise the is defined as: + + ::= * [ ';' | ] + +However, since all elements are optional, we will typically +show it as []. + +The parser deals with managing the tokens that come out of the lexer. +Once the parser gets to a state where it has something that needs to +be converted to a Python object and returned, it uses the decoder to +make that conversion. + +Throughout this module, various parser functions will take a *tokens: +collections.abc.Generator* parameter. In all cases, *tokens* is +expected to be a *generator iterator* which provides ``pvl.token.Token`` +objects. It should allow for a generated object to be 'returned' +via the generator's send() function. When parsing the first object +from *tokens*, if an unexpected object is encountered, it will +'return' the object to *tokens*, and raise a ``ValueError``, so +that ``try``-``except`` blocks can be used, and the *generator +iterator* is left in a good state. However, if a parsing anomaly +is discovered deeper in parsing a PVL sequence, then a ``ValueError`` +will be thrown into the *tokens* generator iterator (via .throw()). +""" + +# Copyright 2015, 2017, 2019-2020, ``pvl`` library authors. +# +# Reuse is permitted under the terms of the license. +# The AUTHORS file and the LICENSE file are at the +# top level of this library. + +import collections.abc as abc +import re + +from .collections import MutableMappingSequence, PVLModule, PVLGroup, PVLObject +from .token import Token +from .grammar import PVLGrammar, OmniGrammar +from .decoder import PVLDecoder, OmniDecoder +from .lexer import lexer as Lexer +from .exceptions import LexerError, ParseError, linecount + + +class EmptyValueAtLine(str): + """Empty string to be used as a placeholder for a parameter without + a value. + + When a label contains a parameter without a value, it is normally + considered a broken label in PVL. To allow parsing to continue, + we can rectify the broken parameter-value pair by setting the + value to have a value of EmptyValueAtLine, which is an empty + string (and can be treated as such) with some additional properties. + + The argument *lineno* should be the line number of the error from + the original document, which will be available as a property. + + Examples:: + >>> from pvl.parser import EmptyValueAtLine + >>> EV1 = EmptyValueAtLine(1) + >>> EV1 + EmptyValueAtLine(1 does not have a value. Treat as an empty string) + >>> EV1.lineno + 1 + >>> print(EV1) + + + >>> EV1 + 'foo' + 'foo' + >>> # Can be turned into an integer and float as 0: + >>> int(EV1) + 0 + >>> float(EV1) + 0.0 + """ + + def __new__(cls, lineno, *args, **kwargs): + self = super(EmptyValueAtLine, cls).__new__(cls, "") + self.lineno = lineno + return self + + def __int__(self): + return 0 + + def __float__(self): + return 0.0 + + def __repr__(self): + return ( + "{}({} does not ".format(type(self).__name__, self.lineno) + + "have a value. Treat as an empty string)" + ) + + +class PVLParser(object): + """A parser based on the rules in the CCSDS-641.0-B-2 'Blue Book' + which defines the PVL language. + + :param grammar: A pvl.grammar object, if None or not specified, it will + be set to the grammar parameter of *decoder* (if + *decoder* is not None) or will default to + :class:`pvl.grammar.OmniGrammar()`. + :param decoder: defaults to :class:`pvl.decoder.OmniDecoder()`. + :param lexer_fn: must be a lexer function that takes a ``str``, + a grammar, and a decoder, as :func:`pvl.lexer.lexer()` does, + which is the default if none is given. + :param module_class: must be a subclass of PVLModule, and is the type + of object that will be returned from this parser's :func:`parse()` + function. + :param group_class: must be a subclass of PVLGroup, and is the type + that will be used to hold PVL elements when a PVL Group is + encountered during parsing, and must be able to be added to + via an ``.append()`` function which should take a two-tuple + of name and value. + :param object_class: must be a subclass of PVLObject, and is the type + that will be used to hold PVL elements when a PVL Object is + encountered during parsing, otherwise similar to *group_class*. + """ + + def __init__( + self, + grammar=None, + decoder=None, + lexer_fn=None, + module_class=PVLModule, + group_class=PVLGroup, + object_class=PVLObject, + ): + + self.errors = [] + self.doc = "" + + if lexer_fn is None: + self.lexer = Lexer + else: + self.lexer = lexer_fn + + if grammar is None: + if decoder is not None: + self.grammar = decoder.grammar + else: + self.grammar = OmniGrammar() + elif isinstance(grammar, PVLGrammar): + self.grammar = grammar + else: + raise TypeError("The grammar must be an instance of pvl.grammar.") + + if decoder is None: + self.decoder = OmniDecoder(grammar=self.grammar) + elif isinstance(decoder, PVLDecoder): + self.decoder = decoder + else: + raise TypeError( + "The decode must be an instance of pvl.PVLDecoder." + ) + + if issubclass(module_class, MutableMappingSequence): + self.modcls = module_class + else: + raise TypeError( + "The module_class must be a " + "pvl.collections.MutableMappingSequence." + ) + + if issubclass(group_class, MutableMappingSequence): + self.grpcls = group_class + else: + raise TypeError( + "The group_class must be a " + "pvl.collections.MutableMappingSequence." + ) + + if issubclass(object_class, MutableMappingSequence): + self.objcls = object_class + else: + raise TypeError( + "The object_class must be a " + "pvl.collections.MutableMappingSequence." + ) + + def parse(self, s: str): + """Converts the string, *s* to a PVLModule.""" + self.doc = s + tokens = self.lexer(s, g=self.grammar, d=self.decoder) + module = self.parse_module(tokens) + module.errors = sorted(self.errors) + return module + + def aggregation_cls(self, begin: str): + """Returns an initiated object of the group_class or object_class + as specified on this parser's creation, according to the value + of *begin*. If *begin* does not match the Group or Object + keywords for this parser's grammar, then it will raise a + ValueError. + """ + begin_fold = begin.casefold() + for gk in self.grammar.group_keywords.keys(): + if begin_fold == gk.casefold(): + return self.grpcls() + + for ok in self.grammar.object_keywords.keys(): + if begin_fold == ok.casefold(): + return self.objcls() + + raise ValueError( + f'The value "{begin}" did not match a Begin ' + "Aggregation Statement." + ) + + def parse_module(self, tokens: abc.Generator): + """Parses the tokens for a PVL Module. + + ::= + ( | * | )* + [] + + """ + m = self.modcls() + + parsing = True + while parsing: + # print(f'top of while parsing: {m}') + parsing = False + for p in ( + self.parse_aggregation_block, + self.parse_assignment_statement, + self.parse_end_statement, + ): + try: + self.parse_WSC_until(None, tokens) + # t = next(tokens) + # print(f'next token: {t}, {t.pos}') + # tokens.send(t) + parsed = p(tokens) + # print(f'parsed: {parsed}') + if parsed is None: # because parse_end_statement returned + return m + else: + m.append(*parsed) + parsing = True + except LexerError: + raise + except ValueError: + pass + try: + (m, keep_parsing) = self.parse_module_post_hook(m, tokens) + if keep_parsing: + parsing = True + else: + return m + except Exception: + pass + + # print(f'got to bottom: {m}') + t = next(tokens) + tokens.throw( + ValueError, + "Expecting an Aggregation Block, an Assignment " + "Statement, or an End Statement, but found " + f'"{t}" ', + ) + + def parse_module_post_hook( + self, module: MutableMappingSequence, tokens: abc.Generator + ): + """This function is meant to be overridden by subclasses + that may want to perform some extra processing if + 'normal' parse_module() operations fail to complete. + See OmniParser for an example. + + This function shall return a two-tuple, with the first item + being the *module* (altered by processing or unaltered), and + the second item being a boolean that will signal whether + the tokens should continue to be parsed to accumulate more + elements into the returned *module*, or whether the + *module* is in a good state and should be returned by + parse_module(). + + If the operations within this function are unsuccessful, + it should raise an exception (any exception descended from + Exception), which will result in the operation of parse_module() + as if it were not overridden. + """ + raise Exception + + def parse_aggregation_block(self, tokens: abc.Generator): # noqa: C901 + """Parses the tokens for an Aggregation Block, and returns + the modcls object that is the result of the parsing and + decoding. + + ::= + (* (Assignment-Statement | Aggregation-Block) *)+ + + + The Begin-Aggregation-Statement Name must match the Block-Name + in the paired End-Aggregation-Statement if a Block-Name is + present in the End-Aggregation-Statement. + """ + (begin, block_name) = self.parse_begin_aggregation_statement(tokens) + + agg = self.aggregation_cls(begin) + + while True: + self.parse_WSC_until(None, tokens) + try: + agg.append(*self.parse_aggregation_block(tokens)) + except LexerError: + raise + except ValueError: + try: + agg.append(*self.parse_assignment_statement(tokens)) + # print(f'agg: {agg}') + # t = next(tokens) + # print(f'next token is: {t}') + # tokens.send(t) + except LexerError: + raise + except ValueError: + # t = next(tokens) + # print(f'parsing agg block, next token is: {t}') + # tokens.send(t) + try: + self.parse_end_aggregation(begin, block_name, tokens) + break + except LexerError: + raise + except ValueError as ve: + try: + (agg, keep_parsing) = self.parse_module_post_hook( + agg, tokens + ) + if not keep_parsing: + raise ve + except Exception: + raise ve + + return block_name, agg + + def parse_around_equals(self, tokens: abc.Generator) -> None: + """Parses white space and comments on either side + of an equals sign. + + *tokens* is expected to be a *generator iterator* which + provides ``pvl.token`` objects. + + This is shared functionality for Begin Aggregation Statements + and Assignment Statements. It basically covers parsing + anything that has a syntax diagram like this: + + * '=' * + + """ + if not self.parse_WSC_until("=", tokens): + try: + t = next(tokens) + tokens.send(t) + raise ValueError(f'Expecting "=", got: {t}') + except StopIteration: + raise ParseError('Expecting "=", but ran out of tokens.') + + self.parse_WSC_until(None, tokens) + return + + def parse_begin_aggregation_statement( + self, tokens: abc.Generator + ) -> tuple: + """Parses the tokens for a Begin Aggregation Statement, and returns + the name Block Name as a ``str``. + + ::= + * '=' * + [] + + Where ::= + + """ + try: + begin = next(tokens) + if not begin.is_begin_aggregation(): + tokens.send(begin) + raise ValueError( + "Expecting a Begin-Aggegation-Statement, but " + f"found: {begin}" + ) + except StopIteration: + raise ValueError( + "Ran out of tokens before starting to parse " + "a Begin-Aggegation-Statement." + ) + + try: + self.parse_around_equals(tokens) + except ValueError: + tokens.throw( + ValueError, f'Expecting an equals sign after "{begin}" ' + ) + + block_name = next(tokens) + if not block_name.is_parameter_name(): + tokens.throw( + ValueError, + f'Expecting a Block-Name after "{begin} =" ' + f'but found: "{block_name}"', + ) + + self.parse_statement_delimiter(tokens) + + return begin, str(block_name) + + def parse_end_aggregation( + self, begin_agg: str, block_name: str, tokens: abc.Generator + ) -> None: + """Parses the tokens for an End Aggregation Statement. + + ::= + [* '=' * + ] [] + + Where ::= + + """ + end_agg = next(tokens) + + # Need to do a little song and dance to case-independently + # match the keys: + for k in self.grammar.aggregation_keywords.keys(): + if k.casefold() == begin_agg.casefold(): + truecase_begin = k + break + if ( + end_agg.casefold() + != self.grammar.aggregation_keywords[truecase_begin].casefold() + ): + tokens.send(end_agg) + raise ValueError( + "Expecting an End-Aggegation-Statement that " + "matched the Begin-Aggregation_Statement, " + f'"{begin_agg}" but found: {end_agg}' + ) + + try: + self.parse_around_equals(tokens) + except (ParseError, ValueError): # No equals statement, which is fine. + self.parse_statement_delimiter(tokens) + return None + + t = next(tokens) + if t != block_name: + tokens.send(t) + tokens.throw( + ValueError, + f'Expecting a Block-Name after "{end_agg} =" ' + f'that matches "{block_name}", but found: ' + f'"{t}"', + ) + + self.parse_statement_delimiter(tokens) + + return None + + def parse_end_statement(self, tokens: abc.Generator) -> None: + """Parses the tokens for an End Statement. + + ::= "END" ( * | [] ) + + """ + try: + end = next(tokens) + if not end.is_end_statement(): + tokens.send(end) + raise ValueError( + "Expecting an End Statement, like " + f'"{self.grammar.end_statements}" but found ' + f'"{end}"' + ) + + # The following commented code was originally put in place to deal + # with the possible future situation of being able to process + # the possible comment after an end-statement. + # In practice, an edge case was discovered (Issue 104) where "data" + # after an END statement *all* properly converted to UTF with no + # whitespace characters. So this request for the next token + # resulted in lexing more than 100 million "valid characters" + # and did not return in a prompt manner. If we ever enable + # processing of comments, we'll have to figure out how to handle + # this case. An alternate to removing this code is to leave it + # but put in a limit on the size that a lexeme can grow to, + # but that implies an additional if-statement for each character. + # This is the better solution for now. + # try: + # t = next(tokens) + # if t.is_WSC(): + # # maybe process comment + # return + # else: + # tokens.send(t) + # return + # except LexerError: + # pass + except StopIteration: + pass + + return + + def parse_assignment_statement(self, tokens: abc.Generator) -> tuple: + """Parses the tokens for an Assignment Statement. + + The returned two-tuple contains the Parameter Name in the + first element, and the Value in the second. + + ::= * '=' * + [] + + """ + try: + t = next(tokens) + if t.is_parameter_name(): + parameter_name = str(t) + else: + tokens.send(t) + raise ValueError( + "Expecting a Parameter Name, but " f'found: "{t}"' + ) + except StopIteration: + raise ValueError( + "Ran out of tokens before starting to parse " + "an Assignment-Statement." + ) + + self.parse_around_equals(tokens) + + try: + # print(f'parameter name: {parameter_name}') + value = self.parse_value(tokens) + except StopIteration: + raise ParseError( + "Ran out of tokens to parse after the equals " + "sign in an Assignment-Statement: " + f'"{parameter_name} =".', + t, + ) + + self.parse_statement_delimiter(tokens) + + return parameter_name, value + + @staticmethod + def parse_WSC_until(token: str, tokens: abc.Generator) -> bool: + """Consumes objects from *tokens*, if the object's *.is_WSC()* + function returns *True*, it will continue until *token* is + encountered and will return *True*. If it encounters an object + that does not meet these conditions, it will 'return' that + object to *tokens* and will return *False*. + + *tokens* is expected to be a *generator iterator* which + provides ``pvl.token`` objects. + """ + for t in tokens: + if t == token: + return True + elif t.is_WSC(): + # If there's a comment, could parse here. + pass + else: + tokens.send(t) + return False + + def _parse_set_seq(self, delimiters, tokens: abc.Generator) -> list: + """The internal parsing of PVL Sets and Sequences are very + similar, and this function provides that shared logic. + + *delimiters* are a two-tuple containing the start and end + characters for the PVL Set or Sequence. + """ + t = next(tokens) + if t != delimiters[0]: + tokens.send(t) + raise ValueError( + f'Expecting a begin delimiter "{delimiters[0]}" ' + f'but found: "{t}"' + ) + set_seq = list() + # Initial WSC and/or empty + if self.parse_WSC_until(delimiters[1], tokens): + return set_seq + + # First item: + set_seq.append(self.parse_value(tokens)) + if self.parse_WSC_until(delimiters[1], tokens): + return set_seq + + # Remaining items, if any + for t in tokens: + # print(f'in loop, t: {t}, set_seq: {set_seq}') + if t == ",": + self.parse_WSC_until(None, tokens) # consume WSC after ',' + set_seq.append(self.parse_value(tokens)) + if self.parse_WSC_until(delimiters[1], tokens): + return set_seq + else: + tokens.send(t) + tokens.throw( + ValueError, + "While parsing, expected a comma (,)" f'but found: "{t}"', + ) + + def parse_set(self, tokens: abc.Generator) -> frozenset: + """Parses a PVL Set. + + ::= "{" * + [ * ( "," * * )* ] + "}" + + Returns the decoded as a Python ``frozenset``. The PVL + specification doesn't seem to indicate that a PVL Set + has distinct values (like a Python ``set``), only that the + ordering of the values is unimportant. For now, we will + implement PVL Sets as Python ``frozenset`` objects. + + They are returned as ``frozenset`` objects because PVL Sets + can contain as their elements other PVL Sets, but since Python + ``set`` objects are non-hashable, they cannot be members of a set, + however, ``frozenset`` objects can. + """ + return frozenset( + self._parse_set_seq(self.grammar.set_delimiters, tokens) + ) + + def parse_sequence(self, tokens: abc.Generator) -> list: + """Parses a PVL Sequence. + + ::= "(" * + [ * ( "," * * )* ] + ")" + + Returns the decoded as a Python ``list``. + """ + return self._parse_set_seq(self.grammar.sequence_delimiters, tokens) + + @staticmethod + def parse_statement_delimiter(tokens: abc.Generator) -> bool: + """Parses the tokens for a Statement Delimiter. + + *tokens* is expected to be a *generator iterator* which + provides ``pvl.token`` objects. + + ::= * + ( | | ';' | ) + + Although the above structure comes from Figure 2-4 + of the Blue Book, the and + elements are redundant with the presence of [WSC]* + so it can be simplified to: + + ::= * [ ';' | ] + + Typically written []. + """ + for t in tokens: + if t.is_WSC(): + # If there's a comment, could parse here. + pass + elif t.is_delimiter(): + return True + else: + tokens.send(t) # Put the next token back into the generator + return False + + def parse_value(self, tokens: abc.Generator): + """Parses PVL Values. + + ::= ( | | ) + [* ] + + Returns the decoded as an appropriate Python object. + """ + value = None + + try: + t = next(tokens) + value = self.decoder.decode_simple_value(t) + except ValueError: + tokens.send(t) + for p in ( + self.parse_set, + self.parse_sequence, + self.parse_value_post_hook, + ): + try: + value = p(tokens) + break + except LexerError: + # A LexerError is a subclass of ValueError, but + # if we get a LexerError, that's a problem and + # we need to raise it, and not let it pass. + raise + except ValueError: + # Getting a ValueError is a normal consequence of + # one of the parsing strategies not working, + # this pass allows us to go to the next one. + pass + else: + tokens.throw( + ValueError, + "Was expecting a Simple Value, or the " + "beginning of a Set or Sequence, but " + f'found: "{t}"', + ) + + # print(f'in parse_value, value is: {value}') + self.parse_WSC_until(None, tokens) + try: + return self.parse_units(value, tokens) + except (ValueError, StopIteration): + return value + + def parse_value_post_hook(self, tokens): + """This function is meant to be overridden by subclasses + that may want to perform some extra processing if + 'normal' parse_value() operations fail to yield a value. + See OmniParser for an example. + + This function shall return an appropriate Python value, + similar to what parse_value() would return. + + If the operations within this function are unsuccessful, + it should raise a ValueError which will result in the + operation of parse_value() as if it were not overridden. + """ + raise ValueError + + def parse_units(self, value, tokens: abc.Generator) -> str: + """Parses PVL Units Expression. + + ::= "<" [] + [] ">" + + and + + ::= + [ [ | ]* + ] + + Returns the *value* and the as a ``Units()`` + object. + """ + t = next(tokens) + + if not t.startswith(self.grammar.units_delimiters[0]): + tokens.send(t) + raise ValueError( + "Was expecting the start units delimiter, " + + '"{}" '.format(self.grammar.units_delimiters[0]) + + f'but found "{t}"' + ) + + if not t.endswith(self.grammar.units_delimiters[1]): + tokens.send(t) + raise ValueError( + "Was expecting the end units delimiter, " + + '"{}" '.format(self.grammar.units_delimiters[1]) + + f'at the end, but found "{t}"' + ) + + delim_strip = t.strip("".join(self.grammar.units_delimiters)) + + units_value = delim_strip.strip("".join(self.grammar.whitespace)) + + for d in self.grammar.units_delimiters: + if d in units_value: + tokens.throw( + ValueError, + "Was expecting a units character, but found a " + f'unit delimiter, "{d}" instead.', + ) + + return self.decoder.decode_quantity(value, units_value) + + +class ODLParser(PVLParser): + """A parser based on the rules in the PDS3 Standards Reference + (version 3.8, 27 Feb 2009) Chapter 12: Object Description + Language Specification and Usage. + + It extends PVLParser. + """ + + def parse_set(self, tokens: abc.Generator) -> set: + """Overrides the parent function to return + the decoded as a Python ``set``. + + The ODL specification only allows scalar_values in Sets, + since ODL Sets cannot contain other ODL Sets, an ODL Set + can be represented as a Python ``set`` (unlike PVL Sets, + which must be represented as a Python ``frozenset`` objects). + """ + return set(self._parse_set_seq(self.grammar.set_delimiters, tokens)) + + def parse_units(self, value, tokens: abc.Generator) -> str: + """Extends the parent function, since ODL only allows units + on numeric values, any others will result in a ValueError. + """ + + if isinstance(value, int) or isinstance(value, float): + return super().parse_units(value, tokens) + + else: + raise ValueError( + "ODL Units Expressions can only follow " "numeric values." + ) + + +class OmniParser(PVLParser): + """A permissive PVL/ODL/ISIS label parser that attempts to parse + all forms of "PVL" that are thrown at it. + """ + + def _empty_value(self, pos): + eq_pos = self.doc.rfind("=", 0, pos) + lc = linecount(self.doc, eq_pos) + self.errors.append(lc) + return EmptyValueAtLine(lc) + + def parse(self, s: str): + """Extends the parent function. + + If *any* line ends with a dash (-) followed by a carriage + return, form-feed, or newline, plus one or more whitespace + characters on the following line, then those characters, and + all whitespace characters that begin the next line will + be removed. + """ + nodash = re.sub(r"-[\n\r\f]\s*", "", s) + self.doc = nodash + + return super().parse(nodash) + + def parse_module_post_hook( + self, module: MutableMappingSequence, tokens: abc.Generator + ): + """Overrides the parent function to allow for more + permissive parsing. If an Assignment-Statement + is blank, then the value will be assigned an + EmptyValueAtLine object. + """ + # It enables this by checking to see if the next thing is an + # '=' which means there was an empty assignment at the previous + # equals sign, and then unwinding the stack to give the + # previous assignment the EmptyValueAtLine() object and trying + # to continue parsing. + + # print('in hook') + try: + t = next(tokens) + if t == "=" and len(module) != 0: + (last_k, last_v) = module[-1] + last_token = Token( + last_v, grammar=self.grammar, decoder=self.decoder + ) + if last_token.is_parameter_name(): + # Fix the previous entry + module.pop() + module.append(last_k, self._empty_value(t.pos)) + # Now use last_token as the parameter name + # for the next assignment, and we must + # reproduce the last part of parse-assignment: + try: + # print(f'parameter name: {last_token}') + self.parse_WSC_until(None, tokens) + value = self.parse_value(tokens) + self.parse_statement_delimiter(tokens) + module.append(str(last_token), value) + except StopIteration: + module.append( + str(last_token), self._empty_value(t.pos + 1) + ) + return module, False # return through parse_module() + else: + tokens.send(t) + else: + # The next token isn't an equals sign or the module is + # empty, so we want return the token and signal + # parse_module() that it should ignore us. + tokens.send(t) + raise Exception + + # Peeking at the next token gives us the opportunity to + # see if we're at the end of tokens, which we want to handle. + t = next(tokens) + tokens.send(t) + return module, True # keep parsing + except StopIteration: + # If we're out of tokens, that's okay. + return module, False # return through parse_module() + + def parse_assignment_statement(self, tokens: abc.Generator) -> tuple: + """Extends the parent function to allow for more + permissive parsing. If an Assignment-Statement + is blank, then the value will be assigned an + EmptyValueAtLine object. + """ + try: + return super().parse_assignment_statement(tokens) + except ParseError as err: + if err.token is not None: + after_eq = self.doc.find("=", err.token.pos) + 1 + return str(err.token), self._empty_value(after_eq) + else: + raise + + def parse_value_post_hook(self, tokens: abc.Generator): + """Overrides the parent function to allow for more + permissive parsing. + + If the next token is a reserved word or delimiter, + then it is returned to the *tokens* and an + EmptyValueAtLine object is returned as the value. + """ + + t = next(tokens) + # print(f't: {t}') + truecase_reserved = [ + x.casefold() for x in self.grammar.reserved_keywords + ] + trucase_delim = [x.casefold() for x in self.grammar.delimiters] + if t.casefold() in (truecase_reserved + trucase_delim): + # print(f'kw: {kw}') + # if kw.casefold() == t.casefold(): + # print('match') + tokens.send(t) + return self._empty_value(t.pos) + else: + raise ValueError diff --git a/hirise_blender/pvl/pvl_translate.py b/hirise_blender/pvl/pvl_translate.py new file mode 100644 index 0000000..9a786ed --- /dev/null +++ b/hirise_blender/pvl/pvl_translate.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +"""A program for converting PVL text to a specific PVL dialect. + +The ``pvl_translate`` program will read a file with PVL text (any +of the kinds of files that :func:`pvl.load` reads) or STDIN and +will convert that PVL text to a particular PVL dialect. It is not +particularly robust, and if it cannot make simple conversions, it +will raise errors. +""" + +# Copyright 2020-2021, ``pvl`` library authors. +# +# Reuse is permitted under the terms of the license. +# The AUTHORS file and the LICENSE file are at the +# top level of this library. + +import argparse +import json +import os +import sys + +import pvl +from .encoder import PVLEncoder, ODLEncoder, ISISEncoder, PDSLabelEncoder + + +class Writer(object): + """Base class for writers. Descendents must implement dump(). + """ + + def dump(self, dictlike: dict, outfile: os.PathLike): + raise Exception + + +class PVLWriter(Writer): + def __init__(self, encoder): + self.encoder = encoder + + def dump(self, dictlike: dict, outfile: os.PathLike): + return pvl.dump(dictlike, outfile, encoder=self.encoder) + + +class JSONWriter(Writer): + def dump(self, dictlike: dict, outfile: os.PathLike): + return json.dump(dictlike, outfile) + + +formats = dict( + PDS3=PVLWriter(PDSLabelEncoder()), + ODL=PVLWriter(ODLEncoder()), + ISIS=PVLWriter(ISISEncoder()), + PVL=PVLWriter(PVLEncoder()), + JSON=JSONWriter(), +) + + +def arg_parser(formats): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "-of", + "--output_format", + required=True, + choices=formats.keys(), + help="Select the format to create the new file as.", + ) + parser.add_argument( + "infile", + nargs="?", + type=argparse.FileType("r"), + default=sys.stdin, + help="file containing PVL text to translate, " "defaults to STDIN.", + ) + parser.add_argument( + "outfile", + nargs="?", + type=argparse.FileType("w"), + default=sys.stdout, + help="file to write translated PVL to, defaults " "to STDOUT.", + ) + parser.add_argument("--version", action="version", version=pvl.__version__) + return parser + + +def main(argv=None): + args = arg_parser(formats).parse_args(argv) + + some_pvl = pvl.load(args.infile) + + formats[args.output_format].dump(some_pvl, args.outfile) + return diff --git a/hirise_blender/pvl/pvl_validate.py b/hirise_blender/pvl/pvl_validate.py new file mode 100644 index 0000000..96b9cbb --- /dev/null +++ b/hirise_blender/pvl/pvl_validate.py @@ -0,0 +1,253 @@ +# -*- coding: utf-8 -*- +"""A program for testing and validating PVL text. + +The ``pvl_validate`` program will read a file with PVL text (any of +the kinds of files that :func:`pvl.load` reads) and will report +on which of the various PVL dialects were able to load that PVL +text, and then also reports on whether the ``pvl`` library can encode +the Python Objects back out to PVL text. + +You can imagine some PVL text that could be loaded, but is not able +to be written out in a particular strict PVL dialect (like PDS3 +labels). +""" + +# Copyright 2020-2021, ``pvl`` library authors. +# +# Reuse is permitted under the terms of the license. +# The AUTHORS file and the LICENSE file are at the +# top level of this library. + +import argparse +import logging +from collections import OrderedDict + +import pvl +from .lexer import LexerError +from .grammar import ( + PVLGrammar, + ODLGrammar, + PDSGrammar, + ISISGrammar, + OmniGrammar, +) +from .parser import ParseError, PVLParser, ODLParser, OmniParser +from .decoder import PVLDecoder, ODLDecoder, PDSLabelDecoder, OmniDecoder +from .encoder import PVLEncoder, ODLEncoder, ISISEncoder, PDSLabelEncoder + +# Some assembly required for the dialects. +# We are going to be explicit here, because these arguments are +# are different than the defaults for these classes, especially for the +# parsers and decoders, as we want to be strict and not permissive here. +_pvl_g = PVLGrammar() +_pvl_d = PVLDecoder(grammar=_pvl_g) +_odl_g = ODLGrammar() +_odl_d = ODLDecoder(grammar=_odl_g) +_pds_g = PDSGrammar() +_pds_d = PDSLabelDecoder(grammar=_pds_g) +_isis_g = ISISGrammar() +_isis_d = OmniDecoder(grammar=_isis_g) +_omni_g = OmniGrammar() +_omni_d = OmniDecoder(grammar=_omni_g) + +dialects = OrderedDict( + PDS3=dict( + parser=ODLParser(grammar=_pds_g, decoder=_pds_d), + grammar=_pds_g, + decoder=_pds_d, + encoder=PDSLabelEncoder(grammar=_pds_g, decoder=_pds_d), + ), + ODL=dict( + parser=ODLParser(grammar=_odl_g, decoder=_odl_d), + grammar=_odl_g, + decoder=_odl_d, + encoder=ODLEncoder(grammar=_odl_g, decoder=_odl_d), + ), + PVL=dict( + parser=PVLParser(grammar=_pvl_g, decoder=_pvl_d), + grammar=_pvl_g, + decoder=_pvl_d, + encoder=PVLEncoder(grammar=_pvl_g, decoder=_pvl_d), + ), + ISIS=dict( + parser=OmniParser(grammar=_isis_g, decoder=_isis_d), + grammar=_isis_g, + decoder=_isis_d, + encoder=ISISEncoder(grammar=_isis_g, decoder=_isis_d), + ), + Omni=dict( + parser=OmniParser(grammar=_omni_g, decoder=_omni_d), + grammar=_omni_g, + decoder=_omni_d, + encoder=PVLEncoder(grammar=_omni_g, decoder=_omni_d), + ), +) + + +def arg_parser(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument( + "-v", + "--verbose", + action="count", + default=0, + help="Will report the errors that are encountered. A second v will " + "include tracebacks for non-pvl exceptions. ", + ) + p.add_argument("--version", action="version", version=pvl.__version__) + p.add_argument( + "file", nargs="+", help="file containing PVL text to validate." + ) + return p + + +def main(argv=None): + args = arg_parser().parse_args(argv) + + logging.basicConfig( + format="%(levelname)s: %(message)s", level=(60 - 20 * args.verbose) + ) + + results_list = list() + for f in args.file: + pvl_text = pvl.get_text_from(f) + + results = dict() + + for k, v in dialects.items(): + results[k] = pvl_flavor(pvl_text, k, v, f, args.verbose) + + results_list.append((f, results)) + + # Writing the flavors out again to preserve order. + if args.verbose > 0: + print(f"pvl library version: {pvl.__version__}") + print(report(results_list, list(dialects.keys()))) + return + + +def pvl_flavor( + text, dialect, decenc: dict, filename, verbose=False +) -> tuple((bool, bool)): + """Returns a two-tuple of booleans which indicate + whether the *text* could be loaded and then encoded. + + The first boolean in the two-tuple indicates whether the *text* + could be loaded with the given parser, grammar, and decoder. + The second indicates whether the loaded PVL object could be + encoded with the given encoder, grammar, and decoder. If the + first element is False, the second will be None. + """ + loads = None + encodes = None + try: + some_pvl = pvl.loads(text, **decenc) + loads = True + + try: + pvl.dumps(some_pvl, **decenc) + encodes = True + except (LexerError, ParseError, ValueError) as err: + logging.error(f"{dialect} encode error {filename} {err}") + encodes = False + except (LexerError, ParseError) as err: + logging.error(f"{dialect} load error {filename} {err}") + loads = False + except: # noqa E722 + if verbose <= 1: + logging.error( + f"{dialect} load error {filename}, try -vv for more info." + ) + else: + logging.exception(f"{dialect} load error {filename}") + logging.error(f"End {dialect} load error {filename}") + loads = False + + return loads, encodes + + +def report(reports: list, flavors: list) -> str: + """Returns a multi-line string which is the + pretty-printed report given the list of + *reports*. + """ + if len(reports[0][1]) != len(flavors): + raise IndexError( + "The length of the report list keys " + f"({len(reports[0][1])}) " + "and the length of the flavors list " + f"({len(flavors)}) aren't the same." + ) + + if len(reports) > 1: + return report_many(reports, flavors) + + r = reports[0][1] + + lines = list() + loads = {True: "Loads", False: "does NOT load"} + encodes = {True: "Encodes", False: "does NOT encode", None: ""} + + col1w = len(max(flavors, key=len)) + col2w = len(max(loads.values(), key=len)) + col3w = len(max(encodes.values(), key=len)) + + for k in flavors: + lines.append( + build_line( + [k, loads[r[k][0]], encodes[r[k][1]]], [col1w, col2w, col3w] + ) + ) + return "\n".join(lines) + + +def report_many(r_list: list, flavors: list) -> str: + """Returns a multi-line, table-like string which + is the pretty-printed report of the items in *r_list*. + """ + + lines = list() + loads = {True: "L", False: "No L"} + encodes = {True: "E", False: "No E", None: ""} + + col1w = len(max([x[0] for x in r_list], key=len)) + col2w = len(max(loads.values(), key=len)) + col3w = len(max(encodes.values(), key=len)) + flavorw = col2w + col3w + 1 + + header = ["File"] + flavors + headerw = [col1w] + [flavorw] * len(flavors) + rule = [" " * col1w] + [" " * flavorw] * len(flavors) + + rule_line = build_line(rule, headerw).replace("|", "+").replace(" ", "-") + lines.append(rule_line) + lines.append(build_line(header, headerw)) + lines.append(rule_line) + + for r in r_list: + cells = [r[0]] + widths = [col1w] + for f in flavors: + # cells.append(loads[r[1][f][0]] + ' ' + encodes[r[1][f][1]]) + cells.append( + "{0:^{w2}} {1:^{w3}}".format( + loads[r[1][f][0]], encodes[r[1][f][1]], w2=col2w, w3=col3w + ) + ) + widths.append(flavorw) + lines.append(build_line(cells, widths)) + + return "\n".join(lines) + + +def build_line(elements: list, widths: list, sep=" | ") -> str: + """Returns a string formatted from the *elements* and *widths* + provided. + """ + cells = list() + cells.append("{0:<{width}}".format(elements[0], width=widths[0])) + + for e, w in zip(elements[1:], widths[1:]): + cells.append("{0:^{width}}".format(e, width=w)) + + return sep.join(cells) diff --git a/hirise_blender/pvl/token.py b/hirise_blender/pvl/token.py new file mode 100644 index 0000000..7478554 --- /dev/null +++ b/hirise_blender/pvl/token.py @@ -0,0 +1,335 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019-2020, ``pvl`` library authors. +# +# Reuse is permitted under the terms of the license. +# The AUTHORS file and the LICENSE file are at the +# top level of this library. + + +from .decoder import PVLDecoder +from .grammar import PVLGrammar + + +class Token(str): + """A PVL-aware string. + + :var content: A string that is the Token text. + + :var grammar: A pvl.grammar object, if None or not specified, it will + be set to the grammar parameter of *decoder* (if + *decoder* is not None) or will default to PVLGrammar(). + + :var decoder: A pvl.decoder object, defaults to + PVLDecoder(grammar=*grammar*). + + :var pos: Integer that describes the starting position of this + Token in the source string, defaults to zero. + """ + + def __new__(cls, content, grammar=None, decoder=None, pos=0): + return str.__new__(cls, content) + + def __init__(self, content, grammar=None, decoder=None, pos=0): + if grammar is None: + if decoder is not None: + self.grammar = decoder.grammar + else: + self.grammar = PVLGrammar() + elif isinstance(grammar, PVLGrammar): + self.grammar = grammar + else: + raise TypeError("The grammar object is not of type PVLGrammar.") + + if decoder is None: + self.decoder = PVLDecoder(grammar=self.grammar) + elif isinstance(decoder, PVLDecoder): + self.decoder = decoder + else: + raise TypeError("The decoder object is not of type PVLDecoder.") + + self.pos = pos + + def __repr__(self): + return f"{self.__class__.__name__}('{self}', " f"'{self.grammar}')" + + def __index__(self): + if self.is_decimal(): + try: + return self.decoder.decode_non_decimal(str(self)) + except ValueError: + if int(str(self)) == float(str(self)): + return int(str(self)) + + raise ValueError(f"The {self:r} cannot be used as an index.") + + def __float__(self): + return float(self.decoder.decode_decimal(str(self))) + + def split(self, sep=None, maxsplit=-1) -> list: + """Extends ``str.split()`` that calling split() on a Token + returns a list of Tokens. + """ + str_list = super().split(sep, maxsplit) + tkn_list = list() + for t in str_list: + tkn_list.append( + Token(t, grammar=self.grammar, decoder=self.decoder) + ) + return tkn_list + + def replace(self, *args): + """Extends ``str.replace()`` to return a Token.""" + return Token( + super().replace(*args), grammar=self.grammar, decoder=self.decoder + ) + + def lstrip(self, chars=None): + """Extends ``str.lstrip()`` to strip whitespace according + to the definition of whitespace in the Token's grammar + instead of the default Python whitespace definition. + """ + return self._strip(super().lstrip, chars) + + def rstrip(self, chars=None): + """Extends ``str.rstrip()`` to strip whitespace according + to the definition of whitespace in the Token's grammar + instead of the default Python whitespace definition. + """ + return self._strip(super().rstrip, chars) + + def strip(self, chars=None): + """Extends ``str.strip()`` to strip whitespace according + to the definition of whitespace in the Token's grammar + instead of the default Python whitespace definition. + """ + return self._strip(super().strip, chars) + + def _strip(self, strip_func, chars=None): + # Shared functionality for the various strip functions. + if chars is None: + chars = "".join(self.grammar.whitespace) + return Token( + strip_func(chars), grammar=self.grammar, decoder=self.decoder + ) + + def isspace(self) -> bool: + """Overrides ``str.isspace()`` to be the same as Token's + is_space() function, so that we don't get inconsisent + behavior if someone forgets an underbar. + """ + # So that we don't get inconsisent behavior + # if someone forgets an underbar. + return self.is_space() + + def is_space(self) -> bool: + """Return true if the Token contains whitespace according + to the definition of whitespace in the Token's grammar + and there is at least one character, false otherwise. + """ + if len(self) == 0: + return False + + return all(c in self.grammar.whitespace for c in self) + + def is_WSC(self) -> bool: + """Return true if the Token is white space characters or comments + according to the Token's grammar, false otherwise. + """ + if self.is_comment(): + return True + + if self.is_space(): + return True + + for ws in reversed(self.grammar.whitespace): + temp = self.replace(ws, " ") + + return all(t.is_comment() for t in temp.split()) + + def is_comment(self) -> bool: + """Return true if the Token is a comment according to the + Token's grammar (defined as beginning and ending with + comment delimieters), false otherwise. + """ + for pair in self.grammar.comments: + if self.startswith(pair[0]) and self.endswith(pair[1]): + return True + return False + + def is_quote(self) -> bool: + """Return true if the Token is a quote character + according to the Token's grammar, false otherwise. + """ + if self in self.grammar.quotes: + return True + else: + return False + + def is_quoted_string(self) -> bool: + """Return true if the Token can be converted to a quoted + string by the Token's decoder, false otherwise. + """ + try: + self.decoder.decode_quoted_string(self) + return True + except ValueError: + return False + + def is_delimiter(self) -> bool: + """Return true if the Token is a delimiter character + (e.g. the ';' in PVL) according to the Token's grammar, + false otherwise. + """ + if self in self.grammar.delimiters: + return True + return False + + def is_begin_aggregation(self) -> bool: + """Return true if the Token is a begin aggregation + keyword (e.g. 'BEGIN_GROUP' in PVL) according to + the Token's grammar, false otherwise. + """ + for k in self.grammar.aggregation_keywords.keys(): + if self.casefold() == k.casefold(): + return True + return False + + def is_unquoted_string(self) -> bool: + """Return false if the Token has any + reserved characters, comment characters, whitespace + characters or could be interpreted as a number, + date, or time according to the Token's grammar, + true otherwise. + """ + for char in self.grammar.reserved_characters: + if char in self: + return False + + for pair in self.grammar.comments: + if pair[0] in self: + return False + if pair[1] in self: + return False + + if self.is_numeric() or self.is_datetime(): + return False + + for char in self.grammar.whitespace: + if char in self: + return False + + return True + + def is_string(self) -> bool: + """Return true if either the Token's is_quoted_string() + or is_unquoted_string() return true, false otherwise. + """ + if self.is_quoted_string() or self.is_unquoted_string(): + return True + return False + + def is_parameter_name(self) -> bool: + """Return true if the Token is an unquoted string that + isn't a reserved_keyword according to the Token's + grammar, false otherwise. + """ + for word in self.grammar.reserved_keywords: + if word.casefold() == self.casefold(): + return False + + return self.is_unquoted_string() + + def is_end_statement(self) -> bool: + """Return true if the Token matches an end statement + from its grammar, false otherwise. + """ + for e in self.grammar.end_statements: + if e.casefold() == self.casefold(): + return True + return False + + def isnumeric(self) -> bool: + """Overrides ``str.isnumeric()`` to be the same as Token's + is_numeric() function, so that we don't get inconsisent behavior + if someone forgets an underbar. + """ + return self.is_numeric() + + def is_numeric(self) -> bool: + """Return true if the Token's is_decimal() or is_non_decimal() + functions return true, false otherwise. + """ + if self.is_decimal() or self.is_non_decimal(): + return True + + return False + + def is_decimal(self) -> bool: + """Return true if the Token's decoder can convert the Token + to a decimal value, false otherwise. + """ + try: + self.decoder.decode_decimal(self) + return True + except ValueError: + return False + + def is_non_decimal(self) -> bool: + """Return true if the Token's decoder can convert the Token + to a numeric non-decimal value, false otherwise. + """ + try: + self.decoder.decode_non_decimal(self) + return True + except ValueError: + return False + + # Took these out, since some grammars allow a much wider + # range of radix values. + # + # def is_binary(self) -> bool: + # if self.grammar.binary_re.fullmatch(self) is None: + # return False + # else: + # return True + + # def is_octal(self) -> bool: + # if self.grammar.octal_re.fullmatch(self) is None: + # return False + # else: + # return True + + # def is_hex(self) -> bool: + # if self.grammar.hex_re.fullmatch(self) is None: + # return False + # else: + # return True + + def is_datetime(self) -> bool: + """Return true if the Token's decoder can convert the Token + to a datetime, false otherwise. + + Separate is_date() or is_time() functions aren't needed, + since PVL parsing doesn't distinguish between them. + If a user needs that distinction the decoder's + decode_datetime(self) function should return a datetime + time, date, or datetime object, as appropriate, and + a user can use isinstance() to check. + """ + try: + self.decoder.decode_datetime(self) + return True + except ValueError: + return False + + def is_simple_value(self) -> bool: + """Return true if the Token's decoder can convert the Token + to a 'simple value', however the decoder defines that, false + otherwise. + """ + try: + self.decoder.decode_simple_value(self) + return True + except ValueError: + return False diff --git a/hirise_blender/six/__init__.py b/hirise_blender/six/__init__.py new file mode 100644 index 0000000..9fed1ef --- /dev/null +++ b/hirise_blender/six/__init__.py @@ -0,0 +1,3 @@ +from .six import string_types, integer_types + +__all__ = ["string_types", "integer_types"] \ No newline at end of file diff --git a/hirise_blender/six/__pycache__/__init__.cpython-310.pyc b/hirise_blender/six/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..0c2b13e Binary files /dev/null and b/hirise_blender/six/__pycache__/__init__.cpython-310.pyc differ diff --git a/hirise_blender/six/__pycache__/six.cpython-310.pyc b/hirise_blender/six/__pycache__/six.cpython-310.pyc new file mode 100644 index 0000000..87452d0 Binary files /dev/null and b/hirise_blender/six/__pycache__/six.cpython-310.pyc differ diff --git a/hirise_blender/six/six.py b/hirise_blender/six/six.py new file mode 100644 index 0000000..d4fe984 --- /dev/null +++ b/hirise_blender/six/six.py @@ -0,0 +1,998 @@ +# Copyright (c) 2010-2020 Benjamin Peterson +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +"""Utilities for writing code that runs on Python 2 and 3""" + +from __future__ import absolute_import + +import functools +import itertools +import operator +import sys +import types + +__author__ = "Benjamin Peterson " +__version__ = "1.16.0" + + +# Useful for very coarse version differentiation. +PY2 = sys.version_info[0] == 2 +PY3 = sys.version_info[0] == 3 +PY34 = sys.version_info[0:2] >= (3, 4) + +if PY3: + string_types = str, + integer_types = int, + class_types = type, + text_type = str + binary_type = bytes + + MAXSIZE = sys.maxsize +else: + string_types = basestring, + integer_types = (int, long) + class_types = (type, types.ClassType) + text_type = unicode + binary_type = str + + if sys.platform.startswith("java"): + # Jython always uses 32 bits. + MAXSIZE = int((1 << 31) - 1) + else: + # It's possible to have sizeof(long) != sizeof(Py_ssize_t). + class X(object): + + def __len__(self): + return 1 << 31 + try: + len(X()) + except OverflowError: + # 32-bit + MAXSIZE = int((1 << 31) - 1) + else: + # 64-bit + MAXSIZE = int((1 << 63) - 1) + del X + +if PY34: + from importlib.util import spec_from_loader +else: + spec_from_loader = None + + +def _add_doc(func, doc): + """Add documentation to a function.""" + func.__doc__ = doc + + +def _import_module(name): + """Import module, returning the module after the last dot.""" + __import__(name) + return sys.modules[name] + + +class _LazyDescr(object): + + def __init__(self, name): + self.name = name + + def __get__(self, obj, tp): + result = self._resolve() + setattr(obj, self.name, result) # Invokes __set__. + try: + # This is a bit ugly, but it avoids running this again by + # removing this descriptor. + delattr(obj.__class__, self.name) + except AttributeError: + pass + return result + + +class MovedModule(_LazyDescr): + + def __init__(self, name, old, new=None): + super(MovedModule, self).__init__(name) + if PY3: + if new is None: + new = name + self.mod = new + else: + self.mod = old + + def _resolve(self): + return _import_module(self.mod) + + def __getattr__(self, attr): + _module = self._resolve() + value = getattr(_module, attr) + setattr(self, attr, value) + return value + + +class _LazyModule(types.ModuleType): + + def __init__(self, name): + super(_LazyModule, self).__init__(name) + self.__doc__ = self.__class__.__doc__ + + def __dir__(self): + attrs = ["__doc__", "__name__"] + attrs += [attr.name for attr in self._moved_attributes] + return attrs + + # Subclasses should override this + _moved_attributes = [] + + +class MovedAttribute(_LazyDescr): + + def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None): + super(MovedAttribute, self).__init__(name) + if PY3: + if new_mod is None: + new_mod = name + self.mod = new_mod + if new_attr is None: + if old_attr is None: + new_attr = name + else: + new_attr = old_attr + self.attr = new_attr + else: + self.mod = old_mod + if old_attr is None: + old_attr = name + self.attr = old_attr + + def _resolve(self): + module = _import_module(self.mod) + return getattr(module, self.attr) + + +class _SixMetaPathImporter(object): + + """ + A meta path importer to import six.moves and its submodules. + + This class implements a PEP302 finder and loader. It should be compatible + with Python 2.5 and all existing versions of Python3 + """ + + def __init__(self, six_module_name): + self.name = six_module_name + self.known_modules = {} + + def _add_module(self, mod, *fullnames): + for fullname in fullnames: + self.known_modules[self.name + "." + fullname] = mod + + def _get_module(self, fullname): + return self.known_modules[self.name + "." + fullname] + + def find_module(self, fullname, path=None): + if fullname in self.known_modules: + return self + return None + + def find_spec(self, fullname, path, target=None): + if fullname in self.known_modules: + return spec_from_loader(fullname, self) + return None + + def __get_module(self, fullname): + try: + return self.known_modules[fullname] + except KeyError: + raise ImportError("This loader does not know module " + fullname) + + def load_module(self, fullname): + try: + # in case of a reload + return sys.modules[fullname] + except KeyError: + pass + mod = self.__get_module(fullname) + if isinstance(mod, MovedModule): + mod = mod._resolve() + else: + mod.__loader__ = self + sys.modules[fullname] = mod + return mod + + def is_package(self, fullname): + """ + Return true, if the named module is a package. + + We need this method to get correct spec objects with + Python 3.4 (see PEP451) + """ + return hasattr(self.__get_module(fullname), "__path__") + + def get_code(self, fullname): + """Return None + + Required, if is_package is implemented""" + self.__get_module(fullname) # eventually raises ImportError + return None + get_source = get_code # same as get_code + + def create_module(self, spec): + return self.load_module(spec.name) + + def exec_module(self, module): + pass + +_importer = _SixMetaPathImporter(__name__) + + +class _MovedItems(_LazyModule): + + """Lazy loading of moved objects""" + __path__ = [] # mark as package + + +_moved_attributes = [ + MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"), + MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"), + MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"), + MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"), + MovedAttribute("intern", "__builtin__", "sys"), + MovedAttribute("map", "itertools", "builtins", "imap", "map"), + MovedAttribute("getcwd", "os", "os", "getcwdu", "getcwd"), + MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"), + MovedAttribute("getoutput", "commands", "subprocess"), + MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"), + MovedAttribute("reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"), + MovedAttribute("reduce", "__builtin__", "functools"), + MovedAttribute("shlex_quote", "pipes", "shlex", "quote"), + MovedAttribute("StringIO", "StringIO", "io"), + MovedAttribute("UserDict", "UserDict", "collections", "IterableUserDict", "UserDict"), + MovedAttribute("UserList", "UserList", "collections"), + MovedAttribute("UserString", "UserString", "collections"), + MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"), + MovedAttribute("zip", "itertools", "builtins", "izip", "zip"), + MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"), + MovedModule("builtins", "__builtin__"), + MovedModule("configparser", "ConfigParser"), + MovedModule("collections_abc", "collections", "collections.abc" if sys.version_info >= (3, 3) else "collections"), + MovedModule("copyreg", "copy_reg"), + MovedModule("dbm_gnu", "gdbm", "dbm.gnu"), + MovedModule("dbm_ndbm", "dbm", "dbm.ndbm"), + MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread" if sys.version_info < (3, 9) else "_thread"), + MovedModule("http_cookiejar", "cookielib", "http.cookiejar"), + MovedModule("http_cookies", "Cookie", "http.cookies"), + MovedModule("html_entities", "htmlentitydefs", "html.entities"), + MovedModule("html_parser", "HTMLParser", "html.parser"), + MovedModule("http_client", "httplib", "http.client"), + MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"), + MovedModule("email_mime_image", "email.MIMEImage", "email.mime.image"), + MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"), + MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"), + MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"), + MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"), + MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"), + MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"), + MovedModule("cPickle", "cPickle", "pickle"), + MovedModule("queue", "Queue"), + MovedModule("reprlib", "repr"), + MovedModule("socketserver", "SocketServer"), + MovedModule("_thread", "thread", "_thread"), + MovedModule("tkinter", "Tkinter"), + MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"), + MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"), + MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"), + MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"), + MovedModule("tkinter_tix", "Tix", "tkinter.tix"), + MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"), + MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"), + MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"), + MovedModule("tkinter_colorchooser", "tkColorChooser", + "tkinter.colorchooser"), + MovedModule("tkinter_commondialog", "tkCommonDialog", + "tkinter.commondialog"), + MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"), + MovedModule("tkinter_font", "tkFont", "tkinter.font"), + MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"), + MovedModule("tkinter_tksimpledialog", "tkSimpleDialog", + "tkinter.simpledialog"), + MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"), + MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"), + MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"), + MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"), + MovedModule("xmlrpc_client", "xmlrpclib", "xmlrpc.client"), + MovedModule("xmlrpc_server", "SimpleXMLRPCServer", "xmlrpc.server"), +] +# Add windows specific modules. +if sys.platform == "win32": + _moved_attributes += [ + MovedModule("winreg", "_winreg"), + ] + +for attr in _moved_attributes: + setattr(_MovedItems, attr.name, attr) + if isinstance(attr, MovedModule): + _importer._add_module(attr, "moves." + attr.name) +del attr + +_MovedItems._moved_attributes = _moved_attributes + +moves = _MovedItems(__name__ + ".moves") +_importer._add_module(moves, "moves") + + +class Module_six_moves_urllib_parse(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_parse""" + + +_urllib_parse_moved_attributes = [ + MovedAttribute("ParseResult", "urlparse", "urllib.parse"), + MovedAttribute("SplitResult", "urlparse", "urllib.parse"), + MovedAttribute("parse_qs", "urlparse", "urllib.parse"), + MovedAttribute("parse_qsl", "urlparse", "urllib.parse"), + MovedAttribute("urldefrag", "urlparse", "urllib.parse"), + MovedAttribute("urljoin", "urlparse", "urllib.parse"), + MovedAttribute("urlparse", "urlparse", "urllib.parse"), + MovedAttribute("urlsplit", "urlparse", "urllib.parse"), + MovedAttribute("urlunparse", "urlparse", "urllib.parse"), + MovedAttribute("urlunsplit", "urlparse", "urllib.parse"), + MovedAttribute("quote", "urllib", "urllib.parse"), + MovedAttribute("quote_plus", "urllib", "urllib.parse"), + MovedAttribute("unquote", "urllib", "urllib.parse"), + MovedAttribute("unquote_plus", "urllib", "urllib.parse"), + MovedAttribute("unquote_to_bytes", "urllib", "urllib.parse", "unquote", "unquote_to_bytes"), + MovedAttribute("urlencode", "urllib", "urllib.parse"), + MovedAttribute("splitquery", "urllib", "urllib.parse"), + MovedAttribute("splittag", "urllib", "urllib.parse"), + MovedAttribute("splituser", "urllib", "urllib.parse"), + MovedAttribute("splitvalue", "urllib", "urllib.parse"), + MovedAttribute("uses_fragment", "urlparse", "urllib.parse"), + MovedAttribute("uses_netloc", "urlparse", "urllib.parse"), + MovedAttribute("uses_params", "urlparse", "urllib.parse"), + MovedAttribute("uses_query", "urlparse", "urllib.parse"), + MovedAttribute("uses_relative", "urlparse", "urllib.parse"), +] +for attr in _urllib_parse_moved_attributes: + setattr(Module_six_moves_urllib_parse, attr.name, attr) +del attr + +Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes + +_importer._add_module(Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"), + "moves.urllib_parse", "moves.urllib.parse") + + +class Module_six_moves_urllib_error(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_error""" + + +_urllib_error_moved_attributes = [ + MovedAttribute("URLError", "urllib2", "urllib.error"), + MovedAttribute("HTTPError", "urllib2", "urllib.error"), + MovedAttribute("ContentTooShortError", "urllib", "urllib.error"), +] +for attr in _urllib_error_moved_attributes: + setattr(Module_six_moves_urllib_error, attr.name, attr) +del attr + +Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes + +_importer._add_module(Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"), + "moves.urllib_error", "moves.urllib.error") + + +class Module_six_moves_urllib_request(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_request""" + + +_urllib_request_moved_attributes = [ + MovedAttribute("urlopen", "urllib2", "urllib.request"), + MovedAttribute("install_opener", "urllib2", "urllib.request"), + MovedAttribute("build_opener", "urllib2", "urllib.request"), + MovedAttribute("pathname2url", "urllib", "urllib.request"), + MovedAttribute("url2pathname", "urllib", "urllib.request"), + MovedAttribute("getproxies", "urllib", "urllib.request"), + MovedAttribute("Request", "urllib2", "urllib.request"), + MovedAttribute("OpenerDirector", "urllib2", "urllib.request"), + MovedAttribute("HTTPDefaultErrorHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPRedirectHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPCookieProcessor", "urllib2", "urllib.request"), + MovedAttribute("ProxyHandler", "urllib2", "urllib.request"), + MovedAttribute("BaseHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPPasswordMgr", "urllib2", "urllib.request"), + MovedAttribute("HTTPPasswordMgrWithDefaultRealm", "urllib2", "urllib.request"), + MovedAttribute("AbstractBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("ProxyBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("AbstractDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("ProxyDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPSHandler", "urllib2", "urllib.request"), + MovedAttribute("FileHandler", "urllib2", "urllib.request"), + MovedAttribute("FTPHandler", "urllib2", "urllib.request"), + MovedAttribute("CacheFTPHandler", "urllib2", "urllib.request"), + MovedAttribute("UnknownHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPErrorProcessor", "urllib2", "urllib.request"), + MovedAttribute("urlretrieve", "urllib", "urllib.request"), + MovedAttribute("urlcleanup", "urllib", "urllib.request"), + MovedAttribute("URLopener", "urllib", "urllib.request"), + MovedAttribute("FancyURLopener", "urllib", "urllib.request"), + MovedAttribute("proxy_bypass", "urllib", "urllib.request"), + MovedAttribute("parse_http_list", "urllib2", "urllib.request"), + MovedAttribute("parse_keqv_list", "urllib2", "urllib.request"), +] +for attr in _urllib_request_moved_attributes: + setattr(Module_six_moves_urllib_request, attr.name, attr) +del attr + +Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes + +_importer._add_module(Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"), + "moves.urllib_request", "moves.urllib.request") + + +class Module_six_moves_urllib_response(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_response""" + + +_urllib_response_moved_attributes = [ + MovedAttribute("addbase", "urllib", "urllib.response"), + MovedAttribute("addclosehook", "urllib", "urllib.response"), + MovedAttribute("addinfo", "urllib", "urllib.response"), + MovedAttribute("addinfourl", "urllib", "urllib.response"), +] +for attr in _urllib_response_moved_attributes: + setattr(Module_six_moves_urllib_response, attr.name, attr) +del attr + +Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes + +_importer._add_module(Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"), + "moves.urllib_response", "moves.urllib.response") + + +class Module_six_moves_urllib_robotparser(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_robotparser""" + + +_urllib_robotparser_moved_attributes = [ + MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"), +] +for attr in _urllib_robotparser_moved_attributes: + setattr(Module_six_moves_urllib_robotparser, attr.name, attr) +del attr + +Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes + +_importer._add_module(Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"), + "moves.urllib_robotparser", "moves.urllib.robotparser") + + +class Module_six_moves_urllib(types.ModuleType): + + """Create a six.moves.urllib namespace that resembles the Python 3 namespace""" + __path__ = [] # mark as package + parse = _importer._get_module("moves.urllib_parse") + error = _importer._get_module("moves.urllib_error") + request = _importer._get_module("moves.urllib_request") + response = _importer._get_module("moves.urllib_response") + robotparser = _importer._get_module("moves.urllib_robotparser") + + def __dir__(self): + return ['parse', 'error', 'request', 'response', 'robotparser'] + +_importer._add_module(Module_six_moves_urllib(__name__ + ".moves.urllib"), + "moves.urllib") + + +def add_move(move): + """Add an item to six.moves.""" + setattr(_MovedItems, move.name, move) + + +def remove_move(name): + """Remove item from six.moves.""" + try: + delattr(_MovedItems, name) + except AttributeError: + try: + del moves.__dict__[name] + except KeyError: + raise AttributeError("no such move, %r" % (name,)) + + +if PY3: + _meth_func = "__func__" + _meth_self = "__self__" + + _func_closure = "__closure__" + _func_code = "__code__" + _func_defaults = "__defaults__" + _func_globals = "__globals__" +else: + _meth_func = "im_func" + _meth_self = "im_self" + + _func_closure = "func_closure" + _func_code = "func_code" + _func_defaults = "func_defaults" + _func_globals = "func_globals" + + +try: + advance_iterator = next +except NameError: + def advance_iterator(it): + return it.next() +next = advance_iterator + + +try: + callable = callable +except NameError: + def callable(obj): + return any("__call__" in klass.__dict__ for klass in type(obj).__mro__) + + +if PY3: + def get_unbound_function(unbound): + return unbound + + create_bound_method = types.MethodType + + def create_unbound_method(func, cls): + return func + + Iterator = object +else: + def get_unbound_function(unbound): + return unbound.im_func + + def create_bound_method(func, obj): + return types.MethodType(func, obj, obj.__class__) + + def create_unbound_method(func, cls): + return types.MethodType(func, None, cls) + + class Iterator(object): + + def next(self): + return type(self).__next__(self) + + callable = callable +_add_doc(get_unbound_function, + """Get the function out of a possibly unbound function""") + + +get_method_function = operator.attrgetter(_meth_func) +get_method_self = operator.attrgetter(_meth_self) +get_function_closure = operator.attrgetter(_func_closure) +get_function_code = operator.attrgetter(_func_code) +get_function_defaults = operator.attrgetter(_func_defaults) +get_function_globals = operator.attrgetter(_func_globals) + + +if PY3: + def iterkeys(d, **kw): + return iter(d.keys(**kw)) + + def itervalues(d, **kw): + return iter(d.values(**kw)) + + def iteritems(d, **kw): + return iter(d.items(**kw)) + + def iterlists(d, **kw): + return iter(d.lists(**kw)) + + viewkeys = operator.methodcaller("keys") + + viewvalues = operator.methodcaller("values") + + viewitems = operator.methodcaller("items") +else: + def iterkeys(d, **kw): + return d.iterkeys(**kw) + + def itervalues(d, **kw): + return d.itervalues(**kw) + + def iteritems(d, **kw): + return d.iteritems(**kw) + + def iterlists(d, **kw): + return d.iterlists(**kw) + + viewkeys = operator.methodcaller("viewkeys") + + viewvalues = operator.methodcaller("viewvalues") + + viewitems = operator.methodcaller("viewitems") + +_add_doc(iterkeys, "Return an iterator over the keys of a dictionary.") +_add_doc(itervalues, "Return an iterator over the values of a dictionary.") +_add_doc(iteritems, + "Return an iterator over the (key, value) pairs of a dictionary.") +_add_doc(iterlists, + "Return an iterator over the (key, [values]) pairs of a dictionary.") + + +if PY3: + def b(s): + return s.encode("latin-1") + + def u(s): + return s + unichr = chr + import struct + int2byte = struct.Struct(">B").pack + del struct + byte2int = operator.itemgetter(0) + indexbytes = operator.getitem + iterbytes = iter + import io + StringIO = io.StringIO + BytesIO = io.BytesIO + del io + _assertCountEqual = "assertCountEqual" + if sys.version_info[1] <= 1: + _assertRaisesRegex = "assertRaisesRegexp" + _assertRegex = "assertRegexpMatches" + _assertNotRegex = "assertNotRegexpMatches" + else: + _assertRaisesRegex = "assertRaisesRegex" + _assertRegex = "assertRegex" + _assertNotRegex = "assertNotRegex" +else: + def b(s): + return s + # Workaround for standalone backslash + + def u(s): + return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape") + unichr = unichr + int2byte = chr + + def byte2int(bs): + return ord(bs[0]) + + def indexbytes(buf, i): + return ord(buf[i]) + iterbytes = functools.partial(itertools.imap, ord) + import StringIO + StringIO = BytesIO = StringIO.StringIO + _assertCountEqual = "assertItemsEqual" + _assertRaisesRegex = "assertRaisesRegexp" + _assertRegex = "assertRegexpMatches" + _assertNotRegex = "assertNotRegexpMatches" +_add_doc(b, """Byte literal""") +_add_doc(u, """Text literal""") + + +def assertCountEqual(self, *args, **kwargs): + return getattr(self, _assertCountEqual)(*args, **kwargs) + + +def assertRaisesRegex(self, *args, **kwargs): + return getattr(self, _assertRaisesRegex)(*args, **kwargs) + + +def assertRegex(self, *args, **kwargs): + return getattr(self, _assertRegex)(*args, **kwargs) + + +def assertNotRegex(self, *args, **kwargs): + return getattr(self, _assertNotRegex)(*args, **kwargs) + + +if PY3: + exec_ = getattr(moves.builtins, "exec") + + def reraise(tp, value, tb=None): + try: + if value is None: + value = tp() + if value.__traceback__ is not tb: + raise value.with_traceback(tb) + raise value + finally: + value = None + tb = None + +else: + def exec_(_code_, _globs_=None, _locs_=None): + """Execute code in a namespace.""" + if _globs_ is None: + frame = sys._getframe(1) + _globs_ = frame.f_globals + if _locs_ is None: + _locs_ = frame.f_locals + del frame + elif _locs_ is None: + _locs_ = _globs_ + exec("""exec _code_ in _globs_, _locs_""") + + exec_("""def reraise(tp, value, tb=None): + try: + raise tp, value, tb + finally: + tb = None +""") + + +if sys.version_info[:2] > (3,): + exec_("""def raise_from(value, from_value): + try: + raise value from from_value + finally: + value = None +""") +else: + def raise_from(value, from_value): + raise value + + +print_ = getattr(moves.builtins, "print", None) +if print_ is None: + def print_(*args, **kwargs): + """The new-style print function for Python 2.4 and 2.5.""" + fp = kwargs.pop("file", sys.stdout) + if fp is None: + return + + def write(data): + if not isinstance(data, basestring): + data = str(data) + # If the file has an encoding, encode unicode with it. + if (isinstance(fp, file) and + isinstance(data, unicode) and + fp.encoding is not None): + errors = getattr(fp, "errors", None) + if errors is None: + errors = "strict" + data = data.encode(fp.encoding, errors) + fp.write(data) + want_unicode = False + sep = kwargs.pop("sep", None) + if sep is not None: + if isinstance(sep, unicode): + want_unicode = True + elif not isinstance(sep, str): + raise TypeError("sep must be None or a string") + end = kwargs.pop("end", None) + if end is not None: + if isinstance(end, unicode): + want_unicode = True + elif not isinstance(end, str): + raise TypeError("end must be None or a string") + if kwargs: + raise TypeError("invalid keyword arguments to print()") + if not want_unicode: + for arg in args: + if isinstance(arg, unicode): + want_unicode = True + break + if want_unicode: + newline = unicode("\n") + space = unicode(" ") + else: + newline = "\n" + space = " " + if sep is None: + sep = space + if end is None: + end = newline + for i, arg in enumerate(args): + if i: + write(sep) + write(arg) + write(end) +if sys.version_info[:2] < (3, 3): + _print = print_ + + def print_(*args, **kwargs): + fp = kwargs.get("file", sys.stdout) + flush = kwargs.pop("flush", False) + _print(*args, **kwargs) + if flush and fp is not None: + fp.flush() + +_add_doc(reraise, """Reraise an exception.""") + +if sys.version_info[0:2] < (3, 4): + # This does exactly the same what the :func:`py3:functools.update_wrapper` + # function does on Python versions after 3.2. It sets the ``__wrapped__`` + # attribute on ``wrapper`` object and it doesn't raise an error if any of + # the attributes mentioned in ``assigned`` and ``updated`` are missing on + # ``wrapped`` object. + def _update_wrapper(wrapper, wrapped, + assigned=functools.WRAPPER_ASSIGNMENTS, + updated=functools.WRAPPER_UPDATES): + for attr in assigned: + try: + value = getattr(wrapped, attr) + except AttributeError: + continue + else: + setattr(wrapper, attr, value) + for attr in updated: + getattr(wrapper, attr).update(getattr(wrapped, attr, {})) + wrapper.__wrapped__ = wrapped + return wrapper + _update_wrapper.__doc__ = functools.update_wrapper.__doc__ + + def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS, + updated=functools.WRAPPER_UPDATES): + return functools.partial(_update_wrapper, wrapped=wrapped, + assigned=assigned, updated=updated) + wraps.__doc__ = functools.wraps.__doc__ + +else: + wraps = functools.wraps + + +def with_metaclass(meta, *bases): + """Create a base class with a metaclass.""" + # This requires a bit of explanation: the basic idea is to make a dummy + # metaclass for one level of class instantiation that replaces itself with + # the actual metaclass. + class metaclass(type): + + def __new__(cls, name, this_bases, d): + if sys.version_info[:2] >= (3, 7): + # This version introduced PEP 560 that requires a bit + # of extra care (we mimic what is done by __build_class__). + resolved_bases = types.resolve_bases(bases) + if resolved_bases is not bases: + d['__orig_bases__'] = bases + else: + resolved_bases = bases + return meta(name, resolved_bases, d) + + @classmethod + def __prepare__(cls, name, this_bases): + return meta.__prepare__(name, bases) + return type.__new__(metaclass, 'temporary_class', (), {}) + + +def add_metaclass(metaclass): + """Class decorator for creating a class with a metaclass.""" + def wrapper(cls): + orig_vars = cls.__dict__.copy() + slots = orig_vars.get('__slots__') + if slots is not None: + if isinstance(slots, str): + slots = [slots] + for slots_var in slots: + orig_vars.pop(slots_var) + orig_vars.pop('__dict__', None) + orig_vars.pop('__weakref__', None) + if hasattr(cls, '__qualname__'): + orig_vars['__qualname__'] = cls.__qualname__ + return metaclass(cls.__name__, cls.__bases__, orig_vars) + return wrapper + + +def ensure_binary(s, encoding='utf-8', errors='strict'): + """Coerce **s** to six.binary_type. + + For Python 2: + - `unicode` -> encoded to `str` + - `str` -> `str` + + For Python 3: + - `str` -> encoded to `bytes` + - `bytes` -> `bytes` + """ + if isinstance(s, binary_type): + return s + if isinstance(s, text_type): + return s.encode(encoding, errors) + raise TypeError("not expecting type '%s'" % type(s)) + + +def ensure_str(s, encoding='utf-8', errors='strict'): + """Coerce *s* to `str`. + + For Python 2: + - `unicode` -> encoded to `str` + - `str` -> `str` + + For Python 3: + - `str` -> `str` + - `bytes` -> decoded to `str` + """ + # Optimization: Fast return for the common case. + if type(s) is str: + return s + if PY2 and isinstance(s, text_type): + return s.encode(encoding, errors) + elif PY3 and isinstance(s, binary_type): + return s.decode(encoding, errors) + elif not isinstance(s, (text_type, binary_type)): + raise TypeError("not expecting type '%s'" % type(s)) + return s + + +def ensure_text(s, encoding='utf-8', errors='strict'): + """Coerce *s* to six.text_type. + + For Python 2: + - `unicode` -> `unicode` + - `str` -> `unicode` + + For Python 3: + - `str` -> `str` + - `bytes` -> decoded to `str` + """ + if isinstance(s, binary_type): + return s.decode(encoding, errors) + elif isinstance(s, text_type): + return s + else: + raise TypeError("not expecting type '%s'" % type(s)) + + +def python_2_unicode_compatible(klass): + """ + A class decorator that defines __unicode__ and __str__ methods under Python 2. + Under Python 3 it does nothing. + + To support Python 2 and 3 with a single code base, define a __str__ method + returning text and apply this decorator to the class. + """ + if PY2: + if '__str__' not in klass.__dict__: + raise ValueError("@python_2_unicode_compatible cannot be applied " + "to %s because it doesn't define __str__()." % + klass.__name__) + klass.__unicode__ = klass.__str__ + klass.__str__ = lambda self: self.__unicode__().encode('utf-8') + return klass + + +# Complete the moves implementation. +# This code is at the end of this module to speed up module loading. +# Turn this module into a package. +__path__ = [] # required for PEP 302 and PEP 451 +__package__ = __name__ # see PEP 366 @ReservedAssignment +if globals().get("__spec__") is not None: + __spec__.submodule_search_locations = [] # PEP 451 @UndefinedVariable +# Remove other six meta path importers, since they cause problems. This can +# happen if six is removed from sys.modules and then reloaded. (Setuptools does +# this for some reason.) +if sys.meta_path: + for i, importer in enumerate(sys.meta_path): + # Here's some real nastiness: Another "instance" of the six module might + # be floating around. Therefore, we can't use isinstance() to check for + # the six meta path importer, since the other six instance will have + # inserted an importer with different class. + if (type(importer).__name__ == "_SixMetaPathImporter" and + importer.name == __name__): + del sys.meta_path[i] + break + del i, importer +# Finally, add the importer to the meta path import hook. +sys.meta_path.append(_importer) diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..d11fc0e --- /dev/null +++ b/poetry.lock @@ -0,0 +1,180 @@ +[[package]] +name = "attrs" +version = "22.1.0" +description = "Classes Without Boilerplate" +category = "main" +optional = false +python-versions = ">=3.5" + +[package.extras] +dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy (>=0.900,!=0.940)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "sphinx", "sphinx-notfound-page", "zope.interface"] +docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] +tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "zope.interface"] +tests_no_zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] + +[[package]] +name = "colorama" +version = "0.4.5" +description = "Cross-platform colored terminal text." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "iniconfig" +version = "1.1.1" +description = "iniconfig: brain-dead simple config-ini parsing" +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "numpy" +version = "1.23.4" +description = "NumPy is the fundamental package for array computing with Python." +category = "main" +optional = false +python-versions = ">=3.8" + +[[package]] +name = "packaging" +version = "21.3" +description = "Core utilities for Python packages" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" + +[[package]] +name = "pluggy" +version = "1.0.0" +description = "plugin and hook calling mechanisms for python" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "pyparsing" +version = "3.0.9" +description = "pyparsing module - Classes and methods to define and execute parsing grammars" +category = "main" +optional = false +python-versions = ">=3.6.8" + +[package.extras] +diagrams = ["jinja2", "railroad-diagrams"] + +[[package]] +name = "pytest" +version = "7.1.3" +description = "pytest: simple powerful testing with Python" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +tomli = ">=1.0.0" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +category = "main" +optional = false +python-versions = ">=3.7" + +[metadata] +lock-version = "1.1" +python-versions = "^3.10" +content-hash = "89f4bd34fda9d9f711e38dacb160780ecc3a847c867c3a83a4580cfa683e55ea" + +[metadata.files] +attrs = [ + {file = "attrs-22.1.0-py2.py3-none-any.whl", hash = "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"}, + {file = "attrs-22.1.0.tar.gz", hash = "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6"}, +] +colorama = [ + {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, + {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, +] +iniconfig = [ + {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, + {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, +] +numpy = [ + {file = "numpy-1.23.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:95d79ada05005f6f4f337d3bb9de8a7774f259341c70bc88047a1f7b96a4bcb2"}, + {file = "numpy-1.23.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:926db372bc4ac1edf81cfb6c59e2a881606b409ddc0d0920b988174b2e2a767f"}, + {file = "numpy-1.23.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c237129f0e732885c9a6076a537e974160482eab8f10db6292e92154d4c67d71"}, + {file = "numpy-1.23.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8365b942f9c1a7d0f0dc974747d99dd0a0cdfc5949a33119caf05cb314682d3"}, + {file = "numpy-1.23.4-cp310-cp310-win32.whl", hash = "sha256:2341f4ab6dba0834b685cce16dad5f9b6606ea8a00e6da154f5dbded70fdc4dd"}, + {file = "numpy-1.23.4-cp310-cp310-win_amd64.whl", hash = "sha256:d331afac87c92373826af83d2b2b435f57b17a5c74e6268b79355b970626e329"}, + {file = "numpy-1.23.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:488a66cb667359534bc70028d653ba1cf307bae88eab5929cd707c761ff037db"}, + {file = "numpy-1.23.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ce03305dd694c4873b9429274fd41fc7eb4e0e4dea07e0af97a933b079a5814f"}, + {file = "numpy-1.23.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8981d9b5619569899666170c7c9748920f4a5005bf79c72c07d08c8a035757b0"}, + {file = "numpy-1.23.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a70a7d3ce4c0e9284e92285cba91a4a3f5214d87ee0e95928f3614a256a1488"}, + {file = "numpy-1.23.4-cp311-cp311-win32.whl", hash = "sha256:5e13030f8793e9ee42f9c7d5777465a560eb78fa7e11b1c053427f2ccab90c79"}, + {file = "numpy-1.23.4-cp311-cp311-win_amd64.whl", hash = "sha256:7607b598217745cc40f751da38ffd03512d33ec06f3523fb0b5f82e09f6f676d"}, + {file = "numpy-1.23.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7ab46e4e7ec63c8a5e6dbf5c1b9e1c92ba23a7ebecc86c336cb7bf3bd2fb10e5"}, + {file = "numpy-1.23.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a8aae2fb3180940011b4862b2dd3756616841c53db9734b27bb93813cd79fce6"}, + {file = "numpy-1.23.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c053d7557a8f022ec823196d242464b6955a7e7e5015b719e76003f63f82d0f"}, + {file = "numpy-1.23.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0882323e0ca4245eb0a3d0a74f88ce581cc33aedcfa396e415e5bba7bf05f68"}, + {file = "numpy-1.23.4-cp38-cp38-win32.whl", hash = "sha256:dada341ebb79619fe00a291185bba370c9803b1e1d7051610e01ed809ef3a4ba"}, + {file = "numpy-1.23.4-cp38-cp38-win_amd64.whl", hash = "sha256:0fe563fc8ed9dc4474cbf70742673fc4391d70f4363f917599a7fa99f042d5a8"}, + {file = "numpy-1.23.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c67b833dbccefe97cdd3f52798d430b9d3430396af7cdb2a0c32954c3ef73894"}, + {file = "numpy-1.23.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f76025acc8e2114bb664294a07ede0727aa75d63a06d2fae96bf29a81747e4a7"}, + {file = "numpy-1.23.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:12ac457b63ec8ded85d85c1e17d85efd3c2b0967ca39560b307a35a6703a4735"}, + {file = "numpy-1.23.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95de7dc7dc47a312f6feddd3da2500826defdccbc41608d0031276a24181a2c0"}, + {file = "numpy-1.23.4-cp39-cp39-win32.whl", hash = "sha256:f2f390aa4da44454db40a1f0201401f9036e8d578a25f01a6e237cea238337ef"}, + {file = "numpy-1.23.4-cp39-cp39-win_amd64.whl", hash = "sha256:f260da502d7441a45695199b4e7fd8ca87db659ba1c78f2bbf31f934fe76ae0e"}, + {file = "numpy-1.23.4-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:61be02e3bf810b60ab74e81d6d0d36246dbfb644a462458bb53b595791251911"}, + {file = "numpy-1.23.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:296d17aed51161dbad3c67ed6d164e51fcd18dbcd5dd4f9d0a9c6055dce30810"}, + {file = "numpy-1.23.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:4d52914c88b4930dafb6c48ba5115a96cbab40f45740239d9f4159c4ba779962"}, + {file = "numpy-1.23.4.tar.gz", hash = "sha256:ed2cc92af0efad20198638c69bb0fc2870a58dabfba6eb722c933b48556c686c"}, +] +packaging = [ + {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, + {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, +] +pluggy = [ + {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, + {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, +] +py = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] +pyparsing = [ + {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, + {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, +] +pytest = [ + {file = "pytest-7.1.3-py3-none-any.whl", hash = "sha256:1377bda3466d70b55e3f5cecfa55bb7cfcf219c7964629b967c37cf0bda818b7"}, + {file = "pytest-7.1.3.tar.gz", hash = "sha256:4f365fec2dff9c1162f834d9f18af1ba13062db0c708bf7b946f8a5c76180c39"}, +] +tomli = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..d2119aa --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,17 @@ +[tool.poetry] +name = "hirise-blender" +version = "0.1.0" +description = "" +authors = ["Daniel Cellucci "] +readme = "README.md" +packages = [{include = "hirise_blender"}] + +[tool.poetry.dependencies] +python = "^3.10" +numpy = "^1.23.4" +pytest = "^7.1.3" + + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api"