Source code for pyffile

#
##
##  SPDX-FileCopyrightText: © 2007-2024 Benedict Verhegghe <bverheg@gmail.com>
##  SPDX-License-Identifier: GPL-3.0-or-later
##
##  This file is part of pyFormex 3.5  (Thu Feb  8 19:11:13 CET 2024)
##  pyFormex is a tool for generating, manipulating and transforming 3D
##  geometrical models by sequences of mathematical operations.
##  Home page: https://pyformex.org
##  Project page: https://savannah.nongnu.org/projects/pyformex/
##  Development: https://gitlab.com/bverheg/pyformex
##  Distributed under the GNU General Public License version 3 or later.
##
##  This program is free software: you can redistribute it and/or modify
##  it under the terms of the GNU General Public License as published by
##  the Free Software Foundation, either version 3 of the License, or
##  (at your option) any later version.
##
##  This program is distributed in the hope that it will be useful,
##  but WITHOUT ANY WARRANTY; without even the implied warranty of
##  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
##  GNU General Public License for more details.
##
##  You should have received a copy of the GNU General Public License
##  along with this program.  If not, see http://www.gnu.org/licenses/.
##
"""pyffile.py

Functions for reading and writing a project in pyFormex legacy PYF format.
The use of the PYF format is deprecated. It is highly recommended to save
your projects in PZF or HDF5 format. See :mod:`pzffile` and :mod:`hdf5file`.
"""
import gzip
import pickle
import importlib

import pyformex as pf
from pyformex import Path

__all__ = ['PyfFile']

_signature_ = pf.fullversion()
default_protocol = pickle.DEFAULT_PROTOCOL

module_relocations = {
    'plugins.mesh': 'pyformex.mesh',
    'plugins.surface': 'pyformex.trisurface',
    'plugins.trisurface': 'pyformex.trisurface',
    'pyformex.plugins.mesh': 'pyformex.mesh',
    'pyformex.plugins.surface': 'pyformex.trisurface',
    'pyformex.plugins.trisurface': 'pyformex.trisurface',
}

class_relocations = {
    'coords.Coords': 'pyformex.coords.Coords',
    'coords.BoundVectors': 'pyformex.plugins.alt.BoundVectors',
    'elements.Element': 'pyformex.elements.ElementType',
    'elements.Line2': 'pyformex.elements.Line2',
    'elements.Tri3': 'pyformex.elements.Tri3',
    'elements.Quad4': 'pyformex.elements.Quad4',
    'mesh.Mesh': 'pyformex.mesh.Mesh',
    'formex.Formex': 'pyformex.formex.Formex'
}


[docs]def find_class(module, name): """Find a class whose name or module has changed""" if pf.debugon(pf.DEBUG.PROJECT): print(f"I want to import {name} from {module}") clas = f"{module}.{name}" if pf.debugon(pf.DEBUG.PROJECT): print("Object is", clas) if clas in class_relocations: module = class_relocations[clas] lastdot = module.rfind('.') module, name = module[:lastdot], module[lastdot+1:] if pf.debugon(pf.DEBUG.PROJECT): print(f" I will try {name} from module {module} instead") elif module in module_relocations: module = module_relocations[module] if pf.debugon(pf.DEBUG.PROJECT): print(f" I will try module {module} instead") mod = importlib.import_module(module) clas = getattr(mod, name) if pf.debugon(pf.DEBUG.PROJECT): print("Success: Got", clas.__class__.__name__) return clas
[docs]class Unpickler(pickle.Unpickler): """Customized Unpickler class""" def __init__(self, f, try_resolve=True): """Initialize the Unpickler""" pickle.Unpickler.__init__(self, f, encoding='latin1') self.try_resolve = try_resolve if not try_resolve: if pf.debugon(pf.DEBUG.PROJECT): print("NOT TRYING TO RESOLVE RELOCATIONS: " "YOU MAY GET INTO TROUBLE")
[docs] def find_class(self, module, name): if pf.debugon(pf.DEBUG.PROJECT): print(f"FIND MODULE {module} NAME {name}") clas = pickle.Unpickler.find_class(self, module, name) if not clas: clas = find_class(module, name) return clas
_old_format = """.. Old project format ------------------ This is an old format project file. Unless you need to read this project file from an older pyFormex version, we strongly advise you to convert the project file to the latest format. Otherwise future versions of pyFormex might not be able to read it back. """ # TODO: Schedule the removal of WRITING pyf files, # probably with a long deprecation period
[docs]class PyfFile: """Persistent storage of pyFormex data. The PYF format file is the persistent storage format that has been present in pyFormex since the early days. Nowadays the newer PZF format (see :class:`PzfFile`) is prefered over PYF, but we keep this format available and supported for compatibility with old projects. A pyFormex :class:`PyfFile` is a Python dict containing named data of any kind, and can be saved to a file to create persistence over different pyFormex sessions. Two formats are available: PYF and PZF. The :class:`PyfFile` class is used by pyFormex for the ``pyformex.PF`` global variable that collects variables exported from pyFormex scripts. While projects are mostly handled through the pyFormex GUI, notably the *File* menu, the user may also create and handle his own PyfFile objects from a script. Because of the way pyFormex PyfFiles are written to file, there may be problems when trying to read a project file that was created with another pyFormex version. Problems may occur if the project contains data of a class whose implementation has changed, or whose definition has been relocated. Our policy is to provide backward compatibility: newer versions of pyFormex will normally read the older project formats. Saving is always done in the newest format, and these can generally not be read back by older program versions (unless you are prepared to do some hacking). .. warning:: Compatibility issues. Occasionally you may run into problems when reading back an old project file, especially when it was created by an unreleased (development) version of pyFormex. Because pyFormex is evolving fast, we can not test the full compatibility with every revision You can file a support request on the pyFormex `support tracker`_. and we will try to add the required conversion code to pyFormex. The project files are mainly intended as a means to easily save lots of data of any kind and to restore them in the same session or a later session, to pass them to another user (with the same or later pyFormex version), to store them over a medium time period. Occasionally opening and saving back your project files with newer pyFormex versions may help to avoid read-back problems over longer time. For a problemless long time storage of Geometry type objects you may consider to write them to a pyFormex Geometry file (.pgf) instead, since this uses a stable ascii based format. It can however (currently) only store obects of class Geometry or one of its subclasses. Parameters ---------- filename: :term:`path_like` The name of the file where the PyfFile data will be saved. If the file exists (and `access` is not `w`), it should be a previously saved PyfFile and an attempt will be made to load the data from this file into the PyfFile. If this fails, an error is raised. If the file exists and `access` is `w`, it will be overwritten, destroying any previous contents. If no filename is specified, a temporary file will be created when the PyfFile is saved for the first time. The file with not be automatically deleted. The generated name can be retrieved from the filename attribute. access: str, optional The access mode of the project file: 'wr' (default), 'rw', 'w' or 'r'. If the string contains an 'r' the data from an existing file will be read into the PyfFile. If the string starts with an 'r', the file should exist. If the string contains a 'w', the data can be written back to the file. The 'r' access mode is a read-only mode. ====== =============== ============ =================== access File must exist File is read File can be written ====== =============== ============ =================== r yes yes no rw yes yes yes wr no if it exists yes w no no yes ====== =============== ============ =================== convert: bool, optional If True (default), and the file is opened for reading, an attempt is made to open old projects in a compatibility mode, doing the necessary conversions to new data formats. If convert is set to False, only the latest format can be read and older formats will generate an error. signature: str, optional A text that will be written in the header record of the file. This can e.g. be used to record format version info. compression: int (0-9) The compression level. For large data sets, compression leads to much smaller files. 0 is no compression, 9 is maximal compression. The default is 4. binary: bool If False and no compression is used, storage is done in an ASCII format, allowing to edit the file. Otherwise, storage uses a binary format. Using binary=False is deprecated. data: dict, optional A dict-like object to initialize the PyfFile contents. These data may override values read from the file. protocol: int,optional The protocol to be used in pickling the data to file. If not specified, the highest protocol supported by the pickle module is used. Examples -------- >>> d = dict(a=1,b=2,c=3,d=[1,2,3],e={'f':4,'g':5}) >>> print(d) {'a': 1, 'b': 2, 'c': 3, 'd': [1, 2, 3], 'e': ...} >>> from pyformex.filetools import TempFile >>> with TempFile() as tmp: ... PyfFile(tmp.path).save(d, quiet=True) ... d.clear() ... print(d) ... d.update(PyfFile(tmp.path).load(quiet=True)) {} >>> print(d) {'a': 1, 'b': 2, 'c': 3, 'd': [1, 2, 3], 'e': ...} """ # Historically there have been a few different formats of the # written PyfFile files (.pyf). This variable holds the latest # version. latest_format = 3 def __init__(self, filename, access='wr', *, convert=True, signature=_signature_, compression=5, binary=True, protocol=default_protocol): """Create a new PyfFile.""" self.filename = Path(filename) self.access = access self.signature = str(signature) self.gzip = compression if compression in range(1, 10) else 0 self.mode = 'b' if binary or compression > 0 else '' if protocol is None: protocol = default_protocol self.protocol = min(protocol, pickle.HIGHEST_PROTOCOL) \ if self.mode == 'b' else 0 if self.filename.exists() and self.access == 'w': # destroy existing contents self.filename.truncate() def __str__(self): return ( f"PyfFile: {self.filename}\n" f" access: {self.access}, mode: {self.mode}, gzip: {self.gzip}\n" f" signature: {self.signature}\n" f" contents: {self.contents()}\n" ) def _header_data(self): """Construct the data to be saved in the header.""" store_attr = ['signature', 'gzip', 'mode', 'autofile', '_autoscript_'] store_vals = [getattr(self, k, None) for k in store_attr] return dict([(k, v) for k, v in zip(store_attr, store_vals) if v is not None])
[docs] def save(self, project, quiet=False): """Save the project to file. Parameters ---------- project: dict The project dict to store. Make sure everything in it can be pickled. """ if 'w' not in self.access: if pf.debugon(pf.DEBUG.PROJECT): print("Not saving because PyfFile file opened readonly") return if not quiet: print( f"Saving project {self.filename} with protocol {self.protocol}, " f"mode {self.mode} and compression {self.gzip}") with self.filename.open('w'+self.mode) as f: # write header header = f"{self._header_data()}\n".encode('utf-8') f.write(header) f.flush() if self.gzip: pyf = gzip.GzipFile(mode='w'+self.mode, compresslevel=self.gzip, fileobj=f) pickle.dump(dict(project), pyf, self.protocol) pyf.close() else: pickle.dump(dict(project), f, self.protocol) self.hits = 0
[docs] def readHeader(self, f, quiet=False): """Read the header from a project file. f is the file opened for reading. Tries to read the header from different legacy formats, and if successful, adjusts the project attributes according to the values in the header. Returns the open file if successful. """ self.format = -1 fpos = f.tell() s = f.readline() if pf.debugon(pf.DEBUG.PROJECT): print("PyfFile Header:", s) # Try subsequent formats try: # newest format has header in text format header = eval(s) self.__dict__.update(header) self.format = 3 except Exception: # try OLD new format: the first pickle contains attributes try: p = pickle.load(f) self.__dict__.update(p) self.format = 2 except Exception: s = s.strip() if not quiet: print(f"Header = '{s}'") if s == 'gzip' or s == '' or 'pyFormex' in s: # transitional format self.gzip = 5 self.format = 1 # NOT SURE IF THIS IS OK, NEED EXAMPLE FILE f.seek(fpos) else: # headerless format f.seek(0) self.gzip = 0 self.format = 0 return f
[docs] def load(self, try_resolve=True, quiet=False): """Load a project from file. The loaded definitions will update the current project. Returns ------- dict: The project dict that was saved on the file. """ if not quiet: print(f"Reading project file: {self.filename}") with self.filename.open('rb') as f: f = self.readHeader(f, quiet) if self.format < self.__class__.latest_format: if not quiet: print(f"Format looks like {self.format}") pf.warning(_old_format) elif self.signature != _signature_: pf.warning( f"The project was written with {self.signature}, while you " f"are running {_signature_}. If the latter is the newer " "one, this should probably not cause any problems. Saving " "is always done in the current format of the running " "version. Save your project and this message will be " "avoided on the next reopening.") pos = f.tell() if self.gzip > 0: if not quiet: print("Unpickling gzip") pyf = gzip.GzipFile(fileobj=f, mode='rb') project = Unpickler(pyf, try_resolve).load() pyf.close() else: if not quiet: print("Unpickling clear") f.seek(pos) project = Unpickler(f, try_resolve).load() return project
[docs] def convert(self, filename=None): """Convert an old format project file. The project file is read, and if successful, is immediately saved. By default, this will overwrite the original file. If a filename is specified, the converted data are saved to that file. In both cases, access is set to 'wr', so the tha saved data can be read back immediately. """ project = self.load(try_resolve=True) print(f"GOT KEYS {list(project.keys())}") if filename is not None: self.filename = Path(filename) self.access = 'w' print(f"Will now save to {self.filename}") self.save(project)
[docs] def uncompress(self, verbose=True): # noqa: C901 """Uncompress a compressed project file. The project file is read, and if successful, is written back in uncompressed format. This allows to make conversions of the data inside. """ import utils if self.filename is None: return if verbose: print(f"Uncompressing project file: {self.filename}") with self.filename.open('rb') as f: f = self.readHeader(f) if verbose: print(self.format, self.gzip) if self.gzip: try: pyf = gzip.GzipFile(self.filename, 'r', self.gzip, f) except Exception: self.gzip = 0 if self.gzip: fn = self.filename.replace('.pyf', '_uncompressed.pyf') fu = open(fn, 'w'+self.mode) h = self._header_data() h['gzip'] = 0 fu.write(f"{h}\n") while True: x = pyf.read() if x: fu.write(x) else: break fu.close() if verbose: print(f"Uncompressed {self.filename} to {fn}") else: utils.warn("warn_project_compression")
[docs] def delete(self): """Unrecoverably delete the project file.""" if self.filename: Path(self.filename).remove()
# End