diff --git a/alfred-workflow.zip b/alfred-workflow.zip index 9c641198..fee13124 100644 Binary files a/alfred-workflow.zip and b/alfred-workflow.zip differ diff --git a/doc/howto.rst b/doc/howto.rst index 2eb09d11..e760d707 100644 --- a/doc/howto.rst +++ b/doc/howto.rst @@ -110,6 +110,8 @@ with the ``libraries`` argument: sys.exit(wf.run(main)) +.. _persistent-data: + Persistent data =============== @@ -123,7 +125,17 @@ attributes/methods to make it easier to access these directories: - :meth:`datafile(filename) ` — The full path to ``filename`` under the data directory. - :meth:`cachefile(filename) ` — The full path to ``filename`` under the cache directory. -There are also corresponding features related to the root directory of your Workflow +The cache directory may be deleted during system maintenance, and is thus only +suitable for temporary data or data that is easily recreated. +:class:`Workflow `'s cache methods reflect this, +and make it easy to replace cached data that are too old. +See :ref:`Caching data ` for more details. + +The data directory is intended for more permanent, user-generated data, or data +that cannot be otherwise easily recreated. See :ref:`Storing data ` +for details. + +There are also simliar methods related to the root directory of your Workflow (where ``info.plist`` and your code are): - :attr:`~workflow.workflow.Workflow.workflowdir` — The full path to your Workflow's root directory. @@ -135,7 +147,83 @@ may help you with development/debugging. In addition, :class:`Workflow ` also provides a convenient interface for storing persistent settings with :attr:`Workflow.settings `. +See :ref:`Settings ` and :ref:`Keychain access ` for more +information on storing settings and sensitive data. + +.. _caching-data: + +Caching data +------------ + +:class:`Workflow ` provides a few methods to simplify +caching data that is slow to retrieve or expensive to generate. The main method +is :meth:`Workflow.cached_data() `, which +takes a name under which the data should be cached, a callable to retrieve +the data if they aren't in the cache (or are too old), and a maximum age in seconds +for the cached data: + +.. code-block:: python + :linenos: + + from workflow import web, Workflow + + def get_data(): + return web.get('https://example.com/api/stuff').json() + + wf = Workflow() + data = wf.cached_data('stuff', get_data, max_age=600) + +To retrieve data only if they are in the cache, call with ``None`` as the +data-retrieval function (which is the default): + +.. code-block:: python + :linenos: + + data = wf.cached_data('stuff', max_age=600) + +**Note**: This will return ``None`` if there are no corresponding data in the +cache. + +This is useful if you want to update your cache in the background, so it doesn't +impact your Workflow's responsiveness in Alfred. (See +:ref:`the tutorial ` for an example of how to run an update +script in the background.) + +Passing ``max_age=0`` will return the cached data regardless of age. + + +.. _storing-data: + +Storing data +------------ + +:class:`Workflow ` provides two methods to store +and retrieve permanent data: +:meth:`store_data() ` and +:meth:`stored_data() `. + +.. code-block:: python + :linenos: + from workflow import Workflow + + wf = Workflow() + wf.store_data('name', data) + # data will be `None` if there is nothing stored under `name` + data = wf.stored_data('name') + +These methods do not support the data expiry features of the cached data methods, +but you can specify your own serializer for each datastore, making it simple +to store data in, e.g., JSON or YAML format. + +You should use these methods (and not the data caching ones) if the data you +are saving should not be deleted as part of system maintenance. + +If you want to specify your own file format/serializer, please see +:ref:`Serialization ` for details. + + +.. _settings: Settings -------- @@ -145,7 +233,7 @@ of :class:`dict` that automatically saves its contents to the ``settings.json`` file in your Workflow's data directory when it is changed. :class:`~workflow.workflow.Settings` can be used just like a normal :class:`dict` -with the caveat that all keys and values must be serialisable to JSON. +with the caveat that all keys and values must be serializable to JSON. If you need to store arbitrary data, you can use the :ref:`cached data API `. @@ -154,6 +242,8 @@ If you need to store data securely (such as passwords and API keys), the OS X Keychain. +.. _keychain: + Keychain access --------------- @@ -191,50 +281,6 @@ Example usage: See :ref:`the relevant part of the tutorial ` for a full example. - -.. _caching-data: - -Caching data ------------- - -:class:`Workflow ` provides a few methods to simplify -caching data that is slow to retrieve or expensive to generate. The main method -is :meth:`Workflow.cached_data() `, which -takes a name under which the data should be cached, a callable to retrieve -the data if they aren't in the cache (or are too old), and a maximum age in seconds -for the cached data: - -.. code-block:: python - :linenos: - - from workflow import web, Workflow - - def get_data(): - return web.get('https://example.com/api/stuff').json() - - wf = Workflow() - data = wf.cached_data('stuff', get_data, max_age=600) - -To only retrieve data if they are in the cache, call with ``None`` as the -data-retrieval function (which is the default): - -.. code-block:: python - :linenos: - - data = wf.cached_data('stuff', max_age=600) - -**Note**: This will return ``None`` if there are no corresponding data in the -cache. - -This is useful if you want to update your cache in the background, so it doesn't -impact your Workflow's responsiveness in Alfred. (See -:ref:`the tutorial ` for an example of how to run an update -script in the background.) - -Passing ``max_age=0`` will return the cached data regardless of age. - - - .. _filtering: Searching/filtering data @@ -525,7 +571,9 @@ are parsed). - ``workflow:opendata`` — Open the Workflow's data directory. - ``workflow:openworkflow`` — Open the Workflow's root directory (where ``info.plist`` is). - ``workflow:openterm`` — Open a Terminal window in the Workflow's root directory. -- ``workflow:delcache`` — Delete any data cached by the Workflow. +- ``workflow:reset`` — Delete the Workflow's settings, cache and saved data. +- ``workflow:delcache`` — Delete the Workflow's cache. +- ``workflow:deldata`` — Delete the Workflow's saved data. - ``workflow:delsettings`` — Delete the Workflow's settings file (which contains the data stored using :attr:`Workflow.settings `). - ``workflow:foldingon`` — Force diacritic folding in search keys (e.g. convert *ü* to *ue*) - ``workflow:foldingoff`` — Never fold diacritics in search keys @@ -540,3 +588,74 @@ You can turn off magic arguments by passing ``capture_args=False`` to :meth:`~workflow.workflow.Workflow.open_log`, :meth:`~workflow.workflow.Workflow.clear_cache` and :meth:`~workflow.workflow.Workflow.clear_settings` methods directly, perhaps assigning them to your own Keywords. + + +.. _serialization: + +Serialization +============= + +By default, both cache and data files are cached using :mod:`cPickle`. This +provides a great compromise in terms of speed and ability to store arbitrary +objects. + +When it comes to cache data, it is strongly recommended to stick with +the default. :mod:`cPickle` is very fast and fully supports standard Python +data structures (``dict``, ``list``, ``tuple``, ``set`` etc.). + +If you need the ability to customise caching, you can change the default +cache serialization format to :mod:`pickle` thus: + +.. code-block:: python + :linenos: + + wf.cache_serializer = 'pickle' + +In the case of stored data, you are free to specify either a global default +serializer of one for each individual datastore: + +.. code-block:: python + :linenos: + + # Use `pickle` as the global default serializer + wf.data_serializer = 'pickle' + + # Use the JSON serializer only for these data + wf.store_data('name', data, serializer='json') + +This is primarily so you can create files that are human-readable or useable +by non-Python programs. + +By default, ``cpickle``, ``pickle`` and ``json`` serializers are available. + +You can also register your own custom serializers using the +:class:`~workflow.workflow.SerializerManager` interface. + +To register a new serializer, call the ``register`` method of the ``workflow.manager`` +object: + +.. code-block:: python + :linenos: + + from workflow import Workflow, manager + + wf = Workflow() + manager.register('format', object_with_load_and_dump_methods) + + wf.store_data('name', data, serializer='format') + +A serializer *must* conform to this interface (like :mod:`json` and :mod:`pickle`): + +.. code-block:: python + :linenos: + + serializer.load(file_obj) + serializer.dump(obj, file_obj) + + +**Note:** The name you use for your serializer will be the file extension +of the stored file. + +The :meth:`stored_data() ` method can +automatically determine the serialization of the stored data, provided the +relevant serializer is registered. If it isn't, an exception will be raised. diff --git a/doc/tutorial2.rst b/doc/tutorial2.rst index 143afc28..09a7b4b4 100644 --- a/doc/tutorial2.rst +++ b/doc/tutorial2.rst @@ -264,7 +264,7 @@ Saving settings Saving the API key was pretty easy (1 line of code). :class:`~workflow.workflow.Settings` is a special dictionary that automatically saves itself when you change its contents. It can be used much like a normal dictionary with the caveat that all -values must be serialisable to JSON as the settings are saved as a JSON file in +values must be serializable to JSON as the settings are saved as a JSON file in the Workflow's data directory. Very simple, yes, but secure? No. A better place to save the API key would be diff --git a/tests/test_workflow.py b/tests/test_workflow.py index 77e62df8..af270934 100644 --- a/tests/test_workflow.py +++ b/tests/test_workflow.py @@ -31,7 +31,8 @@ KeychainError, MATCH_ALL, MATCH_ALLCHARS, MATCH_ATOM, MATCH_CAPITALS, MATCH_STARTSWITH, MATCH_SUBSTRING, MATCH_INITIALS_CONTAIN, - MATCH_INITIALS_STARTSWITH) + MATCH_INITIALS_STARTSWITH, + manager) # info.plist settings BUNDLE_ID = 'net.deanishe.alfred-workflow' @@ -49,6 +50,85 @@ def tearDown(): pass +class SerializerTests(unittest.TestCase): + + def setUp(self): + self.serializers = ['json', 'cpickle', 'pickle'] + self.tempdir = tempfile.mkdtemp() + + def tearDown(self): + if os.path.exists(self.tempdir): + shutil.rmtree(self.tempdir) + + def _is_serializer(self, obj): + self.assertTrue(hasattr(obj, 'load')) + self.assertTrue(hasattr(obj, 'dump')) + + def test_default_serializers(self): + """Default serializers""" + for name in self.serializers: + self._is_serializer(manager.serializer(name)) + + self.assertEqual(set(self.serializers), set(manager.serializers)) + + def test_serialization(self): + """Dump/load data""" + data = {'arg1': 'value1', 'arg2': 'value2'} + + for name in self.serializers: + serializer = manager.serializer(name) + path = os.path.join(self.tempdir, 'test.{}'.format(name)) + self.assertFalse(os.path.exists(path)) + + with open(path, 'wb') as file_obj: + serializer.dump(data, file_obj) + + self.assertTrue(os.path.exists(path)) + + with open(path, 'rb') as file_obj: + data2 = serializer.load(file_obj) + + self.assertEqual(data, data2) + + os.unlink(path) + + def test_register_unregister(self): + """Register/unregister serializers""" + serializers = {} + for name in self.serializers: + serializer = manager.serializer(name) + self._is_serializer(serializer) + + for name in self.serializers: + serializer = manager.unregister(name) + self._is_serializer(serializer) + serializers[name] = serializer + + for name in self.serializers: + self.assertEqual(manager.serializer(name), None) + + for name in self.serializers: + with self.assertRaises(ValueError): + manager.unregister(name) + + for name in self.serializers: + serializer = serializers[name] + manager.register(name, serializer) + + def test_register_invalid(self): + """Register invalid serializer""" + class Thing(object): + pass + invalid1 = Thing() + invalid2 = Thing() + setattr(invalid2, 'load', lambda x: x) + + with self.assertRaises(AttributeError): + manager.register('bork', invalid1) + with self.assertRaises(AttributeError): + manager.register('bork', invalid2) + + class WorkflowTests(unittest.TestCase): def setUp(self): @@ -106,8 +186,7 @@ def setUp(self): } def tearDown(self): - self.wf.clear_cache() - self.wf.clear_settings() + self.wf.reset() try: self.wf.delete_password(self.account) except PasswordNotFound: @@ -310,10 +389,10 @@ def somedata(): cachepath = wf.cachefile('somedir') os.makedirs(cachepath) wf.cached_data('test', somedata) - self.assertTrue(os.path.exists(wf.cachefile('test.cache'))) + self.assertTrue(os.path.exists(wf.cachefile('test.cpickle'))) with self.assertRaises(SystemExit): wf.args - self.assertFalse(os.path.exists(wf.cachefile('test.cache'))) + self.assertFalse(os.path.exists(wf.cachefile('test.cpickle'))) finally: sys.argv = oargs[:] @@ -337,7 +416,7 @@ def test_cached_data_deleted(self): self.assertEqual(data, d) ret = self.wf.cache_data('test', None) self.assertEquals(ret, None) - self.assertFalse(os.path.exists(self.wf.cachefile('test.cache'))) + self.assertFalse(os.path.exists(self.wf.cachefile('test.cpickle'))) # Test alternate code path for non-existent file self.assertEqual(self.wf.cache_data('test', None), None) @@ -395,10 +474,148 @@ def test_cache_fresh(self): self.assertTrue(self.wf.cached_data_fresh('test', max_age=10)) def test_cache_fresh_non_existent(self): - """Non-existant cache data is not fresh""" + """Non-existent cache data is not fresh""" self.assertEqual(self.wf.cached_data_fresh('popsicle', max_age=10000), False) + def test_cache_serializer(self): + """Cache serializer""" + self.assertEqual(self.wf.cache_serializer, 'cpickle') + with self.assertRaises(ValueError): + self.wf.cache_serializer = 'non-existent' + self.assertEqual(self.wf.cache_serializer, 'cpickle') + self.wf.cache_serializer = 'pickle' + self.assertEqual(self.wf.cache_serializer, 'pickle') + + def test_alternative_cache_serializer(self): + """Alternative cache serializer""" + data = {'key1': 'value1'} + self.assertEqual(self.wf.cache_serializer, 'cpickle') + self.wf.cache_data('test', data) + self.assertTrue(os.path.exists(self.wf.cachefile('test.cpickle'))) + self.assertEqual(data, self.wf.cached_data('test')) + + self.wf.cache_serializer = 'pickle' + self.assertEqual(None, self.wf.cached_data('test')) + self.wf.cache_data('test', data) + self.assertTrue(os.path.exists(self.wf.cachefile('test.pickle'))) + self.assertEqual(data, self.wf.cached_data('test')) + + self.wf.cache_serializer = 'json' + self.assertEqual(None, self.wf.cached_data('test')) + self.wf.cache_data('test', data) + self.assertTrue(os.path.exists(self.wf.cachefile('test.json'))) + self.assertEqual(data, self.wf.cached_data('test')) + + def test_custom_cache_serializer(self): + """Custom cache serializer""" + data = {'key1': 'value1'} + + class MySerializer(object): + @classmethod + def load(self, file_obj): + return json.load(file_obj) + + @classmethod + def dump(self, obj, file_obj): + return json.dump(obj, file_obj, indent=2) + + manager.register('spoons', MySerializer) + self.assertFalse(os.path.exists(self.wf.cachefile('test.spoons'))) + self.wf.cache_serializer = 'spoons' + self.wf.cache_data('test', data) + self.assertTrue(os.path.exists(self.wf.cachefile('test.spoons'))) + self.assertEqual(data, self.wf.cached_data('test')) + + def test_data_serializer(self): + """Data serializer""" + self.assertEqual(self.wf.data_serializer, 'cpickle') + with self.assertRaises(ValueError): + self.wf.data_serializer = 'non-existent' + self.assertEqual(self.wf.data_serializer, 'cpickle') + self.wf.data_serializer = 'pickle' + self.assertEqual(self.wf.data_serializer, 'pickle') + + def test_alternative_data_serializer(self): + """Alternative data serializer""" + data = {'key7': 'value7'} + + self.assertEqual(self.wf.data_serializer, 'cpickle') + self.wf.store_data('test', data) + for path in self._stored_data_paths('test', 'cpickle'): + self.assertTrue(os.path.exists(path)) + self.assertEqual(data, self.wf.stored_data('test')) + + self.wf.data_serializer = 'pickle' + self.assertEqual(data, self.wf.stored_data('test')) + self.wf.store_data('test', data) + for path in self._stored_data_paths('test', 'pickle'): + self.assertTrue(os.path.exists(path)) + self.assertEqual(data, self.wf.stored_data('test')) + + self.wf.data_serializer = 'json' + self.assertEqual(data, self.wf.stored_data('test')) + self.wf.store_data('test', data) + for path in self._stored_data_paths('test', 'json'): + self.assertTrue(os.path.exists(path)) + self.assertEqual(data, self.wf.stored_data('test')) + + def test_non_existent_stored_data(self): + """Non-existent stored data""" + self.assertIsNone(self.wf.stored_data('banjo magic')) + + def test_borked_stored_data(self): + """Borked stored data""" + data = {'key7': 'value7'} + + self.wf.store_data('test', data) + metadata, datapath = self._stored_data_paths('test', 'cpickle') + os.unlink(metadata) + self.assertEqual(self.wf.stored_data('test'), None) + + self.wf.store_data('test', data) + metadata, datapath = self._stored_data_paths('test', 'cpickle') + os.unlink(datapath) + self.assertIsNone(self.wf.stored_data('test')) + + self.wf.store_data('test', data) + metadata, datapath = self._stored_data_paths('test', 'cpickle') + with open(metadata, 'wb') as file_obj: + file_obj.write('bangers and mash') + self.wf.logger.debug('Changed format to `bangers and mash`') + with self.assertRaises(ValueError): + self.wf.stored_data('test') + + def test_reject_settings(self): + """Disallow settings.json""" + data = {'key7': 'value7'} + + self.wf.data_serializer = 'json' + + with self.assertRaises(ValueError): + self.wf.store_data('settings', data) + + def test_invalid_data_serializer(self): + """Invalid data serializer""" + data = {'key7': 'value7'} + + with self.assertRaises(ValueError): + self.wf.store_data('test', data, 'spong') + + def test_delete_stored_data(self): + """Delete stored data""" + data = {'key7': 'value7'} + + paths = self._stored_data_paths('test', 'cpickle') + + self.wf.store_data('test', data) + self.assertEqual(data, self.wf.stored_data('test')) + self.wf.store_data('test', None) + self.assertEqual(None, self.wf.stored_data('test')) + + for p in paths: + self.assertFalse(os.path.exists(p)) + def test_keychain(self): """Save/get/delete password""" self.assertRaises(PasswordNotFound, @@ -562,6 +779,12 @@ def _print_results(self, results): for item, score, rule in results: print('{!r} (rule {}) : {}'.format(item[0], rule, score)) + def _stored_data_paths(self, name, serializer): + """Return list of paths created when storing data""" + metadata = self.wf.datafile('.{}.alfred-workflow'.format(name)) + datapath = self.wf.datafile('{}.{}'.format(name, serializer)) + return [metadata, datapath] + class SettingsTests(unittest.TestCase): diff --git a/workflow/__init__.py b/workflow/__init__.py index 0bf6d3de..afa23340 100644 --- a/workflow/__init__.py +++ b/workflow/__init__.py @@ -108,7 +108,7 @@ def main(wf): """ -__version__ = '1.7.1' +__version__ = '1.8' from .workflow import Workflow, PasswordNotFound, KeychainError diff --git a/workflow/workflow.py b/workflow/workflow.py index e6917634..1e6a79f5 100644 --- a/workflow/workflow.py +++ b/workflow/workflow.py @@ -15,6 +15,49 @@ The :class:`Item` and :class:`Settings` classes are supporting classes, which are meant to be accessed via :class:`Workflow` instances. +Classes :class:`SerializerManager`, :class:`JSONSerializer`, +:class:`CPickleSerializer` and :class:`PickleSerializer` are part of +the data/cache serialization features of :class:`Workflow`, accessible +by the module-level ``manager`` object. + +To register a new serializer, do: + +.. code-block:: python + :linenos: + + from workflow import Workflow, manager + + + class MySerializer(object): + + @classmethod + def load(cls, file_obj): + # load data from file_obj + + @classmethod + def dump(cls, data, file_obj): + # write data to file_obj + + manager.register('myformat', MySerializer()) + + +The name under which you register your serializer will be used as the +file extension of any saved files. + +To set the default serializer for cached data, +set :attr:`Workflow.cache_serializer`, and to set the default +serializer for stored data, set :attr:`Workflow.data_serializer`. + +Cached data is stored in the Workflow's cache directory, which is intended +for temporary and easily regenerated data. + +Stored data is stored in the Workflow's data directory, which is intended +for data that is user-generated or not easily recreated. + +The default serializer for both cached and stored data is ``cpickle``. + +For more information, please see :ref:`Persistent data `. + """ from __future__ import print_function, unicode_literals @@ -28,7 +71,8 @@ import unicodedata import shutil import json -import cPickle as pickle +import cPickle +import pickle import time import logging import logging.handlers @@ -447,6 +491,162 @@ def isascii(text): # Implementation classes #################################################################### +class SerializerManager(object): + """Contains registered serializers. + + A configured instance of this class is available at ``workflow.manager``. + + Use :meth:`register()` to register new (or replace + existing) serializers, which you can specify by name when calling + :class:`Workflow` data storage methods. + + A ``serializer`` object must have ``load()`` and ``dump()`` methods + that work the same way as in the built-in :mod:`json` and + :mod:`pickle` libraries, i.e.: + + .. code-block:: python + :linenos: + + # Reading + data = serializer.load(open('filename', 'rb')) + # Writing + serializer.dump(data, open('filename', 'wb')) + + There are 3 pre-configured serializers: ``json``, ``pickle`` + and ``cpickle``. The default is ``cpickle``, as it is very fast and + can handle most Python objects. + + If you need custom pickling, use the ``pickle`` serializer instead. + + Be careful using ``json``: JSON only supports a subset of Python's + native data types (e.g., no ``tuple`` or :class:`set`) and + doesn't, for example, support ``dict`` keys that aren't strings. + + See the built-in :mod:`cPickle`, :mod:`pickle` and :mod:`json` + libraries for more information on the serialization formats. + + """ + + def __init__(self): + self._serializers = {} + + def register(self, name, serializer): + """Register ``serializer`` object under ``name``. + + Raises :class:`AttributeError` if ``serializer`` in invalid. + + **Note:** ``name`` will be used as the file extension of the + saved files. + + :param name: Name to register ``serializer`` under + :type name: ``unicode`` or ``str`` + :param serilializer: object with ``load()`` and ``dump()`` + methods + + """ + + # Basic validation + getattr(serializer, 'load') + getattr(serializer, 'dump') + + self._serializers[name] = serializer + + def serializer(self, name): + """Return serializer object for ``name`` or ``None`` if no such + serializer is registered + + :param name: Name of serializer to return + :type name: ``unicode`` or ``str`` + :returns: serializer object or ``None`` + + """ + + return self._serializers.get(name) + + def unregister(self, name): + """Remove registered serializer with ``name`` + + Raises a :class:`ValueError` if there is no such registered + serializer. + + :param name: Name of serializer to remove + :type name: ``unicode`` or ``str`` + :returns: serializer object + + """ + + if name not in self._serializers: + raise ValueError('No such serializer registered : {}'.format(name)) + + serializer = self._serializers[name] + del self._serializers[name] + + return serializer + + @property + def serializers(self): + """Return names of registered serializers""" + return sorted(self._serializers.keys()) + + +class JSONSerializer(object): + """Wrapper around :mod:`json`. Sets ``indent`` and ``encoding``. + + Use this serializer if you need readable data files. JSON doesn't + support Python objects as well as ``cPickle``/``pickle``, so be + careful which data you try to serialize as JSON. + + """ + + @classmethod + def load(cls, file_obj): + return json.load(file_obj) + + @classmethod + def dump(cls, obj, file_obj): + return json.dump(obj, file_obj, indent=2, encoding='utf-8') + + +class CPickleSerializer(object): + """Wrapper around :mod:`cPickle`. Sets ``protocol``. + + This is the default serializer and the best combination of speed and + flexibility. + + """ + + @classmethod + def load(cls, file_obj): + return cPickle.load(file_obj) + + @classmethod + def dump(cls, obj, file_obj): + return cPickle.dump(obj, file_obj, protocol=-1) + + +class PickleSerializer(object): + """Wrapper around :mod:`pickle`. Sets ``protocol``. + + Use this serializer if you need to add custom pickling. + + """ + + @classmethod + def load(cls, file_obj): + return pickle.load(file_obj) + + @classmethod + def dump(cls, obj, file_obj): + return pickle.dump(obj, file_obj, protocol=-1) + + +# Set up default manager and register built-in serializers +manager = SerializerManager() +manager.register('cpickle', CPickleSerializer) +manager.register('pickle', PickleSerializer) +manager.register('json', JSONSerializer) + + class Item(object): """Represents a feedback item for Alfred. Generates Alfred-compliant XML for a single item. @@ -619,6 +819,8 @@ def __init__(self, default_settings=None, input_encoding='utf-8', self._settings = None self._bundleid = None self._name = None + self._cache_serializer = 'cpickle' + self._data_serializer = 'cpickle' # info.plist should be in the directory above this one self._info_plist = self.workflowfile('info.plist') self._info = None @@ -781,9 +983,15 @@ def args(self): if 'workflow:openlog' in args: msg = 'Opening workflow log file' self.open_log() + elif 'workflow:reset' in args: + self.reset() + msg = 'Reset workflow' elif 'workflow:delcache' in args: self.clear_cache() msg = 'Deleted workflow cache' + elif 'workflow:deldata' in args: + self.clear_data() + msg = 'Deleted workflow data' elif 'workflow:delsettings' in args: self.clear_settings() msg = 'Deleted workflow settings' @@ -1007,6 +1215,191 @@ def settings(self): self._default_settings) return self._settings + @property + def cache_serializer(self): + """Name of default cache serializer. + + This serializer is used by :meth:`cache_data()` and + :meth:`cached_data()` + + See :class:`SerializerManager` for details. + + :returns: serializer name + :rtype: ``unicode`` + + """ + + return self._cache_serializer + + @cache_serializer.setter + def cache_serializer(self, serializer_name): + """Set the default cache serialization format. + + This serializer is used by :meth:`cache_data()` and + :meth:`cached_data()` + + The specified serializer must already by registered with the + :class:`SerializerManager` at `~workflow.workflow.manager`, + otherwise a :class:`ValueError` will be raised. + + :param serializer_name: Name of default serializer to use. + :type serializer_name: + + """ + + if manager.serializer(serializer_name) is None: + raise ValueError( + 'Unknown serializer : `{}`. Register your serializer ' + 'with `manager` first.'.format(serializer_name)) + + self.logger.debug( + 'default cache serializer set to `{}`'.format(serializer_name)) + + self._cache_serializer = serializer_name + + @property + def data_serializer(self): + """Name of default data serializer. + + This serializer is used by :meth:`store_data()` and + :meth:`stored_data()` + + See :class:`SerializerManager` for details. + + :returns: serializer name + :rtype: ``unicode`` + + """ + + return self._data_serializer + + @data_serializer.setter + def data_serializer(self, serializer_name): + """Set the default cache serialization format. + + This serializer is used by :meth:`store_data()` and + :meth:`stored_data()` + + The specified serializer must already by registered with the + :class:`SerializerManager` at `~workflow.workflow.manager`, + otherwise a :class:`ValueError` will be raised. + + :param serializer_name: Name of default serializer to use. + :type serializer_name: + + """ + + if manager.serializer(serializer_name) is None: + raise ValueError( + 'Unknown serializer : `{}`. Register your serializer ' + 'with `manager` first.'.format(serializer_name)) + + self.logger.debug( + 'default data serializer set to `{}`'.format(serializer_name)) + + self._data_serializer = serializer_name + + def stored_data(self, name): + """Retrieve data from data directory. Returns ``None`` if there + is no data stored. + + :param name: name of datastore + :type name: ``unicode`` + + """ + + metadata_path = self.datafile('.{}.alfred-workflow'.format(name)) + + if not os.path.exists(metadata_path): + self.logger.debug('No data stored for `{}`'.format(name)) + return None + + with open(metadata_path, 'rb') as file_obj: + serializer_name = file_obj.read().strip() + + serializer = manager.serializer(serializer_name) + + if serializer is None: + raise ValueError( + 'Unknown serializer `{}`. Register a corresponding serializer ' + 'with `manager.register()` to load this data.'.format( + serializer_name)) + + self.logger.debug('Data `{}` stored in `{}` format'.format( + name, serializer_name)) + + filename = '{}.{}'.format(name, serializer_name) + data_path = self.datafile(filename) + + if not os.path.exists(data_path): + self.logger.debug('No data stored for `{}`'.format(name)) + if os.path.exists(metadata_path): + os.unlink(metadata_path) + + return None + + with open(data_path, 'rb') as file_obj: + data = serializer.load(file_obj) + + self.logger.debug('Stored data loaded from : {}'.format(data_path)) + + return data + + def store_data(self, name, data, serializer=None): + """Save data to data directory. + + If ``data`` is ``None``, the datastore will be deleted. + + :param name: name of datastore + :type name: ``unicode`` + :param data: object(s) to store + :type data: artibrary Python objects. **Note:** some serializers + can only handled certain types of data. + :param serializer: name of serializer to use. + See :class:`SerializerManager` for more information. + :type serializer: ``unicode`` + :returns: data in datastore or ``None`` + + """ + + serializer_name = serializer or self.data_serializer + + if serializer_name == 'json' and name == 'settings': + raise ValueError( + 'Cannot save data to `settings` with format `json`. ' + "This would overwrite Alfred-Workflow's settings file.") + + serializer = manager.serializer(serializer_name) + + if serializer is None: + raise ValueError( + 'Invalid serializer `{}`. Register your serializer with ' + '`manager.register()` first.'.format(serializer_name)) + + # In order for `stored_data()` to be able to load data stored with + # an arbitrary serializer, yet still have meaningful file extensions, + # the format (i.e. extension) is saved to an accompanying file + metadata_path = self.datafile('.{}.alfred-workflow'.format(name)) + filename = '{}.{}'.format(name, serializer_name) + data_path = self.datafile(filename) + + if data is None: # Delete cached data + for path in (metadata_path, data_path): + if os.path.exists(path): + os.unlink(path) + self.logger.debug('Deleted data file : {}'.format(path)) + + return + + # Save file extension + with open(metadata_path, 'wb') as file_obj: + file_obj.write(serializer_name) + + with open(data_path, 'wb') as file_obj: + serializer.dump(data, file_obj) + + self.logger.debug('Stored data saved at : {}'.format(data_path)) + def cached_data(self, name, data_func=None, max_age=60): """Retrieve data from cache or re-generate and re-cache data if stale/non-existant. If ``max_age`` is 0, return cached data no @@ -1024,17 +1417,24 @@ def cached_data(self, name, data_func=None, max_age=60): """ - cache_path = self.cachefile('%s.cache' % name) + serializer = manager.serializer(self.cache_serializer) + + cache_path = self.cachefile('%s.%s' % (name, self.cache_serializer)) age = self.cached_data_age(name) + if (age < max_age or max_age == 0) and os.path.exists(cache_path): + with open(cache_path, 'rb') as file: self.logger.debug('Loading cached data from : %s', cache_path) - return pickle.load(file) + return serializer.load(file) + if not data_func: return None + data = data_func() self.cache_data(name, data) + return data def cache_data(self, name, data): @@ -1049,7 +1449,9 @@ def cache_data(self, name, data): """ - cache_path = self.cachefile('%s.cache' % name) + serializer = manager.serializer(self.cache_serializer) + + cache_path = self.cachefile('%s.%s' % (name, self.cache_serializer)) if data is None: if os.path.exists(cache_path): @@ -1058,7 +1460,8 @@ def cache_data(self, name, data): return with open(cache_path, 'wb') as file: - pickle.dump(data, file, protocol=-1) + serializer.dump(data, file) + self.logger.debug('Cached data saved at : %s', cache_path) def cached_data_fresh(self, name, max_age): @@ -1074,8 +1477,10 @@ def cached_data_fresh(self, name, max_age): """ age = self.cached_data_age(name) + if not age: return False + return age < max_age def cached_data_age(self, name): @@ -1089,9 +1494,11 @@ def cached_data_age(self, name): """ - cache_path = self.cachefile('%s.cache' % name) + cache_path = self.cachefile('%s.%s' % (name, self.cache_serializer)) + if not os.path.exists(cache_path): return 0 + return time.time() - os.stat(cache_path).st_mtime def filter(self, query, items, key=lambda x: x, ascending=False, @@ -1513,14 +1920,11 @@ def delete_password(self, account, service=None): def clear_cache(self): """Delete all files in workflow cache directory.""" - if os.path.exists(self.cachedir): - for filename in os.listdir(self.cachedir): - path = os.path.join(self.cachedir, filename) - if os.path.isdir(path): - shutil.rmtree(path) - else: - os.unlink(path) - self.logger.debug('Deleted : %r', path) + self._delete_directory_contents(self.cachedir) + + def clear_data(self): + """Delete all files in workflow data directory.""" + self._delete_directory_contents(self.datadir) def clear_settings(self): """Delete settings file.""" @@ -1528,6 +1932,12 @@ def clear_settings(self): os.unlink(self.settings_path) self.logger.debug('Deleted : %r', self.settings_path) + def reset(self): + """Delete settings, cache and data""" + self.clear_cache() + self.clear_data() + self.clear_settings() + def open_log(self): """Open log file in standard application (usually Console.app).""" subprocess.call(['open', self.logfile]) # pragma: no cover @@ -1606,6 +2016,23 @@ def fold_to_ascii(self, text): return unicode(unicodedata.normalize('NFKD', text).encode('ascii', 'ignore')) + def _delete_directory_contents(self, dirpath): + """Delete all files in a directory + + :param dirpath: path to directory to clear + :type dirpath: ``unicode`` or ``str`` + + """ + + if os.path.exists(dirpath): + for filename in os.listdir(dirpath): + path = os.path.join(dirpath, filename) + if os.path.isdir(path): + shutil.rmtree(path) + else: + os.unlink(path) + self.logger.debug('Deleted : %r', path) + def _load_info_plist(self): """Load workflow info from ``info.plist``