diff --git a/CHANGES.md b/CHANGES.md index a475ae52..7e0ed50e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,6 +3,10 @@ The released versions correspond to PyPi releases. ## Version 4.1.0 (as yet unreleased) +#### New Features + * Added some support for pandas (`read_csv`, `read_excel`) to work with + the fake filesystem (see [#531](../../issues/531)) + #### Fixes * Do not override global warnings setting in `Deprecator` (see [#526](../../issues/526)) diff --git a/docs/usage.rst b/docs/usage.rst index 60a618fd..f3f5be65 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -356,6 +356,17 @@ if the real user is a root user (e.g. has the user ID 0). If you want to run your tests as a non-root user regardless of the actual user rights, you may want to set this to ``False``. +use_known_patches +~~~~~~~~~~~~~~~~~ +If this is set to ``True`` (the default), ``pyfakefs`` patches some +libraries that are known to not work out of the box, to be able work with the +fake filesystem. Currently, this includes patches for the ``pandas`` methods +``read_csv`` and ``read_excel`` - more may follow. This flag is +there to be able to disable this functionality in case it causes any +problems. It may be removed or replaced by a more fine-grained argument in +future releases. + + Using convenience methods ------------------------- While ``pyfakefs`` can be used just with the standard Python file system @@ -605,9 +616,6 @@ A list of Python modules that are known to not work correctly with sufficient demand. - the ``Pillow`` image library does not work with pyfakefs at least if writing JPEG files (see `this issue `__) -- ``pandas`` (the Python data analysis library) uses its own internal file - system access, written in C, and does therefore not work with pyfakefs - (see `this issue `__) If you are not sure if a module can be handled, or how to do it, you can always write a new issue, of course! diff --git a/extra_requirements.txt b/extra_requirements.txt index 93525526..b9267c4d 100644 --- a/extra_requirements.txt +++ b/extra_requirements.txt @@ -9,5 +9,10 @@ # available at the time of writing. pathlib2>=2.3.2 - scandir>=1.8 + +# pandas + xlrd are used to test pandas-specific patches to allow +# pyfakefs to work with pandas +# we use the latest version to see any problems with new versions +pandas +xlrd diff --git a/pyfakefs/fake_filesystem_unittest.py b/pyfakefs/fake_filesystem_unittest.py index dc68e0e0..44ca7257 100644 --- a/pyfakefs/fake_filesystem_unittest.py +++ b/pyfakefs/fake_filesystem_unittest.py @@ -47,6 +47,8 @@ from pyfakefs.deprecator import Deprecator from pyfakefs.fake_filesystem import set_uid, set_gid, reset_ids from pyfakefs.helpers import IS_PYPY +from pyfakefs.patched_packages import get_modules_to_patch, \ + get_classes_to_patch, get_fake_module_classes try: from importlib.machinery import ModuleSpec @@ -74,7 +76,8 @@ def patchfs(_func=None, *, additional_skip_names=None, modules_to_reload=None, modules_to_patch=None, - allow_root_user=True): + allow_root_user=True, + use_known_patches=True): """Convenience decorator to use patcher with additional parameters in a test function. @@ -96,7 +99,8 @@ def wrapped(*args, **kwargs): additional_skip_names=additional_skip_names, modules_to_reload=modules_to_reload, modules_to_patch=modules_to_patch, - allow_root_user=allow_root_user) as p: + allow_root_user=allow_root_user, + use_known_patches=use_known_patches) as p: kwargs['fs'] = p.fs return f(*args, **kwargs) @@ -117,7 +121,8 @@ def load_doctests(loader, tests, ignore, module, additional_skip_names=None, modules_to_reload=None, modules_to_patch=None, - allow_root_user=True): # pylint: disable=unused-argument + allow_root_user=True, + use_known_patches=True): # pylint: disable=unused-argument """Load the doctest tests for the specified module into unittest. Args: loader, tests, ignore : arguments passed in from `load_tests()` @@ -129,7 +134,8 @@ def load_doctests(loader, tests, ignore, module, _patcher = Patcher(additional_skip_names=additional_skip_names, modules_to_reload=modules_to_reload, modules_to_patch=modules_to_patch, - allow_root_user=allow_root_user) + allow_root_user=allow_root_user, + use_known_patches=use_known_patches) globs = _patcher.replace_globs(vars(module)) tests.addTests(doctest.DocTestSuite(module, globs=globs, @@ -155,6 +161,8 @@ class TestCaseMixin: modules_to_patch: A dictionary of fake modules mapped to the fully qualified patched module names. Can be used to add patching of modules not provided by `pyfakefs`. + use_known_patches: If True (the default), some patches for commonly + used packges are applied which make them usable with pyfakes. If you specify some of these attributes here and you have DocTests, consider also specifying the same arguments to :py:func:`load_doctests`. @@ -190,7 +198,8 @@ def setUpPyfakefs(self, additional_skip_names=None, modules_to_reload=None, modules_to_patch=None, - allow_root_user=True): + allow_root_user=True, + use_known_patches=True): """Bind the file-related modules to the :py:class:`pyfakefs` fake file system instead of the real file system. Also bind the fake `open()` function. @@ -212,7 +221,8 @@ def setUpPyfakefs(self, additional_skip_names=additional_skip_names, modules_to_reload=modules_to_reload, modules_to_patch=modules_to_patch, - allow_root_user=allow_root_user + allow_root_user=allow_root_user, + use_known_patches=use_known_patches ) self._stubber.setUp() @@ -247,7 +257,8 @@ def __init__(self, methodName='runTest', additional_skip_names=None, modules_to_reload=None, modules_to_patch=None, - allow_root_user=True): + allow_root_user=True, + use_known_patches=True): """Creates the test class instance and the patcher used to stub out file system related modules. @@ -261,6 +272,7 @@ def __init__(self, methodName='runTest', self.modules_to_reload = modules_to_reload self.modules_to_patch = modules_to_patch self.allow_root_user = allow_root_user + self.use_known_patches = use_known_patches @Deprecator('add_real_file') def copyRealFile(self, real_file_path, fake_file_path=None, @@ -337,7 +349,7 @@ class Patcher: def __init__(self, additional_skip_names=None, modules_to_reload=None, modules_to_patch=None, - allow_root_user=True): + allow_root_user=True, use_known_patches=True): """For a description of the arguments, see TestCase.__init__""" if not allow_root_user: @@ -361,6 +373,12 @@ def __init__(self, additional_skip_names=None, self.modules_to_reload = modules_to_reload or [] + if use_known_patches: + modules_to_patch = modules_to_patch or {} + modules_to_patch.update(get_modules_to_patch()) + self._class_modules.update(get_classes_to_patch()) + self._fake_module_classes.update(get_fake_module_classes()) + if modules_to_patch is not None: for name, fake_module in modules_to_patch.items(): self._fake_module_classes[name] = fake_module @@ -516,7 +534,8 @@ def _find_modules(self): # where py.error has no __name__ attribute # see https://github.com/pytest-dev/py/issues/73 continue - + if name == 'pandas.io.parsers': + print(name) module_items = module.__dict__.copy().items() # suppress specific pytest warning - see #466 @@ -588,6 +607,8 @@ def start_patching(self): self._patching = True for name, modules in self._modules.items(): + if name == 'TextFileReader': + print(name, modules) for module, attr in modules: self._stubs.smart_set( module, name, self.fake_modules[attr]) diff --git a/pyfakefs/patched_packages.py b/pyfakefs/patched_packages.py new file mode 100644 index 00000000..f115839a --- /dev/null +++ b/pyfakefs/patched_packages.py @@ -0,0 +1,109 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Provides patches for some commonly used modules that enable them to work +with pyfakefs. +""" +import sys + +try: + import pandas.io.parsers as parsers +except ImportError: + parsers = None + +try: + import xlrd +except ImportError: + xlrd = None + + +def get_modules_to_patch(): + modules_to_patch = {} + if xlrd is not None: + modules_to_patch['xlrd'] = XLRDModule + return modules_to_patch + + +def get_classes_to_patch(): + classes_to_patch = {} + if parsers is not None: + classes_to_patch[ + 'TextFileReader' + ] = 'pandas.io.parsers' + return classes_to_patch + + +def get_fake_module_classes(): + fake_module_classes = {} + if parsers is not None: + fake_module_classes[ + 'TextFileReader' + ] = FakeTextFileReader + return fake_module_classes + + +if xlrd is not None: + class XLRDModule: + """Patches the xlrd module, which is used as the default Excel file + reader by pandas. Disables using memory mapped files, which are + implemented platform-specific on OS level.""" + + def __init__(self, _): + self._xlrd_module = xlrd + + def open_workbook(self, filename=None, + logfile=sys.stdout, + verbosity=0, + use_mmap=False, + file_contents=None, + encoding_override=None, + formatting_info=False, + on_demand=False, + ragged_rows=False): + return self._xlrd_module.open_workbook( + filename, logfile, verbosity, False, file_contents, + encoding_override, formatting_info, on_demand, ragged_rows) + + def __getattr__(self, name): + """Forwards any unfaked calls to the standard xlrd module.""" + return getattr(self._xlrd_module, name) + +if parsers is not None: + # we currently need to add fake modules for both the parser module and + # the contained text reader - maybe this can be simplified + + class FakeTextFileReader: + fake_parsers = None + + def __init__(self, filesystem): + if self.fake_parsers is None: + self.__class__.fake_parsers = ParsersModule(filesystem) + + def __call__(self, *args, **kwargs): + return self.fake_parsers.TextFileReader(*args, **kwargs) + + def __getattr__(self, name): + return getattr(self.fake_parsers.TextFileReader, name) + + class ParsersModule: + def __init__(self, _): + self._parsers_module = parsers + + class TextFileReader(parsers.TextFileReader): + def __init__(self, *args, **kwargs): + kwargs['engine'] = 'python' + super().__init__(*args, **kwargs) + + def __getattr__(self, name): + """Forwards any unfaked calls to the standard xlrd module.""" + return getattr(self._parsers_module, name) diff --git a/pyfakefs/tests/all_tests.py b/pyfakefs/tests/all_tests.py index d0971083..331a3d17 100644 --- a/pyfakefs/tests/all_tests.py +++ b/pyfakefs/tests/all_tests.py @@ -18,18 +18,22 @@ import sys import unittest -from pyfakefs.tests import dynamic_patch_test, fake_stat_time_test -from pyfakefs.tests import example_test -from pyfakefs.tests import fake_filesystem_glob_test -from pyfakefs.tests import fake_filesystem_shutil_test -from pyfakefs.tests import fake_filesystem_test -from pyfakefs.tests import fake_filesystem_unittest_test -from pyfakefs.tests import fake_filesystem_vs_real_test -from pyfakefs.tests import fake_open_test -from pyfakefs.tests import fake_os_test -from pyfakefs.tests import fake_pathlib_test -from pyfakefs.tests import fake_tempfile_test -from pyfakefs.tests import mox3_stubout_test +from pyfakefs.tests import ( + dynamic_patch_test, + fake_stat_time_test, + example_test, + fake_filesystem_glob_test, + fake_filesystem_shutil_test, + fake_filesystem_test, + fake_filesystem_unittest_test, + fake_filesystem_vs_real_test, + fake_open_test, + fake_os_test, + fake_pathlib_test, + fake_tempfile_test, + patched_packages_test, + mox3_stubout_test +) class AllTests(unittest.TestSuite): @@ -50,7 +54,8 @@ def suite(self): # pylint: disable-msg=C6409 loader.loadTestsFromModule(example_test), loader.loadTestsFromModule(mox3_stubout_test), loader.loadTestsFromModule(dynamic_patch_test), - loader.loadTestsFromModule(fake_pathlib_test) + loader.loadTestsFromModule(fake_pathlib_test), + loader.loadTestsFromModule(patched_packages_test) ]) return self diff --git a/pyfakefs/tests/fixtures/excel_test.xlsx b/pyfakefs/tests/fixtures/excel_test.xlsx new file mode 100644 index 00000000..6b6b64d5 Binary files /dev/null and b/pyfakefs/tests/fixtures/excel_test.xlsx differ diff --git a/pyfakefs/tests/patched_packages_test.py b/pyfakefs/tests/patched_packages_test.py new file mode 100644 index 00000000..bb2ed6d1 --- /dev/null +++ b/pyfakefs/tests/patched_packages_test.py @@ -0,0 +1,52 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Provides patches for some commonly used modules that enable them to work +with pyfakefs. +""" +import os + +from pyfakefs import fake_filesystem_unittest + +try: + import pandas as pd +except ImportError: + pd = None + +try: + import xlrd +except ImportError: + xlrd = None + + +class TestPatchedPackages(fake_filesystem_unittest.TestCase): + def setUp(self): + self.setUpPyfakefs() + + if pd is not None: + def test_load_csv(self): + path = '/foo/bar.csv' + self.fs.create_file(path, contents='1,2,3,4') + df = pd.read_csv(path) + assert (df.columns == ['1', '2', '3', '4']).all() + + if pd is not None and xlrd is not None: + def test_load_excel(self): + path = '/foo/bar.xlsx' + src_path = os.path.dirname(os.path.abspath(__file__)) + src_path = os.path.join(src_path, 'fixtures', 'excel_test.xlsx') + # map the file into another location to be sure that + # the real fs is not used + self.fs.add_real_file(src_path, target_path=path) + df = pd.read_excel(path) + assert (df.columns == [1, 2, 3, 4]).all()