From 85a1ab31c58196615ca09494fae885a577faa816 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 19 Nov 2019 14:14:24 -0800 Subject: [PATCH 1/5] DEPR: enforce deprecations in core.internals --- doc/source/whatsnew/v1.0.0.rst | 3 ++- pandas/core/internals/blocks.py | 20 +++----------------- pandas/tests/internals/test_internals.py | 10 ++-------- 3 files changed, 7 insertions(+), 26 deletions(-) diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst index 98d861d999ea9..178fbdf827650 100644 --- a/doc/source/whatsnew/v1.0.0.rst +++ b/doc/source/whatsnew/v1.0.0.rst @@ -280,7 +280,8 @@ or ``matplotlib.Axes.plot``. See :ref:`plotting.formatters` for more. - Removed the previously deprecated ``assert_raises_regex`` function in ``pandas.util.testing`` (:issue:`29174`) - Removed :meth:`Index.is_lexsorted_for_tuple` (:issue:`29305`) - Removed support for nexted renaming in :meth:`DataFrame.aggregate`, :meth:`Series.aggregate`, :meth:`DataFrameGroupBy.aggregate`, :meth:`SeriesGroupBy.aggregate`, :meth:`Rolling.aggregate` (:issue:`29608`) -- +- :func:`core.internals.blocks.make_block` no longer accepts the "fastpath" keyword(:issue:`19265`) +- :meth:`Block.make_block_same_class` no longer accepts the "dtype" keyword(:issue:`19434`) .. _whatsnew_1000.performance: diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 7ace80415c846..86d8dc9372033 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -251,21 +251,13 @@ def make_block(self, values, placement=None): return make_block(values, placement=placement, ndim=self.ndim) - def make_block_same_class(self, values, placement=None, ndim=None, dtype=None): + def make_block_same_class(self, values, placement=None, ndim=None): """ Wrap given values in a block of same type as self. """ - if dtype is not None: - # issue 19431 fastparquet is passing this - warnings.warn( - "dtype argument is deprecated, will be removed in a future release.", - FutureWarning, - ) if placement is None: placement = self.mgr_locs if ndim is None: ndim = self.ndim - return make_block( - values, placement=placement, ndim=ndim, klass=self.__class__, dtype=dtype - ) + return make_block(values, placement=placement, ndim=ndim, klass=self.__class__) def __repr__(self) -> str: # don't want to print out all of the items here @@ -2999,7 +2991,7 @@ def get_block_type(values, dtype=None): return cls -def make_block(values, placement, klass=None, ndim=None, dtype=None, fastpath=None): +def make_block(values, placement, klass=None, ndim=None, dtype=None): # Ensure that we don't allow PandasArray / PandasDtype in internals. # For now, blocks should be backed by ndarrays when possible. if isinstance(values, ABCPandasArray): @@ -3010,12 +3002,6 @@ def make_block(values, placement, klass=None, ndim=None, dtype=None, fastpath=No if isinstance(dtype, PandasDtype): dtype = dtype.numpy_dtype - if fastpath is not None: - # GH#19265 pyarrow is passing this - warnings.warn( - "fastpath argument is deprecated, will be removed in a future release.", - FutureWarning, - ) if klass is None: dtype = dtype or values.dtype klass = get_block_type(values, dtype) diff --git a/pandas/tests/internals/test_internals.py b/pandas/tests/internals/test_internals.py index c98bdab0df766..4acd80e29cb95 100644 --- a/pandas/tests/internals/test_internals.py +++ b/pandas/tests/internals/test_internals.py @@ -311,7 +311,8 @@ def test_delete(self): def test_make_block_same_class(self): # issue 19431 block = create_block("M8[ns, US/Eastern]", [3]) - with tm.assert_produces_warning(FutureWarning, check_stacklevel=False): + with pytest.raises(TypeError, match="unexpected keyword"): + # Deprecation enforced as of GH#???? block.make_block_same_class(block.values, dtype=block.values.dtype) @@ -1255,13 +1256,6 @@ def test_holder(typestr, holder): assert blk._holder is holder -def test_deprecated_fastpath(): - # GH#19265 - values = np.random.rand(3, 3) - with tm.assert_produces_warning(FutureWarning, check_stacklevel=False): - make_block(values, placement=np.arange(3), fastpath=True) - - def test_validate_ndim(): values = np.array([1.0, 2.0]) placement = slice(2) From 362723803601a88a748c26318510a5990038d64d Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 19 Nov 2019 16:52:25 -0800 Subject: [PATCH 2/5] remove test --- pandas/tests/internals/test_internals.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/pandas/tests/internals/test_internals.py b/pandas/tests/internals/test_internals.py index 4acd80e29cb95..abe2ddf955ad8 100644 --- a/pandas/tests/internals/test_internals.py +++ b/pandas/tests/internals/test_internals.py @@ -308,13 +308,6 @@ def test_delete(self): with pytest.raises(Exception): newb.delete(3) - def test_make_block_same_class(self): - # issue 19431 - block = create_block("M8[ns, US/Eastern]", [3]) - with pytest.raises(TypeError, match="unexpected keyword"): - # Deprecation enforced as of GH#???? - block.make_block_same_class(block.values, dtype=block.values.dtype) - class TestDatetimeBlock: def test_can_hold_element(self): From f0dc98825920e019c210f00bb64b28b26eaa566a Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Thu, 21 Nov 2019 11:41:53 -0800 Subject: [PATCH 3/5] bump fastparquet to 0.3.0 --- doc/source/whatsnew/v1.0.0.rst | 65 ++++++++++++++++++++++++++++++++++ environment.yml | 2 +- pandas/compat/_optional.py | 2 +- requirements-dev.txt | 2 +- 4 files changed, 68 insertions(+), 3 deletions(-) diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst index b3130ab3ba068..0c95a7f4d1d34 100644 --- a/doc/source/whatsnew/v1.0.0.rst +++ b/doc/source/whatsnew/v1.0.0.rst @@ -235,6 +235,71 @@ The following methods now also correctly output values for unobserved categories df.groupby(["cat_1", "cat_2"], observed=False)["value"].count() +.. _whatsnew_1000.api_breaking.deps: + +Increased minimum versions for dependencies +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Some minimum supported versions of dependencies were updated (:issue:`29723`). +If installed, we now require: + ++-----------------+-----------------+----------+ +| Package | Minimum Version | Required | ++=================+=================+==========+ +| numpy | 1.13.3 | X | ++-----------------+-----------------+----------+ +| pytz | 2015.4 | X | ++-----------------+-----------------+----------+ +| python-dateutil | 2.6.1 | X | ++-----------------+-----------------+----------+ +| bottleneck | 1.2.1 | | ++-----------------+-----------------+----------+ +| numexpr | 2.6.2 | | ++-----------------+-----------------+----------+ +| pytest (dev) | 4.0.2 | | ++-----------------+-----------------+----------+ + +For `optional libraries `_ the general recommendation is to use the latest version. +The following table lists the lowest version per library that is currently being tested throughout the development of pandas. +Optional libraries below the lowest tested version may still work, but are not considered supported. + ++-----------------+-----------------+ +| Package | Minimum Version | ++=================+=================+ +| beautifulsoup4 | 4.6.0 | ++-----------------+-----------------+ +| fastparquet | 0.3.0 | ++-----------------+-----------------+ +| gcsfs | 0.2.2 | ++-----------------+-----------------+ +| lxml | 3.8.0 | ++-----------------+-----------------+ +| matplotlib | 2.2.2 | ++-----------------+-----------------+ +| openpyxl | 2.4.8 | ++-----------------+-----------------+ +| pyarrow | 0.9.0 | ++-----------------+-----------------+ +| pymysql | 0.7.1 | ++-----------------+-----------------+ +| pytables | 3.4.2 | ++-----------------+-----------------+ +| scipy | 0.19.0 | ++-----------------+-----------------+ +| sqlalchemy | 1.1.4 | ++-----------------+-----------------+ +| xarray | 0.8.2 | ++-----------------+-----------------+ +| xlrd | 1.1.0 | ++-----------------+-----------------+ +| xlsxwriter | 0.9.8 | ++-----------------+-----------------+ +| xlwt | 1.2.0 | ++-----------------+-----------------+ + +See :ref:`install.dependencies` and :ref:`install.optional_dependencies` for more. + + .. _whatsnew_1000.api.other: Other API changes diff --git a/environment.yml b/environment.yml index 54c99f415165d..be55886bb5664 100644 --- a/environment.yml +++ b/environment.yml @@ -75,7 +75,7 @@ dependencies: # optional for io - beautifulsoup4>=4.6.0 # pandas.read_html - - fastparquet>=0.2.1 # pandas.read_parquet, DataFrame.to_parquet + - fastparquet>=0.3.0 # pandas.read_parquet, DataFrame.to_parquet - html5lib # pandas.read_html - lxml # pandas.read_html - openpyxl # pandas.read_excel, DataFrame.to_excel, pandas.ExcelWriter, pandas.ExcelFile diff --git a/pandas/compat/_optional.py b/pandas/compat/_optional.py index fc66502710b0c..8c56373759176 100644 --- a/pandas/compat/_optional.py +++ b/pandas/compat/_optional.py @@ -8,7 +8,7 @@ VERSIONS = { "bs4": "4.6.0", "bottleneck": "1.2.1", - "fastparquet": "0.2.1", + "fastparquet": "0.3.0", "gcsfs": "0.2.2", "lxml.etree": "3.8.0", "matplotlib": "2.2.2", diff --git a/requirements-dev.txt b/requirements-dev.txt index 87b348c39a17b..ce5a98c0ef905 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -48,7 +48,7 @@ matplotlib>=2.2.2 numexpr>=2.6.8 scipy>=1.1 beautifulsoup4>=4.6.0 -fastparquet>=0.2.1 +fastparquet>=0.3.0 html5lib lxml openpyxl From b9306a9e32d5a7200ddf721f4ff8949e638e1f1c Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Thu, 21 Nov 2019 14:53:45 -0800 Subject: [PATCH 4/5] update ci deps --- ci/deps/azure-windows-36.yaml | 2 +- ci/deps/travis-36-cov.yaml | 2 +- ci/deps/travis-36-locale.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/deps/azure-windows-36.yaml b/ci/deps/azure-windows-36.yaml index e3ad1d8371623..d9534912627f0 100644 --- a/ci/deps/azure-windows-36.yaml +++ b/ci/deps/azure-windows-36.yaml @@ -16,7 +16,7 @@ dependencies: # pandas dependencies - blosc - bottleneck - - fastparquet>=0.2.1 + - fastparquet>=0.3.0 - matplotlib=3.0.2 - numexpr - numpy=1.15.* diff --git a/ci/deps/travis-36-cov.yaml b/ci/deps/travis-36-cov.yaml index 9148e0d4b29d9..778aaf1855968 100644 --- a/ci/deps/travis-36-cov.yaml +++ b/ci/deps/travis-36-cov.yaml @@ -18,7 +18,7 @@ dependencies: - botocore>=1.11 - cython>=0.29.13 - dask - - fastparquet>=0.2.1 + - fastparquet>=0.3.0 - gcsfs - geopandas - html5lib diff --git a/ci/deps/travis-36-locale.yaml b/ci/deps/travis-36-locale.yaml index 3199ee037bc0a..e75061be25998 100644 --- a/ci/deps/travis-36-locale.yaml +++ b/ci/deps/travis-36-locale.yaml @@ -16,7 +16,7 @@ dependencies: - beautifulsoup4 - blosc=1.14.3 - python-blosc - - fastparquet=0.2.1 + - fastparquet=0.3.0 - gcsfs=0.2.2 - html5lib - ipython From 305fa9acb9bb23afe3d830b0e6754d86fc1416c1 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Thu, 21 Nov 2019 17:07:47 -0800 Subject: [PATCH 5/5] bump fastparquet to 0.3.2 --- ci/deps/azure-windows-36.yaml | 2 +- ci/deps/travis-36-cov.yaml | 2 +- ci/deps/travis-36-locale.yaml | 2 +- doc/source/getting_started/install.rst | 2 +- doc/source/whatsnew/v1.0.0.rst | 2 +- environment.yml | 2 +- pandas/compat/_optional.py | 2 +- pandas/tests/io/test_parquet.py | 2 +- requirements-dev.txt | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ci/deps/azure-windows-36.yaml b/ci/deps/azure-windows-36.yaml index d9534912627f0..ec4aff41f1967 100644 --- a/ci/deps/azure-windows-36.yaml +++ b/ci/deps/azure-windows-36.yaml @@ -16,7 +16,7 @@ dependencies: # pandas dependencies - blosc - bottleneck - - fastparquet>=0.3.0 + - fastparquet>=0.3.2 - matplotlib=3.0.2 - numexpr - numpy=1.15.* diff --git a/ci/deps/travis-36-cov.yaml b/ci/deps/travis-36-cov.yaml index 778aaf1855968..81e5f6516fe69 100644 --- a/ci/deps/travis-36-cov.yaml +++ b/ci/deps/travis-36-cov.yaml @@ -18,7 +18,7 @@ dependencies: - botocore>=1.11 - cython>=0.29.13 - dask - - fastparquet>=0.3.0 + - fastparquet>=0.3.2 - gcsfs - geopandas - html5lib diff --git a/ci/deps/travis-36-locale.yaml b/ci/deps/travis-36-locale.yaml index e75061be25998..9058f25c9344d 100644 --- a/ci/deps/travis-36-locale.yaml +++ b/ci/deps/travis-36-locale.yaml @@ -16,7 +16,7 @@ dependencies: - beautifulsoup4 - blosc=1.14.3 - python-blosc - - fastparquet=0.3.0 + - fastparquet=0.3.2 - gcsfs=0.2.2 - html5lib - ipython diff --git a/doc/source/getting_started/install.rst b/doc/source/getting_started/install.rst index 663948fd46cf6..a9e853c887636 100644 --- a/doc/source/getting_started/install.rst +++ b/doc/source/getting_started/install.rst @@ -250,7 +250,7 @@ SQLAlchemy 1.1.4 SQL support for databases other tha SciPy 0.19.0 Miscellaneous statistical functions XLsxWriter 0.9.8 Excel writing blosc Compression for msgpack -fastparquet 0.2.1 Parquet reading / writing +fastparquet 0.3.2 Parquet reading / writing gcsfs 0.2.2 Google Cloud Storage access html5lib HTML parser for read_html (see :ref:`note `) lxml 3.8.0 HTML parser for read_html (see :ref:`note `) diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst index 0c95a7f4d1d34..0d8f39755f2d6 100644 --- a/doc/source/whatsnew/v1.0.0.rst +++ b/doc/source/whatsnew/v1.0.0.rst @@ -268,7 +268,7 @@ Optional libraries below the lowest tested version may still work, but are not c +=================+=================+ | beautifulsoup4 | 4.6.0 | +-----------------+-----------------+ -| fastparquet | 0.3.0 | +| fastparquet | 0.3.2 | +-----------------+-----------------+ | gcsfs | 0.2.2 | +-----------------+-----------------+ diff --git a/environment.yml b/environment.yml index be55886bb5664..848825c37a160 100644 --- a/environment.yml +++ b/environment.yml @@ -75,7 +75,7 @@ dependencies: # optional for io - beautifulsoup4>=4.6.0 # pandas.read_html - - fastparquet>=0.3.0 # pandas.read_parquet, DataFrame.to_parquet + - fastparquet>=0.3.2 # pandas.read_parquet, DataFrame.to_parquet - html5lib # pandas.read_html - lxml # pandas.read_html - openpyxl # pandas.read_excel, DataFrame.to_excel, pandas.ExcelWriter, pandas.ExcelFile diff --git a/pandas/compat/_optional.py b/pandas/compat/_optional.py index 8c56373759176..b0a7f243ca14c 100644 --- a/pandas/compat/_optional.py +++ b/pandas/compat/_optional.py @@ -8,7 +8,7 @@ VERSIONS = { "bs4": "4.6.0", "bottleneck": "1.2.1", - "fastparquet": "0.3.0", + "fastparquet": "0.3.2", "gcsfs": "0.2.2", "lxml.etree": "3.8.0", "matplotlib": "2.2.2", diff --git a/pandas/tests/io/test_parquet.py b/pandas/tests/io/test_parquet.py index bcbbee3b86769..3e687d185df84 100644 --- a/pandas/tests/io/test_parquet.py +++ b/pandas/tests/io/test_parquet.py @@ -531,7 +531,7 @@ def test_additional_extension_arrays(self, pa): class TestParquetFastParquet(Base): - @td.skip_if_no("fastparquet", min_version="0.2.1") + @td.skip_if_no("fastparquet", min_version="0.3.2") def test_basic(self, fp, df_full): df = df_full diff --git a/requirements-dev.txt b/requirements-dev.txt index ce5a98c0ef905..4d0e7ee904294 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -48,7 +48,7 @@ matplotlib>=2.2.2 numexpr>=2.6.8 scipy>=1.1 beautifulsoup4>=4.6.0 -fastparquet>=0.3.0 +fastparquet>=0.3.2 html5lib lxml openpyxl