Resolves a testing issue that can happen with Pandas 1.1

`assert_series_equal` was failing on datetime-like objects.

Also upgraded packages used for developing Pemi.

docker/Dockerfile.spark

@@ -6,10 +6,10 @@ RUN apt-get update && apt-get -y install \
     && rm -rf /var/lib/apt/lists/*
 
 # SPARK
-ARG SPARK_ARCHIVE=https://archive.apache.org/dist/spark/spark-2.4.0/spark-2.4.0-bin-hadoop2.7.tgz
+ARG SPARK_ARCHIVE=https://archive.apache.org/dist/spark/spark-2.4.6/spark-2.4.6-bin-hadoop2.7.tgz
 RUN curl -s $SPARK_ARCHIVE | tar -xz -C /usr/local/
 
-ENV SPARK_HOME /usr/local/spark-2.4.0-bin-hadoop2.7
+ENV SPARK_HOME /usr/local/spark-2.4.6-bin-hadoop2.7
 ENV PATH $PATH:$SPARK_HOME/bin
 
 # for High-availability like zoo-keeper's leader election

docs/README.rst

@@ -367,7 +367,7 @@ Once the containers are built, spin up the containers to run the tests
 
 And then run the tests using something like (you may prefer different pytest options):
 
-    >>> inv test --pytest="-s -x -vv --tb=short --color=yes tests"
+    >>> inv test --opts="-s -x -vv --tb=short --color=yes tests"
 
 The test container also launches a local Jupyter notebook server.  This can be a convenient tool to
 have when developing Pemi.  To access the notebook severs, just visit http://localhost:8890/lab

pemi/data.py

@@ -74,7 +74,15 @@ def digits(precision, scale):
         FloatField:    lambda field: FAKER.pyfloat,
         DecimalField:  _fake_decimal.__func__,
         BooleanField:  lambda field: FAKER.pybool,
-        JsonField:     lambda field: lambda: FAKER.pydict(5, True, 'str', 'int', 'date')
+        JsonField:     lambda field: lambda: FAKER.pydict(
+            nb_elements=5,
+            variable_nb_elements=True,
+            value_types=[
+                'str',
+                'int',
+                'date'
+            ],
+        )
     }
     #pylint: enable=no-member
 

pemi/pipes/pd.py

@@ -11,7 +11,8 @@ def flow(self):
 
 
 class PdConcatPipe(pemi.pipes.patterns.ConcatPipe):
-    def __init__(self, *, concat_opts={}, **params):
+    def __init__(self, *, concat_opts=None, **params):
+        concat_opts = concat_opts or {}
         super().__init__(**params)
         self.concat_opts = concat_opts
 

pemi/testing.py

@@ -435,7 +435,9 @@ def _then(case):
                 actual = target[case].data[[target_field]]
 
             try:
-                assert_series_equal(actual[target_field], expected[source_field], check_names=False)
+                assert_series_equal(actual[target_field], expected[source_field],
+                                    check_names=False, check_dtype=False,
+                                    check_datetimelike_compat=True)
             except AssertionError as err:
                 raise AssertionError(
                     'Source field {} not copied to target field {}: {}'.format(
@@ -504,7 +506,8 @@ def _then(case):
             for source_field, target_field in mapping:
                 try:
                     assert_series_equal(actual[target_field], expected[source_field],
-                                        check_names=False, check_dtype=False)
+                                        check_names=False, check_dtype=False,
+                                        check_datetimelike_compat=True)
                 except AssertionError as err:
                     raise AssertionError(
                         'Source field {} not copied to target field {}: {}'.format(

requirements/all.txt

@@ -2,98 +2,104 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile -U requirements/all.in
+#    pip-compile requirements/all.in
 #
 alabaster==0.7.12         # via sphinx
-appnope==0.1.0            # via ipython
-astroid==2.2.5            # via pylint
-atomicwrites==1.3.0       # via pytest
-attrs==19.1.0             # via jsonschema, pytest
-babel==2.6.0              # via sphinx
-backcall==0.1.0           # via ipython
-bleach==3.1.0             # via nbconvert
-certifi==2019.3.9         # via requests
+appnope==0.1.0            # via ipykernel, ipython
+argon2-cffi==20.1.0       # via notebook
+astroid==2.4.2            # via pylint
+attrs==19.3.0             # via jsonschema, pytest
+babel==2.8.0              # via sphinx
+backcall==0.2.0           # via ipython
+bleach==3.1.5             # via nbconvert
+certifi==2020.6.20        # via requests
+cffi==1.14.1              # via argon2-cffi
 chardet==3.0.4            # via requests
-dask==1.2.0
-decorator==4.4.0          # via ipython, traitlets
+dask==2.22.0              # via -r requirements/base.in
+decorator==4.4.2          # via ipython, traitlets
 defusedxml==0.6.0         # via nbconvert
-docutils==0.14            # via sphinx
+docutils==0.16            # via sphinx
 entrypoints==0.3          # via nbconvert
-factory-boy==2.11.1
-faker==1.0.5
-graphviz==0.10.1
-idna==2.8                 # via requests
-imagesize==1.1.0          # via sphinx
-invoke==1.2.0
-ipykernel==5.1.0          # via ipywidgets, jupyter, jupyter-console, notebook, qtconsole
-ipython-genutils==0.2.0   # via nbformat, notebook, qtconsole, traitlets
-ipython==7.4.0            # via ipykernel, ipywidgets, jupyter-console
-ipywidgets==7.4.2         # via jupyter
-isort==4.3.17             # via pylint
-jedi==0.13.3              # via ipython
-jinja2==2.10.1            # via nbconvert, notebook, sphinx
-jsonschema==3.0.1         # via jupyterlab-server, nbformat
-jupyter-client==5.2.4     # via ipykernel, jupyter-console, notebook, qtconsole
-jupyter-console==6.0.0    # via jupyter
-jupyter-core==4.4.0       # via jupyter-client, nbconvert, nbformat, notebook, qtconsole
-jupyter==1.0.0
-jupyterlab-server==0.2.0  # via jupyterlab
-jupyterlab==0.35.4
-lazy-object-proxy==1.3.1  # via astroid
+factory-boy==2.12.0       # via -r requirements/base.in
+faker==4.1.1              # via -r requirements/base.in, factory-boy
+graphviz==0.14.1          # via -r requirements/base.in
+idna==2.10                # via requests
+imagesize==1.2.0          # via sphinx
+importlib-metadata==1.7.0  # via jsonschema, pluggy, pytest
+iniconfig==1.0.1          # via pytest
+invoke==1.4.1             # via -r requirements/base.in
+ipykernel==5.3.4          # via notebook
+ipython-genutils==0.2.0   # via nbformat, notebook, traitlets
+ipython==7.16.1           # via -r requirements/base.in, ipykernel
+isort==4.3.21             # via pylint
+jedi==0.17.2              # via ipython
+jinja2==2.11.2            # via jupyterlab, jupyterlab-server, nbconvert, notebook, sphinx
+json5==0.9.5              # via jupyterlab-server
+jsonschema==3.2.0         # via jupyterlab-server, nbformat
+jupyter-client==6.1.6     # via ipykernel, notebook
+jupyter-core==4.6.3       # via jupyter-client, nbconvert, nbformat, notebook
+jupyterlab-server==1.2.0  # via jupyterlab
+jupyterlab==2.2.2         # via -r requirements/base.in
+lazy-object-proxy==1.4.3  # via astroid
 markupsafe==1.1.1         # via jinja2
 mccabe==0.6.1             # via pylint
 mistune==0.8.4            # via nbconvert
-more-itertools==7.0.0     # via pytest
-nbconvert==5.4.1          # via jupyter, notebook
-nbformat==4.4.0           # via ipywidgets, nbconvert, notebook
-notebook==5.7.8           # via jupyter, jupyterlab, jupyterlab-server, widgetsnbextension
-numpy==1.16.3             # via pandas
-packaging==19.0           # via sphinx
-pandas-mapper==0.1.2
-pandas==0.24.2
+more-itertools==8.4.0     # via pytest
+nbconvert==5.6.1          # via notebook
+nbformat==5.0.7           # via nbconvert, notebook
+notebook==6.1.1           # via jupyterlab, jupyterlab-server
+numpy==1.19.1             # via pandas
+packaging==20.4           # via bleach, pytest, sphinx
+pandas-mapper==0.1.4      # via -r requirements/base.in
+pandas==1.1.0             # via -r requirements/base.in, pandas-mapper
 pandocfilters==1.4.2      # via nbconvert
-parso==0.4.0              # via jedi
-pexpect==4.7.0            # via ipython
+parso==0.7.1              # via jedi
+pexpect==4.8.0            # via ipython
 pickleshare==0.7.5        # via ipython
-pluggy==0.9.0             # via pytest
-prometheus-client==0.6.0  # via notebook
-prompt-toolkit==2.0.9     # via ipython, jupyter-console
-psycopg2==2.8.2
+pluggy==0.13.1            # via pytest
+prometheus-client==0.8.0  # via notebook
+prompt-toolkit==3.0.5     # via ipython
+psycopg2-binary==2.8.5    # via -r requirements/base.in
 ptyprocess==0.6.0         # via pexpect, terminado
 py4j==0.10.7              # via pyspark
-py==1.8.0                 # via pytest
-pygments==2.3.1           # via ipython, jupyter-console, nbconvert, qtconsole, sphinx
-pylint==2.3.1
-pyparsing==2.4.0          # via packaging
-pyrsistent==0.14.11       # via jsonschema
-pyspark==2.4.0
-pytest==4.4.1
-python-dateutil==2.8.0    # via faker, jupyter-client, pandas
-pytz==2019.1              # via babel, pandas
-pyzmq==18.0.1             # via jupyter-client, notebook
-qtconsole==4.4.3          # via jupyter
-requests==2.21.0          # via sphinx
+py==1.9.0                 # via pytest
+pycparser==2.20           # via cffi
+pygments==2.6.1           # via ipython, nbconvert, sphinx
+pylint==2.5.3             # via -r requirements/base.in
+pyparsing==2.4.7          # via packaging
+pyrsistent==0.16.0        # via jsonschema
+pyspark==2.4.6            # via -r requirements/spark.in
+pytest==6.0.1             # via -r requirements/base.in
+python-dateutil==2.8.1    # via faker, jupyter-client, pandas
+pytz==2020.1              # via babel, pandas
+pyyaml==5.3.1             # via dask
+pyzmq==19.0.2             # via jupyter-client, notebook
+requests==2.24.0          # via jupyterlab-server, sphinx
 send2trash==1.5.0         # via notebook
-six==1.12.0               # via astroid, bleach, faker, jsonschema, packaging, prompt-toolkit, pyrsistent, pytest, python-dateutil, traitlets
-snowballstemmer==1.2.1    # via sphinx
-sphinx-rtd-theme==0.4.3
-sphinx==2.0.1
-sphinxcontrib-applehelp==1.0.1  # via sphinx
-sphinxcontrib-devhelp==1.0.1  # via sphinx
-sphinxcontrib-htmlhelp==1.0.2  # via sphinx
+six==1.15.0               # via argon2-cffi, astroid, bleach, jsonschema, packaging, pyrsistent, python-dateutil, traitlets
+snowballstemmer==2.0.0    # via sphinx
+sphinx-rtd-theme==0.5.0   # via -r requirements/doc.in
+sphinx==3.1.2             # via -r requirements/doc.in, sphinx-rtd-theme
+sphinxcontrib-applehelp==1.0.2  # via sphinx
+sphinxcontrib-devhelp==1.0.2  # via sphinx
+sphinxcontrib-htmlhelp==1.0.3  # via sphinx
 sphinxcontrib-jsmath==1.0.1  # via sphinx
-sphinxcontrib-qthelp==1.0.2  # via sphinx
-sphinxcontrib-serializinghtml==1.1.3  # via sphinx
-sqlalchemy==1.3.3
-tabulate==0.8.3
-terminado==0.8.2          # via notebook
-testpath==0.4.2           # via nbconvert
-text-unidecode==1.2       # via faker
-tornado==6.0.2            # via ipykernel, jupyter-client, notebook, terminado
-traitlets==4.3.2          # via ipykernel, ipython, ipywidgets, jupyter-client, jupyter-core, nbconvert, nbformat, notebook, qtconsole
-typed-ast==1.3.4          # via astroid
-urllib3==1.24.2           # via requests
-wcwidth==0.1.7            # via prompt-toolkit
+sphinxcontrib-qthelp==1.0.3  # via sphinx
+sphinxcontrib-serializinghtml==1.1.4  # via sphinx
+sqlalchemy==1.3.18        # via -r requirements/base.in
+tabulate==0.8.7           # via -r requirements/base.in
+terminado==0.8.3          # via notebook
+testpath==0.4.4           # via nbconvert
+text-unidecode==1.3       # via faker
+toml==0.10.1              # via pylint, pytest
+tornado==6.0.4            # via ipykernel, jupyter-client, jupyterlab, notebook, terminado
+traitlets==4.3.3          # via ipykernel, ipython, jupyter-client, jupyter-core, nbconvert, nbformat, notebook
+typed-ast==1.4.1          # via astroid
+urllib3==1.25.10          # via requests
+wcwidth==0.2.5            # via prompt-toolkit
 webencodings==0.5.1       # via bleach
-widgetsnbextension==3.4.2  # via ipywidgets
-wrapt==1.11.1             # via astroid
+wrapt==1.12.1             # via astroid
+zipp==3.1.0               # via importlib-metadata
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools

requirements/base.in

@@ -4,10 +4,10 @@ dask
 graphviz
 faker
 sqlalchemy
-psycopg2
+psycopg2-binary
 pandas-mapper
 
-jupyter>=0.30
+ipython
 jupyterlab
 
 pytest