get tests to pass on Py27 (#2812)

* get tests to pass on Py27

* enable Py27 builds under Appveyor and TravisCI

* fix utils.open invocation

.travis.yml

@@ -32,6 +32,9 @@ matrix:
     - python: '3.6'
       env: TOXENV="py36-linux"
 
+    - python: '2.7'
+      env: TOXENV="py27-linux"
+
 
 install:
   - pip install tox

appveyor.yml

@@ -31,6 +31,11 @@ environment:
       PYTHON_ARCH: "64"
       TOXENV: "py36-win"
 
+    - PYTHON: "C:\\Python27-x64"
+      PYTHON_VERSION: "2.7.17"
+      PYTHON_ARCH: "64"
+      TOXENV: "py27-win"
+
 init:
   - "ECHO %PYTHON% %PYTHON_VERSION% %PYTHON_ARCH%"
   - "ECHO \"%APPVEYOR_SCHEDULED_BUILD%\""

gensim/corpora/dictionary.py

@@ -9,7 +9,10 @@
 from __future__ import with_statement
 
 from collections import defaultdict
-from collections.abc import Mapping
+try:
+    from collections.abc import Mapping
+except ImportError:
+    from collections import Mapping
 import sys
 import logging
 import itertools

gensim/corpora/sharded_corpus.py

@@ -24,14 +24,19 @@
 import math
 import numpy
 import scipy.sparse as sparse
-import time
 
 from six.moves import range
 
 import gensim
 from gensim.corpora import IndexedCorpus
 from gensim.interfaces import TransformedCorpus
 
+import six
+if six.PY2:
+    from time import time as perf_counter
+else:
+    from time import perf_counter
+
 logger = logging.getLogger(__name__)
 
 #: Specifies which dtype should be used for serializing the shards.
@@ -280,12 +285,12 @@ def init_shards(self, output_prefix, corpus, shardsize=4096, dtype=_default_dtyp
         self.dim = proposed_dim
         self.offsets = [0]
 
-        start_time = time.perf_counter()
+        start_time = perf_counter()
 
         logger.info('Running init from corpus.')
 
         for n, doc_chunk in enumerate(gensim.utils.grouper(corpus, chunksize=shardsize)):
-            logger.info('Chunk no. %d at %f s', n, time.perf_counter() - start_time)
+            logger.info('Chunk no. %d at %f s', n, perf_counter() - start_time)
 
             current_shard = numpy.zeros((len(doc_chunk), self.dim), dtype=dtype)
             logger.debug('Current chunk dimension: %d x %d', len(doc_chunk), self.dim)
@@ -300,7 +305,7 @@ def init_shards(self, output_prefix, corpus, shardsize=4096, dtype=_default_dtyp
 
             self.save_shard(current_shard)
 
-        end_time = time.perf_counter()
+        end_time = perf_counter()
         logger.info('Built %d shards in %f s.', self.n_shards, end_time - start_time)
 
     def init_by_clone(self):

gensim/models/doc2vec.py

@@ -71,7 +71,10 @@
     from Queue import Queue  # noqa:F401
 
 from collections import namedtuple, defaultdict
-from collections.abc import Iterable
+try:
+    from collections.abc import Iterable
+except ImportError:
+    from collections import Iterable
 from timeit import default_timer
 
 from numpy import zeros, float32 as REAL, empty, ones, \

gensim/models/fasttext.py

@@ -285,7 +285,10 @@
 import numpy as np
 from numpy import ones, vstack, float32 as REAL
 import six
-from collections.abc import Iterable
+try:
+    from collections.abc import Iterable
+except ImportError:
+    from collections import Iterable
 
 import gensim.models._fasttext_bin
 

gensim/models/hdpmodel.py

@@ -52,7 +52,6 @@
 from __future__ import with_statement
 
 import logging
-import time
 import warnings
 
 import numpy as np
@@ -65,6 +64,12 @@
 
 from gensim.utils import deprecated
 
+import six
+if six.PY2:
+    from time import time as perf_counter
+else:
+    from time import perf_counter
+
 logger = logging.getLogger(__name__)
 
 meanchangethresh = 0.00001
@@ -464,7 +469,7 @@ def update(self, corpus):
         """
         save_freq = max(1, int(10000 / self.chunksize))  # save every 10k docs, roughly
         chunks_processed = 0
-        start_time = time.perf_counter()
+        start_time = perf_counter()
 
         while True:
             for chunk in utils.grouper(corpus, self.chunksize):
@@ -513,7 +518,7 @@ def update_finished(self, start_time, chunks_processed, docs_processed):
             (self.max_chunks and chunks_processed == self.max_chunks)
 
             # time limit reached
-            or (self.max_time and time.perf_counter() - start_time > self.max_time)
+            or (self.max_time and perf_counter() - start_time > self.max_time)
 
             # no limits and whole corpus has been processed once
             or (not self.max_chunks and not self.max_time and docs_processed >= self.m_D))

gensim/test/test_corpora_dictionary.py

@@ -7,7 +7,10 @@
 Unit tests for the `corpora.Dictionary` class.
 """
 
-from collections.abc import Mapping
+try:
+    from collections.abc import Mapping
+except ImportError:
+    from collections import Mapping
 from itertools import chain
 import logging
 import unittest

gensim/test/test_utils_any2vec.py

@@ -24,7 +24,7 @@
 
 def save_dict_to_word2vec_formated_file(fname, word2vec_dict):
 
-    with gensim.utils.open(fname, "bw") as f:
+    with gensim.utils.open(fname, "wb") as f:
 
         num_words = len(word2vec_dict)
         vector_length = len(list(word2vec_dict.values())[0])

setup.py

@@ -20,8 +20,9 @@
 from setuptools import setup, find_packages, Extension
 from setuptools.command.build_ext import build_ext
 
-if sys.version_info[:2] < (3, 5):
-    raise Exception('This version of gensim needs 3.5 or later.')
+PY2 = sys.version_info[:2] == (2, 7)
+if sys.version_info[:2] < (3, 5) and not PY2:
+    raise Exception('This version of gensim requires Py2.7, or Py3.5 or greater')
 
 c_extensions = {
     'gensim.models.word2vec_inner': 'gensim/models/word2vec_inner.c',
@@ -316,7 +317,18 @@ def run(self):
 if (3, 0) < sys.version_info < (3, 7):
     linux_testenv.extend(['nmslib'])
 
-NUMPY_STR = 'numpy >= 1.11.3'
+if PY2:
+    #
+    # https://www.scipy.org/scipylib/faq.html#python-version-support
+    #
+    NUMPY_STR = 'numpy <= 1.16.1'
+    SCIPY_STR = 'scipy <= 1.2.3'
+    SO_STR = 'smart_open == 1.10.1'
+else:
+    NUMPY_STR = 'numpy >= 1.11.3'
+    SCIPY_STR = 'scipy >= 0.18.1'
+    SO_STR = 'smart_open'
+
 #
 # We pin the Cython version for reproducibility.  We expect our extensions
 # to build with any sane version of Cython, so we should update this pin
@@ -326,9 +338,9 @@ def run(self):
 
 install_requires = [
     NUMPY_STR,
-    'scipy >= 0.18.1',
+    SCIPY_STR,
     'six >= 1.5.0',
-    'smart_open >= 1.8.1',
+    SO_STR,
 ]
 
 setup_requires = [NUMPY_STR]

tox.ini

@@ -1,6 +1,6 @@
 [tox]
 minversion = 2.0
-envlist = {py36,py37,py38}-{win,linux}, flake8, docs, docs-upload, download-wheels, upload-wheels, test-pypi
+envlist = {py27,py36,py37,py38}-{win,linux}, flake8, docs, docs-upload, download-wheels, upload-wheels, test-pypi
 skipsdist = True
 platform = linux: linux
            win: win64
@@ -17,7 +17,7 @@ ignore = F821  ; TODO remove me when all examples in docstrings will be executab
 exclude=.venv, .git, .tox, dist, doc, build, gensim/models/deprecated
 
 [pytest]
-addopts = -rfxEXs --durations=20 --showlocals --reruns 3 --reruns-delay 1
+addopts = -rfxEXs --durations=20 --showlocals
 
 [testenv]
 recreate = True