added requirement handler, for usage check comment in file (#8887)

* added requirement handler, for usage check comment in file * more changes and fixes * experiments * multiversion, added dockerfile, added multiversion_testing.py, added conda_manipulator * changes * mega changes that twirl the world, haha, just changes that make multiversion testing possible * minor changes for jax imports * lint fixes * changes * lint * lint * lint * lint Co-authored-by: Rishabh Kumar <[email protected]>
ivy-llc · Jan 24, 2023 · ecabace · ecabace
1 parent 9ec2b69
commit ecabace
Show file tree

Hide file tree

Showing 22 changed files with 888 additions and 115 deletions.
diff --git a/docker/MultiversionDockerFile b/docker/MultiversionDockerFile
@@ -0,0 +1,32 @@
+FROM ubuntu:20.04
+WORKDIR /ivy
+
+COPY ../docker/multicondaenv.yml .
+COPY ../docker/multiversion_framework_directory.py .
+COPY ../docker/multiversion_testing_requirements.txt .
+COPY ../docker/run_multiversion_framework_directory.sh .
+
+
+
+
+# Install miniconda
+ENV CONDA_DIR /opt/miniconda
+
+RUN apt clean && \
+    rm -rf /var/lib/apt/lists/* && \
+    apt-get update && \
+    apt-get install -y wget
+
+RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-py38_22.11.1-1-Linux-x86_64.sh -O ~/miniconda.sh && \
+     /bin/bash ~/miniconda.sh -b -p /opt/miniconda
+
+
+ENV PATH=$CONDA_DIR/bin:$PATH
+RUN conda env create -f multicondaenv.yml
+
+RUN pip3 install --no-cache-dir --no-deps -r multiversion_testing_requirements.txt --target=/opt/miniconda/envs/multienv/lib/python3.8/site-packages
+RUN ./run_multiversion_framework_directory.sh
+# Make RUN commands use the new environment:
+SHELL ["conda", "run", "-n", "multienv", "/bin/bash", "-c"]
+
+
diff --git a/docker/build_multiversiondockerfile.sh b/docker/build_multiversiondockerfile.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker build --progress=plain -t experiment_conda   -f MultiversionDockerFile ..
diff --git a/docker/multicondaenv.yml b/docker/multicondaenv.yml
@@ -0,0 +1,6 @@
+name: multienv
+channels:
+  - conda-forge
+dependencies:
+  - python=3.8.0
+
diff --git a/docker/multiversion_framework_directory.py b/docker/multiversion_framework_directory.py
@@ -0,0 +1,48 @@
+# flake8: noqa
+import os
+import subprocess
+
+
+def directory_generator(req, base="/opt/miniconda/fw/"):
+    for versions in req:
+        pkg, ver = versions.split("/")
+        path = base + pkg + "/" + ver
+        if not os.path.exists(path):
+            install_pkg(path, pkg + "==" + ver)
+
+
+def install_pkg(path, pkg, base="fw/"):
+    if pkg.split("==")[0] == "torch":
+        subprocess.run(
+            f"pip3 install {pkg} --default-timeout=100 -f https://download.pytorch.org/whl/torch_stable.html --target={path}",
+            shell=True,
+        )
+    elif pkg.split("==")[0] == "jaxlib":
+        subprocess.run(
+            f"pip3 install {pkg} --default-timeout=100 -f https://storage.googleapis.com/jax-releases/jax_releases.html  --target={path}",
+            shell=True,
+        )
+    else:
+        subprocess.run(
+            f"pip3 install {pkg} --default-timeout=100  --target={path}", shell=True
+        )
+
+
+torch_req = ["torch/1.4.0", "torch/1.5.0", "torch/1.10.1"]
+tensorflow_req = [
+    "tensorflow/2.2.0",
+    "tensorflow/2.2.1",
+    "tensorflow/2.2.2",
+    "tensorflow/2.4.4",
+    "tensorflow/2.9.0",
+    "tensorflow/2.9.1",
+]
+jax_req = ["jax/0.1.60", "jax/0.1.61"]
+jaxlib_req = ["jaxlib/0.1.50", "jaxlib/0.1.60", "jaxlib/0.1.61"]
+numpy_req = ["numpy/1.17.3", "numpy/1.17.4", "numpy/1.23.1", "numpy/1.24.0"]
+
+directory_generator(torch_req)
+directory_generator(tensorflow_req)
+directory_generator(jax_req)
+directory_generator(numpy_req)
+directory_generator(jaxlib_req)
diff --git a/docker/multiversion_testing_requirements.txt b/docker/multiversion_testing_requirements.txt
@@ -0,0 +1,22 @@
+h5py==3.7.0
+pytest==7.1.2
+networkx==2.8.4
+hypothesis==6.55.0
+pymongo==4.3.3
+redis==4.3.4
+matplotlib==3.5.2
+opencv-python==4.6.0.66 # mod_name=cv2
+tensorflow-addons==0.17.1 # mod_name=tensorflow_addons
+tensorflow-probability==0.17.0 # mod_name=tensorflow_probability
+functorch==0.1.1
+scipy==1.8.1
+dm-haiku==0.0.6 # mod_name=haiku
+pydriller
+tqdm
+coverage
+einops
+psutil
+termcolor
+colorama
+packaging
+nvidia-ml-py<=11.495.46 # mod_name=pynvml
diff --git a/docker/run_multiversion_framework_directory.sh b/docker/run_multiversion_framework_directory.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+cd ivy
+python multiversion_framework_drectory.py
diff --git a/ivy/inspection.py b/ivy/inspection.py
@@ -1,6 +1,7 @@
 # global
 from typing import get_type_hints
 
+
 # local
 import ivy
 
@@ -103,7 +104,11 @@ def fn_array_spec(fn):
         specification
 
     """
-    type_hints = get_type_hints(fn)
+    try:  # this is because it raises error if python version 3.8.0, in certain cases
+        type_hints = get_type_hints(fn)
+    except Exception as e:
+        type_hints = dict()
+        print(f"exception found:{e} resorting to type_hindts=dict()")
     array_idxs = list()
     for i, (k, v) in enumerate(type_hints.items()):
         a_idxs = _get_array_idxs(v)

diff --git a/ivy_tests/config.py b/ivy_tests/config.py
@@ -0,0 +1,63 @@
+# flake8: noqa
+import os
+import sys
+import importlib
+
+global_temp_sys_module = {}
+
+
+def allow_global_framework_imports(fw=["numpy/1.23.1"]):
+    # since no framework installed right now we quickly store a copy of the sys.modules
+    global global_temp_sys_module
+    if not global_temp_sys_module:
+        global_temp_sys_module = sys.modules.copy()
+    for framework in fw:
+        sys.path.insert(1, os.path.abspath("/opt/miniconda/fw/" + framework))
+
+
+def try_except():
+    try:
+        import numpy
+    except ImportError:
+        allow_global_framework_imports()
+
+
+def return_global_temp_sys_module():
+    return global_temp_sys_module
+
+
+def reset_sys_modules_to_base():
+    if global_temp_sys_module != sys.modules:
+        sys.modules.clear()
+        sys.modules.update(global_temp_sys_module)
+
+
+# to import a specific pkg along with version name, to be used by the test functions
+def custom_import(
+    pkg, base="/opt/miniconda/fw/", globally_done=None
+):  # format is pkg_name/version , globally_done means
+    # if we have imported any framework before globally
+    if globally_done:  # i.e import numpy etc
+        if pkg == globally_done:
+            ret = importlib.import_module(pkg.split("/")[0])
+            return ret
+        sys.path.remove(os.path.abspath(base + globally_done))
+        temp = sys.modules.copy()
+        sys.modules.clear()
+        sys.modules.update(global_temp_sys_module)
+        sys.path.insert(1, os.path.abspath(base + pkg))
+        ret = importlib.import_module(pkg.split("/")[0])
+        sys.path.remove(os.path.abspath(base + pkg))
+        sys.path.insert(1, os.path.abspath(base + globally_done))
+        sys.modules.clear()
+        sys.modules.update(temp)
+        return ret
+
+    temp = sys.modules.copy()
+    sys.path.insert(1, os.path.abspath(base + pkg))
+    ret = importlib.import_module(pkg.split("/")[0])
+    sys.path.remove(os.path.abspath(base + pkg))
+    sys.modules.clear()
+    sys.modules.update(temp)
+
+    return ret
diff --git a/ivy_tests/test_ivy/__init__.py b/ivy_tests/test_ivy/__init__.py
@@ -1,3 +1,7 @@
+from .. import config
+
+if hasattr(config, "try_except"):
+    config.try_except()
 from . import helpers
 
 test_shapes = ((), (1,), (2, 1), (1, 2, 3))
diff --git a/ivy_tests/test_ivy/conftest.py b/ivy_tests/test_ivy/conftest.py
@@ -3,13 +3,28 @@
 import pytest
 from typing import Dict
 
+
+mod_frontend = {
+    "tensorflow": None,
+    "numpy": None,
+    "jax": None,
+    "torch": None,
+}  # multiversion
+mod_backend = {
+    "tensorflow": None,
+    "numpy": None,
+    "jax": None,
+    "torch": None,
+}  # multiversion
+
+
 # local
 import ivy_tests.test_ivy.helpers.test_parameter_flags as pf
 from ivy import DefaultDevice
 from ivy_tests.test_ivy.helpers import globals as test_globals
 from ivy_tests.test_ivy.helpers.available_frameworks import available_frameworks
 
-
+available_frameworks = available_frameworks()
 GENERAL_CONFIG_DICT = {}
 UNSET_TEST_CONFIG = {"list": [], "flag": []}
 UNSET_TEST_API_CONFIG = {"list": [], "flag": []}
@@ -21,6 +36,7 @@
 
 
 def pytest_configure(config):
+    global available_frameworks
     # device
     raw_value = config.getoption("--device")
     if raw_value == "all":
@@ -35,6 +51,15 @@ def pytest_configure(config):
     else:
         backend_strs = raw_value.split(",")
 
+    # frontend
+
+    frontend = config.getoption("--frontend")
+
+    if frontend:
+        frontend_strs = frontend.split(",")
+        for i in frontend_strs:
+            mod_frontend[i.split("/")[0]] = i
+
     # compile_graph
     raw_value = config.getoption("--compile_graph")
     if raw_value == "both":
@@ -56,14 +81,27 @@ def pytest_configure(config):
         for device in devices:
             for compile_graph in compile_modes:
                 for implicit in implicit_modes:
-                    TEST_PARAMS_CONFIG.append(
-                        (
-                            device,
-                            test_globals.FWS_DICT[backend_str](),
-                            compile_graph,
-                            implicit,
+                    if "/" in backend_str:
+                        mod_backend[backend_str.split("/")[0]] = backend_str
+                        TEST_PARAMS_CONFIG.append(
+                            (
+                                device,
+                                test_globals.FWS_DICT[backend_str.split("/")[0]](
+                                    backend_str
+                                ),
+                                compile_graph,
+                                implicit,
+                            )
+                        )
+                    else:
+                        TEST_PARAMS_CONFIG.append(
+                            (
+                                device,
+                                test_globals.FWS_DICT[backend_str](),
+                                compile_graph,
+                                implicit,
+                            )
                         )
-                    )
 
     process_cl_flags(config)
 
@@ -153,7 +191,7 @@ def pytest_addoption(parser):
     parser.addoption("-B", "--backend", action="store", default="all")
     parser.addoption("--compile_graph", action="store_true")
     parser.addoption("--with_implicit", action="store_true")
-
+    parser.addoption("--frontend", action="store", default=None)
     parser.addoption("--skip-variable-testing", action="store_true")
     parser.addoption("--skip-native-array-testing", action="store_true")
     parser.addoption("--skip-out-testing", action="store_true")
@@ -167,7 +205,6 @@ def pytest_addoption(parser):
     parser.addoption("--with-nestable-testing", action="store_true")
     parser.addoption("--with-instance-method-testing", action="store_true")
     parser.addoption("--with-gradient-testing", action="store_true")
-
     parser.addoption("--no-extra-testing", action="store_true")
     parser.addoption(
         "--my_test_dump",

diff --git a/ivy_tests/test_ivy/helpers/available_frameworks.py b/ivy_tests/test_ivy/helpers/available_frameworks.py
@@ -1,33 +1,40 @@
 # A list of available backends that can be used for testing.
 
-available_frameworks = ["numpy", "jax", "tensorflow", "torch"]
 
-try:
-    import jax
-
-    assert jax, "jax is imported to see if the user has it installed"
-except ImportError:
-    available_frameworks.remove("jax")
-
-try:
-    import tensorflow as tf
-
-    assert tf, "tensorflow is imported to see if the user has it installed"
-except ImportError:
-    available_frameworks.remove("tensorflow")
-
-try:
-    import torch
-
-    assert torch, "torch is imported to see if the user has it installed"
-except ImportError:
-    available_frameworks.remove("torch")
-
-if "tensorflow" in available_frameworks:
-    ground_truth = "tensorflow"
-elif "torch" in available_frameworks:
-    ground_truth = "torch"
-elif "jax" in available_frameworks:
-    ground_truth = "jax"
-else:
-    ground_truth = "numpy"
+def available_frameworks():
+    available_frameworks_lis = ["numpy", "jax", "tensorflow", "torch"]
+    try:
+        import jax
+
+        assert jax, "jax is imported to see if the user has it installed"
+    except ImportError:
+        available_frameworks_lis.remove("jax")
+
+    try:
+        import tensorflow as tf
+
+        assert tf, "tensorflow is imported to see if the user has it installed"
+    except ImportError:
+        available_frameworks_lis.remove("tensorflow")
+
+    try:
+        import torch
+
+        assert torch, "torch is imported to see if the user has it installed"
+    except ImportError:
+        available_frameworks_lis.remove("torch")
+    return available_frameworks_lis
+
+
+def ground_truth():
+    available_framework_lis = available_frameworks()
+    g_truth = ""
+    if "tensorflow" in available_framework_lis:
+        g_truth = "tensorflow"
+    elif "torch" in available_framework_lis:
+        g_truth = "torch"
+    elif "jax" in available_framework_lis:
+        g_truth = "jax"
+    else:
+        g_truth = "numpy"
+    return g_truth
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		#!/bin/bash

		docker build --progress=plain -t experiment_conda -f MultiversionDockerFile ..