From aebbc36bf69c691d33d46b69ab3519b68843b412 Mon Sep 17 00:00:00 2001 From: RickSanchezStoic <57310695+RickSanchezStoic@users.noreply.github.com> Date: Thu, 23 Feb 2023 19:12:38 +0530 Subject: [PATCH] Docker exploration multiversion (#10865) * added requirement handler, for usage check comment in file * more changes and fixes * experiments * multiversion, added dockerfile, added multiversion_testing.py, added conda_manipulator * changes * mega changes that twirl the world, haha, just changes that make multiversion testing possible * minor changes for jax imports * lint fixes * changes * lint * lint * lint * lint * multiversion * lint errors * Update ivy.iml * Update ivy.iml * Update ivy.iml * Update misc.xml * multiversion frontend test * minor changes * lint * changes * lint * changes * changes * changes * changes * multiversion frontend, dtype handler added to subprocess, numpy bfloat16 dependency on tensorflow or jax eliminatedgit add -u! * lint * lint * lint * lint * small change * fixes for jax * frontend method test for multiversion * changes * fixed dtype handling for multiversion --------- Co-authored-by: Rishabh Kumar --- ivy_tests/test_ivy/conftest.py | 3 + .../hypothesis_helpers/array_helpers.py | 2 + .../hypothesis_helpers/dtype_helpers.py | 32 ++++++++- multiversion_frontend_test.py | 72 +++++++++++++------ 4 files changed, 86 insertions(+), 23 deletions(-) diff --git a/ivy_tests/test_ivy/conftest.py b/ivy_tests/test_ivy/conftest.py index c352003e359db..8a64611ca4652 100644 --- a/ivy_tests/test_ivy/conftest.py +++ b/ivy_tests/test_ivy/conftest.py @@ -60,6 +60,8 @@ def pytest_configure(config): frontend_strs = frontend.split(",") for i in frontend_strs: process = subprocess.Popen( + + [ "/opt/miniconda/envs/multienv/bin/python", "multiversion_frontend_test.py", @@ -73,6 +75,7 @@ def pytest_configure(config): ) mod_frontend[i.split("/")[0]] = [i, process] + # compile_graph raw_value = config.getoption("--compile_graph") if raw_value == "both": diff --git a/ivy_tests/test_ivy/helpers/hypothesis_helpers/array_helpers.py b/ivy_tests/test_ivy/helpers/hypothesis_helpers/array_helpers.py index 05efd7e2e547f..c2d5facd39bbc 100644 --- a/ivy_tests/test_ivy/helpers/hypothesis_helpers/array_helpers.py +++ b/ivy_tests/test_ivy/helpers/hypothesis_helpers/array_helpers.py @@ -1043,8 +1043,10 @@ def array_values( except Exception: # enables bfloat16 behavior with possibly no side-effects + import paddle_bfloat # noqa + array = np.asarray(values, dtype=dtype) if isinstance(shape, (tuple, list)): diff --git a/ivy_tests/test_ivy/helpers/hypothesis_helpers/dtype_helpers.py b/ivy_tests/test_ivy/helpers/hypothesis_helpers/dtype_helpers.py index 9ba9c34869105..04289b5cf4f46 100644 --- a/ivy_tests/test_ivy/helpers/hypothesis_helpers/dtype_helpers.py +++ b/ivy_tests/test_ivy/helpers/hypothesis_helpers/dtype_helpers.py @@ -125,9 +125,35 @@ def get_dtypes( # FN_DTYPES & BACKEND_DTYPES & FRONTEND_DTYPES & GROUND_TRUTH_DTYPES # If being called from a frontend test - if test_globals.CURRENT_FRONTEND is not test_globals._Notsetval: # NOQA - frontend_dtypes = retrieval_fn(test_globals.CURRENT_FRONTEND(), kind) - valid_dtypes = valid_dtypes.intersection(frontend_dtypes) + if test_globals.CURRENT_FRONTEND is not test_globals._Notsetval or isinstance( + test_globals.CURRENT_FRONTEND_STR, list + ): # NOQA + if isinstance(test_globals.CURRENT_FRONTEND_STR, list): + process = test_globals.CURRENT_FRONTEND_STR[1] + try: + process.stdin.write("1" + "\n") + process.stdin.write(f"{str(retrieval_fn.__name__)}"+ "\n") + process.stdin.write(f"{str(kind)}" + "\n") + process.stdin.write(f"{test_globals.CURRENT_DEVICE}"+ "\n") + process.stdin.write(f"{test_globals.CURRENT_RUNNING_TEST.fn_tree}"+ "\n") + process.stdin.flush() + except Exception as e: + print( + "Something bad happened to the subprocess, here are the logs:\n\n" + ) + print(process.stdout.readlines()) + raise e + frontend_ret = process.stdout.readline() + if frontend_ret: + frontend_ret = jsonpickle.loads(make_json_pickable(frontend_ret)) + else: + print(process.stderr.readlines()) + raise Exception + frontend_dtypes = frontend_ret + valid_dtypes = valid_dtypes.intersection(frontend_dtypes) + else: + frontend_dtypes = retrieval_fn(test_globals.CURRENT_FRONTEND(), kind) + valid_dtypes = valid_dtypes.intersection(frontend_dtypes) # Make sure we return dtypes that are compatiable with ground truth backend ground_truth_is_set = ( diff --git a/multiversion_frontend_test.py b/multiversion_frontend_test.py index f0edf23145887..c1c02e8e7a117 100644 --- a/multiversion_frontend_test.py +++ b/multiversion_frontend_test.py @@ -3,6 +3,7 @@ import sys import jsonpickle import importlib +from ivy_tests.test_ivy.helpers.testing_helpers import _import_fn, _get_supported_devices_dtypes def available_frameworks(): @@ -59,32 +60,59 @@ def __init__(self, native_class): self._native_class = native_class -def _get_type_dict(framework): - return { - "valid": framework.valid_dtypes, - "numeric": framework.valid_numeric_dtypes, - "float": framework.valid_float_dtypes, - "integer": framework.valid_int_dtypes, - "unsigned": framework.valid_uint_dtypes, - "signed_integer": tuple( +def _get_fn_dtypes(framework,fn_tree, device=None,kind="valid"): + callable_fn, fn_name, fn_mod = _import_fn(fn_tree) + supported_device_dtypes = _get_supported_devices_dtypes(fn_name, fn_mod) + return supported_device_dtypes[framework][ + device + ][kind] + +def _get_type_dict(framework,fn_tree, device=None,kind="valid"): + if kind == "valid": + return framework.valid_dtypes + elif kind == "numeric": + return framework.valid_numeric_dtypes + elif kind == "integer": + return framework.valid_int_dtypes + elif kind == "float": + return framework.valid_float_dtypes + elif kind == "unsigned": + return framework.valid_int_dtypes + elif kind == "signed_integer": + return tuple( set(framework.valid_int_dtypes).difference(framework.valid_uint_dtypes) - ), - "complex": framework.valid_complex_dtypes, - "real_and_complex": tuple( + ) + elif kind == "complex": + return framework.valid_complex_dtypes + elif kind == "real_and_complex": + return tuple( set(framework.valid_numeric_dtypes).union(framework.valid_complex_dtypes) - ), - "float_and_complex": tuple( + ) + elif kind == "float_and_complex": + return tuple( set(framework.valid_float_dtypes).union(framework.valid_complex_dtypes) - ), - "bool": tuple( + ) + elif kind == "bool": + return tuple( set(framework.valid_dtypes).difference(framework.valid_numeric_dtypes) - ), - } + ) + else: + raise RuntimeError("{} is an unknown kind!".format(kind)) def dtype_handler(framework): - framework = importlib.import_module("ivy.functional.backends." + framework) - dtypes = _get_type_dict(framework) + z=input() + retrieval_fn=globals()[z] + z=input() + kind=z + z=input() + device=z + z=input() + fn_tree=z + + if retrieval_fn.__name__== '_get_type_dict': + framework = importlib.import_module("ivy.functional.backends." + framework) + dtypes = retrieval_fn(framework,fn_tree,device,kind) dtypes = jsonpickle.dumps(dtypes) print(dtypes) @@ -244,10 +272,14 @@ def test_frontend_method(): ) frontend_ret = frontend_fw.__dict__[func](*args_frontend, **kwargs_frontend) - frontend_ret = ivy.to_numpy(frontend_ret) + if isinstance(frontend_ret,tuple) or isinstance(frontend_ret,list): + frontend_ret=ivy.nested_map(frontend_ret,ivy.to_numpy) + else: + frontend_ret = ivy.to_numpy(frontend_ret) frontend_ret = jsonpickle.dumps(frontend_ret) print(frontend_ret) except EOFError: continue except Exception as e: + print(frontend_ret.shape) raise e