From 54dcf2951e610bf35dad10b9cf2b0003bfdc84c9 Mon Sep 17 00:00:00 2001 From: Julio Perez <37191411+jperez999@users.noreply.github.com> Date: Thu, 13 Jul 2023 10:26:55 -0400 Subject: [PATCH] migrated base runtime to core (#380) * migrated base runtime to core * add comments to init file * change runtime imports --- merlin/systems/dag/runtimes/__init__.py | 4 +- merlin/systems/dag/runtimes/base_runtime.py | 59 ------------------- merlin/systems/dag/runtimes/triton/runtime.py | 2 +- .../runtimes/local/ops/fil/test_lightgbm.py | 2 +- .../runtimes/local/ops/fil/test_sklearn.py | 2 +- .../runtimes/local/ops/fil/test_xgboost.py | 2 +- .../local/ops/nvtabular/test_ensemble.py | 2 +- .../local/ops/tensorflow/test_ensemble.py | 2 +- .../dag/runtimes/local/ops/torch/test_op.py | 2 +- 9 files changed, 8 insertions(+), 69 deletions(-) delete mode 100644 merlin/systems/dag/runtimes/base_runtime.py diff --git a/merlin/systems/dag/runtimes/__init__.py b/merlin/systems/dag/runtimes/__init__.py index a6e44cb75..94951cb20 100644 --- a/merlin/systems/dag/runtimes/__init__.py +++ b/merlin/systems/dag/runtimes/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2023, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,5 +14,3 @@ # limitations under the License. # # flake8: noqa - -from .base_runtime import Runtime diff --git a/merlin/systems/dag/runtimes/base_runtime.py b/merlin/systems/dag/runtimes/base_runtime.py deleted file mode 100644 index c68eeaadb..000000000 --- a/merlin/systems/dag/runtimes/base_runtime.py +++ /dev/null @@ -1,59 +0,0 @@ -# -# Copyright (c) 2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from merlin.core.protocols import Transformable -from merlin.dag import Graph -from merlin.dag.executors import LocalExecutor - - -class Runtime: - """A Systems Graph Runtime. - - This class can be used as a base class for custom runtimes. - """ - - def __init__(self, executor=None): - """Construct a Runtime. - - Parameters - ---------- - executor : Executor, optional - The Graph Executor to use to use for the transform, by default None - """ - self.executor = executor or LocalExecutor() - self.op_table = {} - - def transform(self, graph: Graph, transformable: Transformable): - """Run the graph with the input data. - - Parameters - ---------- - graph : Graph - Graph of nodes container operator chains for data manipulation. - transformable : Transformable - Input data to transform in graph. - - Returns - ------- - Transformable - Input data after it has been transformed via graph. - """ - return self.executor.transform(transformable, [graph.output_node]) - - def export(self): - """Optional method. - Implemented for runtimes that require an exported artifact to transform. - """ - raise NotImplementedError diff --git a/merlin/systems/dag/runtimes/triton/runtime.py b/merlin/systems/dag/runtimes/triton/runtime.py index e5e61c29d..2b685d42d 100644 --- a/merlin/systems/dag/runtimes/triton/runtime.py +++ b/merlin/systems/dag/runtimes/triton/runtime.py @@ -23,6 +23,7 @@ from google.protobuf import text_format from merlin.dag import postorder_iter_nodes +from merlin.dag.runtime import Runtime from merlin.systems.dag.ops import compute_dims from merlin.systems.dag.ops.compat import ( cuml_ensemble, @@ -32,7 +33,6 @@ xgboost, ) from merlin.systems.dag.ops.workflow import TransformWorkflow -from merlin.systems.dag.runtimes import Runtime from merlin.systems.dag.runtimes.triton.ops.operator import TritonOperator, add_model_param from merlin.systems.dag.runtimes.triton.ops.workflow import TransformWorkflowTriton diff --git a/tests/unit/systems/dag/runtimes/local/ops/fil/test_lightgbm.py b/tests/unit/systems/dag/runtimes/local/ops/fil/test_lightgbm.py index 14f2fa41e..9e67128e2 100644 --- a/tests/unit/systems/dag/runtimes/local/ops/fil/test_lightgbm.py +++ b/tests/unit/systems/dag/runtimes/local/ops/fil/test_lightgbm.py @@ -3,11 +3,11 @@ import pytest from merlin.dag import ColumnSelector +from merlin.dag.runtime import Runtime from merlin.dtypes.shape import Shape from merlin.schema import ColumnSchema, Schema from merlin.systems.dag.ensemble import Ensemble from merlin.systems.dag.ops.fil import PredictForest -from merlin.systems.dag.runtimes.base_runtime import Runtime from merlin.table import TensorTable sklearn_datasets = pytest.importorskip("sklearn.datasets") diff --git a/tests/unit/systems/dag/runtimes/local/ops/fil/test_sklearn.py b/tests/unit/systems/dag/runtimes/local/ops/fil/test_sklearn.py index 57d49ddd7..589f7c63e 100644 --- a/tests/unit/systems/dag/runtimes/local/ops/fil/test_sklearn.py +++ b/tests/unit/systems/dag/runtimes/local/ops/fil/test_sklearn.py @@ -3,11 +3,11 @@ import pytest from merlin.dag import ColumnSelector +from merlin.dag.runtime import Runtime from merlin.dtypes.shape import Shape from merlin.schema import ColumnSchema, Schema from merlin.systems.dag.ensemble import Ensemble from merlin.systems.dag.ops.fil import PredictForest -from merlin.systems.dag.runtimes.base_runtime import Runtime from merlin.table import TensorTable sklearn_datasets = pytest.importorskip("sklearn.datasets") diff --git a/tests/unit/systems/dag/runtimes/local/ops/fil/test_xgboost.py b/tests/unit/systems/dag/runtimes/local/ops/fil/test_xgboost.py index 70d194085..85c0317c9 100644 --- a/tests/unit/systems/dag/runtimes/local/ops/fil/test_xgboost.py +++ b/tests/unit/systems/dag/runtimes/local/ops/fil/test_xgboost.py @@ -4,11 +4,11 @@ from merlin.core.compat import HAS_GPU from merlin.dag import ColumnSelector +from merlin.dag.runtime import Runtime from merlin.dtypes.shape import Shape from merlin.schema import ColumnSchema, Schema from merlin.systems.dag.ensemble import Ensemble from merlin.systems.dag.ops.fil import PredictForest -from merlin.systems.dag.runtimes.base_runtime import Runtime from merlin.table import TensorTable sklearn_datasets = pytest.importorskip("sklearn.datasets") diff --git a/tests/unit/systems/dag/runtimes/local/ops/nvtabular/test_ensemble.py b/tests/unit/systems/dag/runtimes/local/ops/nvtabular/test_ensemble.py index 250246b1f..846c3c809 100644 --- a/tests/unit/systems/dag/runtimes/local/ops/nvtabular/test_ensemble.py +++ b/tests/unit/systems/dag/runtimes/local/ops/nvtabular/test_ensemble.py @@ -16,7 +16,7 @@ import numpy as np import pytest -from merlin.systems.dag.runtimes.base_runtime import Runtime +from merlin.dag.runtime import Runtime from merlin.table import TensorTable from nvtabular import Workflow from nvtabular import ops as wf_ops diff --git a/tests/unit/systems/dag/runtimes/local/ops/tensorflow/test_ensemble.py b/tests/unit/systems/dag/runtimes/local/ops/tensorflow/test_ensemble.py index c45ef4007..c9e78197c 100644 --- a/tests/unit/systems/dag/runtimes/local/ops/tensorflow/test_ensemble.py +++ b/tests/unit/systems/dag/runtimes/local/ops/tensorflow/test_ensemble.py @@ -17,8 +17,8 @@ import pytest from merlin.dag import ColumnSelector +from merlin.dag.runtime import Runtime from merlin.schema import Tags -from merlin.systems.dag.runtimes.base_runtime import Runtime from nvtabular import Workflow from nvtabular import ops as wf_ops diff --git a/tests/unit/systems/dag/runtimes/local/ops/torch/test_op.py b/tests/unit/systems/dag/runtimes/local/ops/torch/test_op.py index 790d8d795..5d73e1a94 100644 --- a/tests/unit/systems/dag/runtimes/local/ops/torch/test_op.py +++ b/tests/unit/systems/dag/runtimes/local/ops/torch/test_op.py @@ -18,9 +18,9 @@ import numpy as np import pytest +from merlin.dag.runtime import Runtime from merlin.schema import ColumnSchema, Schema from merlin.systems.dag.ensemble import Ensemble -from merlin.systems.dag.runtimes.base_runtime import Runtime from merlin.table import TensorTable torch = pytest.importorskip("torch")