From ed96ab6804ff6cbb231465cb4f19223b1a0b237a Mon Sep 17 00:00:00 2001 From: Misha Chornyi Date: Thu, 5 Dec 2024 15:02:53 -0800 Subject: [PATCH 1/2] Update onnxruntime version --- build.py | 4 ++-- qa/common/gen_qa_model_repository | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build.py b/build.py index 509b2c4eaa..3c0ba555bc 100755 --- a/build.py +++ b/build.py @@ -75,8 +75,8 @@ "triton_container_version": "24.12", "upstream_container_version": "24.12", "ort_version": "1.20.1", - "ort_openvino_version": "2024.4.0", - "standalone_openvino_version": "2024.4.0", + "ort_openvino_version": "2024.5.0", + "standalone_openvino_version": "2024.5.0", "dcgm_version": "3.3.6", "vllm_version": "0.5.5", "rhel_py_version": "3.12.3", diff --git a/qa/common/gen_qa_model_repository b/qa/common/gen_qa_model_repository index f4cdc2b6a2..32e2492ad4 100755 --- a/qa/common/gen_qa_model_repository +++ b/qa/common/gen_qa_model_repository @@ -55,7 +55,7 @@ ONNX_VERSION=1.16.1 ONNX_OPSET=0 # OPENVINO version -OPENVINO_VERSION=2024.4.0 +OPENVINO_VERSION=2024.5.0 UBUNTU_IMAGE=${UBUNTU_IMAGE:=ubuntu:22.04} PYTORCH_IMAGE=${PYTORCH_IMAGE:=nvcr.io/nvidia/pytorch:$TRITON_VERSION-py3} From ca4d30918c6efc1631ad4980194cb48cbcb57b47 Mon Sep 17 00:00:00 2001 From: Misha Chornyi Date: Tue, 10 Dec 2024 16:28:13 -0800 Subject: [PATCH 2/2] Revert "build: Update OpenVINO model generation script with new API (#7811)" This reverts commit 71ca0c551b2cb2959c39dc89f24d448741002ca2. --- qa/common/gen_qa_dyna_sequence_models.py | 28 ++++++++++++---------- qa/common/gen_qa_identity_models.py | 16 ++++++------- qa/common/gen_qa_models.py | 30 +++++++++++------------- qa/common/gen_qa_reshape_models.py | 14 ++++++----- qa/common/gen_qa_sequence_models.py | 20 +++++++++------- 5 files changed, 56 insertions(+), 52 deletions(-) diff --git a/qa/common/gen_qa_dyna_sequence_models.py b/qa/common/gen_qa_dyna_sequence_models.py index 94087cb8c6..bcb5a3a2b4 100755 --- a/qa/common/gen_qa_dyna_sequence_models.py +++ b/qa/common/gen_qa_dyna_sequence_models.py @@ -1292,26 +1292,27 @@ def create_openvino_modelfile(models_dir, model_version, max_batch, dtype, shape ) model_version_dir = models_dir + "/" + model_name + "/" + str(model_version) - in0 = ov.opset1.parameter(shape=batch_dim + shape, dtype=dtype, name="INPUT") - start = ov.opset1.parameter(shape=batch_dim + shape, dtype=dtype, name="START") - end = ov.opset1.parameter(shape=batch_dim + shape, dtype=dtype, name="END") - ready = ov.opset1.parameter(shape=batch_dim + shape, dtype=dtype, name="READY") - corrid = ov.opset1.parameter(shape=batch_dim + shape, dtype=dtype, name="CORRID") + in0 = ng.parameter(shape=batch_dim + shape, dtype=dtype, name="INPUT") + start = ng.parameter(shape=batch_dim + shape, dtype=dtype, name="START") + end = ng.parameter(shape=batch_dim + shape, dtype=dtype, name="END") + ready = ng.parameter(shape=batch_dim + shape, dtype=dtype, name="READY") + corrid = ng.parameter(shape=batch_dim + shape, dtype=dtype, name="CORRID") - tmp1 = ov.opset1.add(in0, start) - tmp2 = ov.opset1.multiply(end, corrid) - tmp = ov.opset1.add(tmp1, tmp2) - op0 = ov.opset1.multiply(tmp, ready, name="OUTPUT") + tmp1 = ng.add(in0, start) + tmp2 = ng.multiply(end, corrid) + tmp = ng.add(tmp1, tmp2) + op0 = ng.multiply(tmp, ready, name="OUTPUT") - model = ov.Model([op0], [in0, start, end, ready, corrid], model_name) + function = ng.impl.Function([op0], [in0, start, end, ready, corrid], model_name) + ie_network = IENetwork(ng.impl.Function.to_capsule(function)) try: os.makedirs(model_version_dir) except OSError as ex: pass # ignore existing dir - ov.serialize( - model, model_version_dir + "/model.xml", model_version_dir + "/model.bin" + ie_network.serialize( + model_version_dir + "/model.xml", model_version_dir + "/model.bin" ) @@ -1559,7 +1560,8 @@ def create_models(models_dir, dtype, shape, no_batch=True): import torch from torch import nn if FLAGS.openvino: - import openvino.runtime as ov + from openvino.inference_engine import IENetwork + import ngraph as ng import test_util as tu diff --git a/qa/common/gen_qa_identity_models.py b/qa/common/gen_qa_identity_models.py index abc57ab48a..7b7066dbb9 100755 --- a/qa/common/gen_qa_identity_models.py +++ b/qa/common/gen_qa_identity_models.py @@ -574,18 +574,17 @@ def create_openvino_modelfile( in_name = "INPUT{}".format(io_num) out_name = "OUTPUT{}".format(io_num) openvino_inputs.append( - ov.opset1.parameter(shape=batch_dim + shape, dtype=dtype, name=in_name) - ) - openvino_outputs.append( - ov.opset1.result(openvino_inputs[io_num], name=out_name) + ng.parameter(shape=batch_dim + shape, dtype=dtype, name=in_name) ) + openvino_outputs.append(ng.result(openvino_inputs[io_num], name=out_name)) - model = ov.Model(openvino_outputs, openvino_inputs, model_name) + function = ng.impl.Function(openvino_outputs, openvino_inputs, model_name) + ie_network = IENetwork(ng.impl.Function.to_capsule(function)) os.makedirs(model_version_dir, exist_ok=True) - ov.serialize( - model, model_version_dir + "/model.xml", model_version_dir + "/model.bin" + ie_network.serialize( + model_version_dir + "/model.xml", model_version_dir + "/model.bin" ) @@ -1318,7 +1317,8 @@ def create_models(models_dir, dtype, shape, io_cnt=1, no_batch=True): ): import tensorrt as trt if FLAGS.openvino: - import openvino.runtime as ov + from openvino.inference_engine import IENetwork + import ngraph as ng import test_util as tu diff --git a/qa/common/gen_qa_models.py b/qa/common/gen_qa_models.py index 1af7762869..efe3fca1e9 100755 --- a/qa/common/gen_qa_models.py +++ b/qa/common/gen_qa_models.py @@ -1809,31 +1809,28 @@ def create_openvino_modelfile( ) model_version_dir = models_dir + "/" + model_name + "/" + str(model_version) - in0 = ov.opset1.parameter( - shape=batch_dim + input_shape, dtype=input_dtype, name="INPUT0" - ) - in1 = ov.opset1.parameter( - shape=batch_dim + input_shape, dtype=input_dtype, name="INPUT1" - ) + in0 = ng.parameter(shape=batch_dim + input_shape, dtype=input_dtype, name="INPUT0") + in1 = ng.parameter(shape=batch_dim + input_shape, dtype=input_dtype, name="INPUT1") - r0 = ov.opset1.add(in0, in1) if not swap else ov.opset1.subtract(in0, in1) - r1 = ov.opset1.subtract(in0, in1) if not swap else ov.opset1.add(in0, in1) + r0 = ng.add(in0, in1) if not swap else ng.subtract(in0, in1) + r1 = ng.subtract(in0, in1) if not swap else ng.add(in0, in1) - result0 = ov.opset1.reshape(r0, batch_dim + output0_shape, special_zero=False) - result1 = ov.opset1.reshape(r1, batch_dim + output1_shape, special_zero=False) + result0 = ng.reshape(r0, batch_dim + output0_shape, special_zero=False) + result1 = ng.reshape(r1, batch_dim + output1_shape, special_zero=False) - op0 = ov.opset1.convert(result0, destination_type=output0_dtype, name="OUTPUT0") - op1 = ov.opset1.convert(result1, destination_type=output1_dtype, name="OUTPUT1") + op0 = ng.convert(result0, destination_type=output0_dtype, name="OUTPUT0") + op1 = ng.convert(result1, destination_type=output1_dtype, name="OUTPUT1") - model = ov.Model([op0, op1], [in0, in1], model_name) + function = ng.impl.Function([op0, op1], [in0, in1], model_name) + ie_network = IENetwork(ng.impl.Function.to_capsule(function)) try: os.makedirs(model_version_dir) except OSError as ex: pass # ignore existing dir - ov.serialize( - model, model_version_dir + "/model.xml", model_version_dir + "/model.bin" + ie_network.serialize( + model_version_dir + "/model.xml", model_version_dir + "/model.bin" ) @@ -2489,7 +2486,8 @@ def create_fixed_models( import torch from torch import nn if FLAGS.openvino: - import openvino.runtime as ov + from openvino.inference_engine import IENetwork + import ngraph as ng import test_util as tu diff --git a/qa/common/gen_qa_reshape_models.py b/qa/common/gen_qa_reshape_models.py index a27355574f..3abb2e978a 100755 --- a/qa/common/gen_qa_reshape_models.py +++ b/qa/common/gen_qa_reshape_models.py @@ -934,13 +934,13 @@ def create_openvino_modelfile( in_name = "INPUT{}".format(io_num) out_name = "OUTPUT{}".format(io_num) openvino_inputs.append( - ov.opset1.parameter( + ng.parameter( shape=batch_dim + input_shapes[io_num], dtype=dtype, name=in_name ) ) openvino_outputs.append( - ov.opset1.reshape( + ng.reshape( openvino_inputs[io_num], batch_dim + output_shapes[io_num], name=out_name, @@ -948,15 +948,16 @@ def create_openvino_modelfile( ) ) - model = ov.Model(openvino_outputs, openvino_inputs, model_name) + function = ng.impl.Function(openvino_outputs, openvino_inputs, model_name) + ie_network = IENetwork(ng.impl.Function.to_capsule(function)) try: os.makedirs(model_version_dir) except OSError as ex: pass # ignore existing dir - ov.serialize( - model, model_version_dir + "/model.xml", model_version_dir + "/model.bin" + ie_network.serialize( + model_version_dir + "/model.xml", model_version_dir + "/model.bin" ) @@ -1462,7 +1463,8 @@ def create_openvino_models( import torch from torch import nn if FLAGS.openvino: - import openvino.runtime as ov + from openvino.inference_engine import IENetwork + import ngraph as ng import test_util as tu diff --git a/qa/common/gen_qa_sequence_models.py b/qa/common/gen_qa_sequence_models.py index 25c35b7603..42557c7747 100755 --- a/qa/common/gen_qa_sequence_models.py +++ b/qa/common/gen_qa_sequence_models.py @@ -1145,22 +1145,23 @@ def create_openvino_modelfile(models_dir, model_version, max_batch, dtype, shape ) model_version_dir = models_dir + "/" + model_name + "/" + str(model_version) - in0 = ov.opset1.parameter(shape=batch_dim + shape, dtype=dtype, name="INPUT") - start = ov.opset1.parameter(shape=batch_dim + shape, dtype=dtype, name="START") - ready = ov.opset1.parameter(shape=batch_dim + shape, dtype=dtype, name="READY") + in0 = ng.parameter(shape=batch_dim + shape, dtype=dtype, name="INPUT") + start = ng.parameter(shape=batch_dim + shape, dtype=dtype, name="START") + ready = ng.parameter(shape=batch_dim + shape, dtype=dtype, name="READY") - tmp = ov.opset1.add(in0, start) - op0 = ov.opset1.multiply(tmp, ready, name="OUTPUT") + tmp = ng.add(in0, start) + op0 = ng.multiply(tmp, ready, name="OUTPUT") - model = ov.Model([op0], [in0, start, ready], model_name) + function = ng.impl.Function([op0], [in0, start, ready], model_name) + ie_network = IENetwork(ng.impl.Function.to_capsule(function)) try: os.makedirs(model_version_dir) except OSError as ex: pass # ignore existing dir - ov.serialize( - model, model_version_dir + "/model.xml", model_version_dir + "/model.bin" + ie_network.serialize( + model_version_dir + "/model.xml", model_version_dir + "/model.bin" ) @@ -1420,7 +1421,8 @@ def create_models(models_dir, dtype, shape, no_batch=True): import torch from torch import nn if FLAGS.openvino: - import openvino.runtime as ov + from openvino.inference_engine import IENetwork + import ngraph as ng import test_util as tu