[GPU] LSTMSequence and LSTMCell optimization #26767
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| // Copyright (C) 2018-2024 Intel Corporation | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
| // | ||
|
|
||
| #pragma once | ||
| #include "primitive.hpp" | ||
| #include "activation.hpp" | ||
| #include <vector> | ||
| #include <algorithm> | ||
| #include "intel_gpu/graph/serialization/activation_serializer.hpp" | ||
| #include "rnn.hpp" | ||
|
|
||
|
|
||
| namespace cldnn { | ||
|
|
||
| struct lstm_cell : public RNNParams<lstm_cell> { | ||
| CLDNN_DECLARE_PRIMITIVE(lstm_cell) | ||
| using vec_activation = std::vector<activation_func>; | ||
| using vec_activation_param = std::vector<activation_additional_params>; | ||
| using RNNParams::RNNParams; | ||
| lstm_cell(const lstm_cell&) = default; | ||
| lstm_cell() : RNNParams() {} | ||
| }; | ||
| } // namespace cldnn |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,203 @@ | ||
| // Copyright (C) 2018-2024 Intel Corporation | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
| // | ||
|
|
||
| #pragma once | ||
| #include "primitive.hpp" | ||
| #include "activation.hpp" | ||
| #include <vector> | ||
| #include <algorithm> | ||
| #include <string> | ||
| #include "intel_gpu/graph/serialization/activation_serializer.hpp" | ||
|
|
||
| namespace cldnn { | ||
|
|
||
| /// @brief Weights orders | ||
| /// @details Specifies the order in which the weights are concatenated. | ||
| /// e.g. [i, o, f, z] : [input, output, forget, block] | ||
| /// ONNX order: iofz | ||
| /// Caffe order: ifoz | ||
| /// pyTorch order: izof | ||
| /// OV order: fizo | ||
| enum class lstm_weights_order { | ||
| iofz, | ||
| ifoz, | ||
| izof, | ||
| fizo | ||
| }; | ||
|
|
||
| template <typename PType> | ||
| struct RNNParams : public primitive_base<PType> { | ||
| RNNParams() : primitive_base<PType>("", {}) {} | ||
| RNNParams(const RNNParams&) = default; | ||
| RNNParams(const primitive_id& id, | ||
p-durandin marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| const input_info& x, | ||
| const input_info& initial_hidden_state, | ||
| const input_info& initial_cell_state, | ||
| const input_info& W, | ||
| const input_info& R, | ||
| const input_info& B, | ||
| const input_info& seq_lenghts, | ||
| const primitive_id& out1_prim_id = "", | ||
| const primitive_id& out2_prim_id = "", | ||
| const float clip = 0, | ||
| bool input_forget = false, | ||
| const std::vector<activation_func>& activations = {activation_func::logistic, | ||
| activation_func::hyperbolic_tan, | ||
| activation_func::hyperbolic_tan}, | ||
| const std::vector<activation_additional_params>& activation_params = {}, | ||
| const lstm_weights_order& offset_order = lstm_weights_order::iofz, | ||
| const ov::op::RecurrentSequenceDirection direction = ov::op::RecurrentSequenceDirection::FORWARD, | ||
| const padding& output_padding = padding(), | ||
|
There was a problem hiding this comment. I think padding arg is not needed as it's always set as default |
||
| const int num_outputs = 1) | ||
| : primitive_base<PType>(id, {x}, num_outputs, {optional_data_type()}, {output_padding}), | ||
| x(x), | ||
| initial_hidden_state(initial_hidden_state), | ||
| initial_cell_state(initial_cell_state), | ||
| W(W), | ||
| R(R), | ||
| B(B), | ||
| seq_lenghts(seq_lenghts), | ||
| out1_prim_id(out1_prim_id), | ||
| out2_prim_id(out2_prim_id), | ||
| clip(clip), | ||
| input_forget(input_forget), | ||
| activations(activations), | ||
| activation_params(activation_params), | ||
| offset_order(offset_order), | ||
| direction(direction) { | ||
| std::vector<std::string> pids{initial_hidden_state.pid, initial_cell_state.pid, W.pid, R.pid, B.pid, seq_lenghts.pid, out1_prim_id, out2_prim_id}; | ||
| for (auto pid : pids) { | ||
| if (!pid.empty()) { | ||
| primitive_base<PType>::input.push_back(pid); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| input_info x; | ||
| input_info initial_hidden_state; | ||
| input_info initial_cell_state; | ||
| input_info W; | ||
| input_info R; | ||
| input_info B; | ||
| input_info seq_lenghts; | ||
| primitive_id out1_prim_id; | ||
| primitive_id out2_prim_id; | ||
| /// @brief Cell clip threshold T. It is applied to the input of activations [-T, T]. No clip is applied if it is not specified. | ||
| float clip; | ||
| bool input_forget; | ||
| /// @brief A list of 3 activation functions for the input, output, forget, cell, and hidden. | ||
| std::vector<activation_func> activations; | ||
| /// @brief Optional scaling values used by some activation functions. The values are consumed in the order of activation functions. | ||
| std::vector<activation_additional_params> activation_params; | ||
| /// @brief Weights, recurrent weights, and biases order. [iofz] : ONNX, [ifoz] : Caffe | ||
| lstm_weights_order offset_order; | ||
| /// @brief direction of LSTMSequence - only FORWARD or REVERSE, currently BIDIRECTIONAL not supported | ||
| ov::op::RecurrentSequenceDirection direction; | ||
|
|
||
| int num_directions() const { | ||
| return direction == ov::op::RecurrentSequenceDirection::BIDIRECTIONAL ? 2 : 1; | ||
| } | ||
|
|
||
| size_t hash() const override { | ||
| size_t seed = primitive::hash(); | ||
| seed = hash_combine(seed, x.pid); | ||
| seed = hash_combine(seed, initial_hidden_state.pid); | ||
| seed = hash_combine(seed, initial_cell_state.pid); | ||
| seed = hash_combine(seed, seq_lenghts.pid); | ||
| seed = hash_combine(seed, W.pid); | ||
| seed = hash_combine(seed, R.pid); | ||
| seed = hash_combine(seed, B.pid); | ||
|
There was a problem hiding this comment. Comparison and hashing of the primitive ids prevents primitive reuse if we have multiple instances of the same op. So you shall just hash/compare only presence flag to all inputs. As an example you can use |
||
| seed = hash_combine(seed, out1_prim_id); | ||
| seed = hash_combine(seed, out2_prim_id); | ||
| seed = hash_combine(seed, clip); | ||
| seed = hash_range(seed, activations.begin(), activations.end()); | ||
| for (auto& act_param : activation_params) { | ||
| seed = hash_combine(seed, act_param.a); | ||
| seed = hash_combine(seed, act_param.b); | ||
| } | ||
| seed = hash_combine(seed, offset_order); | ||
| seed = hash_combine(seed, direction); | ||
| return seed; | ||
| } | ||
|
|
||
| bool operator==(const primitive& rhs) const override { | ||
| if (!primitive::compare_common_params(rhs)) | ||
| return false; | ||
|
|
||
| auto rhs_casted = downcast<const PType>(rhs); | ||
| bool act_params_eq = activation_params.size() == rhs_casted.activation_params.size(); | ||
| for (size_t i = 0; i < activation_params.size(); ++i) { | ||
| act_params_eq &= activation_params[i].a == rhs_casted.activation_params[i].a && | ||
| activation_params[i].b == rhs_casted.activation_params[i].b; | ||
| } | ||
|
|
||
| #define cmp_fields(name) name == rhs_casted.name | ||
| return act_params_eq && | ||
| cmp_fields(x) && | ||
| cmp_fields(initial_hidden_state) && | ||
| cmp_fields(initial_cell_state) && | ||
| cmp_fields(seq_lenghts) && | ||
| cmp_fields(W) && | ||
| cmp_fields(R) && | ||
| cmp_fields(B) && | ||
|
There was a problem hiding this comment. Here you also shouldn't compare string values, but rather check presence of inputs |
||
| cmp_fields(out1_prim_id) && | ||
| cmp_fields(out2_prim_id) && | ||
| cmp_fields(clip) && | ||
| cmp_fields(activations) && | ||
| cmp_fields(offset_order) && | ||
| cmp_fields(direction); | ||
| #undef cmp_fields | ||
| } | ||
|
|
||
| void save(BinaryOutputBuffer& ob) const override { | ||
| primitive_base<PType>::save(ob); | ||
| ob << x; | ||
| ob << initial_hidden_state; | ||
| ob << initial_cell_state; | ||
| ob << W; | ||
| ob << R; | ||
| ob << B; | ||
| ob << seq_lenghts; | ||
| ob << out1_prim_id; | ||
| ob << out2_prim_id; | ||
| ob << clip; | ||
| ob << activations; | ||
| ob << activation_params; | ||
| ob << make_data(&offset_order, sizeof(lstm_weights_order)); | ||
| ob << make_data(&direction, sizeof(ov::op::RecurrentSequenceDirection)); | ||
| } | ||
|
|
||
| void load(BinaryInputBuffer& ib) override{ | ||
| primitive_base<PType>::load(ib); | ||
| ib >> x; | ||
| ib >> initial_hidden_state; | ||
| ib >> initial_cell_state; | ||
| ib >> W; | ||
| ib >> R; | ||
| ib >> B; | ||
| ib >> seq_lenghts; | ||
| ib >> out1_prim_id; | ||
| ib >> out2_prim_id; | ||
| ib >> clip; | ||
| ib >> activations; | ||
| ib >> activation_params; | ||
| ib >> make_data(&offset_order, sizeof(lstm_weights_order)); | ||
| ib >> make_data(&direction, sizeof(ov::op::RecurrentSequenceDirection)); | ||
| } | ||
| }; | ||
|
|
||
| struct lstm_seq : public RNNParams<lstm_seq> { | ||
| CLDNN_DECLARE_PRIMITIVE(lstm_seq) | ||
| using vec_activation = std::vector<activation_func>; | ||
| using vec_activation_param = std::vector<activation_additional_params>; | ||
| using RNNParams::RNNParams; | ||
| lstm_seq() : RNNParams() { | ||
| weights = W.pid; | ||
| input = x.pid; | ||
| } | ||
| lstm_seq(const lstm_seq&) = default; | ||
| primitive_id input; | ||
| primitive_id weights; | ||
| }; | ||
| } //namespace cldnn | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Why do you still keep (and use)
lstm_eltprimitive given that you introducelstm_cellandlstm_seq? I'd expect that it's not needed anymoreThere was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
there is poor performance of onednn on case of seq_len = 1 , so I don't update it
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Could you try enabling ngraph pass for decomposition in such case? Ideally we need to get rid of this lstm_elt primitive and related decomposition code in program builder to switch to new shape inference completely
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
lstm_cell which will be used in such case is too slow
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
One of the initial goals of this patch was removing this lstm decomposition on the plugin side to bunch of custom primitives (and thus removing lstm_elt primitive). And that's still needed.
Also, as I can see, lstm_cell primitive is not used at all currently, which means there's no sense to add it. So my suggestion is to continue perf tuning then.