diff --git a/bessctl/static/pipeline.js b/bessctl/static/pipeline.js
index f0041745c4..94c45abeef 100644
--- a/bessctl/static/pipeline.js
+++ b/bessctl/static/pipeline.js
@@ -134,7 +134,7 @@ function graph_to_dot(modules) {
var ogates = module.show_ogates ? gates_to_str(module.ogates, 'ogate') : '';
nodes += `
- ${module_name} [shape=plaintext label=
+ "${module_name}" [shape=plaintext label=
<
${igates}
${module_name} |
diff --git a/core/dpdk.cc b/core/dpdk.cc
index ffade53059..964d513c2c 100644
--- a/core/dpdk.cc
+++ b/core/dpdk.cc
@@ -123,8 +123,10 @@ void init_eal(int dpdk_mb_per_socket, std::string nonworker_corelist) {
"--legacy-mem",
};
+ if (FLAGS_iova != "")
+ rte_args.Append({"--iova", FLAGS_iova});
+
if (dpdk_mb_per_socket <= 0) {
- rte_args.Append({"--iova", (FLAGS_iova != "") ? FLAGS_iova : "va"});
rte_args.Append({"--no-huge"});
// even if we opt out of using hugepages, many DPDK libraries still rely on
@@ -132,8 +134,6 @@ void init_eal(int dpdk_mb_per_socket, std::string nonworker_corelist) {
// memory in advance. We allocate 512MB (this is shared among nodes).
rte_args.Append({"-m", "512"});
} else {
- rte_args.Append({"--iova", (FLAGS_iova != "") ? FLAGS_iova : "pa"});
-
std::string opt_socket_mem = std::to_string(dpdk_mb_per_socket);
for (int i = 1; i < NumNumaNodes(); i++) {
opt_socket_mem += "," + std::to_string(dpdk_mb_per_socket);
diff --git a/core/drivers/pmd.cc b/core/drivers/pmd.cc
index fd4a7b0572..dbebbe5513 100644
--- a/core/drivers/pmd.cc
+++ b/core/drivers/pmd.cc
@@ -32,10 +32,18 @@
#include
#include
+#include
#include "../utils/ether.h"
#include "../utils/format.h"
+// TODO: Replace with one time initialized key during InitDriver?
+static uint8_t rss_key[40] = {0xD8, 0x2A, 0x6C, 0x5A, 0xDD, 0x3B, 0x9D, 0x1E,
+ 0x14, 0xCE, 0x2F, 0x37, 0x86, 0xB2, 0x69, 0xF0,
+ 0x44, 0x31, 0x7E, 0xA2, 0x07, 0xA5, 0x0A, 0x99,
+ 0x49, 0xC6, 0xA4, 0xFE, 0x0C, 0x4F, 0x59, 0x02,
+ 0xD4, 0x44, 0xE2, 0x4A, 0xDB, 0xE1, 0x05, 0x82};
+
static const rte_eth_conf default_eth_conf(const rte_eth_dev_info &dev_info,
int nb_rxq) {
rte_eth_conf ret = {};
@@ -45,8 +53,8 @@ static const rte_eth_conf default_eth_conf(const rte_eth_dev_info &dev_info,
ret.rxmode.offloads = 0;
ret.rx_adv_conf.rss_conf = {
- .rss_key = nullptr,
- .rss_key_len = 0,
+ .rss_key = rss_key,
+ .rss_key_len = sizeof(rss_key),
.rss_hf = (ETH_RSS_IP | ETH_RSS_UDP | ETH_RSS_TCP | ETH_RSS_SCTP) &
dev_info.flow_type_rss_offloads,
};
@@ -199,6 +207,122 @@ static CommandResponse find_dpdk_vdev(const std::string &vdev,
return CommandSuccess();
}
+CommandResponse flow_create_one(dpdk_port_t port_id,
+ const uint32_t &flow_profile, int size,
+ uint64_t rss_types,
+ rte_flow_item_type *pattern) {
+ struct rte_flow_item items[size];
+ memset(items, 0, sizeof(items));
+
+ for (int i = 0; i < size; i++) {
+ items[i].type = pattern[i];
+ items[i].spec = nullptr;
+ items[i].mask = nullptr;
+ }
+
+ struct rte_flow *handle;
+ struct rte_flow_error err;
+ memset(&err, 0, sizeof(err));
+
+ struct rte_flow_action actions[2];
+ memset(actions, 0, sizeof(actions));
+
+ struct rte_flow_attr attributes;
+ memset(&attributes, 0, sizeof(attributes));
+ attributes.ingress = 1;
+
+ struct rte_flow_action_rss action_rss;
+ memset(&action_rss, 0, sizeof(action_rss));
+ action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT;
+ action_rss.key_len = 0;
+ action_rss.types = rss_types;
+
+ actions[0].type = RTE_FLOW_ACTION_TYPE_RSS;
+ actions[0].conf = &action_rss;
+ actions[1].type = RTE_FLOW_ACTION_TYPE_END;
+
+ int ret = rte_flow_validate(port_id, &attributes, items, actions, &err);
+ if (ret)
+ return CommandFailure(EINVAL,
+ "Port %u: Failed to validate flow profile %u %s",
+ port_id, flow_profile, err.message);
+
+ handle = rte_flow_create(port_id, &attributes, items, actions, &err);
+ if (handle == nullptr)
+ return CommandFailure(EINVAL, "Port %u: Failed to create flow %s", port_id,
+ err.message);
+
+ return CommandSuccess();
+}
+
+#define NUM_ELEMENTS(x) (sizeof(x) / sizeof((x)[0]))
+
+enum FlowProfile : uint32_t
+{
+ profileN3 = 3,
+ profileN6 = 6,
+ profileN9 = 9,
+};
+
+CommandResponse flow_create(dpdk_port_t port_id, const uint32_t &flow_profile) {
+ CommandResponse err;
+
+ rte_flow_item_type N39_NSA[] = {
+ RTE_FLOW_ITEM_TYPE_ETH, RTE_FLOW_ITEM_TYPE_IPV4, RTE_FLOW_ITEM_TYPE_UDP,
+ RTE_FLOW_ITEM_TYPE_GTPU, RTE_FLOW_ITEM_TYPE_IPV4,
+ RTE_FLOW_ITEM_TYPE_END};
+
+ rte_flow_item_type N39_SA[] = {
+ RTE_FLOW_ITEM_TYPE_ETH, RTE_FLOW_ITEM_TYPE_IPV4, RTE_FLOW_ITEM_TYPE_UDP,
+ RTE_FLOW_ITEM_TYPE_GTPU, RTE_FLOW_ITEM_TYPE_GTP_PSC,
+ RTE_FLOW_ITEM_TYPE_IPV4,
+ RTE_FLOW_ITEM_TYPE_END};
+
+ rte_flow_item_type N6[] = {
+ RTE_FLOW_ITEM_TYPE_ETH, RTE_FLOW_ITEM_TYPE_IPV4,
+ RTE_FLOW_ITEM_TYPE_END};
+
+ switch (flow_profile) {
+ uint64_t rss_types;
+ // N3 traffic with and without PDU Session container
+ case profileN3:
+ rss_types = ETH_RSS_IPV4 | ETH_RSS_L3_SRC_ONLY;
+ err = flow_create_one(port_id, flow_profile, NUM_ELEMENTS(N39_NSA),
+ rss_types, N39_NSA);
+ if (err.error().code() != 0) {
+ return err;
+ }
+
+ err = flow_create_one(port_id, flow_profile, NUM_ELEMENTS(N39_SA),
+ rss_types, N39_SA);
+ break;
+
+ // N6 traffic
+ case profileN6:
+ rss_types = ETH_RSS_IPV4 | ETH_RSS_L3_DST_ONLY;
+ err = flow_create_one(port_id, flow_profile, NUM_ELEMENTS(N6),
+ rss_types, N6);
+ break;
+
+ // N9 traffic with and without PDU Session container
+ case profileN9:
+ rss_types = ETH_RSS_IPV4 | ETH_RSS_L3_DST_ONLY;
+ err = flow_create_one(port_id, flow_profile, NUM_ELEMENTS(N39_NSA),
+ rss_types, N39_NSA);
+ if (err.error().code() != 0) {
+ return err;
+ }
+
+ err = flow_create_one(port_id, flow_profile, NUM_ELEMENTS(N39_SA),
+ rss_types, N39_SA);
+ break;
+
+ default:
+ return CommandFailure(EINVAL, "Unknown flow profile %u", flow_profile);
+ }
+ return err;
+}
+
CommandResponse PMDPort::Init(const bess::pb::PMDPortArg &arg) {
dpdk_port_t ret_port_id = DPDK_PORT_UNKNOWN;
@@ -245,16 +369,28 @@ CommandResponse PMDPort::Init(const bess::pb::PMDPortArg &arg) {
if (arg.loopback()) {
eth_conf.lpbk_mode = 1;
}
+ if (arg.hwcksum()) {
+ eth_conf.rxmode.offloads = DEV_RX_OFFLOAD_IPV4_CKSUM |
+ DEV_RX_OFFLOAD_UDP_CKSUM |
+ DEV_RX_OFFLOAD_TCP_CKSUM;
+ }
ret = rte_eth_dev_configure(ret_port_id, num_rxq, num_txq, ð_conf);
if (ret != 0) {
+ VLOG(1) << "Failed to configure with hardware checksum offload. "
+ << "Create PMDPort without hardware offload";
return CommandFailure(-ret, "rte_eth_dev_configure() failed");
}
- int sid = rte_eth_dev_socket_id(ret_port_id);
+ int sid = arg.socket_case() == bess::pb::PMDPortArg::kSocketId ?
+ arg.socket_id() : rte_eth_dev_socket_id(ret_port_id);
+ /* if socket_id is invalid, set to 0 */
if (sid < 0 || sid > RTE_MAX_NUMA_NODES) {
- sid = 0; // if socket_id is invalid, set to 0
+ LOG(WARNING) << "Invalid socket, falling back... ";
+ sid = 0;
}
+ LOG(INFO) << "Initializing Port:" << ret_port_id
+ << " with memory from socket " << sid;
eth_rxconf = dev_info.default_rxconf;
eth_rxconf.rx_drop_en = 1;
@@ -308,7 +444,12 @@ CommandResponse PMDPort::Init(const bess::pb::PMDPortArg &arg) {
}
}
- rte_eth_promiscuous_enable(ret_port_id);
+ if (arg.promiscuous_mode()) {
+ ret = rte_eth_promiscuous_enable(ret_port_id);
+ if (ret != 0) {
+ return CommandFailure(-ret, "rte_eth_promiscuous_enable() failed");
+ }
+ }
int offload_mask = 0;
offload_mask |= arg.vlan_offload_rx_strip() ? ETH_VLAN_STRIP_OFFLOAD : 0;
@@ -327,7 +468,8 @@ CommandResponse PMDPort::Init(const bess::pb::PMDPortArg &arg) {
}
dpdk_port_id_ = ret_port_id;
- int numa_node = rte_eth_dev_socket_id(static_cast(ret_port_id));
+ int numa_node = arg.socket_case() == bess::pb::PMDPortArg::kSocketId ?
+ sid : rte_eth_dev_socket_id(ret_port_id);
node_placement_ =
numa_node == -1 ? UNCONSTRAINED_SOCKET : (1ull << numa_node);
@@ -339,6 +481,15 @@ CommandResponse PMDPort::Init(const bess::pb::PMDPortArg &arg) {
driver_ = dev_info.driver_name ?: "unknown";
+ if (arg.flow_profiles_size() > 0){
+ for (int i = 0; i < arg.flow_profiles_size(); ++i) {
+ err = flow_create(ret_port_id, arg.flow_profiles(i));
+ if (err.error().code() != 0) {
+ return err;
+ }
+ }
+ }
+
return CommandSuccess();
}
@@ -448,9 +599,10 @@ void PMDPort::CollectStats(bool reset) {
port_stats_.inc.dropped = stats.imissed;
- // i40e/net_e1000_igb PMD drivers, ixgbevf and net_bonding vdevs don't support
- // per-queue stats
- if (driver_ == "net_i40e" || driver_ == "net_i40e_vf" ||
+ // ice/i40e/net_e1000_igb PMD drivers, ixgbevf and net_bonding vdevs don't
+ // support per-queue stats
+ if (driver_ == "net_ice" || driver_ == "net_iavf" ||
+ driver_ == "net_i40e" || driver_ == "net_i40e_vf" ||
driver_ == "net_ixgbe_vf" || driver_ == "net_bonding" ||
driver_ == "net_e1000_igb") {
// NOTE:
diff --git a/core/modules/exact_match.cc b/core/modules/exact_match.cc
index 072a0333dd..e50cf5cce6 100644
--- a/core/modules/exact_match.cc
+++ b/core/modules/exact_match.cc
@@ -50,20 +50,21 @@ const Commands ExactMatch::cmds = {
{"set_runtime_config", "ExactMatchConfig",
MODULE_CMD_FUNC(&ExactMatch::SetRuntimeConfig), Command::THREAD_UNSAFE},
{"add", "ExactMatchCommandAddArg", MODULE_CMD_FUNC(&ExactMatch::CommandAdd),
- Command::THREAD_UNSAFE},
+ Command::THREAD_SAFE},
{"delete", "ExactMatchCommandDeleteArg",
- MODULE_CMD_FUNC(&ExactMatch::CommandDelete), Command::THREAD_UNSAFE},
+ MODULE_CMD_FUNC(&ExactMatch::CommandDelete), Command::THREAD_SAFE},
{"clear", "EmptyArg", MODULE_CMD_FUNC(&ExactMatch::CommandClear),
- Command::THREAD_UNSAFE},
+ Command::THREAD_SAFE},
{"set_default_gate", "ExactMatchCommandSetDefaultGateArg",
MODULE_CMD_FUNC(&ExactMatch::CommandSetDefaultGate),
Command::THREAD_SAFE}};
CommandResponse ExactMatch::AddFieldOne(const bess::pb::Field &field,
const bess::pb::FieldData &mask,
- int idx) {
+ int idx, Type t) {
int size = field.num_bytes();
uint64_t mask64 = 0;
+
if (mask.encoding_case() == bess::pb::FieldData::kValueInt) {
mask64 = mask.value_int();
} else if (mask.encoding_case() == bess::pb::FieldData::kValueBin) {
@@ -73,12 +74,15 @@ CommandResponse ExactMatch::AddFieldOne(const bess::pb::Field &field,
Error ret;
if (field.position_case() == bess::pb::Field::kAttrName) {
- ret = table_.AddField(this, field.attr_name(), size, mask64, idx);
+ ret = (t == FIELD_TYPE)
+ ? table_.AddField(this, field.attr_name(), size, mask64, idx)
+ : AddValue(this, field.attr_name(), size, mask64, idx);
if (ret.first) {
return CommandFailure(ret.first, "%s", ret.second.c_str());
}
} else if (field.position_case() == bess::pb::Field::kOffset) {
- ret = table_.AddField(field.offset(), size, mask64, idx);
+ ret = (t == FIELD_TYPE) ? table_.AddField(field.offset(), size, mask64, idx)
+ : AddValue(field.offset(), size, mask64, idx);
if (ret.first) {
return CommandFailure(ret.first, "%s", ret.second.c_str());
}
@@ -103,9 +107,9 @@ CommandResponse ExactMatch::Init(const bess::pb::ExactMatchArg &arg) {
if (empty_masks_) {
bess::pb::FieldData emptymask;
- err = AddFieldOne(arg.fields(i), emptymask, i);
+ err = AddFieldOne(arg.fields(i), emptymask, i, FIELD_TYPE);
} else {
- err = AddFieldOne(arg.fields(i), arg.masks(i), i);
+ err = AddFieldOne(arg.fields(i), arg.masks(i), i, FIELD_TYPE);
}
if (err.error().code() != 0) {
@@ -113,8 +117,24 @@ CommandResponse ExactMatch::Init(const bess::pb::ExactMatchArg &arg) {
}
}
- default_gate_ = DROP_GATE;
+ empty_masks_ = arg.masksv_size() == 0;
+ for (auto i = 0; i < arg.values_size(); ++i) {
+ CommandResponse err;
+
+ if (empty_masks_) {
+ bess::pb::FieldData emptymask;
+ err = AddFieldOne(arg.values(i), emptymask, i, VALUE_TYPE);
+ } else {
+ err = AddFieldOne(arg.values(i), arg.masksv(i), i, VALUE_TYPE);
+ }
+
+ if (err.error().code() != 0) {
+ return err;
+ }
+ }
+ default_gate_ = DROP_GATE;
+ table_.Init(arg.entries());
return CommandSuccess();
}
@@ -157,7 +177,7 @@ CommandResponse ExactMatch::GetRuntimeConfig(const bess::pb::EmptyArg &) {
auto const &value = kv.second;
rule_t *rule = r.add_rules();
- rule->set_gate(value);
+ rule->set_gate(value.gate);
for (size_t i = 0; i < table_.num_fields(); i++) {
const ExactMatchField &f = table_.get_field(i);
bess::pb::FieldData *field = rule->add_fields();
@@ -199,9 +219,30 @@ Error ExactMatch::AddRule(const bess::pb::ExactMatchCommandAddArg &arg) {
}
ExactMatchRuleFields rule;
- RuleFieldsFromPb(arg.fields(), &rule);
+ ExactMatchRuleFields action;
+ Error err;
+ ValueTuple t;
+
+ /* clear value tuple */
+ memset(&t.action, 0, sizeof(t.action));
+ /* set gate */
+ t.gate = gate;
+ RuleFieldsFromPb(arg.fields(), &rule, FIELD_TYPE);
+ /* check whether values match with the the table's */
+ if (arg.values_size() != (ssize_t)num_values())
+ return std::make_pair(
+ EINVAL, bess::utils::Format(
+ "rule has incorrect number of values. Need %d, has %d",
+ (int)num_values(), arg.values_size()));
+ /* check if values are non-zero */
+ if (arg.values_size() > 0) {
+ RuleFieldsFromPb(arg.values(), &action, VALUE_TYPE);
+
+ if ((err = CreateValue(t.action, action)).first != 0)
+ return err;
+ }
- return table_.AddRule(gate, rule);
+ return table_.AddRule(t, rule);
}
// Uses an ExactMatchConfig to restore this module's runtime config.
@@ -221,6 +262,53 @@ CommandResponse ExactMatch::SetRuntimeConfig(
return CommandSuccess();
}
+void ExactMatch::setValues(bess::Packet *pkt, ExactMatchKey &action) {
+ size_t num_values_ = num_values();
+
+ for (size_t i = 0; i < num_values_; i++) {
+ int value_size = get_value(i).size;
+ int value_pos = get_value(i).pos;
+ int value_off = get_value(i).offset;
+ int value_attr_id = get_value(i).attr_id;
+ uint8_t *data = pkt->head_data() + value_off;
+
+ if (value_attr_id < 0) { /* if it is offset-based */
+ memcpy(data, reinterpret_cast(&action) + value_pos,
+ value_size);
+ } else { /* if it is attribute-based */
+ switch (value_size) {
+ case 1:
+ set_attr(this, value_attr_id, pkt,
+ *((uint8_t *)((uint8_t *)&action + value_pos)));
+ break;
+ case 2:
+ set_attr(this, value_attr_id, pkt,
+ *((uint16_t *)((uint8_t *)&action + value_pos)));
+ break;
+ case 4:
+ set_attr(this, value_attr_id, pkt,
+ *((uint32_t *)((uint8_t *)&action + value_pos)));
+ break;
+ case 8:
+ set_attr(this, value_attr_id, pkt,
+ *((uint64_t *)((uint8_t *)&action + value_pos)));
+ break;
+ default: {
+ typedef struct {
+ uint8_t bytes[bess::metadata::kMetadataAttrMaxSize];
+ } value_t;
+ void *mt_ptr =
+ _ptr_attr_with_offset(attr_offset(value_attr_id), pkt);
+ bess::utils::CopySmall(
+ mt_ptr,
+ reinterpret_cast(((uint8_t *)(&action)) + value_pos),
+ value_size);
+ } break;
+ }
+ }
+ }
+}
+
void ExactMatch::ProcessBatch(Context *ctx, bess::PacketBatch *batch) {
gate_idx_t default_gate;
ExactMatchKey keys[bess::PacketBatch::kMaxBurst] __ymm_aligned;
@@ -237,9 +325,23 @@ void ExactMatch::ProcessBatch(Context *ctx, bess::PacketBatch *batch) {
table_.MakeKeys(batch, buffer_fn, keys);
int cnt = batch->cnt();
- for (int i = 0; i < cnt; i++) {
- bess::Packet *pkt = batch->pkts()[i];
- EmitPacket(ctx, pkt, table_.Find(keys[i], default_gate));
+ Value default_value(default_gate);
+
+ int icnt=0;
+ for(int lcnt=0; lcnt=64) ? 64 : cnt-lcnt ;
+ ValueTuple *res[icnt];
+ uint64_t hit_mask = table_.Find(keys+lcnt, res, icnt);
+
+ for (int j = 0; j < icnt; j++) {
+ if ((hit_mask & ((uint64_t)1ULL << j)) == 0)
+ EmitPacket(ctx, batch->pkts()[j+lcnt], default_gate);
+ else {
+ setValues(batch->pkts()[j+lcnt], res[j]->action);
+ EmitPacket(ctx, batch->pkts()[j+lcnt], res[j]->gate);
+ }
+ }
}
}
@@ -250,20 +352,34 @@ std::string ExactMatch::GetDesc() const {
void ExactMatch::RuleFieldsFromPb(
const RepeatedPtrField &fields,
- bess::utils::ExactMatchRuleFields *rule) {
+ bess::utils::ExactMatchRuleFields *rule, Type type) {
for (auto i = 0; i < fields.size(); i++) {
- int field_size = table_.get_field(i).size;
+ (void)type;
+ int field_size =
+ (type == FIELD_TYPE) ? table_.get_field(i).size : get_value(i).size;
+ int attr_id = (type == FIELD_TYPE) ? table_.get_field(i).attr_id
+ : get_value(i).attr_id;
bess::pb::FieldData current = fields.Get(i);
-
if (current.encoding_case() == bess::pb::FieldData::kValueBin) {
const std::string &f_obj = fields.Get(i).value_bin();
rule->push_back(std::vector(f_obj.begin(), f_obj.end()));
} else {
rule->emplace_back();
- uint64_t rule64 = current.value_int();
+ uint64_t rule64 = 0;
+ if (attr_id < 0) {
+ if (!bess::utils::uint64_to_bin(&rule64, current.value_int(),
+ field_size, 1)) {
+ std::cerr << "idx " << i << ": not a correct" << field_size
+ << "-byte value\n";
+ return;
+ }
+ } else {
+ rule64 = current.value_int();
+ }
for (int j = 0; j < field_size; j++) {
rule->back().push_back(rule64 & 0xFFULL);
+ DLOG(INFO) << "Pushed " << std::hex << (rule64 & 0xFFULL) << " to rule.";
rule64 >>= 8;
}
}
@@ -289,7 +405,7 @@ CommandResponse ExactMatch::CommandDelete(
}
ExactMatchRuleFields rule;
- RuleFieldsFromPb(arg.fields(), &rule);
+ RuleFieldsFromPb(arg.fields(), &rule, FIELD_TYPE);
Error ret = table_.DeleteRule(rule);
if (ret.first) {
@@ -310,4 +426,8 @@ CommandResponse ExactMatch::CommandSetDefaultGate(
return CommandSuccess();
}
+void ExactMatch::DeInit() {
+ table_.DeInit();
+}
+
ADD_MODULE(ExactMatch, "em", "Multi-field classifier with an exact match table")
diff --git a/core/modules/exact_match.h b/core/modules/exact_match.h
index f7560e9700..d7e3531ed3 100644
--- a/core/modules/exact_match.h
+++ b/core/modules/exact_match.h
@@ -37,13 +37,36 @@
#include "../module.h"
#include "../pb/module_msg.pb.h"
#include "../utils/exact_match_table.h"
+#include "../utils/format.h"
-using google::protobuf::RepeatedPtrField;
+using bess::utils::Error;
using bess::utils::ExactMatchField;
using bess::utils::ExactMatchKey;
using bess::utils::ExactMatchRuleFields;
using bess::utils::ExactMatchTable;
-using bess::utils::Error;
+using google::protobuf::RepeatedPtrField;
+
+typedef enum { FIELD_TYPE = 0, VALUE_TYPE } Type;
+
+class ExactMatch;
+class Value {
+ friend class ExactMatch;
+
+ public:
+ Value(gate_idx_t g = 0) : gate(g) {}
+ Value(const Value &v) : gate(v.gate) {}
+ gate_idx_t gate;
+};
+
+class ValueTuple : public Value {
+ friend class ExactMatch;
+
+ public:
+ ValueTuple() : Value(), action() {}
+ ValueTuple(Value v) : Value(v), action() {}
+
+ ExactMatchKey action;
+};
class ExactMatch final : public Module {
public:
@@ -51,7 +74,14 @@ class ExactMatch final : public Module {
static const Commands cmds;
- ExactMatch() : Module(), default_gate_(), table_() {
+ ExactMatch()
+ : Module(),
+ default_gate_(),
+ raw_value_size_(),
+ total_value_size_(),
+ num_values_(),
+ values_(),
+ table_() {
max_allowed_workers_ = Worker::kMaxWorkers;
}
@@ -60,6 +90,9 @@ class ExactMatch final : public Module {
std::string GetDesc() const override;
CommandResponse Init(const bess::pb::ExactMatchArg &arg);
+
+ void DeInit() override;
+
CommandResponse GetInitialArg(const bess::pb::EmptyArg &arg);
CommandResponse GetRuntimeConfig(const bess::pb::EmptyArg &arg);
CommandResponse SetRuntimeConfig(const bess::pb::ExactMatchConfig &arg);
@@ -72,15 +105,153 @@ class ExactMatch final : public Module {
private:
CommandResponse AddFieldOne(const bess::pb::Field &field,
- const bess::pb::FieldData &mask, int idx);
+ const bess::pb::FieldData &mask, int idx, Type t);
void RuleFieldsFromPb(const RepeatedPtrField &fields,
- bess::utils::ExactMatchRuleFields *rule);
+ bess::utils::ExactMatchRuleFields *rule, Type type);
Error AddRule(const bess::pb::ExactMatchCommandAddArg &arg);
+ size_t num_values() const { return num_values_; }
+ ExactMatchField *getVals() { return values_; };
+ Error gather_value(const ExactMatchRuleFields &fields, ExactMatchKey *key) {
+ if (fields.size() != num_values_) {
+ return std::make_pair(
+ EINVAL, bess::utils::Format("rule should have %zu fields (has %zu)",
+ num_values_, fields.size()));
+ }
+
+ *key = {};
+
+ for (size_t i = 0; i < fields.size(); i++) {
+ int field_size = values_[i].size;
+ int field_pos = values_[i].pos;
+
+ const std::vector &f_obj = fields[i];
+
+ if (static_cast(field_size) != f_obj.size()) {
+ return std::make_pair(
+ EINVAL,
+ bess::utils::Format("rule field %zu should have size %d (has %zu)",
+ i, field_size, f_obj.size()));
+ }
+
+ memcpy(reinterpret_cast(key) + field_pos, f_obj.data(),
+ field_size);
+ }
+
+ return std::make_pair(0, bess::utils::Format("Success"));
+ }
+ // Helper for public AddField functions.
+ // DoAddValue inserts `field` as the `idx`th field for this table.
+ // If `mt_attr_name` is set, the `offset` field of `field` will be ignored and
+ // the inserted field will use the offset of `mt_attr_name` as reported by the
+ // module `m`.
+ // Returns 0 on success, non-zero errno on failure.
+ Error DoAddValue(const ExactMatchField &value,
+ const std::string &mt_attr_name, int idx,
+ Module *m = nullptr) {
+ if (idx >= MAX_FIELDS) {
+ return std::make_pair(
+ EINVAL,
+ bess::utils::Format("idx %d is not in [0,%d)", idx, MAX_FIELDS));
+ }
+ ExactMatchField *v = &values_[idx];
+ v->size = value.size;
+ if (v->size < 1 || v->size > MAX_FIELD_SIZE) {
+ return std::make_pair(
+ EINVAL, bess::utils::Format("idx %d: 'size' must be in [1,%d]", idx,
+ MAX_FIELD_SIZE));
+ }
+
+ if (mt_attr_name.length() > 0) {
+ v->attr_id = m->AddMetadataAttr(
+ mt_attr_name, v->size, bess::metadata::Attribute::AccessMode::kWrite);
+ if (v->attr_id < 0) {
+ return std::make_pair(
+ -v->attr_id,
+ bess::utils::Format("idx %d: add_metadata_attr() failed", idx));
+ }
+ } else {
+ v->attr_id = -1;
+ v->offset = value.offset;
+ if (v->offset < 0 || v->offset > 1024) {
+ return std::make_pair(
+ EINVAL, bess::utils::Format("idx %d: invalid 'offset'", idx));
+ }
+ }
+
+ int force_be = (v->attr_id < 0);
+
+ if (value.mask == 0) {
+ /* by default all bits are considered */
+ v->mask = bess::utils::SetBitsHigh(v->size * 8);
+ } else {
+ if (!bess::utils::uint64_to_bin(&v->mask, value.mask, v->size,
+ bess::utils::is_be_system() | force_be)) {
+ return std::make_pair(
+ EINVAL, bess::utils::Format("idx %d: not a valid %d-byte mask", idx,
+ v->size));
+ }
+ }
+
+ if (v->mask == 0) {
+ return std::make_pair(EINVAL,
+ bess::utils::Format("idx %d: empty mask", idx));
+ }
+
+ num_values_++;
+
+ v->pos = raw_value_size_;
+ raw_value_size_ += v->size;
+ total_value_size_ = align_ceil(raw_value_size_, sizeof(uint64_t));
+ return std::make_pair(0, bess::utils::Format("Success"));
+ }
+ // Returns the ith value.
+ const ExactMatchField &get_value(size_t i) const { return values_[i]; }
+ // Set the `idx`th field of this table to one at offset `offset` bytes into a
+ // buffer with length `size` and mask `mask`.
+ // Returns 0 on success, non-zero errno on failure.
+ Error AddValue(int offset, int size, uint64_t mask, int idx) {
+ ExactMatchField v = {
+ .mask = mask, .attr_id = 0, .offset = offset, .pos = 0, .size = size};
+ return DoAddValue(v, "", idx, nullptr);
+ }
+
+ // Set the `idx`th field of this table to one at the offset of the
+ // `mt_attr_name` metadata field as seen by module `m`, with length `size` and
+ // mask `mask`.
+ // Returns 0 on success, non-zero errno on failure.
+ Error AddValue(Module *m, const std::string &mt_attr_name, int size,
+ uint64_t mask, int idx) {
+ ExactMatchField v = {
+ .mask = mask, .attr_id = 0, .offset = 0, .pos = 0, .size = size};
+ return DoAddValue(v, mt_attr_name, idx, m);
+ }
+ Error CreateValue(ExactMatchKey &v, const ExactMatchRuleFields &values) {
+ Error err;
+
+ if (values.size() == 0) {
+ return std::make_pair(EINVAL, "rule has no values");
+ }
+
+ if ((err = gather_value(values, &v)).first != 0) {
+ return err;
+ }
+
+ return std::make_pair(0, bess::utils::Format("Success"));
+ }
+ void setValues(bess::Packet *pkt, ExactMatchKey &action);
gate_idx_t default_gate_;
bool empty_masks_; // mainly for GetInitialArg
- ExactMatchTable table_;
+ // unaligend key size, used as an accumulator for calls to AddField()
+ size_t raw_value_size_;
+
+ // aligned total key size
+ size_t total_value_size_;
+
+ size_t num_values_;
+ ExactMatchField values_[MAX_FIELDS];
+ ExactMatchTable table_;
};
#endif // BESS_MODULES_EXACTMATCH_H_
diff --git a/core/modules/ip_checksum.cc b/core/modules/ip_checksum.cc
index ecaba24d7d..47f86bae84 100644
--- a/core/modules/ip_checksum.cc
+++ b/core/modules/ip_checksum.cc
@@ -56,8 +56,8 @@ void IPChecksum::ProcessBatch(Context *ctx, bess::PacketBatch *batch) {
data = qinq + 1;
ether_type = qinq->ether_type;
if (ether_type != be16_t(Ethernet::Type::kVlan)) {
- EmitPacket(ctx, batch->pkts()[i], FORWARD_GATE);
- continue;
+ EmitPacket(ctx, batch->pkts()[i], FORWARD_GATE);
+ continue;
}
}
@@ -75,7 +75,17 @@ void IPChecksum::ProcessBatch(Context *ctx, bess::PacketBatch *batch) {
}
if (verify_) {
- EmitPacket(ctx, batch->pkts()[i], (VerifyIpv4Checksum(*ip)) ? FORWARD_GATE : FAIL_GATE);
+ if (hw_) {
+ struct rte_mbuf *m = (struct rte_mbuf *)batch->pkts()[i];
+ if (unlikely((m->ol_flags & PKT_RX_IP_CKSUM_MASK) ==
+ PKT_RX_IP_CKSUM_BAD))
+ EmitPacket(ctx, (bess::Packet *)m, FAIL_GATE);
+ else
+ EmitPacket(ctx, (bess::Packet *)m, FORWARD_GATE);
+ } else {
+ EmitPacket(ctx, batch->pkts()[i],
+ (VerifyIpv4Checksum(*ip)) ? FORWARD_GATE : FAIL_GATE);
+ }
} else {
ip->checksum = CalculateIpv4Checksum(*ip);
EmitPacket(ctx, batch->pkts()[i], FORWARD_GATE);
@@ -85,6 +95,7 @@ void IPChecksum::ProcessBatch(Context *ctx, bess::PacketBatch *batch) {
CommandResponse IPChecksum::Init(const bess::pb::IPChecksumArg &arg) {
verify_ = arg.verify();
+ hw_ = arg.hw();
return CommandSuccess();
}
diff --git a/core/modules/ip_checksum.h b/core/modules/ip_checksum.h
index f0f1d079ff..94e0d7246a 100644
--- a/core/modules/ip_checksum.h
+++ b/core/modules/ip_checksum.h
@@ -37,7 +37,9 @@
// Compute IP checksum on packet
class IPChecksum final : public Module {
public:
- IPChecksum() : Module(), verify_(false) { max_allowed_workers_ = Worker::kMaxWorkers; }
+ IPChecksum() : Module(), verify_(false) {
+ max_allowed_workers_ = Worker::kMaxWorkers;
+ }
/* Gates: (0) Default, (1) Drop */
static const gate_idx_t kNumOGates = 2;
@@ -47,6 +49,8 @@ class IPChecksum final : public Module {
private:
/* enable checksum verification */
bool verify_;
+ /* enable hardware offload */
+ bool hw_;
};
#endif // BESS_MODULES_IP_CHECKSUM_H_
diff --git a/core/modules/ip_lookup.cc b/core/modules/ip_lookup.cc
index 3c8b115717..7e976d413c 100644
--- a/core/modules/ip_lookup.cc
+++ b/core/modules/ip_lookup.cc
@@ -32,7 +32,6 @@
#include
#include
-#include
#include "../utils/bits.h"
#include "../utils/ether.h"
@@ -54,15 +53,26 @@ const Commands IPLookup::cmds = {
Command::THREAD_UNSAFE}};
CommandResponse IPLookup::Init(const bess::pb::IPLookupArg &arg) {
+#if RTE_VERSION < RTE_VERSION_NUM(19, 11, 0, 0)
struct rte_lpm_config conf = {
.max_rules = arg.max_rules() ? arg.max_rules() : 1024,
.number_tbl8s = arg.max_tbl8s() ? arg.max_tbl8s() : 128,
.flags = 0,
};
+#else
+ conf.type = RTE_FIB_DIR24_8;
+ conf.default_nh = DROP_GATE;
+ conf.max_routes = arg.max_rules() ? (int)arg.max_rules() : 1024;
+ conf.dir24_8.nh_sz = RTE_FIB_DIR24_8_4B;
+ conf.dir24_8.num_tbl8 = arg.max_tbl8s() ? arg.max_tbl8s() : 128;
+#endif
default_gate_ = DROP_GATE;
-
- lpm_ = rte_lpm_create(name().c_str(), /* socket_id = */ 0, &conf);
+#if RTE_VERSION < RTE_VERSION_NUM(19, 11, 0, 0)
+ lpm_ = rte_lpm_create(name().c_str(), /* socket_id = */ rte_socket_id(), &conf);
+#else
+ lpm_ = rte_fib_create(name().c_str(), /* socket_id = */ rte_socket_id(), &conf);
+#endif
if (!lpm_) {
return CommandFailure(rte_errno, "DPDK error: %s", rte_strerror(rte_errno));
@@ -73,7 +83,11 @@ CommandResponse IPLookup::Init(const bess::pb::IPLookupArg &arg) {
void IPLookup::DeInit() {
if (lpm_) {
+#if RTE_VERSION < RTE_VERSION_NUM(19, 11, 0, 0)
rte_lpm_free(lpm_);
+#else
+ rte_fib_free(lpm_);
+#endif
}
}
@@ -86,6 +100,7 @@ void IPLookup::ProcessBatch(Context *ctx, bess::PacketBatch *batch) {
int cnt = batch->cnt();
int i;
+#if RTE_VERSION < RTE_VERSION_NUM(19, 11, 0, 0)
#if VECTOR_OPTIMIZATION
// Convert endianness for four addresses at the same time
const __m128i bswap_mask =
@@ -148,6 +163,29 @@ void IPLookup::ProcessBatch(Context *ctx, bess::PacketBatch *batch) {
EmitPacket(ctx, batch->pkts()[i], default_gate);
}
}
+#else /* RTE_VERSION >= 19.11 */
+ Ethernet *eth;
+ Ipv4 *ip;
+ int ret;
+ uint32_t ip_list[cnt];
+ uint64_t next_hops[cnt];
+
+ for (i = 0; i < cnt; i++) {
+ eth = batch->pkts()[i]->head_data();
+ ip = (Ipv4 *)(eth + 1);
+ ip_list[i] = ip->dst.value();
+ }
+
+ ret = rte_fib_lookup_bulk(lpm_, ip_list, next_hops, cnt);
+
+ if (ret != 0)
+ RunNextModule(ctx, batch);
+ else
+ for (i = 0; i < cnt; i++) {
+ EmitPacket(ctx, batch->pkts()[i], (next_hops[i] == DROP_GATE) ? default_gate_ : next_hops[i]);
+ }
+ USED(default_gate);
+#endif
}
ParsedPrefix IPLookup::ParseIpv4Prefix(
@@ -201,7 +239,12 @@ CommandResponse IPLookup::CommandAdd(
default_gate_ = gate;
} else {
be32_t net_addr = std::get<2>(prefix);
+#if RTE_VERSION < RTE_VERSION_NUM(19, 11, 0, 0)
int ret = rte_lpm_add(lpm_, net_addr.value(), prefix_len, gate);
+#else
+ uint64_t next_hop = (uint64_t)gate;
+ int ret = rte_fib_add(lpm_, net_addr.value(), prefix_len, next_hop);
+#endif
if (ret) {
return CommandFailure(-ret, "rpm_lpm_add() failed");
}
@@ -223,7 +266,11 @@ CommandResponse IPLookup::CommandDelete(
default_gate_ = DROP_GATE;
} else {
be32_t net_addr = std::get<2>(prefix);
+#if RTE_VERSION < RTE_VERSION_NUM(19, 11, 0, 0)
int ret = rte_lpm_delete(lpm_, net_addr.value(), prefix_len);
+#else
+ int ret = rte_fib_delete(lpm_, net_addr.value(), prefix_len);
+#endif
if (ret) {
return CommandFailure(-ret, "rpm_lpm_delete() failed");
}
@@ -233,7 +280,17 @@ CommandResponse IPLookup::CommandDelete(
}
CommandResponse IPLookup::CommandClear(const bess::pb::EmptyArg &) {
+#if RTE_VERSION < RTE_VERSION_NUM(19, 11, 0, 0)
rte_lpm_delete_all(lpm_);
+#else
+ /* rte_fib_delete_all(lpm_) does not exist! */
+ rte_fib_free(lpm_);
+
+ lpm_ = rte_fib_create(name().c_str(), /* socket_id = */ 0, &conf);
+ if (!lpm_) {
+ return CommandFailure(rte_errno, "DPDK error: %s", rte_strerror(rte_errno));
+ }
+#endif
return CommandSuccess();
}
diff --git a/core/modules/ip_lookup.h b/core/modules/ip_lookup.h
index de020e7f43..a866ad085e 100644
--- a/core/modules/ip_lookup.h
+++ b/core/modules/ip_lookup.h
@@ -34,6 +34,15 @@
#include "../module.h"
#include "../pb/module_msg.pb.h"
#include "../utils/endian.h"
+#include
+#if RTE_VERSION < RTE_VERSION_NUM(19, 11, 0, 0)
+#include
+#else
+#define USED(x) (void)(x)
+extern "C" {
+#include
+}
+#endif
using bess::utils::be32_t;
using ParsedPrefix = std::tuple;
@@ -59,7 +68,12 @@ class IPLookup final : public Module {
CommandResponse CommandClear(const bess::pb::EmptyArg &arg);
private:
+#if RTE_VERSION < RTE_VERSION_NUM(19, 11, 0, 0)
struct rte_lpm *lpm_;
+#else
+ struct rte_fib *lpm_;
+ struct rte_fib_conf conf;
+#endif
gate_idx_t default_gate_;
ParsedPrefix ParseIpv4Prefix(const std::string &prefix, uint64_t prefix_len);
};
diff --git a/core/modules/l4_checksum.cc b/core/modules/l4_checksum.cc
index 3ad1f23d68..20faa85c33 100644
--- a/core/modules/l4_checksum.cc
+++ b/core/modules/l4_checksum.cc
@@ -63,27 +63,52 @@ void L4Checksum::ProcessBatch(Context *ctx, bess::PacketBatch *batch) {
Udp *udp =
reinterpret_cast(reinterpret_cast(ip) + ip_bytes);
if (verify_) {
- EmitPacket(ctx, batch->pkts()[i],
- (VerifyIpv4UdpChecksum(*ip, *udp)) ? FORWARD_GATE : FAIL_GATE);
+ if (hw_) {
+ struct rte_mbuf *m = (struct rte_mbuf *)batch->pkts()[i];
+ if (unlikely((m->ol_flags & PKT_RX_L4_CKSUM_MASK) ==
+ PKT_RX_L4_CKSUM_BAD))
+ EmitPacket(ctx, (bess::Packet *)m, FAIL_GATE);
+ else
+ EmitPacket(ctx, (bess::Packet *)m, FORWARD_GATE);
+ } else {
+ EmitPacket(
+ ctx, batch->pkts()[i],
+ (VerifyIpv4UdpChecksum(*ip, *udp)) ? FORWARD_GATE : FAIL_GATE);
+ }
} else {
- udp->checksum = CalculateIpv4UdpChecksum(*ip, *udp);
- EmitPacket(ctx, batch->pkts()[i], FORWARD_GATE);
+ udp->checksum = CalculateIpv4UdpChecksum(*ip, *udp);
+ EmitPacket(ctx, batch->pkts()[i], FORWARD_GATE);
}
} else if (ip->protocol == Ipv4::Proto::kTcp) {
size_t ip_bytes = (ip->header_length) << 2;
Tcp *tcp =
reinterpret_cast(reinterpret_cast(ip) + ip_bytes);
- if (verify_)
- EmitPacket(ctx, batch->pkts()[i],
- (VerifyIpv4TcpChecksum(*ip, *tcp)) ? FORWARD_GATE : FAIL_GATE);
- else
- tcp->checksum = CalculateIpv4TcpChecksum(*ip, *tcp);
+ if (verify_) {
+ if (hw_) {
+ struct rte_mbuf *m = (struct rte_mbuf *)batch->pkts()[i];
+ if (unlikely((m->ol_flags & PKT_RX_L4_CKSUM_MASK) ==
+ PKT_RX_L4_CKSUM_BAD))
+ EmitPacket(ctx, (bess::Packet *)m, FAIL_GATE);
+ else
+ EmitPacket(ctx, (bess::Packet *)m, FORWARD_GATE);
+ } else {
+ EmitPacket(
+ ctx, batch->pkts()[i],
+ (VerifyIpv4TcpChecksum(*ip, *tcp)) ? FORWARD_GATE : FAIL_GATE);
+ }
+ } else {
+ tcp->checksum = CalculateIpv4TcpChecksum(*ip, *tcp);
+ EmitPacket(ctx, batch->pkts()[i], FORWARD_GATE);
+ }
+ } else { /* fail-safe condition */
+ EmitPacket(ctx, batch->pkts()[i], FORWARD_GATE);
}
}
}
CommandResponse L4Checksum::Init(const bess::pb::L4ChecksumArg &arg) {
verify_ = arg.verify();
+ hw_ = arg.hw();
return CommandSuccess();
}
diff --git a/core/modules/l4_checksum.h b/core/modules/l4_checksum.h
index 27ab8f98f1..56d8ae5c20 100644
--- a/core/modules/l4_checksum.h
+++ b/core/modules/l4_checksum.h
@@ -36,7 +36,9 @@
// Compute L4 checksum on packet
class L4Checksum final : public Module {
public:
- L4Checksum() : Module(), verify_(false) { max_allowed_workers_ = Worker::kMaxWorkers; }
+ L4Checksum() : Module(), verify_(false) {
+ max_allowed_workers_ = Worker::kMaxWorkers;
+ }
/* Gates: (0) Default, (1) Drop */
static const gate_idx_t kNumOGates = 2;
@@ -46,6 +48,7 @@ class L4Checksum final : public Module {
private:
bool verify_;
+ bool hw_;
};
#endif // BESS_MODULES_L4_CHECKSUM_H_
diff --git a/core/modules/nat.h b/core/modules/nat.h
index ff29dbf555..4ec594de92 100644
--- a/core/modules/nat.h
+++ b/core/modules/nat.h
@@ -133,6 +133,7 @@ struct PortRange {
// igate/ogate 1: reverse dir
class NAT final : public Module {
public:
+ NAT() { max_allowed_workers_ = 2; }
enum Direction {
kForward = 0, // internal -> external
kReverse = 1, // external -> internal
diff --git a/core/modules/wildcard_match.cc b/core/modules/wildcard_match.cc
index 1b7bfeb6cb..e4c88d9526 100644
--- a/core/modules/wildcard_match.cc
+++ b/core/modules/wildcard_match.cc
@@ -37,15 +37,31 @@
#include "../utils/format.h"
using bess::metadata::Attribute;
+enum { FieldType = 0, ValueType };
// dst = src & mask. len must be a multiple of sizeof(uint64_t)
-static inline void mask(wm_hkey_t *dst, const wm_hkey_t &src,
+static inline void mask(wm_hkey_t &dst, const wm_hkey_t &src,
const wm_hkey_t &mask, size_t len) {
promise(len >= sizeof(uint64_t));
promise(len <= sizeof(wm_hkey_t));
for (size_t i = 0; i < len / 8; i++) {
- dst->u64_arr[i] = src.u64_arr[i] & mask.u64_arr[i];
+ dst.u64_arr[i] = src.u64_arr[i] & mask.u64_arr[i];
+ }
+}
+static inline void mask_bulk(const wm_hkey_t *src, void *dst, void **dsptr,
+ const wm_hkey_t &mask, int keys, size_t len) {
+ promise(len >= sizeof(uint64_t));
+ promise(len <= sizeof(wm_hkey_t));
+ size_t i = 0;
+ wm_hkey_t *dst1 = (wm_hkey_t *)dst;
+ wm_hkey_t **dstptr = (wm_hkey_t **)dsptr;
+
+ for (int j = 0; j < keys; j++) {
+ for (i = 0; i < len / 8; i++) {
+ dst1[j].u64_arr[i] = src[j].u64_arr[i] & mask.u64_arr[i];
+ }
+ dstptr[j] = &dst1[j];
}
}
@@ -63,17 +79,17 @@ const Commands WildcardMatch::cmds = {
{"set_runtime_config", "WildcardMatchConfig",
MODULE_CMD_FUNC(&WildcardMatch::SetRuntimeConfig), Command::THREAD_UNSAFE},
{"add", "WildcardMatchCommandAddArg",
- MODULE_CMD_FUNC(&WildcardMatch::CommandAdd), Command::THREAD_UNSAFE},
+ MODULE_CMD_FUNC(&WildcardMatch::CommandAdd), Command::THREAD_SAFE},
{"delete", "WildcardMatchCommandDeleteArg",
- MODULE_CMD_FUNC(&WildcardMatch::CommandDelete), Command::THREAD_UNSAFE},
+ MODULE_CMD_FUNC(&WildcardMatch::CommandDelete), Command::THREAD_SAFE},
{"clear", "EmptyArg", MODULE_CMD_FUNC(&WildcardMatch::CommandClear),
- Command::THREAD_UNSAFE},
+ Command::THREAD_SAFE},
{"set_default_gate", "WildcardMatchCommandSetDefaultGateArg",
MODULE_CMD_FUNC(&WildcardMatch::CommandSetDefaultGate),
Command::THREAD_SAFE}};
CommandResponse WildcardMatch::AddFieldOne(const bess::pb::Field &field,
- struct WmField *f) {
+ struct WmField *f, uint8_t type) {
f->size = field.num_bytes();
if (f->size < 1 || f->size > MAX_FIELD_SIZE) {
@@ -88,7 +104,10 @@ CommandResponse WildcardMatch::AddFieldOne(const bess::pb::Field &field,
}
} else if (field.position_case() == bess::pb::Field::kAttrName) {
const char *attr = field.attr_name().c_str();
- f->attr_id = AddMetadataAttr(attr, f->size, Attribute::AccessMode::kRead);
+ f->attr_id =
+ (type == FieldType)
+ ? AddMetadataAttr(attr, f->size, Attribute::AccessMode::kRead)
+ : AddMetadataAttr(attr, f->size, Attribute::AccessMode::kWrite);
if (f->attr_id < 0) {
return CommandFailure(-f->attr_id, "add_metadata_attr() failed");
}
@@ -119,57 +138,221 @@ CommandResponse WildcardMatch::Init(const bess::pb::WildcardMatchArg &arg) {
f.pos = size_acc;
- err = AddFieldOne(field, &f);
+ err = AddFieldOne(field, &f, FieldType);
if (err.error().code() != 0) {
return err;
}
size_acc += f.size;
}
-
default_gate_ = DROP_GATE;
total_key_size_ = align_ceil(size_acc, sizeof(uint64_t));
+ entries_ = arg.entries();
+ // reset size_acc
+ size_acc = 0;
+ for (int i = 0; i < arg.values_size(); i++) {
+ const auto &value = arg.values(i);
+ CommandResponse err;
+ values_.emplace_back();
+ struct WmField &v = values_.back();
+
+ v.pos = size_acc;
+
+ err = AddFieldOne(value, &v, ValueType);
+ if (err.error().code() != 0) {
+ return err;
+ }
+
+ size_acc += v.size;
+ }
+
+ total_value_size_ = align_ceil(size_acc, sizeof(uint64_t));
return CommandSuccess();
}
inline gate_idx_t WildcardMatch::LookupEntry(const wm_hkey_t &key,
- gate_idx_t def_gate) {
+ gate_idx_t def_gate,
+ bess::Packet *pkt) {
struct WmData result = {
- .priority = INT_MIN, .ogate = def_gate,
- };
-
+ .priority = INT_MIN, .ogate = def_gate, .keyv = {{0}}};
for (auto &tuple : tuples_) {
+ if (tuple.occupied == 0)
+ continue;
const auto &ht = tuple.ht;
wm_hkey_t key_masked;
+ mask(key_masked, key, tuple.mask, total_key_size_);
+ WmData *entry = nullptr;
+ ht->find_dpdk(&key_masked, ((void **)&entry));
+ if (entry && entry->priority >= result.priority) {
+ result = *entry;
+ }
+ }
- mask(&key_masked, key, tuple.mask, total_key_size_);
-
- const auto *entry =
- ht.Find(key_masked, wm_hash(total_key_size_), wm_eq(total_key_size_));
+ /* if lookup was successful, then set values (if possible) */
+ if (result.ogate != default_gate_) {
+ size_t num_values_ = values_.size();
+ for (size_t i = 0; i < num_values_; i++) {
+ int value_size = values_[i].size;
+ int value_pos = values_[i].pos;
+ int value_off = values_[i].offset;
+ int value_attr_id = values_[i].attr_id;
+ uint8_t *data = pkt->head_data() + value_off;
+
+ DLOG(INFO) << "off: " << (int)value_off << ", sz: " << value_size;
+
+ if (value_attr_id < 0) { /* if it is offset-based */
+ memcpy(data, reinterpret_cast(&result.keyv) + value_pos,
+ value_size);
+ } else { /* if it is attribute-based */
+ typedef struct {
+ uint8_t bytes[bess::metadata::kMetadataAttrMaxSize];
+ } value_t;
+ uint8_t *buf = (uint8_t *)&result.keyv + value_pos;
+
+ DLOG(INFO) << "Setting value " << std::hex
+ << *(reinterpret_cast(buf))
+ << " for attr_id: " << value_attr_id
+ << " of size: " << value_size
+ << " at value_pos: " << value_pos;
+
+ switch (value_size) {
+ case 1:
+ set_attr(this, value_attr_id, pkt, *((uint8_t *)buf));
+ break;
+ case 2:
+ set_attr(this, value_attr_id, pkt,
+ *((uint16_t *)((uint8_t *)buf)));
+ break;
+ case 4:
+ set_attr(this, value_attr_id, pkt,
+ *((uint32_t *)((uint8_t *)buf)));
+ break;
+ case 8:
+ set_attr(this, value_attr_id, pkt,
+ *((uint64_t *)((uint8_t *)buf)));
+ break;
+ default: {
+ void *mt_ptr =
+ _ptr_attr_with_offset(attr_offset(value_attr_id), pkt);
+ bess::utils::CopySmall(mt_ptr, buf, value_size);
+ } break;
+ }
+ }
+ }
+ }
+ return result.ogate;
+}
- if (entry && entry->second.priority >= result.priority) {
- result = entry->second;
+inline bool WildcardMatch::LookupBulkEntry(wm_hkey_t *key, gate_idx_t def_gate,
+ int packeti, gate_idx_t *Outgate,
+ int cnt, bess::PacketBatch *batch) {
+ bess::Packet *pkt = nullptr;
+ struct WmData *result[cnt];
+ uint64_t prev_hitmask = 0;
+ uint64_t hitmask = 0;
+ wm_hkey_t key_masked[cnt];
+ WmData *entry[cnt];
+ wm_hkey_t **key_ptr[cnt];
+
+ for (auto tuple = tuples_.begin(); tuple != tuples_.end(); ++tuple) {
+ if (tuple->occupied == 0)
+ continue;
+ const auto &ht = tuple->ht;
+ mask_bulk(key, key_masked, (void **)key_ptr, tuple->mask, cnt,
+ total_key_size_);
+ int num = ht->lookup_bulk_data((const void **)key_ptr, cnt, &hitmask,
+ (void **)entry);
+ if (num == 0)
+ continue;
+
+ for (int init = 0; (init < cnt) && (num); init++) {
+ if ((hitmask & ((uint64_t)1 << init))) {
+ if ((prev_hitmask & ((uint64_t)1 << init)) == 0)
+ result[init] = entry[init];
+ else if ((prev_hitmask & ((uint64_t)1 << init)) &&
+ (entry[init]->priority >= result[init]->priority)) {
+ result[init] = entry[init];
+ }
+
+ num--;
+ }
}
+ prev_hitmask = prev_hitmask | hitmask;
}
- return result.ogate;
+ for (int init = 0; init < cnt; init++) {
+ /* if lookup was successful, then set values (if possible) */
+ if (prev_hitmask && (prev_hitmask & ((uint64_t)1 << init))) {
+ pkt = batch->pkts()[packeti + init];
+ size_t num_values_ = values_.size();
+ for (size_t i = 0; i < num_values_; i++) {
+ int value_size = values_[i].size;
+ int value_pos = values_[i].pos;
+ int value_off = values_[i].offset;
+ int value_attr_id = values_[i].attr_id;
+ uint8_t *data = pkt->head_data() + value_off;
+
+ DLOG(INFO) << "off: " << (int)value_off << ", sz: " << value_size
+ << std::endl;
+ if (value_attr_id < 0) { /* if it is offset-based */
+ memcpy(data,
+ reinterpret_cast(&result[init]->keyv) + value_pos,
+ value_size);
+ } else { /* if it is attribute-based */
+ typedef struct {
+ uint8_t bytes[bess::metadata::kMetadataAttrMaxSize];
+ } value_t;
+ uint8_t *buf = (uint8_t *)&result[init]->keyv + value_pos;
+
+ DLOG(INFO) << "Setting value " << std::hex
+ << *(reinterpret_cast(buf))
+ << " for attr_id: " << value_attr_id
+ << " of size: " << value_size
+ << " at value_pos: " << value_pos << std::endl;
+
+ switch (value_size) {
+ case 1:
+ set_attr(this, value_attr_id, pkt, *((uint8_t *)buf));
+ break;
+ case 2:
+ set_attr(this, value_attr_id, pkt,
+ *((uint16_t *)((uint8_t *)buf)));
+ break;
+ case 4:
+ set_attr(this, value_attr_id, pkt,
+ *((uint32_t *)((uint8_t *)buf)));
+ break;
+ case 8:
+ set_attr(this, value_attr_id, pkt,
+ *((uint64_t *)((uint8_t *)buf)));
+ break;
+ default: {
+ void *mt_ptr = _ptr_attr_with_offset(
+ attr_offset(value_attr_id), pkt);
+ bess::utils::CopySmall(mt_ptr, buf, value_size);
+ } break;
+ }
+ }
+ }
+ Outgate[init] = result[init]->ogate;
+ } else
+ Outgate[init] = def_gate;
+ }
+ return 1;
}
void WildcardMatch::ProcessBatch(Context *ctx, bess::PacketBatch *batch) {
gate_idx_t default_gate;
-
wm_hkey_t keys[bess::PacketBatch::kMaxBurst] __ymm_aligned;
-
int cnt = batch->cnt();
+ gate_idx_t Outgate[cnt];
// Initialize the padding with zero
for (int i = 0; i < cnt; i++) {
keys[i].u64_arr[(total_key_size_ - 1) / 8] = 0;
}
-
default_gate = ACCESS_ONCE(default_gate_);
-
for (const auto &field : fields_) {
int offset;
int pos = field.pos;
@@ -196,9 +379,25 @@ void WildcardMatch::ProcessBatch(Context *ctx, bess::PacketBatch *batch) {
}
}
- for (int i = 0; i < cnt; i++) {
- bess::Packet *pkt = batch->pkts()[i];
- EmitPacket(ctx, pkt, LookupEntry(keys[i], default_gate));
+ if(cnt>64)
+ {
+ int icnt=0;
+ for(int lcnt=0; lcnt=64) ? 64 : cnt-lcnt ;
+ LookupBulkEntry(&keys[lcnt], default_gate, lcnt, Outgate, icnt, batch);
+ for (int j = 0; j < icnt; j++)
+ {
+ EmitPacket(ctx, batch->pkts()[j+lcnt], Outgate[j]);
+ }
+ }
+ }
+ else
+ {
+ LookupBulkEntry(keys, default_gate, 0, Outgate, cnt, batch);
+ for (int j = 0; j < cnt; j++) {
+ EmitPacket(ctx, batch->pkts()[j], Outgate[j]);
+ }
}
}
@@ -206,7 +405,9 @@ std::string WildcardMatch::GetDesc() const {
int num_rules = 0;
for (const auto &tuple : tuples_) {
- num_rules += tuple.ht.Count();
+ if (tuple.occupied == 0)
+ continue;
+ num_rules += tuple.ht->Count();
}
return bess::utils::Format("%zu fields, %d rules", fields_.size(), num_rules);
@@ -275,55 +476,99 @@ CommandResponse WildcardMatch::ExtractKeyMask(const T &arg, wm_hkey_t *key,
return CommandSuccess();
}
-int WildcardMatch::FindTuple(wm_hkey_t *mask) {
- int i = 0;
+template
+CommandResponse WildcardMatch::ExtractValue(const T &arg, wm_hkey_t *keyv) {
+ if ((size_t)arg.valuesv_size() != values_.size()) {
+ return CommandFailure(EINVAL, "must specify %zu values", values_.size());
+ }
- for (const auto &tuple : tuples_) {
- if (memcmp(&tuple.mask, mask, total_key_size_) == 0) {
+ memset(keyv, 0, sizeof(*keyv));
+
+ for (size_t i = 0; i < values_.size(); i++) {
+ int value_size = values_[i].size;
+ int value_pos = values_[i].pos;
+
+ uint64_t v = 0;
+
+ bess::pb::FieldData valuedata = arg.valuesv(i);
+ if (valuedata.encoding_case() == bess::pb::FieldData::kValueInt) {
+ if (!bess::utils::uint64_to_bin(&v, valuedata.value_int(), value_size,
+ false)) {
+ return CommandFailure(EINVAL, "idx %zu: not a correct %d-byte value", i,
+ value_size);
+ }
+ } else if (valuedata.encoding_case() == bess::pb::FieldData::kValueBin) {
+ bess::utils::Copy(reinterpret_cast(&v),
+ valuedata.value_bin().c_str(),
+ valuedata.value_bin().size());
+ }
+
+ // Use memcpy, not utils::Copy, to workaround the false positive warning
+ // in g++-8
+ memcpy(reinterpret_cast(keyv) + value_pos, &v, value_size);
+ }
+
+ return CommandSuccess();
+}
+
+int WildcardMatch::FindTuple(wm_hkey_t *mask) {
+ for (auto i = 0; i < MAX_TUPLES; i++) {
+ if ((tuples_[i].occupied) &&
+ (memcmp(&tuples_[i].mask, mask, total_key_size_) == 0)) {
return i;
}
- i++;
}
return -ENOENT;
}
int WildcardMatch::AddTuple(wm_hkey_t *mask) {
- if (tuples_.size() >= MAX_TUPLES) {
- return -ENOSPC;
+ CuckooMap *temp = nullptr;
+ for (int i = 0; i < MAX_TUPLES; i++) {
+ if (tuples_[i].occupied == 0) {
+ bess::utils::Copy(&tuples_[i].mask, mask, sizeof(*mask));
+ tuples_[i].params.key_len = total_key_size_;
+ if (entries_) {
+ tuples_[i].params.entries = entries_;
+ }
+ temp = new CuckooMap(
+ 0, 0, &tuples_[i].params);
+ if (temp == nullptr)
+ return -ENOSPC;
+ if (temp->hash == 0) {
+ delete temp;
+ return -ENOSPC;
+ }
+ void *temp1 = tuples_[i].ht;
+ tuples_[i].ht = temp;
+ if (temp1)
+ delete (
+ static_cast *>(
+ temp1));
+ tuples_[i].occupied = 1;
+ return i;
+ }
}
-
- tuples_.emplace_back();
- struct WmTuple &tuple = tuples_.back();
- bess::utils::Copy(&tuple.mask, mask, sizeof(*mask));
-
- return int(tuples_.size() - 1);
+ return -ENOSPC;
}
-int WildcardMatch::DelEntry(int idx, wm_hkey_t *key) {
- struct WmTuple &tuple = tuples_[idx];
- int ret =
- tuple.ht.Remove(*key, wm_hash(total_key_size_), wm_eq(total_key_size_));
- if (ret) {
- return ret;
+bool WildcardMatch::DelEntry(int idx, wm_hkey_t *key) {
+ int ret = tuples_[idx].ht->Remove(*key, wm_hash(total_key_size_),
+ wm_eq(total_key_size_));
+ if (ret >= 0) {
+ return true;
}
-
- if (tuple.ht.Count() == 0) {
- tuples_.erase(tuples_.begin() + idx);
+ if (tuples_[idx].ht->Count() == 0) {
}
-
- return 0;
+ return false;
}
CommandResponse WildcardMatch::CommandAdd(
const bess::pb::WildcardMatchCommandAddArg &arg) {
gate_idx_t gate = arg.gate();
int priority = arg.priority();
-
wm_hkey_t key = {{0}};
wm_hkey_t mask = {{0}};
-
struct WmData data;
-
CommandResponse err = ExtractKeyMask(arg, &key, &mask);
if (err.error().code() != 0) {
return err;
@@ -333,9 +578,13 @@ CommandResponse WildcardMatch::CommandAdd(
return CommandFailure(EINVAL, "Invalid gate: %hu", gate);
}
+ err = ExtractValue(arg, &(data.keyv));
+ if (err.error().code() != 0) {
+ return err;
+ }
+
data.priority = priority;
data.ogate = gate;
-
int idx = FindTuple(&mask);
if (idx < 0) {
idx = AddTuple(&mask);
@@ -343,13 +592,10 @@ CommandResponse WildcardMatch::CommandAdd(
return CommandFailure(-idx, "failed to add a new wildcard pattern");
}
}
-
- auto *ret = tuples_[idx].ht.Insert(key, data, wm_hash(total_key_size_),
- wm_eq(total_key_size_));
- if (ret == nullptr) {
+ struct WmData *data_t = new WmData(data);
+ int ret = tuples_[idx].ht->insert_dpdk(&key, data_t);
+ if (ret < 0)
return CommandFailure(EINVAL, "failed to add a rule");
- }
-
return CommandSuccess();
}
@@ -383,7 +629,10 @@ CommandResponse WildcardMatch::CommandClear(const bess::pb::EmptyArg &) {
void WildcardMatch::Clear() {
for (auto &tuple : tuples_) {
- tuple.ht.Clear();
+ if (tuple.occupied) {
+ tuple.occupied = 0;
+ tuple.ht->Clear();
+ }
}
}
@@ -405,17 +654,26 @@ CommandResponse WildcardMatch::GetInitialArg(const bess::pb::EmptyArg &) {
// Retrieves a WildcardMatchConfig that would restore this module's
// runtime configuration.
CommandResponse WildcardMatch::GetRuntimeConfig(const bess::pb::EmptyArg &) {
+ std::pair entry;
bess::pb::WildcardMatchConfig resp;
using rule_t = bess::pb::WildcardMatchCommandAddArg;
-
+ const wm_hkey_t *key = 0;
+ WmData *data;
+ uint32_t *next = 0;
resp.set_default_gate(default_gate_);
// Each tuple provides a single mask, which may have many data-matches.
for (auto &tuple : tuples_) {
+ if (tuple.occupied == 0)
+ continue;
wm_hkey_t mask = tuple.mask;
// Each entry in the hash table has priority, ogate, and the data
// (one datum per field, under the mask for this field).
- for (auto &entry : tuple.ht) {
+ // using rte method
+ while ((tuple.ht->Iterate((const void **)&key, (void **)&data, next)) >=
+ (int)0) {
+ entry.first = *key;
+ entry.second = *data;
// Create the rule instance
rule_t *rule = resp.add_rules();
rule->set_priority(entry.second.priority);
@@ -480,5 +738,14 @@ CommandResponse WildcardMatch::SetRuntimeConfig(
return CommandSuccess();
}
+void WildcardMatch::DeInit() {
+ for (auto &tuple : tuples_) {
+ if (!tuple.ht)
+ continue;
+ tuple.ht->DeInit();
+ tuple.ht = NULL;
+ }
+}
+
ADD_MODULE(WildcardMatch, "wm",
"Multi-field classifier with a wildcard match table")
diff --git a/core/modules/wildcard_match.h b/core/modules/wildcard_match.h
index 5a8d9e8142..6e7703dcfa 100644
--- a/core/modules/wildcard_match.h
+++ b/core/modules/wildcard_match.h
@@ -39,12 +39,13 @@
#include "../pb/module_msg.pb.h"
#include "../utils/cuckoo_map.h"
-using bess::utils::HashResult;
using bess::utils::CuckooMap;
+using bess::utils::HashResult;
-#define MAX_TUPLES 8
+#define MAX_TUPLES 16
#define MAX_FIELDS 8
#define MAX_FIELD_SIZE 8
+#define BULK_SIZE 32
static_assert(MAX_FIELD_SIZE <= sizeof(uint64_t),
"field cannot be larger than 8 bytes");
@@ -54,11 +55,6 @@ static_assert(MAX_FIELD_SIZE <= sizeof(uint64_t),
#error this code assumes little endian architecture (x86)
#endif
-struct WmData {
- int priority;
- gate_idx_t ogate;
-};
-
struct WmField {
int attr_id; /* -1 for offset-based fields */
@@ -75,6 +71,12 @@ struct wm_hkey_t {
uint64_t u64_arr[MAX_FIELDS];
};
+struct WmData {
+ int priority;
+ gate_idx_t ogate;
+ wm_hkey_t keyv;
+};
+
class wm_eq {
public:
explicit wm_eq(size_t len) : len_(len) {}
@@ -127,6 +129,12 @@ class wm_hash {
private:
size_t len_;
};
+struct rte_hash_parameters dpdk_params1 {
+ .name = "test2", .entries = 1 << 15, .reserved = 0,
+ .key_len = sizeof(wm_hkey_t), .hash_func = rte_hash_crc,
+ .hash_func_init_val = 0, .socket_id = (int)rte_socket_id(),
+ .extra_flag = RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY
+};
class WildcardMatch final : public Module {
public:
@@ -135,12 +143,21 @@ class WildcardMatch final : public Module {
static const Commands cmds;
WildcardMatch()
- : Module(), default_gate_(), total_key_size_(), fields_(), tuples_() {
+ : Module(),
+ default_gate_(),
+ total_key_size_(),
+ total_value_size_(),
+ fields_(),
+ values_(),
+ tuples_() {
max_allowed_workers_ = Worker::kMaxWorkers;
+ { tuples_.resize(MAX_TUPLES); }
}
CommandResponse Init(const bess::pb::WildcardMatchArg &arg);
+ void DeInit() override;
+
void ProcessBatch(Context *ctx, bess::PacketBatch *batch) override;
std::string GetDesc() const override;
@@ -157,30 +174,49 @@ class WildcardMatch final : public Module {
private:
struct WmTuple {
- CuckooMap ht;
+ bool occupied;
+ CuckooMap *ht;
wm_hkey_t mask;
+ struct rte_hash_parameters params;
+ std::string hash_name;
+ WmTuple() : occupied(0), ht(0) {
+ params = dpdk_params1;
+ std::ostringstream address;
+ address << this;
+ hash_name = "Wild" + address.str();
+ params.name = hash_name.c_str();
+ }
};
- gate_idx_t LookupEntry(const wm_hkey_t &key, gate_idx_t def_gate);
+ gate_idx_t LookupEntry(const wm_hkey_t &key, gate_idx_t def_gate,
+ bess::Packet *pkt);
- CommandResponse AddFieldOne(const bess::pb::Field &field, struct WmField *f);
+ bool LookupBulkEntry(wm_hkey_t *key, gate_idx_t def_gate, int i,
+ gate_idx_t *Outgate, int cnt, bess::PacketBatch *batch);
+
+ CommandResponse AddFieldOne(const bess::pb::Field &field, struct WmField *f,
+ uint8_t type);
template
CommandResponse ExtractKeyMask(const T &arg, wm_hkey_t *key, wm_hkey_t *mask);
+ template
+ CommandResponse ExtractValue(const T &arg, wm_hkey_t *keyv);
int FindTuple(wm_hkey_t *mask);
int AddTuple(wm_hkey_t *mask);
- int DelEntry(int idx, wm_hkey_t *key);
-
+ bool DelEntry(int idx, wm_hkey_t *key);
void Clear();
-
gate_idx_t default_gate_;
- size_t total_key_size_; /* a multiple of sizeof(uint64_t) */
+ size_t total_key_size_; /* a multiple of sizeof(uint64_t) */
+ size_t total_value_size_; /* a multiple of sizeof(uint64_t) */
+ size_t entries_; /* a power of 2 */
// TODO(melvinw): this can be refactored to use ExactMatchTable
std::vector fields_;
- std::vector tuples_;
+ std::vector values_;
+ std::vector tuples_; //[MAX_TUPLES];
+ std::vector data_;
};
#endif // BESS_MODULES_WILDCARDMATCH_H_
diff --git a/core/utils/cuckoo_map.h b/core/utils/cuckoo_map.h
index c855b93dea..6213ce7d92 100644
--- a/core/utils/cuckoo_map.h
+++ b/core/utils/cuckoo_map.h
@@ -50,6 +50,8 @@
#include "../debug.h"
#include "common.h"
+#include
+#include
namespace bess {
namespace utils {
@@ -74,7 +76,13 @@ typedef uint32_t EntryIndex;
template ,
typename E = std::equal_to>
class CuckooMap {
+ private:
+ bool IsDpdk = false;
+ uint32_t key_len = 0;
+ rte_hash_parameters rt;
+
public:
+ struct rte_hash* hash = nullptr;
typedef std::pair Entry;
class iterator {
@@ -149,20 +157,29 @@ class CuckooMap {
};
CuckooMap(size_t reserve_buckets = kInitNumBucket,
- size_t reserve_entries = kInitNumEntries)
+ size_t reserve_entries = kInitNumEntries, void* dpdk_params = 0)
: bucket_mask_(reserve_buckets - 1),
num_entries_(0),
buckets_(reserve_buckets),
entries_(reserve_entries),
free_entry_indices_() {
- // the number of buckets must be a power of 2
- CHECK_EQ(align_ceil_pow2(reserve_buckets), reserve_buckets);
+ if (dpdk_params) {
+ if (hash == NULL) {
+ rt = *((rte_hash_parameters*)dpdk_params);
+ hash = rte_hash_create(&rt);
+ if (hash == NULL)
+ return;
+ }
+ IsDpdk = true;
+ } else {
+ // the number of buckets must be a power of 2
+ CHECK_EQ(align_ceil_pow2(reserve_buckets), reserve_buckets);
- for (int i = reserve_entries - 1; i >= 0; --i) {
- free_entry_indices_.push(i);
+ for (int i = reserve_entries - 1; i >= 0; --i) {
+ free_entry_indices_.push(i);
+ }
}
}
-
// Not allowing copying for now
CuckooMap(CuckooMap&) = delete;
CuckooMap& operator=(CuckooMap&) = delete;
@@ -220,6 +237,50 @@ class CuckooMap {
return DoEmplace(key, hasher, eq, std::move(value));
}
+ int insert_dpdk(const void* key, void* data = 0, hash_sig_t sig = 0) {
+ if (IsDpdk) {
+ if (data && !sig)
+ return rte_hash_add_key_data(hash, key, data);
+ if (data && sig)
+ return rte_hash_add_key_with_hash_data(hash, key, sig, data);
+ if (!data && !sig)
+ return rte_hash_add_key(hash, key);
+ }
+ return -1;
+ }
+
+ int find_dpdk(const void* key, void** data = 0,
+ hash_sig_t sig = 0)
+ {
+ if (IsDpdk) {
+ if (data && !sig)
+ return rte_hash_lookup_data(hash, key, data);
+ if (data && sig)
+ return rte_hash_lookup_with_hash_data(hash, key, sig, data);
+ if (!data && !sig)
+ return rte_hash_lookup(hash, key);
+ if (!data && sig)
+ return rte_hash_lookup_with_hash(hash, key, sig);
+ }
+ return -1;
+ }
+
+ int find_dpdk(const void* key, void** data = 0,
+ hash_sig_t sig = 0) const
+ {
+ if (IsDpdk) {
+ if (data && !sig)
+ return rte_hash_lookup_data(hash, key, data);
+ if (data && sig)
+ return rte_hash_lookup_with_hash_data(hash, key, sig, data);
+ if (!data && !sig)
+ return rte_hash_lookup(hash, key);
+ if (!data && sig)
+ return rte_hash_lookup_with_hash(hash, key, sig);
+ }
+ return -1;
+ }
+
// Emplace/update-in-place a key value pair
// On success returns a pointer to the inserted entry, nullptr otherwise.
// NOTE: when Emplace() returns nullptr, the constructor of `V` may not be
@@ -255,6 +316,13 @@ class CuckooMap {
// Remove the stored entry by the key
// Return false if not exist.
bool Remove(const K& key, const H& hasher = H(), const E& eq = E()) {
+ if (IsDpdk) {
+ int ret = rte_hash_del_key(hash, &key);
+ if (ret < 0)
+ return false;
+ else
+ return true;
+ }
HashResult pri = Hash(key, hasher);
if (RemoveFromBucket(pri, pri & bucket_mask_, key, eq)) {
return true;
@@ -267,6 +335,12 @@ class CuckooMap {
}
void Clear() {
+ if (IsDpdk) {
+ if (hash) {
+ rte_hash_reset(hash);
+ }
+ return;
+ }
buckets_.clear();
entries_.clear();
@@ -286,7 +360,36 @@ class CuckooMap {
}
// Return the number of stored entries
- size_t Count() const { return num_entries_; }
+ size_t Count() const {
+ if (IsDpdk)
+ return rte_hash_count(hash);
+ else
+ return num_entries_;
+ }
+
+ void DeInit() {
+ if (IsDpdk) {
+ if (hash) {
+ rte_hash_free(hash);
+ hash = nullptr;
+ }
+ return;
+ }
+ }
+
+ // bulk data look up bess func
+ int32_t lookup_bulk_data(const void** keys, uint32_t num_keys,
+ uint64_t* hit_mask, void* data[]) {
+ if (IsDpdk)
+ return rte_hash_lookup_bulk_data(hash, keys, num_keys, hit_mask, data);
+ return -1;
+ }
+ // iterate for dpdk hash
+ int32_t Iterate(const void** key, void** data, uint32_t* next) {
+ if (IsDpdk)
+ return rte_hash_iterate(hash, key, data, next);
+ return -1;
+ }
protected:
// Tunable macros
diff --git a/core/utils/exact_match_table.h b/core/utils/exact_match_table.h
index 7c0cfda426..f5238dd5fb 100644
--- a/core/utils/exact_match_table.h
+++ b/core/utils/exact_match_table.h
@@ -155,15 +155,17 @@ typedef std::vector> ExactMatchRuleFields;
template
class ExactMatchTable {
public:
+ struct rte_hash_parameters dpdk_params {
+ .name = "test1", .entries = 1 << 15, .reserved = 0,
+ .key_len = sizeof(ExactMatchKey), .hash_func = rte_hash_crc,
+ .hash_func_init_val = 0, .socket_id = (int)rte_socket_id(),
+ .extra_flag = RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY
+ };
+
using EmTable =
CuckooMap;
-
ExactMatchTable()
- : raw_key_size_(),
- total_key_size_(),
- num_fields_(),
- fields_(),
- table_() {}
+ : raw_key_size_(), total_key_size_(), num_fields_(), fields_() {}
// Add a new rule.
//
@@ -183,10 +185,9 @@ class ExactMatchTable {
if ((err = gather_key(fields, &key)).first != 0) {
return err;
}
-
- table_.Insert(key, val, ExactMatchKeyHash(total_key_size_),
- ExactMatchKeyEq(total_key_size_));
-
+ const void *Key_t = (const void *)&key;
+ T *val_t = new T(val);
+ table_->insert_dpdk(Key_t, val_t);
return MakeError(0);
}
@@ -207,8 +208,8 @@ class ExactMatchTable {
return err;
}
- bool ret = table_.Remove(key, ExactMatchKeyHash(total_key_size_),
- ExactMatchKeyEq(total_key_size_));
+ bool ret = table_->Remove(key, ExactMatchKeyHash(total_key_size_),
+ ExactMatchKeyEq(total_key_size_));
if (!ret) {
return MakeError(ENOENT, "rule doesn't exist");
}
@@ -217,9 +218,11 @@ class ExactMatchTable {
}
// Remove all rules from the table.
- void ClearRules() { table_.Clear(); }
+ void ClearRules() { table_->Clear(); }
- size_t Size() const { return table_.Count(); }
+ void DeInit() { table_->DeInit(); }
+
+ size_t Size() const { return table_->Count(); }
// Extract an ExactMatchKey from `buf` based on the fields that have been
// added to this table.
@@ -272,23 +275,27 @@ class ExactMatchTable {
// Returns the value if `key` matches a rule, otherwise `default_value`.
T Find(const ExactMatchKey &key, const T &default_value) const {
const auto &table = table_;
- const auto *entry = table.Find(key, ExactMatchKeyHash(total_key_size_),
- ExactMatchKeyEq(total_key_size_));
- return entry ? entry->second : default_value;
+ void *data = nullptr;
+ table->find_dpdk(&key, &data);
+ if (data) {
+ T data_t = *((T *)data);
+ return data_t;
+ } else
+ return default_value;
}
// Find entries for `n` `keys` in the table and store their values in in
// `vals`. Keys without entries will have their corresponding entires in
// `vals` set to `default_value`.
- void Find(const ExactMatchKey *keys, T *vals, size_t n,
- T default_value) const {
+ uint64_t Find(ExactMatchKey *keys, T **vals, int n) {
const auto &table = table_;
- for (size_t i = 0; i < n; i++) {
- const auto *entry =
- table.Find(keys[i], ExactMatchKeyHash(total_key_size_),
- ExactMatchKeyEq(total_key_size_));
- vals[i] = entry ? entry->second : default_value;
- }
+ uint64_t hit_mask = 0;
+ ExactMatchKey *key_ptr[n];
+ for (int h = 0; h < n; h++)
+ key_ptr[h] = &keys[h];
+ table->lookup_bulk_data((const void **)&key_ptr, n, &hit_mask,
+ (void **)vals);
+ return hit_mask;
}
uint32_t total_key_size() const { return total_key_size_; }
@@ -318,9 +325,23 @@ class ExactMatchTable {
// Returns the ith field.
const ExactMatchField &get_field(size_t i) const { return fields_[i]; }
- typename EmTable::iterator begin() { return table_.begin(); }
+ typename EmTable::iterator begin() { return table_->begin(); }
+
+ typename EmTable::iterator end() { return table_->end(); }
- typename EmTable::iterator end() { return table_.end(); }
+ void Init(uint32_t entries) {
+ std::ostringstream address;
+ address << &table_;
+ std::string name = "Exactmatch" + address.str();
+ dpdk_params.name = name.c_str();
+ dpdk_params.key_len = total_key_size();
+ if (entries) {
+ dpdk_params.entries = entries;
+ }
+ table_.reset(
+ new CuckooMap(
+ 0, 0, &dpdk_params));
+ }
private:
Error MakeError(int code, const std::string &msg = "") {
@@ -438,23 +459,21 @@ class ExactMatchTable {
f->pos = raw_key_size_;
raw_key_size_ += f->size;
total_key_size_ = align_ceil(raw_key_size_, sizeof(uint64_t));
-
return MakeError(0);
}
// unaligend key size, used as an accumulator for calls to AddField()
size_t raw_key_size_;
-
// aligned total key size
size_t total_key_size_;
-
size_t num_fields_;
ExactMatchField fields_[MAX_FIELDS];
-
- EmTable table_;
+ std::unique_ptr<
+ CuckooMap>
+ table_;
};
-} // namespace bess
} // namespace utils
+} // namespace bess
#endif // BESS_UTILS_EXACT_MATCH_TABLE_H_
diff --git a/protobuf/bess_msg.proto b/protobuf/bess_msg.proto
index 9e1b79a337..9b30de3447 100644
--- a/protobuf/bess_msg.proto
+++ b/protobuf/bess_msg.proto
@@ -42,6 +42,8 @@ import "error.proto";
package bess.pb;
+option go_package = "github.com/omec-project/upf-epc/pfcpiface/bess_pb";
+
message EmptyRequest {
}
diff --git a/protobuf/error.proto b/protobuf/error.proto
index a018ee8e06..04b352c543 100644
--- a/protobuf/error.proto
+++ b/protobuf/error.proto
@@ -31,6 +31,8 @@ syntax = "proto3";
package bess.pb;
+option go_package = "github.com/omec-project/upf-epc/pfcpiface/bess_pb";
+
message Error {
int32 code = 1; // 0 for success, errno (>0) for failure
string errmsg = 2;
diff --git a/protobuf/module_msg.proto b/protobuf/module_msg.proto
index e00a463a35..3cbc328790 100644
--- a/protobuf/module_msg.proto
+++ b/protobuf/module_msg.proto
@@ -31,6 +31,9 @@
syntax = "proto3";
package bess.pb;
+
+option go_package = "github.com/omec-project/upf-epc/pfcpiface/bess_pb";
+
import "util_msg.proto";
// Module-specific messages.
@@ -69,6 +72,7 @@ message BPFCommandClearArg {
message ExactMatchCommandAddArg {
uint64 gate = 1; /// The gate to forward out packets that mach this rule.
repeated FieldData fields = 2; /// The exact match values to check for
+ repeated FieldData values = 3; /// The exact match values to check for
}
/**
@@ -377,6 +381,7 @@ message WildcardMatchCommandAddArg {
int64 priority = 2; ///If a packet matches multiple rules, the rule with higher priority will be applied. If priorities are equal behavior is undefined.
repeated FieldData values = 3; /// The values to check for in each field.
repeated FieldData masks = 4; /// The bitmask for each field -- set `0x0` to ignore the field altogether.
+ repeated FieldData valuesv = 5; /// The values to check for in each fieldv.
}
/**
@@ -501,6 +506,9 @@ message EtherEncapArg {
message ExactMatchArg {
repeated Field fields = 1; ///A list of ExactMatch Fields
repeated FieldData masks = 2; /// mask(i) corresponds to the mask for field(i)
+ repeated Field values = 3; /// A list of ExactMatch Values
+ repeated FieldData masksv = 4; /// mask(i) corresponds to the mask for value(i)
+ uint64 entries = 5;
}
/**
@@ -996,6 +1004,7 @@ message SourceArg {
*/
message IPChecksumArg {
bool verify = 1; /// check checksum
+ bool hw = 2; /// enable hardware offload
}
/**
@@ -1009,6 +1018,85 @@ message IPChecksumArg {
*/
message L4ChecksumArg {
bool verify = 1; /// check checksum
+ bool hw = 2; /// enable hardware offload
+}
+
+/**
+ * The GtpuEcho module processes the GTPv1 echo packet and prepares
+ * corresponding IP packet containing GTP echo response. It assumes
+ * Recovery IE is always zero.
+ *
+ * __Input Gates__: 1
+ * __Output Gates__: 1
+ */
+message GtpuEchoArg {
+ uint32 s1u_sgw_ip = 1; /// IP address of S1U interface
+}
+
+/**
+ * The IPDefrag module scans the IP datagram and checks whether
+ * it is fragmented. It returns a fully reassembled datagram or
+ * an unfragmented IP datagram
+ *
+ * __Input Gates__: 1
+ * __Output Gates__: 1
+ */
+message IPDefragArg {
+ uint32 num_flows = 1; /// max number of flows the module can handle
+ int32 numa = 2; /// numa placement for ip frags memory management
+}
+
+/**
+ * The IPDFrag module scans the IP datagram and checks whether
+ * it needs to be fragmented.
+ *
+ * __Input Gates__: 1
+ * __Output Gates__: 1
+ */
+message IPFragArg {
+ int32 mtu = 1; /// full Ethernet frame size (including CRC) for encapsulated ipv4 frag datagrams
+}
+
+/**
+ * The Counter module has a command `add(...)` which takes one
+ * parameters. This function accepts the counter id of a
+ * session record.
+ * Example use in bessctl: `counter.add(ctr_id=0x1)`
+ */
+message CounterAddArg {
+ uint32 ctr_id = 1; /// counter id
+}
+
+/**
+ * The Counter module has a command `remove(...)` which takes one
+ * parameter. This function accepts ctr_id, and removes the
+ * respective counter.
+ * Example use in bessctl: `counter.remove(ctr_id=0x1)`
+ */
+message CounterRemoveArg {
+ uint32 ctr_id = 1; /// counter id
+}
+
+/**
+ * The Counter module counts the number of packets and bytes it passes
+ *
+ * __Input Gates__: 1
+ * __Output Gates__: 1
+*/
+message CounterArg {
+ string name_id = 1; /// Name of the counter_id
+ bool check_exist = 2; /// verify each counter pre-exists before any operation (default = False)
+ uint32 total = 3; /// Total number of entries it can support
+}
+
+/**
+ * The GtpuEncap module inserts GTP header in an ethernet frame
+ *
+ * __Input Gates__: 1
+ * __Output Gates__: 1
+*/
+message GtpuEncapArg {
+ bool add_psc = 1; /// Add PDU session container in encap (default = False)
}
/**
@@ -1151,6 +1239,8 @@ message VXLANEncapArg {
*/
message WildcardMatchArg {
repeated Field fields = 1; /// A list of WildcardMatch fields.
+ repeated Field values = 2; /// A list of WildcardMatch values.
+ uint64 entries = 3;
}
/**
@@ -1206,3 +1296,79 @@ message MplsPopArg {
message WorkerSplitArg {
map worker_gates = 1; // ogate -> worker mask
}
+
+message QosArg {
+ repeated Field fields = 1;
+ repeated Field values = 2;
+ uint64 entries = 3;
+}
+
+message QosCommandAddArg {
+ uint64 gate = 1;
+ uint64 cir = 2;
+ uint64 pir = 3;
+ uint64 cbs = 4;
+ uint64 pbs = 5;
+ uint64 ebs = 6;
+ oneof optional_deduct_len {
+ int64 deduct_len = 9;
+ }
+ repeated FieldData fields = 7;
+ repeated FieldData values = 8;
+}
+
+message QosCommandDeleteArg {
+ repeated FieldData fields = 2;
+}
+
+/**
+ * The function `clear()` for WildcardMatch takes no parameters, it clears
+ * all state in the WildcardMatch module (is equivalent to calling delete for all rules)
+ */
+message QosCommandClearArg {
+}
+
+/**
+ * For traffic which does not match any rule in the WildcardMatch module,
+ * the `set_default_gate(...)` function specifies which gate to send this extra traffic to.
+ */
+message QosCommandSetDefaultGateArg {
+ uint64 gate = 1;
+}
+
+message FlowMeasureArg {
+ string flag_attr_name = 1;
+ uint64 entries = 2;
+ bool leader = 3; // If true, this module will decide the buffer side
+}
+message FlowMeasureCommandReadArg {
+ bool clear = 1; // If true, the data will be all cleared after read
+ repeated double latency_percentiles = 2; /// ascending list of real numbers in [0.0, 100.0]
+ repeated double jitter_percentiles = 3; /// ascending list of real numbers in [0.0, 100.0]
+ uint64 flag_to_read = 4; /// Which buffer to read from
+}
+message FlowMeasureReadResponse {
+ message Statistic {
+ message Histogram {
+ uint64 count = 1; /// Total # of measured data points, including above_range
+ uint64 above_range = 2; /// # of data points for the "too large value" bucket
+ uint64 resolution_ns = 8; /// resolution of measured data
+ uint64 min_ns = 3;
+ uint64 avg_ns = 4;
+ uint64 max_ns = 5;
+ uint64 total_ns = 6;
+ repeated uint64 percentile_values_ns = 7;
+ }
+ uint64 fseid = 1;
+ uint64 pdr = 2;
+ Histogram latency = 3;
+ Histogram jitter = 4;
+ uint64 total_bytes = 11;
+ uint64 total_packets = 12;
+ }
+ repeated Statistic statistics = 1;
+}
+message FlowMeasureCommandFlipArg {}
+message FlowMeasureFlipResponse {
+ uint64 old_flag = 1;
+}
diff --git a/protobuf/ports/port_msg.proto b/protobuf/ports/port_msg.proto
index ff6e65ede2..7896a22a84 100644
--- a/protobuf/ports/port_msg.proto
+++ b/protobuf/ports/port_msg.proto
@@ -31,6 +31,8 @@ syntax = "proto3";
package bess.pb;
+option go_package = "github.com/omec-project/upf-epc/pfcpiface/bess_pb";
+
message PCAPPortArg {
string dev = 1;
}
@@ -47,6 +49,15 @@ message PMDPortArg {
bool vlan_offload_rx_strip = 5;
bool vlan_offload_rx_filter = 6;
bool vlan_offload_rx_qinq = 7;
+ oneof socket {
+ int32 socket_id = 8;
+ }
+ bool promiscuous_mode = 9;
+ bool hwcksum = 10;
+
+ // N3 -> 3; N6 -> 6; N9 -> 9
+ // [3] or [6, 9]
+ repeated uint32 flow_profiles = 11;
}
message UnixSocketPortArg {
diff --git a/protobuf/service.proto b/protobuf/service.proto
index ae1aba0533..706a283a46 100644
--- a/protobuf/service.proto
+++ b/protobuf/service.proto
@@ -33,6 +33,8 @@ import "bess_msg.proto";
package bess.pb;
+option go_package = "github.com/omec-project/upf-epc/pfcpiface/bess_pb";
+
service BESSControl {
// -------------------------------------------------------------------------
diff --git a/protobuf/util_msg.proto b/protobuf/util_msg.proto
index b1050abfd1..4df01fc67b 100644
--- a/protobuf/util_msg.proto
+++ b/protobuf/util_msg.proto
@@ -32,6 +32,8 @@ syntax="proto3";
package bess.pb;
+option go_package = "github.com/omec-project/upf-epc/pfcpiface/bess_pb";
+
/// The Field message represents one field in a packet -- either stored in metadata or in the packet body.
message Field {
oneof position {