diff --git a/bessctl/static/pipeline.js b/bessctl/static/pipeline.js index f0041745c4..94c45abeef 100644 --- a/bessctl/static/pipeline.js +++ b/bessctl/static/pipeline.js @@ -134,7 +134,7 @@ function graph_to_dot(modules) { var ogates = module.show_ogates ? gates_to_str(module.ogates, 'ogate') : ''; nodes += ` - ${module_name} [shape=plaintext label= + "${module_name}" [shape=plaintext label= < ${igates} diff --git a/core/dpdk.cc b/core/dpdk.cc index ffade53059..964d513c2c 100644 --- a/core/dpdk.cc +++ b/core/dpdk.cc @@ -123,8 +123,10 @@ void init_eal(int dpdk_mb_per_socket, std::string nonworker_corelist) { "--legacy-mem", }; + if (FLAGS_iova != "") + rte_args.Append({"--iova", FLAGS_iova}); + if (dpdk_mb_per_socket <= 0) { - rte_args.Append({"--iova", (FLAGS_iova != "") ? FLAGS_iova : "va"}); rte_args.Append({"--no-huge"}); // even if we opt out of using hugepages, many DPDK libraries still rely on @@ -132,8 +134,6 @@ void init_eal(int dpdk_mb_per_socket, std::string nonworker_corelist) { // memory in advance. We allocate 512MB (this is shared among nodes). rte_args.Append({"-m", "512"}); } else { - rte_args.Append({"--iova", (FLAGS_iova != "") ? FLAGS_iova : "pa"}); - std::string opt_socket_mem = std::to_string(dpdk_mb_per_socket); for (int i = 1; i < NumNumaNodes(); i++) { opt_socket_mem += "," + std::to_string(dpdk_mb_per_socket); diff --git a/core/drivers/pmd.cc b/core/drivers/pmd.cc index fd4a7b0572..dbebbe5513 100644 --- a/core/drivers/pmd.cc +++ b/core/drivers/pmd.cc @@ -32,10 +32,18 @@ #include #include +#include #include "../utils/ether.h" #include "../utils/format.h" +// TODO: Replace with one time initialized key during InitDriver? +static uint8_t rss_key[40] = {0xD8, 0x2A, 0x6C, 0x5A, 0xDD, 0x3B, 0x9D, 0x1E, + 0x14, 0xCE, 0x2F, 0x37, 0x86, 0xB2, 0x69, 0xF0, + 0x44, 0x31, 0x7E, 0xA2, 0x07, 0xA5, 0x0A, 0x99, + 0x49, 0xC6, 0xA4, 0xFE, 0x0C, 0x4F, 0x59, 0x02, + 0xD4, 0x44, 0xE2, 0x4A, 0xDB, 0xE1, 0x05, 0x82}; + static const rte_eth_conf default_eth_conf(const rte_eth_dev_info &dev_info, int nb_rxq) { rte_eth_conf ret = {}; @@ -45,8 +53,8 @@ static const rte_eth_conf default_eth_conf(const rte_eth_dev_info &dev_info, ret.rxmode.offloads = 0; ret.rx_adv_conf.rss_conf = { - .rss_key = nullptr, - .rss_key_len = 0, + .rss_key = rss_key, + .rss_key_len = sizeof(rss_key), .rss_hf = (ETH_RSS_IP | ETH_RSS_UDP | ETH_RSS_TCP | ETH_RSS_SCTP) & dev_info.flow_type_rss_offloads, }; @@ -199,6 +207,122 @@ static CommandResponse find_dpdk_vdev(const std::string &vdev, return CommandSuccess(); } +CommandResponse flow_create_one(dpdk_port_t port_id, + const uint32_t &flow_profile, int size, + uint64_t rss_types, + rte_flow_item_type *pattern) { + struct rte_flow_item items[size]; + memset(items, 0, sizeof(items)); + + for (int i = 0; i < size; i++) { + items[i].type = pattern[i]; + items[i].spec = nullptr; + items[i].mask = nullptr; + } + + struct rte_flow *handle; + struct rte_flow_error err; + memset(&err, 0, sizeof(err)); + + struct rte_flow_action actions[2]; + memset(actions, 0, sizeof(actions)); + + struct rte_flow_attr attributes; + memset(&attributes, 0, sizeof(attributes)); + attributes.ingress = 1; + + struct rte_flow_action_rss action_rss; + memset(&action_rss, 0, sizeof(action_rss)); + action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT; + action_rss.key_len = 0; + action_rss.types = rss_types; + + actions[0].type = RTE_FLOW_ACTION_TYPE_RSS; + actions[0].conf = &action_rss; + actions[1].type = RTE_FLOW_ACTION_TYPE_END; + + int ret = rte_flow_validate(port_id, &attributes, items, actions, &err); + if (ret) + return CommandFailure(EINVAL, + "Port %u: Failed to validate flow profile %u %s", + port_id, flow_profile, err.message); + + handle = rte_flow_create(port_id, &attributes, items, actions, &err); + if (handle == nullptr) + return CommandFailure(EINVAL, "Port %u: Failed to create flow %s", port_id, + err.message); + + return CommandSuccess(); +} + +#define NUM_ELEMENTS(x) (sizeof(x) / sizeof((x)[0])) + +enum FlowProfile : uint32_t +{ + profileN3 = 3, + profileN6 = 6, + profileN9 = 9, +}; + +CommandResponse flow_create(dpdk_port_t port_id, const uint32_t &flow_profile) { + CommandResponse err; + + rte_flow_item_type N39_NSA[] = { + RTE_FLOW_ITEM_TYPE_ETH, RTE_FLOW_ITEM_TYPE_IPV4, RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_GTPU, RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_END}; + + rte_flow_item_type N39_SA[] = { + RTE_FLOW_ITEM_TYPE_ETH, RTE_FLOW_ITEM_TYPE_IPV4, RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_GTPU, RTE_FLOW_ITEM_TYPE_GTP_PSC, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_END}; + + rte_flow_item_type N6[] = { + RTE_FLOW_ITEM_TYPE_ETH, RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_END}; + + switch (flow_profile) { + uint64_t rss_types; + // N3 traffic with and without PDU Session container + case profileN3: + rss_types = ETH_RSS_IPV4 | ETH_RSS_L3_SRC_ONLY; + err = flow_create_one(port_id, flow_profile, NUM_ELEMENTS(N39_NSA), + rss_types, N39_NSA); + if (err.error().code() != 0) { + return err; + } + + err = flow_create_one(port_id, flow_profile, NUM_ELEMENTS(N39_SA), + rss_types, N39_SA); + break; + + // N6 traffic + case profileN6: + rss_types = ETH_RSS_IPV4 | ETH_RSS_L3_DST_ONLY; + err = flow_create_one(port_id, flow_profile, NUM_ELEMENTS(N6), + rss_types, N6); + break; + + // N9 traffic with and without PDU Session container + case profileN9: + rss_types = ETH_RSS_IPV4 | ETH_RSS_L3_DST_ONLY; + err = flow_create_one(port_id, flow_profile, NUM_ELEMENTS(N39_NSA), + rss_types, N39_NSA); + if (err.error().code() != 0) { + return err; + } + + err = flow_create_one(port_id, flow_profile, NUM_ELEMENTS(N39_SA), + rss_types, N39_SA); + break; + + default: + return CommandFailure(EINVAL, "Unknown flow profile %u", flow_profile); + } + return err; +} + CommandResponse PMDPort::Init(const bess::pb::PMDPortArg &arg) { dpdk_port_t ret_port_id = DPDK_PORT_UNKNOWN; @@ -245,16 +369,28 @@ CommandResponse PMDPort::Init(const bess::pb::PMDPortArg &arg) { if (arg.loopback()) { eth_conf.lpbk_mode = 1; } + if (arg.hwcksum()) { + eth_conf.rxmode.offloads = DEV_RX_OFFLOAD_IPV4_CKSUM | + DEV_RX_OFFLOAD_UDP_CKSUM | + DEV_RX_OFFLOAD_TCP_CKSUM; + } ret = rte_eth_dev_configure(ret_port_id, num_rxq, num_txq, ð_conf); if (ret != 0) { + VLOG(1) << "Failed to configure with hardware checksum offload. " + << "Create PMDPort without hardware offload"; return CommandFailure(-ret, "rte_eth_dev_configure() failed"); } - int sid = rte_eth_dev_socket_id(ret_port_id); + int sid = arg.socket_case() == bess::pb::PMDPortArg::kSocketId ? + arg.socket_id() : rte_eth_dev_socket_id(ret_port_id); + /* if socket_id is invalid, set to 0 */ if (sid < 0 || sid > RTE_MAX_NUMA_NODES) { - sid = 0; // if socket_id is invalid, set to 0 + LOG(WARNING) << "Invalid socket, falling back... "; + sid = 0; } + LOG(INFO) << "Initializing Port:" << ret_port_id + << " with memory from socket " << sid; eth_rxconf = dev_info.default_rxconf; eth_rxconf.rx_drop_en = 1; @@ -308,7 +444,12 @@ CommandResponse PMDPort::Init(const bess::pb::PMDPortArg &arg) { } } - rte_eth_promiscuous_enable(ret_port_id); + if (arg.promiscuous_mode()) { + ret = rte_eth_promiscuous_enable(ret_port_id); + if (ret != 0) { + return CommandFailure(-ret, "rte_eth_promiscuous_enable() failed"); + } + } int offload_mask = 0; offload_mask |= arg.vlan_offload_rx_strip() ? ETH_VLAN_STRIP_OFFLOAD : 0; @@ -327,7 +468,8 @@ CommandResponse PMDPort::Init(const bess::pb::PMDPortArg &arg) { } dpdk_port_id_ = ret_port_id; - int numa_node = rte_eth_dev_socket_id(static_cast(ret_port_id)); + int numa_node = arg.socket_case() == bess::pb::PMDPortArg::kSocketId ? + sid : rte_eth_dev_socket_id(ret_port_id); node_placement_ = numa_node == -1 ? UNCONSTRAINED_SOCKET : (1ull << numa_node); @@ -339,6 +481,15 @@ CommandResponse PMDPort::Init(const bess::pb::PMDPortArg &arg) { driver_ = dev_info.driver_name ?: "unknown"; + if (arg.flow_profiles_size() > 0){ + for (int i = 0; i < arg.flow_profiles_size(); ++i) { + err = flow_create(ret_port_id, arg.flow_profiles(i)); + if (err.error().code() != 0) { + return err; + } + } + } + return CommandSuccess(); } @@ -448,9 +599,10 @@ void PMDPort::CollectStats(bool reset) { port_stats_.inc.dropped = stats.imissed; - // i40e/net_e1000_igb PMD drivers, ixgbevf and net_bonding vdevs don't support - // per-queue stats - if (driver_ == "net_i40e" || driver_ == "net_i40e_vf" || + // ice/i40e/net_e1000_igb PMD drivers, ixgbevf and net_bonding vdevs don't + // support per-queue stats + if (driver_ == "net_ice" || driver_ == "net_iavf" || + driver_ == "net_i40e" || driver_ == "net_i40e_vf" || driver_ == "net_ixgbe_vf" || driver_ == "net_bonding" || driver_ == "net_e1000_igb") { // NOTE: diff --git a/core/modules/exact_match.cc b/core/modules/exact_match.cc index 072a0333dd..e50cf5cce6 100644 --- a/core/modules/exact_match.cc +++ b/core/modules/exact_match.cc @@ -50,20 +50,21 @@ const Commands ExactMatch::cmds = { {"set_runtime_config", "ExactMatchConfig", MODULE_CMD_FUNC(&ExactMatch::SetRuntimeConfig), Command::THREAD_UNSAFE}, {"add", "ExactMatchCommandAddArg", MODULE_CMD_FUNC(&ExactMatch::CommandAdd), - Command::THREAD_UNSAFE}, + Command::THREAD_SAFE}, {"delete", "ExactMatchCommandDeleteArg", - MODULE_CMD_FUNC(&ExactMatch::CommandDelete), Command::THREAD_UNSAFE}, + MODULE_CMD_FUNC(&ExactMatch::CommandDelete), Command::THREAD_SAFE}, {"clear", "EmptyArg", MODULE_CMD_FUNC(&ExactMatch::CommandClear), - Command::THREAD_UNSAFE}, + Command::THREAD_SAFE}, {"set_default_gate", "ExactMatchCommandSetDefaultGateArg", MODULE_CMD_FUNC(&ExactMatch::CommandSetDefaultGate), Command::THREAD_SAFE}}; CommandResponse ExactMatch::AddFieldOne(const bess::pb::Field &field, const bess::pb::FieldData &mask, - int idx) { + int idx, Type t) { int size = field.num_bytes(); uint64_t mask64 = 0; + if (mask.encoding_case() == bess::pb::FieldData::kValueInt) { mask64 = mask.value_int(); } else if (mask.encoding_case() == bess::pb::FieldData::kValueBin) { @@ -73,12 +74,15 @@ CommandResponse ExactMatch::AddFieldOne(const bess::pb::Field &field, Error ret; if (field.position_case() == bess::pb::Field::kAttrName) { - ret = table_.AddField(this, field.attr_name(), size, mask64, idx); + ret = (t == FIELD_TYPE) + ? table_.AddField(this, field.attr_name(), size, mask64, idx) + : AddValue(this, field.attr_name(), size, mask64, idx); if (ret.first) { return CommandFailure(ret.first, "%s", ret.second.c_str()); } } else if (field.position_case() == bess::pb::Field::kOffset) { - ret = table_.AddField(field.offset(), size, mask64, idx); + ret = (t == FIELD_TYPE) ? table_.AddField(field.offset(), size, mask64, idx) + : AddValue(field.offset(), size, mask64, idx); if (ret.first) { return CommandFailure(ret.first, "%s", ret.second.c_str()); } @@ -103,9 +107,9 @@ CommandResponse ExactMatch::Init(const bess::pb::ExactMatchArg &arg) { if (empty_masks_) { bess::pb::FieldData emptymask; - err = AddFieldOne(arg.fields(i), emptymask, i); + err = AddFieldOne(arg.fields(i), emptymask, i, FIELD_TYPE); } else { - err = AddFieldOne(arg.fields(i), arg.masks(i), i); + err = AddFieldOne(arg.fields(i), arg.masks(i), i, FIELD_TYPE); } if (err.error().code() != 0) { @@ -113,8 +117,24 @@ CommandResponse ExactMatch::Init(const bess::pb::ExactMatchArg &arg) { } } - default_gate_ = DROP_GATE; + empty_masks_ = arg.masksv_size() == 0; + for (auto i = 0; i < arg.values_size(); ++i) { + CommandResponse err; + + if (empty_masks_) { + bess::pb::FieldData emptymask; + err = AddFieldOne(arg.values(i), emptymask, i, VALUE_TYPE); + } else { + err = AddFieldOne(arg.values(i), arg.masksv(i), i, VALUE_TYPE); + } + + if (err.error().code() != 0) { + return err; + } + } + default_gate_ = DROP_GATE; + table_.Init(arg.entries()); return CommandSuccess(); } @@ -157,7 +177,7 @@ CommandResponse ExactMatch::GetRuntimeConfig(const bess::pb::EmptyArg &) { auto const &value = kv.second; rule_t *rule = r.add_rules(); - rule->set_gate(value); + rule->set_gate(value.gate); for (size_t i = 0; i < table_.num_fields(); i++) { const ExactMatchField &f = table_.get_field(i); bess::pb::FieldData *field = rule->add_fields(); @@ -199,9 +219,30 @@ Error ExactMatch::AddRule(const bess::pb::ExactMatchCommandAddArg &arg) { } ExactMatchRuleFields rule; - RuleFieldsFromPb(arg.fields(), &rule); + ExactMatchRuleFields action; + Error err; + ValueTuple t; + + /* clear value tuple */ + memset(&t.action, 0, sizeof(t.action)); + /* set gate */ + t.gate = gate; + RuleFieldsFromPb(arg.fields(), &rule, FIELD_TYPE); + /* check whether values match with the the table's */ + if (arg.values_size() != (ssize_t)num_values()) + return std::make_pair( + EINVAL, bess::utils::Format( + "rule has incorrect number of values. Need %d, has %d", + (int)num_values(), arg.values_size())); + /* check if values are non-zero */ + if (arg.values_size() > 0) { + RuleFieldsFromPb(arg.values(), &action, VALUE_TYPE); + + if ((err = CreateValue(t.action, action)).first != 0) + return err; + } - return table_.AddRule(gate, rule); + return table_.AddRule(t, rule); } // Uses an ExactMatchConfig to restore this module's runtime config. @@ -221,6 +262,53 @@ CommandResponse ExactMatch::SetRuntimeConfig( return CommandSuccess(); } +void ExactMatch::setValues(bess::Packet *pkt, ExactMatchKey &action) { + size_t num_values_ = num_values(); + + for (size_t i = 0; i < num_values_; i++) { + int value_size = get_value(i).size; + int value_pos = get_value(i).pos; + int value_off = get_value(i).offset; + int value_attr_id = get_value(i).attr_id; + uint8_t *data = pkt->head_data() + value_off; + + if (value_attr_id < 0) { /* if it is offset-based */ + memcpy(data, reinterpret_cast(&action) + value_pos, + value_size); + } else { /* if it is attribute-based */ + switch (value_size) { + case 1: + set_attr(this, value_attr_id, pkt, + *((uint8_t *)((uint8_t *)&action + value_pos))); + break; + case 2: + set_attr(this, value_attr_id, pkt, + *((uint16_t *)((uint8_t *)&action + value_pos))); + break; + case 4: + set_attr(this, value_attr_id, pkt, + *((uint32_t *)((uint8_t *)&action + value_pos))); + break; + case 8: + set_attr(this, value_attr_id, pkt, + *((uint64_t *)((uint8_t *)&action + value_pos))); + break; + default: { + typedef struct { + uint8_t bytes[bess::metadata::kMetadataAttrMaxSize]; + } value_t; + void *mt_ptr = + _ptr_attr_with_offset(attr_offset(value_attr_id), pkt); + bess::utils::CopySmall( + mt_ptr, + reinterpret_cast(((uint8_t *)(&action)) + value_pos), + value_size); + } break; + } + } + } +} + void ExactMatch::ProcessBatch(Context *ctx, bess::PacketBatch *batch) { gate_idx_t default_gate; ExactMatchKey keys[bess::PacketBatch::kMaxBurst] __ymm_aligned; @@ -237,9 +325,23 @@ void ExactMatch::ProcessBatch(Context *ctx, bess::PacketBatch *batch) { table_.MakeKeys(batch, buffer_fn, keys); int cnt = batch->cnt(); - for (int i = 0; i < cnt; i++) { - bess::Packet *pkt = batch->pkts()[i]; - EmitPacket(ctx, pkt, table_.Find(keys[i], default_gate)); + Value default_value(default_gate); + + int icnt=0; + for(int lcnt=0; lcnt=64) ? 64 : cnt-lcnt ; + ValueTuple *res[icnt]; + uint64_t hit_mask = table_.Find(keys+lcnt, res, icnt); + + for (int j = 0; j < icnt; j++) { + if ((hit_mask & ((uint64_t)1ULL << j)) == 0) + EmitPacket(ctx, batch->pkts()[j+lcnt], default_gate); + else { + setValues(batch->pkts()[j+lcnt], res[j]->action); + EmitPacket(ctx, batch->pkts()[j+lcnt], res[j]->gate); + } + } } } @@ -250,20 +352,34 @@ std::string ExactMatch::GetDesc() const { void ExactMatch::RuleFieldsFromPb( const RepeatedPtrField &fields, - bess::utils::ExactMatchRuleFields *rule) { + bess::utils::ExactMatchRuleFields *rule, Type type) { for (auto i = 0; i < fields.size(); i++) { - int field_size = table_.get_field(i).size; + (void)type; + int field_size = + (type == FIELD_TYPE) ? table_.get_field(i).size : get_value(i).size; + int attr_id = (type == FIELD_TYPE) ? table_.get_field(i).attr_id + : get_value(i).attr_id; bess::pb::FieldData current = fields.Get(i); - if (current.encoding_case() == bess::pb::FieldData::kValueBin) { const std::string &f_obj = fields.Get(i).value_bin(); rule->push_back(std::vector(f_obj.begin(), f_obj.end())); } else { rule->emplace_back(); - uint64_t rule64 = current.value_int(); + uint64_t rule64 = 0; + if (attr_id < 0) { + if (!bess::utils::uint64_to_bin(&rule64, current.value_int(), + field_size, 1)) { + std::cerr << "idx " << i << ": not a correct" << field_size + << "-byte value\n"; + return; + } + } else { + rule64 = current.value_int(); + } for (int j = 0; j < field_size; j++) { rule->back().push_back(rule64 & 0xFFULL); + DLOG(INFO) << "Pushed " << std::hex << (rule64 & 0xFFULL) << " to rule."; rule64 >>= 8; } } @@ -289,7 +405,7 @@ CommandResponse ExactMatch::CommandDelete( } ExactMatchRuleFields rule; - RuleFieldsFromPb(arg.fields(), &rule); + RuleFieldsFromPb(arg.fields(), &rule, FIELD_TYPE); Error ret = table_.DeleteRule(rule); if (ret.first) { @@ -310,4 +426,8 @@ CommandResponse ExactMatch::CommandSetDefaultGate( return CommandSuccess(); } +void ExactMatch::DeInit() { + table_.DeInit(); +} + ADD_MODULE(ExactMatch, "em", "Multi-field classifier with an exact match table") diff --git a/core/modules/exact_match.h b/core/modules/exact_match.h index f7560e9700..d7e3531ed3 100644 --- a/core/modules/exact_match.h +++ b/core/modules/exact_match.h @@ -37,13 +37,36 @@ #include "../module.h" #include "../pb/module_msg.pb.h" #include "../utils/exact_match_table.h" +#include "../utils/format.h" -using google::protobuf::RepeatedPtrField; +using bess::utils::Error; using bess::utils::ExactMatchField; using bess::utils::ExactMatchKey; using bess::utils::ExactMatchRuleFields; using bess::utils::ExactMatchTable; -using bess::utils::Error; +using google::protobuf::RepeatedPtrField; + +typedef enum { FIELD_TYPE = 0, VALUE_TYPE } Type; + +class ExactMatch; +class Value { + friend class ExactMatch; + + public: + Value(gate_idx_t g = 0) : gate(g) {} + Value(const Value &v) : gate(v.gate) {} + gate_idx_t gate; +}; + +class ValueTuple : public Value { + friend class ExactMatch; + + public: + ValueTuple() : Value(), action() {} + ValueTuple(Value v) : Value(v), action() {} + + ExactMatchKey action; +}; class ExactMatch final : public Module { public: @@ -51,7 +74,14 @@ class ExactMatch final : public Module { static const Commands cmds; - ExactMatch() : Module(), default_gate_(), table_() { + ExactMatch() + : Module(), + default_gate_(), + raw_value_size_(), + total_value_size_(), + num_values_(), + values_(), + table_() { max_allowed_workers_ = Worker::kMaxWorkers; } @@ -60,6 +90,9 @@ class ExactMatch final : public Module { std::string GetDesc() const override; CommandResponse Init(const bess::pb::ExactMatchArg &arg); + + void DeInit() override; + CommandResponse GetInitialArg(const bess::pb::EmptyArg &arg); CommandResponse GetRuntimeConfig(const bess::pb::EmptyArg &arg); CommandResponse SetRuntimeConfig(const bess::pb::ExactMatchConfig &arg); @@ -72,15 +105,153 @@ class ExactMatch final : public Module { private: CommandResponse AddFieldOne(const bess::pb::Field &field, - const bess::pb::FieldData &mask, int idx); + const bess::pb::FieldData &mask, int idx, Type t); void RuleFieldsFromPb(const RepeatedPtrField &fields, - bess::utils::ExactMatchRuleFields *rule); + bess::utils::ExactMatchRuleFields *rule, Type type); Error AddRule(const bess::pb::ExactMatchCommandAddArg &arg); + size_t num_values() const { return num_values_; } + ExactMatchField *getVals() { return values_; }; + Error gather_value(const ExactMatchRuleFields &fields, ExactMatchKey *key) { + if (fields.size() != num_values_) { + return std::make_pair( + EINVAL, bess::utils::Format("rule should have %zu fields (has %zu)", + num_values_, fields.size())); + } + + *key = {}; + + for (size_t i = 0; i < fields.size(); i++) { + int field_size = values_[i].size; + int field_pos = values_[i].pos; + + const std::vector &f_obj = fields[i]; + + if (static_cast(field_size) != f_obj.size()) { + return std::make_pair( + EINVAL, + bess::utils::Format("rule field %zu should have size %d (has %zu)", + i, field_size, f_obj.size())); + } + + memcpy(reinterpret_cast(key) + field_pos, f_obj.data(), + field_size); + } + + return std::make_pair(0, bess::utils::Format("Success")); + } + // Helper for public AddField functions. + // DoAddValue inserts `field` as the `idx`th field for this table. + // If `mt_attr_name` is set, the `offset` field of `field` will be ignored and + // the inserted field will use the offset of `mt_attr_name` as reported by the + // module `m`. + // Returns 0 on success, non-zero errno on failure. + Error DoAddValue(const ExactMatchField &value, + const std::string &mt_attr_name, int idx, + Module *m = nullptr) { + if (idx >= MAX_FIELDS) { + return std::make_pair( + EINVAL, + bess::utils::Format("idx %d is not in [0,%d)", idx, MAX_FIELDS)); + } + ExactMatchField *v = &values_[idx]; + v->size = value.size; + if (v->size < 1 || v->size > MAX_FIELD_SIZE) { + return std::make_pair( + EINVAL, bess::utils::Format("idx %d: 'size' must be in [1,%d]", idx, + MAX_FIELD_SIZE)); + } + + if (mt_attr_name.length() > 0) { + v->attr_id = m->AddMetadataAttr( + mt_attr_name, v->size, bess::metadata::Attribute::AccessMode::kWrite); + if (v->attr_id < 0) { + return std::make_pair( + -v->attr_id, + bess::utils::Format("idx %d: add_metadata_attr() failed", idx)); + } + } else { + v->attr_id = -1; + v->offset = value.offset; + if (v->offset < 0 || v->offset > 1024) { + return std::make_pair( + EINVAL, bess::utils::Format("idx %d: invalid 'offset'", idx)); + } + } + + int force_be = (v->attr_id < 0); + + if (value.mask == 0) { + /* by default all bits are considered */ + v->mask = bess::utils::SetBitsHigh(v->size * 8); + } else { + if (!bess::utils::uint64_to_bin(&v->mask, value.mask, v->size, + bess::utils::is_be_system() | force_be)) { + return std::make_pair( + EINVAL, bess::utils::Format("idx %d: not a valid %d-byte mask", idx, + v->size)); + } + } + + if (v->mask == 0) { + return std::make_pair(EINVAL, + bess::utils::Format("idx %d: empty mask", idx)); + } + + num_values_++; + + v->pos = raw_value_size_; + raw_value_size_ += v->size; + total_value_size_ = align_ceil(raw_value_size_, sizeof(uint64_t)); + return std::make_pair(0, bess::utils::Format("Success")); + } + // Returns the ith value. + const ExactMatchField &get_value(size_t i) const { return values_[i]; } + // Set the `idx`th field of this table to one at offset `offset` bytes into a + // buffer with length `size` and mask `mask`. + // Returns 0 on success, non-zero errno on failure. + Error AddValue(int offset, int size, uint64_t mask, int idx) { + ExactMatchField v = { + .mask = mask, .attr_id = 0, .offset = offset, .pos = 0, .size = size}; + return DoAddValue(v, "", idx, nullptr); + } + + // Set the `idx`th field of this table to one at the offset of the + // `mt_attr_name` metadata field as seen by module `m`, with length `size` and + // mask `mask`. + // Returns 0 on success, non-zero errno on failure. + Error AddValue(Module *m, const std::string &mt_attr_name, int size, + uint64_t mask, int idx) { + ExactMatchField v = { + .mask = mask, .attr_id = 0, .offset = 0, .pos = 0, .size = size}; + return DoAddValue(v, mt_attr_name, idx, m); + } + Error CreateValue(ExactMatchKey &v, const ExactMatchRuleFields &values) { + Error err; + + if (values.size() == 0) { + return std::make_pair(EINVAL, "rule has no values"); + } + + if ((err = gather_value(values, &v)).first != 0) { + return err; + } + + return std::make_pair(0, bess::utils::Format("Success")); + } + void setValues(bess::Packet *pkt, ExactMatchKey &action); gate_idx_t default_gate_; bool empty_masks_; // mainly for GetInitialArg - ExactMatchTable table_; + // unaligend key size, used as an accumulator for calls to AddField() + size_t raw_value_size_; + + // aligned total key size + size_t total_value_size_; + + size_t num_values_; + ExactMatchField values_[MAX_FIELDS]; + ExactMatchTable table_; }; #endif // BESS_MODULES_EXACTMATCH_H_ diff --git a/core/modules/ip_checksum.cc b/core/modules/ip_checksum.cc index ecaba24d7d..47f86bae84 100644 --- a/core/modules/ip_checksum.cc +++ b/core/modules/ip_checksum.cc @@ -56,8 +56,8 @@ void IPChecksum::ProcessBatch(Context *ctx, bess::PacketBatch *batch) { data = qinq + 1; ether_type = qinq->ether_type; if (ether_type != be16_t(Ethernet::Type::kVlan)) { - EmitPacket(ctx, batch->pkts()[i], FORWARD_GATE); - continue; + EmitPacket(ctx, batch->pkts()[i], FORWARD_GATE); + continue; } } @@ -75,7 +75,17 @@ void IPChecksum::ProcessBatch(Context *ctx, bess::PacketBatch *batch) { } if (verify_) { - EmitPacket(ctx, batch->pkts()[i], (VerifyIpv4Checksum(*ip)) ? FORWARD_GATE : FAIL_GATE); + if (hw_) { + struct rte_mbuf *m = (struct rte_mbuf *)batch->pkts()[i]; + if (unlikely((m->ol_flags & PKT_RX_IP_CKSUM_MASK) == + PKT_RX_IP_CKSUM_BAD)) + EmitPacket(ctx, (bess::Packet *)m, FAIL_GATE); + else + EmitPacket(ctx, (bess::Packet *)m, FORWARD_GATE); + } else { + EmitPacket(ctx, batch->pkts()[i], + (VerifyIpv4Checksum(*ip)) ? FORWARD_GATE : FAIL_GATE); + } } else { ip->checksum = CalculateIpv4Checksum(*ip); EmitPacket(ctx, batch->pkts()[i], FORWARD_GATE); @@ -85,6 +95,7 @@ void IPChecksum::ProcessBatch(Context *ctx, bess::PacketBatch *batch) { CommandResponse IPChecksum::Init(const bess::pb::IPChecksumArg &arg) { verify_ = arg.verify(); + hw_ = arg.hw(); return CommandSuccess(); } diff --git a/core/modules/ip_checksum.h b/core/modules/ip_checksum.h index f0f1d079ff..94e0d7246a 100644 --- a/core/modules/ip_checksum.h +++ b/core/modules/ip_checksum.h @@ -37,7 +37,9 @@ // Compute IP checksum on packet class IPChecksum final : public Module { public: - IPChecksum() : Module(), verify_(false) { max_allowed_workers_ = Worker::kMaxWorkers; } + IPChecksum() : Module(), verify_(false) { + max_allowed_workers_ = Worker::kMaxWorkers; + } /* Gates: (0) Default, (1) Drop */ static const gate_idx_t kNumOGates = 2; @@ -47,6 +49,8 @@ class IPChecksum final : public Module { private: /* enable checksum verification */ bool verify_; + /* enable hardware offload */ + bool hw_; }; #endif // BESS_MODULES_IP_CHECKSUM_H_ diff --git a/core/modules/ip_lookup.cc b/core/modules/ip_lookup.cc index 3c8b115717..7e976d413c 100644 --- a/core/modules/ip_lookup.cc +++ b/core/modules/ip_lookup.cc @@ -32,7 +32,6 @@ #include #include -#include #include "../utils/bits.h" #include "../utils/ether.h" @@ -54,15 +53,26 @@ const Commands IPLookup::cmds = { Command::THREAD_UNSAFE}}; CommandResponse IPLookup::Init(const bess::pb::IPLookupArg &arg) { +#if RTE_VERSION < RTE_VERSION_NUM(19, 11, 0, 0) struct rte_lpm_config conf = { .max_rules = arg.max_rules() ? arg.max_rules() : 1024, .number_tbl8s = arg.max_tbl8s() ? arg.max_tbl8s() : 128, .flags = 0, }; +#else + conf.type = RTE_FIB_DIR24_8; + conf.default_nh = DROP_GATE; + conf.max_routes = arg.max_rules() ? (int)arg.max_rules() : 1024; + conf.dir24_8.nh_sz = RTE_FIB_DIR24_8_4B; + conf.dir24_8.num_tbl8 = arg.max_tbl8s() ? arg.max_tbl8s() : 128; +#endif default_gate_ = DROP_GATE; - - lpm_ = rte_lpm_create(name().c_str(), /* socket_id = */ 0, &conf); +#if RTE_VERSION < RTE_VERSION_NUM(19, 11, 0, 0) + lpm_ = rte_lpm_create(name().c_str(), /* socket_id = */ rte_socket_id(), &conf); +#else + lpm_ = rte_fib_create(name().c_str(), /* socket_id = */ rte_socket_id(), &conf); +#endif if (!lpm_) { return CommandFailure(rte_errno, "DPDK error: %s", rte_strerror(rte_errno)); @@ -73,7 +83,11 @@ CommandResponse IPLookup::Init(const bess::pb::IPLookupArg &arg) { void IPLookup::DeInit() { if (lpm_) { +#if RTE_VERSION < RTE_VERSION_NUM(19, 11, 0, 0) rte_lpm_free(lpm_); +#else + rte_fib_free(lpm_); +#endif } } @@ -86,6 +100,7 @@ void IPLookup::ProcessBatch(Context *ctx, bess::PacketBatch *batch) { int cnt = batch->cnt(); int i; +#if RTE_VERSION < RTE_VERSION_NUM(19, 11, 0, 0) #if VECTOR_OPTIMIZATION // Convert endianness for four addresses at the same time const __m128i bswap_mask = @@ -148,6 +163,29 @@ void IPLookup::ProcessBatch(Context *ctx, bess::PacketBatch *batch) { EmitPacket(ctx, batch->pkts()[i], default_gate); } } +#else /* RTE_VERSION >= 19.11 */ + Ethernet *eth; + Ipv4 *ip; + int ret; + uint32_t ip_list[cnt]; + uint64_t next_hops[cnt]; + + for (i = 0; i < cnt; i++) { + eth = batch->pkts()[i]->head_data(); + ip = (Ipv4 *)(eth + 1); + ip_list[i] = ip->dst.value(); + } + + ret = rte_fib_lookup_bulk(lpm_, ip_list, next_hops, cnt); + + if (ret != 0) + RunNextModule(ctx, batch); + else + for (i = 0; i < cnt; i++) { + EmitPacket(ctx, batch->pkts()[i], (next_hops[i] == DROP_GATE) ? default_gate_ : next_hops[i]); + } + USED(default_gate); +#endif } ParsedPrefix IPLookup::ParseIpv4Prefix( @@ -201,7 +239,12 @@ CommandResponse IPLookup::CommandAdd( default_gate_ = gate; } else { be32_t net_addr = std::get<2>(prefix); +#if RTE_VERSION < RTE_VERSION_NUM(19, 11, 0, 0) int ret = rte_lpm_add(lpm_, net_addr.value(), prefix_len, gate); +#else + uint64_t next_hop = (uint64_t)gate; + int ret = rte_fib_add(lpm_, net_addr.value(), prefix_len, next_hop); +#endif if (ret) { return CommandFailure(-ret, "rpm_lpm_add() failed"); } @@ -223,7 +266,11 @@ CommandResponse IPLookup::CommandDelete( default_gate_ = DROP_GATE; } else { be32_t net_addr = std::get<2>(prefix); +#if RTE_VERSION < RTE_VERSION_NUM(19, 11, 0, 0) int ret = rte_lpm_delete(lpm_, net_addr.value(), prefix_len); +#else + int ret = rte_fib_delete(lpm_, net_addr.value(), prefix_len); +#endif if (ret) { return CommandFailure(-ret, "rpm_lpm_delete() failed"); } @@ -233,7 +280,17 @@ CommandResponse IPLookup::CommandDelete( } CommandResponse IPLookup::CommandClear(const bess::pb::EmptyArg &) { +#if RTE_VERSION < RTE_VERSION_NUM(19, 11, 0, 0) rte_lpm_delete_all(lpm_); +#else + /* rte_fib_delete_all(lpm_) does not exist! */ + rte_fib_free(lpm_); + + lpm_ = rte_fib_create(name().c_str(), /* socket_id = */ 0, &conf); + if (!lpm_) { + return CommandFailure(rte_errno, "DPDK error: %s", rte_strerror(rte_errno)); + } +#endif return CommandSuccess(); } diff --git a/core/modules/ip_lookup.h b/core/modules/ip_lookup.h index de020e7f43..a866ad085e 100644 --- a/core/modules/ip_lookup.h +++ b/core/modules/ip_lookup.h @@ -34,6 +34,15 @@ #include "../module.h" #include "../pb/module_msg.pb.h" #include "../utils/endian.h" +#include +#if RTE_VERSION < RTE_VERSION_NUM(19, 11, 0, 0) +#include +#else +#define USED(x) (void)(x) +extern "C" { +#include +} +#endif using bess::utils::be32_t; using ParsedPrefix = std::tuple; @@ -59,7 +68,12 @@ class IPLookup final : public Module { CommandResponse CommandClear(const bess::pb::EmptyArg &arg); private: +#if RTE_VERSION < RTE_VERSION_NUM(19, 11, 0, 0) struct rte_lpm *lpm_; +#else + struct rte_fib *lpm_; + struct rte_fib_conf conf; +#endif gate_idx_t default_gate_; ParsedPrefix ParseIpv4Prefix(const std::string &prefix, uint64_t prefix_len); }; diff --git a/core/modules/l4_checksum.cc b/core/modules/l4_checksum.cc index 3ad1f23d68..20faa85c33 100644 --- a/core/modules/l4_checksum.cc +++ b/core/modules/l4_checksum.cc @@ -63,27 +63,52 @@ void L4Checksum::ProcessBatch(Context *ctx, bess::PacketBatch *batch) { Udp *udp = reinterpret_cast(reinterpret_cast(ip) + ip_bytes); if (verify_) { - EmitPacket(ctx, batch->pkts()[i], - (VerifyIpv4UdpChecksum(*ip, *udp)) ? FORWARD_GATE : FAIL_GATE); + if (hw_) { + struct rte_mbuf *m = (struct rte_mbuf *)batch->pkts()[i]; + if (unlikely((m->ol_flags & PKT_RX_L4_CKSUM_MASK) == + PKT_RX_L4_CKSUM_BAD)) + EmitPacket(ctx, (bess::Packet *)m, FAIL_GATE); + else + EmitPacket(ctx, (bess::Packet *)m, FORWARD_GATE); + } else { + EmitPacket( + ctx, batch->pkts()[i], + (VerifyIpv4UdpChecksum(*ip, *udp)) ? FORWARD_GATE : FAIL_GATE); + } } else { - udp->checksum = CalculateIpv4UdpChecksum(*ip, *udp); - EmitPacket(ctx, batch->pkts()[i], FORWARD_GATE); + udp->checksum = CalculateIpv4UdpChecksum(*ip, *udp); + EmitPacket(ctx, batch->pkts()[i], FORWARD_GATE); } } else if (ip->protocol == Ipv4::Proto::kTcp) { size_t ip_bytes = (ip->header_length) << 2; Tcp *tcp = reinterpret_cast(reinterpret_cast(ip) + ip_bytes); - if (verify_) - EmitPacket(ctx, batch->pkts()[i], - (VerifyIpv4TcpChecksum(*ip, *tcp)) ? FORWARD_GATE : FAIL_GATE); - else - tcp->checksum = CalculateIpv4TcpChecksum(*ip, *tcp); + if (verify_) { + if (hw_) { + struct rte_mbuf *m = (struct rte_mbuf *)batch->pkts()[i]; + if (unlikely((m->ol_flags & PKT_RX_L4_CKSUM_MASK) == + PKT_RX_L4_CKSUM_BAD)) + EmitPacket(ctx, (bess::Packet *)m, FAIL_GATE); + else + EmitPacket(ctx, (bess::Packet *)m, FORWARD_GATE); + } else { + EmitPacket( + ctx, batch->pkts()[i], + (VerifyIpv4TcpChecksum(*ip, *tcp)) ? FORWARD_GATE : FAIL_GATE); + } + } else { + tcp->checksum = CalculateIpv4TcpChecksum(*ip, *tcp); + EmitPacket(ctx, batch->pkts()[i], FORWARD_GATE); + } + } else { /* fail-safe condition */ + EmitPacket(ctx, batch->pkts()[i], FORWARD_GATE); } } } CommandResponse L4Checksum::Init(const bess::pb::L4ChecksumArg &arg) { verify_ = arg.verify(); + hw_ = arg.hw(); return CommandSuccess(); } diff --git a/core/modules/l4_checksum.h b/core/modules/l4_checksum.h index 27ab8f98f1..56d8ae5c20 100644 --- a/core/modules/l4_checksum.h +++ b/core/modules/l4_checksum.h @@ -36,7 +36,9 @@ // Compute L4 checksum on packet class L4Checksum final : public Module { public: - L4Checksum() : Module(), verify_(false) { max_allowed_workers_ = Worker::kMaxWorkers; } + L4Checksum() : Module(), verify_(false) { + max_allowed_workers_ = Worker::kMaxWorkers; + } /* Gates: (0) Default, (1) Drop */ static const gate_idx_t kNumOGates = 2; @@ -46,6 +48,7 @@ class L4Checksum final : public Module { private: bool verify_; + bool hw_; }; #endif // BESS_MODULES_L4_CHECKSUM_H_ diff --git a/core/modules/nat.h b/core/modules/nat.h index ff29dbf555..4ec594de92 100644 --- a/core/modules/nat.h +++ b/core/modules/nat.h @@ -133,6 +133,7 @@ struct PortRange { // igate/ogate 1: reverse dir class NAT final : public Module { public: + NAT() { max_allowed_workers_ = 2; } enum Direction { kForward = 0, // internal -> external kReverse = 1, // external -> internal diff --git a/core/modules/wildcard_match.cc b/core/modules/wildcard_match.cc index 1b7bfeb6cb..e4c88d9526 100644 --- a/core/modules/wildcard_match.cc +++ b/core/modules/wildcard_match.cc @@ -37,15 +37,31 @@ #include "../utils/format.h" using bess::metadata::Attribute; +enum { FieldType = 0, ValueType }; // dst = src & mask. len must be a multiple of sizeof(uint64_t) -static inline void mask(wm_hkey_t *dst, const wm_hkey_t &src, +static inline void mask(wm_hkey_t &dst, const wm_hkey_t &src, const wm_hkey_t &mask, size_t len) { promise(len >= sizeof(uint64_t)); promise(len <= sizeof(wm_hkey_t)); for (size_t i = 0; i < len / 8; i++) { - dst->u64_arr[i] = src.u64_arr[i] & mask.u64_arr[i]; + dst.u64_arr[i] = src.u64_arr[i] & mask.u64_arr[i]; + } +} +static inline void mask_bulk(const wm_hkey_t *src, void *dst, void **dsptr, + const wm_hkey_t &mask, int keys, size_t len) { + promise(len >= sizeof(uint64_t)); + promise(len <= sizeof(wm_hkey_t)); + size_t i = 0; + wm_hkey_t *dst1 = (wm_hkey_t *)dst; + wm_hkey_t **dstptr = (wm_hkey_t **)dsptr; + + for (int j = 0; j < keys; j++) { + for (i = 0; i < len / 8; i++) { + dst1[j].u64_arr[i] = src[j].u64_arr[i] & mask.u64_arr[i]; + } + dstptr[j] = &dst1[j]; } } @@ -63,17 +79,17 @@ const Commands WildcardMatch::cmds = { {"set_runtime_config", "WildcardMatchConfig", MODULE_CMD_FUNC(&WildcardMatch::SetRuntimeConfig), Command::THREAD_UNSAFE}, {"add", "WildcardMatchCommandAddArg", - MODULE_CMD_FUNC(&WildcardMatch::CommandAdd), Command::THREAD_UNSAFE}, + MODULE_CMD_FUNC(&WildcardMatch::CommandAdd), Command::THREAD_SAFE}, {"delete", "WildcardMatchCommandDeleteArg", - MODULE_CMD_FUNC(&WildcardMatch::CommandDelete), Command::THREAD_UNSAFE}, + MODULE_CMD_FUNC(&WildcardMatch::CommandDelete), Command::THREAD_SAFE}, {"clear", "EmptyArg", MODULE_CMD_FUNC(&WildcardMatch::CommandClear), - Command::THREAD_UNSAFE}, + Command::THREAD_SAFE}, {"set_default_gate", "WildcardMatchCommandSetDefaultGateArg", MODULE_CMD_FUNC(&WildcardMatch::CommandSetDefaultGate), Command::THREAD_SAFE}}; CommandResponse WildcardMatch::AddFieldOne(const bess::pb::Field &field, - struct WmField *f) { + struct WmField *f, uint8_t type) { f->size = field.num_bytes(); if (f->size < 1 || f->size > MAX_FIELD_SIZE) { @@ -88,7 +104,10 @@ CommandResponse WildcardMatch::AddFieldOne(const bess::pb::Field &field, } } else if (field.position_case() == bess::pb::Field::kAttrName) { const char *attr = field.attr_name().c_str(); - f->attr_id = AddMetadataAttr(attr, f->size, Attribute::AccessMode::kRead); + f->attr_id = + (type == FieldType) + ? AddMetadataAttr(attr, f->size, Attribute::AccessMode::kRead) + : AddMetadataAttr(attr, f->size, Attribute::AccessMode::kWrite); if (f->attr_id < 0) { return CommandFailure(-f->attr_id, "add_metadata_attr() failed"); } @@ -119,57 +138,221 @@ CommandResponse WildcardMatch::Init(const bess::pb::WildcardMatchArg &arg) { f.pos = size_acc; - err = AddFieldOne(field, &f); + err = AddFieldOne(field, &f, FieldType); if (err.error().code() != 0) { return err; } size_acc += f.size; } - default_gate_ = DROP_GATE; total_key_size_ = align_ceil(size_acc, sizeof(uint64_t)); + entries_ = arg.entries(); + // reset size_acc + size_acc = 0; + for (int i = 0; i < arg.values_size(); i++) { + const auto &value = arg.values(i); + CommandResponse err; + values_.emplace_back(); + struct WmField &v = values_.back(); + + v.pos = size_acc; + + err = AddFieldOne(value, &v, ValueType); + if (err.error().code() != 0) { + return err; + } + + size_acc += v.size; + } + + total_value_size_ = align_ceil(size_acc, sizeof(uint64_t)); return CommandSuccess(); } inline gate_idx_t WildcardMatch::LookupEntry(const wm_hkey_t &key, - gate_idx_t def_gate) { + gate_idx_t def_gate, + bess::Packet *pkt) { struct WmData result = { - .priority = INT_MIN, .ogate = def_gate, - }; - + .priority = INT_MIN, .ogate = def_gate, .keyv = {{0}}}; for (auto &tuple : tuples_) { + if (tuple.occupied == 0) + continue; const auto &ht = tuple.ht; wm_hkey_t key_masked; + mask(key_masked, key, tuple.mask, total_key_size_); + WmData *entry = nullptr; + ht->find_dpdk(&key_masked, ((void **)&entry)); + if (entry && entry->priority >= result.priority) { + result = *entry; + } + } - mask(&key_masked, key, tuple.mask, total_key_size_); - - const auto *entry = - ht.Find(key_masked, wm_hash(total_key_size_), wm_eq(total_key_size_)); + /* if lookup was successful, then set values (if possible) */ + if (result.ogate != default_gate_) { + size_t num_values_ = values_.size(); + for (size_t i = 0; i < num_values_; i++) { + int value_size = values_[i].size; + int value_pos = values_[i].pos; + int value_off = values_[i].offset; + int value_attr_id = values_[i].attr_id; + uint8_t *data = pkt->head_data() + value_off; + + DLOG(INFO) << "off: " << (int)value_off << ", sz: " << value_size; + + if (value_attr_id < 0) { /* if it is offset-based */ + memcpy(data, reinterpret_cast(&result.keyv) + value_pos, + value_size); + } else { /* if it is attribute-based */ + typedef struct { + uint8_t bytes[bess::metadata::kMetadataAttrMaxSize]; + } value_t; + uint8_t *buf = (uint8_t *)&result.keyv + value_pos; + + DLOG(INFO) << "Setting value " << std::hex + << *(reinterpret_cast(buf)) + << " for attr_id: " << value_attr_id + << " of size: " << value_size + << " at value_pos: " << value_pos; + + switch (value_size) { + case 1: + set_attr(this, value_attr_id, pkt, *((uint8_t *)buf)); + break; + case 2: + set_attr(this, value_attr_id, pkt, + *((uint16_t *)((uint8_t *)buf))); + break; + case 4: + set_attr(this, value_attr_id, pkt, + *((uint32_t *)((uint8_t *)buf))); + break; + case 8: + set_attr(this, value_attr_id, pkt, + *((uint64_t *)((uint8_t *)buf))); + break; + default: { + void *mt_ptr = + _ptr_attr_with_offset(attr_offset(value_attr_id), pkt); + bess::utils::CopySmall(mt_ptr, buf, value_size); + } break; + } + } + } + } + return result.ogate; +} - if (entry && entry->second.priority >= result.priority) { - result = entry->second; +inline bool WildcardMatch::LookupBulkEntry(wm_hkey_t *key, gate_idx_t def_gate, + int packeti, gate_idx_t *Outgate, + int cnt, bess::PacketBatch *batch) { + bess::Packet *pkt = nullptr; + struct WmData *result[cnt]; + uint64_t prev_hitmask = 0; + uint64_t hitmask = 0; + wm_hkey_t key_masked[cnt]; + WmData *entry[cnt]; + wm_hkey_t **key_ptr[cnt]; + + for (auto tuple = tuples_.begin(); tuple != tuples_.end(); ++tuple) { + if (tuple->occupied == 0) + continue; + const auto &ht = tuple->ht; + mask_bulk(key, key_masked, (void **)key_ptr, tuple->mask, cnt, + total_key_size_); + int num = ht->lookup_bulk_data((const void **)key_ptr, cnt, &hitmask, + (void **)entry); + if (num == 0) + continue; + + for (int init = 0; (init < cnt) && (num); init++) { + if ((hitmask & ((uint64_t)1 << init))) { + if ((prev_hitmask & ((uint64_t)1 << init)) == 0) + result[init] = entry[init]; + else if ((prev_hitmask & ((uint64_t)1 << init)) && + (entry[init]->priority >= result[init]->priority)) { + result[init] = entry[init]; + } + + num--; + } } + prev_hitmask = prev_hitmask | hitmask; } - return result.ogate; + for (int init = 0; init < cnt; init++) { + /* if lookup was successful, then set values (if possible) */ + if (prev_hitmask && (prev_hitmask & ((uint64_t)1 << init))) { + pkt = batch->pkts()[packeti + init]; + size_t num_values_ = values_.size(); + for (size_t i = 0; i < num_values_; i++) { + int value_size = values_[i].size; + int value_pos = values_[i].pos; + int value_off = values_[i].offset; + int value_attr_id = values_[i].attr_id; + uint8_t *data = pkt->head_data() + value_off; + + DLOG(INFO) << "off: " << (int)value_off << ", sz: " << value_size + << std::endl; + if (value_attr_id < 0) { /* if it is offset-based */ + memcpy(data, + reinterpret_cast(&result[init]->keyv) + value_pos, + value_size); + } else { /* if it is attribute-based */ + typedef struct { + uint8_t bytes[bess::metadata::kMetadataAttrMaxSize]; + } value_t; + uint8_t *buf = (uint8_t *)&result[init]->keyv + value_pos; + + DLOG(INFO) << "Setting value " << std::hex + << *(reinterpret_cast(buf)) + << " for attr_id: " << value_attr_id + << " of size: " << value_size + << " at value_pos: " << value_pos << std::endl; + + switch (value_size) { + case 1: + set_attr(this, value_attr_id, pkt, *((uint8_t *)buf)); + break; + case 2: + set_attr(this, value_attr_id, pkt, + *((uint16_t *)((uint8_t *)buf))); + break; + case 4: + set_attr(this, value_attr_id, pkt, + *((uint32_t *)((uint8_t *)buf))); + break; + case 8: + set_attr(this, value_attr_id, pkt, + *((uint64_t *)((uint8_t *)buf))); + break; + default: { + void *mt_ptr = _ptr_attr_with_offset( + attr_offset(value_attr_id), pkt); + bess::utils::CopySmall(mt_ptr, buf, value_size); + } break; + } + } + } + Outgate[init] = result[init]->ogate; + } else + Outgate[init] = def_gate; + } + return 1; } void WildcardMatch::ProcessBatch(Context *ctx, bess::PacketBatch *batch) { gate_idx_t default_gate; - wm_hkey_t keys[bess::PacketBatch::kMaxBurst] __ymm_aligned; - int cnt = batch->cnt(); + gate_idx_t Outgate[cnt]; // Initialize the padding with zero for (int i = 0; i < cnt; i++) { keys[i].u64_arr[(total_key_size_ - 1) / 8] = 0; } - default_gate = ACCESS_ONCE(default_gate_); - for (const auto &field : fields_) { int offset; int pos = field.pos; @@ -196,9 +379,25 @@ void WildcardMatch::ProcessBatch(Context *ctx, bess::PacketBatch *batch) { } } - for (int i = 0; i < cnt; i++) { - bess::Packet *pkt = batch->pkts()[i]; - EmitPacket(ctx, pkt, LookupEntry(keys[i], default_gate)); + if(cnt>64) + { + int icnt=0; + for(int lcnt=0; lcnt=64) ? 64 : cnt-lcnt ; + LookupBulkEntry(&keys[lcnt], default_gate, lcnt, Outgate, icnt, batch); + for (int j = 0; j < icnt; j++) + { + EmitPacket(ctx, batch->pkts()[j+lcnt], Outgate[j]); + } + } + } + else + { + LookupBulkEntry(keys, default_gate, 0, Outgate, cnt, batch); + for (int j = 0; j < cnt; j++) { + EmitPacket(ctx, batch->pkts()[j], Outgate[j]); + } } } @@ -206,7 +405,9 @@ std::string WildcardMatch::GetDesc() const { int num_rules = 0; for (const auto &tuple : tuples_) { - num_rules += tuple.ht.Count(); + if (tuple.occupied == 0) + continue; + num_rules += tuple.ht->Count(); } return bess::utils::Format("%zu fields, %d rules", fields_.size(), num_rules); @@ -275,55 +476,99 @@ CommandResponse WildcardMatch::ExtractKeyMask(const T &arg, wm_hkey_t *key, return CommandSuccess(); } -int WildcardMatch::FindTuple(wm_hkey_t *mask) { - int i = 0; +template +CommandResponse WildcardMatch::ExtractValue(const T &arg, wm_hkey_t *keyv) { + if ((size_t)arg.valuesv_size() != values_.size()) { + return CommandFailure(EINVAL, "must specify %zu values", values_.size()); + } - for (const auto &tuple : tuples_) { - if (memcmp(&tuple.mask, mask, total_key_size_) == 0) { + memset(keyv, 0, sizeof(*keyv)); + + for (size_t i = 0; i < values_.size(); i++) { + int value_size = values_[i].size; + int value_pos = values_[i].pos; + + uint64_t v = 0; + + bess::pb::FieldData valuedata = arg.valuesv(i); + if (valuedata.encoding_case() == bess::pb::FieldData::kValueInt) { + if (!bess::utils::uint64_to_bin(&v, valuedata.value_int(), value_size, + false)) { + return CommandFailure(EINVAL, "idx %zu: not a correct %d-byte value", i, + value_size); + } + } else if (valuedata.encoding_case() == bess::pb::FieldData::kValueBin) { + bess::utils::Copy(reinterpret_cast(&v), + valuedata.value_bin().c_str(), + valuedata.value_bin().size()); + } + + // Use memcpy, not utils::Copy, to workaround the false positive warning + // in g++-8 + memcpy(reinterpret_cast(keyv) + value_pos, &v, value_size); + } + + return CommandSuccess(); +} + +int WildcardMatch::FindTuple(wm_hkey_t *mask) { + for (auto i = 0; i < MAX_TUPLES; i++) { + if ((tuples_[i].occupied) && + (memcmp(&tuples_[i].mask, mask, total_key_size_) == 0)) { return i; } - i++; } return -ENOENT; } int WildcardMatch::AddTuple(wm_hkey_t *mask) { - if (tuples_.size() >= MAX_TUPLES) { - return -ENOSPC; + CuckooMap *temp = nullptr; + for (int i = 0; i < MAX_TUPLES; i++) { + if (tuples_[i].occupied == 0) { + bess::utils::Copy(&tuples_[i].mask, mask, sizeof(*mask)); + tuples_[i].params.key_len = total_key_size_; + if (entries_) { + tuples_[i].params.entries = entries_; + } + temp = new CuckooMap( + 0, 0, &tuples_[i].params); + if (temp == nullptr) + return -ENOSPC; + if (temp->hash == 0) { + delete temp; + return -ENOSPC; + } + void *temp1 = tuples_[i].ht; + tuples_[i].ht = temp; + if (temp1) + delete ( + static_cast *>( + temp1)); + tuples_[i].occupied = 1; + return i; + } } - - tuples_.emplace_back(); - struct WmTuple &tuple = tuples_.back(); - bess::utils::Copy(&tuple.mask, mask, sizeof(*mask)); - - return int(tuples_.size() - 1); + return -ENOSPC; } -int WildcardMatch::DelEntry(int idx, wm_hkey_t *key) { - struct WmTuple &tuple = tuples_[idx]; - int ret = - tuple.ht.Remove(*key, wm_hash(total_key_size_), wm_eq(total_key_size_)); - if (ret) { - return ret; +bool WildcardMatch::DelEntry(int idx, wm_hkey_t *key) { + int ret = tuples_[idx].ht->Remove(*key, wm_hash(total_key_size_), + wm_eq(total_key_size_)); + if (ret >= 0) { + return true; } - - if (tuple.ht.Count() == 0) { - tuples_.erase(tuples_.begin() + idx); + if (tuples_[idx].ht->Count() == 0) { } - - return 0; + return false; } CommandResponse WildcardMatch::CommandAdd( const bess::pb::WildcardMatchCommandAddArg &arg) { gate_idx_t gate = arg.gate(); int priority = arg.priority(); - wm_hkey_t key = {{0}}; wm_hkey_t mask = {{0}}; - struct WmData data; - CommandResponse err = ExtractKeyMask(arg, &key, &mask); if (err.error().code() != 0) { return err; @@ -333,9 +578,13 @@ CommandResponse WildcardMatch::CommandAdd( return CommandFailure(EINVAL, "Invalid gate: %hu", gate); } + err = ExtractValue(arg, &(data.keyv)); + if (err.error().code() != 0) { + return err; + } + data.priority = priority; data.ogate = gate; - int idx = FindTuple(&mask); if (idx < 0) { idx = AddTuple(&mask); @@ -343,13 +592,10 @@ CommandResponse WildcardMatch::CommandAdd( return CommandFailure(-idx, "failed to add a new wildcard pattern"); } } - - auto *ret = tuples_[idx].ht.Insert(key, data, wm_hash(total_key_size_), - wm_eq(total_key_size_)); - if (ret == nullptr) { + struct WmData *data_t = new WmData(data); + int ret = tuples_[idx].ht->insert_dpdk(&key, data_t); + if (ret < 0) return CommandFailure(EINVAL, "failed to add a rule"); - } - return CommandSuccess(); } @@ -383,7 +629,10 @@ CommandResponse WildcardMatch::CommandClear(const bess::pb::EmptyArg &) { void WildcardMatch::Clear() { for (auto &tuple : tuples_) { - tuple.ht.Clear(); + if (tuple.occupied) { + tuple.occupied = 0; + tuple.ht->Clear(); + } } } @@ -405,17 +654,26 @@ CommandResponse WildcardMatch::GetInitialArg(const bess::pb::EmptyArg &) { // Retrieves a WildcardMatchConfig that would restore this module's // runtime configuration. CommandResponse WildcardMatch::GetRuntimeConfig(const bess::pb::EmptyArg &) { + std::pair entry; bess::pb::WildcardMatchConfig resp; using rule_t = bess::pb::WildcardMatchCommandAddArg; - + const wm_hkey_t *key = 0; + WmData *data; + uint32_t *next = 0; resp.set_default_gate(default_gate_); // Each tuple provides a single mask, which may have many data-matches. for (auto &tuple : tuples_) { + if (tuple.occupied == 0) + continue; wm_hkey_t mask = tuple.mask; // Each entry in the hash table has priority, ogate, and the data // (one datum per field, under the mask for this field). - for (auto &entry : tuple.ht) { + // using rte method + while ((tuple.ht->Iterate((const void **)&key, (void **)&data, next)) >= + (int)0) { + entry.first = *key; + entry.second = *data; // Create the rule instance rule_t *rule = resp.add_rules(); rule->set_priority(entry.second.priority); @@ -480,5 +738,14 @@ CommandResponse WildcardMatch::SetRuntimeConfig( return CommandSuccess(); } +void WildcardMatch::DeInit() { + for (auto &tuple : tuples_) { + if (!tuple.ht) + continue; + tuple.ht->DeInit(); + tuple.ht = NULL; + } +} + ADD_MODULE(WildcardMatch, "wm", "Multi-field classifier with a wildcard match table") diff --git a/core/modules/wildcard_match.h b/core/modules/wildcard_match.h index 5a8d9e8142..6e7703dcfa 100644 --- a/core/modules/wildcard_match.h +++ b/core/modules/wildcard_match.h @@ -39,12 +39,13 @@ #include "../pb/module_msg.pb.h" #include "../utils/cuckoo_map.h" -using bess::utils::HashResult; using bess::utils::CuckooMap; +using bess::utils::HashResult; -#define MAX_TUPLES 8 +#define MAX_TUPLES 16 #define MAX_FIELDS 8 #define MAX_FIELD_SIZE 8 +#define BULK_SIZE 32 static_assert(MAX_FIELD_SIZE <= sizeof(uint64_t), "field cannot be larger than 8 bytes"); @@ -54,11 +55,6 @@ static_assert(MAX_FIELD_SIZE <= sizeof(uint64_t), #error this code assumes little endian architecture (x86) #endif -struct WmData { - int priority; - gate_idx_t ogate; -}; - struct WmField { int attr_id; /* -1 for offset-based fields */ @@ -75,6 +71,12 @@ struct wm_hkey_t { uint64_t u64_arr[MAX_FIELDS]; }; +struct WmData { + int priority; + gate_idx_t ogate; + wm_hkey_t keyv; +}; + class wm_eq { public: explicit wm_eq(size_t len) : len_(len) {} @@ -127,6 +129,12 @@ class wm_hash { private: size_t len_; }; +struct rte_hash_parameters dpdk_params1 { + .name = "test2", .entries = 1 << 15, .reserved = 0, + .key_len = sizeof(wm_hkey_t), .hash_func = rte_hash_crc, + .hash_func_init_val = 0, .socket_id = (int)rte_socket_id(), + .extra_flag = RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY +}; class WildcardMatch final : public Module { public: @@ -135,12 +143,21 @@ class WildcardMatch final : public Module { static const Commands cmds; WildcardMatch() - : Module(), default_gate_(), total_key_size_(), fields_(), tuples_() { + : Module(), + default_gate_(), + total_key_size_(), + total_value_size_(), + fields_(), + values_(), + tuples_() { max_allowed_workers_ = Worker::kMaxWorkers; + { tuples_.resize(MAX_TUPLES); } } CommandResponse Init(const bess::pb::WildcardMatchArg &arg); + void DeInit() override; + void ProcessBatch(Context *ctx, bess::PacketBatch *batch) override; std::string GetDesc() const override; @@ -157,30 +174,49 @@ class WildcardMatch final : public Module { private: struct WmTuple { - CuckooMap ht; + bool occupied; + CuckooMap *ht; wm_hkey_t mask; + struct rte_hash_parameters params; + std::string hash_name; + WmTuple() : occupied(0), ht(0) { + params = dpdk_params1; + std::ostringstream address; + address << this; + hash_name = "Wild" + address.str(); + params.name = hash_name.c_str(); + } }; - gate_idx_t LookupEntry(const wm_hkey_t &key, gate_idx_t def_gate); + gate_idx_t LookupEntry(const wm_hkey_t &key, gate_idx_t def_gate, + bess::Packet *pkt); - CommandResponse AddFieldOne(const bess::pb::Field &field, struct WmField *f); + bool LookupBulkEntry(wm_hkey_t *key, gate_idx_t def_gate, int i, + gate_idx_t *Outgate, int cnt, bess::PacketBatch *batch); + + CommandResponse AddFieldOne(const bess::pb::Field &field, struct WmField *f, + uint8_t type); template CommandResponse ExtractKeyMask(const T &arg, wm_hkey_t *key, wm_hkey_t *mask); + template + CommandResponse ExtractValue(const T &arg, wm_hkey_t *keyv); int FindTuple(wm_hkey_t *mask); int AddTuple(wm_hkey_t *mask); - int DelEntry(int idx, wm_hkey_t *key); - + bool DelEntry(int idx, wm_hkey_t *key); void Clear(); - gate_idx_t default_gate_; - size_t total_key_size_; /* a multiple of sizeof(uint64_t) */ + size_t total_key_size_; /* a multiple of sizeof(uint64_t) */ + size_t total_value_size_; /* a multiple of sizeof(uint64_t) */ + size_t entries_; /* a power of 2 */ // TODO(melvinw): this can be refactored to use ExactMatchTable std::vector fields_; - std::vector tuples_; + std::vector values_; + std::vector tuples_; //[MAX_TUPLES]; + std::vector data_; }; #endif // BESS_MODULES_WILDCARDMATCH_H_ diff --git a/core/utils/cuckoo_map.h b/core/utils/cuckoo_map.h index c855b93dea..6213ce7d92 100644 --- a/core/utils/cuckoo_map.h +++ b/core/utils/cuckoo_map.h @@ -50,6 +50,8 @@ #include "../debug.h" #include "common.h" +#include +#include namespace bess { namespace utils { @@ -74,7 +76,13 @@ typedef uint32_t EntryIndex; template , typename E = std::equal_to> class CuckooMap { + private: + bool IsDpdk = false; + uint32_t key_len = 0; + rte_hash_parameters rt; + public: + struct rte_hash* hash = nullptr; typedef std::pair Entry; class iterator { @@ -149,20 +157,29 @@ class CuckooMap { }; CuckooMap(size_t reserve_buckets = kInitNumBucket, - size_t reserve_entries = kInitNumEntries) + size_t reserve_entries = kInitNumEntries, void* dpdk_params = 0) : bucket_mask_(reserve_buckets - 1), num_entries_(0), buckets_(reserve_buckets), entries_(reserve_entries), free_entry_indices_() { - // the number of buckets must be a power of 2 - CHECK_EQ(align_ceil_pow2(reserve_buckets), reserve_buckets); + if (dpdk_params) { + if (hash == NULL) { + rt = *((rte_hash_parameters*)dpdk_params); + hash = rte_hash_create(&rt); + if (hash == NULL) + return; + } + IsDpdk = true; + } else { + // the number of buckets must be a power of 2 + CHECK_EQ(align_ceil_pow2(reserve_buckets), reserve_buckets); - for (int i = reserve_entries - 1; i >= 0; --i) { - free_entry_indices_.push(i); + for (int i = reserve_entries - 1; i >= 0; --i) { + free_entry_indices_.push(i); + } } } - // Not allowing copying for now CuckooMap(CuckooMap&) = delete; CuckooMap& operator=(CuckooMap&) = delete; @@ -220,6 +237,50 @@ class CuckooMap { return DoEmplace(key, hasher, eq, std::move(value)); } + int insert_dpdk(const void* key, void* data = 0, hash_sig_t sig = 0) { + if (IsDpdk) { + if (data && !sig) + return rte_hash_add_key_data(hash, key, data); + if (data && sig) + return rte_hash_add_key_with_hash_data(hash, key, sig, data); + if (!data && !sig) + return rte_hash_add_key(hash, key); + } + return -1; + } + + int find_dpdk(const void* key, void** data = 0, + hash_sig_t sig = 0) + { + if (IsDpdk) { + if (data && !sig) + return rte_hash_lookup_data(hash, key, data); + if (data && sig) + return rte_hash_lookup_with_hash_data(hash, key, sig, data); + if (!data && !sig) + return rte_hash_lookup(hash, key); + if (!data && sig) + return rte_hash_lookup_with_hash(hash, key, sig); + } + return -1; + } + + int find_dpdk(const void* key, void** data = 0, + hash_sig_t sig = 0) const + { + if (IsDpdk) { + if (data && !sig) + return rte_hash_lookup_data(hash, key, data); + if (data && sig) + return rte_hash_lookup_with_hash_data(hash, key, sig, data); + if (!data && !sig) + return rte_hash_lookup(hash, key); + if (!data && sig) + return rte_hash_lookup_with_hash(hash, key, sig); + } + return -1; + } + // Emplace/update-in-place a key value pair // On success returns a pointer to the inserted entry, nullptr otherwise. // NOTE: when Emplace() returns nullptr, the constructor of `V` may not be @@ -255,6 +316,13 @@ class CuckooMap { // Remove the stored entry by the key // Return false if not exist. bool Remove(const K& key, const H& hasher = H(), const E& eq = E()) { + if (IsDpdk) { + int ret = rte_hash_del_key(hash, &key); + if (ret < 0) + return false; + else + return true; + } HashResult pri = Hash(key, hasher); if (RemoveFromBucket(pri, pri & bucket_mask_, key, eq)) { return true; @@ -267,6 +335,12 @@ class CuckooMap { } void Clear() { + if (IsDpdk) { + if (hash) { + rte_hash_reset(hash); + } + return; + } buckets_.clear(); entries_.clear(); @@ -286,7 +360,36 @@ class CuckooMap { } // Return the number of stored entries - size_t Count() const { return num_entries_; } + size_t Count() const { + if (IsDpdk) + return rte_hash_count(hash); + else + return num_entries_; + } + + void DeInit() { + if (IsDpdk) { + if (hash) { + rte_hash_free(hash); + hash = nullptr; + } + return; + } + } + + // bulk data look up bess func + int32_t lookup_bulk_data(const void** keys, uint32_t num_keys, + uint64_t* hit_mask, void* data[]) { + if (IsDpdk) + return rte_hash_lookup_bulk_data(hash, keys, num_keys, hit_mask, data); + return -1; + } + // iterate for dpdk hash + int32_t Iterate(const void** key, void** data, uint32_t* next) { + if (IsDpdk) + return rte_hash_iterate(hash, key, data, next); + return -1; + } protected: // Tunable macros diff --git a/core/utils/exact_match_table.h b/core/utils/exact_match_table.h index 7c0cfda426..f5238dd5fb 100644 --- a/core/utils/exact_match_table.h +++ b/core/utils/exact_match_table.h @@ -155,15 +155,17 @@ typedef std::vector> ExactMatchRuleFields; template class ExactMatchTable { public: + struct rte_hash_parameters dpdk_params { + .name = "test1", .entries = 1 << 15, .reserved = 0, + .key_len = sizeof(ExactMatchKey), .hash_func = rte_hash_crc, + .hash_func_init_val = 0, .socket_id = (int)rte_socket_id(), + .extra_flag = RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY + }; + using EmTable = CuckooMap; - ExactMatchTable() - : raw_key_size_(), - total_key_size_(), - num_fields_(), - fields_(), - table_() {} + : raw_key_size_(), total_key_size_(), num_fields_(), fields_() {} // Add a new rule. // @@ -183,10 +185,9 @@ class ExactMatchTable { if ((err = gather_key(fields, &key)).first != 0) { return err; } - - table_.Insert(key, val, ExactMatchKeyHash(total_key_size_), - ExactMatchKeyEq(total_key_size_)); - + const void *Key_t = (const void *)&key; + T *val_t = new T(val); + table_->insert_dpdk(Key_t, val_t); return MakeError(0); } @@ -207,8 +208,8 @@ class ExactMatchTable { return err; } - bool ret = table_.Remove(key, ExactMatchKeyHash(total_key_size_), - ExactMatchKeyEq(total_key_size_)); + bool ret = table_->Remove(key, ExactMatchKeyHash(total_key_size_), + ExactMatchKeyEq(total_key_size_)); if (!ret) { return MakeError(ENOENT, "rule doesn't exist"); } @@ -217,9 +218,11 @@ class ExactMatchTable { } // Remove all rules from the table. - void ClearRules() { table_.Clear(); } + void ClearRules() { table_->Clear(); } - size_t Size() const { return table_.Count(); } + void DeInit() { table_->DeInit(); } + + size_t Size() const { return table_->Count(); } // Extract an ExactMatchKey from `buf` based on the fields that have been // added to this table. @@ -272,23 +275,27 @@ class ExactMatchTable { // Returns the value if `key` matches a rule, otherwise `default_value`. T Find(const ExactMatchKey &key, const T &default_value) const { const auto &table = table_; - const auto *entry = table.Find(key, ExactMatchKeyHash(total_key_size_), - ExactMatchKeyEq(total_key_size_)); - return entry ? entry->second : default_value; + void *data = nullptr; + table->find_dpdk(&key, &data); + if (data) { + T data_t = *((T *)data); + return data_t; + } else + return default_value; } // Find entries for `n` `keys` in the table and store their values in in // `vals`. Keys without entries will have their corresponding entires in // `vals` set to `default_value`. - void Find(const ExactMatchKey *keys, T *vals, size_t n, - T default_value) const { + uint64_t Find(ExactMatchKey *keys, T **vals, int n) { const auto &table = table_; - for (size_t i = 0; i < n; i++) { - const auto *entry = - table.Find(keys[i], ExactMatchKeyHash(total_key_size_), - ExactMatchKeyEq(total_key_size_)); - vals[i] = entry ? entry->second : default_value; - } + uint64_t hit_mask = 0; + ExactMatchKey *key_ptr[n]; + for (int h = 0; h < n; h++) + key_ptr[h] = &keys[h]; + table->lookup_bulk_data((const void **)&key_ptr, n, &hit_mask, + (void **)vals); + return hit_mask; } uint32_t total_key_size() const { return total_key_size_; } @@ -318,9 +325,23 @@ class ExactMatchTable { // Returns the ith field. const ExactMatchField &get_field(size_t i) const { return fields_[i]; } - typename EmTable::iterator begin() { return table_.begin(); } + typename EmTable::iterator begin() { return table_->begin(); } + + typename EmTable::iterator end() { return table_->end(); } - typename EmTable::iterator end() { return table_.end(); } + void Init(uint32_t entries) { + std::ostringstream address; + address << &table_; + std::string name = "Exactmatch" + address.str(); + dpdk_params.name = name.c_str(); + dpdk_params.key_len = total_key_size(); + if (entries) { + dpdk_params.entries = entries; + } + table_.reset( + new CuckooMap( + 0, 0, &dpdk_params)); + } private: Error MakeError(int code, const std::string &msg = "") { @@ -438,23 +459,21 @@ class ExactMatchTable { f->pos = raw_key_size_; raw_key_size_ += f->size; total_key_size_ = align_ceil(raw_key_size_, sizeof(uint64_t)); - return MakeError(0); } // unaligend key size, used as an accumulator for calls to AddField() size_t raw_key_size_; - // aligned total key size size_t total_key_size_; - size_t num_fields_; ExactMatchField fields_[MAX_FIELDS]; - - EmTable table_; + std::unique_ptr< + CuckooMap> + table_; }; -} // namespace bess } // namespace utils +} // namespace bess #endif // BESS_UTILS_EXACT_MATCH_TABLE_H_ diff --git a/protobuf/bess_msg.proto b/protobuf/bess_msg.proto index 9e1b79a337..9b30de3447 100644 --- a/protobuf/bess_msg.proto +++ b/protobuf/bess_msg.proto @@ -42,6 +42,8 @@ import "error.proto"; package bess.pb; +option go_package = "github.com/omec-project/upf-epc/pfcpiface/bess_pb"; + message EmptyRequest { } diff --git a/protobuf/error.proto b/protobuf/error.proto index a018ee8e06..04b352c543 100644 --- a/protobuf/error.proto +++ b/protobuf/error.proto @@ -31,6 +31,8 @@ syntax = "proto3"; package bess.pb; +option go_package = "github.com/omec-project/upf-epc/pfcpiface/bess_pb"; + message Error { int32 code = 1; // 0 for success, errno (>0) for failure string errmsg = 2; diff --git a/protobuf/module_msg.proto b/protobuf/module_msg.proto index e00a463a35..3cbc328790 100644 --- a/protobuf/module_msg.proto +++ b/protobuf/module_msg.proto @@ -31,6 +31,9 @@ syntax = "proto3"; package bess.pb; + +option go_package = "github.com/omec-project/upf-epc/pfcpiface/bess_pb"; + import "util_msg.proto"; // Module-specific messages. @@ -69,6 +72,7 @@ message BPFCommandClearArg { message ExactMatchCommandAddArg { uint64 gate = 1; /// The gate to forward out packets that mach this rule. repeated FieldData fields = 2; /// The exact match values to check for + repeated FieldData values = 3; /// The exact match values to check for } /** @@ -377,6 +381,7 @@ message WildcardMatchCommandAddArg { int64 priority = 2; ///If a packet matches multiple rules, the rule with higher priority will be applied. If priorities are equal behavior is undefined. repeated FieldData values = 3; /// The values to check for in each field. repeated FieldData masks = 4; /// The bitmask for each field -- set `0x0` to ignore the field altogether. + repeated FieldData valuesv = 5; /// The values to check for in each fieldv. } /** @@ -501,6 +506,9 @@ message EtherEncapArg { message ExactMatchArg { repeated Field fields = 1; ///A list of ExactMatch Fields repeated FieldData masks = 2; /// mask(i) corresponds to the mask for field(i) + repeated Field values = 3; /// A list of ExactMatch Values + repeated FieldData masksv = 4; /// mask(i) corresponds to the mask for value(i) + uint64 entries = 5; } /** @@ -996,6 +1004,7 @@ message SourceArg { */ message IPChecksumArg { bool verify = 1; /// check checksum + bool hw = 2; /// enable hardware offload } /** @@ -1009,6 +1018,85 @@ message IPChecksumArg { */ message L4ChecksumArg { bool verify = 1; /// check checksum + bool hw = 2; /// enable hardware offload +} + +/** + * The GtpuEcho module processes the GTPv1 echo packet and prepares + * corresponding IP packet containing GTP echo response. It assumes + * Recovery IE is always zero. + * + * __Input Gates__: 1 + * __Output Gates__: 1 + */ +message GtpuEchoArg { + uint32 s1u_sgw_ip = 1; /// IP address of S1U interface +} + +/** + * The IPDefrag module scans the IP datagram and checks whether + * it is fragmented. It returns a fully reassembled datagram or + * an unfragmented IP datagram + * + * __Input Gates__: 1 + * __Output Gates__: 1 + */ +message IPDefragArg { + uint32 num_flows = 1; /// max number of flows the module can handle + int32 numa = 2; /// numa placement for ip frags memory management +} + +/** + * The IPDFrag module scans the IP datagram and checks whether + * it needs to be fragmented. + * + * __Input Gates__: 1 + * __Output Gates__: 1 + */ +message IPFragArg { + int32 mtu = 1; /// full Ethernet frame size (including CRC) for encapsulated ipv4 frag datagrams +} + +/** + * The Counter module has a command `add(...)` which takes one + * parameters. This function accepts the counter id of a + * session record. + * Example use in bessctl: `counter.add(ctr_id=0x1)` + */ +message CounterAddArg { + uint32 ctr_id = 1; /// counter id +} + +/** + * The Counter module has a command `remove(...)` which takes one + * parameter. This function accepts ctr_id, and removes the + * respective counter. + * Example use in bessctl: `counter.remove(ctr_id=0x1)` + */ +message CounterRemoveArg { + uint32 ctr_id = 1; /// counter id +} + +/** + * The Counter module counts the number of packets and bytes it passes + * + * __Input Gates__: 1 + * __Output Gates__: 1 +*/ +message CounterArg { + string name_id = 1; /// Name of the counter_id + bool check_exist = 2; /// verify each counter pre-exists before any operation (default = False) + uint32 total = 3; /// Total number of entries it can support +} + +/** + * The GtpuEncap module inserts GTP header in an ethernet frame + * + * __Input Gates__: 1 + * __Output Gates__: 1 +*/ +message GtpuEncapArg { + bool add_psc = 1; /// Add PDU session container in encap (default = False) } /** @@ -1151,6 +1239,8 @@ message VXLANEncapArg { */ message WildcardMatchArg { repeated Field fields = 1; /// A list of WildcardMatch fields. + repeated Field values = 2; /// A list of WildcardMatch values. + uint64 entries = 3; } /** @@ -1206,3 +1296,79 @@ message MplsPopArg { message WorkerSplitArg { map worker_gates = 1; // ogate -> worker mask } + +message QosArg { + repeated Field fields = 1; + repeated Field values = 2; + uint64 entries = 3; +} + +message QosCommandAddArg { + uint64 gate = 1; + uint64 cir = 2; + uint64 pir = 3; + uint64 cbs = 4; + uint64 pbs = 5; + uint64 ebs = 6; + oneof optional_deduct_len { + int64 deduct_len = 9; + } + repeated FieldData fields = 7; + repeated FieldData values = 8; +} + +message QosCommandDeleteArg { + repeated FieldData fields = 2; +} + +/** + * The function `clear()` for WildcardMatch takes no parameters, it clears + * all state in the WildcardMatch module (is equivalent to calling delete for all rules) + */ +message QosCommandClearArg { +} + +/** + * For traffic which does not match any rule in the WildcardMatch module, + * the `set_default_gate(...)` function specifies which gate to send this extra traffic to. + */ +message QosCommandSetDefaultGateArg { + uint64 gate = 1; +} + +message FlowMeasureArg { + string flag_attr_name = 1; + uint64 entries = 2; + bool leader = 3; // If true, this module will decide the buffer side +} +message FlowMeasureCommandReadArg { + bool clear = 1; // If true, the data will be all cleared after read + repeated double latency_percentiles = 2; /// ascending list of real numbers in [0.0, 100.0] + repeated double jitter_percentiles = 3; /// ascending list of real numbers in [0.0, 100.0] + uint64 flag_to_read = 4; /// Which buffer to read from +} +message FlowMeasureReadResponse { + message Statistic { + message Histogram { + uint64 count = 1; /// Total # of measured data points, including above_range + uint64 above_range = 2; /// # of data points for the "too large value" bucket + uint64 resolution_ns = 8; /// resolution of measured data + uint64 min_ns = 3; + uint64 avg_ns = 4; + uint64 max_ns = 5; + uint64 total_ns = 6; + repeated uint64 percentile_values_ns = 7; + } + uint64 fseid = 1; + uint64 pdr = 2; + Histogram latency = 3; + Histogram jitter = 4; + uint64 total_bytes = 11; + uint64 total_packets = 12; + } + repeated Statistic statistics = 1; +} +message FlowMeasureCommandFlipArg {} +message FlowMeasureFlipResponse { + uint64 old_flag = 1; +} diff --git a/protobuf/ports/port_msg.proto b/protobuf/ports/port_msg.proto index ff6e65ede2..7896a22a84 100644 --- a/protobuf/ports/port_msg.proto +++ b/protobuf/ports/port_msg.proto @@ -31,6 +31,8 @@ syntax = "proto3"; package bess.pb; +option go_package = "github.com/omec-project/upf-epc/pfcpiface/bess_pb"; + message PCAPPortArg { string dev = 1; } @@ -47,6 +49,15 @@ message PMDPortArg { bool vlan_offload_rx_strip = 5; bool vlan_offload_rx_filter = 6; bool vlan_offload_rx_qinq = 7; + oneof socket { + int32 socket_id = 8; + } + bool promiscuous_mode = 9; + bool hwcksum = 10; + + // N3 -> 3; N6 -> 6; N9 -> 9 + // [3] or [6, 9] + repeated uint32 flow_profiles = 11; } message UnixSocketPortArg { diff --git a/protobuf/service.proto b/protobuf/service.proto index ae1aba0533..706a283a46 100644 --- a/protobuf/service.proto +++ b/protobuf/service.proto @@ -33,6 +33,8 @@ import "bess_msg.proto"; package bess.pb; +option go_package = "github.com/omec-project/upf-epc/pfcpiface/bess_pb"; + service BESSControl { // ------------------------------------------------------------------------- diff --git a/protobuf/util_msg.proto b/protobuf/util_msg.proto index b1050abfd1..4df01fc67b 100644 --- a/protobuf/util_msg.proto +++ b/protobuf/util_msg.proto @@ -32,6 +32,8 @@ syntax="proto3"; package bess.pb; +option go_package = "github.com/omec-project/upf-epc/pfcpiface/bess_pb"; + /// The Field message represents one field in a packet -- either stored in metadata or in the packet body. message Field { oneof position {
${module_name}