From f29de0d2802a7f2d110637b9e7576cc1e5ce727e Mon Sep 17 00:00:00 2001 From: Hans Gaiser Date: Mon, 22 Sep 2014 10:34:32 +0200 Subject: [PATCH] Add more debug and weights --- random/src/connection.h | 4 +- random/src/main.cpp | 69 ++++++++++++++++++++------ random/src/objectness_location_prior.h | 8 +-- random/src/objectness_prior.h | 46 ++++++++++++++--- random/src/segment.h | 16 +++--- random/src/selection_prior.h | 20 +++++++- 6 files changed, 127 insertions(+), 36 deletions(-) diff --git a/random/src/connection.h b/random/src/connection.h index ccd48a4..f0ef24c 100644 --- a/random/src/connection.h +++ b/random/src/connection.h @@ -5,8 +5,8 @@ class Connection { public: - Connection(const Segment & a_, const Segment & b_, uint8_t flags) : - a(a_.id), b(b_.id), similarity(exp(Segment::computeSimilarity(a_, b_, flags))) + Connection(const Segment & a_, const Segment & b_, uint8_t flags, const cv::Mat & weights) : + a(a_.id), b(b_.id), similarity(exp(Segment::computeSimilarity(a_, b_, flags, weights))) { } diff --git a/random/src/main.cpp b/random/src/main.cpp index e04f823..609d903 100644 --- a/random/src/main.cpp +++ b/random/src/main.cpp @@ -13,12 +13,16 @@ using namespace boost::python; -//#define DEBUG 1 std::random_device rd_; std::mt19937 gen_(rd_()); -cv::Mat get_bboxes_(cv::Mat image, cv::Mat seg, cv::Mat edge, uint8_t flags, uint32_t n, std::string selection_prior) { +UniformPrior up; +LocationPrior lp; +ObjectnessPrior op; +ObjectnessLocationPrior olp; + +cv::Mat get_bboxes_(const cv::Mat & image, const cv::Mat & seg, const cv::Mat & edge, uint8_t flags, const cv::Mat & weights, uint32_t n, std::string selection_prior) { double max_id_; cv::minMaxIdx(seg, nullptr, &max_id_); int max_id = max_id_; @@ -82,24 +86,27 @@ cv::Mat get_bboxes_(cv::Mat image, cv::Mat seg, cv::Mat edge, uint8_t flags, uin SelectionPriorMap prior; if (selection_prior == "uniform") { - UniformPrior up; prior = up.computeSelectionPrior(image, segments); } else if (selection_prior == "location") { - LocationPrior lp; prior = lp.computeSelectionPrior(image, segments); } else if (selection_prior == "objectness") { - ObjectnessPrior op; prior = op.computeSelectionPrior(image, segments); } - else if (selection_prior == "objectness-location") { - ObjectnessLocationPrior olp; + else if (selection_prior == "objectness-slow") { prior = olp.computeSelectionPrior(image, segments); } else { throw std::runtime_error("Unknown selection prior method: " + selection_prior); } +#ifdef DEBUG + prior.visualize(seg); +#endif + +#ifdef DEBUG + cv::namedWindow("SelectionLikelihood", cv::WINDOW_AUTOSIZE); +#endif Segment s; for (uint32_t i = 0; i < n; i++) { @@ -108,13 +115,37 @@ cv::Mat get_bboxes_(cv::Mat image, cv::Mat seg, cv::Mat edge, uint8_t flags, uin cv::Rect r(s.min_p, s.max_p); while (stop.stop(image, r) == false) { +#ifdef DEBUG + cv::Mat red = edge * 0.5; + cv::Mat green; + s.mask.copyTo(green); + green *= 255; +#endif float sum = 0.f; std::vector connections; for (auto n: s.neighbours) { - connections.push_back(Connection(s, segments[n], flags)); + connections.push_back(Connection(s, segments[n], flags, weights)); sum += (connections.end() - 1)->similarity; } +#ifdef DEBUG + cv::Mat blue = cv::Mat::zeros(seg.size(), CV_8UC1); + float max_sim = 0.f; + for (auto & c: connections) { + if (c.similarity > max_sim) + max_sim = c.similarity; + } + for (auto & c: connections) { + blue += segments[c.b].mask * 255 * (c.similarity / max_sim); + std::cout << c.similarity << std::endl; + } + std::cout << std::endl; + cv::Mat visualization; + cv::merge({blue, green, red}, visualization); + cv::imshow("SelectionLikelihood", visualization); + cv::waitKey(); +#endif + std::uniform_real_distribution dis(0.f, sum); float rnd = dis(gen_); @@ -154,13 +185,13 @@ cv::Mat get_bboxes_(cv::Mat image, cv::Mat seg, cv::Mat edge, uint8_t flags, uin return bboxes; } -PyObject * get_bboxes(PyObject * image_, PyObject * seg_, PyObject * edge_, uint8_t flags, uint32_t n, std::string selection_prior) { +PyObject * get_bboxes(PyObject * image_, PyObject * seg_, PyObject * edge_, uint8_t flags, PyObject * weights_, uint32_t n, std::string selection_prior) { NDArrayConverter cvt; - cv::Mat image, seg, edge; - image = cvt.toMat(image_); - seg = cvt.toMat(seg_); - edge = cvt.toMat(edge_); - return cvt.toNDArray(get_bboxes_(image, seg, edge, flags, n, selection_prior)); + cv::Mat image = cvt.toMat(image_); + cv::Mat seg = cvt.toMat(seg_); + cv::Mat edge = cvt.toMat(edge_); + cv::Mat weights = cvt.toMat(weights_); + return cvt.toNDArray(get_bboxes_(image, seg, edge, flags, weights, n, selection_prior)); } static void init_ar() { @@ -186,6 +217,14 @@ int main(int argc, char * argv[]) { uint8_t iterations = atoi(argv[4]); std::string selection_prior = std::string(argv[5]); + cv::Mat weights(1, 5, CV_32FC1); + weights.at(0) = -0.00697891f; + weights.at(1) = 0.f; + weights.at(2) = 6.65155577f; + weights.at(3) = 1.16613659f; + weights.at(4) = -7.59423175f; + weights = cv::Mat::ones(1, 5, CV_32FC1); + // cv::namedWindow("Image", cv::WINDOW_NORMAL); // cv::imshow("Image", (seg == 338) * 255); // cv::waitKey(); @@ -193,7 +232,7 @@ int main(int argc, char * argv[]) { // cv::namedWindow("Image", cv::WINDOW_NORMAL); // while (cv::waitKey() != 'q') { std::clock_t begin = std::clock(); - cv::Mat bboxes = get_bboxes_(image, seg, edge, COLOR_SIMILARITY | TEXTURE_SIMILARITY/* | SIZE_SIMILARITY | BBOX_SIMILARITY*/, iterations, "objectness"); + cv::Mat bboxes = get_bboxes_(image, seg, edge, COLOR_SIMILARITY /*| TEXTURE_SIMILARITY*/ | SIZE_SIMILARITY | BBOX_SIMILARITY, weights, iterations, selection_prior); std::clock_t end = std::clock(); double elapsed_secs = double(end - begin) / CLOCKS_PER_SEC; std::cout << "Times passed in seconds: " << elapsed_secs << std::endl; diff --git a/random/src/objectness_location_prior.h b/random/src/objectness_location_prior.h index 8094cd4..e2ed0cc 100644 --- a/random/src/objectness_location_prior.h +++ b/random/src/objectness_location_prior.h @@ -42,8 +42,8 @@ class ObjectnessLocationPrior : public SelectionPrior { } } - uint32_t radius = (image.cols*image.cols + image.rows*image.rows) / 4; - cv::Point center = cv::Point(image.cols / 2, image.rows / 2); + //uint32_t radius = (image.cols*image.cols + image.rows*image.rows) / 4; + //cv::Point center = cv::Point(image.cols / 2, image.rows / 2); float sum = 0.f; for (const auto & s: segments) { @@ -51,8 +51,8 @@ class ObjectnessLocationPrior : public SelectionPrior { cv::rectangle(mask, s.min_p, s.max_p, cv::Scalar(1), CV_FILLED); float score = cv::mean(likelihood, mask)[0]; - cv::Point diff = (s.min_p + s.max_p) * 0.5 - center; - score *= exp(-diff.dot(diff) / float(radius) * deviation); + //cv::Point diff = (s.min_p + s.max_p) * 0.5 - center; + //score *= exp(-diff.dot(diff) / float(radius) * deviation); sum += score; prior.insert({s.id, score}); diff --git a/random/src/objectness_prior.h b/random/src/objectness_prior.h index f28e07d..67df663 100644 --- a/random/src/objectness_prior.h +++ b/random/src/objectness_prior.h @@ -6,17 +6,50 @@ class ObjectnessPrior : public SelectionPrior { public: - ObjectnessPrior() { + ObjectnessPrior() : + obj_bbox(2, 8, 1) + { obj.loadTrainedModel("/Users/hans/MasterThesis/code/cpp/random/model/ObjNessB2W8MAXBGR"); + obj_bbox.loadTrainedModel("/Users/hans/MasterThesis/code/cpp/random/model/ObjNessB2W8MAXBGR"); + } + + float scoreBBox(const cv::Mat & img) { + cv::Mat img_small; + if (img.size() != cv::Size(8,8)) { + cv::resize(img, img_small, cv::Size(8,8), 0, 0, cv::INTER_CUBIC); + } else + img_small = img; + + ValStructVec boxes; + obj_bbox.getObjBndBoxes(img_small, boxes, 1); + for (int i = 0; i < boxes.size(); i++) { + std::cout << "value: " << boxes(i) << " rect: " << boxes[i] << std::endl; + } + + return boxes(0); + } + + float boxOverlap(cv::Vec4i bbox1, cv::Vec4i bbox2) { + cv::Vec4i bi(max(bbox1[0],bbox2[0]), max(bbox1[1],bbox2[1]), min(bbox1[2],bbox2[2]), min(bbox1[3],bbox2[3])); + int iw = bi[2] - bi[0]; + int ih = bi[3] - bi[1]; + if (iw > 0 && ih > 0) { + float overlap = iw * ih / float((bbox2[2] - bbox2[0]) * (bbox2[3] - bbox2[1])); + return overlap; + } + + return 0.f; } virtual SelectionPriorMap computeSelectionPrior(const cv::Mat & image, const std::vector & segments) { SelectionPriorMap prior; - cv::Mat likelihood = cv::Mat::zeros(image.size(), CV_32FC1); ValStructVec boxes; obj.getObjBndBoxes(image, boxes, 130); + //std::cout << "estimated: " << +// scoreBBox(image(cv::Range(boxes[0][1], boxes[0][3]), cv::Range(boxes[0][0], boxes[0][2]))); +// std::cout << "actual: " << boxes(0) << std::endl; // cv::Mat test = cv::imread("../../../images/horse.jpg"); // cv::Mat test; image.copyTo(test); @@ -34,17 +67,15 @@ class ObjectnessPrior : public SelectionPrior { for (const auto & s: segments) { float score = 0.f; for (int i = 0; i < boxes.size(); i++) { - const cv::Vec4i & box = boxes[i]; - if (s.min_p.x > box[0] && s.min_p.x < box[2] && - s.min_p.y > box[1] && s.min_p.y < box[3] && - s.max_p.x > box[0] && s.max_p.y < box[2] && - s.max_p.y > box[1] && s.max_p.y < box[3]) + //std::cout << jaccardSimilarity(boxes[i], cv::Vec4i(s.min_p.x, s.min_p.y, s.max_p.x, s.max_p.y)) << std::endl; + if (boxOverlap(boxes[i], cv::Vec4i(s.min_p.x, s.min_p.y, s.max_p.x, s.max_p.y)) > 0.3) score += boxes(i); } sum += score; prior.insert({s.id, score}); } +// cv::Mat likelihood = cv::Mat::zeros(image.size(), CV_32FC1); // float max = boxes(0); // float min = boxes(boxes.size() - 1); // for (int i = 0; i < boxes.size(); i++) { @@ -82,4 +113,5 @@ class ObjectnessPrior : public SelectionPrior { protected: Objectness obj; + Objectness obj_bbox; }; diff --git a/random/src/segment.h b/random/src/segment.h index ac77aef..d213347 100644 --- a/random/src/segment.h +++ b/random/src/segment.h @@ -74,30 +74,32 @@ class Segment return 0.f; } - float similarity = 0.f; - + float color_similarity = 0.f; if (flags & COLOR_SIMILARITY) { for (uint64_t i = 0; i < a.color_hist.size(); i++) - similarity += std::min(a.color_hist[i], b.color_hist[i]); + color_similarity += weights.at(0) * std::min(a.color_hist[i], b.color_hist[i]); } + float texture_similarity = 0.f; if (flags & TEXTURE_SIMILARITY) { for (uint64_t i = 0; i < a.texture_hist.size(); i++) - similarity += std::min(a.texture_hist[i], b.texture_hist[i]); + texture_similarity += weights.at(1) * std::min(a.texture_hist[i], b.texture_hist[i]); } + float size_similarity = 0.f; if (flags & SIZE_SIMILARITY) { - similarity += 1.f - float(a.size + b.size) / a.im_size; + size_similarity += weights.at(2) * (1.f - float(a.size + b.size) / a.im_size); } + float bbox_similarity = 0.f; if (flags & BBOX_SIMILARITY) { cv::Point min_p(std::min(a.min_p.x, b.min_p.x), std::min(a.min_p.y, b.min_p.y)); cv::Point max_p(std::max(a.max_p.x, b.max_p.x), std::max(a.max_p.y, b.max_p.y)); int bbox_size = (max_p.x - min_p.x) * (max_p.y - min_p.y); - similarity += 1.f - float(bbox_size - a.size - b.size) / a.im_size; + bbox_similarity += weights.at(3) * (1.f - float(bbox_size - a.size - b.size) / a.im_size); } - return similarity; + return color_similarity + texture_similarity + size_similarity + bbox_similarity + weights.at(4); } static Segment merge(const Segment & a, const Segment & b) { diff --git a/random/src/selection_prior.h b/random/src/selection_prior.h index 6918842..554869a 100644 --- a/random/src/selection_prior.h +++ b/random/src/selection_prior.h @@ -18,7 +18,25 @@ class SelectionPriorMap : public std::unordered_map { } } - return -1; + return size() - 1; + } + + void visualize(const cv::Mat & seg) { + float max = 0.f; + for (auto & p: *this) { + if (p.second > max) { + max = p.second; + } + } + + cv::Mat visualization = cv::Mat::zeros(seg.size(), CV_8UC1); + for (auto & p: *this) { + visualization += (seg == p.first) * (p.second / max); + } + + cv::namedWindow("SelectionPrior", cv::WINDOW_AUTOSIZE); + cv::imshow("SelectionPrior", visualization); + cv::waitKey(); } protected: