diff --git a/random/src/connection.h b/random/src/connection.h index f0ef24c..c7cc9bc 100644 --- a/random/src/connection.h +++ b/random/src/connection.h @@ -1,12 +1,10 @@ #pragma once -#include "segment.h" - class Connection { public: - Connection(const Segment & a_, const Segment & b_, uint8_t flags, const cv::Mat & weights) : - a(a_.id), b(b_.id), similarity(exp(Segment::computeSimilarity(a_, b_, flags, weights))) + Connection(int a_, int b_, float sim) : + a(a_), b(b_), similarity(exp(sim)) { } diff --git a/random/src/features_segment.h b/random/src/features_segment.h new file mode 100644 index 0000000..3174d3b --- /dev/null +++ b/random/src/features_segment.h @@ -0,0 +1,154 @@ +#pragma once + +#include +#include +#include "segment.h" + +enum { + COLOR_SIMILARITY = 0x1, + TEXTURE_SIMILARITY = 0x2, + SIZE_SIMILARITY = 0x4, + BBOX_SIMILARITY = 0x8, +}; + +#define HIST_SIZE 25 +#define HIST_STEP int(255 / HIST_SIZE) + +class FeaturesSegment: public Segment +{ +public: + FeaturesSegment(int id, int nchannels, cv::Size size, uint8_t f, const cv::Mat & w) : + Segment(id, size), + nchannels(nchannels), + flags(f), + weights(w) + { + color_hist.assign(nchannels * HIST_SIZE, 0.f); + texture_hist.assign(HIST_SIZE, 0.f); + } + + virtual void addPoint(const cv::Mat & image, const cv::Mat & texture, cv::Point point) override { + Segment::addPoint(image, texture, point); + size++; + if (nchannels == 1) { + cv::Vec color = image.at>(point); + for (int i = 0; i < nchannels; i++) + color_hist[i*HIST_SIZE + color[i] / HIST_STEP]++; + } else if (nchannels == 3) { + cv::Vec color = image.at>(point); + for (int i = 0; i < nchannels; i++) + color_hist[i*HIST_SIZE + color[i] / HIST_STEP]++; + } + uint8_t text = texture.at(point); + texture_hist[text / HIST_STEP]++; + } + + virtual void finalizeSetup() override { + cv::normalize(color_hist, color_hist, 1, 0, cv::NORM_L1); + cv::normalize(texture_hist, texture_hist, 1, 0, cv::NORM_L1); + } + + virtual float computeSimilarity(const Segment * b_) override { + const FeaturesSegment * b = dynamic_cast(b_); + if (weights.type() != CV_32FC1) { + std::cerr << "Error! Weights are not one channel float " << weights << std::endl; + return 0.f; + } + if (weights.total() != 5) { + std::cerr << "Error! Not enough weights: " << weights << std::endl; + return 0.f; + } + if (im_size != b->im_size) { + std::cerr << "Error! Segments are from different images?" << std::endl; + return 0.f; + } + if (size == 0 || b->size == 0) { + std::cout << "Warning! Invalid segment." << std::endl; + return 0.f; + } + if (color_hist.size() != b->color_hist.size() || texture_hist.size() != b->texture_hist.size()) { + std::cout << "Error! Segments histograms don't match!" << std::endl; + return 0.f; + } + + float color_similarity = 0.f; + if (flags & COLOR_SIMILARITY) { + for (uint64_t i = 0; i < color_hist.size(); i++) + color_similarity += weights.at(0) * std::min(color_hist[i], b->color_hist[i]); + } + + float texture_similarity = 0.f; + if (flags & TEXTURE_SIMILARITY) { + for (uint64_t i = 0; i < texture_hist.size(); i++) + texture_similarity += weights.at(1) * std::min(texture_hist[i], b->texture_hist[i]); + } + + float size_similarity = 0.f; + if (flags & SIZE_SIMILARITY) { + size_similarity += weights.at(2) * (1.f - float(size + b->size) / im_size); + } + + float bbox_similarity = 0.f; + if (flags & BBOX_SIMILARITY) { + cv::Point combined_min(std::min(min_p.x, b->min_p.x), std::min(min_p.y, b->min_p.y)); + cv::Point combined_max(std::max(max_p.x, b->max_p.x), std::max(max_p.y, b->max_p.y)); + int bbox_size = (combined_max.x - combined_min.x) * (combined_max.y - combined_min.y); + bbox_similarity += weights.at(3) * (1.f - float(bbox_size - size - b->size) / im_size); + } + + return color_similarity + texture_similarity + size_similarity + bbox_similarity + weights.at(4); + } + + std::unique_ptr merge(const Segment * b_) override { + const FeaturesSegment * b = dynamic_cast(b_); + std::unique_ptr s_(new FeaturesSegment(-1, nchannels, im_size_cv, flags, weights)); + FeaturesSegment * s = dynamic_cast(s_.get()); + s->size = size + b->size; +#ifdef DEBUG + cv::bitwise_or(mask, b->mask, s->mask); +#endif + + s->min_p = cv::Point(std::min(min_p.x, b->min_p.x), std::min(min_p.y, b->min_p.y)); + s->max_p = cv::Point(std::max(max_p.x, b->max_p.x), std::max(max_p.y, b->max_p.y)); + + if (color_hist.size() != b->color_hist.size() || texture_hist.size() != b->texture_hist.size()) + return s_; + + for (uint64_t i = 0; i < color_hist.size(); i++) { + s->color_hist[i] = (color_hist[i] * size + b->size * b->color_hist[i]) / s->size; + } + + for (uint64_t i = 0; i < texture_hist.size(); i++) + s->texture_hist[i] = (size * texture_hist[i] + b->size * b->texture_hist[i]) / s->size; + //s->texture_hist = (size * texture_hist + b->size * b->texture_hist) / s->size; + + + s->history.insert(history.begin(), history.end()); + s->history.insert(b->history.begin(), b->history.end()); + + s->neighbours.insert(neighbours.begin(), neighbours.end()); + s->neighbours.insert(b->neighbours.begin(), b->neighbours.end()); + + for (const auto & h: s->history) + s->neighbours.erase(h); + + return s_; + } + + std::ostream & output(std::ostream & out) const { + out << ""; + } + + inline cv::Rect bbox() const { + return cv::Rect(min_p, max_p); + } + + std::vector color_hist; + std::vector texture_hist; + int nchannels; + uint8_t flags; + const cv::Mat & weights; +}; diff --git a/random/src/location_prior.h b/random/src/location_prior.h index 1587936..b388275 100644 --- a/random/src/location_prior.h +++ b/random/src/location_prior.h @@ -6,25 +6,27 @@ const float deviation = 4.f; class LocationPrior : public SelectionPrior { public: - virtual SelectionPriorMap computeSelectionPrior(const cv::Mat & image, const std::vector & segments) { + virtual SelectionPriorMap computeSelectionPrior(const cv::Mat & image, const std::vector> & segments) { SelectionPriorMap prior; uint32_t radius = (image.cols*image.cols + image.rows*image.rows) / 4; cv::Point center = cv::Point(image.cols / 2, image.rows / 2); float sum = 0.f; - for (const auto s: segments) { - if (s.empty()) + for (const auto & s: segments) { + if (s->empty()) continue; - cv::Point diff = (s.min_p + s.max_p) * 0.5 - center; + cv::Point diff = (s->min_p + s->max_p) * 0.5 - center; float likelihood = exp(-diff.dot(diff) / float(radius) * deviation); sum += likelihood; - prior.insert({s.id, likelihood}); + prior.insert({s->id, likelihood}); + std::cout << "size: " << prior.size() << " id: " << s->id << std::endl; } for (auto & p: prior) { p.second /= sum; + std::cout << "likelihood: " << p.second << std::endl; } return prior; diff --git a/random/src/main.cpp b/random/src/main.cpp index 609d903..382d346 100644 --- a/random/src/main.cpp +++ b/random/src/main.cpp @@ -1,14 +1,20 @@ +#define DEBUG 1 + //#include "CVBoostConverter.hpp" #include +#include +#include #include "conversion.h" #include +#include "features_segment.h" #include "connection.h" #include "adjacency.h" #include "uniform_prior.h" #include "location_prior.h" #include "objectness_prior.h" #include "objectness_location_prior.h" + #include "random_stopping_criterion.h" using namespace boost::python; @@ -27,12 +33,12 @@ cv::Mat get_bboxes_(const cv::Mat & image, const cv::Mat & seg, const cv::Mat & cv::minMaxIdx(seg, nullptr, &max_id_); int max_id = max_id_; - std::vector segments; + std::vector> segments; segments.reserve(max_id); int nchannels = image.channels(); cv::Size size = image.size(); for (int i = 0; i <= max_id; i++) { - segments.push_back(Segment(i, nchannels, size)); + segments.push_back(std::unique_ptr(new FeaturesSegment(i, nchannels, size, flags, weights))); } { @@ -41,17 +47,14 @@ cv::Mat get_bboxes_(const cv::Mat & image, const cv::Mat & seg, const cv::Mat & for (int j = 0; j < image.cols; j++) { cv::Point p(j, i); uint16_t id = seg.at(p); - if (nchannels == 1) - segments[id].addPoint(image.at>(p), edge.at(p), p); - else if (nchannels == 3) - segments[id].addPoint(image.at(p), edge.at(p), p); + segments[id]->addPoint(image, edge, p); if (i < image.rows - 1) { uint16_t n = seg.at(i+1, j); if (n != id && adjacency.get(id, n) == false) { adjacency.get(id, n) = true; - segments[id].addNeighbour(n); - segments[n].addNeighbour(id); + segments[id]->addNeighbour(n); + segments[n]->addNeighbour(id); } } @@ -59,8 +62,8 @@ cv::Mat get_bboxes_(const cv::Mat & image, const cv::Mat & seg, const cv::Mat & uint16_t n = seg.at(i, j+1); if (n != id && adjacency.get(id, n) == false) { adjacency.get(id, n) = true; - segments[id].addNeighbour(n); - segments[n].addNeighbour(id); + segments[id]->addNeighbour(n); + segments[n]->addNeighbour(id); } } } @@ -70,13 +73,13 @@ cv::Mat get_bboxes_(const cv::Mat & image, const cv::Mat & seg, const cv::Mat & cv::Mat bboxes; float similarity_sum = 0.f; for (auto & s: segments) { - s.normalizeHistogram(); + s->finalizeSetup(); cv::Mat bbox = cv::Mat(1, 4, CV_32SC1); - bbox.at(0) = s.min_p.x; - bbox.at(1) = s.min_p.y; - bbox.at(2) = s.max_p.x; - bbox.at(3) = s.max_p.y; + bbox.at(0) = s->min_p.x; + bbox.at(1) = s->min_p.y; + bbox.at(2) = s->max_p.x; + bbox.at(3) = s->max_p.y; if (bboxes.empty()) bboxes = bbox; else @@ -108,25 +111,31 @@ cv::Mat get_bboxes_(const cv::Mat & image, const cv::Mat & seg, const cv::Mat & cv::namedWindow("SelectionLikelihood", cv::WINDOW_AUTOSIZE); #endif - Segment s; for (uint32_t i = 0; i < n; i++) { - s = segments[prior.poll()]; + std::unique_ptr & s = segments[prior.poll()]; RandomStoppingCriterion stop; - cv::Rect r(s.min_p, s.max_p); + cv::Rect r(s->min_p, s->max_p); while (stop.stop(image, r) == false) { #ifdef DEBUG cv::Mat red = edge * 0.5; cv::Mat green; - s.mask.copyTo(green); + s->mask.copyTo(green); green *= 255; + std::cout << "history:" << std::endl; + for (const auto & h: s->history) + std::cout << h << ", "; + std::cout << std::endl; #endif float sum = 0.f; std::vector connections; - for (auto n: s.neighbours) { - connections.push_back(Connection(s, segments[n], flags, weights)); + std::cout << "neighbours:" << std::endl; + for (auto n: s->neighbours) { + connections.push_back(Connection(s->id, n, s->computeSimilarity(segments[n].get()))); sum += (connections.end() - 1)->similarity; + std::cout << n << ", "; } + std::cout << std::endl; #ifdef DEBUG cv::Mat blue = cv::Mat::zeros(seg.size(), CV_8UC1); @@ -136,9 +145,19 @@ cv::Mat get_bboxes_(const cv::Mat & image, const cv::Mat & seg, const cv::Mat & max_sim = c.similarity; } for (auto & c: connections) { - blue += segments[c.b].mask * 255 * (c.similarity / max_sim); + blue += segments[c.b]->mask * 255 * (c.similarity / max_sim); std::cout << c.similarity << std::endl; } + + for (int i = 0; i < blue.total(); i++) { + if (blue.at(i) && green.at(i)) { + // std::cout << "blue: " << (int)blue.at(i) << " green: " << (int)green.at(i) << std::endl; + for (auto & c: connections) { + if (segments[c.b]->mask.at(i)) + std::cout << "<<<<<<<<<<<<<<<<<<< " << c.b << " <<<<<<<<<<<<<<<<<<<<<" << std::endl; + } + } + } std::cout << std::endl; cv::Mat visualization; cv::merge({blue, green, red}, visualization); @@ -153,17 +172,17 @@ cv::Mat get_bboxes_(const cv::Mat & image, const cv::Mat & seg, const cv::Mat & for (auto & c: connections) { sum += c.similarity; if (sum >= rnd) { - s = Segment::merge(s, segments[c.b]); + s = s->merge(segments[c.b].get()); break; } } } cv::Mat bbox = cv::Mat(1, 4, CV_32SC1); - bbox.at(0) = s.min_p.x; - bbox.at(1) = s.min_p.y; - bbox.at(2) = s.max_p.x; - bbox.at(3) = s.max_p.y; + bbox.at(0) = s->min_p.x; + bbox.at(1) = s->min_p.y; + bbox.at(2) = s->max_p.x; + bbox.at(3) = s->max_p.y; if (bboxes.empty()) bboxes = bbox; else @@ -176,7 +195,7 @@ cv::Mat get_bboxes_(const cv::Mat & image, const cv::Mat & seg, const cv::Mat & cv::cvtColor(draw, draw, cv::COLOR_BGR2GRAY); cv::Rect r(cv::Point(bboxes.at(0), bboxes.at(1)), cv::Point(bboxes.at(2), bboxes.at(3))); cv::rectangle(draw, r, cv::Scalar(255)); - cv::addWeighted(draw, 0.5, s.mask * 255, 0.5, 0.0, draw); + cv::addWeighted(draw, 0.5, segments[n - 1]->mask * 255, 0.5, 0.0, draw); cv::imshow("Image", draw); std::cout << bboxes << std::endl; diff --git a/random/src/objectness_location_prior.h b/random/src/objectness_location_prior.h index e2ed0cc..f477a89 100644 --- a/random/src/objectness_location_prior.h +++ b/random/src/objectness_location_prior.h @@ -10,7 +10,7 @@ class ObjectnessLocationPrior : public SelectionPrior { obj.loadTrainedModel("/Users/hans/MasterThesis/code/cpp/random/model/ObjNessB2W8MAXBGR"); } - virtual SelectionPriorMap computeSelectionPrior(const cv::Mat & image, const std::vector & segments) { + virtual SelectionPriorMap computeSelectionPrior(const cv::Mat & image, const std::vector> & segments) { SelectionPriorMap prior; cv::Mat likelihood = cv::Mat::zeros(image.size(), CV_32FC1); @@ -48,14 +48,14 @@ class ObjectnessLocationPrior : public SelectionPrior { float sum = 0.f; for (const auto & s: segments) { cv::Mat mask = cv::Mat::zeros(image.size(), CV_8UC1); - cv::rectangle(mask, s.min_p, s.max_p, cv::Scalar(1), CV_FILLED); + cv::rectangle(mask, s->min_p, s->max_p, cv::Scalar(1), CV_FILLED); float score = cv::mean(likelihood, mask)[0]; //cv::Point diff = (s.min_p + s.max_p) * 0.5 - center; //score *= exp(-diff.dot(diff) / float(radius) * deviation); sum += score; - prior.insert({s.id, score}); + prior.insert({s->id, score}); } for (auto & p: prior) { diff --git a/random/src/objectness_prior.h b/random/src/objectness_prior.h index 67df663..5129a70 100644 --- a/random/src/objectness_prior.h +++ b/random/src/objectness_prior.h @@ -41,7 +41,7 @@ class ObjectnessPrior : public SelectionPrior { return 0.f; } - virtual SelectionPriorMap computeSelectionPrior(const cv::Mat & image, const std::vector & segments) { + virtual SelectionPriorMap computeSelectionPrior(const cv::Mat & image, const std::vector> & segments) { SelectionPriorMap prior; @@ -68,11 +68,11 @@ class ObjectnessPrior : public SelectionPrior { float score = 0.f; for (int i = 0; i < boxes.size(); i++) { //std::cout << jaccardSimilarity(boxes[i], cv::Vec4i(s.min_p.x, s.min_p.y, s.max_p.x, s.max_p.y)) << std::endl; - if (boxOverlap(boxes[i], cv::Vec4i(s.min_p.x, s.min_p.y, s.max_p.x, s.max_p.y)) > 0.3) + if (boxOverlap(boxes[i], cv::Vec4i(s->min_p.x, s->min_p.y, s->max_p.x, s->max_p.y)) > 0.3) score += boxes(i); } sum += score; - prior.insert({s.id, score}); + prior.insert({s->id, score}); } // cv::Mat likelihood = cv::Mat::zeros(image.size(), CV_32FC1); diff --git a/random/src/segment.h b/random/src/segment.h index d213347..3cad3e4 100644 --- a/random/src/segment.h +++ b/random/src/segment.h @@ -2,46 +2,25 @@ #include -enum { - COLOR_SIMILARITY = 0x1, - TEXTURE_SIMILARITY = 0x2, - SIZE_SIMILARITY = 0x4, - BBOX_SIMILARITY = 0x8, -}; - -#define HIST_SIZE 25 -#define HIST_STEP int(255 / HIST_SIZE) - class Segment { public: - Segment() {} - Segment(int id, int nchannels, cv::Size size) : + Segment(int id, cv::Size size) : #ifdef DEBUG mask(cv::Mat::zeros(size, CV_8UC1)), #endif + id(id), size(0), - im_size(size.area()), - im_size_cv(size), min_p(size.width, size.height), max_p(0, 0), - id(id), - nchannels(nchannels) + im_size(size.area()), + im_size_cv(size) { - color_hist.assign(nchannels * HIST_SIZE, 0.f); - texture_hist.assign(HIST_SIZE, 0.f); history.insert(id); } - template - void addPoint(cv::Vec<_Tp, n> col, uint8_t text, cv::Point point) { + virtual void addPoint(const cv::Mat & image, const cv::Mat & texture, cv::Point point) { size++; - - for (int i = 0; i < n; i++) - color_hist[i*HIST_SIZE + col[i] / HIST_STEP]++; - - texture_hist[text / HIST_STEP]++; - #ifdef DEBUG mask.at(point) = 1; #endif @@ -54,115 +33,32 @@ class Segment neighbours.insert(n); } - void normalizeHistogram() { - cv::normalize(color_hist, color_hist, 1, 0, cv::NORM_L1); - cv::normalize(texture_hist, texture_hist, 1, 0, cv::NORM_L1); - } + virtual float computeSimilarity(const Segment * b) = 0; - static float computeSimilarity(const Segment & a, const Segment & b, uint8_t flags) - { - if (a.im_size != b.im_size) { - std::cerr << "Error! Segments are from different images?" << std::endl; - return 0.f; - } - if (a.size == 0 || b.size == 0) { - std::cout << "Warning! Invalid segment." << std::endl; - return 0.f; - } - if (a.color_hist.size() != b.color_hist.size() || a.texture_hist.size() != b.texture_hist.size()) { - std::cout << "Error! Segments histograms don't match!" << std::endl; - return 0.f; - } - - float color_similarity = 0.f; - if (flags & COLOR_SIMILARITY) { - for (uint64_t i = 0; i < a.color_hist.size(); i++) - color_similarity += weights.at(0) * std::min(a.color_hist[i], b.color_hist[i]); - } - - float texture_similarity = 0.f; - if (flags & TEXTURE_SIMILARITY) { - for (uint64_t i = 0; i < a.texture_hist.size(); i++) - texture_similarity += weights.at(1) * std::min(a.texture_hist[i], b.texture_hist[i]); - } - - float size_similarity = 0.f; - if (flags & SIZE_SIMILARITY) { - size_similarity += weights.at(2) * (1.f - float(a.size + b.size) / a.im_size); - } - - float bbox_similarity = 0.f; - if (flags & BBOX_SIMILARITY) { - cv::Point min_p(std::min(a.min_p.x, b.min_p.x), std::min(a.min_p.y, b.min_p.y)); - cv::Point max_p(std::max(a.max_p.x, b.max_p.x), std::max(a.max_p.y, b.max_p.y)); - int bbox_size = (max_p.x - min_p.x) * (max_p.y - min_p.y); - bbox_similarity += weights.at(3) * (1.f - float(bbox_size - a.size - b.size) / a.im_size); - } - - return color_similarity + texture_similarity + size_similarity + bbox_similarity + weights.at(4); - } - - static Segment merge(const Segment & a, const Segment & b) { - Segment s(-1, a.nchannels, a.im_size_cv); - s.size = a.size + b.size; -#ifdef DEBUG - cv::bitwise_or(a.mask, b.mask, s.mask); -#endif + virtual std::unique_ptr merge(const Segment * b) = 0; - if (a.color_hist.size() != b.color_hist.size() || a.texture_hist.size() != b.texture_hist.size()) - return s; + virtual void finalizeSetup() = 0; - for (uint64_t i = 0; i < a.color_hist.size(); i++) { - s.color_hist[i] = (a.color_hist[i] * a.size + b.size * b.color_hist[i]) / s.size; - } - - for (uint64_t i = 0; i < a.texture_hist.size(); i++) - s.texture_hist[i] = (a.size * a.texture_hist[i] + b.size * b.texture_hist[i]) / s.size; - //s.texture_hist = (a.size * a.texture_hist + b.size * b.texture_hist) / s.size; - - s.min_p = cv::Point(std::min(a.min_p.x, b.min_p.x), std::min(a.min_p.y, b.min_p.y)); - s.max_p = cv::Point(std::max(a.max_p.x, b.max_p.x), std::max(a.max_p.y, b.max_p.y)); - - s.history.insert(a.history.begin(), a.history.end()); - s.history.insert(b.history.begin(), b.history.end()); - - s.neighbours.insert(a.neighbours.begin(), a.neighbours.end()); - s.neighbours.insert(b.neighbours.begin(), b.neighbours.end()); - - for (const auto & h: s.history) - s.neighbours.erase(h); - - return s; - } - - friend std::ostream& operator<<(std::ostream &out, Segment & s) { - out << ""; - } - - inline cv::Rect bbox() { - return cv::Rect(min_p, max_p); - } + virtual std::ostream & output(std::ostream &) const = 0; inline bool empty() const { return size == 0; } + int size; std::set neighbours; std::set history; #ifdef DEBUG cv::Mat mask; #endif - int size; - int im_size; - cv::Size im_size_cv; - std::vector color_hist; - std::vector texture_hist; + int id; cv::Point min_p; cv::Point max_p; - int id; - int nchannels; + int im_size; + cv::Size im_size_cv; }; +std::ostream & operator<<(std::ostream & os, const Segment & b) { + return b.output(os); +} + diff --git a/random/src/selection_prior.h b/random/src/selection_prior.h index 554869a..edcba7b 100644 --- a/random/src/selection_prior.h +++ b/random/src/selection_prior.h @@ -51,5 +51,5 @@ std::uniform_real_distribution SelectionPriorMap::dis(0.f, 1.f); class SelectionPrior { public: - virtual SelectionPriorMap computeSelectionPrior(const cv::Mat & image, const std::vector & segments) = 0; + virtual SelectionPriorMap computeSelectionPrior(const cv::Mat & image, const std::vector> & segments) = 0; }; diff --git a/random/src/uniform_prior.h b/random/src/uniform_prior.h index 4d945ff..52f44b7 100644 --- a/random/src/uniform_prior.h +++ b/random/src/uniform_prior.h @@ -4,16 +4,16 @@ class UniformPrior : public SelectionPrior { public: - virtual SelectionPriorMap computeSelectionPrior(const cv::Mat & image, const std::vector & segments) { + virtual SelectionPriorMap computeSelectionPrior(const cv::Mat & image, const std::vector> & segments) { SelectionPriorMap prior; int sum = 0; - for (const auto s: segments) { - if (s.empty()) + for (const auto & s: segments) { + if (s->empty()) continue; sum++; - prior.insert({s.id, 1}); + prior.insert({s->id, 1}); } for (auto & p: prior) {