Add more debug and weights

random/src/connection.h

@@ -5,8 +5,8 @@
 class Connection
 {
 public:
-	Connection(const Segment & a_, const Segment & b_, uint8_t flags) :
-		a(a_.id), b(b_.id), similarity(exp(Segment::computeSimilarity(a_, b_, flags)))
+	Connection(const Segment & a_, const Segment & b_, uint8_t flags, const cv::Mat & weights) :
+		a(a_.id), b(b_.id), similarity(exp(Segment::computeSimilarity(a_, b_, flags, weights)))
 	{
 	}
 

random/src/main.cpp

@@ -13,12 +13,16 @@
 
 using namespace boost::python;
 
-//#define DEBUG 1
 
 std::random_device rd_;
 std::mt19937 gen_(rd_());
 
-cv::Mat get_bboxes_(cv::Mat image, cv::Mat seg, cv::Mat edge, uint8_t flags, uint32_t n, std::string selection_prior) {
+UniformPrior up;
+LocationPrior lp;
+ObjectnessPrior op;
+ObjectnessLocationPrior olp;
+
+cv::Mat get_bboxes_(const cv::Mat & image, const cv::Mat & seg, const cv::Mat & edge, uint8_t flags, const cv::Mat & weights, uint32_t n, std::string selection_prior) {
 	double max_id_;
 	cv::minMaxIdx(seg, nullptr, &max_id_);
 	int max_id = max_id_;
@@ -82,24 +86,27 @@ cv::Mat get_bboxes_(cv::Mat image, cv::Mat seg, cv::Mat edge, uint8_t flags, uin
 	SelectionPriorMap prior;
 
 	if (selection_prior == "uniform") {
-		UniformPrior up;
 		prior = up.computeSelectionPrior(image, segments);
 	}
 	else if (selection_prior == "location") {
-		LocationPrior lp;
 		prior = lp.computeSelectionPrior(image, segments);
 	}
 	else if (selection_prior == "objectness") {
-		ObjectnessPrior op;
 		prior = op.computeSelectionPrior(image, segments);
 	}
-	else if (selection_prior == "objectness-location") {
-		ObjectnessLocationPrior olp;
+	else if (selection_prior == "objectness-slow") {
 		prior = olp.computeSelectionPrior(image, segments);
 	}
 	else {
 		throw std::runtime_error("Unknown selection prior method: " + selection_prior);
 	}
+#ifdef DEBUG
+	prior.visualize(seg);
+#endif
+
+#ifdef DEBUG
+	cv::namedWindow("SelectionLikelihood", cv::WINDOW_AUTOSIZE);
+#endif
 
 	Segment s;
 	for (uint32_t i = 0; i < n; i++) {
@@ -108,13 +115,37 @@ cv::Mat get_bboxes_(cv::Mat image, cv::Mat seg, cv::Mat edge, uint8_t flags, uin
 		cv::Rect r(s.min_p, s.max_p);
 
 		while (stop.stop(image, r) == false) {
+#ifdef DEBUG
+			cv::Mat red = edge * 0.5;
+			cv::Mat green;
+			s.mask.copyTo(green);
+			green *= 255;
+#endif
 			float sum = 0.f;
 			std::vector<Connection> connections;
 			for (auto n: s.neighbours) {
-				connections.push_back(Connection(s, segments[n], flags));
+				connections.push_back(Connection(s, segments[n], flags, weights));
 				sum += (connections.end() - 1)->similarity;
 			}
 
+#ifdef DEBUG
+			cv::Mat blue = cv::Mat::zeros(seg.size(), CV_8UC1);
+			float max_sim = 0.f;
+			for (auto & c: connections) {
+				if (c.similarity > max_sim)
+					max_sim = c.similarity;
+			}
+			for (auto & c: connections) {
+				blue += segments[c.b].mask * 255 * (c.similarity / max_sim);
+				std::cout << c.similarity << std::endl;
+			}
+			std::cout << std::endl;
+			cv::Mat visualization;
+			cv::merge({blue, green, red}, visualization);
+			cv::imshow("SelectionLikelihood", visualization);
+			cv::waitKey();
+#endif
+
 			std::uniform_real_distribution<float> dis(0.f, sum);
 			float rnd = dis(gen_);
 
@@ -154,13 +185,13 @@ cv::Mat get_bboxes_(cv::Mat image, cv::Mat seg, cv::Mat edge, uint8_t flags, uin
 	return bboxes;
 }
 
-PyObject * get_bboxes(PyObject * image_, PyObject * seg_, PyObject * edge_, uint8_t flags, uint32_t n, std::string selection_prior) {
+PyObject * get_bboxes(PyObject * image_, PyObject * seg_, PyObject * edge_, uint8_t flags, PyObject * weights_, uint32_t n, std::string selection_prior) {
 	NDArrayConverter cvt;
-	cv::Mat image, seg, edge;
-	image = cvt.toMat(image_);
-	seg = cvt.toMat(seg_);
-	edge = cvt.toMat(edge_);
-	return cvt.toNDArray(get_bboxes_(image, seg, edge, flags, n, selection_prior));
+	cv::Mat image   = cvt.toMat(image_);
+	cv::Mat seg     = cvt.toMat(seg_);
+	cv::Mat edge    = cvt.toMat(edge_);
+	cv::Mat weights = cvt.toMat(weights_);
+	return cvt.toNDArray(get_bboxes_(image, seg, edge, flags, weights, n, selection_prior));
 }
 
 static void init_ar() {
@@ -186,14 +217,22 @@ int main(int argc, char * argv[]) {
 	uint8_t iterations = atoi(argv[4]);
 	std::string selection_prior = std::string(argv[5]);
 
+	cv::Mat weights(1, 5, CV_32FC1);
+	weights.at<float>(0) = -0.00697891f;
+	weights.at<float>(1) = 0.f;
+	weights.at<float>(2) = 6.65155577f;
+	weights.at<float>(3) = 1.16613659f;
+	weights.at<float>(4) = -7.59423175f;
+	weights = cv::Mat::ones(1, 5, CV_32FC1);
+
 // 	cv::namedWindow("Image", cv::WINDOW_NORMAL);
 // 	cv::imshow("Image", (seg == 338) * 255);
 // 	cv::waitKey();
 
 // 	cv::namedWindow("Image", cv::WINDOW_NORMAL);
 //	while (cv::waitKey() != 'q') {
 		std::clock_t begin = std::clock();
-		cv::Mat bboxes = get_bboxes_(image, seg, edge, COLOR_SIMILARITY | TEXTURE_SIMILARITY/* | SIZE_SIMILARITY | BBOX_SIMILARITY*/, iterations, "objectness"); 
+		cv::Mat bboxes = get_bboxes_(image, seg, edge, COLOR_SIMILARITY /*| TEXTURE_SIMILARITY*/  | SIZE_SIMILARITY | BBOX_SIMILARITY, weights, iterations, selection_prior); 
 		std::clock_t end = std::clock();
 		double elapsed_secs = double(end - begin) / CLOCKS_PER_SEC;
 		std::cout << "Times passed in seconds: " << elapsed_secs << std::endl;

random/src/objectness_location_prior.h

@@ -42,17 +42,17 @@ class ObjectnessLocationPrior : public SelectionPrior {
 			}
 		}
 
-		uint32_t radius = (image.cols*image.cols + image.rows*image.rows) / 4;
-		cv::Point center = cv::Point(image.cols / 2, image.rows / 2);
+		//uint32_t radius = (image.cols*image.cols + image.rows*image.rows) / 4;
+		//cv::Point center = cv::Point(image.cols / 2, image.rows / 2);
 
 		float sum = 0.f;
 		for (const auto & s: segments) {
 			cv::Mat mask = cv::Mat::zeros(image.size(), CV_8UC1);
 			cv::rectangle(mask, s.min_p, s.max_p, cv::Scalar(1), CV_FILLED);
 			float score = cv::mean(likelihood, mask)[0];
 
-			cv::Point diff = (s.min_p + s.max_p) * 0.5 - center;
-			score *= exp(-diff.dot(diff) / float(radius) * deviation);
+			//cv::Point diff = (s.min_p + s.max_p) * 0.5 - center;
+			//score *= exp(-diff.dot(diff) / float(radius) * deviation);
 
 			sum += score;
 			prior.insert({s.id, score});

random/src/objectness_prior.h

@@ -6,17 +6,50 @@
 
 class ObjectnessPrior : public SelectionPrior {
 public:
-	ObjectnessPrior() {
+	ObjectnessPrior() :
+		obj_bbox(2, 8, 1)
+	{
 		obj.loadTrainedModel("/Users/hans/MasterThesis/code/cpp/random/model/ObjNessB2W8MAXBGR");
+		obj_bbox.loadTrainedModel("/Users/hans/MasterThesis/code/cpp/random/model/ObjNessB2W8MAXBGR");
+	}
+
+	float scoreBBox(const cv::Mat & img) {
+		cv::Mat img_small;
+		if (img.size() != cv::Size(8,8)) {
+			cv::resize(img, img_small, cv::Size(8,8), 0, 0, cv::INTER_CUBIC);
+		} else
+			img_small = img;
+
+		ValStructVec<float, Vec4i> boxes;
+		obj_bbox.getObjBndBoxes(img_small, boxes, 1);
+		for (int i = 0; i < boxes.size(); i++) {
+			std::cout << "value: " << boxes(i) << " rect: " << boxes[i] << std::endl;
+		}
+
+		return boxes(0);
+	}
+
+	float boxOverlap(cv::Vec4i bbox1, cv::Vec4i bbox2) {
+		cv::Vec4i bi(max(bbox1[0],bbox2[0]), max(bbox1[1],bbox2[1]), min(bbox1[2],bbox2[2]), min(bbox1[3],bbox2[3]));
+		int iw = bi[2] - bi[0];
+		int ih = bi[3] - bi[1];
+		if (iw > 0 && ih > 0) {
+			float overlap = iw * ih / float((bbox2[2] - bbox2[0]) * (bbox2[3] - bbox2[1]));
+			return overlap;
+		}
+
+		return 0.f;
 	}
 
 	virtual SelectionPriorMap computeSelectionPrior(const cv::Mat & image, const std::vector<Segment> & segments) {
 		SelectionPriorMap prior;
 
-		cv::Mat likelihood = cv::Mat::zeros(image.size(), CV_32FC1);
 
 		ValStructVec<float, Vec4i> boxes;
 		obj.getObjBndBoxes(image, boxes, 130);
+		//std::cout << "estimated: " << 
+// 		scoreBBox(image(cv::Range(boxes[0][1], boxes[0][3]), cv::Range(boxes[0][0], boxes[0][2])));
+// 		std::cout << "actual: " << boxes(0) << std::endl;
 
 // 		cv::Mat test = cv::imread("../../../images/horse.jpg");
 // 		cv::Mat test; image.copyTo(test);
@@ -34,17 +67,15 @@ class ObjectnessPrior : public SelectionPrior {
 		for (const auto & s: segments) {
 			float score = 0.f;
 			for (int i = 0; i < boxes.size(); i++) {
-				const cv::Vec4i & box = boxes[i];
-				if (s.min_p.x > box[0] && s.min_p.x < box[2] &&
-					s.min_p.y > box[1] && s.min_p.y < box[3] &&
-					s.max_p.x > box[0] && s.max_p.y < box[2] &&
-					s.max_p.y > box[1] && s.max_p.y < box[3])
+				//std::cout << jaccardSimilarity(boxes[i], cv::Vec4i(s.min_p.x, s.min_p.y, s.max_p.x, s.max_p.y)) << std::endl; 
+				if (boxOverlap(boxes[i], cv::Vec4i(s.min_p.x, s.min_p.y, s.max_p.x, s.max_p.y)) > 0.3)
 					score += boxes(i);
 			}
 			sum += score;
 			prior.insert({s.id, score});
 		}
 
+// 		cv::Mat likelihood = cv::Mat::zeros(image.size(), CV_32FC1);
 // 		float max = boxes(0);
 // 		float min = boxes(boxes.size() - 1);
 // 		for (int i = 0; i < boxes.size(); i++) {
@@ -82,4 +113,5 @@ class ObjectnessPrior : public SelectionPrior {
 
 protected:
 	Objectness obj;
+	Objectness obj_bbox;
 };

random/src/segment.h

@@ -74,30 +74,32 @@ class Segment
 			return 0.f;
 		}
 
-		float similarity = 0.f;
-
+		float color_similarity = 0.f;
 		if (flags & COLOR_SIMILARITY) {
 			for (uint64_t i = 0; i < a.color_hist.size(); i++)
-				similarity += std::min(a.color_hist[i], b.color_hist[i]);
+				color_similarity += weights.at<float>(0) * std::min(a.color_hist[i], b.color_hist[i]);
 		}
 
+		float texture_similarity = 0.f;
 		if (flags & TEXTURE_SIMILARITY) {
 			for (uint64_t i = 0; i < a.texture_hist.size(); i++)
-				similarity += std::min(a.texture_hist[i], b.texture_hist[i]);
+				texture_similarity += weights.at<float>(1) * std::min(a.texture_hist[i], b.texture_hist[i]);
 		}
 
+		float size_similarity = 0.f;
 		if (flags & SIZE_SIMILARITY) {
-			similarity += 1.f - float(a.size + b.size) / a.im_size;
+			size_similarity += weights.at<float>(2) * (1.f - float(a.size + b.size) / a.im_size);
 		}
 
+		float bbox_similarity = 0.f;
 		if (flags & BBOX_SIMILARITY) {
 			cv::Point min_p(std::min(a.min_p.x, b.min_p.x), std::min(a.min_p.y, b.min_p.y));
 			cv::Point max_p(std::max(a.max_p.x, b.max_p.x), std::max(a.max_p.y, b.max_p.y));
 			int bbox_size = (max_p.x - min_p.x) * (max_p.y - min_p.y);
-			similarity += 1.f - float(bbox_size - a.size - b.size) / a.im_size;
+			bbox_similarity += weights.at<float>(3) * (1.f - float(bbox_size - a.size - b.size) / a.im_size);
 		}
 
-		return similarity;
+		return color_similarity + texture_similarity + size_similarity + bbox_similarity + weights.at<float>(4);
 	}
 
 	static Segment merge(const Segment & a, const Segment & b) {

random/src/selection_prior.h

@@ -18,7 +18,25 @@ class SelectionPriorMap : public std::unordered_map<uint32_t, float> {
 			}
 		}
 
-		return -1;
+		return size() - 1;
+	}
+
+	void visualize(const cv::Mat & seg) {
+		float max = 0.f;
+		for (auto & p: *this) {
+			if (p.second > max) {
+				max = p.second;
+			}
+		}
+
+		cv::Mat visualization = cv::Mat::zeros(seg.size(), CV_8UC1);
+		for (auto & p: *this) {
+			visualization += (seg == p.first) * (p.second / max);
+		}
+
+		cv::namedWindow("SelectionPrior", cv::WINDOW_AUTOSIZE);
+		cv::imshow("SelectionPrior", visualization);
+		cv::waitKey();
 	}
 
 protected: