iol
module.h
1 /*
2  * Copyright (C) 2011 Department of Robotics Brain and Cognitive Sciences - Istituto Italiano di Tecnologia
3  * Author: Ugo Pattacini
4  * email: ugo.pattacini@iit.it
5  * Permission is granted to copy, distribute, and/or modify this program
6  * under the terms of the GNU General Public License, version 2 or any
7  * later version published by the Free Software Foundation.
8  *
9  * A copy of the license can be found at
10  * http://www.robotcub.org/icub/license/gpl.txt
11  *
12  * This program is distributed in the hope that it will be useful, but
13  * WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
15  * Public License for more details
16 */
17 
18 #ifndef __MODULE_H__
19 #define __MODULE_H__
20 
21 #include <mutex>
22 #include <string>
23 #include <vector>
24 #include <deque>
25 #include <map>
26 
27 #include <yarp/os/all.h>
28 #include <yarp/sig/all.h>
29 #include <iCub/ctrl/filters.h>
30 
31 #include <opencv2/core.hpp>
32 #include <opencv2/opencv.hpp>
33 #include <opencv2/tracking.hpp>
34 
35 #include "utils.h"
36 #include "classifierHandling.h"
37 
38 #define RET_INVALID -1
39 
40 using namespace std;
41 using namespace yarp::os;
42 using namespace yarp::sig;
43 using namespace iCub::ctrl;
44 
45 
46 /**********************************************************/
47 class Tracker
48 {
49 protected:
50  enum { idle, init, no_need, tracking };
51 
52  string trackerType;
53  int trackerState;
54  double trackerTmo;
55  double trackerTimer;
56 
57  cv::Rect2d trackerResult;
58  cv::Ptr<cv::Tracker> tracker;
59 
60 public:
61  Tracker(const string &trackerType_="BOOSTING", const double trackerTmo_=0.0);
62  void prepare();
63  void latchBBox(const cv::Rect &bbox);
64  void track(ImageOf<PixelBgr> &img);
65  bool is_tracking(cv::Rect &bbox) const;
66 };
67 
68 
69 /**********************************************************/
70 class Manager : public RFModule
71 {
72 protected:
73  RpcServer rpcPort;
74  RpcServer rpcHuman;
75  RpcClient rpcClassifier;
76  RpcClient rpcMotor;
77  RpcClient rpcMotorGrasp;
78  RpcClient rpcReachCalib;
79  RpcClient rpcGet3D;
80  RpcClient rpcMotorStop;
81  RpcClient rpcMemory;
82  StopCmdPort rxMotorStop;
83  PointedLocationPort pointedLoc;
84  MemoryReporter memoryReporter;
85 
86  BufferedPort<Bottle> blobExtractor;
87  BufferedPort<Bottle> histObjLocPort;
88  BufferedPort<Property> recogTriggerPort;
89  BufferedPort<ImageOf<PixelBgr> > imgIn;
90  BufferedPort<ImageOf<PixelBgr> > imgOut;
91  BufferedPort<ImageOf<PixelBgr> > imgRtLocOut;
92  BufferedPort<ImageOf<PixelBgr> > imgTrackOut;
93  BufferedPort<ImageOf<PixelBgr> > imgHistogram;
94  Port imgClassifier;
95 
96  Speaker speaker;
97  Attention attention;
98  RtLocalization rtLocalization;
99  Exploration exploration;
100  MemoryUpdater memoryUpdater;
101  ClassifiersDataBase db;
102  map<string,int> memoryIds;
103 
104  ImageOf<PixelBgr> img;
105  ImageOf<PixelBgr> imgRtLoc;
106  mutex mutexResources;
107  mutex mutexResourcesMemory;
108  mutex mutexAttention;
109  mutex mutexMemoryUpdate;
110  mutex mutexGet3D;
111 
112  string name;
113  string camera;
114  string objectToBeKinCalibrated;
115  bool busy;
116  bool scheduleLoadMemory;
117  bool enableInterrupt;
118  bool actionInterrupted;
119  bool skipGazeHoming;
120  bool doAttention;
121  bool trainOnFlipped;
122  bool trainBurst;
123  bool skipLearningUponSuccess;
124  double blobs_detection_timeout;
125  double improve_train_period;
126  double classification_threshold;
127  double blockEyes;
128  Bottle* dropPosition;
129 
130  string tracker_type;
131  double tracker_timeout;
132  vector<int> tracker_min_blob_size;
133  map<string,Tracker> trackersPool;
134 
135  map<string,Filter*> histFiltersPool;
136  int histFilterLength;
137  deque<cv::Scalar> histColorsCode;
138 
139  bool trackStopGood;
140  bool whatGood;
141  cv::Point trackStopLocation;
142  cv::Point whatLocation;
143  cv::Point clickLocation;
144 
145  double lastBlobsArrivalTime;
146  Bottle lastBlobs;
147  Bottle memoryBlobs;
148  Bottle memoryScores;
149 
150  Vector skim_blobs_x_bounds;
151  Vector skim_blobs_y_bounds;
152  Vector histObjLocation;
153 
154  friend class Attention;
155  friend class RtLocalization;
156  friend class Exploration;
157  friend class MemoryUpdater;
158  friend class MemoryReporter;
159 
160  int processHumanCmd(const Bottle &cmd, Bottle &b);
161  Bottle skimBlobs(const Bottle &blobs);
162  bool thresBBox(cv::Rect &bbox, const Image &img);
163  Bottle getBlobs();
164  cv::Point getBlobCOG(const Bottle &blobs, const int i);
165  bool get3DPosition(const cv::Point &point, Vector &x);
166  void acquireImage(const bool rtlocalization=false);
167  void drawBlobs(const Bottle &blobs, const int i, Bottle *scores=NULL);
168  void rotate(cv::Mat &src, const double angle, cv::Mat &dst);
169  void drawScoresHistogram(const Bottle &blobs, const Bottle &scores, const int i);
170  void loadMemory();
171  void updateClassifierInMemory(Classifier *pClassifier);
172  Vector updateObjCartPosInMemory(const string &object, const Bottle &blobs, const int i);
173  void triggerRecogInfo(const string &object, const Bottle &blobs, const int i, const string &recogType);
174  int findClosestBlob(const Bottle &blobs, const cv::Point &loc);
175  int findClosestBlob(const Bottle &blobs, const Vector &loc);
176  Bottle classify(const Bottle &blobs, const bool rtlocalization=false);
177  void burst(const string &tag="");
178  void train(const string &object, const Bottle &blobs, const int i);
179  void improve_train(const string &object, const Bottle &blobs, const int i);
180  void home(const string &part="all");
181  void calibTable();
182  bool calibKinStart(const string &object, const string &hand, const Vector &x, const int recogBlob);
183  void calibKinStop();
184  void addDropPosition(Bottle &cmd);
185  void motorHelper(const string &cmd, const string &object);
186  bool getCalibratedLocation(const string &object, string &hand, const Vector &x, Vector &y);
187  Vector applyObjectPosOffsets(const string &object, const string &hand);
188  bool interruptableAction(const string &action, deque<string> *param, const string &object, const Vector &x, Vector &y);
189  void point(const string &object);
190  void look(const string &object);
191  void look(const Bottle &blobs, const int i, const Bottle &options=Bottle());
192  int recognize(const string &object, Bottle &blobs, Classifier **ppClassifier=NULL);
193  int recognize(Bottle &blobs, Bottle &scores, string &object);
194  void execName(const string &object);
195  void execForget(const string &object);
196  void execWhere(const string &object, const Bottle &blobs, const int recogBlob, Classifier *pClassifier, const string &recogType);
197  void execWhat(const Bottle &blobs, const int pointedBlob, const Bottle &scores, const string &object);
198  void execThis(const string &object, const string &detectedObject, const Bottle &blobs, const int &pointedBlob);
199  void execExplore(const string &object);
200  void execReinforce(const string &object, const Vector &position);
201  void execInterruptableAction(const string &action, const string &object, const Vector &x, const Bottle &blobs, const int recogBlob);
202  void switchAttention();
203  void doLocalization();
204  bool get3DPositionFromMemory(const string &object, Vector &position, const bool lockMemory=true);
205  bool doExploration(const string &object, const Vector &position);
206  void updateMemory();
207 
208 public:
209  void interruptMotor();
210  void reinstateMotor(const bool saySorry=true);
211  bool configure(ResourceFinder &rf);
212  bool interruptModule();
213  bool close();
214  bool updateModule();
215  bool respond(const Bottle &command, Bottle &reply);
216  double getPeriod();
217 };
218 
219 #endif
220 
Generated by doxygen 1.9.1