Fraxinus  16.5.0-fx-rc6
An IGT application
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
cxImageStreamerOpenCV.cpp
Go to the documentation of this file.
1 /*=========================================================================
2 This file is part of CustusX, an Image Guided Therapy Application.
3 
4 Copyright (c) 2008-2014, SINTEF Department of Medical Technology
5 All rights reserved.
6 
7 Redistribution and use in source and binary forms, with or without
8 modification, are permitted provided that the following conditions are met:
9 
10 1. Redistributions of source code must retain the above copyright notice,
11  this list of conditions and the following disclaimer.
12 
13 2. Redistributions in binary form must reproduce the above copyright notice,
14  this list of conditions and the following disclaimer in the documentation
15  and/or other materials provided with the distribution.
16 
17 3. Neither the name of the copyright holder nor the names of its contributors
18  may be used to endorse or promote products derived from this software
19  without specific prior written permission.
20 
21 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
22 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
24 DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
25 FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26 DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
27 SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
28 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
29 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 =========================================================================*/
32 #include "cxImageStreamerOpenCV.h"
33 #include "cxConfig.h"
34 
35 #include <QCoreApplication>
36 #include <QTimer>
37 #include <QTime>
38 #include <QHostAddress>
39 #include "vtkImageData.h"
40 #include "vtkSmartPointer.h"
41 #include "vtkMetaImageReader.h"
42 #include "vtkImageImport.h"
43 #include "vtkLookupTable.h"
44 #include "vtkImageMapToColors.h"
45 #include "vtkMetaImageWriter.h"
46 
47 #include "cxTypeConversions.h"
49 #include "cxStringHelpers.h"
50 #include "cxDoubleProperty.h"
51 #include "cxBoolProperty.h"
52 #include "cxLogger.h"
53 #include "cxVideoServerConfig.h"
54 
55 #ifdef CX_USE_OpenCV
56 #include <opencv2/highgui/highgui.hpp>
57 #include <opencv2/imgproc/imgproc.hpp>
58 #endif
59 
60 namespace cx
61 {
62 
63 std::vector<PropertyPtr> ImageStreamerOpenCVArguments::getSettings(QDomElement root)
64 {
65  std::vector<PropertyPtr> retval;
66  retval.push_back(this->getVideoPortOption(root));
67  retval.push_back(this->getPrintPropertiesOption(root));
68  return retval;
69 }
70 
72 {
73  DoublePropertyPtr retval;
74  retval = DoubleProperty::initialize("videoport", "Video Port", "Select video source as an integer from 0 and up.", 0, DoubleRange(0, 10, 1), 0, root);
75  retval->setGuiRepresentation(DoublePropertyBase::grSPINBOX);
76  retval->setGroup("OpenCV");
77  return retval;
78 }
79 
81 {
82  BoolPropertyPtr retval;
83  bool defaultValue = false;
84  retval = BoolProperty::initialize("properties", "Print Properties",
85  "When starting OpenCV, print properties to console",
86  defaultValue, root);
87  retval->setAdvanced(true);
88  retval->setGroup("OpenCV");
89  return retval;
90 }
91 
93 {
94  StringMap retval;
95  retval["--type"] = "OpenCV";
96  retval["--videoport"] = this->getVideoPortOption(root)->getValueAsVariant().toString();
97  if (this->getPrintPropertiesOption(root)->getValue())
98  retval["--properties"] = "1";
99  return retval;
100 }
101 
103 {
104  QStringList retval;
105 #ifdef CX_USE_OpenCV
106  retval << "--videoport: video id, default=0";
107  retval << "--in_width: width of incoming image, default=camera";
108  retval << "--in_height: height of incoming image, default=camera";
109  retval << "--out_width: width of outgoing image, default=camera";
110  retval << "--out_height: width of outgoing image, default=camera";
111  retval << "--properties: dump image properties";
112 #endif
113  return retval;
114 }
115 
116 } // namespace cx
117 
118 //------------------------------------------------------------
119 //------------------------------------------------------------
120 //------------------------------------------------------------
121 
122 namespace
123 {
124 
125 //------------------------------------------------------------
126 // Function to generate random matrix.
127 void GetRandomTestMatrix(igtl::Matrix4x4& matrix)
128 {
129  //float position[3];
130  //float orientation[4];
131 
132  matrix[0][0] = 1.0;
133  matrix[1][0] = 0.0;
134  matrix[2][0] = 0.0;
135  matrix[3][0] = 0.0;
136  matrix[0][1] = 0.0;
137  matrix[1][1] = -1.0;
138  matrix[2][1] = 0.0;
139  matrix[3][1] = 0.0;
140  matrix[0][2] = 0.0;
141  matrix[1][2] = 0.0;
142  matrix[2][2] = 1.0;
143  matrix[3][2] = 0.0;
144  matrix[0][3] = 0.0;
145  matrix[1][3] = 0.0;
146  matrix[2][3] = 0.0;
147  matrix[3][3] = 1.0;
148 }
149 
150 }
151 
152 namespace cx
153 {
155 {
156  mGrabbing = false;
157  mAvailableImage = false;
158  setSendInterval(40);
159 
160 #ifdef CX_USE_OpenCV
161  mVideoCapture.reset(new cv::VideoCapture());
162 #endif
163  mSendTimer = new QTimer(this);
164  connect(mSendTimer, SIGNAL(timeout()), this, SLOT(send())); // this signal will be executed in the thread of THIS, i.e. the main thread.
165 }
166 
168 {
169  this->deinitialize_local();
170 }
171 
173 {
174  return "OpenCV";
175 }
176 
178 {
180 }
181 
182 
184 {
186 }
187 
188 void ImageStreamerOpenCV::deinitialize_local()
189 {
190 #ifdef CX_USE_OpenCV
191  while (mGrabbing) // grab() method seems to call processmessages itself...
192  qApp->processEvents();
193  mVideoCapture->release();
194  mVideoCapture.reset(new cv::VideoCapture());
195 #endif
196 }
197 
198 void ImageStreamerOpenCV::initialize_local()
199 {
200 #ifdef CX_USE_OpenCV
201 
202  if (!mArguments.count("videoport"))
203  mArguments["videoport"] = "0";
204  if (!mArguments.count("out_width"))
205  mArguments["out_width"] = "";
206  if (!mArguments.count("out_height"))
207  mArguments["out_height"] = "";
208  if (!mArguments.count("in_width"))
209  mArguments["in_width"] = "";
210  if (!mArguments.count("in_height"))
211  mArguments["in_height"] = "";
212 
213  QString videoSource = mArguments["videoport"];
214  int videoport = convertStringWithDefault(mArguments["videoport"], 0);
215 
216  bool sourceIsInt = false;
217  videoSource.toInt(&sourceIsInt);
218 
219  if (sourceIsInt){
220  // open device (camera)
221  mVideoCapture->open(videoport);
222  }
223  else{
224  // open file
225  mVideoCapture->open(videoSource.toStdString().c_str());
226  }
227 
228  if (!mVideoCapture->isOpened())
229  {
230  cerr << "ImageStreamerOpenCV: Failed to open a video device or video file!\n" << endl;
231  return;
232  }
233 
234  // try one grab before accepting the streamer
235  // - this fails if no camera is attached
236  try
237  {
238  mVideoCapture->grab();
239  }
240  catch(cv::Exception& e)
241  {
242  CX_LOG_ERROR() << "OpenCV failed with message: " << e.what();
243  mVideoCapture->release();
244  return;
245  }
246 
247 
248  {
249  //determine default values
250  int default_width = mVideoCapture->get(CV_CAP_PROP_FRAME_WIDTH);
251  int default_height = mVideoCapture->get(CV_CAP_PROP_FRAME_HEIGHT);
252 
253  //set input size
254  int in_width = convertStringWithDefault(mArguments["in_width"], default_width);
255  int in_height = convertStringWithDefault(mArguments["in_height"], default_height);
256  mVideoCapture->set(CV_CAP_PROP_FRAME_WIDTH, in_width);
257  mVideoCapture->set(CV_CAP_PROP_FRAME_HEIGHT, in_height);
258 
259  //set output size (resize)
260  int out_width = convertStringWithDefault(mArguments["out_width"], in_width);
261  int out_height = convertStringWithDefault(mArguments["out_height"], in_height);
262  mRescaleSize.setWidth(out_width);
263  mRescaleSize.setHeight(out_height);
264 
265  if (mArguments.count("properties"))
266  this->dumpProperties();
267 
268  std::cout << "ImageStreamerOpenCV: Started streaming from openCV device "
269  << videoSource.toStdString()
270  << ", size=(" << in_width << "," << in_height << ")";
271  if (( in_width!=mRescaleSize.width() )|| (in_height!=mRescaleSize.height()))
272  std::cout << ". Scaled to (" << mRescaleSize.width() << "," << mRescaleSize.height() << ")";
273 
274  std::cout << std::endl;
275  }
276 #endif
277 }
278 
280 {
281 #ifdef CX_USE_OpenCV
282  this->initialize_local();
283 
284  if (!mSendTimer || !mVideoCapture->isOpened())
285  {
286  reportError("ImageStreamerOpenCV: Failed to start streaming: Not initialized.");
287  return;
288  }
289 
290  mSender = sender;
291  mSendTimer->start(getSendInterval());
292  this->continousGrabEvent(); // instead of grabtimer
293 
294 #else
295  reportWarning("ImageStreamerOpenCV: Failed to start streaming: CX_USE_OpenCV not defined.");
296 #endif //CX_USE_OpenCV
297 }
298 
300 {
301  if (!mSendTimer)
302  return;
303  mSendTimer->stop();
304  mSender.reset();
305 
306  this->deinitialize_local();
307 }
308 
310 {
311  return (mSendTimer && mVideoCapture->isOpened());
312 }
313 
314 void ImageStreamerOpenCV::dumpProperties()
315 {
316 #ifdef CX_USE_OpenCV
317  this->dumpProperty(CV_CAP_PROP_POS_MSEC, "CV_CAP_PROP_POS_MSEC");
318  this->dumpProperty(CV_CAP_PROP_POS_FRAMES, "CV_CAP_PROP_POS_FRAMES");
319  this->dumpProperty(CV_CAP_PROP_POS_AVI_RATIO, "CV_CAP_PROP_POS_AVI_RATIO");
320  this->dumpProperty(CV_CAP_PROP_FRAME_WIDTH, "CV_CAP_PROP_FRAME_WIDTH");
321  this->dumpProperty(CV_CAP_PROP_FRAME_HEIGHT, "CV_CAP_PROP_FRAME_HEIGHT");
322  this->dumpProperty(CV_CAP_PROP_FPS, "CV_CAP_PROP_FPS");
323  this->dumpProperty(CV_CAP_PROP_FOURCC, "CV_CAP_PROP_FOURCC");
324  this->dumpProperty(CV_CAP_PROP_FRAME_COUNT, "CV_CAP_PROP_FRAME_COUNT");
325  this->dumpProperty(CV_CAP_PROP_FORMAT, "CV_CAP_PROP_FORMAT");
326  this->dumpProperty(CV_CAP_PROP_MODE, "CV_CAP_PROP_MODE");
327  this->dumpProperty(CV_CAP_PROP_BRIGHTNESS, "CV_CAP_PROP_BRIGHTNESS");
328  this->dumpProperty(CV_CAP_PROP_CONTRAST, "CV_CAP_PROP_CONTRAST");
329  this->dumpProperty(CV_CAP_PROP_SATURATION, "CV_CAP_PROP_SATURATION");
330  this->dumpProperty(CV_CAP_PROP_HUE, "CV_CAP_PROP_HUE");
331  this->dumpProperty(CV_CAP_PROP_GAIN, "CV_CAP_PROP_GAIN");
332  this->dumpProperty(CV_CAP_PROP_EXPOSURE, "CV_CAP_PROP_EXPOSURE");
333  this->dumpProperty(CV_CAP_PROP_CONVERT_RGB, "CV_CAP_PROP_CONVERT_RGB");
334  // this->dumpProperty(CV_CAP_PROP_WHITE_BALANCE, "CV_CAP_PROP_WHITE_BALANCE");
335  this->dumpProperty(CV_CAP_PROP_RECTIFICATION, "CV_CAP_PROP_RECTIFICATION");
336 #endif
337 }
338 
339 void ImageStreamerOpenCV::dumpProperty(int val, QString name)
340 {
341 #ifdef CX_USE_OpenCV
342  double value = mVideoCapture->get(val);
343  if (value != -1)
344  std::cout << "Property " << name.toStdString() << " : " << mVideoCapture->get(val) << std::endl;
345 #endif
346 }
347 
353 void ImageStreamerOpenCV::continousGrabEvent()
354 {
355  if (!mSendTimer->isActive())
356  return;
357  this->grab();
358  QMetaObject::invokeMethod(this, "continousGrabEvent", Qt::QueuedConnection);
359 }
360 
361 void ImageStreamerOpenCV::grab()
362 {
363 #ifdef CX_USE_OpenCV
364  if (!mVideoCapture->isOpened())
365  {
366  return;
367  }
368 
369  mGrabbing = true;
370  // grab images from camera to opencv internal buffer, do not process
371  bool success = mVideoCapture->grab();
372  if (success)
373  mLastGrabTime = QDateTime::currentDateTime();
374  mAvailableImage = success;
375  mGrabbing = false;
376 #endif
377 }
378 
379 void ImageStreamerOpenCV::send()
380 {
381  if (!mSender || !mSender->isReady())
382  return;
383  if (!mAvailableImage)
384  {
385 // reportDebug("dropped resend of frame");
386  return;
387  }
388  PackagePtr package(new Package());
389  package->mImage = this->getImageMessage();
390  mSender->send(package);
391  mAvailableImage = false;
392 
393 // static int counter=0;
394 // if (++counter%50==0)
395 // std::cout << "=== ImageStreamerOpenCV send: " << start.msecsTo(QTime::currentTime()) << " ms" << std::endl;
396 }
397 
398 ImagePtr ImageStreamerOpenCV::getImageMessage()
399 {
400 #ifdef CX_USE_OpenCV
401  if (!mVideoCapture->isOpened())
402  return ImagePtr();
403 
404  QTime start = QTime::currentTime();
405 
406  cv::Mat frame_source;
407  // mVideoCapture >> frame_source;
408  if (!mVideoCapture->retrieve(frame_source, 0))
409  return ImagePtr();
410 
411  if (this->thread() == QCoreApplication::instance()->thread() && !mSender)
412  {
413  cv::imshow("ImageStreamerOpenCV", frame_source);
414  }
415 
416  cv::Mat frame = frame_source;
417  if (( frame.cols!=mRescaleSize.width() )|| (frame.rows!=mRescaleSize.height()))
418  {
419  cv::resize(frame_source, frame, cv::Size(mRescaleSize.width(), mRescaleSize.height()), 0, 0, CV_INTER_LINEAR);
420  }
421 
422  vtkImageDataPtr raw = this->convertTovtkImageData(frame);
423  ImagePtr image(new Image("openCV", raw));
424  image->setAcquisitionTime(mLastGrabTime);
425  return image;
426 #else
427  return ImagePtr();
428 #endif
429 }
430 
431 vtkImageDataPtr ImageStreamerOpenCV::convertTovtkImageData(cv::Mat& frame)
432 {
433  vtkImageDataPtr retval = vtkImageDataPtr::New();
434 
435  Eigen::Array3i dim(frame.cols, frame.rows, 1);
436 // Eigen::Array3f spacing(1,1);
437  retval->SetDimensions(dim.data());
438  retval->SetSpacing(1,1,1);
439 
440  int dataType = -1;
441 
442 // if (frame.channels() == 3 || frame.channels() == 4) // dropped support for alpha - dont think openCV uses this.
443  if (frame.channels() == 3)
444  {
445  dataType = VTK_UNSIGNED_CHAR;
446  }
447  else if (frame.channels() == 1)
448  {
449  // dropped support: must iterate over using shorts later on.
450 // if (frame.depth() == 16)
451 // {
452 // dataType = VTK_UNSIGNED_SHORT;
453 // }
454  if (frame.depth() == 8)
455  {
456  dataType = VTK_UNSIGNED_CHAR;
457  }
458  }
459 
460  if (dataType == -1)
461  {
462  std::cerr << "unknown image type" << std::endl;
463  return vtkImageDataPtr();
464  }
465 
466  retval->AllocateScalars(dataType, frame.channels());
467 
468  //------------------------------------------------------------
469  // Create a new IMAGE type message
470 // IGTLinkImageMessage::Pointer imgMsg = IGTLinkImageMessage::New();
471 // imgMsg->SetDimensions(size);
472 // imgMsg->SetSpacing(spacingF);
473 // imgMsg->SetScalarType(scalarType);
474 // imgMsg->SetSubVolume(svsize, svoffset);
475 // imgMsg->AllocateScalars();
476 // imgMsg->SetTimeStamp(timestamp);
477 
478  unsigned char* dest = reinterpret_cast<unsigned char*> (retval->GetScalarPointer());
479  uchar* src = frame.data;
480  int N = frame.rows * frame.cols;
481 // QString colorFormat;
482 
483  if (frame.channels() == 3)
484  {
485  if (frame.isContinuous())
486  {
487  // 3-channel continous colors
488  for (int i = 0; i < N; ++i)
489  {
490 // *destPtr++ = 255;
491  *dest++ = src[2]; // R
492  *dest++ = src[1]; // G
493  *dest++ = src[0]; // B
494  src += 3;
495  }
496  }
497  else
498  {
499 // std::cout << "noncontinous conversion, rows=" << size[1] << std::endl;
500  for (int i=0; i<dim[1]; ++i)
501  {
502  const uchar* src = frame.ptr<uchar>(i);
503  for (int j=0; j<dim[0]; ++j)
504  {
505 // *destPtr++ = 255;
506  *dest++ = src[2];
507  *dest++ = src[1];
508  *dest++ = src[0];
509  src += 3;
510  }
511  }
512  }
513 // colorFormat = "ARGB";
514  }
515  if (frame.channels() == 1)
516  {
517  if (!frame.isContinuous())
518  {
519  std::cout << "Error: Non-continous frame data." << std::endl;
520  return vtkImageDataPtr();
521  }
522 
523  // BW camera
524  for (int i = 0; i < N; ++i)
525  {
526  *dest++ = *src++;
527  }
528 // colorFormat = "R";
529  }
530 
531  return retval;
532 
533 }
534 
535 //------------------------------------------------------------
536 //------------------------------------------------------------
537 //------------------------------------------------------------
538 
539 }
static BoolPropertyPtr initialize(const QString &uid, QString name, QString help, bool value, QDomNode root=QDomNode())
void reportError(QString msg)
Definition: cxLogger.cpp:92
DoublePropertyBasePtr getVideoPortOption(QDomElement root)
void setSendInterval(int milliseconds)
how often an image should be sent (in milliseconds)
Definition: cxStreamer.cpp:43
virtual void initialize(StringMap arguments)
Utility class for describing a bounded numeric range.
Definition: cxDoubleRange.h:53
boost::shared_ptr< class Image > ImagePtr
Definition: cxDicomWidget.h:48
virtual QStringList getArgumentDescription()
BoolPropertyBasePtr getPrintPropertiesOption(QDomElement root)
StringMap convertToCommandLineArguments(QDomElement root)
std::map< QString, QString > StringMap
void reportWarning(QString msg)
Definition: cxLogger.cpp:91
#define CX_LOG_ERROR
Definition: cxLogger.h:114
boost::shared_ptr< class BoolPropertyBase > BoolPropertyBasePtr
boost::shared_ptr< class DoublePropertyBase > DoublePropertyBasePtr
boost::shared_ptr< class DoubleProperty > DoublePropertyPtr
boost::shared_ptr< struct Package > PackagePtr
static DoublePropertyPtr initialize(const QString &uid, QString name, QString help, double value, DoubleRange range, int decimals, QDomNode root=QDomNode())
QTimer * mSendTimer
Definition: cxStreamer.h:87
virtual void startStreaming(SenderPtr sender)
unsigned char uchar
int convertStringWithDefault(QString text, int def)
boost::shared_ptr< class BoolProperty > BoolPropertyPtr
boost::shared_ptr< Sender > SenderPtr
Definition: cxSender.h:85
vtkSmartPointer< class vtkImageData > vtkImageDataPtr
SenderPtr mSender
Definition: cxStreamer.h:86
virtual void initialize(StringMap arguments)
Definition: cxStreamer.cpp:80
int getSendInterval() const
how often an image should be sent (in milliseconds)
Definition: cxStreamer.cpp:48
virtual std::vector< PropertyPtr > getSettings(QDomElement root)