diff --git a/apps/specular_estimation/CMakeFiles/specular_estimation.dir/src/PhotometricStereo.cc.o b/apps/specular_estimation/CMakeFiles/specular_estimation.dir/src/PhotometricStereo.cc.o index fadb42bda7c2ea136bf434869d06753e8271ca11..540ba83948787cdb8abf565161409a49e31d4306 100644 Binary files a/apps/specular_estimation/CMakeFiles/specular_estimation.dir/src/PhotometricStereo.cc.o and b/apps/specular_estimation/CMakeFiles/specular_estimation.dir/src/PhotometricStereo.cc.o differ diff --git a/apps/specular_estimation/CMakeFiles/specular_estimation.dir/src/specular_estimation.cc.o b/apps/specular_estimation/CMakeFiles/specular_estimation.dir/src/specular_estimation.cc.o index dfc38612e7ed82561406f9c68d1a6fa660a8d93b..374201375dab997b7a111a5c3b3e0519d7d34f82 100644 Binary files a/apps/specular_estimation/CMakeFiles/specular_estimation.dir/src/specular_estimation.cc.o and b/apps/specular_estimation/CMakeFiles/specular_estimation.dir/src/specular_estimation.cc.o differ diff --git a/apps/specular_estimation/src/Charuco.h b/apps/specular_estimation/src/Charuco.h index 3441706c9694f5257ded48f5554ac9c0b5764b3c..57fbcecb2e4ba244ec9014adc5a1984a8a22cb2b 100644 --- a/apps/specular_estimation/src/Charuco.h +++ b/apps/specular_estimation/src/Charuco.h @@ -27,6 +27,7 @@ void detectArucoMarkers(int dictionaryId, bool showRejected, bool estimatePose, void createArucoDictionary(std::vector< cv::Mat >& arucoMarkers, int dictionaryId); cv::Mat drawCharuco(cv::Ptr<cv::aruco::CharucoBoard> board, int rows, int cols, bool saveCharuco, bool displayCharuco); void calibrateCamera(int squaresX, int squaresY, float squareLength, float markerLength, int dictionaryId, std::vector<cv::Mat> charucoImages, cv::Mat& cameraMatrix, cv::Mat& distCoeffs); +void calibrateCamera(std::string charucoPath, cv::Mat& cameraMatrix, cv::Mat& distortionCoefficients); void loadCharucoImages(std::vector< cv::Mat >& charucoImages, std::string charucoPath, std::string charucoName); void undistortCharuco(cv::Ptr<cv::aruco::CharucoBoard> board, cv::Mat inputImage, cv::Mat boardImage, cv::Mat& undistortedCharuco, cv::Mat cameraMatrix, cv::Mat distortionCoefficients, cv::Vec3d& rvec, cv::Vec3d& tvec, cv::Mat& perspectiveTransform, bool displayUndistortedCharuco); @@ -428,7 +429,7 @@ void estimatePoseFromCharuco(cv::Ptr<cv::aruco::CharucoBoard> board, cv::Mat inp } } -void calibrateCamera(int squaresX, int squaresY, float squareLength, float markerLength, int dictionaryId, std::vector< cv::Mat > charucoImages, cv::Mat& cameraMatrix, cv::Mat& distortionCoefficients) { +void calibrateCamera(int squaresX, int squaresY, float squareLength, float markerLength, int dictionaryId, std::vector<cv::Mat> charucoImages, cv::Mat& cameraMatrix, cv::Mat& distortionCoefficients) { std::string outputFile = "cameraCalibration"; @@ -586,6 +587,180 @@ void calibrateCamera(int squaresX, int squaresY, float squareLength, float marke } } + +void calibrateCamera(std::string charucoPath, cv::Mat& cameraMatrix, cv::Mat& distortionCoefficients) { + + // Load a premade dictionary of ArUco markers + int dictionaryId = 0; + //cv::Ptr<cv::aruco::Dictionary> dictionary = cv::aruco::getPredefinedDictionary(cv::aruco::PREDEFINED_DICTIONARY_NAME(dictionaryId)); + + // Create a ChArUco board + int squaresX = 11; + int squaresY = 9; + float squareLength = 0.04f; + float markerLength = 0.02f; + //cv::Ptr<cv::aruco::CharucoBoard> charucoBoard = cv::aruco::CharucoBoard::create(squaresX, squaresY, squareLength, markerLength, dictionary); + + std::cout << "Loading ChArUco images.\n"; + std::vector<cv::Mat> charucoImages = loadImages(charucoPath); + + std::string outputFile = "cameraCalibration"; + + bool showChessboardCorners = false; + + int calibrationFlags = 0; + float aspectRatio = 1; + + bool zt = true; // Assume zero tangential distortion + bool pc = true; // Fix the principal point at the center + + if (zt) calibrationFlags |= cv::CALIB_ZERO_TANGENT_DIST; + if (pc) calibrationFlags |= cv::CALIB_FIX_PRINCIPAL_POINT; + + cv::Ptr<cv::aruco::DetectorParameters> detectorParams = cv::aruco::DetectorParameters::create(); + + bool refindStrategy = false; // Apply refind strategy + int camId = 0; // Camera id if input doesnt come from video (-v) + cv::String video; + + cv::VideoCapture inputVideo; + int waitTime = 0; + + cv::Ptr<cv::aruco::Dictionary> dictionary = cv::aruco::getPredefinedDictionary(cv::aruco::PREDEFINED_DICTIONARY_NAME(dictionaryId)); + + // create charuco board object + cv::Ptr<cv::aruco::CharucoBoard> charucoboard = cv::aruco::CharucoBoard::create(squaresX, squaresY, squareLength, markerLength, dictionary); + cv::Ptr<cv::aruco::Board> board = charucoboard.staticCast<cv::aruco::Board>(); + + // collect data from each frame + std::vector< std::vector< std::vector< cv::Point2f > > > allCorners; + std::vector< std::vector< int > > allIds; + std::vector< cv::Mat > allImgs; + cv::Size imgSize; + + for (unsigned int i = 0; i < charucoImages.size(); i++) { + cv::Mat imageCopy; + + std::vector< int > ids; + std::vector< std::vector< cv::Point2f > > corners, rejected; + + // detect markers + cv::aruco::detectMarkers(charucoImages[i], dictionary, corners, ids, detectorParams, rejected); + + // refind strategy to detect more markers + if (refindStrategy) cv::aruco::refineDetectedMarkers(charucoImages[i], board, corners, ids, rejected); + + // interpolate charuco corners + cv::Mat currentCharucoCorners, currentCharucoIds; + if (ids.size() > 0) { + cv::aruco::interpolateCornersCharuco(corners, ids, charucoImages[i], charucoboard, currentCharucoCorners, currentCharucoIds); + } + + // draw results + charucoImages[i].copyTo(imageCopy); + if (ids.size() > 0) { + cv::aruco::drawDetectedMarkers(imageCopy, corners); + } + + if (currentCharucoCorners.total() > 0) { + cv::aruco::drawDetectedCornersCharuco(imageCopy, currentCharucoCorners, currentCharucoIds); + } + + //cv::putText(charucoImages[i], "Press 'c' to add current frame. 'ESC' to finish and calibrate", cv::Point(10, 20), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(255, 0, 0), 2); + + //cv::namedWindow( "Camera Calibration", cv::WINDOW_NORMAL ); + //cv::imshow("Camera Calibration", imageCopy); + + if (ids.size() > 0) { + allCorners.push_back(corners); + allIds.push_back(ids); + allImgs.push_back(charucoImages[i]); + imgSize = charucoImages[i].size(); + } + + //cv::waitKey(0); + } + + if (allIds.size() < 1) { + std::cerr << "Not enough captures for calibration" << std::endl; + return; + } + + std::vector< cv::Mat > rvecs, tvecs; + double repError; + + if (calibrationFlags & cv::CALIB_FIX_ASPECT_RATIO) { + cameraMatrix = cv::Mat::eye(3, 3, CV_64F); + cameraMatrix.at< double >(0, 0) = aspectRatio; + } + + // prepare data for calibration + std::vector< std::vector< cv::Point2f > > allCornersConcatenated; + std::vector< int > allIdsConcatenated; + std::vector< int > markerCounterPerFrame; + markerCounterPerFrame.reserve(allCorners.size()); + for (unsigned int i = 0; i < allCorners.size(); i++) { + markerCounterPerFrame.push_back((int)allCorners[i].size()); + for (unsigned int j = 0; j < allCorners[i].size(); j++) { + allCornersConcatenated.push_back(allCorners[i][j]); + allIdsConcatenated.push_back(allIds[i][j]); + } + } + + // calibrate camera using aruco markers + double arucoRepErr; + arucoRepErr = cv::aruco::calibrateCameraAruco(allCornersConcatenated, allIdsConcatenated, markerCounterPerFrame, board, imgSize, cameraMatrix, distortionCoefficients, cv::noArray(), cv::noArray(), calibrationFlags); + + // prepare data for charuco calibration + int nFrames = (int)allCorners.size(); + std::vector< cv::Mat > allCharucoCorners; + std::vector< cv::Mat > allCharucoIds; + std::vector< cv::Mat > filteredImages; + allCharucoCorners.reserve(nFrames); + allCharucoIds.reserve(nFrames); + + for (int i = 0; i < nFrames; i++) { + // interpolate using camera parameters + cv::Mat currentCharucoCorners, currentCharucoIds; + cv::aruco::interpolateCornersCharuco(allCorners[i], allIds[i], allImgs[i], charucoboard, currentCharucoCorners, currentCharucoIds, cameraMatrix, distortionCoefficients); + + allCharucoCorners.push_back(currentCharucoCorners); + allCharucoIds.push_back(currentCharucoIds); + filteredImages.push_back(allImgs[i]); + } + + if (allCharucoCorners.size() < 4) { + std::cerr << "Not enough corners for calibration" << std::endl; + return; + } + + // calibrate camera using charuco + repError = cv::aruco::calibrateCameraCharuco(allCharucoCorners, allCharucoIds, charucoboard, imgSize, cameraMatrix, distortionCoefficients, rvecs, tvecs, calibrationFlags); + + //saveCameraParams(outputFile, imgSize, aspectRatio, calibrationFlags, cameraMatrix, distortionCoefficients, repError); + + std::cout << "Rep Error: " << repError << std::endl; + std::cout << "Rep Error Aruco: " << arucoRepErr << std::endl; + //std::cout << "Calibration saved to " << outputFile << std::endl; + + // show interpolated charuco corners for debugging + if (showChessboardCorners) { + for (unsigned int frame = 0; frame < filteredImages.size(); frame++) { + cv::Mat imageCopy = filteredImages[frame].clone(); + if (allIds[frame].size() > 0) { + + if (allCharucoCorners[frame].total() > 0) { + cv::aruco::drawDetectedCornersCharuco(imageCopy, allCharucoCorners[frame], allCharucoIds[frame]); + } + } + + cv::namedWindow("Interpolated ChArUco Markers", cv::WINDOW_NORMAL); + cv::imshow("Interpolated ChArUco Markers", imageCopy); + } + } +} + + void loadCharucoImages(std::vector< cv::Mat >& charucoImages, std::string charucoPath, std::string charucoName) { bool loadedImages = false; int imageNumber = 0; diff --git a/apps/specular_estimation/src/OpenCV.h b/apps/specular_estimation/src/OpenCV.h index e7db8a89c2322f070ab42d7cc9b123159970a815..2522596a07aca30d73b549ae61e86da8aea2d2df 100644 --- a/apps/specular_estimation/src/OpenCV.h +++ b/apps/specular_estimation/src/OpenCV.h @@ -639,14 +639,15 @@ std::vector<cv::Mat> loadMaterialImages(std::string modelPath, int& originalWidt } else { + originalWidth = materialImage.cols; + originalHeight = materialImage.rows; + if (imageScale > 1) { cv::resize(materialImage, materialImage, cv::Size(), 1.0 / imageScale, 1.0 / imageScale, cv::INTER_AREA); } materialImages.push_back(materialImage); - imageNumber++; - std::cout << "Loaded material image " << imageNumber << std::endl; } } diff --git a/apps/specular_estimation/src/PhotometricStereo.cc b/apps/specular_estimation/src/PhotometricStereo.cc index 5e60d16154bdd8aac5fb128aea30a06f0b632988..2a9c06ea133a4497dfafe05b73acb8d37b12be09 100755 --- a/apps/specular_estimation/src/PhotometricStereo.cc +++ b/apps/specular_estimation/src/PhotometricStereo.cc @@ -181,24 +181,38 @@ photometricStereo::photometricStereo(int n, int startI, int endI, std::string pa } photometricStereo::photometricStereo(int imageNumber, std::string modelPath, std::string calibrationPath, std::string macbethPath, std::string imageName, std::string calibration, double discardRatio): mDiscardRatio(discardRatio), imageNum(imageNumber) { - - mp2Images = readImages(modelPath); + int imageScale = 4; + mp2Images = readImages(modelPath, imageScale); std::cout << "Loaded material images.\n"; - mp2CalibrationImages = readImages(calibrationPath); + mp2CalibrationImages = readImages(calibrationPath, imageScale); std::cout << "Loaded calibration images.\n"; - mp2MacbethImages = readImages(macbethPath); - std::cout << "Loaded macbeth images.\n"; + //mp2MacbethImages = readImages(macbethPath); + //std::cout << "Loaded macbeth images.\n"; + + width = mp2Images[0].cols; + height = mp2Images[0].rows; std::string maskName = calibrationPath + "mask.png"; cv::Mat p2Mask = cv::imread(maskName, cv::IMREAD_GRAYSCALE); + if (imageScale != 1) + cv::resize(p2Mask, p2Mask, cv::Size(), 1.0 / imageScale, 1.0 / imageScale, cv::INTER_AREA); mp2Mask.push_back(p2Mask); mMaskNames.push_back(maskName); + + maskName = calibrationPath + "white.png"; p2Mask = cv::imread(maskName, cv::IMREAD_GRAYSCALE); + if (imageScale != 1) + cv::resize(p2Mask, p2Mask, cv::Size(), 1.0 / imageScale, 1.0 / imageScale, cv::INTER_AREA); mp2Mask.push_back(p2Mask); mMaskNames.push_back(maskName); } +photometricStereo::photometricStereo(std::vector<cv::Mat> mp2Images, std::vector<cv::Mat> mp2CalibrationImages, std::vector<cv::Mat> mp2Mask, int imageNumber, double discardRatio): mDiscardRatio(discardRatio), imageNum(imageNumber) { + width = mp2Images[0].cols; + height = mp2Images[0].rows; +} + /* photometricStereo::photometricStereo(int imageNumber, std::string modelPath, std::string calibrationPath, std::string macbethPath, std::string imageName, std::string calibration, double discardRatio): mDiscardRatio(discardRatio), imageNum(imageNumber) { bool loadedImages = false; @@ -334,7 +348,7 @@ bool photometricStereo::readImage() { return true; } -std::vector<cv::Mat> photometricStereo::readImages(std::string path) { +std::vector<cv::Mat> photometricStereo::readImages(std::string path, int imageScale = 1) { std::vector<cv::Mat> images; bool loadedImages = false; @@ -357,6 +371,9 @@ std::vector<cv::Mat> photometricStereo::readImages(std::string path) { loadedImages = true; } else { + if (imageScale != 1) { + cv::resize(image, image, cv::Size(), 1.0 / imageScale, 1.0 / imageScale, cv::INTER_AREA); + } images.push_back(image); imageNumber++; //std::cout << "Loaded image " << imageNumber << std::endl; @@ -510,15 +527,16 @@ void photometricStereo::getLightInformation(const int metalIndex) { double lightIntensity; //detectMacbeth(mMacbethNames[0]); - cv::minMaxLoc(mp2MacbethImages[i], NULL, &lightIntensity, NULL, NULL); + //cv::minMaxLoc(mp2MacbethImages[i], NULL, &lightIntensity, NULL, NULL); + lightIntensity = 1; m_light[i].mIntensity = lightIntensity; //m_light[i].mIntensity = macbethIntensity[i]; //std::cout << "Light intensity: " << m_light[i].mIntensity << std::endl; } } -// calculate the norm and albedo for every pixel -// here, threshold is for the dark discard +// calculate the normal and albedo for every pixel +// threshold is for the dark discard void photometricStereo::getPixelNormAndAlbedo(const int objectIndex, cv::Mat lightDirections) { // calculate the pixel num //std::vector<double> pixelThreshold; @@ -640,21 +658,21 @@ void photometricStereo::getPixelNormAndAlbedo(const int objectIndex, cv::Mat lig } } -// calculate the norm and albedo for every pixel -// here, threshold is for the dark discard +// calculate the normal and albedo for every pixel +// threshold is for the dark discard void photometricStereo::getPixelNormAndAlbedo(const int objectIndex) { // calculate the pixel num //std::vector<double> pixelThreshold; for(int i = 0; i < mp2Mask[objectIndex].rows; i++) { for(int j = 0; j < mp2Mask[objectIndex].cols; j++) { - if(mp2Mask[objectIndex].at<unsigned char>(i, j) >= 255) { + //if(mp2Mask[objectIndex].at<unsigned char>(i, j) == 255) { mObjectX.push_back(j); mObjectY.push_back(i); - } + //} } } - //std::cout << "Calculated the pixel coordinates within the mask\n"; + std::cout << "Calculated the pixel coordinates within the mask\n"; std::vector<std::vector<double>> allPixelValue; allPixelValue.reserve(imageNum); @@ -668,7 +686,7 @@ void photometricStereo::getPixelNormAndAlbedo(const int objectIndex) { } } - //std::cout << "Store all pixel values in a vector\n"; + std::cout << "Store all pixel values in a vector\n"; // threshold for every image; std::vector<double> allThreshold; @@ -680,7 +698,7 @@ void photometricStereo::getPixelNormAndAlbedo(const int objectIndex) { quickSelect(tmpPixelValue, 0, objectPixelNum - 1, thresholdIndex)); } - //std::cout << "Threshold for every image\n"; + std::cout << "Threshold for every image\n"; mN.create(3, objectPixelNum, CV_64F); mAlbedo.create(1, objectPixelNum, CV_64F); diff --git a/apps/specular_estimation/src/PhotometricStereo.h b/apps/specular_estimation/src/PhotometricStereo.h index b0ae968cbfd8af516bb2a1bcb1b14b931c265767..e759ccaf011c3f7d562fe5d22f2ab1b8020fa281 100755 --- a/apps/specular_estimation/src/PhotometricStereo.h +++ b/apps/specular_estimation/src/PhotometricStereo.h @@ -34,12 +34,13 @@ class photometricStereo { // use image with equal distance photometricStereo(int n, int startI, int endI, std::string path, std::string metal1Phere1Name, std::string metal2Phere1Name, std::string lambertPhereName, std::string objectName, double discardRatio = 0.1); photometricStereo(int imageNumber, std::string modelPath, std::string calibrationPath, std::string macbethPath, std::string imageName, std::string calibration, double discardRatio = 0.1); + photometricStereo(std::vector<cv::Mat> mp2Images, std::vector<cv::Mat> mp2CalibrationImages, std::vector<cv::Mat> mp2Mask, int imageNumber = 0, double discardRatio = 0.1); photometricStereo(const photometricStereo&) = delete; photometricStereo& operator = (const photometricStereo&) = delete; ~photometricStereo(); bool readImage(); // read the images and masks according to the ImageNames bool readImage(std::string modelPath, std::string calibrationPath, std::string macbethPath); - std::vector<cv::Mat> readImages(std::string path); + std::vector<cv::Mat> readImages(std::string path, int imageScale); void getLightInformation(const int metalIndex, const int lambIndex); void getLightInformation(const int metalIndex); void getPixelNormAndAlbedo(const int objectIndex); @@ -81,6 +82,8 @@ class photometricStereo { std::string calibrationPath; std::string imageName; std::string calibration; + int width; + int height; }; } diff --git a/apps/specular_estimation/src/specular_estimation.cc b/apps/specular_estimation/src/specular_estimation.cc index f4153d33f4a40a4f0b57cb45c6c34a22640698a2..d9fc1d6503ee125d256765a7d846c3f19cc703d4 100644 --- a/apps/specular_estimation/src/specular_estimation.cc +++ b/apps/specular_estimation/src/specular_estimation.cc @@ -56,7 +56,7 @@ void specularEstimation(std::string imageName, std::string calibration, double R // Define the paths for the model images, calibration images and the albedo const std::string imagesPath = "/home/thomas/Documents/"; - const std::string folderPath = "2017-12-04"; + const std::string folderPath = "2018-04-18"; const std::string materialPath = imagesPath + folderPath + "/" + imageName + "/" + imageName + "."; const std::string calibrationPath = imagesPath + folderPath + "/" + calibration + "/" + calibration + "."; const std::string charucoPath = imagesPath + folderPath + "/charuco/charuco."; @@ -65,23 +65,37 @@ void specularEstimation(std::string imageName, std::string calibration, double R const std::string normalPath = materialPath + "normal.png"; double LightDistance = 1.0, LightIntensity = 1.0, Gain = 1.0, Bias = 0.0; - int numberOfLights = 6, width = 1092, height = 728; + int numberOfLights = 6, width = 1092, height = 732; cv::Mat residualImage, albedo, normalMap, heightMap; - std::vector<cv::Mat> materialImages, charucoImages, calibrationImages; + std::vector<cv::Mat> materialImages, charucoImages, calibrationImages, mp2Mask; cv::Mat lightDirections = (cv::Mat_<float>(6,3) << 0.447712, 0.138562, 0.883377, 0.228758, -0.106536, 0.967636, 0.1, 0.0705882, 0.99248, 0.000653595, -0.0718954, 0.997412, -0.139216, -0.12549, 0.982279, -0.494771, 0.115033, 0.861376); if (!synthetic) { - int originalWidth, originalHeight, imageScale = 1; + int originalWidth, originalHeight, imageScale = 1; + cv::Mat cameraMatrix, distortionCoefficients, lightDirectionsPerspective; + + calibrateCamera(charucoPath, cameraMatrix, distortionCoefficients); + std::cout << "Camera parameters obtained.\n"; + + /* //std::cout << "Loading material images.\n"; - //materialImages = loadMaterialImages(materialPath, originalWidth, originalHeight, width, height, imageScale); + materialImages = loadMaterialImages(materialPath, originalWidth, originalHeight, width, height, imageScale); //std::cout << "Loading light source calibration images.\n"; - //calibrationImages = loadImages(calibrationPath); - //numberOfLights = calibrationImages.size(); + calibrationImages = loadImages(calibrationPath); + numberOfLights = calibrationImages.size(); + + std::string maskName = calibrationPath + "mask.png"; + cv::Mat p2Mask = cv::imread(maskName, cv::IMREAD_GRAYSCALE); + mp2Mask.push_back(p2Mask); + maskName = calibrationPath + "white.png"; + p2Mask = cv::imread(maskName, cv::IMREAD_GRAYSCALE); + //p2Mask = cv::Mat(p2Mask.size(), CV_8U, cv::Scalar(255)); + mp2Mask.push_back(p2Mask);*/ //cv::Mat lightDirectionsInverted; - cv::Mat cameraMatrix, distortionCoefficients, lightDirectionsPerspective; + //cv::Mat cameraMatrix, distortionCoefficients, lightDirectionsPerspective; /*cv::Mat lightDirections = (cv::Mat_<float>(6,3) << -0.57735, -0.57735, 0.57735, 0, 0.7071, 0.7071, 0, -0.7071, 0.7071, 0.7071, 0, 0.7071, -0.7071, 0, 0.7071, 0.57735, 0.57735, 0.57735);*/ //cv::invert(lightDirections, lightDirectionsInverted, cv::DECOMP_SVD); @@ -89,7 +103,9 @@ void specularEstimation(std::string imageName, std::string calibration, double R //charucoAlignment(charucoPath, charucoImages, materialImages, numberOfLights, width, height, lightDirections, lightDirectionsPerspective, cameraMatrix, distortionCoefficients); + //phoSte::photometricStereo PhotometricStereo(materialImages, calibrationImages, mp2Mask, numberOfLights, 0); phoSte::photometricStereo PhotometricStereo(numberOfLights, materialPath, calibrationPath, macbethPath, imageName, calibration, 0); + //PhotometricStereo.readImage(materialPath, calibrationPath, macbethPath); //cv::Mat calibrationMask = loadCalibrationMask(calibrationPath, height, width); //calibrationBoundingBox = getBoundingBox(calibrationMask); diff --git a/bin/specular_estimation b/bin/specular_estimation index 820d8011d34964bb0333787b0f6523776886cf3b..05d138dc76b37f7b723a99e82ee3c01b97583c7b 100755 Binary files a/bin/specular_estimation and b/bin/specular_estimation differ