diff --git a/apps/specular_estimation/CMakeFiles/specular_estimation.dir/src/specular_estimation.cc.o b/apps/specular_estimation/CMakeFiles/specular_estimation.dir/src/specular_estimation.cc.o
index dfb93a91c75d7f7857f4da4335851fee671bc3ed..b3973e1a6bf385a384000e9b3788f3a39a798915 100644
Binary files a/apps/specular_estimation/CMakeFiles/specular_estimation.dir/src/specular_estimation.cc.o and b/apps/specular_estimation/CMakeFiles/specular_estimation.dir/src/specular_estimation.cc.o differ
diff --git a/apps/specular_estimation/src/OpenCV.h b/apps/specular_estimation/src/OpenCV.h
index 45275791a991bce833001852c66ccc1ab81b4e27..866dbe6c8772cd4e8342dd1d11be2457462f5e12 100644
--- a/apps/specular_estimation/src/OpenCV.h
+++ b/apps/specular_estimation/src/OpenCV.h
@@ -135,10 +135,10 @@ void loadSurfaceNormals(cv::Mat lightDirections, int width, int height, int imag
cv::FileStorage storageWrite;
if (!normals.data) {
- //normals = cv::Mat(height, width, CV_32FC3, cv::Scalar::all(0));
- normals = cv::Mat(height, width, CV_32FC3, cv::Scalar(0,0,1));
+ normals = cv::Mat(height, width, CV_32FC3, cv::Scalar::all(0));
+ //normals = cv::Mat(height, width, CV_32FC3, cv::Scalar(0,0,1));
// Estimate the surface normals and p, q gradients
- //getSurfaceNormals(normals, pGradients, qGradients, lightDirectionsInverted, modelImages);
+ getSurfaceNormals(normals, pGradients, qGradients, lightDirectionsInverted, modelImages);
std::cout << "Surface normals have been estimated.\n\n";
storageWrite.open(normalMapName, cv::FileStorage::WRITE);
diff --git a/apps/specular_estimation/src/OpenGL.h b/apps/specular_estimation/src/OpenGL.h
index 08fda0c3449f6fec15ecedf1da16c006902905b2..1a59035961a322964ee7034ea75f806972449191 100644
--- a/apps/specular_estimation/src/OpenGL.h
+++ b/apps/specular_estimation/src/OpenGL.h
@@ -48,7 +48,7 @@ void renderShadowMap(GLuint quad_programID, GLuint depthTexture, GLuint texID, G
void computeResidual(int imageNumber, int windowHeight, int windowWidth, std::string modelPath, std::vector < cv::Mat > textureImages, cv::Mat& residual, cv::Vec3d& residuals, double& sum, float SpecularIntensity, float SpecularPower);
double returnResidual(int imageNumber, int windowHeight, int windowWidth, std::vector < cv::Mat > textureImages, cv::Mat& residual, cv::Vec3d& residuals, float SpecularIntensity, float SpecularPower);
void computeResiduals(cv::Vec3d& residual, std::vector< cv::Vec3d >& residuals, double& totalResidual, cv::Mat& residualImage, bool calculateResidual, glm::mat4 depthProjectionMatrix, glm::mat4 depthViewMatrix, int width, int height, glm::vec3& position, float& horizontalAngle, float& verticalAngle, float& FoV, glm::vec3& lightInvDir, cv::Mat lightDirections, std::vector< cv::Mat > textureImages, int& lightNumber, int numberOfLights, GLuint& SpecularIntensityID, float& SpecularIntensity, GLuint& SpecularPowerID, float& SpecularPower, GLuint programID, GLuint ModelMatrixID, GLuint ViewMatrixID, GLuint DepthBiasID, GLuint lightInvDirID, GLuint Texture, GLuint TextureID, GLuint depthTexture, GLuint ShadowMapID, GLuint vertexbuffer, GLuint uvbuffer, GLuint normalbuffer, GLuint elementbuffer, std::vector<unsigned int> indices, GLuint MatrixID, std::string modelPath, double& x, Problem& problem);
-void viewModel(cv::Vec3d& residual, std::vector< cv::Vec3d >& residuals, double& totalResidual, cv::Mat& residualImage, bool calculateResidual, glm::mat4 depthProjectionMatrix, glm::mat4 depthViewMatrix, int width, int height, glm::vec3& position, float& horizontalAngle, float& verticalAngle, float& FoV, glm::vec3& lightInvDir, cv::Mat lightDirections, std::vector< cv::Mat > textureImages, int& lightNumber, int numberOfLights, GLuint& SpecularIntensityID, float& SpecularIntensity, GLuint& SpecularPowerID, float& SpecularPower, GLuint programID, GLuint ModelMatrixID, GLuint ViewMatrixID, GLuint DepthBiasID, GLuint lightInvDirID, GLuint Texture, GLuint TextureID, GLuint depthTexture, GLuint ShadowMapID, GLuint vertexbuffer, GLuint uvbuffer, GLuint normalbuffer, GLuint elementbuffer, std::vector<unsigned int> indices, GLuint MatrixID);
+void viewModel(cv::Vec3d& residual, std::vector< cv::Vec3d >& residuals, double& totalResidual, cv::Mat& residualImage, bool calculateResidual, glm::mat4 depthProjectionMatrix, glm::mat4 depthViewMatrix, int width, int height, glm::vec3& position, float& horizontalAngle, float& verticalAngle, float& FoV, glm::vec3& lightInvDir, cv::Mat lightDirections, std::vector< cv::Mat > textureImages, int& lightNumber, int numberOfLights, GLuint& SpecularIntensityID, float& SpecularIntensity, GLuint& SpecularPowerID, float& SpecularPower, GLuint programID, GLuint ModelMatrixID, GLuint ViewMatrixID, GLuint DepthBiasID, GLuint lightInvDirID, GLuint Texture, GLuint TextureID, GLuint depthTexture, GLuint ShadowMapID, GLuint vertexbuffer, GLuint uvbuffer, GLuint normalbuffer, GLuint elementbuffer, std::vector<unsigned int> indices, GLuint MatrixID, std::string modelPath);
void initialiseOpenGL(cv::Mat cv_depth, cv::Mat cv_normals, cv::Mat cv_texture, std::vector< cv::Mat > textureImages, cv::Mat lightDirections, int& windowWidth, int& windowHeight, glm::mat4& depthProjectionMatrix, glm::mat4& depthViewMatrix, glm::vec3& position, float& horizontalAngle, float& verticalAngle, float& FoV, glm::vec3& lightInvDir, GLuint& programID, GLuint& MatrixID, GLuint& ModelMatrixID, GLuint& ViewMatrixID, GLuint& DepthBiasID, GLuint& lightInvDirID, GLuint& Texture, GLuint& TextureID, GLuint& depthTexture, GLuint& ShadowMapID, GLuint& vertexbuffer, GLuint& uvbuffer, GLuint& normalbuffer, GLuint& elementbuffer, std::vector<unsigned int> & indices, GLuint& depthProgramID, GLuint& quad_programID, GLuint& FramebufferName, GLuint& quad_vertexbuffer, GLuint& VertexArrayID, GLuint& SpecularIntensityID, float& SpecularIntensity, GLuint& SpecularPowerID, float& SpecularPower) {
// Initialise the GLFW library first
@@ -487,8 +487,10 @@ void computeResidual(int imageNumber, int windowHeight, int windowWidth, std::st
// Calculate the absolute differences between the model and the photograph
cv::absdiff(temp, textureImages[imageNumber], residual);
+ sumOfSquaredDifferences = residual;
+
// Square each element
- multiply(residual, residual, sumOfSquaredDifferences);
+ multiply(sumOfSquaredDifferences, sumOfSquaredDifferences, sumOfSquaredDifferences);
std::ostringstream stm;
stm << imageNumber;
@@ -500,12 +502,13 @@ void computeResidual(int imageNumber, int windowHeight, int windowWidth, std::st
//cv::namedWindow("Sum of Squared Differences", CV_WINDOW_NORMAL);
//cv::imshow("Sum of Squared Differences", sumOfSquaredDifferences);
cv::imwrite(modelPath + "model." + number + ".png", temp);
- cv::imwrite(modelPath + "residual." + number + ".png", sumOfSquaredDifferences);
+ cv::imwrite(modelPath + "residual." + number + ".png", residual);
// Crop a 1-pixel border around the residual
sumOfSquaredDifferences = sumOfSquaredDifferences(cv::Rect(1, 1, windowWidth-2, windowHeight-2));
- sum = (cv::sum(sumOfSquaredDifferences)[0] + cv::sum(sumOfSquaredDifferences)[1] + cv::sum(sumOfSquaredDifferences)[2])/(windowHeight * windowWidth);
+ //sum = (cv::sum(sumOfSquaredDifferences)[0] + cv::sum(sumOfSquaredDifferences)[1] + cv::sum(sumOfSquaredDifferences)[2])/(windowHeight-2 * windowWidth-2);
+ sum = (cv::sum(sumOfSquaredDifferences)[0] + cv::sum(sumOfSquaredDifferences)[1] + cv::sum(sumOfSquaredDifferences)[2]);
residuals = cv::Vec3d(cv::sum(sumOfSquaredDifferences)[0], cv::sum(sumOfSquaredDifferences)[1], cv::sum(sumOfSquaredDifferences)[2]);
@@ -516,7 +519,8 @@ void computeResidual(int imageNumber, int windowHeight, int windowWidth, std::st
sum = cv::sum(residual)[0];
*/
- std::cout << "Image " << imageNumber << ", sum of squared differences per pixel: R = " << cv::sum(sumOfSquaredDifferences)[2]/(windowHeight * windowWidth) << ", G = " << cv::sum(sumOfSquaredDifferences)[1]/(windowHeight * windowWidth) << ", B = " << cv::sum(sumOfSquaredDifferences)[0]/(windowHeight * windowWidth) << ", specular intensity = " << SpecularIntensity << ", specular power = " << SpecularPower << std::endl;
+ //std::cout << "Image " << imageNumber << ", SSD per pixel: R = " << cv::sum(sumOfSquaredDifferences)[2]/(windowHeight * windowWidth) << ", G = " << cv::sum(sumOfSquaredDifferences)[1]/(windowHeight * windowWidth) << ", B = " << cv::sum(sumOfSquaredDifferences)[0]/(windowHeight * windowWidth) << ", specular intensity = " << SpecularIntensity << ", specular power = " << SpecularPower << std::endl;
+ //std::cout << "Image " << imageNumber << ", average sum of squared differences: R = " << sum << ", specular intensity = " << SpecularIntensity << ", specular power = " << SpecularPower << std::endl;
}
double returnResidual(int imageNumber, int windowHeight, int windowWidth, std::vector < cv::Mat > textureImages, cv::Mat& residual, cv::Vec3d& residuals, float SpecularIntensity, float SpecularPower) {
@@ -620,7 +624,7 @@ void computeResiduals(cv::Vec3d& residual, std::vector< cv::Vec3d >& residuals,
lightNumber = 0;
}
-void viewModel(cv::Vec3d& residual, std::vector< cv::Vec3d >& residuals, double& totalResidual, cv::Mat& residualImage, bool calculateResidual, glm::mat4 depthProjectionMatrix, glm::mat4 depthViewMatrix, int width, int height, glm::vec3& position, float& horizontalAngle, float& verticalAngle, float& FoV, glm::vec3& lightInvDir, cv::Mat lightDirections, std::vector< cv::Mat > textureImages, int& lightNumber, int numberOfLights, GLuint& SpecularIntensityID, float& SpecularIntensity, GLuint& SpecularPowerID, float& SpecularPower, GLuint programID, GLuint ModelMatrixID, GLuint ViewMatrixID, GLuint DepthBiasID, GLuint lightInvDirID, GLuint Texture, GLuint TextureID, GLuint depthTexture, GLuint ShadowMapID, GLuint vertexbuffer, GLuint uvbuffer, GLuint normalbuffer, GLuint elementbuffer, std::vector<unsigned int> indices, GLuint MatrixID) {
+void viewModel(cv::Vec3d& residual, std::vector< cv::Vec3d >& residuals, double& totalResidual, cv::Mat& residualImage, bool calculateResidual, glm::mat4 depthProjectionMatrix, glm::mat4 depthViewMatrix, int width, int height, glm::vec3& position, float& horizontalAngle, float& verticalAngle, float& FoV, glm::vec3& lightInvDir, cv::Mat lightDirections, std::vector< cv::Mat > textureImages, int& lightNumber, int numberOfLights, GLuint& SpecularIntensityID, float& SpecularIntensity, GLuint& SpecularPowerID, float& SpecularPower, GLuint programID, GLuint ModelMatrixID, GLuint ViewMatrixID, GLuint DepthBiasID, GLuint lightInvDirID, GLuint Texture, GLuint TextureID, GLuint depthTexture, GLuint ShadowMapID, GLuint vertexbuffer, GLuint uvbuffer, GLuint normalbuffer, GLuint elementbuffer, std::vector<unsigned int> indices, GLuint MatrixID, std::string modelPath) {
do {
// Render the shadows
glm::mat4 depthModelMatrix = glm::mat4(1.0);
@@ -631,6 +635,7 @@ void viewModel(cv::Vec3d& residual, std::vector< cv::Vec3d >& residuals, double&
glm::mat4 ModelMatrix = glm::mat4(1.0);
glm::mat4 MVP, ViewMatrix, depthBiasMVP, ProjectionMatrix;
bool perspectiveProjection, shadowControl;
+ calculateResidual = false;
//computeMatricesFromInputs(windowWidth, windowHeight, position, horizontalAngle, verticalAngle, FoV, mouseSpeed, ProjectionMatrix, ViewMatrix, lightInvDir, depthProjectionMatrix, depthViewMatrix, perspectiveProjection, shadowControl, SpecularIntensityID, SpecularIntensity, SpecularPowerID, SpecularPower, calculateResidual);
computeMatricesFromLights(width, height, position, horizontalAngle, verticalAngle, FoV, ProjectionMatrix, ViewMatrix, lightInvDir, depthProjectionMatrix, depthViewMatrix, perspectiveProjection = false, lightDirections, textureImages, lightNumber, numberOfLights, SpecularIntensityID, SpecularIntensity, SpecularPowerID, SpecularPower, calculateResidual);
MVP = ProjectionMatrix * ViewMatrix * ModelMatrix;
@@ -650,7 +655,14 @@ void viewModel(cv::Vec3d& residual, std::vector< cv::Vec3d >& residuals, double&
// Optionally render the shadowmap (for debug only)
//void renderShadowMap(GLuint quad_programID, GLuint depthTexture, GLuint texID, GLuint quad_vertexbuffer);
-
+
+ if (calculateResidual) {
+ //computeResidual(lightNumber, height, width, modelPath, textureImages, residualImage, residual, totalResidual, SpecularIntensity, SpecularPower);
+ double x = SpecularIntensity;
+ ceres::Problem problem;
+ computeResiduals(residual, residuals, totalResidual, residualImage, calculateResidual, depthProjectionMatrix, depthViewMatrix, width, height, position, horizontalAngle, verticalAngle, FoV, lightInvDir, lightDirections, textureImages, lightNumber, numberOfLights, SpecularIntensityID, SpecularIntensity, SpecularPowerID, SpecularPower, programID, ModelMatrixID, ViewMatrixID, DepthBiasID, lightInvDirID, Texture, TextureID, depthTexture, ShadowMapID, vertexbuffer, uvbuffer, normalbuffer, elementbuffer, indices, MatrixID, modelPath, x, problem);
+ std::cout << "Average residual = " << totalResidual << ", specular intensity = " << SpecularIntensity << ", specular power = " << SpecularPower << std::endl;
+ }
/*
@@ -701,6 +713,15 @@ void computeMatricesFromLights(int windowWidth, int windowHeight, glm::vec3& pos
// Up vector
glm::vec3 up = glm::cross(right, direction);
+ float modifier = 1.0f;
+
+ // Control modifiers
+ if (glfwGetKey(window, GLFW_KEY_LEFT_SHIFT) == GLFW_PRESS) {
+ modifier = 0.1f;
+ } else if (glfwGetKey(window, GLFW_KEY_LEFT_CONTROL) == GLFW_PRESS) {
+ modifier = 0.01f;
+ }
+
lightInvDir = glm::vec3(lightDirections.at<float>(lightNumber, 1), -lightDirections.at<float>(lightNumber, 0), lightDirections.at<float>(lightNumber, 2));
// Change light direction
@@ -732,15 +753,15 @@ void computeMatricesFromLights(int windowWidth, int windowHeight, glm::vec3& pos
// Modify specular power
if (glfwGetKey(window, GLFW_KEY_7) == GLFW_PRESS) {
//if (glfwGetKey(window, GLFW_KEY_7) == GLFW_RELEASE) {
- if (SpecularPower > 0.1f) {
- SpecularPower -= 0.1f;
+ if (SpecularPower > 0.1*modifier) {
+ SpecularPower -= 0.1*modifier;
calculateResidual = true;
}
//}
}
else if (glfwGetKey(window, GLFW_KEY_8) == GLFW_PRESS) {
//if (glfwGetKey(window, GLFW_KEY_8) == GLFW_RELEASE) {
- SpecularPower += 0.1f;
+ SpecularPower += 0.1*modifier;
calculateResidual = true;
//}
}
@@ -748,15 +769,15 @@ void computeMatricesFromLights(int windowWidth, int windowHeight, glm::vec3& pos
// Modify specular intensity
if (glfwGetKey(window, GLFW_KEY_9) == GLFW_PRESS) {
//if (glfwGetKey(window, GLFW_KEY_9) == GLFW_RELEASE) {
- if (SpecularIntensity > 0.1f) {
- SpecularIntensity -= 0.1f;
+ if (SpecularIntensity > 0.1*modifier) {
+ SpecularIntensity -= 0.1*modifier;
calculateResidual = true;
}
//}
}
else if (glfwGetKey(window, GLFW_KEY_0) == GLFW_PRESS) {
//if (glfwGetKey(window, GLFW_KEY_0) == GLFW_RELEASE) {
- SpecularIntensity += 0.1f;
+ SpecularIntensity += 0.1*modifier;
calculateResidual = true;
//}
}
diff --git a/apps/specular_estimation/src/specular_estimation.cc b/apps/specular_estimation/src/specular_estimation.cc
index f667c46125c986e3d6e91c05d655999cdb721321..f5a72e278764bedcd1cc9ab725d48c0f311b78fb 100644
--- a/apps/specular_estimation/src/specular_estimation.cc
+++ b/apps/specular_estimation/src/specular_estimation.cc
@@ -142,9 +142,15 @@ int main(int argc, char** argv) {
std::cout << summary.BriefReport() << "\n";
std::cout << "Specular Intensity: " << initial_x << " -> " << x << "\n";
- viewModel(residual, residuals, totalResidual, residualImage, calculateResidual, depthProjectionMatrix, depthViewMatrix, width, height, position, horizontalAngle, verticalAngle, FoV, lightInvDir, lightDirections, textureImages, lightNumber, numberOfLights, SpecularIntensityID, SpecularIntensity, SpecularPowerID, SpecularPower, programID, ModelMatrixID, ViewMatrixID, DepthBiasID, lightInvDirID, Texture, TextureID, depthTexture, ShadowMapID, vertexbuffer, uvbuffer, normalbuffer, elementbuffer, indices, MatrixID);
+ viewModel(residual, residuals, totalResidual, residualImage, calculateResidual, depthProjectionMatrix, depthViewMatrix, width, height, position, horizontalAngle, verticalAngle, FoV, lightInvDir, lightDirections, textureImages, lightNumber, numberOfLights, SpecularIntensityID, SpecularIntensity, SpecularPowerID, SpecularPower, programID, ModelMatrixID, ViewMatrixID, DepthBiasID, lightInvDirID, Texture, TextureID, depthTexture, ShadowMapID, vertexbuffer, uvbuffer, normalbuffer, elementbuffer, indices, MatrixID, modelPath);
terminateOpenGL(vertexbuffer, uvbuffer, normalbuffer, elementbuffer, programID, depthProgramID, quad_programID, Texture, FramebufferName, depthTexture, quad_vertexbuffer, VertexArrayID);
+ // wood6: Average residual = 8.92126e+08, difference = -27795, specular intensity = 0.234, specular power = 2
+ // shirt6: Average residual = 6.80489e+08, difference = -38358, specular intensity = 0.0450001, specular power = 2
+ // greenpaper6: Average residual = 8.46285e+08, difference = -12212, specular intensity = 0.177, specular power = 2
+ // plastic6: Average residual = 9.30516e+08, difference = -30793, specular intensity = 0.42, specular power = 2
+
+
return 0;
}
diff --git a/bin/specular_estimation b/bin/specular_estimation
index a190f979325200bd142905ed12407a9b0fcf043d..cf077aa8ad088eed03d0a8a8b39d7130a0004304 100755
Binary files a/bin/specular_estimation and b/bin/specular_estimation differ