Commit d8bc42fb authored by Matthew Waters's avatar Matthew Waters

glutils: fix matrix operations everywhere

- correct the matrix multiplication
- Use column-major matrices
  - reverse order of matrix multiplications

https://bugzilla.gnome.org/show_bug.cgi?id=785980
parent 05a2aa33
......@@ -2237,10 +2237,14 @@ gst_glimage_sink_on_draw (GstGLImageSink * gl_sink)
gst_buffer_get_video_affine_transformation_meta
(gl_sink->stored_buffer[0]);
gst_gl_get_affine_transformation_meta_as_ndc_ext (af_meta, matrix);
if (gl_sink->transform_matrix) {
gfloat tmp[16];
if (gl_sink->transform_matrix)
gst_gl_multiply_matrix4 (gl_sink->transform_matrix, matrix, matrix);
gst_gl_get_affine_transformation_meta_as_ndc_ext (af_meta, tmp);
gst_gl_multiply_matrix4 (tmp, gl_sink->transform_matrix, matrix);
} else {
gst_gl_get_affine_transformation_meta_as_ndc_ext (af_meta, matrix);
}
gst_gl_shader_set_uniform_matrix_4fv (gl_sink->redisplay_shader,
"u_transformation", 1, FALSE, matrix);
......
......@@ -770,20 +770,6 @@ gst_gl_transformation_gl_start (GstGLBaseFilter * base_filter)
return TRUE;
}
static const gfloat from_ndc_matrix[] = {
0.5f, 0.0f, 0.0, 0.5f,
0.0f, 0.5f, 0.0, 0.5f,
0.0f, 0.0f, 0.5, 0.5f,
0.0f, 0.0f, 0.0, 1.0f,
};
static const gfloat to_ndc_matrix[] = {
2.0f, 0.0f, 0.0, -1.0f,
0.0f, 2.0f, 0.0, -1.0f,
0.0f, 0.0f, 2.0, -1.0f,
0.0f, 0.0f, 0.0, 1.0f,
};
static GstFlowReturn
gst_gl_transformation_prepare_output_buffer (GstBaseTransform * trans,
GstBuffer * inbuf, GstBuffer ** outbuf)
......@@ -794,7 +780,8 @@ gst_gl_transformation_prepare_output_buffer (GstBaseTransform * trans,
if (transformation->downstream_supports_affine_meta &&
gst_video_info_is_equal (&filter->in_info, &filter->out_info)) {
GstVideoAffineTransformationMeta *af_meta;
graphene_matrix_t upstream_matrix, from_ndc, to_ndc, tmp, tmp2, inv_aspect;
graphene_matrix_t upstream_matrix, tmp, tmp2, inv_aspect, yflip;
float upstream[16], downstream[16];
*outbuf = gst_buffer_make_writable (inbuf);
......@@ -805,19 +792,23 @@ gst_gl_transformation_prepare_output_buffer (GstBaseTransform * trans,
GST_LOG_OBJECT (trans, "applying transformation to existing affine "
"transformation meta");
gst_gl_get_affine_transformation_meta_as_ndc_ext (af_meta, upstream);
/* apply the transformation to the existing affine meta */
graphene_matrix_init_from_float (&from_ndc, from_ndc_matrix);
graphene_matrix_init_from_float (&to_ndc, to_ndc_matrix);
graphene_matrix_init_from_float (&upstream_matrix, af_meta->matrix);
graphene_matrix_init_from_float (&upstream_matrix, upstream);
graphene_matrix_init_scale (&inv_aspect, transformation->aspect, -1., 1.);
graphene_matrix_init_scale (&yflip, 1., -1., 1.);
graphene_matrix_init_scale (&inv_aspect, transformation->aspect, 1., 1.);
/* invert the aspect effects */
graphene_matrix_multiply (&upstream_matrix, &inv_aspect, &tmp2);
/* apply the transformation */
graphene_matrix_multiply (&tmp2, &transformation->mvp_matrix, &tmp);
/* and undo yflip */
graphene_matrix_multiply (&tmp, &yflip, &tmp2);
graphene_matrix_multiply (&from_ndc, &upstream_matrix, &tmp);
graphene_matrix_multiply (&tmp, &transformation->mvp_matrix, &tmp2);
graphene_matrix_multiply (&tmp2, &inv_aspect, &tmp);
graphene_matrix_multiply (&tmp, &to_ndc, &tmp2);
graphene_matrix_to_float (&tmp2, downstream);
gst_gl_set_affine_transformation_meta_from_ndc_ext (af_meta, downstream);
graphene_matrix_to_float (&tmp2, af_meta->matrix);
return GST_FLOW_OK;
}
......
......@@ -108,24 +108,24 @@ gst_gl_context_gen_shader (GstGLContext * context, const gchar * vert_src,
}
static const gfloat identity_matrix[] = {
1.0f, 0.0f, 0.0, 0.0f,
0.0f, 1.0f, 0.0, 0.0f,
0.0f, 0.0f, 1.0, 0.0f,
0.0f, 0.0f, 0.0, 1.0f,
1.0, 0.0, 0.0, 0.0,
0.0, 1.0, 0.0, 0.0,
0.0, 0.0, 1.0, 0.0,
0.0, 0.0, 0.0, 1.0,
};
static const gfloat from_ndc_matrix[] = {
0.5f, 0.0f, 0.0, 0.5f,
0.0f, 0.5f, 0.0, 0.5f,
0.0f, 0.0f, 0.5, 0.5f,
0.0f, 0.0f, 0.0, 1.0f,
0.5, 0.0, 0.0, 0.0,
0.0, 0.5, 0.0, 0.0,
0.0, 0.0, 0.5, 0.0,
0.5, 0.5, 0.5, 1.0,
};
static const gfloat to_ndc_matrix[] = {
2.0f, 0.0f, 0.0, -1.0f,
0.0f, 2.0f, 0.0, -1.0f,
0.0f, 0.0f, 2.0, -1.0f,
0.0f, 0.0f, 0.0, 1.0f,
2.0, 0.0, 0.0, 0.0,
0.0, 2.0, 0.0, 0.0,
0.0, 0.0, 2.0, 0.0,
-1.0, -1.0, -1.0, 1.0,
};
void
......@@ -137,10 +137,10 @@ gst_gl_multiply_matrix4 (const gfloat * a, const gfloat * b, gfloat * result)
if (!a || !b || !result)
return;
for (i = 0; i < 4; i++) {
for (j = 0; j < 4; j++) {
for (i = 0; i < 4; i++) { /* column */
for (j = 0; j < 4; j++) { /* row */
for (k = 0; k < 4; k++) {
tmp[i + (j * 4)] += a[i + (k * 4)] * b[k + (j * 4)];
tmp[j + (i * 4)] += a[k + (i * 4)] * b[j + (k * 4)];
}
}
}
......@@ -159,9 +159,20 @@ void gst_gl_get_affine_transformation_meta_as_ndc_ext
matrix[i] = identity_matrix[i];
}
} else {
gfloat tmp[16] = { 0.0f };
float tmp[16];
gst_gl_multiply_matrix4 (from_ndc_matrix, meta->matrix, tmp);
gst_gl_multiply_matrix4 (tmp, to_ndc_matrix, matrix);
gst_gl_multiply_matrix4 (to_ndc_matrix, meta->matrix, tmp);
gst_gl_multiply_matrix4 (tmp, from_ndc_matrix, matrix);
}
}
void gst_gl_set_affine_transformation_meta_from_ndc_ext
(GstVideoAffineTransformationMeta * meta, const gfloat * matrix)
{
float tmp[16];
g_return_if_fail (meta != NULL);
gst_gl_multiply_matrix4 (from_ndc_matrix, matrix, tmp);
gst_gl_multiply_matrix4 (tmp, to_ndc_matrix, meta->matrix);
}
......@@ -31,6 +31,7 @@ gboolean gst_gl_context_gen_shader (GstGLContext * context,
void gst_gl_multiply_matrix4 (const gfloat * a, const gfloat * b, gfloat * result);
void gst_gl_get_affine_transformation_meta_as_ndc_ext (GstVideoAffineTransformationMeta *
meta, gfloat * matrix);
void gst_gl_set_affine_transformation_meta_from_ndc_ext (GstVideoAffineTransformationMeta * meta, const gfloat * matrix);
G_END_DECLS
......
......@@ -779,32 +779,35 @@ gst_gl_value_set_texture_target_from_mask (GValue * value,
ret = TRUE;
}
g_value_unset(&item);
g_value_unset (&item);
return ret;
}
}
static const gfloat identity_matrix[] = {
1.0f, 0.0f, 0.0, 0.0f,
0.0f, 1.0f, 0.0, 0.0f,
0.0f, 0.0f, 1.0, 0.0f,
0.0f, 0.0f, 0.0, 1.0f,
1.0, 0.0, 0.0, 0.0,
0.0, 1.0, 0.0, 0.0,
0.0, 0.0, 1.0, 0.0,
0.0, 0.0, 0.0, 1.0,
};
static const gfloat from_ndc_matrix[] = {
0.5f, 0.0f, 0.0, 0.5f,
0.0f, 0.5f, 0.0, 0.5f,
0.0f, 0.0f, 0.5, 0.5f,
0.0f, 0.0f, 0.0, 1.0f,
0.5, 0.0, 0.0, 0.0,
0.0, 0.5, 0.0, 0.0,
0.0, 0.0, 0.5, 0.0,
0.5, 0.5, 0.5, 1.0,
};
static const gfloat to_ndc_matrix[] = {
2.0f, 0.0f, 0.0, -1.0f,
0.0f, 2.0f, 0.0, -1.0f,
0.0f, 0.0f, 2.0, -1.0f,
0.0f, 0.0f, 0.0, 1.0f,
2.0, 0.0, 0.0, 0.0,
0.0, 2.0, 0.0, 0.0,
0.0, 0.0, 2.0, 0.0,
-1.0, -1.0, -1.0, 1.0,
};
/* multiplies two 4x4 matrices, @a X @b, and stores the result in @result
* https://en.wikipedia.org/wiki/Matrix_multiplication
*/
static void
gst_gl_multiply_matrix4 (const gfloat * a, const gfloat * b, gfloat * result)
{
......@@ -813,11 +816,10 @@ gst_gl_multiply_matrix4 (const gfloat * a, const gfloat * b, gfloat * result)
if (!a || !b || !result)
return;
for (i = 0; i < 4; i++) {
for (j = 0; j < 4; j++) {
for (i = 0; i < 4; i++) { /* column */
for (j = 0; j < 4; j++) { /* row */
for (k = 0; k < 4; k++) {
tmp[i + (j * 4)] += a[i + (k * 4)] * b[k + (j * 4)];
tmp[j + (i * 4)] += a[k + (i * 4)] * b[j + (k * 4)];
}
}
}
......@@ -826,6 +828,19 @@ gst_gl_multiply_matrix4 (const gfloat * a, const gfloat * b, gfloat * result)
result[i] = tmp[i];
}
/*
* gst_gl_get_affine_transformation_meta_as_ndc:
* @meta: (nullable): a #GstVideoAffineTransformationMeta
* @matrix: (out): result of the 4x4 matrix
*
* Retrieves the stored 4x4 affine transformation matrix stored in @meta in
* NDC coordinates. if @meta is NULL, an identity matrix is returned.
*
* NDC is a left-handed coordinate sytem
* - x - [-1, 1] - +ve X moves right
* - y - [-1, 1] - +ve Y moves up
* - z - [-1, 1] - +ve Z moves into
*/
void
gst_gl_get_affine_transformation_meta_as_ndc (GstVideoAffineTransformationMeta *
meta, gfloat * matrix)
......@@ -837,9 +852,22 @@ gst_gl_get_affine_transformation_meta_as_ndc (GstVideoAffineTransformationMeta *
matrix[i] = identity_matrix[i];
}
} else {
gfloat tmp[16] = { 0.0f };
float tmp[16];
gst_gl_multiply_matrix4 (from_ndc_matrix, meta->matrix, tmp);
gst_gl_multiply_matrix4 (tmp, to_ndc_matrix, matrix);
/* change of basis multiplications */
gst_gl_multiply_matrix4 (to_ndc_matrix, meta->matrix, tmp);
gst_gl_multiply_matrix4 (tmp, from_ndc_matrix, matrix);
}
}
void gst_gl_set_affine_transformation_meta_from_ndc
(GstVideoAffineTransformationMeta * meta, const gfloat * matrix)
{
float tmp[16];
g_return_if_fail (meta != NULL);
/* change of basis multiplications */
gst_gl_multiply_matrix4 (from_ndc_matrix, matrix, tmp);
gst_gl_multiply_matrix4 (tmp, to_ndc_matrix, meta->matrix);
}
......@@ -27,6 +27,7 @@ G_BEGIN_DECLS
G_GNUC_INTERNAL gboolean gst_gl_run_query (GstElement * element, GstQuery * query, GstPadDirection direction);
G_GNUC_INTERNAL void gst_gl_get_affine_transformation_meta_as_ndc (GstVideoAffineTransformationMeta * meta, gfloat * matrix);
G_GNUC_INTERNAL void gst_gl_set_affine_transformation_meta_from_ndc (GstVideoAffineTransformationMeta * meta, const gfloat * matrix);
G_END_DECLS
......
......@@ -209,6 +209,7 @@ check_gl=libs/gstglcontext \
libs/gstglmemory \
libs/gstglupload \
libs/gstglcolorconvert \
libs/gstglmatrix \
libs/gstglquery \
libs/gstglsl \
libs/gstglheaders \
......@@ -588,6 +589,16 @@ libs_gstglcolorconvert_LDADD = \
$(GST_PLUGINS_BASE_LIBS) $(GST_VIDEO_LIBS) $(GST_BASE_LIBS) \
$(GST_LIBS) $(LDADD)
libs_gstglmatrix_CFLAGS = \
$(GST_PLUGINS_BAD_CFLAGS) $(GST_PLUGINS_BASE_CFLAGS) \
-DGST_USE_UNSTABLE_API \
$(GST_BASE_CFLAGS) $(GST_CFLAGS) $(AM_CFLAGS)
libs_gstglmatrix_LDADD = \
$(top_builddir)/gst-libs/gst/gl/libgstgl-@GST_API_VERSION@.la \
$(GST_PLUGINS_BASE_LIBS) $(GST_VIDEO_LIBS) $(GST_BASE_LIBS) \
$(GST_LIBS) $(LDADD)
libs_gstglquery_LDADD = \
$(top_builddir)/gst-libs/gst/gl/libgstgl-@GST_API_VERSION@.la \
$(GST_PLUGINS_BASE_LIBS) $(GST_BASE_LIBS) $(GST_LIBS) $(LDADD)
......
......@@ -13,3 +13,4 @@ gstglcolorconvert
gstglsl
gstglquery
gstglheaders
gstglmatrix
/* GStreamer
*
* Copyright (C) 2014 Matthew Waters <ystreet00@gmail.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifdef HAVE_CONFIG_H
# include "config.h"
#endif
#include <gst/gl/gstglutils.c>
#undef GST_CAT_DEFAULT
#include <gst/check/gstcheck.h>
#define EPSILON 0.0001f
#define FEQ(a,b) (fabs(a-b) < EPSILON)
static void
debug_matrix (const float *m)
{
int i;
for (i = 0; i < 4; i++) {
GST_DEBUG ("%10.4f %10.4f %10.4f %10.4f", m[i * 4 + 0], m[i * 4 + 1],
m[i * 4 + 2], m[i * 4 + 3]);
}
}
GST_START_TEST (test_matrix_multiply)
{
/* A * B == C */
const float A[] = {
1., 1., 2., 5.,
0., 3., 0., 1.,
2., 0., 3., 1.,
3., 2., 1., 0.,
};
const float B[] = {
3., 1., 0., 2.,
1., 0., 3., 2.,
0., 1., 2., 3.,
3., 2., 1., 0.,
};
const float C[] = {
19., 13., 12., 10.,
6., 2., 10., 6.,
9., 7., 7., 13.,
11., 4., 8., 13.,
};
float res[16];
int i;
gst_gl_multiply_matrix4 (A, B, res);
GST_DEBUG ("result");
debug_matrix (res);
for (i = 0; i < G_N_ELEMENTS (res); i++) {
fail_unless (FEQ (res[i], C[i]), "value %f at index %u does not match "
"expected value %f", res[i], i, C[i]);
}
}
GST_END_TEST;
GST_START_TEST (test_matrix_ndc)
{
GstBuffer *buffer = gst_buffer_new ();
GstVideoAffineTransformationMeta *aff_meta;
float res[16];
int i;
const float m[] = {
1., 0., 0., 0.,
0., 1., 0., 0.,
0., 0., 1., 0.,
0., 0., 0., 1.,
};
const float n[] = {
4., 6., 4., 9.,
1., 5., 8., 2.,
9., 3., 5., 8.,
3., 7., 9., 1.,
};
aff_meta = gst_buffer_add_video_affine_transformation_meta (buffer);
/* test default identity matrix */
gst_gl_get_affine_transformation_meta_as_ndc (aff_meta, res);
GST_DEBUG ("result");
debug_matrix (res);
for (i = 0; i < G_N_ELEMENTS (res); i++) {
fail_unless (FEQ (res[i], m[i]), "value %f at index %u does not match "
"expected value %f", res[i], i, m[i]);
}
/* test setting and receiving the same values */
gst_gl_set_affine_transformation_meta_from_ndc (aff_meta, n);
gst_gl_get_affine_transformation_meta_as_ndc (aff_meta, res);
GST_DEBUG ("result");
debug_matrix (res);
for (i = 0; i < G_N_ELEMENTS (res); i++) {
fail_unless (FEQ (res[i], n[i]), "value %f at index %u does not match "
"expected value %f", res[i], i, n[i]);
}
gst_buffer_unref (buffer);
}
GST_END_TEST;
#if 0
static void
transpose_matrix4 (float *m, float *res)
{
int i, j;
for (i = 0; i < 4; i++) {
for (j = 0; j < 4; j++) {
int idx = i + (j * 4);
int swapped_idx = j + (i * 4);
if (i == j)
fail_unless (idx == swapped_idx);
res[swapped_idx] = m[idx];
}
}
}
static float
dot4 (float *v1, float *v2)
{
GST_TRACE ("%.4f * %.4f + %.4f * %.4f + %.4f * %.4f + %.4f * %.4f",
v1[0], v2[0], v1[1], v2[1], v1[2], v2[2], v1[3], v2[3]);
return v1[0] * v2[0] + v1[1] * v2[1] + v1[2] * v2[2] + v1[3] * v2[3];
}
/* m * v */
static void
_matrix_mult_vertex4 (float *m, float *v, float *res)
{
res[0] = dot4 (&m[0], v);
res[1] = dot4 (&m[4], v);
res[2] = dot4 (&m[8], v);
res[3] = dot4 (&m[12], v);
}
/* v * m */
static void
_vertex_mult_matrix4 (float *v, float *m, float *res)
{
float tmp[16] = { 0., };
transpose_matrix4 (m, tmp);
_matrix_mult_vertex4 (tmp, v, res);
}
GST_START_TEST (test_matrix_vertex_identity)
{
float identity[] = {
1., 0., 0., 0.,
0., 1., 0., 0.,
0., 0., 1., 0.,
0., 0., 0., 1.,
};
float v[] = { 1., 1., 1., 1. };
float res[4] = { 0., };
int i;
_vertex_mult_matrix4 (v, identity, res);
GST_DEBUG ("vertex: %.4f %.4f %.4f %.4f", v[0], v[1], v[2], v[3]);
GST_DEBUG ("result: %.4f %.4f %.4f %.4f", res[0], res[1], res[2], res[3]);
for (i = 0; i < 4; i++) {
fail_unless (FEQ (res[i], v[i]), "value %f at index %u does not match "
"expected value %f", res[i], i, v[i]);
}
_matrix_mult_vertex4 (identity, v, res);
GST_DEBUG ("vertex: %.4f %.4f %.4f %.4f", v[0], v[1], v[2], v[3]);
GST_DEBUG ("result: %.4f %.4f %.4f %.4f", res[0], res[1], res[2], res[3]);
for (i = 0; i < 4; i++) {
fail_unless (FEQ (res[i], v[i]), "value %f at index %u does not match "
"expected value %f", res[i], i, v[i]);
}
}
GST_END_TEST;
GST_START_TEST (test_matrix_vertex_scale)
{
float scale[] = {
1.5, 0., 0., 0.,
0., 2.5, 0., 0.,
0., 0., 3., 0.,
0., 0., 0., 1.,
};
float v[] = { 1., 1., 1., 1. };
float expected[] = { 1.5, 2.5, 3., 1. };
float res[4] = { 0., };
int i;
_vertex_mult_matrix4 (v, scale, res);
GST_DEBUG ("vertex: %.4f %.4f %.4f %.4f", v[0], v[1], v[2], v[3]);
GST_DEBUG ("result: %.4f %.4f %.4f %.4f", res[0], res[1], res[2], res[3]);
for (i = 0; i < 4; i++) {
fail_unless (FEQ (res[i], expected[i]),
"value %f at index %u does not match " "expected value %f", res[i], i,
expected[i]);
}
_matrix_mult_vertex4 (scale, v, res);
GST_DEBUG ("vertex: %.4f %.4f %.4f %.4f", v[0], v[1], v[2], v[3]);
GST_DEBUG ("result: %.4f %.4f %.4f %.4f", res[0], res[1], res[2], res[3]);
for (i = 0; i < 4; i++) {
fail_unless (FEQ (res[i], expected[i]),
"value %f at index %u does not match " "expected value %f", res[i], i,
expected[i]);
}
}
GST_END_TEST;
#endif
static Suite *
gst_gl_upload_suite (void)
{
Suite *s = suite_create ("GstGLMatrix");
TCase *tc_chain = tcase_create ("matrix");
suite_add_tcase (s, tc_chain);
tcase_add_test (tc_chain, test_matrix_multiply);
tcase_add_test (tc_chain, test_matrix_ndc);
return s;
}
GST_CHECK_MAIN (gst_gl_upload);
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment